From e3f9d5753f850cd5f7e831eccdd2897c84869c6c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 19 Dec 2023 15:19:15 +0100 Subject: [PATCH 01/96] added some half working validators --- plugins/nf-validation/build.gradle | 1 + .../validation/JsonSchemaValidator.groovy | 60 ++++++++ .../validation/SchemaValidator.groovy | 132 ++++++++++-------- 3 files changed, 137 insertions(+), 56 deletions(-) create mode 100644 plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy diff --git a/plugins/nf-validation/build.gradle b/plugins/nf-validation/build.gradle index ae2652f..7b28e3d 100644 --- a/plugins/nf-validation/build.gradle +++ b/plugins/nf-validation/build.gradle @@ -55,6 +55,7 @@ dependencies { compileOnly 'org.slf4j:slf4j-api:1.7.10' compileOnly 'org.pf4j:pf4j:3.4.1' api 'com.github.everit-org.json-schema:org.everit.json.schema:1.14.1' + implementation 'net.jimblackler.jsonschemafriend:core:0.12.3' // test configuration testImplementation "io.nextflow:nextflow:$nextflowVersion" diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy new file mode 100644 index 0000000..3e8c70f --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -0,0 +1,60 @@ +package nextflow.validation + +import groovy.util.logging.Slf4j +import groovy.transform.CompileStatic +import net.jimblackler.jsonschemafriend.Schema +import net.jimblackler.jsonschemafriend.SchemaException +import net.jimblackler.jsonschemafriend.MissingPropertyError +import net.jimblackler.jsonschemafriend.SchemaStore +import net.jimblackler.jsonschemafriend.Validator +import org.json.JSONObject +import org.json.JSONArray + +@Slf4j +@CompileStatic +public class JsonSchemaValidator { + + private static Schema schema + private static List errors = [] + + JsonSchemaValidator(String schemaString) { + try { + SchemaStore schemaStore = new SchemaStore(); // Initialize a SchemaStore. + this.schema = schemaStore.loadSchemaJson(schemaString) // Load the schema. + } catch (SchemaException e) { + // TODO handle these exceptions better + e.printStackTrace() + } + } + + public static List validateObject(JSONObject input) { + Validator validator = new Validator() + validator.validate(this.schema, input.toMap(), validationError -> { + if (validationError instanceof MissingPropertyError) { + MissingPropertyError missingPropertyError = (MissingPropertyError) validationError + this.errors.add("* Missing required parameter: --${missingPropertyError.getProperty()}" as String) + } else { + // TODO write custom error messages for other types of errors + this.errors.add("* ${validationError}" as String) + } + }) + return this.errors + } + + public static List validateArray(JSONArray input) { + Validator validator = new Validator() + input.forEach { entry -> + JSONObject jsonEntry = (JSONObject) entry + validator.validate(this.schema, jsonEntry.toMap(), validationError -> { + if (validationError instanceof MissingPropertyError) { + MissingPropertyError missingPropertyError = (MissingPropertyError) validationError + this.errors.add("* Missing required field: --${missingPropertyError.getProperty()}" as String) + } else { + // TODO write custom error messages for other types of errors + this.errors.add("* ${validationError}" as String) + } + }) + } + return this.errors + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index a19715b..c556333 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -363,15 +363,17 @@ class SchemaValidator extends PluginExtensionPoint { //=====================================================================// // Validate parameters against the schema def String schema_string = Files.readString( Path.of(getSchemaPath(baseDir, schemaFilename)) ) - final rawSchema = new JSONObject(new JSONTokener(schema_string)) - final SchemaLoader schemaLoader = SchemaLoader.builder() - .schemaJson(rawSchema) - .addFormatValidator("file-path", new FilePathValidator()) - .addFormatValidator("directory-path", new DirectoryPathValidator()) - .addFormatValidator("path", new PathValidator()) - .addFormatValidator("file-path-pattern", new FilePathPatternValidator()) - .build() - final schema = schemaLoader.load().build() + def validator = new JsonSchemaValidator(schema_string) + // TODO remove these comments once finished + // final rawSchema = new JSONObject(new JSONTokener(schema_string)) + // final SchemaLoader schemaLoader = SchemaLoader.builder() + // .schemaJson(rawSchema) + // .addFormatValidator("file-path", new FilePathValidator()) + // .addFormatValidator("directory-path", new DirectoryPathValidator()) + // .addFormatValidator("path", new PathValidator()) + // .addFormatValidator("file-path-pattern", new FilePathPatternValidator()) + // .build() + // final schema = schemaLoader.load().build() // check for warnings if( this.hasWarnings() ) { @@ -383,29 +385,38 @@ class SchemaValidator extends PluginExtensionPoint { def colors = logColours(useMonochromeLogs) // Validate - try { - if (lenientMode) { - // Create new validator with LENIENT mode - Validator validator = Validator.builder() - .primitiveValidationStrategy(PrimitiveValidationStrategy.LENIENT) - .build(); - validator.performValidation(schema, paramsJSON); - } else { - schema.validate(paramsJSON) - } - if (this.hasErrors()) { - // Needed when validationFailUnrecognisedParams is true - def msg = "${colors.red}The following invalid input values have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" - log.error("ERROR: Validation of pipeline parameters failed!") - throw new SchemaValidationException(msg, this.getErrors()) - } - } catch (ValidationException e) { - JSONObject exceptionJSON = (JSONObject) e.toJSON() - collectErrors(exceptionJSON, paramsJSON, enums, rawSchema) - def msg = "${colors.red}The following invalid input values have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" + // TODO find out how to enable lenient mode + List validationErrors = validator.validateObject(paramsJSON) + this.errors.addAll(validationErrors) + if (this.hasErrors()) { + def msg = "${colors.red}The following invalid input values have been detected:\n\n" + errors.join('\n').trim() + "\n${colors.reset}\n" log.error("ERROR: Validation of pipeline parameters failed!") throw new SchemaValidationException(msg, this.getErrors()) } + // TODO remove these comments once finished + // try { + // if (lenientMode) { + // // Create new validator with LENIENT mode + // Validator validator = Validator.builder() + // .primitiveValidationStrategy(PrimitiveValidationStrategy.LENIENT) + // .build(); + // validator.performValidation(schema, paramsJSON); + // } else { + // schema.validate(paramsJSON) + // } + // if (this.hasErrors()) { + // // Needed when validationFailUnrecognisedParams is true + // def msg = "${colors.red}The following invalid input values have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" + // log.error("ERROR: Validation of pipeline parameters failed!") + // throw new SchemaValidationException(msg, this.getErrors()) + // } + // } catch (ValidationException e) { + // JSONObject exceptionJSON = (JSONObject) e.toJSON() + // collectErrors(exceptionJSON, paramsJSON, enums, rawSchema) + // def msg = "${colors.red}The following invalid input values have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" + // log.error("ERROR: Validation of pipeline parameters failed!") + // throw new SchemaValidationException(msg, this.getErrors()) + // } //=====================================================================// // Look for other schemas to validate @@ -557,15 +568,16 @@ class SchemaValidator extends PluginExtensionPoint { // Load the schema def String schema_string = Files.readString( Path.of(getSchemaPath(baseDir, schemaFilename)) ) - final rawSchema = new JSONObject(new JSONTokener(schema_string)) - final SchemaLoader schemaLoader = SchemaLoader.builder() - .schemaJson(rawSchema) - .addFormatValidator("file-path", new FilePathValidator()) - .addFormatValidator("directory-path", new DirectoryPathValidator()) - .addFormatValidator("path", new PathValidator()) - .addFormatValidator("file-path-pattern", new FilePathPatternValidator()) - .build() - final schema = schemaLoader.load().build() + def validator = new JsonSchemaValidator(schema_string) + // final rawSchema = new JSONObject(new JSONTokener(schema_string)) + // final SchemaLoader schemaLoader = SchemaLoader.builder() + // .schemaJson(rawSchema) + // .addFormatValidator("file-path", new FilePathValidator()) + // .addFormatValidator("directory-path", new DirectoryPathValidator()) + // .addFormatValidator("path", new PathValidator()) + // .addFormatValidator("file-path-pattern", new FilePathPatternValidator()) + // .build() + // final schema = schemaLoader.load().build() // Remove all null values from JSON object // and convert the groovy object to a JSONArray @@ -599,29 +611,37 @@ class SchemaValidator extends PluginExtensionPoint { //=====================================================================// // Validate - try { - // Create new validator with LENIENT mode - Validator validator = Validator.builder() - .primitiveValidationStrategy(PrimitiveValidationStrategy.LENIENT) - .build(); - validator.performValidation(schema, arrayJSON); - if (this.hasErrors()) { - // Needed for custom errors such as pathExists() errors - def colors = logColours(monochrome_logs) - def msg = "${colors.red}The following errors have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" - log.error("ERROR: Validation of '$paramName' file failed!") - throw new SchemaValidationException(msg, this.getErrors()) - } - } catch (ValidationException e) { + def List validationErrors = validator.validateArray(arrayJSON) + this.errors.addAll(validationErrors) + if (this.hasErrors()) { def colors = logColours(monochrome_logs) - JSONObject exceptionJSON = (JSONObject) e.toJSON() - JSONObject objectJSON = new JSONObject(); - objectJSON.put("objects",arrayJSON); - collectErrors(exceptionJSON, objectJSON, enums, rawSchema) def msg = "${colors.red}The following errors have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" log.error("ERROR: Validation of '$paramName' file failed!") throw new SchemaValidationException(msg, this.getErrors()) } + // try { + // // Create new validator with LENIENT mode + // Validator validator = Validator.builder() + // .primitiveValidationStrategy(PrimitiveValidationStrategy.LENIENT) + // .build(); + // validator.performValidation(schema, arrayJSON); + // if (this.hasErrors()) { + // // Needed for custom errors such as pathExists() errors + // def colors = logColours(monochrome_logs) + // def msg = "${colors.red}The following errors have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" + // log.error("ERROR: Validation of '$paramName' file failed!") + // throw new SchemaValidationException(msg, this.getErrors()) + // } + // } catch (ValidationException e) { + // def colors = logColours(monochrome_logs) + // JSONObject exceptionJSON = (JSONObject) e.toJSON() + // JSONObject objectJSON = new JSONObject(); + // objectJSON.put("objects",arrayJSON); + // collectErrors(exceptionJSON, objectJSON, enums, rawSchema) + // def msg = "${colors.red}The following errors have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" + // log.error("ERROR: Validation of '$paramName' file failed!") + // throw new SchemaValidationException(msg, this.getErrors()) + // } return true } From 3531eafaa836e81fcc972b3e84e8ee02f5a8e3f0 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 19 Dec 2023 16:22:45 +0100 Subject: [PATCH 02/96] some small fixes to the error handling --- .../main/nextflow/validation/JsonSchemaValidator.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 3e8c70f..c6c85ff 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -29,10 +29,10 @@ public class JsonSchemaValidator { public static List validateObject(JSONObject input) { Validator validator = new Validator() + println(input.toMap()) validator.validate(this.schema, input.toMap(), validationError -> { if (validationError instanceof MissingPropertyError) { - MissingPropertyError missingPropertyError = (MissingPropertyError) validationError - this.errors.add("* Missing required parameter: --${missingPropertyError.getProperty()}" as String) + this.errors.add("* Missing required parameter: --${validationError.getProperty()}" as String) } else { // TODO write custom error messages for other types of errors this.errors.add("* ${validationError}" as String) @@ -43,12 +43,13 @@ public class JsonSchemaValidator { public static List validateArray(JSONArray input) { Validator validator = new Validator() + Integer entryCount = 0 input.forEach { entry -> + entryCount++ JSONObject jsonEntry = (JSONObject) entry validator.validate(this.schema, jsonEntry.toMap(), validationError -> { if (validationError instanceof MissingPropertyError) { - MissingPropertyError missingPropertyError = (MissingPropertyError) validationError - this.errors.add("* Missing required field: --${missingPropertyError.getProperty()}" as String) + this.errors.add("* Entry ${entryCount}: Missing required field: ${validationError.getProperty()}" as String) } else { // TODO write custom error messages for other types of errors this.errors.add("* ${validationError}" as String) From ed558aec99107e83c18667a0a2d45f957a93a465 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 19 Dec 2023 16:53:54 +0100 Subject: [PATCH 03/96] better handle schema exceptions --- .../validation/JsonSchemaValidator.groovy | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index c6c85ff..62bba72 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -18,20 +18,18 @@ public class JsonSchemaValidator { private static List errors = [] JsonSchemaValidator(String schemaString) { - try { - SchemaStore schemaStore = new SchemaStore(); // Initialize a SchemaStore. - this.schema = schemaStore.loadSchemaJson(schemaString) // Load the schema. - } catch (SchemaException e) { - // TODO handle these exceptions better - e.printStackTrace() - } + SchemaStore schemaStore = new SchemaStore(); // Initialize a SchemaStore. + this.schema = schemaStore.loadSchemaJson(schemaString) // Load the schema. } public static List validateObject(JSONObject input) { Validator validator = new Validator() - println(input.toMap()) validator.validate(this.schema, input.toMap(), validationError -> { - if (validationError instanceof MissingPropertyError) { + if(validationError instanceof SchemaException) { + // TODO handle this better + log.error("* ${validationError.getMessage()}" as String) + } + else if (validationError instanceof MissingPropertyError) { this.errors.add("* Missing required parameter: --${validationError.getProperty()}" as String) } else { // TODO write custom error messages for other types of errors @@ -48,7 +46,11 @@ public class JsonSchemaValidator { entryCount++ JSONObject jsonEntry = (JSONObject) entry validator.validate(this.schema, jsonEntry.toMap(), validationError -> { - if (validationError instanceof MissingPropertyError) { + if(validationError instanceof SchemaException) { + // TODO handle this better + log.error("* ${validationError.getMessage()}" as String) + } + else if (validationError instanceof MissingPropertyError) { this.errors.add("* Entry ${entryCount}: Missing required field: ${validationError.getProperty()}" as String) } else { // TODO write custom error messages for other types of errors From ac811550a2198bec2f378abedd7f067a3a00cd24 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 20 Dec 2023 14:18:16 +0100 Subject: [PATCH 04/96] fix issues with schema + even better errors (wow) --- .../validation/JsonSchemaValidator.groovy | 17 +++++++++++++++-- .../nextflow/validation/SchemaValidator.groovy | 4 ++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 62bba72..e8f034b 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -7,6 +7,7 @@ import net.jimblackler.jsonschemafriend.SchemaException import net.jimblackler.jsonschemafriend.MissingPropertyError import net.jimblackler.jsonschemafriend.SchemaStore import net.jimblackler.jsonschemafriend.Validator +import net.jimblackler.jsonschemafriend.ValidationError import org.json.JSONObject import org.json.JSONArray @@ -31,8 +32,14 @@ public class JsonSchemaValidator { } else if (validationError instanceof MissingPropertyError) { this.errors.add("* Missing required parameter: --${validationError.getProperty()}" as String) + } + else if (validationError instanceof ValidationError) { + def String value = validationError.getObject() + def String paramUri = "${validationError.getUri()}" as String + def String param = paramUri.replaceFirst("#/", "") + def String msg = validationError.getMessage() + this.errors.add("* Error for parameter '${param}' (${value}): ${msg}" as String) } else { - // TODO write custom error messages for other types of errors this.errors.add("* ${validationError}" as String) } }) @@ -52,8 +59,14 @@ public class JsonSchemaValidator { } else if (validationError instanceof MissingPropertyError) { this.errors.add("* Entry ${entryCount}: Missing required field: ${validationError.getProperty()}" as String) + } + else if (validationError instanceof ValidationError) { + def String value = validationError.getObject() + def String fieldUri = "${validationError.getUri()}" as String + def String field = fieldUri.replaceFirst("#/", "") + def String msg = validationError.getMessage() + this.errors.add("* Entry ${entryCount}: Error for field '${field}' (${value}): ${msg}" as String) } else { - // TODO write custom error messages for other types of errors this.errors.add("* ${validationError}" as String) } }) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index c556333..90bbed6 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -491,7 +491,7 @@ class SchemaValidator extends PluginExtensionPoint { def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) // Obtain the type of each variable in the schema - def Map properties = (Map) parsed['items']['properties'] + def Map properties = (Map) parsed['properties'] for (p in properties) { def String key = (String) p.key def Map property = properties[key] as Map @@ -590,7 +590,7 @@ class SchemaValidator extends PluginExtensionPoint { // Check for params with expected values def slurper = new JsonSlurper() def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) - def Map schemaParams = (Map) ["items": parsed.get('items')] + def Map schemaParams = (Map) ["properties": parsed.get('properties')] // Collect expected parameters from the schema def enumsTuple = collectEnums(schemaParams) From 690f1a130b9c2a9930e580de8ed2eebf14366b50 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 20 Dec 2023 14:35:11 +0100 Subject: [PATCH 05/96] more error messages! --- .../validation/JsonSchemaValidator.groovy | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index e8f034b..cac9e1b 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -4,9 +4,10 @@ import groovy.util.logging.Slf4j import groovy.transform.CompileStatic import net.jimblackler.jsonschemafriend.Schema import net.jimblackler.jsonschemafriend.SchemaException -import net.jimblackler.jsonschemafriend.MissingPropertyError import net.jimblackler.jsonschemafriend.SchemaStore import net.jimblackler.jsonschemafriend.Validator +import net.jimblackler.jsonschemafriend.MissingPropertyError +import net.jimblackler.jsonschemafriend.DependencyError import net.jimblackler.jsonschemafriend.ValidationError import org.json.JSONObject import org.json.JSONArray @@ -31,14 +32,19 @@ public class JsonSchemaValidator { log.error("* ${validationError.getMessage()}" as String) } else if (validationError instanceof MissingPropertyError) { + println(validationError.getMessage()) this.errors.add("* Missing required parameter: --${validationError.getProperty()}" as String) } else if (validationError instanceof ValidationError) { - def String value = validationError.getObject() - def String paramUri = "${validationError.getUri()}" as String - def String param = paramUri.replaceFirst("#/", "") - def String msg = validationError.getMessage() - this.errors.add("* Error for parameter '${param}' (${value}): ${msg}" as String) + def String paramUri = validationError.getUri().toString() + if (paramUri == '') { + this.errors.add("* ${validationError.getMessage()}" as String) + return + } + def String param = paramUri.replaceFirst("#/", "") + def String value = validationError.getObject() + def String msg = validationError.getMessage() + this.errors.add("* Error for parameter '${param}' (${value}): ${msg}" as String) } else { this.errors.add("* ${validationError}" as String) } @@ -59,11 +65,15 @@ public class JsonSchemaValidator { } else if (validationError instanceof MissingPropertyError) { this.errors.add("* Entry ${entryCount}: Missing required field: ${validationError.getProperty()}" as String) - } + } else if (validationError instanceof ValidationError) { - def String value = validationError.getObject() - def String fieldUri = "${validationError.getUri()}" as String + def String fieldUri = validationError.getUri().toString() + if (fieldUri == '') { + this.errors.add("* Entry ${entryCount}: ${validationError.getMessage()}" as String) + return + } def String field = fieldUri.replaceFirst("#/", "") + def String value = validationError.getObject() def String msg = validationError.getMessage() this.errors.add("* Entry ${entryCount}: Error for field '${field}' (${value}): ${msg}" as String) } else { From 633206c86e550dae3453b2e7d9c9a2ad546892ab Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 20 Dec 2023 16:38:32 +0100 Subject: [PATCH 06/96] no repetition! --- .../validation/JsonSchemaValidator.groovy | 34 ++++--------------- .../validation/SchemaValidator.groovy | 2 +- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index cac9e1b..6392721 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -24,8 +24,9 @@ public class JsonSchemaValidator { this.schema = schemaStore.loadSchemaJson(schemaString) // Load the schema. } - public static List validateObject(JSONObject input) { + public static List validateObject(JSONObject input, String validationType, Integer entryCount) { Validator validator = new Validator() + def String entryString = entryCount != -1 ? "Entry ${entryCount}: " : "" validator.validate(this.schema, input.toMap(), validationError -> { if(validationError instanceof SchemaException) { // TODO handle this better @@ -33,20 +34,20 @@ public class JsonSchemaValidator { } else if (validationError instanceof MissingPropertyError) { println(validationError.getMessage()) - this.errors.add("* Missing required parameter: --${validationError.getProperty()}" as String) + this.errors.add("* ${entryString}Missing required ${validationType}: --${validationError.getProperty()}" as String) } else if (validationError instanceof ValidationError) { def String paramUri = validationError.getUri().toString() if (paramUri == '') { - this.errors.add("* ${validationError.getMessage()}" as String) + this.errors.add("* ${entryString}${validationError.getMessage()}" as String) return } def String param = paramUri.replaceFirst("#/", "") def String value = validationError.getObject() def String msg = validationError.getMessage() - this.errors.add("* Error for parameter '${param}' (${value}): ${msg}" as String) + this.errors.add("* ${entryString}Error for ${validationType} '${param}' (${value}): ${msg}" as String) } else { - this.errors.add("* ${validationError}" as String) + this.errors.add("* ${entryString}${validationError}" as String) } }) return this.errors @@ -58,28 +59,7 @@ public class JsonSchemaValidator { input.forEach { entry -> entryCount++ JSONObject jsonEntry = (JSONObject) entry - validator.validate(this.schema, jsonEntry.toMap(), validationError -> { - if(validationError instanceof SchemaException) { - // TODO handle this better - log.error("* ${validationError.getMessage()}" as String) - } - else if (validationError instanceof MissingPropertyError) { - this.errors.add("* Entry ${entryCount}: Missing required field: ${validationError.getProperty()}" as String) - } - else if (validationError instanceof ValidationError) { - def String fieldUri = validationError.getUri().toString() - if (fieldUri == '') { - this.errors.add("* Entry ${entryCount}: ${validationError.getMessage()}" as String) - return - } - def String field = fieldUri.replaceFirst("#/", "") - def String value = validationError.getObject() - def String msg = validationError.getMessage() - this.errors.add("* Entry ${entryCount}: Error for field '${field}' (${value}): ${msg}" as String) - } else { - this.errors.add("* ${validationError}" as String) - } - }) + validateObject(jsonEntry, "field", entryCount) } return this.errors } diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 90bbed6..2e97301 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -386,7 +386,7 @@ class SchemaValidator extends PluginExtensionPoint { // Validate // TODO find out how to enable lenient mode - List validationErrors = validator.validateObject(paramsJSON) + List validationErrors = validator.validateObject(paramsJSON, "parameter", -1) this.errors.addAll(validationErrors) if (this.hasErrors()) { def msg = "${colors.red}The following invalid input values have been detected:\n\n" + errors.join('\n').trim() + "\n${colors.reset}\n" From b1ca7c256817d7eb8ac5faa27e4b50b5b3063cea Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 20 Dec 2023 17:08:23 +0100 Subject: [PATCH 07/96] enable format validation --- .../src/main/nextflow/validation/JsonSchemaValidator.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 6392721..9e90f43 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -25,7 +25,7 @@ public class JsonSchemaValidator { } public static List validateObject(JSONObject input, String validationType, Integer entryCount) { - Validator validator = new Validator() + Validator validator = new Validator(true) def String entryString = entryCount != -1 ? "Entry ${entryCount}: " : "" validator.validate(this.schema, input.toMap(), validationError -> { if(validationError instanceof SchemaException) { @@ -54,7 +54,6 @@ public class JsonSchemaValidator { } public static List validateArray(JSONArray input) { - Validator validator = new Validator() Integer entryCount = 0 input.forEach { entry -> entryCount++ From 6d77825e8a7b3f1191b9a03a14dd2d63b4d89705 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 21 Dec 2023 14:16:58 +0100 Subject: [PATCH 08/96] files are now validated as an array of inputs --- .../validation/JsonSchemaValidator.groovy | 67 +++++++++++++------ .../validation/SchemaValidator.groovy | 8 +-- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 9e90f43..ee9025c 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -12,54 +12,83 @@ import net.jimblackler.jsonschemafriend.ValidationError import org.json.JSONObject import org.json.JSONArray +import java.util.regex.Pattern +import java.util.regex.Matcher + @Slf4j @CompileStatic public class JsonSchemaValidator { private static Schema schema private static List errors = [] + private static Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$') JsonSchemaValidator(String schemaString) { SchemaStore schemaStore = new SchemaStore(); // Initialize a SchemaStore. this.schema = schemaStore.loadSchemaJson(schemaString) // Load the schema. } - public static List validateObject(JSONObject input, String validationType, Integer entryCount) { + private static validateObject(Object input, String validationType) { Validator validator = new Validator(true) - def String entryString = entryCount != -1 ? "Entry ${entryCount}: " : "" - validator.validate(this.schema, input.toMap(), validationError -> { - if(validationError instanceof SchemaException) { + validator.validate(this.schema, input, validationError -> { + // Fail on other errors than validation errors + if(validationError !instanceof ValidationError) { // TODO handle this better log.error("* ${validationError.getMessage()}" as String) + return + } + + // Get the name of the parameter and determine if it is a list entry + def Integer entry = 0 + def String name = '' + def String[] uriSplit = validationError.getUri().toString().replaceFirst('#/', '').split('/') + def String error = '' + + if (input instanceof Map) { + name = uriSplit.size() > 0 ? uriSplit[0..-1].join('/') : '' } - else if (validationError instanceof MissingPropertyError) { - println(validationError.getMessage()) - this.errors.add("* ${entryString}Missing required ${validationType}: --${validationError.getProperty()}" as String) + else if (input instanceof List) { + entry = uriSplit[0].toInteger() + 1 + name = uriSplit.size() > 1 ? uriSplit[1..-1].join('/') : '' + } + + // Create custom error messages + if (validationError instanceof MissingPropertyError) { + def String paramUri = validationError.getUri().toString() + error = "Missing required ${validationType}: --${validationError.getProperty()}" as String } else if (validationError instanceof ValidationError) { def String paramUri = validationError.getUri().toString() - if (paramUri == '') { - this.errors.add("* ${entryString}${validationError.getMessage()}" as String) + if (name == '') { + this.errors.add("${validationError.getMessage()}" as String) return } - def String param = paramUri.replaceFirst("#/", "") def String value = validationError.getObject() def String msg = validationError.getMessage() - this.errors.add("* ${entryString}Error for ${validationType} '${param}' (${value}): ${msg}" as String) - } else { - this.errors.add("* ${entryString}${validationError}" as String) + error = "Error for ${validationType} '${name}' (${value}): ${msg}" as String + } + + // Add the error to the list + if (entry > 0) { + this.errors.add("* Entry ${entry}: ${error}" as String) + } + else { + this.errors.add("* ${error}" as String) } }) - return this.errors } - public static List validateArray(JSONArray input) { - Integer entryCount = 0 - input.forEach { entry -> - entryCount++ + public static List validate(JSONArray input, String validationType) { + List> inputList = input.collect { entry -> JSONObject jsonEntry = (JSONObject) entry - validateObject(jsonEntry, "field", entryCount) + jsonEntry.toMap() } + this.validateObject(inputList, validationType) + return this.errors + } + + public static List validate(JSONObject input, String validationType) { + this.validateObject(input.toMap(), validationType) return this.errors } } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 2e97301..6cf2b32 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -386,7 +386,7 @@ class SchemaValidator extends PluginExtensionPoint { // Validate // TODO find out how to enable lenient mode - List validationErrors = validator.validateObject(paramsJSON, "parameter", -1) + List validationErrors = validator.validate(paramsJSON, "parameter") this.errors.addAll(validationErrors) if (this.hasErrors()) { def msg = "${colors.red}The following invalid input values have been detected:\n\n" + errors.join('\n').trim() + "\n${colors.reset}\n" @@ -491,7 +491,7 @@ class SchemaValidator extends PluginExtensionPoint { def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) // Obtain the type of each variable in the schema - def Map properties = (Map) parsed['properties'] + def Map properties = (Map) parsed['items']['properties'] for (p in properties) { def String key = (String) p.key def Map property = properties[key] as Map @@ -590,7 +590,7 @@ class SchemaValidator extends PluginExtensionPoint { // Check for params with expected values def slurper = new JsonSlurper() def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) - def Map schemaParams = (Map) ["properties": parsed.get('properties')] + def Map schemaParams = (Map) ["items": parsed.get('items')] // Collect expected parameters from the schema def enumsTuple = collectEnums(schemaParams) @@ -611,7 +611,7 @@ class SchemaValidator extends PluginExtensionPoint { //=====================================================================// // Validate - def List validationErrors = validator.validateArray(arrayJSON) + def List validationErrors = validator.validate(arrayJSON, "field") this.errors.addAll(validationErrors) if (this.hasErrors()) { def colors = logColours(monochrome_logs) From cc7434d8045df1f30a70f5c6cd7c70d95788d4b2 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 21 Dec 2023 14:42:59 +0100 Subject: [PATCH 09/96] remove everit json schema lib and add org.json --- plugins/nf-validation/build.gradle | 2 +- .../DirectoryPathValidator.groovy | 40 +++++------ .../FilePathPatternValidator.groovy | 58 +++++++-------- .../FormatValidators/FilePathValidator.groovy | 40 +++++------ .../FormatValidators/PathValidator.groovy | 34 ++++----- .../validation/SamplesheetConverter.groovy | 5 -- .../validation/SchemaValidator.groovy | 71 ------------------- 7 files changed, 87 insertions(+), 163 deletions(-) diff --git a/plugins/nf-validation/build.gradle b/plugins/nf-validation/build.gradle index 7b28e3d..456dfc9 100644 --- a/plugins/nf-validation/build.gradle +++ b/plugins/nf-validation/build.gradle @@ -54,7 +54,7 @@ dependencies { compileOnly "io.nextflow:nextflow:$nextflowVersion" compileOnly 'org.slf4j:slf4j-api:1.7.10' compileOnly 'org.pf4j:pf4j:3.4.1' - api 'com.github.everit-org.json-schema:org.everit.json.schema:1.14.1' + implementation 'org.json:json:20231013' implementation 'net.jimblackler.jsonschemafriend:core:0.12.3' // test configuration diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy index 0d935b5..c4ab6a6 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy @@ -1,24 +1,24 @@ -package nextflow.validation +// package nextflow.validation -import java.nio.file.Path -import groovy.util.logging.Slf4j +// import java.nio.file.Path +// import groovy.util.logging.Slf4j -import org.everit.json.schema.FormatValidator -import nextflow.Nextflow +// import org.everit.json.schema.FormatValidator +// import nextflow.Nextflow -@Slf4j -public class DirectoryPathValidator implements FormatValidator { +// @Slf4j +// public class DirectoryPathValidator implements FormatValidator { - @Override - public Optional validate(final String subject) { - if (subject.startsWith('s3://')) { - log.debug("S3 paths are not supported by 'DirectoryPathValidator': '${subject}'") - return Optional.empty() - } - Path file = Nextflow.file(subject) as Path - if (file.exists() && !file.isDirectory()) { - return Optional.of("'${subject}' is not a directory, but a file" as String) - } - return Optional.empty() - } -} \ No newline at end of file +// @Override +// public Optional validate(final String subject) { +// if (subject.startsWith('s3://')) { +// log.debug("S3 paths are not supported by 'DirectoryPathValidator': '${subject}'") +// return Optional.empty() +// } +// Path file = Nextflow.file(subject) as Path +// if (file.exists() && !file.isDirectory()) { +// return Optional.of("'${subject}' is not a directory, but a file" as String) +// } +// return Optional.empty() +// } +// } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy index 4ddaaea..92cd8a2 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy @@ -1,34 +1,34 @@ -package nextflow.validation +// package nextflow.validation -import java.nio.file.Path -import groovy.util.logging.Slf4j +// import java.nio.file.Path +// import groovy.util.logging.Slf4j -import org.everit.json.schema.FormatValidator -import nextflow.Nextflow +// import org.everit.json.schema.FormatValidator +// import nextflow.Nextflow -@Slf4j -public class FilePathPatternValidator implements FormatValidator { +// @Slf4j +// public class FilePathPatternValidator implements FormatValidator { - @Override - public Optional validate(final String subject) { - if (subject.startsWith('s3://')) { - log.debug("S3 paths are not supported by 'FilePathPatternValidator': '${subject}'") - return Optional.empty() - } - ArrayList files = Nextflow.files(subject) - ArrayList errors = [] +// @Override +// public Optional validate(final String subject) { +// if (subject.startsWith('s3://')) { +// log.debug("S3 paths are not supported by 'FilePathPatternValidator': '${subject}'") +// return Optional.empty() +// } +// ArrayList files = Nextflow.files(subject) +// ArrayList errors = [] - if(files.size() == 0) { - return Optional.of("No files were found using the globbing pattern '${subject}'" as String) - } - for( file : files ) { - if (file.isDirectory()) { - errors.add("'${file.toString()}' is not a file, but a directory" as String) - } - } - if(errors.size() > 0) { - return Optional.of(errors.join('\n')) - } - return Optional.empty() - } -} \ No newline at end of file +// if(files.size() == 0) { +// return Optional.of("No files were found using the globbing pattern '${subject}'" as String) +// } +// for( file : files ) { +// if (file.isDirectory()) { +// errors.add("'${file.toString()}' is not a file, but a directory" as String) +// } +// } +// if(errors.size() > 0) { +// return Optional.of(errors.join('\n')) +// } +// return Optional.empty() +// } +// } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy index a49ec6c..c33c4a1 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy @@ -1,24 +1,24 @@ -package nextflow.validation +// package nextflow.validation -import java.nio.file.Path -import groovy.util.logging.Slf4j +// import java.nio.file.Path +// import groovy.util.logging.Slf4j -import org.everit.json.schema.FormatValidator -import nextflow.Nextflow +// import org.everit.json.schema.FormatValidator +// import nextflow.Nextflow -@Slf4j -public class FilePathValidator implements FormatValidator { +// @Slf4j +// public class FilePathValidator implements FormatValidator { - @Override - public Optional validate(final String subject) { - if (subject.startsWith('s3://')) { - log.debug("S3 paths are not supported by 'FilePathValidator': '${subject}'") - return Optional.empty() - } - Path file = Nextflow.file(subject) as Path - if (file.isDirectory()) { - return Optional.of("'${subject}' is not a file, but a directory" as String) - } - return Optional.empty() - } -} \ No newline at end of file +// @Override +// public Optional validate(final String subject) { +// if (subject.startsWith('s3://')) { +// log.debug("S3 paths are not supported by 'FilePathValidator': '${subject}'") +// return Optional.empty() +// } +// Path file = Nextflow.file(subject) as Path +// if (file.isDirectory()) { +// return Optional.of("'${subject}' is not a file, but a directory" as String) +// } +// return Optional.empty() +// } +// } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy index afe82e0..bc7b3f3 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy @@ -1,21 +1,21 @@ -package nextflow.validation +// package nextflow.validation -import java.nio.file.Path -import groovy.util.logging.Slf4j +// import java.nio.file.Path +// import groovy.util.logging.Slf4j -import org.everit.json.schema.FormatValidator -import nextflow.Nextflow +// import org.everit.json.schema.FormatValidator +// import nextflow.Nextflow -@Slf4j -public class PathValidator implements FormatValidator { +// @Slf4j +// public class PathValidator implements FormatValidator { - @Override - public Optional validate(final String subject) { - if (subject.startsWith('s3://')) { - log.debug("S3 paths are not supported by 'PathValidator': '${subject}'") - return Optional.empty() - } - Path file = Nextflow.file(subject) as Path - return Optional.empty() - } -} \ No newline at end of file +// @Override +// public Optional validate(final String subject) { +// if (subject.startsWith('s3://')) { +// log.debug("S3 paths are not supported by 'PathValidator': '${subject}'") +// return Optional.empty() +// } +// Path file = Nextflow.file(subject) as Path +// return Optional.empty() +// } +// } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index 3118bd8..e3af031 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -12,11 +12,6 @@ import java.util.concurrent.CompletableFuture import java.util.concurrent.CompletionException import org.yaml.snakeyaml.Yaml -import org.everit.json.schema.loader.SchemaLoader -import org.everit.json.schema.PrimitiveValidationStrategy -import org.everit.json.schema.ValidationException -import org.everit.json.schema.SchemaException -import org.everit.json.schema.Schema import org.json.JSONArray import org.json.JSONObject import org.json.JSONTokener diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 6cf2b32..6cf2651 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -21,11 +21,6 @@ import nextflow.script.WorkflowMetadata import nextflow.Session import nextflow.util.Duration import nextflow.util.MemoryUnit -import org.everit.json.schema.loader.SchemaLoader -import org.everit.json.schema.PrimitiveValidationStrategy -import org.everit.json.schema.ValidationException -import org.everit.json.schema.Validator -import org.everit.json.schema.Schema import org.json.JSONException import org.json.JSONArray import org.json.JSONObject @@ -364,16 +359,6 @@ class SchemaValidator extends PluginExtensionPoint { // Validate parameters against the schema def String schema_string = Files.readString( Path.of(getSchemaPath(baseDir, schemaFilename)) ) def validator = new JsonSchemaValidator(schema_string) - // TODO remove these comments once finished - // final rawSchema = new JSONObject(new JSONTokener(schema_string)) - // final SchemaLoader schemaLoader = SchemaLoader.builder() - // .schemaJson(rawSchema) - // .addFormatValidator("file-path", new FilePathValidator()) - // .addFormatValidator("directory-path", new DirectoryPathValidator()) - // .addFormatValidator("path", new PathValidator()) - // .addFormatValidator("file-path-pattern", new FilePathPatternValidator()) - // .build() - // final schema = schemaLoader.load().build() // check for warnings if( this.hasWarnings() ) { @@ -393,30 +378,6 @@ class SchemaValidator extends PluginExtensionPoint { log.error("ERROR: Validation of pipeline parameters failed!") throw new SchemaValidationException(msg, this.getErrors()) } - // TODO remove these comments once finished - // try { - // if (lenientMode) { - // // Create new validator with LENIENT mode - // Validator validator = Validator.builder() - // .primitiveValidationStrategy(PrimitiveValidationStrategy.LENIENT) - // .build(); - // validator.performValidation(schema, paramsJSON); - // } else { - // schema.validate(paramsJSON) - // } - // if (this.hasErrors()) { - // // Needed when validationFailUnrecognisedParams is true - // def msg = "${colors.red}The following invalid input values have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" - // log.error("ERROR: Validation of pipeline parameters failed!") - // throw new SchemaValidationException(msg, this.getErrors()) - // } - // } catch (ValidationException e) { - // JSONObject exceptionJSON = (JSONObject) e.toJSON() - // collectErrors(exceptionJSON, paramsJSON, enums, rawSchema) - // def msg = "${colors.red}The following invalid input values have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" - // log.error("ERROR: Validation of pipeline parameters failed!") - // throw new SchemaValidationException(msg, this.getErrors()) - // } //=====================================================================// // Look for other schemas to validate @@ -569,15 +530,6 @@ class SchemaValidator extends PluginExtensionPoint { // Load the schema def String schema_string = Files.readString( Path.of(getSchemaPath(baseDir, schemaFilename)) ) def validator = new JsonSchemaValidator(schema_string) - // final rawSchema = new JSONObject(new JSONTokener(schema_string)) - // final SchemaLoader schemaLoader = SchemaLoader.builder() - // .schemaJson(rawSchema) - // .addFormatValidator("file-path", new FilePathValidator()) - // .addFormatValidator("directory-path", new DirectoryPathValidator()) - // .addFormatValidator("path", new PathValidator()) - // .addFormatValidator("file-path-pattern", new FilePathPatternValidator()) - // .build() - // final schema = schemaLoader.load().build() // Remove all null values from JSON object // and convert the groovy object to a JSONArray @@ -619,29 +571,6 @@ class SchemaValidator extends PluginExtensionPoint { log.error("ERROR: Validation of '$paramName' file failed!") throw new SchemaValidationException(msg, this.getErrors()) } - // try { - // // Create new validator with LENIENT mode - // Validator validator = Validator.builder() - // .primitiveValidationStrategy(PrimitiveValidationStrategy.LENIENT) - // .build(); - // validator.performValidation(schema, arrayJSON); - // if (this.hasErrors()) { - // // Needed for custom errors such as pathExists() errors - // def colors = logColours(monochrome_logs) - // def msg = "${colors.red}The following errors have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" - // log.error("ERROR: Validation of '$paramName' file failed!") - // throw new SchemaValidationException(msg, this.getErrors()) - // } - // } catch (ValidationException e) { - // def colors = logColours(monochrome_logs) - // JSONObject exceptionJSON = (JSONObject) e.toJSON() - // JSONObject objectJSON = new JSONObject(); - // objectJSON.put("objects",arrayJSON); - // collectErrors(exceptionJSON, objectJSON, enums, rawSchema) - // def msg = "${colors.red}The following errors have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" - // log.error("ERROR: Validation of '$paramName' file failed!") - // throw new SchemaValidationException(msg, this.getErrors()) - // } return true } From 3ec99803ef3b17402b6bd8aed9f83135cd72e9d6 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 21 Dec 2023 14:46:04 +0100 Subject: [PATCH 10/96] small typo fix --- .../src/main/nextflow/validation/JsonSchemaValidator.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index ee9025c..70e39ed 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -55,7 +55,7 @@ public class JsonSchemaValidator { // Create custom error messages if (validationError instanceof MissingPropertyError) { def String paramUri = validationError.getUri().toString() - error = "Missing required ${validationType}: --${validationError.getProperty()}" as String + error = "Missing required ${validationType}: ${validationError.getProperty()}" as String } else if (validationError instanceof ValidationError) { def String paramUri = validationError.getUri().toString() From e3a1563eff226ff19be780e0f1e7af2b2ed091cd Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 18 Jan 2024 14:09:12 +0100 Subject: [PATCH 11/96] harrel56 part1 --- plugins/nf-validation/build.gradle | 4 +- .../validation/JsonSchemaValidator.groovy | 171 +++++++++++------- .../validation/SchemaValidator.groovy | 4 +- 3 files changed, 114 insertions(+), 65 deletions(-) diff --git a/plugins/nf-validation/build.gradle b/plugins/nf-validation/build.gradle index 456dfc9..1247983 100644 --- a/plugins/nf-validation/build.gradle +++ b/plugins/nf-validation/build.gradle @@ -54,8 +54,8 @@ dependencies { compileOnly "io.nextflow:nextflow:$nextflowVersion" compileOnly 'org.slf4j:slf4j-api:1.7.10' compileOnly 'org.pf4j:pf4j:3.4.1' - implementation 'org.json:json:20231013' - implementation 'net.jimblackler.jsonschemafriend:core:0.12.3' + implementation 'org.json:json:20230227' + implementation 'dev.harrel:json-schema:1.5.0' // test configuration testImplementation "io.nextflow:nextflow:$nextflowVersion" diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 70e39ed..25a2b56 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -2,15 +2,14 @@ package nextflow.validation import groovy.util.logging.Slf4j import groovy.transform.CompileStatic -import net.jimblackler.jsonschemafriend.Schema -import net.jimblackler.jsonschemafriend.SchemaException -import net.jimblackler.jsonschemafriend.SchemaStore -import net.jimblackler.jsonschemafriend.Validator -import net.jimblackler.jsonschemafriend.MissingPropertyError -import net.jimblackler.jsonschemafriend.DependencyError -import net.jimblackler.jsonschemafriend.ValidationError import org.json.JSONObject import org.json.JSONArray +import dev.harrel.jsonschema.ValidatorFactory +import dev.harrel.jsonschema.Validator +import dev.harrel.jsonschema.EvaluatorFactory +import dev.harrel.jsonschema.FormatEvaluatorFactory +import dev.harrel.jsonschema.JsonNode +import dev.harrel.jsonschema.providers.OrgJsonNode import java.util.regex.Pattern import java.util.regex.Matcher @@ -19,76 +18,126 @@ import java.util.regex.Matcher @CompileStatic public class JsonSchemaValidator { - private static Schema schema + private static ValidatorFactory validator + private static String schema private static List errors = [] private static Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$') JsonSchemaValidator(String schemaString) { - SchemaStore schemaStore = new SchemaStore(); // Initialize a SchemaStore. - this.schema = schemaStore.loadSchemaJson(schemaString) // Load the schema. + this.schema = schemaString + this.validator = new ValidatorFactory() + .withJsonNodeFactory(new OrgJsonNode.Factory()) + // .withEvaluatorFactory(EvaluatorFactory.compose(customFactory, new FormatEvaluatorFactory())) + // .withEvaluatorFactory() => exists keyword should be implemented here } - private static validateObject(Object input, String validationType) { - Validator validator = new Validator(true) - validator.validate(this.schema, input, validationError -> { - // Fail on other errors than validation errors - if(validationError !instanceof ValidationError) { - // TODO handle this better - log.error("* ${validationError.getMessage()}" as String) - return + private static validateObject(JsonNode input, String validationType) { + def Validator.Result result = this.validator.validate(this.schema, input) + for (error : result.getErrors()) { + def String errorString = error.getError() + def String location = error.getInstanceLocation() + def String[] locationList = location.split("/").findAll { it != "" } + def String fieldName = "" + def String capValidationType = "${validationType[0].toUpperCase()}${validationType[1..-1]}" + + if (locationList.size() > 0 && isInteger(location[0]) && validationType == "field") { + def Integer entryInteger = location[0] as Integer + def String entryString = "Entry ${entryInteger + 1}" as String + if(locationList.size() > 1) { + fieldName = "Error for '${locationList[1..-1].join("/")}'" + } else { + fieldName = "${capValidationType} error" + } + this.errors.add("* ${entryString}: ${fieldName}: ${errorString}" as String) + } else if (validationType == "parameter") { + fieldName = locationList.join("/") + if(fieldName != "") { + this.errors.add("* --${fieldName}: ${errorString}" as String) + } else { + this.errors.add("* ${capValidationType} error: ${errorString}" as String) + } + } else { + this.errors.add(errorString) } - // Get the name of the parameter and determine if it is a list entry - def Integer entry = 0 - def String name = '' - def String[] uriSplit = validationError.getUri().toString().replaceFirst('#/', '').split('/') - def String error = '' + } - if (input instanceof Map) { - name = uriSplit.size() > 0 ? uriSplit[0..-1].join('/') : '' - } - else if (input instanceof List) { - entry = uriSplit[0].toInteger() + 1 - name = uriSplit.size() > 1 ? uriSplit[1..-1].join('/') : '' - } + // for (annotation : result.getAnnotations()) { + // println(annotation.getAnnotation()) + // println(annotation.getEvaluationPath()) + // println(annotation.getInstanceLocation()) + // println(annotation.getKeyword()) + // println(annotation.getSchemaLocation()) + // } - // Create custom error messages - if (validationError instanceof MissingPropertyError) { - def String paramUri = validationError.getUri().toString() - error = "Missing required ${validationType}: ${validationError.getProperty()}" as String - } - else if (validationError instanceof ValidationError) { - def String paramUri = validationError.getUri().toString() - if (name == '') { - this.errors.add("${validationError.getMessage()}" as String) - return - } - def String value = validationError.getObject() - def String msg = validationError.getMessage() - error = "Error for ${validationType} '${name}' (${value}): ${msg}" as String - } - // Add the error to the list - if (entry > 0) { - this.errors.add("* Entry ${entry}: ${error}" as String) - } - else { - this.errors.add("* ${error}" as String) - } - }) + // Validator validator = new Validator(true) + // validator.validate(this.schema, input, validationError -> { + // // Fail on other errors than validation errors + // if(validationError !instanceof ValidationError) { + // // TODO handle this better + // log.error("* ${validationError.getMessage()}" as String) + // return + // } + + // // Get the name of the parameter and determine if it is a list entry + // def Integer entry = 0 + // def String name = '' + // def String[] uriSplit = validationError.getUri().toString().replaceFirst('#/', '').split('/') + // def String error = '' + + // if (input instanceof Map) { + // name = uriSplit.size() > 0 ? uriSplit[0..-1].join('/') : '' + // } + // else if (input instanceof List) { + // entry = uriSplit[0].toInteger() + 1 + // name = uriSplit.size() > 1 ? uriSplit[1..-1].join('/') : '' + // } + + // // Create custom error messages + // if (validationError instanceof MissingPropertyError) { + // def String paramUri = validationError.getUri().toString() + // error = "Missing required ${validationType}: ${validationError.getProperty()}" as String + // } + // else if (validationError instanceof ValidationError) { + // def String paramUri = validationError.getUri().toString() + // if (name == '') { + // this.errors.add("${validationError.getMessage()}" as String) + // return + // } + // def String value = validationError.getObject() + // def String msg = validationError.getMessage() + // error = "Error for ${validationType} '${name}' (${value}): ${msg}" as String + // } + + // // Add the error to the list + // if (entry > 0) { + // this.errors.add("* Entry ${entry}: ${error}" as String) + // } + // else { + // this.errors.add("* ${error}" as String) + // } + // }) } - public static List validate(JSONArray input, String validationType) { - List> inputList = input.collect { entry -> - JSONObject jsonEntry = (JSONObject) entry - jsonEntry.toMap() - } - this.validateObject(inputList, validationType) + public static List validate(JSONArray input) { + def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) + this.validateObject(jsonInput, "field") return this.errors } - public static List validate(JSONObject input, String validationType) { - this.validateObject(input.toMap(), validationType) + public static List validate(JSONObject input) { + def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) + this.validateObject(jsonInput, "parameter") return this.errors } + + private static Boolean isInteger(String input) { + try { + input as Integer + return true + } catch (NumberFormatException e) { + return false + } + } } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 6cf2651..aa00122 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -371,7 +371,7 @@ class SchemaValidator extends PluginExtensionPoint { // Validate // TODO find out how to enable lenient mode - List validationErrors = validator.validate(paramsJSON, "parameter") + List validationErrors = validator.validate(paramsJSON) this.errors.addAll(validationErrors) if (this.hasErrors()) { def msg = "${colors.red}The following invalid input values have been detected:\n\n" + errors.join('\n').trim() + "\n${colors.reset}\n" @@ -563,7 +563,7 @@ class SchemaValidator extends PluginExtensionPoint { //=====================================================================// // Validate - def List validationErrors = validator.validate(arrayJSON, "field") + def List validationErrors = validator.validate(arrayJSON) this.errors.addAll(validationErrors) if (this.hasErrors()) { def colors = logColours(monochrome_logs) From ff7c2ba801761cf1cad675611cba1484e8d42015 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 18 Jan 2024 16:07:04 +0100 Subject: [PATCH 12/96] add a double error check --- .../validation/JsonSchemaValidator.groovy | 61 ++----------------- 1 file changed, 4 insertions(+), 57 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 25a2b56..7c556bd 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -35,6 +35,10 @@ public class JsonSchemaValidator { def Validator.Result result = this.validator.validate(this.schema, input) for (error : result.getErrors()) { def String errorString = error.getError() + // Skip double error in the parameter schema + if (errorString.startsWith("Value does not match against the schemas at indexes") && validationType == "parameter") { + continue + } def String location = error.getInstanceLocation() def String[] locationList = location.split("/").findAll { it != "" } def String fieldName = "" @@ -61,63 +65,6 @@ public class JsonSchemaValidator { } } - - // for (annotation : result.getAnnotations()) { - // println(annotation.getAnnotation()) - // println(annotation.getEvaluationPath()) - // println(annotation.getInstanceLocation()) - // println(annotation.getKeyword()) - // println(annotation.getSchemaLocation()) - // } - - - // Validator validator = new Validator(true) - // validator.validate(this.schema, input, validationError -> { - // // Fail on other errors than validation errors - // if(validationError !instanceof ValidationError) { - // // TODO handle this better - // log.error("* ${validationError.getMessage()}" as String) - // return - // } - - // // Get the name of the parameter and determine if it is a list entry - // def Integer entry = 0 - // def String name = '' - // def String[] uriSplit = validationError.getUri().toString().replaceFirst('#/', '').split('/') - // def String error = '' - - // if (input instanceof Map) { - // name = uriSplit.size() > 0 ? uriSplit[0..-1].join('/') : '' - // } - // else if (input instanceof List) { - // entry = uriSplit[0].toInteger() + 1 - // name = uriSplit.size() > 1 ? uriSplit[1..-1].join('/') : '' - // } - - // // Create custom error messages - // if (validationError instanceof MissingPropertyError) { - // def String paramUri = validationError.getUri().toString() - // error = "Missing required ${validationType}: ${validationError.getProperty()}" as String - // } - // else if (validationError instanceof ValidationError) { - // def String paramUri = validationError.getUri().toString() - // if (name == '') { - // this.errors.add("${validationError.getMessage()}" as String) - // return - // } - // def String value = validationError.getObject() - // def String msg = validationError.getMessage() - // error = "Error for ${validationType} '${name}' (${value}): ${msg}" as String - // } - - // // Add the error to the list - // if (entry > 0) { - // this.errors.add("* Entry ${entry}: ${error}" as String) - // } - // else { - // this.errors.add("* ${error}" as String) - // } - // }) } public static List validate(JSONArray input) { From 685f270e15ff35b67bdb685854fed7368372376a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 18 Jan 2024 16:40:20 +0100 Subject: [PATCH 13/96] error messages fixes --- .../validation/JsonSchemaValidator.groovy | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 7c556bd..3043826 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -39,29 +39,38 @@ public class JsonSchemaValidator { if (errorString.startsWith("Value does not match against the schemas at indexes") && validationType == "parameter") { continue } + + // Change some error messages to make them more clear + def String keyword = error.getKeyword() + def String customError = "" + if (keyword == "required") { + def Matcher matcher = errorString =~ ~/\[\[([^\[\]]*)\]\]$/ + def String missingKeywords = matcher.findAll().flatten().last() + customError = "Missing required ${validationType}(s): ${missingKeywords}" + } + def String location = error.getInstanceLocation() def String[] locationList = location.split("/").findAll { it != "" } - def String fieldName = "" - def String capValidationType = "${validationType[0].toUpperCase()}${validationType[1..-1]}" - if (locationList.size() > 0 && isInteger(location[0]) && validationType == "field") { - def Integer entryInteger = location[0] as Integer + if (locationList.size() > 0 && isInteger(locationList[0]) && validationType == "field") { + def Integer entryInteger = locationList[0] as Integer def String entryString = "Entry ${entryInteger + 1}" as String + def String fieldError = "" if(locationList.size() > 1) { - fieldName = "Error for '${locationList[1..-1].join("/")}'" + fieldError = "Error for ${validationType} '${locationList[1..-1].join("/")}': ${customError ?: errorString}" } else { - fieldName = "${capValidationType} error" + fieldError = customError ?: errorString } - this.errors.add("* ${entryString}: ${fieldName}: ${errorString}" as String) + this.errors.add("* ${entryString}: ${fieldError}" as String) } else if (validationType == "parameter") { - fieldName = locationList.join("/") + def String fieldName = locationList.join("/") if(fieldName != "") { - this.errors.add("* --${fieldName}: ${errorString}" as String) + this.errors.add("* --${fieldName}: ${customError ?: errorString}" as String) } else { - this.errors.add("* ${capValidationType} error: ${errorString}" as String) + this.errors.add("* ${customError ?: errorString}" as String) } } else { - this.errors.add(errorString) + this.errors.add(customError ?: errorString) } } From 155903d3680e73d0428bc81451a81f3800ff59d2 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 19 Jan 2024 11:44:11 +0100 Subject: [PATCH 14/96] add format validators --- .../validation/CustomEvaluatorFactory.groovy | 27 ++++++++++ .../FormatDirectoryPathEvaluator.groovy | 37 ++++++++++++++ .../FormatFilePathEvaluator.groovy | 37 ++++++++++++++ .../FormatFilePathPatternEvaluator.groovy | 49 +++++++++++++++++++ .../FormatPathEvaluator.groovy | 34 +++++++++++++ .../DirectoryPathValidator.groovy | 24 --------- .../FilePathPatternValidator.groovy | 34 ------------- .../FormatValidators/FilePathValidator.groovy | 24 --------- .../FormatValidators/PathValidator.groovy | 21 -------- .../validation/JsonSchemaValidator.groovy | 3 +- 10 files changed, 185 insertions(+), 105 deletions(-) create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy delete mode 100644 plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy delete mode 100644 plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy delete mode 100644 plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy delete mode 100644 plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy new file mode 100644 index 0000000..9accc54 --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy @@ -0,0 +1,27 @@ +package nextflow.validation + +import dev.harrel.jsonschema.EvaluatorFactory +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.SchemaParsingContext +import dev.harrel.jsonschema.JsonNode + +class CustomEvaluatorFactory implements EvaluatorFactory { + @Override + public Optional create(SchemaParsingContext ctx, String fieldName, JsonNode schemaNode) { + // Format evaluators + if (fieldName == "format" && schemaNode.isString()) { + def String schemaString = schemaNode.asString() + switch (schemaString) { + case "directory-path": + return Optional.of(new FormatDirectoryPathEvaluator()) + case "file-path": + return Optional.of(new FormatFilePathEvaluator()) + case "path": + return Optional.of(new FormatPathEvaluator()) + case "file-path-pattern": + return Optional.of(new FormatFilePathPatternEvaluator()) + } + } + return Optional.empty(); + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy new file mode 100644 index 0000000..8c51ab9 --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy @@ -0,0 +1,37 @@ +package nextflow.validation + +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.EvaluationContext +import dev.harrel.jsonschema.JsonNode +import nextflow.Nextflow + +import groovy.util.logging.Slf4j +import java.nio.file.Path + +@Slf4j +class FormatDirectoryPathEvaluator implements Evaluator { + // The string should be a directory + + @Override + public Evaluator.Result evaluate(EvaluationContext ctx, JsonNode node) { + // To stay consistent with other keywords, types not applicable to this keyword should succeed + if (!node.isString()) { + return Evaluator.Result.success() + } + + def String value = node.asString() + + // Skip validation of S3 paths for now + if (value.startsWith('s3://')) { + log.debug("S3 paths are not supported by 'FormatDirectoryPathEvaluator': '${value}'") + return Evaluator.Result.success() + } + + // Actual validation logic + def Path file = Nextflow.file(value) as Path + if (file.exists() && !file.isDirectory()) { + return Evaluator.Result.failure("'${value}' is not a directory, but a file" as String) + } + return Evaluator.Result.success() + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy new file mode 100644 index 0000000..2708e4c --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy @@ -0,0 +1,37 @@ +package nextflow.validation + +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.EvaluationContext +import dev.harrel.jsonschema.JsonNode +import nextflow.Nextflow + +import groovy.util.logging.Slf4j +import java.nio.file.Path + +@Slf4j +class FormatFilePathEvaluator implements Evaluator { + // The string should be a file + + @Override + public Evaluator.Result evaluate(EvaluationContext ctx, JsonNode node) { + // To stay consistent with other keywords, types not applicable to this keyword should succeed + if (!node.isString()) { + return Evaluator.Result.success() + } + + def String value = node.asString() + + // Skip validation of S3 paths for now + if (value.startsWith('s3://')) { + log.debug("S3 paths are not supported by 'FormatFilePathEvaluator': '${value}'") + return Evaluator.Result.success() + } + + // Actual validation logic + def Path file = Nextflow.file(value) as Path + if (file.exists() && file.isDirectory()) { + return Evaluator.Result.failure("'${value}' is not a file, but a directory" as String) + } + return Evaluator.Result.success() + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy new file mode 100644 index 0000000..71d9765 --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy @@ -0,0 +1,49 @@ +package nextflow.validation + +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.EvaluationContext +import dev.harrel.jsonschema.JsonNode +import nextflow.Nextflow + +import groovy.util.logging.Slf4j +import java.nio.file.Path + +@Slf4j +class FormatFilePathPatternEvaluator implements Evaluator { + // The string should be a path pattern + + @Override + public Evaluator.Result evaluate(EvaluationContext ctx, JsonNode node) { + // To stay consistent with other keywords, types not applicable to this keyword should succeed + if (!node.isString()) { + return Evaluator.Result.success() + } + + def String value = node.asString() + + // Skip validation of S3 paths for now + if (value.startsWith('s3://')) { + log.debug("S3 paths are not supported by 'FormatFilePathPatternEvaluator': '${value}'") + return Evaluator.Result.success() + } + + // Actual validation logic + println(value) + def List files = Nextflow.files(value) + println(files) + def List errors = [] + + if(files.size() == 0) { + return Evaluator.Result.failure("No files were found using the glob pattern '${value}'" as String) + } + for( file : files ) { + if (file.isDirectory()) { + errors.add("'${file.toString()}' is not a file, but a directory" as String) + } + } + if(errors.size() > 0) { + return Evaluator.Result.failure(errors.join('\n')) + } + return Evaluator.Result.success() + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy new file mode 100644 index 0000000..2749b2a --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy @@ -0,0 +1,34 @@ +package nextflow.validation + +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.EvaluationContext +import dev.harrel.jsonschema.JsonNode +import nextflow.Nextflow + +import groovy.util.logging.Slf4j +import java.nio.file.Path + +@Slf4j +class FormatPathEvaluator implements Evaluator { + // The string should be a path + + @Override + public Evaluator.Result evaluate(EvaluationContext ctx, JsonNode node) { + // To stay consistent with other keywords, types not applicable to this keyword should succeed + if (!node.isString()) { + return Evaluator.Result.success() + } + + def String value = node.asString() + + // Skip validation of S3 paths for now + if (value.startsWith('s3://')) { + log.debug("S3 paths are not supported by 'FormatPathEvaluator': '${value}'") + return Evaluator.Result.success() + } + + // Actual validation logic + def Path file = Nextflow.file(value) as Path + return Evaluator.Result.success() + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy deleted file mode 100644 index c4ab6a6..0000000 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/DirectoryPathValidator.groovy +++ /dev/null @@ -1,24 +0,0 @@ -// package nextflow.validation - -// import java.nio.file.Path -// import groovy.util.logging.Slf4j - -// import org.everit.json.schema.FormatValidator -// import nextflow.Nextflow - -// @Slf4j -// public class DirectoryPathValidator implements FormatValidator { - -// @Override -// public Optional validate(final String subject) { -// if (subject.startsWith('s3://')) { -// log.debug("S3 paths are not supported by 'DirectoryPathValidator': '${subject}'") -// return Optional.empty() -// } -// Path file = Nextflow.file(subject) as Path -// if (file.exists() && !file.isDirectory()) { -// return Optional.of("'${subject}' is not a directory, but a file" as String) -// } -// return Optional.empty() -// } -// } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy deleted file mode 100644 index 92cd8a2..0000000 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathPatternValidator.groovy +++ /dev/null @@ -1,34 +0,0 @@ -// package nextflow.validation - -// import java.nio.file.Path -// import groovy.util.logging.Slf4j - -// import org.everit.json.schema.FormatValidator -// import nextflow.Nextflow - -// @Slf4j -// public class FilePathPatternValidator implements FormatValidator { - -// @Override -// public Optional validate(final String subject) { -// if (subject.startsWith('s3://')) { -// log.debug("S3 paths are not supported by 'FilePathPatternValidator': '${subject}'") -// return Optional.empty() -// } -// ArrayList files = Nextflow.files(subject) -// ArrayList errors = [] - -// if(files.size() == 0) { -// return Optional.of("No files were found using the globbing pattern '${subject}'" as String) -// } -// for( file : files ) { -// if (file.isDirectory()) { -// errors.add("'${file.toString()}' is not a file, but a directory" as String) -// } -// } -// if(errors.size() > 0) { -// return Optional.of(errors.join('\n')) -// } -// return Optional.empty() -// } -// } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy deleted file mode 100644 index c33c4a1..0000000 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/FilePathValidator.groovy +++ /dev/null @@ -1,24 +0,0 @@ -// package nextflow.validation - -// import java.nio.file.Path -// import groovy.util.logging.Slf4j - -// import org.everit.json.schema.FormatValidator -// import nextflow.Nextflow - -// @Slf4j -// public class FilePathValidator implements FormatValidator { - -// @Override -// public Optional validate(final String subject) { -// if (subject.startsWith('s3://')) { -// log.debug("S3 paths are not supported by 'FilePathValidator': '${subject}'") -// return Optional.empty() -// } -// Path file = Nextflow.file(subject) as Path -// if (file.isDirectory()) { -// return Optional.of("'${subject}' is not a file, but a directory" as String) -// } -// return Optional.empty() -// } -// } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy deleted file mode 100644 index bc7b3f3..0000000 --- a/plugins/nf-validation/src/main/nextflow/validation/FormatValidators/PathValidator.groovy +++ /dev/null @@ -1,21 +0,0 @@ -// package nextflow.validation - -// import java.nio.file.Path -// import groovy.util.logging.Slf4j - -// import org.everit.json.schema.FormatValidator -// import nextflow.Nextflow - -// @Slf4j -// public class PathValidator implements FormatValidator { - -// @Override -// public Optional validate(final String subject) { -// if (subject.startsWith('s3://')) { -// log.debug("S3 paths are not supported by 'PathValidator': '${subject}'") -// return Optional.empty() -// } -// Path file = Nextflow.file(subject) as Path -// return Optional.empty() -// } -// } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 3043826..f2bc89d 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -27,8 +27,7 @@ public class JsonSchemaValidator { this.schema = schemaString this.validator = new ValidatorFactory() .withJsonNodeFactory(new OrgJsonNode.Factory()) - // .withEvaluatorFactory(EvaluatorFactory.compose(customFactory, new FormatEvaluatorFactory())) - // .withEvaluatorFactory() => exists keyword should be implemented here + .withEvaluatorFactory(EvaluatorFactory.compose(new CustomEvaluatorFactory(), new FormatEvaluatorFactory())) } private static validateObject(JsonNode input, String validationType) { From 8f4f6236f47a39542a77e8c567fe47aa1a3fd74c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 19 Jan 2024 11:44:27 +0100 Subject: [PATCH 15/96] remove print values --- .../CustomEvaluators/FormatFilePathPatternEvaluator.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy index 71d9765..a7d8cdd 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy @@ -28,9 +28,7 @@ class FormatFilePathPatternEvaluator implements Evaluator { } // Actual validation logic - println(value) def List files = Nextflow.files(value) - println(files) def List errors = [] if(files.size() == 0) { From d937d4f235ef6748d66bdfe84b7daf265a21aa16 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 19 Jan 2024 14:54:01 +0100 Subject: [PATCH 16/96] add exists keyword --- .../CustomEvaluators/ExistsEvaluator.groovy | 45 +++++++++++++++++++ .../validation/JsonSchemaValidator.groovy | 33 ++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy new file mode 100644 index 0000000..87f8e8b --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy @@ -0,0 +1,45 @@ +package nextflow.validation + +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.EvaluationContext +import dev.harrel.jsonschema.JsonNode +import nextflow.Nextflow + +import groovy.util.logging.Slf4j +import java.nio.file.Path + +@Slf4j +class ExistsEvaluator implements Evaluator { + // The string should be a directory + + private final Boolean exists + + ExistsEvaluator(Boolean exists) { + this.exists = exists + } + + @Override + public Evaluator.Result evaluate(EvaluationContext ctx, JsonNode node) { + // To stay consistent with other keywords, types not applicable to this keyword should succeed + if (!node.isString()) { + return Evaluator.Result.success() + } + + def String value = node.asString() + + // Skip validation of S3 paths for now + if (value.startsWith('s3://')) { + log.debug("S3 paths are not supported by 'ExistsEvaluator': '${value}'") + return Evaluator.Result.success() + } + + // Actual validation logic + def Path file = Nextflow.file(value) as Path + if (!file.exists() && this.exists == true) { + return Evaluator.Result.failure("the file or directory '${value}' does not exist" as String) + } else if(file.exists() && this.exists == false) { + return Evaluator.Result.failure("the file or directory '${value}' should not exist" as String) + } + return Evaluator.Result.success() + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index f2bc89d..b9a56cd 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -22,11 +22,44 @@ public class JsonSchemaValidator { private static String schema private static List errors = [] private static Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$') + // private static Dialect customDialect = new Dialect() { + // @Override + // public SpecificationVersion getSpecificationVersion() { + // return SpecificationVersion.DRAFT2020_12; + // } + + // @Override + // public String getMetaSchema() { + // return "https://example.com/custom/schema" + // } + + // @Override + // public EvaluatorFactory getEvaluatorFactory() { + // return new Draft2020EvaluatorFactory() + // } + + // @Override + // public Set getSupportedVocabularies() { + // return Collections.singleton("custom-vocabulary") + // } + + // @Override + // public Set getRequiredVocabularies() { + // return Collections.emptySet() + // } + + // @Override + // public Map getDefaultVocabularyObject() { + // return Collections.singletonMap("custom-vocabulary", true) + // } + // } + JsonSchemaValidator(String schemaString) { this.schema = schemaString this.validator = new ValidatorFactory() .withJsonNodeFactory(new OrgJsonNode.Factory()) + // .withDialect() .withEvaluatorFactory(EvaluatorFactory.compose(new CustomEvaluatorFactory(), new FormatEvaluatorFactory())) } From 14721ad8cc1665fa4b6e04c566840a5fae3d1a8e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 19 Jan 2024 14:54:09 +0100 Subject: [PATCH 17/96] update definitions to defs --- .../nextflow_schema_specification.md | 12 ++++++------ .../pipeline/nextflow_schema.json | 4 ++-- .../pipeline/nextflow_schema.json | 4 ++-- .../pipeline/nextflow_schema.json | 4 ++-- examples/paramsHelp/pipeline/nextflow_schema.json | 4 ++-- .../paramsSummaryLog/pipeline/nextflow_schema.json | 4 ++-- .../paramsSummaryMap/pipeline/nextflow_schema.json | 4 ++-- .../pipeline/nextflow_schema.json | 4 ++-- .../pipeline/nextflow_schema.json | 4 ++-- .../pipeline/nextflow_schema.json | 4 ++-- parameters_meta_schema.json | 4 ++-- .../validation/CustomEvaluatorFactory.groovy | 2 ++ .../nextflow/validation/SchemaValidator.groovy | 14 +++++++------- .../src/testResources/nextflow_schema.json | 12 ++++++------ .../nextflow_schema_file_path_pattern.json | 4 ++-- .../nextflow_schema_with_samplesheet.json | 2 +- ...nextflow_schema_with_samplesheet_converter.json | 2 +- ...nextflow_schema_with_samplesheet_no_header.json | 2 +- .../nextflow_schema_with_samplesheet_no_meta.json | 2 +- 19 files changed, 47 insertions(+), 45 deletions(-) diff --git a/docs/nextflow_schema/nextflow_schema_specification.md b/docs/nextflow_schema/nextflow_schema_specification.md index 586dd11..438aa4e 100644 --- a/docs/nextflow_schema/nextflow_schema_specification.md +++ b/docs/nextflow_schema/nextflow_schema_specification.md @@ -30,14 +30,14 @@ You can find more information about JSON Schema here: ## Definitions -A slightly strange use of a JSON schema standard that we use for Nextflow schema is `definitions`. +A slightly strange use of a JSON schema standard that we use for Nextflow schema is `defs`. JSON schema can group variables together in an `object`, but then the validation expects this structure to exist in the data that it is validating. In reality, we have a very long "flat" list of parameters, all at the top level of `params.foo`. -In order to give some structure to log outputs, documentation and so on, we group parameters into `definitions`. +In order to give some structure to log outputs, documentation and so on, we group parameters into `defs`. Each `definition` is an object with a title, description and so on. -However, as they are under `definitions` scope they are effectively ignored by the validation and so their nested nature is not a problem. +However, as they are under `defs` scope they are effectively ignored by the validation and so their nested nature is not a problem. We then bring the contents of each definition object back to the "flat" top level for validation using a series of `allOf` statements at the end of the schema, which reference the specific definition keys. @@ -47,7 +47,7 @@ which reference the specific definition keys. "$schema": "http://json-schema.org/draft-07/schema", "type": "object", // Definition groups - "definitions": { // (1)! + "defs": { // (1)! "my_group_of_params": { // (2)! "title": "A virtual grouping used for docs and pretty-printing", "type": "object", @@ -64,7 +64,7 @@ which reference the specific definition keys. }, // Contents of each definition group brought into main schema for validation "allOf": [ - { "$ref": "#/definitions/my_group_of_params" } // (6)! + { "$ref": "#/defs/my_group_of_params" } // (6)! ] } ``` @@ -77,7 +77,7 @@ which reference the specific definition keys. 5. Shortened here for the example, see below for full parameter specification. 6. A `$ref` line like this needs to be added for every definition group -Parameters can be described outside of the `definitions` scope, in the regular JSON Schema top-level `properties` scope. +Parameters can be described outside of the `defs` scope, in the regular JSON Schema top-level `properties` scope. However, they will be displayed as ungrouped in tools working off the schema. ## Nested parameters diff --git a/examples/fromSamplesheetBasic/pipeline/nextflow_schema.json b/examples/fromSamplesheetBasic/pipeline/nextflow_schema.json index 3efe8c4..ff6d8ba 100644 --- a/examples/fromSamplesheetBasic/pipeline/nextflow_schema.json +++ b/examples/fromSamplesheetBasic/pipeline/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -33,7 +33,7 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" } ] } diff --git a/examples/fromSamplesheetMeta/pipeline/nextflow_schema.json b/examples/fromSamplesheetMeta/pipeline/nextflow_schema.json index 3efe8c4..ff6d8ba 100644 --- a/examples/fromSamplesheetMeta/pipeline/nextflow_schema.json +++ b/examples/fromSamplesheetMeta/pipeline/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -33,7 +33,7 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" } ] } diff --git a/examples/fromSamplesheetOrder/pipeline/nextflow_schema.json b/examples/fromSamplesheetOrder/pipeline/nextflow_schema.json index 3efe8c4..ff6d8ba 100644 --- a/examples/fromSamplesheetOrder/pipeline/nextflow_schema.json +++ b/examples/fromSamplesheetOrder/pipeline/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -33,7 +33,7 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" } ] } diff --git a/examples/paramsHelp/pipeline/nextflow_schema.json b/examples/paramsHelp/pipeline/nextflow_schema.json index 3efe8c4..ff6d8ba 100644 --- a/examples/paramsHelp/pipeline/nextflow_schema.json +++ b/examples/paramsHelp/pipeline/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -33,7 +33,7 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" } ] } diff --git a/examples/paramsSummaryLog/pipeline/nextflow_schema.json b/examples/paramsSummaryLog/pipeline/nextflow_schema.json index 3efe8c4..ff6d8ba 100644 --- a/examples/paramsSummaryLog/pipeline/nextflow_schema.json +++ b/examples/paramsSummaryLog/pipeline/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -33,7 +33,7 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" } ] } diff --git a/examples/paramsSummaryMap/pipeline/nextflow_schema.json b/examples/paramsSummaryMap/pipeline/nextflow_schema.json index 3efe8c4..ff6d8ba 100644 --- a/examples/paramsSummaryMap/pipeline/nextflow_schema.json +++ b/examples/paramsSummaryMap/pipeline/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -33,7 +33,7 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" } ] } diff --git a/examples/validateParameters/pipeline/nextflow_schema.json b/examples/validateParameters/pipeline/nextflow_schema.json index 3efe8c4..ff6d8ba 100644 --- a/examples/validateParameters/pipeline/nextflow_schema.json +++ b/examples/validateParameters/pipeline/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -33,7 +33,7 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" } ] } diff --git a/examples/validationFailUnrecognisedParams/pipeline/nextflow_schema.json b/examples/validationFailUnrecognisedParams/pipeline/nextflow_schema.json index 3efe8c4..ff6d8ba 100644 --- a/examples/validationFailUnrecognisedParams/pipeline/nextflow_schema.json +++ b/examples/validationFailUnrecognisedParams/pipeline/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -33,7 +33,7 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" } ] } diff --git a/examples/validationWarnUnrecognisedParams/pipeline/nextflow_schema.json b/examples/validationWarnUnrecognisedParams/pipeline/nextflow_schema.json index 3efe8c4..ff6d8ba 100644 --- a/examples/validationWarnUnrecognisedParams/pipeline/nextflow_schema.json +++ b/examples/validationWarnUnrecognisedParams/pipeline/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -33,7 +33,7 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" } ] } diff --git a/parameters_meta_schema.json b/parameters_meta_schema.json index 5d5f58b..cb9b326 100644 --- a/parameters_meta_schema.json +++ b/parameters_meta_schema.json @@ -30,7 +30,7 @@ "type": "string", "const": "object" }, - "definitions": { + "defs": { "title": "Parameter groups", "type": "object", "patternProperties": { @@ -139,7 +139,7 @@ "properties": { "$ref": { "type": "string", - "pattern": "^#/definitions/" + "pattern": "^#/defs/" } } } diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy index 9accc54..ef29555 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy @@ -21,6 +21,8 @@ class CustomEvaluatorFactory implements EvaluatorFactory { case "file-path-pattern": return Optional.of(new FormatFilePathPatternEvaluator()) } + } else if (fieldName == "exists" && schemaNode.isBoolean()) { + return Optional.of(new ExistsEvaluator(schemaNode.asBoolean())) } return Optional.empty(); } diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index aa00122..d93e97a 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -306,14 +306,14 @@ class SchemaValidator extends PluginExtensionPoint { // Check for nextflow core params and unexpected params def slurper = new JsonSlurper() def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) - def Map schemaParams = (Map) parsed.get('definitions') + def Map schemaParams = (Map) parsed.get('defs') def specifiedParamKeys = params.keySet() // Collect expected parameters from the schema def enumsTuple = collectEnums(schemaParams) def List expectedParams = (List) enumsTuple[0] + addExpectedParams() def Map enums = (Map) enumsTuple[1] - // Collect expected parameters from the schema when parameters are specified outside of "definitions" + // Collect expected parameters from the schema when parameters are specified outside of "defs" if (parsed.containsKey('properties')) { def enumsTupleTopLevel = collectEnums(['top_level': ['properties': parsed.get('properties')]]) expectedParams += (List) enumsTupleTopLevel[0] @@ -1019,10 +1019,10 @@ class SchemaValidator extends PluginExtensionPoint { private static LinkedHashMap paramsRead(Path json_schema) throws Exception { def slurper = new JsonSlurper() def Map schema = (Map) slurper.parse( json_schema ) - def Map schema_definitions = (Map) schema.get('definitions') + def Map schema_defs = (Map) schema.get('defs') def Map schema_properties = (Map) schema.get('properties') /* Tree looks like this in nf-core schema - * definitions <- this is what the first get('definitions') gets us + * defs <- this is what the first get('defs') gets us group 1 title description @@ -1040,7 +1040,7 @@ class SchemaValidator extends PluginExtensionPoint { parameter 1 type description - * properties <- parameters can also be ungrouped, outside of definitions + * properties <- parameters can also be ungrouped, outside of defs parameter 1 type description @@ -1048,8 +1048,8 @@ class SchemaValidator extends PluginExtensionPoint { def params_map = new LinkedHashMap() // Grouped params - if (schema_definitions) { - for (group in schema_definitions) { + if (schema_defs) { + for (group in schema_defs) { def Map group_property = (Map) group.value['properties'] // Gets the property object of the group def String title = (String) group.value['title'] def sub_params = new LinkedHashMap() diff --git a/plugins/nf-validation/src/testResources/nextflow_schema.json b/plugins/nf-validation/src/testResources/nextflow_schema.json index 5c8a10d..b12ab76 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -250,19 +250,19 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" }, { - "$ref": "#/definitions/reference_genome_options" + "$ref": "#/defs/reference_genome_options" }, { - "$ref": "#/definitions/institutional_config_options" + "$ref": "#/defs/institutional_config_options" }, { - "$ref": "#/definitions/max_job_request_options" + "$ref": "#/defs/max_job_request_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/defs/generic_options" } ] } diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_file_path_pattern.json b/plugins/nf-validation/src/testResources/nextflow_schema_file_path_pattern.json index e6d6e53..006e945 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_file_path_pattern.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_file_path_pattern.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "file_patterns": { "title": "Input/output options", "type": "object", @@ -19,7 +19,7 @@ }, "allOf": [ { - "$ref": "#/definitions/file_patterns" + "$ref": "#/defs/file_patterns" } ] } diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json index 2473e18..268cd2a 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter.json index af503ce..e2bcbb2 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_header.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_header.json index 6132f32..476cb87 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_header.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_header.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_meta.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_meta.json index 1f4b660..941afc1 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_meta.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_meta.json @@ -4,7 +4,7 @@ "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", From ee1aec9d3ed54855f077d869aaad5cd1e49c9511 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 19 Jan 2024 14:56:04 +0100 Subject: [PATCH 18/96] remove old exists code --- .../validation/SchemaValidator.groovy | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index d93e97a..0a1bbb5 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -321,14 +321,6 @@ class SchemaValidator extends PluginExtensionPoint { } //=====================================================================// - // Check if files or directories exist - def List pathsToCheck = (List) collectExists(schemaParams) - pathsToCheck.each { - if (params[it]) { - pathExists(params[it].toString(), it.toString(), s3PathCheck) - } - } - def Boolean lenientMode = params.validationLenientMode ? params.validationLenientMode : false def Boolean failUnrecognisedParams = params.validationFailUnrecognisedParams ? params.validationFailUnrecognisedParams : false @@ -549,18 +541,6 @@ class SchemaValidator extends PluginExtensionPoint { def List expectedParams = (List) enumsTuple[0] + addExpectedParams() def Map enums = (Map) enumsTuple[1] - //=====================================================================// - // Check if files or directories exist - def List pathsToCheck = (List) collectExists(schemaParams) - pathsToCheck.each { String fieldName -> - for (int i=0; i < arrayJSON.size(); i++) { - def JSONObject entry = arrayJSON.getJSONObject(i) - if ( entry.has(fieldName) ) { - pathExists(entry[fieldName].toString(), " Entry ${(i+1).toString()} - ${fieldName.toString()}", s3PathCheck) - } - } - } - //=====================================================================// // Validate def List validationErrors = validator.validate(arrayJSON) @@ -575,22 +555,6 @@ class SchemaValidator extends PluginExtensionPoint { return true } - - // - // Function to check if a file or directory exists - // - List pathExists(String path, String paramName, Boolean s3PathCheck) { - if (path.startsWith('s3://') && !s3PathCheck) { - log.debug "Ignoring validation of S3 URL path '${path}'".toString() - } else { - def Path file = Nextflow.file(path) as Path - if (!file.exists()) { - errors << "* --${paramName}: the file or directory '${path}' does not exist.".toString() - } - } - } - - // // Function to collect parameters with an exists key in the schema. // From 7f7bbede40397b9117e59270e5078cc0ca636710 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Sat, 20 Jan 2024 15:09:29 +0100 Subject: [PATCH 19/96] implement schema keyword --- .../validation/CustomEvaluatorFactory.groovy | 2 + .../CustomEvaluators/ExistsEvaluator.groovy | 2 +- .../CustomEvaluators/SchemaEvaluator.groovy | 64 +++++ .../validation/JsonSchemaValidator.groovy | 53 +--- .../validation/SamplesheetConverter.groovy | 37 +-- .../validation/SchemaValidator.groovy | 242 +----------------- .../src/main/nextflow/validation/Utils.groovy | 157 ++++++++++++ 7 files changed, 241 insertions(+), 316 deletions(-) create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy create mode 100644 plugins/nf-validation/src/main/nextflow/validation/Utils.groovy diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy index ef29555..1f6fa9c 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy @@ -23,6 +23,8 @@ class CustomEvaluatorFactory implements EvaluatorFactory { } } else if (fieldName == "exists" && schemaNode.isBoolean()) { return Optional.of(new ExistsEvaluator(schemaNode.asBoolean())) + } else if (fieldName == "schema" && schemaNode.isString()) { + return Optional.of(new SchemaEvaluator(schemaNode.asString())) } return Optional.empty(); } diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy index 87f8e8b..2848190 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy @@ -10,7 +10,7 @@ import java.nio.file.Path @Slf4j class ExistsEvaluator implements Evaluator { - // The string should be a directory + // The file should or should not exist private final Boolean exists diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy new file mode 100644 index 0000000..0408727 --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -0,0 +1,64 @@ +package nextflow.validation + +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.EvaluationContext +import dev.harrel.jsonschema.JsonNode +import nextflow.Nextflow +import nextflow.Global +import groovy.json.JsonGenerator +import org.json.JSONArray + +import groovy.util.logging.Slf4j +import java.nio.file.Path +import java.nio.file.Files + +@Slf4j +class SchemaEvaluator implements Evaluator { + // Evaluate the file using the given schema + + private final String schema + + SchemaEvaluator(String schema) { + this.schema = schema + } + + @Override + public Evaluator.Result evaluate(EvaluationContext ctx, JsonNode node) { + // To stay consistent with other keywords, types not applicable to this keyword should succeed + if (!node.isString()) { + return Evaluator.Result.success() + } + + def String value = node.asString() + + // Actual validation logic + def Path file = Nextflow.file(value) + // Don't validate if the file does not exist or is a directory + if(!file.exists() && !file.isDirectory()) { + return Evaluator.Result.success() + } + + def String baseDir = Global.getSession().baseDir + def String schemaFull = Utils.getSchemaPath(baseDir, this.schema) + def List fileMaps = Utils.fileToMaps(file, this.schema, baseDir) + def String schemaContents = Files.readString( Path.of(schemaFull) ) + def validator = new JsonSchemaValidator(schemaContents) + + // Remove all null values from JSON object + // and convert the groovy object to a JSONArray + def jsonGenerator = new JsonGenerator.Options() + .excludeNulls() + .build() + def JSONArray arrayJSON = new JSONArray(jsonGenerator.toJson(fileMaps)) + + def List validationErrors = validator.validate(arrayJSON) + if (validationErrors) { + def List errors = ["Validation of '$value' file failed!" as String] + validationErrors.collect { "\t${it}" as String} + return Evaluator.Result.failure(errors.join("\n")) + } + + log.debug("Validatrion of file '${value}' passed!") + return Evaluator.Result.success() + } + +} \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index b9a56cd..76d5ac7 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -20,51 +20,19 @@ public class JsonSchemaValidator { private static ValidatorFactory validator private static String schema - private static List errors = [] private static Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$') - // private static Dialect customDialect = new Dialect() { - // @Override - // public SpecificationVersion getSpecificationVersion() { - // return SpecificationVersion.DRAFT2020_12; - // } - - // @Override - // public String getMetaSchema() { - // return "https://example.com/custom/schema" - // } - - // @Override - // public EvaluatorFactory getEvaluatorFactory() { - // return new Draft2020EvaluatorFactory() - // } - - // @Override - // public Set getSupportedVocabularies() { - // return Collections.singleton("custom-vocabulary") - // } - - // @Override - // public Set getRequiredVocabularies() { - // return Collections.emptySet() - // } - - // @Override - // public Map getDefaultVocabularyObject() { - // return Collections.singletonMap("custom-vocabulary", true) - // } - // } - JsonSchemaValidator(String schemaString) { this.schema = schemaString this.validator = new ValidatorFactory() .withJsonNodeFactory(new OrgJsonNode.Factory()) - // .withDialect() + // .withDialect() // TODO define the dialect .withEvaluatorFactory(EvaluatorFactory.compose(new CustomEvaluatorFactory(), new FormatEvaluatorFactory())) } - private static validateObject(JsonNode input, String validationType) { + private static List validateObject(JsonNode input, String validationType) { def Validator.Result result = this.validator.validate(this.schema, input) + def List errors = [] for (error : result.getErrors()) { def String errorString = error.getError() // Skip double error in the parameter schema @@ -93,31 +61,30 @@ public class JsonSchemaValidator { } else { fieldError = customError ?: errorString } - this.errors.add("* ${entryString}: ${fieldError}" as String) + errors.add("* ${entryString}: ${fieldError}" as String) } else if (validationType == "parameter") { def String fieldName = locationList.join("/") if(fieldName != "") { - this.errors.add("* --${fieldName}: ${customError ?: errorString}" as String) + errors.add("* --${fieldName}: ${customError ?: errorString}" as String) } else { - this.errors.add("* ${customError ?: errorString}" as String) + errors.add("* ${customError ?: errorString}" as String) } } else { - this.errors.add(customError ?: errorString) + errors.add(customError ?: errorString) } } + return errors } public static List validate(JSONArray input) { def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) - this.validateObject(jsonInput, "field") - return this.errors + return this.validateObject(jsonInput, "field") } public static List validate(JSONObject input) { def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) - this.validateObject(jsonInput, "parameter") - return this.errors + return this.validateObject(jsonInput, "parameter") } private static Boolean isInteger(String input) { diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index e3af031..21f731e 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -22,7 +22,6 @@ import nextflow.Nextflow import nextflow.plugin.extension.Function import nextflow.Session - @Slf4j @CompileStatic class SamplesheetConverter { @@ -61,7 +60,7 @@ class SamplesheetConverter { def List requiredFields = (List) schemaMap["items"]["required"] def Boolean containsHeader = !(allFields.size() == 1 && allFields[0] == "") - def String fileType = getFileType(samplesheetFile) + def String fileType = Utils.getFileType(samplesheetFile) def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null def List> samplesheetList @@ -213,40 +212,6 @@ class SamplesheetConverter { return outputs } - // Function to infer the file type of the samplesheet - public static String getFileType( - Path samplesheetFile - ) { - def String extension = samplesheetFile.getExtension() - if (extension in ["csv", "tsv", "yml", "yaml", "json"]) { - return extension == "yml" ? "yaml" : extension - } - - def String header = getHeader(samplesheetFile) - - def Integer commaCount = header.count(",") - def Integer tabCount = header.count("\t") - - if ( commaCount == tabCount ){ - throw new Exception("Could not derive file type from ${samplesheetFile}. Please specify the file extension (CSV, TSV, YML and YAML are supported).".toString()) - } - if ( commaCount > tabCount ){ - return "csv" - } - else { - return "tsv" - } - } - - // Function to get the header from a CSV or TSV file - public static String getHeader( - Path samplesheetFile - ) { - def String header - samplesheetFile.withReader { header = it.readLine() } - return header - } - // Function to transform an input field from the samplesheet to its desired type private static castToType( String input, diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 0a1bbb5..a8382b7 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -27,9 +27,6 @@ import org.json.JSONObject import org.json.JSONTokener import org.yaml.snakeyaml.Yaml -import static SamplesheetConverter.getHeader -import static SamplesheetConverter.getFileType - @Slf4j @CompileStatic class SchemaValidator extends PluginExtensionPoint { @@ -122,17 +119,6 @@ class SchemaValidator extends PluginExtensionPoint { boolean hasWarnings() { warnings.size()>0 } List getWarnings() { warnings } - // - // Resolve Schema path relative to main workflow directory - // - static String getSchemaPath(String baseDir, String schemaFilename='nextflow_schema.json') { - if (Path.of(schemaFilename).exists()) { - return schemaFilename - } else { - return "${baseDir}/${schemaFilename}" - } - } - // // Find a value in a nested map // @@ -154,7 +140,7 @@ class SchemaValidator extends PluginExtensionPoint { def Boolean skipDuplicateCheck = options?.containsKey('skip_duplicate_check') ? options.skip_duplicate_check as Boolean : params.validationSkipDuplicateCheck ? params.validationSkipDuplicateCheck as Boolean : false def slurper = new JsonSlurper() - def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) + def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) def Map samplesheetValue = (Map) findDeep(parsed, samplesheetParam) def Path samplesheetFile = params[samplesheetParam] as Path if (samplesheetFile == null) { @@ -167,7 +153,7 @@ class SchemaValidator extends PluginExtensionPoint { throw new SchemaValidationException("", []) } else if (samplesheetValue.containsKey('schema')) { - schemaFile = Path.of(getSchemaPath(baseDir, samplesheetValue['schema'].toString())) + schemaFile = Path.of(Utils.getSchemaPath(baseDir, samplesheetValue['schema'].toString())) } else { log.error "Parameter '--$samplesheetParam' does not contain a schema in the parameter schema ($schemaFilename). Unable to create a channel from it." throw new SchemaValidationException("", []) @@ -175,41 +161,6 @@ class SchemaValidator extends PluginExtensionPoint { log.debug "Starting validation: '$samplesheetFile' with '$schemaFile'" - // Validate samplesheet - def String fileType = SamplesheetConverter.getFileType(samplesheetFile) - def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null - def List> fileContent - def Boolean s3PathCheck = params.validationS3PathCheck ? params.validationS3PathCheck : false - def Map types = variableTypes(schemaFile.toString(), baseDir) - def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "") - - if(!containsHeader){ - types = ["empty": types[""]] - } - if(fileType == "yaml"){ - fileContent = new Yaml().load((samplesheetFile.text)).collect { - if(containsHeader) { - return it as Map - } - return ["empty": it] as Map - } - } - else if(fileType == "json"){ - fileContent = new JsonSlurper().parseText(samplesheetFile.text).collect { - if(containsHeader) { - return it as Map - } - return ["empty": it] as Map - } - } - else { - fileContent = samplesheetFile.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"') - } - def List> fileContentCasted = castToType(fileContent, types) - if (validateFile(false, samplesheetFile.toString(), fileContentCasted, schemaFile.toString(), baseDir, s3PathCheck)) { - log.debug "Validation passed: '$samplesheetFile' with '$schemaFile'" - } - // Convert to channel final channel = CH.create() List arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile, skipDuplicateCheck) @@ -305,7 +256,7 @@ class SchemaValidator extends PluginExtensionPoint { //=====================================================================// // Check for nextflow core params and unexpected params def slurper = new JsonSlurper() - def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) + def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) def Map schemaParams = (Map) parsed.get('defs') def specifiedParamKeys = params.keySet() @@ -349,7 +300,7 @@ class SchemaValidator extends PluginExtensionPoint { //=====================================================================// // Validate parameters against the schema - def String schema_string = Files.readString( Path.of(getSchemaPath(baseDir, schemaFilename)) ) + def String schema_string = Files.readString( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) def validator = new JsonSchemaValidator(schema_string) // check for warnings @@ -371,190 +322,9 @@ class SchemaValidator extends PluginExtensionPoint { throw new SchemaValidationException(msg, this.getErrors()) } - //=====================================================================// - // Look for other schemas to validate - for (group in schemaParams) { - def Map properties = (Map) group.value['properties'] - for (p in properties) { - def String key = (String) p.key - if (!params[key]) { - continue - } - def Map property = properties[key] as Map - if (property.containsKey('schema')) { - def String schema_name = getSchemaPath(baseDir, property['schema'].toString()) - def Path file_path - try { - file_path = Nextflow.file(params[key]) as Path - } catch (IllegalArgumentException e) { - errors << "* --${key}: The file path '${params[key]}' is invalid. Unable to validate file.".toString() - continue - } - log.debug "Starting validation: '$key': '$file_path' with '$schema_name'" - def String fileType = SamplesheetConverter.getFileType(file_path) - def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null - def List> fileContent - def Map types = variableTypes(schema_name, baseDir) - def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "") - - if(!containsHeader){ - types = ["empty": types[""]] - } - - if(fileType == "yaml"){ - fileContent = new Yaml().load(file_path.text).collect { - if(containsHeader) { - return it as Map - } - return ["empty": it] as Map - } - } - else if(fileType == "json"){ - fileContent = new JsonSlurper().parseText(file_path.text).collect { - if(containsHeader) { - return it as Map - } - return ["empty": it] as Map - } - } - else { - fileContent = file_path.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"') - } - def List> fileContentCasted = castToType(fileContent, types) - if (validateFile(useMonochromeLogs, key, fileContentCasted, schema_name, baseDir, s3PathCheck)) { - log.debug "Validation passed: '$key': '$file_path' with '$schema_name'" - } - } - } - } - log.debug "Finishing parameters validation" } - - // - // Function to obtain the variable types of properties from a JSON Schema - // - Map variableTypes(String schemaFilename, String baseDir) { - def Map variableTypes = [:] - def String type = '' - - // Read the schema - def slurper = new JsonSlurper() - def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) - - // Obtain the type of each variable in the schema - def Map properties = (Map) parsed['items']['properties'] - for (p in properties) { - def String key = (String) p.key - def Map property = properties[key] as Map - if (property.containsKey('type')) { - if (property['type'] == 'number') { - type = 'float' - } - else { - type = property['type'] - } - variableTypes[key] = type - } - else { - variableTypes[key] = 'string' // If there isn't a type specifyed, return 'string' to avoid having a null value - } - } - - return variableTypes - } - - - // - // Cast a value to the provided type in a Strict mode - // - Set VALID_BOOLEAN_VALUES = ['true', 'false'] as Set - - List castToType(List rows, Map types) { - def List casted = [] - - for( Map row in rows) { - def Map castedRow = [:] - - for (String key in row.keySet()) { - def String str = row[key] - def String type = types[key] - - try { - if( str == null || str == '' ) castedRow[key] = null - else if( type == null ) castedRow[key] = str - else if( type.toLowerCase() == 'boolean' && str.toLowerCase() in VALID_BOOLEAN_VALUES ) castedRow[key] = str.toBoolean() - else if( type.toLowerCase() == 'character' ) castedRow[key] = str.toCharacter() - else if( type.toLowerCase() == 'short' && str.isNumber() ) castedRow[key] = str.toShort() - else if( type.toLowerCase() == 'integer' && str.isInteger() ) castedRow[key] = str.toInteger() - else if( type.toLowerCase() == 'long' && str.isLong() ) castedRow[key] = str.toLong() - else if( type.toLowerCase() == 'float' && str.isFloat() ) castedRow[key] = str.toFloat() - else if( type.toLowerCase() == 'double' && str.isDouble() ) castedRow[key] = str.toDouble() - else if( type.toLowerCase() == 'string' ) castedRow[key] = str - else { - castedRow[key] = str - } - } catch( Exception e ) { - log.warn "Unable to cast value $str to type $type: $e" - castedRow[key] = str - } - - } - - casted = casted + castedRow - } - - return casted - } - - - // - // Function to validate a file by its schema - // - /* groovylint-disable-next-line UnusedPrivateMethodParameter */ - boolean validateFile( - - Boolean monochrome_logs, String paramName, Object fileContent, String schemaFilename, String baseDir, Boolean s3PathCheck = false - - ) { - - // Load the schema - def String schema_string = Files.readString( Path.of(getSchemaPath(baseDir, schemaFilename)) ) - def validator = new JsonSchemaValidator(schema_string) - - // Remove all null values from JSON object - // and convert the groovy object to a JSONArray - def jsonGenerator = new JsonGenerator.Options() - .excludeNulls() - .build() - def JSONArray arrayJSON = new JSONArray(jsonGenerator.toJson(fileContent)) - - //=====================================================================// - // Check for params with expected values - def slurper = new JsonSlurper() - def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) - def Map schemaParams = (Map) ["items": parsed.get('items')] - - // Collect expected parameters from the schema - def enumsTuple = collectEnums(schemaParams) - def List expectedParams = (List) enumsTuple[0] + addExpectedParams() - def Map enums = (Map) enumsTuple[1] - - //=====================================================================// - // Validate - def List validationErrors = validator.validate(arrayJSON) - this.errors.addAll(validationErrors) - if (this.hasErrors()) { - def colors = logColours(monochrome_logs) - def msg = "${colors.red}The following errors have been detected:\n\n" + this.getErrors().join('\n').trim() + "\n${colors.reset}\n" - log.error("ERROR: Validation of '$paramName' file failed!") - throw new SchemaValidationException(msg, this.getErrors()) - } - - return true - } - // // Function to collect parameters with an exists key in the schema. // @@ -637,7 +407,7 @@ class SchemaValidator extends PluginExtensionPoint { String output = '' output += 'Typical pipeline command:\n\n' output += " ${colors.cyan}${command}${colors.reset}\n\n" - Map params_map = paramsLoad( Path.of(getSchemaPath(baseDir, schemaFilename)) ) + Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) Integer max_chars = paramsMaxChars(params_map) + 1 Integer desc_indent = max_chars + 14 Integer dec_linewidth = 160 - desc_indent @@ -752,7 +522,7 @@ class SchemaValidator extends PluginExtensionPoint { // Get pipeline parameters defined in JSON Schema def Map params_summary = [:] - def Map params_map = paramsLoad( Path.of(getSchemaPath(baseDir, schemaFilename)) ) + def Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) for (group in params_map.keySet()) { def sub_params = new LinkedHashMap() def Map group_params = params_map.get(group) as Map // This gets the parameters of that particular group diff --git a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy new file mode 100644 index 0000000..887d20e --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy @@ -0,0 +1,157 @@ +package nextflow.validation + +import org.yaml.snakeyaml.Yaml +import groovy.json.JsonSlurper + +import groovy.util.logging.Slf4j +import java.nio.file.Path + +@Slf4j +public class Utils { + + // Function to infer the file type of a samplesheet + public static String getFileType(Path file) { + def String extension = file.getExtension() + if (extension in ["csv", "tsv", "yml", "yaml", "json"]) { + return extension == "yml" ? "yaml" : extension + } + + def String header = getHeader(file) + + def Integer commaCount = header.count(",") + def Integer tabCount = header.count("\t") + + if ( commaCount == tabCount ){ + log.error("Could not derive file type from ${file}. Please specify the file extension (CSV, TSV, YML, YAML and JSON are supported).".toString()) + } + if ( commaCount > tabCount ){ + return "csv" + } + else { + return "tsv" + } + } + + // Function to get the header from a CSV or TSV file + public static String getHeader(Path file) { + def String header + file.withReader { header = it.readLine() } + return header + } + + // Converts a given file to a map + public static List fileToMaps(Path file, String schemaName, String baseDir) { + def String fileType = Utils.getFileType(file) + def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null + def List> fileContent + def Map types = variableTypes(schemaName, baseDir) + def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "") + + if(!containsHeader){ + types = ["empty": types[""]] + } + if(fileType == "yaml"){ + fileContent = new Yaml().load((file.text)).collect { + if(containsHeader) { + return it as Map + } + return ["empty": it] as Map + } + } + else if(fileType == "json"){ + fileContent = new JsonSlurper().parseText(file.text).collect { + if(containsHeader) { + return it as Map + } + return ["empty": it] as Map + } + } + else { + fileContent = file.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"') + } + def List> fileContentCasted = castToType(fileContent, types) + return fileContentCasted + } + + // + // Cast a value to the provided type in a Strict mode + // + + public static List castToType(List rows, Map types) { + def List casted = [] + def Set validBooleanValues = ['true', 'false'] as Set + + for( Map row in rows) { + def Map castedRow = [:] + + for (String key in row.keySet()) { + def String str = row[key] + def String type = types[key] + + try { + if( str == null || str == '' ) castedRow[key] = null + else if( type == null ) castedRow[key] = str + else if( type.toLowerCase() == 'boolean' && str.toLowerCase() in validBooleanValues ) castedRow[key] = str.toBoolean() + else if( type.toLowerCase() == 'character' ) castedRow[key] = str.toCharacter() + else if( type.toLowerCase() == 'short' && str.isNumber() ) castedRow[key] = str.toShort() + else if( type.toLowerCase() == 'integer' && str.isInteger() ) castedRow[key] = str.toInteger() + else if( type.toLowerCase() == 'long' && str.isLong() ) castedRow[key] = str.toLong() + else if( type.toLowerCase() == 'float' && str.isFloat() ) castedRow[key] = str.toFloat() + else if( type.toLowerCase() == 'double' && str.isDouble() ) castedRow[key] = str.toDouble() + else if( type.toLowerCase() == 'string' ) castedRow[key] = str + else { + castedRow[key] = str + } + } catch( Exception e ) { + log.warn "Unable to cast value $str to type $type: $e" + castedRow[key] = str + } + + } + + casted = casted + castedRow + } + + return casted + } + + // Resolve Schema path relative to main workflow directory + public static String getSchemaPath(String baseDir, String schemaFilename='nextflow_schema.json') { + if (Path.of(schemaFilename).exists()) { + return schemaFilename + } else { + return "${baseDir}/${schemaFilename}" + } + } + + // Function to obtain the variable types of properties from a JSON Schema + public static Map variableTypes(String schemaFilename, String baseDir) { + def Map variableTypes = [:] + def String type = '' + + // Read the schema + def slurper = new JsonSlurper() + def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) + + // Obtain the type of each variable in the schema + def Map properties = (Map) parsed['items']['properties'] + for (p in properties) { + def String key = (String) p.key + def Map property = properties[key] as Map + if (property.containsKey('type')) { + if (property['type'] == 'number') { + type = 'float' + } + else { + type = property['type'] + } + variableTypes[key] = type + } + else { + variableTypes[key] = 'string' // If there isn't a type specifyed, return 'string' to avoid having a null value + } + } + + return variableTypes + } +} \ No newline at end of file From 0dba85b167fef0f7299f2cea7f35fd66fa4a6755 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Sat, 20 Jan 2024 15:15:03 +0100 Subject: [PATCH 20/96] fix for exists keyword and file-path-pattern --- .../validation/CustomEvaluators/ExistsEvaluator.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy index 2848190..f939153 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy @@ -35,6 +35,12 @@ class ExistsEvaluator implements Evaluator { // Actual validation logic def Path file = Nextflow.file(value) as Path + + // Don't evaluate file path patterns + if (file instanceof List) { + return Evaluator.Result.success() + } + if (!file.exists() && this.exists == true) { return Evaluator.Result.failure("the file or directory '${value}' does not exist" as String) } else if(file.exists() && this.exists == false) { From 967373af9d6a8a688298b13af4dc37914a1f7abf Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Sat, 20 Jan 2024 15:54:06 +0100 Subject: [PATCH 21/96] move isInteger to utils --- .../CustomEvaluators/SchemaEvaluator.groovy | 2 +- .../nextflow/validation/JsonSchemaValidator.groovy | 11 +---------- .../src/main/nextflow/validation/Utils.groovy | 9 +++++++++ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index 0408727..87d54e5 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -53,7 +53,7 @@ class SchemaEvaluator implements Evaluator { def List validationErrors = validator.validate(arrayJSON) if (validationErrors) { - def List errors = ["Validation of '$value' file failed!" as String] + validationErrors.collect { "\t${it}" as String} + def List errors = ["Validation of '$value' file failed:" as String] + validationErrors.collect { "\t${it}" as String} return Evaluator.Result.failure(errors.join("\n")) } diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 76d5ac7..20a080e 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -52,7 +52,7 @@ public class JsonSchemaValidator { def String location = error.getInstanceLocation() def String[] locationList = location.split("/").findAll { it != "" } - if (locationList.size() > 0 && isInteger(locationList[0]) && validationType == "field") { + if (locationList.size() > 0 && Utils.isInteger(locationList[0]) && validationType == "field") { def Integer entryInteger = locationList[0] as Integer def String entryString = "Entry ${entryInteger + 1}" as String def String fieldError = "" @@ -86,13 +86,4 @@ public class JsonSchemaValidator { def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) return this.validateObject(jsonInput, "parameter") } - - private static Boolean isInteger(String input) { - try { - input as Integer - return true - } catch (NumberFormatException e) { - return false - } - } } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy index 887d20e..ff94679 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy @@ -154,4 +154,13 @@ public class Utils { return variableTypes } + + public static Boolean isInteger(String input) { + try { + input as Integer + return true + } catch (NumberFormatException e) { + return false + } + } } \ No newline at end of file From c54625fe4dd89fd8264302d8a5e06ca94785cf4c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Sun, 21 Jan 2024 12:37:23 +0100 Subject: [PATCH 22/96] cleaning up and fix a typo --- .../CustomEvaluators/SchemaEvaluator.groovy | 2 +- .../validation/SchemaValidator.groovy | 120 ------------------ 2 files changed, 1 insertion(+), 121 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index 87d54e5..7771335 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -57,7 +57,7 @@ class SchemaEvaluator implements Evaluator { return Evaluator.Result.failure(errors.join("\n")) } - log.debug("Validatrion of file '${value}' passed!") + log.debug("Validation of file '${value}' passed!") return Evaluator.Result.success() } diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index a8382b7..3d5f6c6 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -325,27 +325,6 @@ class SchemaValidator extends PluginExtensionPoint { log.debug "Finishing parameters validation" } - // - // Function to collect parameters with an exists key in the schema. - // - List collectExists(Map schemaParams) { - def exists = [] - for (group in schemaParams) { - def Map properties = (Map) group.value['properties'] - for (p in properties) { - def String key = (String) p.key - def Map property = properties[key] as Map - if (property.containsKey('exists') && property.containsKey('format')) { - if (property['exists'] && (property['format'] == 'file-path' || property['format'] == 'directory-path' || property['format'] == 'path') ) { - exists.push(key) - } - } - } - } - return exists - } - - // // Function to collect enums (options) of a parameter and expected parameters (present in the schema) // @@ -603,105 +582,6 @@ class SchemaValidator extends PluginExtensionPoint { return output } - // - // Loop over nested exceptions and print the causingException - // - private void collectErrors(JSONObject exJSON, JSONObject paramsJSON, Map enums, JSONObject schemaJSON, Integer limit=5) { - def JSONArray causingExceptions = (JSONArray) exJSON['causingExceptions'] - def JSONArray valuesJSON = new JSONArray () - def String validationType = "parameter: --" - if (paramsJSON.has('objects')) { - valuesJSON = (JSONArray) paramsJSON['objects'] - validationType = "value: " - } - def Integer entryNumber = 0 - if (causingExceptions.length() == 0) { - def String pointer = (String) exJSON['pointerToViolation'] - ~/^#\// - def String message = (String) exJSON['message'] - def Pattern p = (Pattern) ~/required key \[([^\]]+)\] not found/ - def Matcher m = message =~ p - // Missing required param - if(m.matches()){ - def List l = m[0] as ArrayList - if (pointer.isNumber()) { - entryNumber = pointer.replace('/', ' - ') as Integer - entryNumber = entryNumber + 1 - errors << "* -- Entry ${entryNumber}: Missing required ${validationType}${l[1]}".toString() - } else { - errors << "* Missing required ${validationType}${l[1]}".toString() - } - } - // Other base-level error - else if(exJSON['pointerToViolation'] == '#'){ - errors << "* ${message}".toString() - } - // Error with specific param - else { - def String param = (String) exJSON['pointerToViolation'] - ~/^#\// - def String paramName = param - def param_val = "" - if (paramsJSON.has('objects')) { - def paramSplit = param.tokenize( '/' ) - int indexInt = paramSplit[0] as int - String paramString = paramSplit[1] as String - paramName = paramString - param_val = valuesJSON[indexInt][paramString].toString() - } else { - param_val = paramsJSON[param].toString() - } - if (enums.containsKey(param)) { - def error_msg = "* --${param}: '${param_val}' is not a valid choice (Available choices" - def List enums_param = (List) enums[param] - if (enums_param.size() > limit) { - errors << "${error_msg} (${limit} of ${enums_param.size()}): ${enums_param[0..limit-1].join(', ')}, ... )".toString() - } else { - errors << "${error_msg}: ${enums_param.join(', ')})".toString() - } - } else { - if (param.contains('/')) { - entryNumber = param.split('/')[0] as Integer - entryNumber = entryNumber + 1 - def String columnName = param.split('/')[1] - paramName = columnName - param = " Entry ${entryNumber} - ${columnName}" - } - // Custom errorMessage - def String errorMessage - try { - errorMessage = schemaJSON['items']['properties'][paramName]['errorMessage'] - } catch (JSONException) { - def Map paramMap = findDeep(schemaJSON.toMap(), paramName) as Map - errorMessage = paramMap['errorMessage'] - } - if (errorMessage) { - log.debug "* --${param}: ${message} (${param_val})".toString() - message = errorMessage - } - errors << "* --${param}: ${message} (${param_val})".toString() - } - } - errors.unique() - } - for (ex in causingExceptions) { - def JSONObject exception = (JSONObject) ex - collectErrors(exception, paramsJSON, enums, schemaJSON) - } - } - - // - // Remove an element from a JSONArray - // - private static JSONArray removeElement(JSONArray json_array, String element) { - def list = [] - int len = json_array.length() - for (int i=0;i Date: Tue, 23 Jan 2024 10:16:35 +0100 Subject: [PATCH 23/96] added uniqueEntries keyword --- .../validation/CustomEvaluatorFactory.groovy | 3 ++ .../UniqueEntriesEvaluator.groovy | 51 +++++++++++++++++++ .../validation/JsonSchemaValidator.groovy | 6 +-- 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy index 1f6fa9c..daab0a3 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy @@ -25,7 +25,10 @@ class CustomEvaluatorFactory implements EvaluatorFactory { return Optional.of(new ExistsEvaluator(schemaNode.asBoolean())) } else if (fieldName == "schema" && schemaNode.isString()) { return Optional.of(new SchemaEvaluator(schemaNode.asString())) + } else if (fieldName == "uniqueEntries" && schemaNode.isArray()) { + return Optional.of(new UniqueEntriesEvaluator(schemaNode.asArray())) } + return Optional.empty(); } } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy new file mode 100644 index 0000000..6b64c81 --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy @@ -0,0 +1,51 @@ +package nextflow.validation + +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.EvaluationContext +import dev.harrel.jsonschema.JsonNode +import dev.harrel.jsonschema.providers.OrgJsonNode +import org.json.JSONObject + +import groovy.json.JsonBuilder +import groovy.util.logging.Slf4j +import java.nio.file.Path + +@Slf4j +class UniqueEntriesEvaluator implements Evaluator { + // Combinations of these columns should be unique + + private final List uniqueEntries + + UniqueEntriesEvaluator(List uniqueEntries) { + this.uniqueEntries = uniqueEntries.collect { it.asString() } + } + + @Override + public Evaluator.Result evaluate(EvaluationContext ctx, JsonNode node) { + // To stay consistent with other keywords, types not applicable to this keyword should succeed + if (!node.isArray()) { + return Evaluator.Result.success() + } + + def List> uniques = [] + def Integer count = 0 + for(nodeEntry : node.asArray()) { + count++ + if(!nodeEntry.isObject()) { + return Evaluator.Result.success() + } + def Map filteredNodes = nodeEntry + .asObject() + .dropWhile { k,v -> !uniqueEntries.contains(k) } + .collectEntries { k,v -> [k, v.asString()] } + for (uniqueNode : uniques) { + if(filteredNodes.equals(uniqueNode)) { + return Evaluator.Result.failure("Detected non-unique combination of the following fields in entry ${count}: ${uniqueEntries}" as String) + } + } + uniques.add(filteredNodes) + } + + return Evaluator.Result.success() + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 20a080e..9ba0773 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -59,9 +59,9 @@ public class JsonSchemaValidator { if(locationList.size() > 1) { fieldError = "Error for ${validationType} '${locationList[1..-1].join("/")}': ${customError ?: errorString}" } else { - fieldError = customError ?: errorString + fieldError = "${customError ?: errorString}" as String } - errors.add("* ${entryString}: ${fieldError}" as String) + errors.add("-> ${entryString}: ${fieldError}" as String) } else if (validationType == "parameter") { def String fieldName = locationList.join("/") if(fieldName != "") { @@ -70,7 +70,7 @@ public class JsonSchemaValidator { errors.add("* ${customError ?: errorString}" as String) } } else { - errors.add(customError ?: errorString) + errors.add("-> ${customError ?: errorString}" as String) } } From b279de4ffb2e40eef0f4674284790eb7c9b01b25 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 23 Jan 2024 11:11:53 +0100 Subject: [PATCH 24/96] fetch values for error messages --- .../CustomEvaluators/SchemaEvaluator.groovy | 2 +- .../UniqueEntriesEvaluator.groovy | 2 +- .../validation/JsonSchemaValidator.groovy | 18 +++++++++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index 7771335..a110fe8 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -53,7 +53,7 @@ class SchemaEvaluator implements Evaluator { def List validationErrors = validator.validate(arrayJSON) if (validationErrors) { - def List errors = ["Validation of '$value' file failed:" as String] + validationErrors.collect { "\t${it}" as String} + def List errors = ["Validation of file failed:"] + validationErrors.collect { "\t${it}" as String} return Evaluator.Result.failure(errors.join("\n")) } diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy index 6b64c81..07a7bef 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy @@ -40,7 +40,7 @@ class UniqueEntriesEvaluator implements Evaluator { .collectEntries { k,v -> [k, v.asString()] } for (uniqueNode : uniques) { if(filteredNodes.equals(uniqueNode)) { - return Evaluator.Result.failure("Detected non-unique combination of the following fields in entry ${count}: ${uniqueEntries}" as String) + return Evaluator.Result.failure("Entry ${count}: Detected non-unique combination of the following fields: ${uniqueEntries}" as String) } } uniques.add(filteredNodes) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 9ba0773..83a6eac 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -4,6 +4,7 @@ import groovy.util.logging.Slf4j import groovy.transform.CompileStatic import org.json.JSONObject import org.json.JSONArray +import org.json.JSONPointer import dev.harrel.jsonschema.ValidatorFactory import dev.harrel.jsonschema.Validator import dev.harrel.jsonschema.EvaluatorFactory @@ -30,7 +31,7 @@ public class JsonSchemaValidator { .withEvaluatorFactory(EvaluatorFactory.compose(new CustomEvaluatorFactory(), new FormatEvaluatorFactory())) } - private static List validateObject(JsonNode input, String validationType) { + private static List validateObject(JsonNode input, String validationType, Object rawJson) { def Validator.Result result = this.validator.validate(this.schema, input) def List errors = [] for (error : result.getErrors()) { @@ -40,6 +41,10 @@ public class JsonSchemaValidator { continue } + def String instanceLocation = error.getInstanceLocation() + def JSONPointer pointer = new JSONPointer(instanceLocation) + def String value = pointer.queryFrom(rawJson) + // Change some error messages to make them more clear def String keyword = error.getKeyword() def String customError = "" @@ -49,15 +54,14 @@ public class JsonSchemaValidator { customError = "Missing required ${validationType}(s): ${missingKeywords}" } - def String location = error.getInstanceLocation() - def String[] locationList = location.split("/").findAll { it != "" } + def String[] locationList = instanceLocation.split("/").findAll { it != "" } if (locationList.size() > 0 && Utils.isInteger(locationList[0]) && validationType == "field") { def Integer entryInteger = locationList[0] as Integer def String entryString = "Entry ${entryInteger + 1}" as String def String fieldError = "" if(locationList.size() > 1) { - fieldError = "Error for ${validationType} '${locationList[1..-1].join("/")}': ${customError ?: errorString}" + fieldError = "Error for ${validationType} '${locationList[1..-1].join("/")}' (${value}): ${customError ?: errorString}" } else { fieldError = "${customError ?: errorString}" as String } @@ -65,7 +69,7 @@ public class JsonSchemaValidator { } else if (validationType == "parameter") { def String fieldName = locationList.join("/") if(fieldName != "") { - errors.add("* --${fieldName}: ${customError ?: errorString}" as String) + errors.add("* --${fieldName} (${value}): ${customError ?: errorString}" as String) } else { errors.add("* ${customError ?: errorString}" as String) } @@ -79,11 +83,11 @@ public class JsonSchemaValidator { public static List validate(JSONArray input) { def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) - return this.validateObject(jsonInput, "field") + return this.validateObject(jsonInput, "field", input) } public static List validate(JSONObject input) { def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) - return this.validateObject(jsonInput, "parameter") + return this.validateObject(jsonInput, "parameter", input) } } \ No newline at end of file From a3d3f93117568901c6e59c586a3081d2e676d168 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 24 Jan 2024 13:42:48 +0100 Subject: [PATCH 25/96] implemented errorMessage --- .../validation/JsonSchemaValidator.groovy | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 83a6eac..dfcdafb 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -5,6 +5,7 @@ import groovy.transform.CompileStatic import org.json.JSONObject import org.json.JSONArray import org.json.JSONPointer +import org.json.JSONPointerException import dev.harrel.jsonschema.ValidatorFactory import dev.harrel.jsonschema.Validator import dev.harrel.jsonschema.EvaluatorFactory @@ -34,6 +35,7 @@ public class JsonSchemaValidator { private static List validateObject(JsonNode input, String validationType, Object rawJson) { def Validator.Result result = this.validator.validate(this.schema, input) def List errors = [] + def JSONObject schemaObject = new JSONObject(this.schema) for (error : result.getErrors()) { def String errorString = error.getError() // Skip double error in the parameter schema @@ -45,13 +47,24 @@ public class JsonSchemaValidator { def JSONPointer pointer = new JSONPointer(instanceLocation) def String value = pointer.queryFrom(rawJson) - // Change some error messages to make them more clear - def String keyword = error.getKeyword() + // Get the errorMessage if there is one + def String schemaLocation = error.getSchemaLocation().replaceFirst(/^[^#]+/, "") + def JSONPointer schemaPointer = new JSONPointer("${schemaLocation}/errorMessage") def String customError = "" - if (keyword == "required") { - def Matcher matcher = errorString =~ ~/\[\[([^\[\]]*)\]\]$/ - def String missingKeywords = matcher.findAll().flatten().last() - customError = "Missing required ${validationType}(s): ${missingKeywords}" + try{ + customError = schemaPointer.queryFrom(schemaObject) ?: "" + } catch (JSONPointerException e) { + customError = "" + } + + // Change some error messages to make them more clear + if (customError == "") { + def String keyword = error.getKeyword() + if (keyword == "required") { + def Matcher matcher = errorString =~ ~/\[\[([^\[\]]*)\]\]$/ + def String missingKeywords = matcher.findAll().flatten().last() + customError = "Missing required ${validationType}(s): ${missingKeywords}" + } } def String[] locationList = instanceLocation.split("/").findAll { it != "" } From e00f576ec814c5f89126d88758a01260456aab93 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 24 Jan 2024 14:07:12 +0100 Subject: [PATCH 26/96] use jsonobject instead of string as schema input --- .../main/nextflow/validation/JsonSchemaValidator.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index dfcdafb..36adc6e 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -21,11 +21,11 @@ import java.util.regex.Matcher public class JsonSchemaValidator { private static ValidatorFactory validator - private static String schema + private static JSONObject schema private static Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$') JsonSchemaValidator(String schemaString) { - this.schema = schemaString + this.schema = new JSONObject(schemaString) this.validator = new ValidatorFactory() .withJsonNodeFactory(new OrgJsonNode.Factory()) // .withDialect() // TODO define the dialect @@ -35,7 +35,6 @@ public class JsonSchemaValidator { private static List validateObject(JsonNode input, String validationType, Object rawJson) { def Validator.Result result = this.validator.validate(this.schema, input) def List errors = [] - def JSONObject schemaObject = new JSONObject(this.schema) for (error : result.getErrors()) { def String errorString = error.getError() // Skip double error in the parameter schema @@ -52,7 +51,7 @@ public class JsonSchemaValidator { def JSONPointer schemaPointer = new JSONPointer("${schemaLocation}/errorMessage") def String customError = "" try{ - customError = schemaPointer.queryFrom(schemaObject) ?: "" + customError = schemaPointer.queryFrom(this.schema) ?: "" } catch (JSONPointerException e) { customError = "" } From e5dd14d62ce5b2e127089e15e75d49474e2324d2 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 24 Jan 2024 14:55:05 +0100 Subject: [PATCH 27/96] remove unused debug message --- .../src/main/nextflow/validation/SchemaValidator.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 3d5f6c6..e7f86a8 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -159,8 +159,6 @@ class SchemaValidator extends PluginExtensionPoint { throw new SchemaValidationException("", []) } - log.debug "Starting validation: '$samplesheetFile' with '$schemaFile'" - // Convert to channel final channel = CH.create() List arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile, skipDuplicateCheck) From 9cb6d1f08aafc550f20c107cfabd459b64fdfd80 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 24 Jan 2024 15:12:34 +0100 Subject: [PATCH 28/96] custom error for samplesheet conversion --- .../nextflow/validation/SchemaValidator.groovy | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index e7f86a8..dfd45ea 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -161,7 +161,20 @@ class SchemaValidator extends PluginExtensionPoint { // Convert to channel final channel = CH.create() - List arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile, skipDuplicateCheck) + def List arrayChannel = [] + try { + arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile, skipDuplicateCheck) + } catch (Exception e) { + log.error( + """ Following error has been found during samplesheet conversion: + ${e} + + Please run validateParameters() first before trying to convert a samplesheet to a channel. + Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ + """ as String + ) + } + session.addIgniter { arrayChannel.each { channel.bind(it) From 8dcbadd13fe249a9f3cba2560c13403b13e2df5e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 24 Jan 2024 15:14:58 +0100 Subject: [PATCH 29/96] custom error for samplesheet conversion --- .../src/main/nextflow/validation/SchemaValidator.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index dfd45ea..4c558db 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -169,8 +169,8 @@ class SchemaValidator extends PluginExtensionPoint { """ Following error has been found during samplesheet conversion: ${e} - Please run validateParameters() first before trying to convert a samplesheet to a channel. - Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ +Please run validateParameters() first before trying to convert a samplesheet to a channel. +Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ """ as String ) } From fcba8843ca534fb0a3f53cad2458b89f2a3c6697 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 25 Jan 2024 10:53:07 +0100 Subject: [PATCH 30/96] implement lenient mode --- .../validation/CustomEvaluatorFactory.groovy | 13 ++- .../CustomTypeEvaluator.groovy | 81 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/CustomTypeEvaluator.groovy diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy index daab0a3..b5282be 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy @@ -1,14 +1,21 @@ package nextflow.validation +import nextflow.Global import dev.harrel.jsonschema.EvaluatorFactory import dev.harrel.jsonschema.Evaluator import dev.harrel.jsonschema.SchemaParsingContext import dev.harrel.jsonschema.JsonNode class CustomEvaluatorFactory implements EvaluatorFactory { + + private Boolean lenientMode + + CustomEvaluatorFactory() { + this.lenientMode = Global.getSession().params.validationLenientMode ?: false + } + @Override public Optional create(SchemaParsingContext ctx, String fieldName, JsonNode schemaNode) { - // Format evaluators if (fieldName == "format" && schemaNode.isString()) { def String schemaString = schemaNode.asString() switch (schemaString) { @@ -27,8 +34,10 @@ class CustomEvaluatorFactory implements EvaluatorFactory { return Optional.of(new SchemaEvaluator(schemaNode.asString())) } else if (fieldName == "uniqueEntries" && schemaNode.isArray()) { return Optional.of(new UniqueEntriesEvaluator(schemaNode.asArray())) + } else if (fieldName == "type" && (schemaNode.isString() || schemaNode.isArray()) && lenientMode) { + return Optional.of(new LenientTypeEvaluator(schemaNode)) } - return Optional.empty(); + return Optional.empty() } } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/CustomTypeEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/CustomTypeEvaluator.groovy new file mode 100644 index 0000000..2bc4079 --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/CustomTypeEvaluator.groovy @@ -0,0 +1,81 @@ +package nextflow.validation + +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.EvaluationContext +import dev.harrel.jsonschema.JsonNode +import dev.harrel.jsonschema.SimpleType +import nextflow.Nextflow + +import groovy.util.logging.Slf4j +import java.nio.file.Path +import java.util.stream.Collectors + +@Slf4j +class LenientTypeEvaluator implements Evaluator { + // Validate against the type + + // private final List types + // private final List lenientTypes = [ + // SimpleType.STRING, + // SimpleType.INTEGER, + // SimpleType.NUMBER, + // SimpleType.BOOLEAN, + // SimpleType.NULL + // ] + + // LenientTypeEvaluator(JsonNode node) { + // if(node.isString()) { + // this.types = [SimpleType.fromName(node.asString())] + // } else { + // this.types = node.asArray().collect { SimpleType.fromName(it.asString()) } + // } + // } + + // @Override + // public Result evaluate(EvaluationContext ctx, JsonNode node) { + // def SimpleType nodeType = node.getNodeType() + // if (types.contains(SimpleType.STRING) && lenientTypes.contains(nodeType)) { + // // println("Lenient: ${node.asString()}") + // return Result.success() + // } else if (types.contains(nodeType) || nodeType == SimpleType.INTEGER && types.contains(SimpleType.NUMBER)) { + // // println("Not lenient: ${node.asString()}") + // return Result.success() + // } else { + // return Result.failure("Value is [${nodeType.getName()}] but should be ${type}" as String) + // } + // } + + private final Set types + private final List lenientTypes = [ + SimpleType.STRING, + SimpleType.INTEGER, + SimpleType.NUMBER, + SimpleType.BOOLEAN, + SimpleType.NULL + ] + + LenientTypeEvaluator(JsonNode node) { + if (node.isString()) { + this.types = [SimpleType.fromName(node.asString())].toSet() + } else { + this.types = node.asArray().stream() + .map(JsonNode::asString) + .map(SimpleType::fromName) + .collect(Collectors.toSet()) + } + } + + @Override + public Result evaluate(EvaluationContext ctx, JsonNode node) { + def SimpleType nodeType = node.getNodeType() + if (types.contains(SimpleType.STRING) && lenientTypes.contains(nodeType)) { + return Result.success() + } + if (types.contains(nodeType) || nodeType == SimpleType.INTEGER && types.contains(SimpleType.NUMBER)) { + return Result.success() + } else { + def List typeNames = types.stream().map(SimpleType::getName).collect(Collectors.toList()) + return Result.failure(String.format("Value is [%s] but should be %s", nodeType.getName(), typeNames)) + } + } +} From 3a978faacc35fa4a8a9c11532ff8851aeab5aee2 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 25 Jan 2024 10:55:34 +0100 Subject: [PATCH 31/96] correctly name a file --- ...tor.groovy => LenientTypeEvaluator.groovy} | 31 ------------------- 1 file changed, 31 deletions(-) rename plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/{CustomTypeEvaluator.groovy => LenientTypeEvaluator.groovy} (57%) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/CustomTypeEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy similarity index 57% rename from plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/CustomTypeEvaluator.groovy rename to plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy index 2bc4079..aa832b6 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/CustomTypeEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy @@ -14,37 +14,6 @@ import java.util.stream.Collectors class LenientTypeEvaluator implements Evaluator { // Validate against the type - // private final List types - // private final List lenientTypes = [ - // SimpleType.STRING, - // SimpleType.INTEGER, - // SimpleType.NUMBER, - // SimpleType.BOOLEAN, - // SimpleType.NULL - // ] - - // LenientTypeEvaluator(JsonNode node) { - // if(node.isString()) { - // this.types = [SimpleType.fromName(node.asString())] - // } else { - // this.types = node.asArray().collect { SimpleType.fromName(it.asString()) } - // } - // } - - // @Override - // public Result evaluate(EvaluationContext ctx, JsonNode node) { - // def SimpleType nodeType = node.getNodeType() - // if (types.contains(SimpleType.STRING) && lenientTypes.contains(nodeType)) { - // // println("Lenient: ${node.asString()}") - // return Result.success() - // } else if (types.contains(nodeType) || nodeType == SimpleType.INTEGER && types.contains(SimpleType.NUMBER)) { - // // println("Not lenient: ${node.asString()}") - // return Result.success() - // } else { - // return Result.failure("Value is [${nodeType.getName()}] but should be ${type}" as String) - // } - // } - private final Set types private final List lenientTypes = [ SimpleType.STRING, From 78c78a8f1c6f45e930242e1e2572746c0c4ef061 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 25 Jan 2024 11:40:04 +0100 Subject: [PATCH 32/96] stay true to the real copy of TypeEvaluator --- .../validation/CustomEvaluators/LenientTypeEvaluator.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy index aa832b6..d8e1f33 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy @@ -9,6 +9,8 @@ import nextflow.Nextflow import groovy.util.logging.Slf4j import java.nio.file.Path import java.util.stream.Collectors +import static java.util.Collections.singleton +import static java.util.Collections.unmodifiableList @Slf4j class LenientTypeEvaluator implements Evaluator { @@ -25,7 +27,7 @@ class LenientTypeEvaluator implements Evaluator { LenientTypeEvaluator(JsonNode node) { if (node.isString()) { - this.types = [SimpleType.fromName(node.asString())].toSet() + this.types = singleton(SimpleType.fromName(node.asString())) } else { this.types = node.asArray().stream() .map(JsonNode::asString) @@ -43,7 +45,7 @@ class LenientTypeEvaluator implements Evaluator { if (types.contains(nodeType) || nodeType == SimpleType.INTEGER && types.contains(SimpleType.NUMBER)) { return Result.success() } else { - def List typeNames = types.stream().map(SimpleType::getName).collect(Collectors.toList()) + def List typeNames = unmodifiableList(types.stream().map(SimpleType::getName).collect(Collectors.toList())) return Result.failure(String.format("Value is [%s] but should be %s", nodeType.getName(), typeNames)) } } From bf19c0ef7a556a6be3624d361f620de6b6921aa1 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 25 Jan 2024 13:07:45 +0100 Subject: [PATCH 33/96] first set of test fixes --- .../nextflow_schema_specification.md | 2 +- .../sample_sheet_schema_examples.md | 2 +- .../sample_sheet_schema_specification.md | 2 +- docs/samplesheets/examples.md | 2 +- docs/schema_input.json | 2 +- .../pipeline/assets/schema_input.json | 2 +- .../pipeline/nextflow_schema.json | 2 +- .../pipeline/assets/schema_input.json | 2 +- .../pipeline/nextflow_schema.json | 2 +- .../pipeline/assets/schema_input.json | 2 +- .../pipeline/nextflow_schema.json | 2 +- .../paramsHelp/pipeline/nextflow_schema.json | 2 +- .../pipeline/nextflow_schema.json | 2 +- .../pipeline/nextflow_schema.json | 2 +- .../pipeline/nextflow_schema.json | 2 +- .../pipeline/nextflow_schema.json | 2 +- .../pipeline/nextflow_schema.json | 2 +- parameters_meta_schema.json | 2 +- .../validation/SchemaValidator.groovy | 2 +- .../PluginExtensionMethodsTest.groovy | 78 +++++-------------- .../src/testResources/nextflow_schema.json | 2 +- .../nextflow_schema_file_path_pattern.json | 2 +- .../nextflow_schema_with_samplesheet.json | 2 +- ...low_schema_with_samplesheet_converter.json | 2 +- ...low_schema_with_samplesheet_no_header.json | 2 +- ...tflow_schema_with_samplesheet_no_meta.json | 2 +- .../src/testResources/no_header_schema.json | 2 +- .../src/testResources/no_meta_schema.json | 2 +- .../src/testResources/samplesheet_schema.json | 2 +- .../src/testResources/schema_input.json | 2 +- 30 files changed, 48 insertions(+), 88 deletions(-) diff --git a/docs/nextflow_schema/nextflow_schema_specification.md b/docs/nextflow_schema/nextflow_schema_specification.md index 438aa4e..07a90aa 100644 --- a/docs/nextflow_schema/nextflow_schema_specification.md +++ b/docs/nextflow_schema/nextflow_schema_specification.md @@ -44,7 +44,7 @@ which reference the specific definition keys. ```json { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", // Definition groups "defs": { // (1)! diff --git a/docs/nextflow_schema/sample_sheet_schema_examples.md b/docs/nextflow_schema/sample_sheet_schema_examples.md index 95595db..14a4f30 100644 --- a/docs/nextflow_schema/sample_sheet_schema_examples.md +++ b/docs/nextflow_schema/sample_sheet_schema_examples.md @@ -20,7 +20,7 @@ You can see this, used for validating sample sheets with `--input` here: [`asset ```json { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/rnaseq/master/assets/schema_input.json", "title": "nf-core/rnaseq pipeline - params.input schema", "description": "Schema for the file provided with params.input", diff --git a/docs/nextflow_schema/sample_sheet_schema_specification.md b/docs/nextflow_schema/sample_sheet_schema_specification.md index 86a44e5..bb7dcd8 100644 --- a/docs/nextflow_schema/sample_sheet_schema_specification.md +++ b/docs/nextflow_schema/sample_sheet_schema_specification.md @@ -32,7 +32,7 @@ So, for CSV sample sheets, the top-level schema should look something like this: ```json { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "array", "items": { "type": "object", diff --git a/docs/samplesheets/examples.md b/docs/samplesheets/examples.md index ba90bb6..434e830 100644 --- a/docs/samplesheets/examples.md +++ b/docs/samplesheets/examples.md @@ -50,7 +50,7 @@ Sometimes you only have one possible input in the pipeline samplesheet. In this ```json { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Schema for the file provided with params.input", "type": "array", "items": { diff --git a/docs/schema_input.json b/docs/schema_input.json index 2673539..a3c9208 100644 --- a/docs/schema_input.json +++ b/docs/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nextflow-io/nf-validation/master/plugins/nf-validation/src/testResources/schema_input.json", "title": "Samplesheet validation schema", "description": "Schema for the samplesheet used in this pipeline", diff --git a/examples/fromSamplesheetBasic/pipeline/assets/schema_input.json b/examples/fromSamplesheetBasic/pipeline/assets/schema_input.json index 06b0bb1..d99e614 100644 --- a/examples/fromSamplesheetBasic/pipeline/assets/schema_input.json +++ b/examples/fromSamplesheetBasic/pipeline/assets/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-validation/example/master/assets/schema_input.json", "title": "nf-validation example - params.input schema", "description": "Schema for the file provided with params.input", diff --git a/examples/fromSamplesheetBasic/pipeline/nextflow_schema.json b/examples/fromSamplesheetBasic/pipeline/nextflow_schema.json index ff6d8ba..6096ceb 100644 --- a/examples/fromSamplesheetBasic/pipeline/nextflow_schema.json +++ b/examples/fromSamplesheetBasic/pipeline/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/examples/fromSamplesheetMeta/pipeline/assets/schema_input.json b/examples/fromSamplesheetMeta/pipeline/assets/schema_input.json index a0d8a02..78b9c00 100644 --- a/examples/fromSamplesheetMeta/pipeline/assets/schema_input.json +++ b/examples/fromSamplesheetMeta/pipeline/assets/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-validation/example/master/assets/schema_input.json", "title": "nf-validation example - params.input schema", "description": "Schema for the file provided with params.input", diff --git a/examples/fromSamplesheetMeta/pipeline/nextflow_schema.json b/examples/fromSamplesheetMeta/pipeline/nextflow_schema.json index ff6d8ba..6096ceb 100644 --- a/examples/fromSamplesheetMeta/pipeline/nextflow_schema.json +++ b/examples/fromSamplesheetMeta/pipeline/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/examples/fromSamplesheetOrder/pipeline/assets/schema_input.json b/examples/fromSamplesheetOrder/pipeline/assets/schema_input.json index 8d1ea9d..9e0f28e 100644 --- a/examples/fromSamplesheetOrder/pipeline/assets/schema_input.json +++ b/examples/fromSamplesheetOrder/pipeline/assets/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-validation/example/master/assets/schema_input.json", "title": "nf-validation example - params.input schema", "description": "Schema for the file provided with params.input", diff --git a/examples/fromSamplesheetOrder/pipeline/nextflow_schema.json b/examples/fromSamplesheetOrder/pipeline/nextflow_schema.json index ff6d8ba..6096ceb 100644 --- a/examples/fromSamplesheetOrder/pipeline/nextflow_schema.json +++ b/examples/fromSamplesheetOrder/pipeline/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/examples/paramsHelp/pipeline/nextflow_schema.json b/examples/paramsHelp/pipeline/nextflow_schema.json index ff6d8ba..6096ceb 100644 --- a/examples/paramsHelp/pipeline/nextflow_schema.json +++ b/examples/paramsHelp/pipeline/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/examples/paramsSummaryLog/pipeline/nextflow_schema.json b/examples/paramsSummaryLog/pipeline/nextflow_schema.json index ff6d8ba..6096ceb 100644 --- a/examples/paramsSummaryLog/pipeline/nextflow_schema.json +++ b/examples/paramsSummaryLog/pipeline/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/examples/paramsSummaryMap/pipeline/nextflow_schema.json b/examples/paramsSummaryMap/pipeline/nextflow_schema.json index ff6d8ba..6096ceb 100644 --- a/examples/paramsSummaryMap/pipeline/nextflow_schema.json +++ b/examples/paramsSummaryMap/pipeline/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/examples/validateParameters/pipeline/nextflow_schema.json b/examples/validateParameters/pipeline/nextflow_schema.json index ff6d8ba..6096ceb 100644 --- a/examples/validateParameters/pipeline/nextflow_schema.json +++ b/examples/validateParameters/pipeline/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/examples/validationFailUnrecognisedParams/pipeline/nextflow_schema.json b/examples/validationFailUnrecognisedParams/pipeline/nextflow_schema.json index ff6d8ba..6096ceb 100644 --- a/examples/validationFailUnrecognisedParams/pipeline/nextflow_schema.json +++ b/examples/validationFailUnrecognisedParams/pipeline/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/examples/validationWarnUnrecognisedParams/pipeline/nextflow_schema.json b/examples/validationWarnUnrecognisedParams/pipeline/nextflow_schema.json index ff6d8ba..6096ceb 100644 --- a/examples/validationWarnUnrecognisedParams/pipeline/nextflow_schema.json +++ b/examples/validationWarnUnrecognisedParams/pipeline/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/parameters_meta_schema.json b/parameters_meta_schema.json index cb9b326..cf26192 100644 --- a/parameters_meta_schema.json +++ b/parameters_meta_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://nextflow.io", "title": "Nextflow Schema Meta-schema", "description": "Meta-schema to validate Nextflow parameter schema files", diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 4c558db..c27c1e0 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -329,7 +329,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ this.errors.addAll(validationErrors) if (this.hasErrors()) { def msg = "${colors.red}The following invalid input values have been detected:\n\n" + errors.join('\n').trim() + "\n${colors.reset}\n" - log.error("ERROR: Validation of pipeline parameters failed!") + log.error("Validation of pipeline parameters failed!") throw new SchemaValidationException(msg, this.getErrors()) } diff --git a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy index 5942d95..ccdb18e 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy @@ -92,7 +92,11 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) - error.message == "The following invalid input values have been detected:\n\n* Missing required parameter: --input\n* Missing required parameter: --outdir\n\n" + error.message == """The following invalid input values have been detected: + +* Missing required parameter(s): input, outdir + +""" !stdout } @@ -431,7 +435,11 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) - error.message == "The following invalid input values have been detected:\n\n* --outdir: expected type: String, found: Integer (10)\n\n" + error.message == """The following invalid input values have been detected: + +* --outdir (10): Value is [integer] but should be [string] + +""" !stdout } @@ -460,33 +468,6 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ !stdout } - def 'should find validation errors for enum' () { - given: - def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() - def SCRIPT_TEXT = """ - params.monochrome_logs = true - params.input = 'src/testResources/correct.csv' - params.outdir = 'src/testResources/testDir' - params.publish_dir_mode = 'incorrect' - params.max_time = '10.day' - include { validateParameters } from 'plugin/nf-validation' - - validateParameters(parameters_schema: '$schema', monochrome_logs: params.monochrome_logs) - """ - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } - - then: - def error = thrown(SchemaValidationException) - error.message == "The following invalid input values have been detected:\n\n* --publish_dir_mode: 'incorrect' is not a valid choice (Available choices (5 of 6): symlink, rellink, link, copy, copyNoFollow, ... )\n\n" - !stdout - } - def 'correct validation of integers' () { given: def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() @@ -560,9 +541,8 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() def SCRIPT_TEXT = """ params.input = 'src/testResources/correct.csv' - params.outdir = 'src/testResources/testDir' + params.outdir = 1 params.validationLenientMode = true - params.max_cpus = '4' include { validateParameters } from 'plugin/nf-validation' validateParameters(parameters_schema: '$schema') @@ -602,33 +582,7 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) - error.message == "The following invalid input values have been detected:\n\n* --max_cpus: expected type: Integer, found: BigDecimal (1.2)\n\n" - !stdout - } - - def 'should fail because of wrong pattern' () { - given: - def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() - def SCRIPT_TEXT = """ - params.monochrome_logs = true - params.input = 'src/testResources/correct.csv' - params.outdir = 'src/testResources/testDir' - params.max_memory = '10' - include { validateParameters } from 'plugin/nf-validation' - - validateParameters(parameters_schema: '$schema', monochrome_logs: params.monochrome_logs) - """ - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } - - then: - def error = thrown(SchemaValidationException) - error.message == '''The following invalid input values have been detected:\n\n* --max_memory: string [10] does not match pattern ^\\d+(\\.\\d+)?\\.?\\s*(K|M|G|T)?B$ (10)\n\n''' + error.message == "The following invalid input values have been detected:\n\n* --max_cpus (1.2): Value is [number] but should be [integer]\n\n" !stdout } @@ -864,7 +818,13 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) - error.message == '''The following errors have been detected:\n\n* -- Entry 1: Missing required value: sample\n* -- Entry 2: Missing required value: sample\n\n''' + error.message == ''' +The following invalid input values have been detected: +* --input (src/testResources/samplesheet_no_required.csv): Validation of file failed: + -> Entry 1: Missing required value: sample + -> Entry 2: Missing required value: sample + +''' !stdout } } diff --git a/plugins/nf-validation/src/testResources/nextflow_schema.json b/plugins/nf-validation/src/testResources/nextflow_schema.json index b12ab76..5bb7c6e 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_file_path_pattern.json b/plugins/nf-validation/src/testResources/nextflow_schema_file_path_pattern.json index 006e945..e0fc8a4 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_file_path_pattern.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_file_path_pattern.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json index 268cd2a..9a8dd14 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter.json index e2bcbb2..b411b16 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_converter.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_header.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_header.json index 476cb87..c37c0b4 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_header.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_header.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_meta.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_meta.json index 941afc1..6c7a022 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_meta.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_no_meta.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", diff --git a/plugins/nf-validation/src/testResources/no_header_schema.json b/plugins/nf-validation/src/testResources/no_header_schema.json index 89194a4..83d1281 100644 --- a/plugins/nf-validation/src/testResources/no_header_schema.json +++ b/plugins/nf-validation/src/testResources/no_header_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Schema for the file provided with params.input", "type": "array", "items": { diff --git a/plugins/nf-validation/src/testResources/no_meta_schema.json b/plugins/nf-validation/src/testResources/no_meta_schema.json index ba22d76..e3a028d 100644 --- a/plugins/nf-validation/src/testResources/no_meta_schema.json +++ b/plugins/nf-validation/src/testResources/no_meta_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Schema for the file provided with params.input", "type": "array", "items": { diff --git a/plugins/nf-validation/src/testResources/samplesheet_schema.json b/plugins/nf-validation/src/testResources/samplesheet_schema.json index f66ed98..580e47f 100644 --- a/plugins/nf-validation/src/testResources/samplesheet_schema.json +++ b/plugins/nf-validation/src/testResources/samplesheet_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Schema for the file provided with params.input", "type": "array", "items": { diff --git a/plugins/nf-validation/src/testResources/schema_input.json b/plugins/nf-validation/src/testResources/schema_input.json index d955d51..a3584fe 100644 --- a/plugins/nf-validation/src/testResources/schema_input.json +++ b/plugins/nf-validation/src/testResources/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nextflow-io/nf-validation/master/plugins/nf-validation/src/testResources/schema_input.json", "title": "Samplesheet validation schema", "description": "Schema for the samplesheet used in this pipeline", From 045c71563b64096db1710fe3d992a93b20dc51b3 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 25 Jan 2024 14:49:26 +0100 Subject: [PATCH 34/96] fix even more tests --- .../CustomEvaluators/SchemaEvaluator.groovy | 3 + .../validation/SchemaValidator.groovy | 2 - .../PluginExtensionMethodsTest.groovy | 74 ++++++++++++------- .../SamplesheetConverterTest.groovy | 65 ---------------- .../nextflow_schema_with_samplesheet.json | 10 ++- .../testResources/samplesheet_no_required.csv | 2 +- .../src/testResources/samplesheet_schema.json | 2 + 7 files changed, 62 insertions(+), 96 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index a110fe8..6b03987 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -35,9 +35,12 @@ class SchemaEvaluator implements Evaluator { def Path file = Nextflow.file(value) // Don't validate if the file does not exist or is a directory if(!file.exists() && !file.isDirectory()) { + log.debug("Could not validate the file ${file.toString()}") return Evaluator.Result.success() } + log.debug("Started validating ${file.toString()}") + def String baseDir = Global.getSession().baseDir def String schemaFull = Utils.getSchemaPath(baseDir, this.schema) def List fileMaps = Utils.fileToMaps(file, this.schema, baseDir) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index c27c1e0..c0c350e 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -283,7 +283,6 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ } //=====================================================================// - def Boolean lenientMode = params.validationLenientMode ? params.validationLenientMode : false def Boolean failUnrecognisedParams = params.validationFailUnrecognisedParams ? params.validationFailUnrecognisedParams : false for (String specifiedParam in specifiedParamKeys) { @@ -324,7 +323,6 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ def colors = logColours(useMonochromeLogs) // Validate - // TODO find out how to enable lenient mode List validationErrors = validator.validate(paramsJSON) this.errors.addAll(validationErrors) if (this.hasErrors()) { diff --git a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy index ccdb18e..eaf6a2d 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy @@ -242,11 +242,15 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) def errorMessages = error.message.readLines() - errorMessages[0] == "\033[0;31mThe following errors have been detected:" - errorMessages[2] == "* -- Entry 1: Missing required value: sample" - errorMessages[3] == "* -- Entry 1 - strandedness: Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded' (weird)" - errorMessages[4] == "* -- Entry 1 - fastq_2: FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz' (test1_fastq2.fasta)" - errorMessages[5] == "* -- Entry 2 - sample: Sample name must be provided and cannot contain spaces (test 2)" + errorMessages[0] == "\033[0;31mThe following invalid input values have been detected:" + errorMessages[1] == "" + errorMessages[2] == "* --input (src/testResources/wrong.csv): Validation of file failed:" + errorMessages[3] == "\t-> Entry 1: Error for field 'strandedness' (weird): Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded'" + errorMessages[4] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): \"test1_fastq2.fasta\" does not match regular expression [^\\S+\\.f(ast)?q\\.gz\$]" + errorMessages[5] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): \"test1_fastq2.fasta\" is longer than 0 characters" + errorMessages[6] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" + errorMessages[7] == "\t-> Entry 1: Missing required field(s): sample" + errorMessages[8] == "\t-> Entry 2: Error for field 'sample' (test 2): Sample name must be provided and cannot contain spaces" !stdout } @@ -271,11 +275,15 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) def errorMessages = error.message.readLines() - errorMessages[0] == "\033[0;31mThe following errors have been detected:" - errorMessages[2] == "* -- Entry 1: Missing required value: sample" - errorMessages[3] == "* -- Entry 1 - strandedness: Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded' (weird)" - errorMessages[4] == "* -- Entry 1 - fastq_2: FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz' (test1_fastq2.fasta)" - errorMessages[5] == "* -- Entry 2 - sample: Sample name must be provided and cannot contain spaces (test 2)" + errorMessages[0] == "\033[0;31mThe following invalid input values have been detected:" + errorMessages[1] == "" + errorMessages[2] == "* --input (src/testResources/wrong.tsv): Validation of file failed:" + errorMessages[3] == "\t-> Entry 1: Error for field 'strandedness' (weird): Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded'" + errorMessages[4] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): \"test1_fastq2.fasta\" does not match regular expression [^\\S+\\.f(ast)?q\\.gz\$]" + errorMessages[5] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): \"test1_fastq2.fasta\" is longer than 0 characters" + errorMessages[6] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" + errorMessages[7] == "\t-> Entry 1: Missing required field(s): sample" + errorMessages[8] == "\t-> Entry 2: Error for field 'sample' (test 2): Sample name must be provided and cannot contain spaces" !stdout } @@ -300,11 +308,15 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) def errorMessages = error.message.readLines() - errorMessages[0] == "\033[0;31mThe following errors have been detected:" - errorMessages[2] == "* -- Entry 1: Missing required value: sample" - errorMessages[3] == "* -- Entry 1 - strandedness: Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded' (weird)" - errorMessages[4] == "* -- Entry 1 - fastq_2: FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz' (test1_fastq2.fasta)" - errorMessages[5] == "* -- Entry 2 - sample: Sample name must be provided and cannot contain spaces (test 2)" + errorMessages[0] == "\033[0;31mThe following invalid input values have been detected:" + errorMessages[1] == "" + errorMessages[2] == "* --input (src/testResources/wrong.yaml): Validation of file failed:" + errorMessages[3] == "\t-> Entry 1: Error for field 'strandedness' (weird): Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded'" + errorMessages[4] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): \"test1_fastq2.fasta\" does not match regular expression [^\\S+\\.f(ast)?q\\.gz\$]" + errorMessages[5] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): \"test1_fastq2.fasta\" is longer than 0 characters" + errorMessages[6] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" + errorMessages[7] == "\t-> Entry 1: Missing required field(s): sample" + errorMessages[8] == "\t-> Entry 2: Error for field 'sample' (test 2): Sample name must be provided and cannot contain spaces" !stdout } @@ -329,11 +341,15 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) def errorMessages = error.message.readLines() - errorMessages[0] == "\033[0;31mThe following errors have been detected:" - errorMessages[2] == "* -- Entry 1: Missing required value: sample" - errorMessages[3] == "* -- Entry 1 - strandedness: Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded' (weird)" - errorMessages[4] == "* -- Entry 1 - fastq_2: FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz' (test1_fastq2.fasta)" - errorMessages[5] == "* -- Entry 2 - sample: Sample name must be provided and cannot contain spaces (test 2)" + errorMessages[0] == "\033[0;31mThe following invalid input values have been detected:" + errorMessages[1] == "" + errorMessages[2] == "* --input (src/testResources/wrong.json): Validation of file failed:" + errorMessages[3] == "\t-> Entry 1: Error for field 'strandedness' (weird): Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded'" + errorMessages[4] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): \"test1_fastq2.fasta\" does not match regular expression [^\\S+\\.f(ast)?q\\.gz\$]" + errorMessages[5] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): \"test1_fastq2.fasta\" is longer than 0 characters" + errorMessages[6] == "\t-> Entry 1: Error for field 'fastq_2' (test1_fastq2.fasta): FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" + errorMessages[7] == "\t-> Entry 1: Missing required field(s): sample" + errorMessages[8] == "\t-> Entry 2: Error for field 'sample' (test 2): Sample name must be provided and cannot contain spaces" !stdout } @@ -794,7 +810,13 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) - error.message == '''The following errors have been detected:\n\n* -- Entry 1 - fastq_1: FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz' (test1_fastq1.txt)\n* -- Entry 2 - fastq_1: FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz' (test2_fastq1.txt)\n\n''' + error.message == """The following invalid input values have been detected: + +* --input (src/testResources/samplesheet_wrong_pattern.csv): Validation of file failed: +\t-> Entry 1: Error for field 'fastq_1' (test1_fastq1.txt): FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz' +\t-> Entry 2: Error for field 'fastq_1' (test2_fastq1.txt): FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz' + +""" !stdout } @@ -805,7 +827,7 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ params.monochrome_logs = true params.input = 'src/testResources/samplesheet_no_required.csv' include { validateParameters } from 'plugin/nf-validation' - + validateParameters(parameters_schema: '$schema', monochrome_logs: params.monochrome_logs) """ @@ -818,11 +840,11 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) - error.message == ''' -The following invalid input values have been detected: + error.message == '''The following invalid input values have been detected: + * --input (src/testResources/samplesheet_no_required.csv): Validation of file failed: - -> Entry 1: Missing required value: sample - -> Entry 2: Missing required value: sample +\t-> Entry 1: Missing required field(s): sample +\t-> Entry 2: Missing required field(s): strandedness, sample ''' !stdout diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy index 444ca15..d2e1faa 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy @@ -318,71 +318,6 @@ class SamplesheetConverterTest extends Dsl2Spec{ stdout.contains("[test1, test2]") } - def 'errors' () { - given: - def SCRIPT_TEXT = ''' - include { fromSamplesheet } from 'plugin/nf-validation' - - params.input = 'src/testResources/errors.csv' - - workflow { - Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter.json").view() - } - ''' - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.startsWith('[[') ? it : null } - - then: - def error = thrown(SchemaValidationException) - def errorMessages = error.message.readLines() - errorMessages[0] == "Samplesheet errors:" - errorMessages[1] == "\tEntry 1: [field_2, field_3] field(s) should be defined when 'field_1' is specified, but the field(s) [field_2] is/are not defined." - errorMessages[2] == "\tEntry 3: The 'field_10' value needs to be unique. 'non_unique' was found at least twice in the samplesheet." - errorMessages[3] == "\tEntry 3: The combination of 'field_11' with fields [field_10] needs to be unique. [field_11:1, field_10:non_unique] was found at least twice." - !stdout - } - - def 'errors before channel conversion' () { - given: - def SCRIPT_TEXT = ''' - include { fromSamplesheet } from 'plugin/nf-validation' - - params.input = 'src/testResources/errorsBeforeConversion.csv' - - workflow { - Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter.json").view() - } - ''' - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.startsWith('[[') ? it : null } - - then: - def error = thrown(SchemaValidationException) - def errorMessages = error.message.readLines() - errorMessages[0] == "\033[0;31mThe following errors have been detected:" - errorMessages[2] == "* -- Entry 1 - field_9: the file or directory 'non_existing_path' does not exist." - errorMessages[3] == "* -- Entry 1 - field_7: the file or directory 'non_existing_file.tsv' does not exist." - errorMessages[4] == '* -- Entry 1 - field_7: string [non_existing_file.tsv] does not match pattern ^.*\\.txt$ (non_existing_file.tsv)' - errorMessages[5] == "* -- Entry 1 - field_8: 'src/testResources/test.txt' is not a directory, but a file (src/testResources/test.txt)" - errorMessages[6] == "* -- Entry 1 - field_5: expected type: Number, found: String (string)" - errorMessages[7] == "* -- Entry 1 - field_6: expected type: Boolean, found: String (20)" - errorMessages[8] == "* -- Entry 2: Missing required value: field_4" - errorMessages[9] == "* -- Entry 2: Missing required value: field_6" - errorMessages[10] == "* -- Entry 3 - field_3: expected type: Boolean, found: String (3333)" - errorMessages[11] == "* -- Entry 3 - field_2: expected type: Integer, found: String (false)" - !stdout - } - def 'duplicates' () { given: def SCRIPT_TEXT = ''' diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json index 9a8dd14..f88e51a 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json @@ -20,9 +20,15 @@ "schema": "src/testResources/samplesheet_schema.json", "description": "Path to comma-separated file containing information about the samples in the experiment.", "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/testpipeline/usage#samplesheet-input).", - "fa_icon": "fas fa-file-csv" + "fa_icon": "fas fa-file-csv", + "exists": true } } } - } + }, + "allOf": [ + { + "$ref": "#/defs/input_output_options" + } + ] } diff --git a/plugins/nf-validation/src/testResources/samplesheet_no_required.csv b/plugins/nf-validation/src/testResources/samplesheet_no_required.csv index 900cad9..2b6af26 100644 --- a/plugins/nf-validation/src/testResources/samplesheet_no_required.csv +++ b/plugins/nf-validation/src/testResources/samplesheet_no_required.csv @@ -1,3 +1,3 @@ fastq_1,fastq_2,strandedness test1_fastq1.fastq.gz,test1_fastq2.fastq.gz,forward -test2_fastq1.fastq.gz,,forward +test2_fastq1.fastq.gz,, diff --git a/plugins/nf-validation/src/testResources/samplesheet_schema.json b/plugins/nf-validation/src/testResources/samplesheet_schema.json index 580e47f..d727b06 100644 --- a/plugins/nf-validation/src/testResources/samplesheet_schema.json +++ b/plugins/nf-validation/src/testResources/samplesheet_schema.json @@ -1,5 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/test/test/master/assets/schema_input.json", + "title": "Test schema for samplesheets", "description": "Schema for the file provided with params.input", "type": "array", "items": { From 0bbe687497a51c42295e273e0ea2b5a51632e71a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 25 Jan 2024 14:52:23 +0100 Subject: [PATCH 35/96] remove validation tests for samplesheet conversion --- .../validation/SamplesheetConverter.groovy | 8 +----- .../validation/SchemaValidator.groovy | 8 ++---- .../SamplesheetConverterTest.groovy | 27 ------------------- 3 files changed, 3 insertions(+), 40 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index 21f731e..263087b 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -50,8 +50,7 @@ class SamplesheetConverter { static List convertToList( Path samplesheetFile, - Path schemaFile, - Boolean skipDuplicateCheck + Path schemaFile ) { def Map schemaMap = (Map) new JsonSlurper().parseText(schemaFile.text) @@ -110,11 +109,6 @@ class SamplesheetConverter { } } - // Check for row uniqueness - if(!skipDuplicateCheck && this.rows.contains(row)) { - def Integer firstDuplicate = this.rows.findIndexOf { it == row } - this.errors << "The samplesheet contains duplicate rows for entry ${firstDuplicate + 1} and entry ${getCount()} (${row})".toString() - } this.rows.add(row) def Map meta = [:] diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index c0c350e..daf3e58 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -137,7 +137,6 @@ class SchemaValidator extends PluginExtensionPoint { // Set defaults for optional inputs def String schemaFilename = options?.containsKey('parameters_schema') ? options.parameters_schema as String : 'nextflow_schema.json' - def Boolean skipDuplicateCheck = options?.containsKey('skip_duplicate_check') ? options.skip_duplicate_check as Boolean : params.validationSkipDuplicateCheck ? params.validationSkipDuplicateCheck as Boolean : false def slurper = new JsonSlurper() def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) @@ -163,7 +162,7 @@ class SchemaValidator extends PluginExtensionPoint { final channel = CH.create() def List arrayChannel = [] try { - arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile, skipDuplicateCheck) + arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile) } catch (Exception e) { log.error( """ Following error has been found during samplesheet conversion: @@ -204,9 +203,6 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ if( !params.containsKey("validationSchemaIgnoreParams") ) { params.validationSchemaIgnoreParams = false } - if( !params.containsKey("validationSkipDuplicateCheck") ) { - params.validationSkipDuplicateCheck = false - } if( !params.containsKey("validationS3PathCheck") ) { params.validationS3PathCheck = false } @@ -258,7 +254,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ false def String schemaFilename = options?.containsKey('parameters_schema') ? options.parameters_schema as String : 'nextflow_schema.json' log.debug "Starting parameters validation" - + // Clean the parameters def cleanedParams = cleanParameters(params) // Convert to JSONObject diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy index d2e1faa..23b5c7b 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy @@ -317,31 +317,4 @@ class SamplesheetConverterTest extends Dsl2Spec{ noExceptionThrown() stdout.contains("[test1, test2]") } - - def 'duplicates' () { - given: - def SCRIPT_TEXT = ''' - include { fromSamplesheet } from 'plugin/nf-validation' - - params.input = 'src/testResources/duplicate.csv' - - workflow { - Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter.json").view() - } - ''' - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.startsWith('[[') ? it : null } - - then: - def error = thrown(SchemaValidationException) - def errorMessages = error.message.readLines() - errorMessages[0] == "Samplesheet errors:" - errorMessages[4] == "\tThe samplesheet contains duplicate rows for entry 2 and entry 3 ([field_4:string1, field_5:25, field_6:false])" - !stdout - } } From f296a124a292e0e11cdb77c44b339e4d99674305 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 25 Jan 2024 15:06:25 +0100 Subject: [PATCH 36/96] modularize the tests --- .../validation/SchemaValidator.groovy | 2 +- .../validation/NfValidationTest.groovy | 75 +++++++ .../nextflow/validation/ParamsHelpTest.groovy | 177 ++++++++++++++++ .../validation/ParamsSummaryLogTest.groovy | 93 ++++++++ ...t.groovy => ValidateParametersTest.groovy} | 198 +----------------- 5 files changed, 354 insertions(+), 191 deletions(-) create mode 100644 plugins/nf-validation/src/test/nextflow/validation/NfValidationTest.groovy create mode 100644 plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy create mode 100644 plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy rename plugins/nf-validation/src/test/nextflow/validation/{PluginExtensionMethodsTest.groovy => ValidateParametersTest.groovy} (80%) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index daf3e58..512ca76 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -170,7 +170,7 @@ class SchemaValidator extends PluginExtensionPoint { Please run validateParameters() first before trying to convert a samplesheet to a channel. Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ - """ as String +""" as String ) } diff --git a/plugins/nf-validation/src/test/nextflow/validation/NfValidationTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/NfValidationTest.groovy new file mode 100644 index 0000000..5582b5e --- /dev/null +++ b/plugins/nf-validation/src/test/nextflow/validation/NfValidationTest.groovy @@ -0,0 +1,75 @@ +package nextflow.validation + +import java.nio.file.Path + +import nextflow.plugin.Plugins +import nextflow.plugin.TestPluginDescriptorFinder +import nextflow.plugin.TestPluginManager +import nextflow.plugin.extension.PluginExtensionProvider +import org.junit.Rule +import org.pf4j.PluginDescriptorFinder +import spock.lang.Shared +import test.Dsl2Spec +import test.OutputCapture +/** + * @author : jorge + * + */ +class NfValidationTest extends Dsl2Spec{ + + @Rule + OutputCapture capture = new OutputCapture() + + + @Shared String pluginsMode + + def setup() { + // reset previous instances + PluginExtensionProvider.reset() + // this need to be set *before* the plugin manager class is created + pluginsMode = System.getProperty('pf4j.mode') + System.setProperty('pf4j.mode', 'dev') + // the plugin root should + def root = Path.of('.').toAbsolutePath().normalize() + def manager = new TestPluginManager(root){ + @Override + protected PluginDescriptorFinder createPluginDescriptorFinder() { + return new TestPluginDescriptorFinder(){ + @Override + protected Path getManifestPath(Path pluginPath) { + return pluginPath.resolve('build/resources/main/META-INF/MANIFEST.MF') + } + } + } + } + Plugins.init(root, 'dev', manager) + } + + def cleanup() { + Plugins.stop() + PluginExtensionProvider.reset() + pluginsMode ? System.setProperty('pf4j.mode',pluginsMode) : System.clearProperty('pf4j.mode') + } + + // + // Params validation tests + // + + def 'should import functions' () { + given: + def SCRIPT_TEXT = ''' + include { validateParameters } from 'plugin/nf-validation' + ''' + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + noExceptionThrown() + !stdout + } +} diff --git a/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy new file mode 100644 index 0000000..ec1853a --- /dev/null +++ b/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy @@ -0,0 +1,177 @@ +package nextflow.validation + +import java.nio.file.Path + +import nextflow.plugin.Plugins +import nextflow.plugin.TestPluginDescriptorFinder +import nextflow.plugin.TestPluginManager +import nextflow.plugin.extension.PluginExtensionProvider +import org.junit.Rule +import org.pf4j.PluginDescriptorFinder +import spock.lang.Shared +import test.Dsl2Spec +import test.OutputCapture + +/** + * @author : Nicolas Vannieuwkerke + * + */ +class ParamsHelpTest extends Dsl2Spec{ + + @Rule + OutputCapture capture = new OutputCapture() + + + @Shared String pluginsMode + + Path root = Path.of('.').toAbsolutePath().normalize() + Path getRoot() { this.root } + String getRootString() { this.root.toString() } + + def setup() { + // reset previous instances + PluginExtensionProvider.reset() + // this need to be set *before* the plugin manager class is created + pluginsMode = System.getProperty('pf4j.mode') + System.setProperty('pf4j.mode', 'dev') + // the plugin root should + def root = this.getRoot() + def manager = new TestPluginManager(root){ + @Override + protected PluginDescriptorFinder createPluginDescriptorFinder() { + return new TestPluginDescriptorFinder(){ + @Override + protected Path getManifestPath(Path pluginPath) { + return pluginPath.resolve('build/resources/main/META-INF/MANIFEST.MF') + } + } + } + } + Plugins.init(root, 'dev', manager) + } + + def cleanup() { + Plugins.stop() + PluginExtensionProvider.reset() + pluginsMode ? System.setProperty('pf4j.mode',pluginsMode) : System.clearProperty('pf4j.mode') + } + + def 'should print a help message' () { + given: + def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + include { paramsHelp } from 'plugin/nf-validation' + + def command = "nextflow run --input samplesheet.csv --outdir -profile docker" + + def help_msg = paramsHelp(command, parameters_schema: '$schema') + log.info help_msg + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('Typical pipeline command:') || + it.contains('nextflow run') || + it.contains('Input/output options') || + it.contains('--input') || + it.contains('--outdir') || + it.contains('--email') || + it.contains('--multiqc_title') || + it.contains('Reference genome options') || + it.contains('--genome') || + it.contains('--fasta') + ? it : null } + + then: + noExceptionThrown() + stdout.size() == 10 + } + + def 'should print a help message with argument options' () { + given: + def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + include { paramsHelp } from 'plugin/nf-validation' + params.validationShowHiddenParams = true + def command = "nextflow run --input samplesheet.csv --outdir -profile docker" + + def help_msg = paramsHelp(command, parameters_schema: '$schema') + log.info help_msg + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('publish_dir_mode') && + it.contains('(accepted: symlink, rellink, link, copy, copyNoFollow') + ? it : null } + + then: + noExceptionThrown() + stdout.size() == 1 + } + + def 'should print a help message of one parameter' () { + given: + def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + include { paramsHelp } from 'plugin/nf-validation' + params.help = 'publish_dir_mode' + + def command = "nextflow run --input samplesheet.csv --outdir -profile docker" + + def help_msg = paramsHelp(command, parameters_schema: '$schema') + log.info help_msg + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.startsWith('--publish_dir_mode') || + it.contains('type :') || + it.contains('default :') || + it.contains('description:') || + it.contains('help_text :') || + it.contains('fa_icon :') || // fa_icon shouldn't be printed + it.contains('enum :') || + it.contains('hidden :') + ? it : null } + + then: + noExceptionThrown() + stdout.size() == 7 + } + + def 'should fail when help param doesnt exist' () { + given: + def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + include { paramsHelp } from 'plugin/nf-validation' + params.help = 'no_exist' + + def command = "nextflow run --input samplesheet.csv --outdir -profile docker" + + def help_msg = paramsHelp(command, parameters_schema: '$schema') + log.info help_msg + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.startsWith('--no_exist') ? it : null } + + then: + def error = thrown(Exception) + error.message == "Specified param 'no_exist' does not exist in JSON schema." + !stdout + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy new file mode 100644 index 0000000..cf2c8b1 --- /dev/null +++ b/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy @@ -0,0 +1,93 @@ +package nextflow.validation + +import java.nio.file.Path + +import nextflow.plugin.Plugins +import nextflow.plugin.TestPluginDescriptorFinder +import nextflow.plugin.TestPluginManager +import nextflow.plugin.extension.PluginExtensionProvider +import org.junit.Rule +import org.pf4j.PluginDescriptorFinder +import spock.lang.Shared +import test.Dsl2Spec +import test.OutputCapture + +/** + * @author : Nicolas Vannieuwkerke + * + */ +class ParamsSummaryLogTest extends Dsl2Spec{ + + @Rule + OutputCapture capture = new OutputCapture() + + + @Shared String pluginsMode + + Path root = Path.of('.').toAbsolutePath().normalize() + Path getRoot() { this.root } + String getRootString() { this.root.toString() } + + def setup() { + // reset previous instances + PluginExtensionProvider.reset() + // this need to be set *before* the plugin manager class is created + pluginsMode = System.getProperty('pf4j.mode') + System.setProperty('pf4j.mode', 'dev') + // the plugin root should + def root = this.getRoot() + def manager = new TestPluginManager(root){ + @Override + protected PluginDescriptorFinder createPluginDescriptorFinder() { + return new TestPluginDescriptorFinder(){ + @Override + protected Path getManifestPath(Path pluginPath) { + return pluginPath.resolve('build/resources/main/META-INF/MANIFEST.MF') + } + } + } + } + Plugins.init(root, 'dev', manager) + } + + def cleanup() { + Plugins.stop() + PluginExtensionProvider.reset() + pluginsMode ? System.setProperty('pf4j.mode',pluginsMode) : System.clearProperty('pf4j.mode') + } + + def 'should print params summary' () { + given: + def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + params.outdir = "outDir" + include { paramsSummaryLog } from 'plugin/nf-validation' + + def summary_params = paramsSummaryLog(workflow, parameters_schema: '$schema') + log.info summary_params + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('Only displaying parameters that differ from the pipeline defaults') || + it.contains('Core Nextflow options') || + it.contains('runName') || + it.contains('launchDir') || + it.contains('workDir') || + it.contains('projectDir') || + it.contains('userName') || + it.contains('profile') || + it.contains('configFiles') || + it.contains('Input/output options') || + it.contains('outdir') + ? it : null } + + then: + noExceptionThrown() + stdout.size() == 11 + stdout ==~ /.*\[0;34moutdir : .\[0;32moutDir.*/ + } +} \ No newline at end of file diff --git a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy similarity index 80% rename from plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy rename to plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy index eaf6a2d..6909b20 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/PluginExtensionMethodsTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy @@ -11,11 +11,12 @@ import org.pf4j.PluginDescriptorFinder import spock.lang.Shared import test.Dsl2Spec import test.OutputCapture + /** - * @author : jorge + * @author : Nicolas Vannieuwkerke * */ -class PluginExtensionMethodsTest extends Dsl2Spec{ +class ValidateParametersTest extends Dsl2Spec{ @Rule OutputCapture capture = new OutputCapture() @@ -23,6 +24,10 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ @Shared String pluginsMode + Path root = Path.of('.').toAbsolutePath().normalize() + Path getRoot() { this.root } + String getRootString() { this.root.toString() } + def setup() { // reset previous instances PluginExtensionProvider.reset() @@ -30,7 +35,7 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ pluginsMode = System.getProperty('pf4j.mode') System.setProperty('pf4j.mode', 'dev') // the plugin root should - def root = Path.of('.').toAbsolutePath().normalize() + def root = this.getRoot() def manager = new TestPluginManager(root){ @Override protected PluginDescriptorFinder createPluginDescriptorFinder() { @@ -51,28 +56,6 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ pluginsMode ? System.setProperty('pf4j.mode',pluginsMode) : System.clearProperty('pf4j.mode') } - // - // Params validation tests - // - - def 'should import functions' () { - given: - def SCRIPT_TEXT = ''' - include { validateParameters } from 'plugin/nf-validation' - ''' - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } - - then: - noExceptionThrown() - !stdout - } - def 'should validate when no params' () { given: def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() @@ -602,171 +585,6 @@ class PluginExtensionMethodsTest extends Dsl2Spec{ !stdout } - // - // --help argument tests - // - - def 'should print a help message' () { - given: - def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() - def SCRIPT_TEXT = """ - include { paramsHelp } from 'plugin/nf-validation' - - def command = "nextflow run --input samplesheet.csv --outdir -profile docker" - - def help_msg = paramsHelp(command, parameters_schema: '$schema') - log.info help_msg - """ - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.contains('Typical pipeline command:') || - it.contains('nextflow run') || - it.contains('Input/output options') || - it.contains('--input') || - it.contains('--outdir') || - it.contains('--email') || - it.contains('--multiqc_title') || - it.contains('Reference genome options') || - it.contains('--genome') || - it.contains('--fasta') - ? it : null } - - then: - noExceptionThrown() - stdout.size() == 10 - } - - def 'should print a help message with argument options' () { - given: - def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() - def SCRIPT_TEXT = """ - include { paramsHelp } from 'plugin/nf-validation' - params.validationShowHiddenParams = true - def command = "nextflow run --input samplesheet.csv --outdir -profile docker" - - def help_msg = paramsHelp(command, parameters_schema: '$schema') - log.info help_msg - """ - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.contains('publish_dir_mode') && - it.contains('(accepted: symlink, rellink, link, copy, copyNoFollow') - ? it : null } - - then: - noExceptionThrown() - stdout.size() == 1 - } - - def 'should print a help message of one parameter' () { - given: - def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() - def SCRIPT_TEXT = """ - include { paramsHelp } from 'plugin/nf-validation' - params.help = 'publish_dir_mode' - - def command = "nextflow run --input samplesheet.csv --outdir -profile docker" - - def help_msg = paramsHelp(command, parameters_schema: '$schema') - log.info help_msg - """ - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.startsWith('--publish_dir_mode') || - it.contains('type :') || - it.contains('default :') || - it.contains('description:') || - it.contains('help_text :') || - it.contains('fa_icon :') || // fa_icon shouldn't be printed - it.contains('enum :') || - it.contains('hidden :') - ? it : null } - - then: - noExceptionThrown() - stdout.size() == 7 - } - - def 'should fail when help param doesnt exist' () { - given: - def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() - def SCRIPT_TEXT = """ - include { paramsHelp } from 'plugin/nf-validation' - params.help = 'no_exist' - - def command = "nextflow run --input samplesheet.csv --outdir -profile docker" - - def help_msg = paramsHelp(command, parameters_schema: '$schema') - log.info help_msg - """ - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.startsWith('--no_exist') ? it : null } - - then: - def error = thrown(Exception) - error.message == "Specified param 'no_exist' does not exist in JSON schema." - !stdout - } - - // - // Summary of params tests - // - - def 'should print params summary' () { - given: - def schema = Path.of('src/testResources/nextflow_schema.json').toAbsolutePath().toString() - def SCRIPT_TEXT = """ - params.outdir = "outDir" - include { paramsSummaryLog } from 'plugin/nf-validation' - - def summary_params = paramsSummaryLog(workflow, parameters_schema: '$schema') - log.info summary_params - """ - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.contains('Only displaying parameters that differ from the pipeline defaults') || - it.contains('Core Nextflow options') || - it.contains('runName') || - it.contains('launchDir') || - it.contains('workDir') || - it.contains('projectDir') || - it.contains('userName') || - it.contains('profile') || - it.contains('configFiles') || - it.contains('Input/output options') || - it.contains('outdir') - ? it : null } - - then: - noExceptionThrown() - stdout.size() == 11 - stdout ==~ /.*\[0;34moutdir : .\[0;32moutDir.*/ - } - - // - // Samplesheet validation tests - // def 'should validate a schema from an input file' () { given: From 3ba471c344cbe3e6f15243dabab476583fdf9054 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Fri, 22 Dec 2023 00:27:39 +0000 Subject: [PATCH 37/96] fix: add explicit check for 0 value if false-y. Add tests to ensure null values still behave correctly and that the new validation works --- .../validation/SchemaValidator.groovy | 2 +- .../validation/ValidateParametersTest.groovy | 51 +++++++++++++++ .../nextflow_schema_required_numerics.json | 62 +++++++++++++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 plugins/nf-validation/src/testResources/nextflow_schema_required_numerics.json diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 512ca76..e3eb691 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -594,7 +594,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ def Map new_params = (Map) params.getClass().newInstance(params) for (p in params) { // remove anything evaluating to false - if (!p['value']) { + if (!p['value'] && p['value'] != 0) { new_params.remove(p.key) } // Cast MemoryUnit to String diff --git a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy index 6909b20..6ef0a49 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy @@ -491,6 +491,57 @@ class ValidateParametersTest extends Dsl2Spec{ !stdout } + def 'correct validation of numerics - 0' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_required_numerics.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + params.monochrome_logs = true + params.input = 'src/testResources/correct.csv' + params.outdir = 'src/testResources/testDir' + params.integer = 0 + params.number = 0 + include { validateParameters } from 'plugin/nf-validation' + + validateParameters(parameters_schema: '$schema', monochrome_logs: params.monochrome_logs) + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + noExceptionThrown() + !stdout + } + + def 'fail validation of numerics - null' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_required_numerics.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + params.monochrome_logs = true + params.input = 'src/testResources/correct.csv' + params.outdir = 'src/testResources/testDir' + include { validateParameters } from 'plugin/nf-validation' + + validateParameters(parameters_schema: '$schema', monochrome_logs: params.monochrome_logs) + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + def error = thrown(SchemaValidationException) + error.message == "The following invalid input values have been detected:\n\n* Missing required parameter: --integer\n* Missing required parameter: --number\n\n" + !stdout + } + def 'correct validation of file-path-pattern - glob' () { given: def schema = Path.of('src/testResources/nextflow_schema_file_path_pattern.json').toAbsolutePath().toString() diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_required_numerics.json b/plugins/nf-validation/src/testResources/nextflow_schema_required_numerics.json new file mode 100644 index 0000000..22c98c4 --- /dev/null +++ b/plugins/nf-validation/src/testResources/nextflow_schema_required_numerics.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", + "title": "nf-core/testpipeline pipeline parameters", + "description": "this is a test", + "type": "object", + "definitions": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["input", "outdir"], + "properties": { + "input": { + "type": "string", + "format": "file-path", + "mimetype": "text/csv", + "pattern": "^\\S+\\.(csv|tsv|yaml|json)$", + "description": "Path to comma-separated file containing information about the samples in the experiment.", + "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/testpipeline/usage#samplesheet-input).", + "fa_icon": "fas fa-file-csv" + }, + "outdir": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" + } + } + }, + "numeric_options": { + "title": "Numeric pipeline options", + "type": "object", + "fa_icon": "fas fa-dna", + "description": "Numeric options to be tested", + "required": ["integer", "number"], + "properties": { + "integer": { + "type": "integer", + "description": "An integer parameter.", + "fa_icon": "fas fa-users-cog", + "help_text": "Integer value" + }, + "number": { + "type": "number", + "description": "A number parameter.", + "fa_icon": "fas fa-users-cog", + "help_text": "Number value" + } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/input_output_options" + }, + { + "$ref": "#/definitions/numeric_options" + } + ] +} From e4635c1418a2aabc769f2803f723cd44322dcea2 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 25 Jan 2024 16:10:44 +0100 Subject: [PATCH 38/96] fix numerics tests --- .../nextflow/validation/ValidateParametersTest.groovy | 6 +++++- .../testResources/nextflow_schema_required_numerics.json | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy index 6ef0a49..ba6162b 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy @@ -538,7 +538,11 @@ class ValidateParametersTest extends Dsl2Spec{ then: def error = thrown(SchemaValidationException) - error.message == "The following invalid input values have been detected:\n\n* Missing required parameter: --integer\n* Missing required parameter: --number\n\n" + error.message == """The following invalid input values have been detected: + +* Missing required parameter(s): number, integer + +""" !stdout } diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_required_numerics.json b/plugins/nf-validation/src/testResources/nextflow_schema_required_numerics.json index 22c98c4..cc27278 100644 --- a/plugins/nf-validation/src/testResources/nextflow_schema_required_numerics.json +++ b/plugins/nf-validation/src/testResources/nextflow_schema_required_numerics.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", "title": "nf-core/testpipeline pipeline parameters", "description": "this is a test", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -53,10 +53,10 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" }, { - "$ref": "#/definitions/numeric_options" + "$ref": "#/defs/numeric_options" } ] } From 6bf066185cb235d9d00e491e31ae5a53c79062e5 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 26 Jan 2024 11:49:29 +0100 Subject: [PATCH 39/96] add a check for older drafts --- .../main/nextflow/validation/JsonSchemaValidator.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 36adc6e..4aae5e8 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -26,6 +26,15 @@ public class JsonSchemaValidator { JsonSchemaValidator(String schemaString) { this.schema = new JSONObject(schemaString) + def JSONPointer schemaPointer = new JSONPointer("#/\$schema") + def String draft = schemaPointer.queryFrom(this.schema) + if(draft != "https://json-schema.org/draft/2020-12/schema") { + log.error("""Failed to load the meta schema: +The used schema draft (${draft}) is not correct, please use \"https://json-schema.org/draft/2020-12/schema\" instead. +See here for more information: https://json-schema.org/specification#migrating-from-older-drafts +""") + throw new SchemaValidationException("", []) + } this.validator = new ValidatorFactory() .withJsonNodeFactory(new OrgJsonNode.Factory()) // .withDialect() // TODO define the dialect From fe77d5058fc19489931da50d9346733e84a765c2 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 26 Jan 2024 11:53:30 +0100 Subject: [PATCH 40/96] fix review comments --- .../nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy | 2 +- .../src/test/nextflow/validation/ValidateParametersTest.groovy | 1 + .../nf-validation/src/testResources/samplesheet_no_required.csv | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index 6b03987..bdf729c 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -34,7 +34,7 @@ class SchemaEvaluator implements Evaluator { // Actual validation logic def Path file = Nextflow.file(value) // Don't validate if the file does not exist or is a directory - if(!file.exists() && !file.isDirectory()) { + if(!file.exists() || file.isDirectory()) { log.debug("Could not validate the file ${file.toString()}") return Evaluator.Result.success() } diff --git a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy index ba6162b..69a5dac 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy @@ -718,6 +718,7 @@ class ValidateParametersTest extends Dsl2Spec{ * --input (src/testResources/samplesheet_no_required.csv): Validation of file failed: \t-> Entry 1: Missing required field(s): sample \t-> Entry 2: Missing required field(s): strandedness, sample +\t-> Entry 3: Missing required field(s): sample ''' !stdout diff --git a/plugins/nf-validation/src/testResources/samplesheet_no_required.csv b/plugins/nf-validation/src/testResources/samplesheet_no_required.csv index 2b6af26..19254fe 100644 --- a/plugins/nf-validation/src/testResources/samplesheet_no_required.csv +++ b/plugins/nf-validation/src/testResources/samplesheet_no_required.csv @@ -1,3 +1,4 @@ fastq_1,fastq_2,strandedness test1_fastq1.fastq.gz,test1_fastq2.fastq.gz,forward test2_fastq1.fastq.gz,, +test3_fastq1.fastq.gz,,forward From 8faed2de7072e7d66eab3c7128cbdc7235aeed6b Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 26 Jan 2024 13:30:21 +0100 Subject: [PATCH 41/96] update changelog --- CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f42554..49fba6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # nextflow-io/nf-validation: Changelog +# Version 2.0.0dev + +:warning: This version contains a number of breaking changes. Please read the changelog carefully before upgrading. :warning: + +To migrate simple schemas, you can simply follow the next steps: +1. Change the `$schema` to `https://json-schema.org/draft/2020-12/schema` +2. Change the `definitions` keyword to `defs` +3. Change the reference links in all `$ref` keywords to use the new `defs` keyword instead of `definitions` + +## New features +- Added the `uniqueEntries` keyword. This keyword takes a list of strings corresponding to names of fields that need to be a unique combination. e.g. `uniqueEntries: ['sample', 'replicate']` will make sure that the combination of the `sample` and `replicate` fields is unique. ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) + + +## Changes +- Changed the used draft for the schema from `draft-07` to `draft-2020-12`. See the [2019-09](https://json-schema.org/draft/2019-09/release-notes) and [2020-12](https://json-schema.org/draft/2020-12/release-notes) release notes for all changes ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) +- Removed all validation code from the `.fromSamplesheet()` channel factory. The validation is now solely done in the `validateParameters()` function. A custom error message will now be displayed if any error has been encountered during the conversion ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) +- Removed the `unique` keyword from the samplesheet schema. You should now use [`uniqueItems`](https://json-schema.org/understanding-json-schema/reference/array#uniqueItems) or `uniqueEntries` instead ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) +- Removed the `skip_duplicate_check` option from the `fromSamplesheet()` channel factory and the `--validationSkipDuplicateCheck` parameter. You should now use the `uniqueEntries` or [`uniqueItems`](https://json-schema.org/understanding-json-schema/reference/array#uniqueItems) keywords in the schema instead ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) + +## Improvements +- Setting the `exists` keyword to `false` will now check if the path does not exist ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) +- The `schema` keyword will now work in all schemas. ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) +- Improved the error messages ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) + # Version 1.1.3 - Asahikawa ## Improvements From 81a708262f0ea64993d73e5c5872aa3786acfca1 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 26 Jan 2024 13:33:12 +0100 Subject: [PATCH 42/96] prettier --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49fba6c..ccc5bae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,21 +5,24 @@ :warning: This version contains a number of breaking changes. Please read the changelog carefully before upgrading. :warning: To migrate simple schemas, you can simply follow the next steps: + 1. Change the `$schema` to `https://json-schema.org/draft/2020-12/schema` 2. Change the `definitions` keyword to `defs` 3. Change the reference links in all `$ref` keywords to use the new `defs` keyword instead of `definitions` ## New features -- Added the `uniqueEntries` keyword. This keyword takes a list of strings corresponding to names of fields that need to be a unique combination. e.g. `uniqueEntries: ['sample', 'replicate']` will make sure that the combination of the `sample` and `replicate` fields is unique. ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) +- Added the `uniqueEntries` keyword. This keyword takes a list of strings corresponding to names of fields that need to be a unique combination. e.g. `uniqueEntries: ['sample', 'replicate']` will make sure that the combination of the `sample` and `replicate` fields is unique. ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) ## Changes + - Changed the used draft for the schema from `draft-07` to `draft-2020-12`. See the [2019-09](https://json-schema.org/draft/2019-09/release-notes) and [2020-12](https://json-schema.org/draft/2020-12/release-notes) release notes for all changes ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) - Removed all validation code from the `.fromSamplesheet()` channel factory. The validation is now solely done in the `validateParameters()` function. A custom error message will now be displayed if any error has been encountered during the conversion ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) - Removed the `unique` keyword from the samplesheet schema. You should now use [`uniqueItems`](https://json-schema.org/understanding-json-schema/reference/array#uniqueItems) or `uniqueEntries` instead ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) - Removed the `skip_duplicate_check` option from the `fromSamplesheet()` channel factory and the `--validationSkipDuplicateCheck` parameter. You should now use the `uniqueEntries` or [`uniqueItems`](https://json-schema.org/understanding-json-schema/reference/array#uniqueItems) keywords in the schema instead ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) ## Improvements + - Setting the `exists` keyword to `false` will now check if the path does not exist ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) - The `schema` keyword will now work in all schemas. ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) - Improved the error messages ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) From d9a47755106b15b5d3a266ab7cc414a2291b2f00 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 26 Jan 2024 16:31:53 +0100 Subject: [PATCH 43/96] add some missing tests --- .../validation/ValidateParametersTest.groovy | 80 +++++++++++++++++++ .../testResources/nextflow_schema_draft7.json | 25 ++++++ .../nextflow_schema_with_exists_false.json | 27 +++++++ ...schema_with_samplesheet_uniqueEntries.json | 34 ++++++++ .../testResources/samplesheet_non_unique.csv | 4 + .../samplesheet_schema_uniqueEntries.json | 41 ++++++++++ 6 files changed, 211 insertions(+) create mode 100644 plugins/nf-validation/src/testResources/nextflow_schema_draft7.json create mode 100644 plugins/nf-validation/src/testResources/nextflow_schema_with_exists_false.json create mode 100644 plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_uniqueEntries.json create mode 100644 plugins/nf-validation/src/testResources/samplesheet_non_unique.csv create mode 100644 plugins/nf-validation/src/testResources/samplesheet_schema_uniqueEntries.json diff --git a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy index 69a5dac..1d4579c 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy @@ -723,4 +723,84 @@ class ValidateParametersTest extends Dsl2Spec{ ''' !stdout } + + def 'should fail because of wrong draft' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_draft7.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + params.monochrome_logs = true + include { validateParameters } from 'plugin/nf-validation' + + validateParameters(parameters_schema: '$schema', monochrome_logs: params.monochrome_logs) + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + def error = thrown(SchemaValidationException) + !stdout + } + + def 'should fail because of existing file' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_with_exists_false.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + params.monochrome_logs = true + params.outdir = "src/testResources/" + include { validateParameters } from 'plugin/nf-validation' + + validateParameters(parameters_schema: '$schema', monochrome_logs: params.monochrome_logs) + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + def error = thrown(SchemaValidationException) + error.message == '''The following invalid input values have been detected: + +* --outdir (src/testResources/): the file or directory 'src/testResources/' should not exist + +''' + !stdout + } + + def 'should fail because of non-unique entries' () { + given: + def schema = Path.of('src/testResources/nextflow_schema_with_samplesheet_uniqueEntries.json').toAbsolutePath().toString() + def SCRIPT_TEXT = """ + params.monochrome_logs = true + params.input = "src/testResources/samplesheet_non_unique.csv" + include { validateParameters } from 'plugin/nf-validation' + + validateParameters(parameters_schema: '$schema', monochrome_logs: params.monochrome_logs) + """ + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null } + + then: + def error = thrown(SchemaValidationException) + error.message == '''The following invalid input values have been detected: + +* --input (src/testResources/samplesheet_non_unique.csv): Validation of file failed: + -> Entry 3: Detected non-unique combination of the following fields: [sample, fastq_1] + +''' + !stdout + } + } diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_draft7.json b/plugins/nf-validation/src/testResources/nextflow_schema_draft7.json new file mode 100644 index 0000000..006e945 --- /dev/null +++ b/plugins/nf-validation/src/testResources/nextflow_schema_draft7.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", + "title": "nf-core/testpipeline pipeline parameters", + "description": "this is a test", + "type": "object", + "defs": { + "file_patterns": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "properties": { + "glob": { + "type": "string", + "format": "file-path-pattern" + } + } + } + }, + "allOf": [ + { + "$ref": "#/defs/file_patterns" + } + ] +} diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_exists_false.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_exists_false.json new file mode 100644 index 0000000..4c30cf9 --- /dev/null +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_exists_false.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", + "title": "nf-core/testpipeline pipeline parameters", + "description": "this is a test", + "type": "object", + "defs": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "properties": { + "outdir": { + "type": "string", + "format": "directory-path", + "exists": false + } + } + } + }, + "allOf": [ + { + "$ref": "#/defs/input_output_options" + } + ] +} diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_uniqueEntries.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_uniqueEntries.json new file mode 100644 index 0000000..1c113b8 --- /dev/null +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet_uniqueEntries.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", + "title": "nf-core/testpipeline pipeline parameters", + "description": "this is a test", + "type": "object", + "defs": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["input"], + "properties": { + "input": { + "type": "string", + "format": "file-path", + "mimetype": "text/csv", + "pattern": "^\\S+\\.csv$", + "schema": "src/testResources/samplesheet_schema_uniqueEntries.json", + "description": "Path to comma-separated file containing information about the samples in the experiment.", + "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/testpipeline/usage#samplesheet-input).", + "fa_icon": "fas fa-file-csv", + "exists": true + } + } + } + }, + "allOf": [ + { + "$ref": "#/defs/input_output_options" + } + ] +} diff --git a/plugins/nf-validation/src/testResources/samplesheet_non_unique.csv b/plugins/nf-validation/src/testResources/samplesheet_non_unique.csv new file mode 100644 index 0000000..c35ba51 --- /dev/null +++ b/plugins/nf-validation/src/testResources/samplesheet_non_unique.csv @@ -0,0 +1,4 @@ +sample,fastq_1,fastq_2,strandedness +test_1,test1_fastq1.fastq.gz,test1_fastq2.fastq.gz,forward +test_2,test2_fastq1.fastq.gz,,unstranded +test_2,test2_fastq1.fastq.gz,,forward diff --git a/plugins/nf-validation/src/testResources/samplesheet_schema_uniqueEntries.json b/plugins/nf-validation/src/testResources/samplesheet_schema_uniqueEntries.json new file mode 100644 index 0000000..6c4396b --- /dev/null +++ b/plugins/nf-validation/src/testResources/samplesheet_schema_uniqueEntries.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/test/test/master/assets/schema_input.json", + "title": "Test schema for samplesheets", + "description": "Schema for the file provided with params.input", + "type": "array", + "uniqueEntries": ["sample", "fastq_1"], + "items": { + "type": "object", + "properties": { + "sample": { + "type": "string", + "pattern": "^\\S+$", + "errorMessage": "Sample name must be provided and cannot contain spaces" + }, + "fastq_1": { + "type": "string", + "pattern": "^\\S+\\.f(ast)?q\\.gz$", + "errorMessage": "FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" + }, + "fastq_2": { + "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", + "anyOf": [ + { + "type": "string", + "pattern": "^\\S+\\.f(ast)?q\\.gz$" + }, + { + "type": "string", + "maxLength": 0 + } + ] + }, + "strandedness": { + "type": "string", + "errorMessage": "Strandedness must be provided and be one of 'forward', 'reverse' or 'unstranded'", + "enum": ["forward", "reverse", "unstranded"] + } + } + } +} From 7e10f7fd381924d2b0ff499d18e3bf65475496c9 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 30 Jan 2024 10:55:39 +0100 Subject: [PATCH 44/96] add array check to fileToMaps --- .../nf-validation/src/main/nextflow/validation/Utils.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy index ff94679..9751949 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy @@ -47,6 +47,12 @@ public class Utils { def Map types = variableTypes(schemaName, baseDir) def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "") + if (types.find{ it.value == "array" } as Boolean && fileType in ["csv", "tsv"]){ + def msg = "Using \"type\": \"array\" in schema with a \".$fileType\" samplesheet is not supported\n" + log.error("ERROR: Validation of pipeline parameters failed!") + throw new SchemaValidationException(msg, []) + } + if(!containsHeader){ types = ["empty": types[""]] } From f102dd7a674da7ac65d3989108a72311d0bdaede Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:56:53 +0100 Subject: [PATCH 45/96] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccc5bae..b808d78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ To migrate simple schemas, you can simply follow the next steps: 1. Change the `$schema` to `https://json-schema.org/draft/2020-12/schema` -2. Change the `definitions` keyword to `defs` +2. Change the `definitions` keyword to `defs`. See [reason](https://json-schema.org/draft/2019-09/release-notes#semi-incompatible-changes). 3. Change the reference links in all `$ref` keywords to use the new `defs` keyword instead of `definitions` ## New features From ca0fb4d8e8865714ee7a8de958c057a3e9c30d45 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 30 Jan 2024 11:33:00 +0100 Subject: [PATCH 46/96] some small fixes to samplesheet convertor --- .../validation/SamplesheetConverter.groovy | 24 +----------- .../SamplesheetConverterTest.groovy | 38 +++++++++---------- .../src/testResources/schema_input.json | 20 ++++++---- .../schema_input_with_arrays.json | 2 +- 4 files changed, 33 insertions(+), 51 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index 657c425..96cff5a 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -60,29 +60,7 @@ class SamplesheetConverter { def Boolean containsHeader = !(allFields.size() == 1 && allFields[0] == "") def String fileType = Utils.getFileType(samplesheetFile) - def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null - def List> samplesheetList - - if(fileType == "yaml"){ - samplesheetList = new Yaml().load((samplesheetFile.text)).collect { - if(containsHeader) { - return it as Map - } - return ["empty": it] as Map - } - } - else if(fileType == "json"){ - samplesheetList = new JsonSlurper().parseText(samplesheetFile.text).collect { - if(containsHeader) { - return it as Map - } - return ["empty": it] as Map - } - } - else { - Path fileSamplesheet = Nextflow.file(samplesheetFile) as Path - samplesheetList = fileSamplesheet.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"') - } + def List> samplesheetList = Utils.fileToMaps(samplesheetFile, schemaFile.toString(), Global.getSession().baseDir.toString()) // Field checks + returning the channels def Map> booleanUniques = [:] diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy index bb0c06a..7ca12d0 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy @@ -77,10 +77,10 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() - stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) + stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) stdout.contains("[[string1:value, string2:value, integer1:0, integer2:0, boolean1:true, boolean2:true], string1, 25.08, false, [], [], [], [], [], itDoesExist]") - stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, [], [], [], unique2, 1, itDoesExist]") - stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) + stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.0, false, [], [], [], unique2, 1, itDoesExist]") + stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) } def 'should work fine - quoted CSV' () { @@ -104,10 +104,10 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() - stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) + stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) stdout.contains("[[string1:value, string2:value, integer1:0, integer2:0, boolean1:true, boolean2:true], string1, 25.08, false, [], [], [], [], [], itDoesExist]") stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, [], [], [], unique2, 1, itDoesExist]") - stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) + stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) } def 'should work fine - TSV' () { @@ -131,10 +131,10 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() - stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) + stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) stdout.contains("[[string1:value, string2:value, integer1:0, integer2:0, boolean1:true, boolean2:true], string1, 25.08, false, [], [], [], [], [], itDoesExist]") stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, [], [], [], unique2, 1, itDoesExist]") - stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) + stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) } def 'should work fine - YAML' () { @@ -158,10 +158,10 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() - stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) + stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) stdout.contains("[[string1:value, string2:value, integer1:0, integer2:0, boolean1:true, boolean2:true], string1, 25.08, false, [], [], [], [], [], itDoesExist]") stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, [], [], [], unique2, 1, itDoesExist]") - stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) + stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) } def 'should work fine - JSON' () { @@ -185,10 +185,10 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() - stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) + stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) stdout.contains("[[string1:value, string2:value, integer1:0, integer2:0, boolean1:true, boolean2:true], string1, 25.08, false, [], [], [], [], [], itDoesExist]") stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, [], [], [], unique2, 1, itDoesExist]") - stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) + stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) } def 'arrays should work fine - YAML' () { @@ -212,9 +212,9 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() - stdout.contains("[[array_meta:null], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) - stdout.contains("[[array_meta:[look, an, array, in, meta]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) - stdout.contains("[[array_meta:null], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String) + stdout.contains("[[array_meta:null], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) + stdout.contains("[[array_meta:[look, an, array, in, meta]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) + stdout.contains("[[array_meta:null], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String) } def 'arrays should work fine - JSON' () { @@ -238,9 +238,9 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() - stdout.contains("[[array_meta:null], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2], [${this.getRootString()}/src/testResources/testDir, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) - stdout.contains("[[array_meta:[look, an, array, in, meta]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) - stdout.contains("[[array_meta:null], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${this.getRootString()}/src/testResources/testDir/testFile.txt], [[${this.getRootString()}/src/testResources/testDir/testFile.txt], [${this.getRootString()}/src/testResources/testDir/testFile.txt, ${this.getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String) + stdout.contains("[[array_meta:null], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) + stdout.contains("[[array_meta:[look, an, array, in, meta]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) + stdout.contains("[[array_meta:null], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String) } def 'array errors before channel conversion - YAML' () { @@ -430,10 +430,10 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() stdout.contains("\tThe samplesheet contains following unchecked field(s): [extraField]") - stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, [], unique1, 1, itDoesExist]" as String) + stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, [], unique1, 1, itDoesExist]" as String) stdout.contains("[[string1:value, string2:value, integer1:0, integer2:0, boolean1:true, boolean2:true], string1, 25, false, [], [], [], [], [], itDoesExist]") stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, [], [], [], unique2, 1, itDoesExist]") - stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${this.getRootString()}/src/testResources/test.txt, ${this.getRootString()}/src/testResources/testDir, [], unique3, 1, itDoesExist]" as String) + stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, [], unique3, 1, itDoesExist]" as String) } def 'no meta' () { diff --git a/plugins/nf-validation/src/testResources/schema_input.json b/plugins/nf-validation/src/testResources/schema_input.json index edbcda7..bf790e2 100644 --- a/plugins/nf-validation/src/testResources/schema_input.json +++ b/plugins/nf-validation/src/testResources/schema_input.json @@ -10,8 +10,7 @@ "field_1": { "type": "string", "meta": ["string1","string2"], - "default": "value", - "dependentRequired": ["field_2", "field_3"] + "default": "value" }, "field_2": { "type": "integer", @@ -49,18 +48,23 @@ "exists": true }, "field_10": { - "type": "string", - "unique": true + "type": "string" }, "field_11": { - "type": "integer", - "unique": ["field_10"] + "type": "integer" }, "field_12": { "type": "string", "default": "itDoesExist" } }, - "required": ["field_4", "field_6"] - } + "required": ["field_4", "field_6"], + "dependentRequired": { + "field_1": ["field_2", "field_3"] + } + }, + "allOf": [ + {"uniqueEntries": ["field_11", "field_10"]}, + {"uniqueEntries": ["field_10"]} + ] } diff --git a/plugins/nf-validation/src/testResources/schema_input_with_arrays.json b/plugins/nf-validation/src/testResources/schema_input_with_arrays.json index 06ebba8..aa7ac61 100644 --- a/plugins/nf-validation/src/testResources/schema_input_with_arrays.json +++ b/plugins/nf-validation/src/testResources/schema_input_with_arrays.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nextflow-io/nf-validation/master/plugins/nf-validation/src/testResources/schema_input.json", "title": "Samplesheet validation schema", "description": "Schema for the samplesheet used in this pipeline", From 58f94d123db89ce519b6f00b854361f138180cab Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 30 Jan 2024 13:11:10 +0100 Subject: [PATCH 47/96] update authors --- .../main/nextflow/validation/CustomEvaluatorFactory.groovy | 4 ++++ .../validation/CustomEvaluators/ExistsEvaluator.groovy | 4 ++++ .../CustomEvaluators/FormatDirectoryPathEvaluator.groovy | 4 ++++ .../CustomEvaluators/FormatFilePathEvaluator.groovy | 4 ++++ .../CustomEvaluators/FormatFilePathPatternEvaluator.groovy | 4 ++++ .../validation/CustomEvaluators/FormatPathEvaluator.groovy | 4 ++++ .../validation/CustomEvaluators/LenientTypeEvaluator.groovy | 4 ++++ .../validation/CustomEvaluators/SchemaEvaluator.groovy | 4 ++++ .../CustomEvaluators/UniqueEntriesEvaluator.groovy | 4 ++++ .../src/main/nextflow/validation/JsonSchemaValidator.groovy | 4 ++++ .../main/nextflow/validation/SamplesheetConverter.groovy | 6 ++++++ .../src/main/nextflow/validation/SchemaValidator.groovy | 6 ++++++ .../nf-validation/src/main/nextflow/validation/Utils.groovy | 6 ++++++ .../src/test/nextflow/validation/NfValidationTest.groovy | 5 +++-- .../src/test/nextflow/validation/ParamsHelpTest.groovy | 5 +++-- .../test/nextflow/validation/ParamsSummaryLogTest.groovy | 5 +++-- .../nextflow/validation/SamplesheetConverterTest.groovy | 5 +++-- .../test/nextflow/validation/ValidateParametersTest.groovy | 5 +++-- 18 files changed, 73 insertions(+), 10 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy index b5282be..0c52cbc 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy @@ -6,6 +6,10 @@ import dev.harrel.jsonschema.Evaluator import dev.harrel.jsonschema.SchemaParsingContext import dev.harrel.jsonschema.JsonNode +/** + * @author : nvnieuwk + */ + class CustomEvaluatorFactory implements EvaluatorFactory { private Boolean lenientMode diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy index f939153..100789b 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/ExistsEvaluator.groovy @@ -8,6 +8,10 @@ import nextflow.Nextflow import groovy.util.logging.Slf4j import java.nio.file.Path +/** + * @author : nvnieuwk + */ + @Slf4j class ExistsEvaluator implements Evaluator { // The file should or should not exist diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy index 8c51ab9..7499aaa 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatDirectoryPathEvaluator.groovy @@ -8,6 +8,10 @@ import nextflow.Nextflow import groovy.util.logging.Slf4j import java.nio.file.Path +/** + * @author : nvnieuwk + */ + @Slf4j class FormatDirectoryPathEvaluator implements Evaluator { // The string should be a directory diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy index 2708e4c..47dd527 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathEvaluator.groovy @@ -8,6 +8,10 @@ import nextflow.Nextflow import groovy.util.logging.Slf4j import java.nio.file.Path +/** + * @author : nvnieuwk + */ + @Slf4j class FormatFilePathEvaluator implements Evaluator { // The string should be a file diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy index a7d8cdd..75c4dad 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatFilePathPatternEvaluator.groovy @@ -8,6 +8,10 @@ import nextflow.Nextflow import groovy.util.logging.Slf4j import java.nio.file.Path +/** + * @author : nvnieuwk + */ + @Slf4j class FormatFilePathPatternEvaluator implements Evaluator { // The string should be a path pattern diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy index 2749b2a..6d3ac92 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/FormatPathEvaluator.groovy @@ -8,6 +8,10 @@ import nextflow.Nextflow import groovy.util.logging.Slf4j import java.nio.file.Path +/** + * @author : nvnieuwk + */ + @Slf4j class FormatPathEvaluator implements Evaluator { // The string should be a path diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy index d8e1f33..6977c05 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/LenientTypeEvaluator.groovy @@ -12,6 +12,10 @@ import java.util.stream.Collectors import static java.util.Collections.singleton import static java.util.Collections.unmodifiableList +/** + * @author : nvnieuwk + */ + @Slf4j class LenientTypeEvaluator implements Evaluator { // Validate against the type diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index bdf729c..b7223a9 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -12,6 +12,10 @@ import groovy.util.logging.Slf4j import java.nio.file.Path import java.nio.file.Files +/** + * @author : nvnieuwk + */ + @Slf4j class SchemaEvaluator implements Evaluator { // Evaluate the file using the given schema diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy index 07a7bef..999c567 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/UniqueEntriesEvaluator.groovy @@ -10,6 +10,10 @@ import groovy.json.JsonBuilder import groovy.util.logging.Slf4j import java.nio.file.Path +/** + * @author : nvnieuwk + */ + @Slf4j class UniqueEntriesEvaluator implements Evaluator { // Combinations of these columns should be unique diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 4aae5e8..d6a8a74 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -16,6 +16,10 @@ import dev.harrel.jsonschema.providers.OrgJsonNode import java.util.regex.Pattern import java.util.regex.Matcher +/** + * @author : nvnieuwk + */ + @Slf4j @CompileStatic public class JsonSchemaValidator { diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index 96cff5a..f9f601a 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -22,6 +22,12 @@ import nextflow.Nextflow import nextflow.plugin.extension.Function import nextflow.Session +/** + * @author : mirpedrol + * @author : nvnieuwk + * @author : awgymer + */ + @Slf4j @CompileStatic class SamplesheetConverter { diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index e3eb691..d55e4ed 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -27,6 +27,12 @@ import org.json.JSONObject import org.json.JSONTokener import org.yaml.snakeyaml.Yaml +/** + * @author : mirpedrol + * @author : nvnieuwk + * @author : KevinMenden + */ + @Slf4j @CompileStatic class SchemaValidator extends PluginExtensionPoint { diff --git a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy index 9751949..9d6e5ab 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy @@ -6,6 +6,12 @@ import groovy.json.JsonSlurper import groovy.util.logging.Slf4j import java.nio.file.Path +/** + * @author : mirpedrol + * @author : nvnieuwk + * @author : KevinMenden + */ + @Slf4j public class Utils { diff --git a/plugins/nf-validation/src/test/nextflow/validation/NfValidationTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/NfValidationTest.groovy index 5582b5e..114922c 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/NfValidationTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/NfValidationTest.groovy @@ -12,8 +12,9 @@ import spock.lang.Shared import test.Dsl2Spec import test.OutputCapture /** - * @author : jorge - * + * @author : mirpedrol + * @author : nvnieuwk + * @author : jorgeaguileraseqera */ class NfValidationTest extends Dsl2Spec{ diff --git a/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy index ec1853a..6c6f435 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy @@ -13,8 +13,9 @@ import test.Dsl2Spec import test.OutputCapture /** - * @author : Nicolas Vannieuwkerke - * + * @author : mirpedrol + * @author : nvnieuwk + * @author : jorgeaguileraseqera */ class ParamsHelpTest extends Dsl2Spec{ diff --git a/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy index cf2c8b1..0b07603 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy @@ -13,8 +13,9 @@ import test.Dsl2Spec import test.OutputCapture /** - * @author : Nicolas Vannieuwkerke - * + * @author : mirpedrol + * @author : nvnieuwk + * @author : jorgeaguileraseqera */ class ParamsSummaryLogTest extends Dsl2Spec{ diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy index 7ca12d0..7aa9d1d 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy @@ -13,8 +13,9 @@ import test.Dsl2Spec import test.OutputCapture /** - * @author : Nicolas Vannieuwkerke - * + * @author : mirpedrol + * @author : nvnieuwk + * @author : jorgeaguileraseqera */ class SamplesheetConverterTest extends Dsl2Spec{ diff --git a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy index 1d4579c..eb76f16 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ValidateParametersTest.groovy @@ -13,8 +13,9 @@ import test.Dsl2Spec import test.OutputCapture /** - * @author : Nicolas Vannieuwkerke - * + * @author : mirpedrol + * @author : nvnieuwk + * @author : jorgeaguileraseqera */ class ValidateParametersTest extends Dsl2Spec{ From 34f95ff6cfb7789d4444f3f708eb924d0387bad6 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:18:43 +0100 Subject: [PATCH 48/96] Update plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- .../src/test/nextflow/validation/SamplesheetConverterTest.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy index 7aa9d1d..69382f7 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy @@ -15,7 +15,6 @@ import test.OutputCapture /** * @author : mirpedrol * @author : nvnieuwk - * @author : jorgeaguileraseqera */ class SamplesheetConverterTest extends Dsl2Spec{ From 0486620ad7d97fb3f33d27ef512327237b3275c9 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:18:55 +0100 Subject: [PATCH 49/96] Update plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- .../src/test/nextflow/validation/ParamsHelpTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy index 6c6f435..f9ee44b 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ParamsHelpTest.groovy @@ -15,7 +15,7 @@ import test.OutputCapture /** * @author : mirpedrol * @author : nvnieuwk - * @author : jorgeaguileraseqera + * @author : KevinMenden */ class ParamsHelpTest extends Dsl2Spec{ From 2127fb80d72ea627b52ddb09c8f13cc3bee8651d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:19:10 +0100 Subject: [PATCH 50/96] Update plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- .../src/test/nextflow/validation/ParamsSummaryLogTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy index 0b07603..0f3b244 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/ParamsSummaryLogTest.groovy @@ -15,7 +15,7 @@ import test.OutputCapture /** * @author : mirpedrol * @author : nvnieuwk - * @author : jorgeaguileraseqera + * @author : KevinMenden */ class ParamsSummaryLogTest extends Dsl2Spec{ From 2634a434f0ec617384bad805b73c789059ea0f51 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 31 Jan 2024 10:55:19 +0100 Subject: [PATCH 51/96] made no-header support more flexible --- .../CustomEvaluators/SchemaEvaluator.groovy | 14 +- .../validation/JsonSchemaValidator.groovy | 16 +- .../validation/SamplesheetConverter.groovy | 198 +++--------- .../validation/SamplesheetConverterOld.groovy | 282 ++++++++++++++++++ .../validation/SchemaValidator.groovy | 20 +- .../src/main/nextflow/validation/Utils.groovy | 148 +++++---- 6 files changed, 431 insertions(+), 247 deletions(-) create mode 100644 plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverterOld.groovy diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index b7223a9..f41bd11 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -5,8 +5,8 @@ import dev.harrel.jsonschema.EvaluationContext import dev.harrel.jsonschema.JsonNode import nextflow.Nextflow import nextflow.Global -import groovy.json.JsonGenerator import org.json.JSONArray +import org.json.JSONObject import groovy.util.logging.Slf4j import java.nio.file.Path @@ -45,19 +45,11 @@ class SchemaEvaluator implements Evaluator { log.debug("Started validating ${file.toString()}") - def String baseDir = Global.getSession().baseDir - def String schemaFull = Utils.getSchemaPath(baseDir, this.schema) - def List fileMaps = Utils.fileToMaps(file, this.schema, baseDir) + def String schemaFull = Utils.getSchemaPath(this.schema) + def JSONArray arrayJSON = Utils.fileToJsonArray(file, Path.of(schemaFull)) def String schemaContents = Files.readString( Path.of(schemaFull) ) def validator = new JsonSchemaValidator(schemaContents) - // Remove all null values from JSON object - // and convert the groovy object to a JSONArray - def jsonGenerator = new JsonGenerator.Options() - .excludeNulls() - .build() - def JSONArray arrayJSON = new JSONArray(jsonGenerator.toJson(fileMaps)) - def List validationErrors = validator.validate(arrayJSON) if (validationErrors) { def List errors = ["Validation of file failed:"] + validationErrors.collect { "\t${it}" as String} diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index d6a8a74..9ba657a 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -4,8 +4,6 @@ import groovy.util.logging.Slf4j import groovy.transform.CompileStatic import org.json.JSONObject import org.json.JSONArray -import org.json.JSONPointer -import org.json.JSONPointerException import dev.harrel.jsonschema.ValidatorFactory import dev.harrel.jsonschema.Validator import dev.harrel.jsonschema.EvaluatorFactory @@ -30,8 +28,7 @@ public class JsonSchemaValidator { JsonSchemaValidator(String schemaString) { this.schema = new JSONObject(schemaString) - def JSONPointer schemaPointer = new JSONPointer("#/\$schema") - def String draft = schemaPointer.queryFrom(this.schema) + def String draft = Utils.getValueFromJson("#/\$schema", this.schema) if(draft != "https://json-schema.org/draft/2020-12/schema") { log.error("""Failed to load the meta schema: The used schema draft (${draft}) is not correct, please use \"https://json-schema.org/draft/2020-12/schema\" instead. @@ -56,18 +53,11 @@ See here for more information: https://json-schema.org/specification#migrating-f } def String instanceLocation = error.getInstanceLocation() - def JSONPointer pointer = new JSONPointer(instanceLocation) - def String value = pointer.queryFrom(rawJson) + def String value = Utils.getValueFromJson(instanceLocation, rawJson) // Get the errorMessage if there is one def String schemaLocation = error.getSchemaLocation().replaceFirst(/^[^#]+/, "") - def JSONPointer schemaPointer = new JSONPointer("${schemaLocation}/errorMessage") - def String customError = "" - try{ - customError = schemaPointer.queryFrom(this.schema) ?: "" - } catch (JSONPointerException e) { - customError = "" - } + def String customError = Utils.getValueFromJson("${schemaLocation}/errorMessage", this.schema) // Change some error messages to make them more clear if (customError == "") { diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index f9f601a..50de35a 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -32,163 +32,66 @@ import nextflow.Session @CompileStatic class SamplesheetConverter { - private static List errors = [] - private static List schemaErrors = [] - private static List warnings = [] - private static List rows = [] - static boolean hasErrors() { errors.size()>0 } - static Set getErrors() { errors.sort().collect { "\t${it}".toString() } as Set } - - static boolean hasSchemaErrors() { schemaErrors.size()>0 } - static Set getSchemaErrors() { schemaErrors.sort().collect { "\t${it}".toString() } as Set } - - static boolean hasWarnings() { warnings.size()>0 } - static Set getWarnings() { warnings.sort().collect { "\t${it}".toString() } as Set } - - private static Integer sampleCount = 0 - - static resetCount(){ sampleCount = 0 } - static increaseCount(){ sampleCount++ } - static Integer getCount(){ sampleCount } + static List convertToList(Path samplesheetFile, Path schemaFile, Boolean header) { + def JSONObject schemaMap = new JSONObject(schemaFile.text) + def JSONArray samplesheetList = Utils.fileToJsonArray(samplesheetFile, schemaFile) - static List convertToList( - Path samplesheetFile, - Path schemaFile - ) { - - def Map schemaMap = (Map) new JsonSlurper().parseText(schemaFile.text) - def Map> schemaFields = (Map) schemaMap["items"]["properties"] - def Set allFields = schemaFields.keySet() - def List requiredFields = (List) schemaMap["items"]["required"] - def Boolean containsHeader = !(allFields.size() == 1 && allFields[0] == "") - - def String fileType = Utils.getFileType(samplesheetFile) - def List> samplesheetList = Utils.fileToMaps(samplesheetFile, schemaFile.toString(), Global.getSession().baseDir.toString()) - - // Field checks + returning the channels - def Map> booleanUniques = [:] - def Map>> listUniques = [:] - def Boolean headerCheck = true this.rows = [] - resetCount() - - def List outputs = samplesheetList.collect { Map fullRow -> - increaseCount() - - Map row = fullRow.findAll { it.value != "" } - def Set rowKeys = containsHeader ? row.keySet() : ["empty"].toSet() - def String entryInfo = fileType in ["yaml", "json"] ? " for entry ${this.getCount()}." : "" - - // Check the header (CSV/TSV) or present fields (YAML) - if(headerCheck) { - def unexpectedFields = containsHeader ? rowKeys - allFields : [] - if(unexpectedFields.size() > 0) { - this.warnings << "The samplesheet contains following unchecked field(s): ${unexpectedFields}${entryInfo}".toString() - } - - if(fileType != 'yaml'){ - headerCheck = false - } - } - - this.rows.add(row) - def Map meta = [:] - def ArrayList output = [] + def Iterator samplesheetIterator = samplesheetList.iterator() - for( Map.Entry field : schemaFields ){ - def String key = containsHeader ? field.key : "empty" - def Object input = row[key] - - // Check if the field is deprecated - if(field['value']['deprecated']){ - this.warnings << "The '${key}' field is deprecated and will no longer be used in the future. Please check the official documentation of the pipeline for more information.".toString() - } - - // Check required dependencies - def List dependencies = field['value']["dependentRequired"] as List - if(input && dependencies) { - def List missingValues = [] - for( dependency in dependencies ){ - if(row[dependency] == "" || !(row[dependency])) { - missingValues.add(dependency) - } - } - if (missingValues) { - this.errors << addSample("${dependencies} field(s) should be defined when '${key}' is specified, but the field(s) ${missingValues} is/are not defined.".toString()) - } - } - - // Check if the field is unique - def unique = field['value']['unique'] - def Boolean uniqueIsList = unique instanceof ArrayList - if(unique && !uniqueIsList){ - if(!(key in booleanUniques)){ - booleanUniques[key] = [] - } - if(input in booleanUniques[key] && input){ - this.errors << addSample("The '${key}' value needs to be unique. '${input}' was found at least twice in the samplesheet.".toString()) - } - booleanUniques[key].add(input as String) - } - else if(unique && uniqueIsList) { - def Map newMap = (Map) row.subMap((List) [key] + (List) unique) - if(!(key in listUniques)){ - listUniques[key] = [] - } - if(newMap in listUniques[key] && input){ - this.errors << addSample("The combination of '${key}' with fields ${unique} needs to be unique. ${newMap} was found at least twice.".toString()) - } - listUniques[key].add(newMap) - } - - // Convert field to a meta field or add it as an input to the channel - def List metaNames = field['value']['meta'] as List - if(metaNames) { - for(name : metaNames) { - meta[name] = (input != '' && input != null) ? - castToNFType(input, field) : - field['value']['default'] != null ? - castToNFType(field['value']['default'], field) : - null - } - } - else { - def inputVal = (input != '' && input != null) ? - castToNFType(input, field) : - field['value']['default'] != null ? - castToNFType(field['value']['default'], field) : - [] - output.add(inputVal) - } - } - // Add meta to the output when a meta field has been created - if(meta != [:]) { output.add(0, meta) } - return output + while (samplesheetIterator.hasNext()) { + println(samplesheetIterator.next()) } - // check for samplesheet errors - if (this.hasErrors()) { - String message = "Samplesheet errors:\n" + this.getErrors().join("\n") - throw new SchemaValidationException(message, this.getErrors() as List) - } + // def List outputs = samplesheetList.collect { fullRow -> - // check for schema errors - if (this.hasSchemaErrors()) { - String message = "Samplesheet schema errors:\n" + this.getSchemaErrors().join("\n") - throw new SchemaValidationException(message, this.getSchemaErrors() as List) - } + // println(fullrow.getClass()) - // check for warnings - if( this.hasWarnings() ) { - def msg = "Samplesheet warnings:\n" + this.getWarnings().join('\n') - log.warn(msg) - } + // def Map row = fullRow.findAll { it.value != "" } + // def Set rowKeys = header ? row.keySet() : ["empty"].toSet() - return outputs + // Check the header (CSV/TSV) or present fields (YAML) + // TODO reimplement warning for unused fields + + // this.rows.add(row) + + // def Map meta = [:] + // def ArrayList output = [] + + // for( Map.Entry field : schemaFields ){ + // def String key = header ? field.key : "empty" + // def Object input = row[key] + + // // Convert field to a meta field or add it as an input to the channel + // def List metaNames = field['value']['meta'] as List + // if(metaNames) { + // for(name : metaNames) { + // meta[name] = (input != '' && input != null) ? + // castToNFType(input, field) : + // field['value']['default'] != null ? + // castToNFType(field['value']['default'], field) : + // null + // } + // } + // else { + // def inputVal = (input != '' && input != null) ? + // castToNFType(input, field) : + // field['value']['default'] != null ? + // castToNFType(field['value']['default'], field) : + // [] + // output.add(inputVal) + // } + // } + // // Add meta to the output when a meta field has been created + // if(meta != [:]) { output.add(0, meta) } + // return [] + // } + + return [] } // Function to transform an input field from the samplesheet to its desired type @@ -274,9 +177,4 @@ class SamplesheetConverter { } } - private static String addSample ( - String message - ) { - return "Entry ${this.getCount()}: ${message}".toString() - } } diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverterOld.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverterOld.groovy new file mode 100644 index 0000000..efda5c8 --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverterOld.groovy @@ -0,0 +1,282 @@ +// package nextflow.validation + +// import groovy.json.JsonSlurper +// import groovy.json.JsonOutput +// import groovy.transform.CompileStatic +// import groovy.util.logging.Slf4j +// import groovyx.gpars.dataflow.DataflowReadChannel +// import groovyx.gpars.dataflow.DataflowWriteChannel + +// import java.nio.file.Path +// import java.util.concurrent.CompletableFuture +// import java.util.concurrent.CompletionException + +// import org.yaml.snakeyaml.Yaml +// import org.json.JSONArray +// import org.json.JSONObject +// import org.json.JSONTokener + +// import nextflow.Channel +// import nextflow.Global +// import nextflow.Nextflow +// import nextflow.plugin.extension.Function +// import nextflow.Session + +// /** +// * @author : mirpedrol +// * @author : nvnieuwk +// * @author : awgymer +// */ + +// @Slf4j +// @CompileStatic +// class SamplesheetConverterOld { + +// private static List errors = [] +// private static List schemaErrors = [] +// private static List warnings = [] + +// private static List rows = [] + +// static boolean hasErrors() { errors.size()>0 } +// static Set getErrors() { errors.sort().collect { "\t${it}".toString() } as Set } + +// static boolean hasSchemaErrors() { schemaErrors.size()>0 } +// static Set getSchemaErrors() { schemaErrors.sort().collect { "\t${it}".toString() } as Set } + +// static boolean hasWarnings() { warnings.size()>0 } +// static Set getWarnings() { warnings.sort().collect { "\t${it}".toString() } as Set } + +// private static Integer sampleCount = 0 + +// static resetCount(){ sampleCount = 0 } +// static increaseCount(){ sampleCount++ } +// static Integer getCount(){ sampleCount } + + +// static List convertToList( +// Path samplesheetFile, +// Path schemaFile +// ) { + +// def Map schemaMap = (Map) new JsonSlurper().parseText(schemaFile.text) +// def Map> schemaFields = (Map) schemaMap["items"]["properties"] +// def Set allFields = schemaFields.keySet() +// def List requiredFields = (List) schemaMap["items"]["required"] +// def Boolean containsHeader = !(allFields.size() == 1 && allFields[0] == "") + +// def String fileType = Utils.getFileType(samplesheetFile) +// def List> samplesheetList = Utils.fileToJsonArray(samplesheetFile, schemaFile.toString(), Global.getSession().baseDir.toString()) + +// // Field checks + returning the channels +// def Map> booleanUniques = [:] +// def Map>> listUniques = [:] +// def Boolean headerCheck = true +// this.rows = [] +// resetCount() + +// def List outputs = samplesheetList.collect { Map fullRow -> +// increaseCount() + +// Map row = fullRow.findAll { it.value != "" } +// def Set rowKeys = containsHeader ? row.keySet() : ["empty"].toSet() +// def String entryInfo = fileType in ["yaml", "json"] ? " for entry ${this.getCount()}." : "" + +// // Check the header (CSV/TSV) or present fields (YAML) +// if(headerCheck) { +// def unexpectedFields = containsHeader ? rowKeys - allFields : [] +// if(unexpectedFields.size() > 0) { +// this.warnings << "The samplesheet contains following unchecked field(s): ${unexpectedFields}${entryInfo}".toString() +// } + +// if(fileType != 'yaml'){ +// headerCheck = false +// } +// } + +// this.rows.add(row) + +// def Map meta = [:] +// def ArrayList output = [] + +// for( Map.Entry field : schemaFields ){ +// def String key = containsHeader ? field.key : "empty" +// def Object input = row[key] + +// // Check if the field is deprecated +// if(field['value']['deprecated']){ +// this.warnings << "The '${key}' field is deprecated and will no longer be used in the future. Please check the official documentation of the pipeline for more information.".toString() +// } + +// // Check required dependencies +// def List dependencies = field['value']["dependentRequired"] as List +// if(input && dependencies) { +// def List missingValues = [] +// for( dependency in dependencies ){ +// if(row[dependency] == "" || !(row[dependency])) { +// missingValues.add(dependency) +// } +// } +// if (missingValues) { +// this.errors << addSample("${dependencies} field(s) should be defined when '${key}' is specified, but the field(s) ${missingValues} is/are not defined.".toString()) +// } +// } + +// // Check if the field is unique +// def unique = field['value']['unique'] +// def Boolean uniqueIsList = unique instanceof ArrayList +// if(unique && !uniqueIsList){ +// if(!(key in booleanUniques)){ +// booleanUniques[key] = [] +// } +// if(input in booleanUniques[key] && input){ +// this.errors << addSample("The '${key}' value needs to be unique. '${input}' was found at least twice in the samplesheet.".toString()) +// } +// booleanUniques[key].add(input as String) +// } +// else if(unique && uniqueIsList) { +// def Map newMap = (Map) row.subMap((List) [key] + (List) unique) +// if(!(key in listUniques)){ +// listUniques[key] = [] +// } +// if(newMap in listUniques[key] && input){ +// this.errors << addSample("The combination of '${key}' with fields ${unique} needs to be unique. ${newMap} was found at least twice.".toString()) +// } +// listUniques[key].add(newMap) +// } + +// // Convert field to a meta field or add it as an input to the channel +// def List metaNames = field['value']['meta'] as List +// if(metaNames) { +// for(name : metaNames) { +// meta[name] = (input != '' && input != null) ? +// castToNFType(input, field) : +// field['value']['default'] != null ? +// castToNFType(field['value']['default'], field) : +// null +// } +// } +// else { +// def inputVal = (input != '' && input != null) ? +// castToNFType(input, field) : +// field['value']['default'] != null ? +// castToNFType(field['value']['default'], field) : +// [] +// output.add(inputVal) +// } +// } +// // Add meta to the output when a meta field has been created +// if(meta != [:]) { output.add(0, meta) } +// return output +// } + +// // check for samplesheet errors +// if (this.hasErrors()) { +// String message = "Samplesheet errors:\n" + this.getErrors().join("\n") +// throw new SchemaValidationException(message, this.getErrors() as List) +// } + +// // check for schema errors +// if (this.hasSchemaErrors()) { +// String message = "Samplesheet schema errors:\n" + this.getSchemaErrors().join("\n") +// throw new SchemaValidationException(message, this.getSchemaErrors() as List) +// } + +// // check for warnings +// if( this.hasWarnings() ) { +// def msg = "Samplesheet warnings:\n" + this.getWarnings().join('\n') +// log.warn(msg) +// } + +// return outputs +// } + +// // Function to transform an input field from the samplesheet to its desired type +// private static castToNFType( +// Object input, +// Map.Entry field +// ) { +// def String type = field['value']['type'] +// def String key = field.key + +// // Recursively call this function for each item in the array if the field is an array-type +// // The returned values are collected into a single array +// if (type == "array") { +// def Map.Entry subfield = (Map.Entry) Map.entry(field.key, field['value']['items']) +// log.debug "subfield = $subfield" +// def ArrayList result = input.collect{ castToNFType(it, subfield) } as ArrayList +// return result +// } + +// def String inputStr = input as String +// // Convert string values +// if(type == "string" || !type) { +// def String result = inputStr as String + +// // Check and convert to the desired format +// def String format = field['value']['format'] +// if(format) { +// if(format == "file-path-pattern") { +// def ArrayList inputFiles = Nextflow.file(inputStr) as ArrayList +// return inputFiles +// } +// if(format.contains("path")) { +// def Path inputFile = Nextflow.file(inputStr) as Path +// return inputFile +// } +// } + + +// // Return the plain string value +// return result +// } + +// // Convert number values +// else if(type == "number") { +// try { +// def int result = inputStr as int +// return result +// } +// catch (NumberFormatException e) { +// log.debug("Could not convert ${input} to an integer. Trying to convert to a float.") +// } + +// try { +// def float result = inputStr as float +// return result +// } +// catch (NumberFormatException e) { +// log.debug("Could not convert ${inputStr} to a float. Trying to convert to a double.") +// } + +// def double result = inputStr as double +// return result +// } + +// // Convert integer values +// else if(type == "integer") { + +// def int result = inputStr as int +// return result +// } + +// // Convert boolean values +// else if(type == "boolean") { + +// if(inputStr.toLowerCase() == "true") { +// return true +// } +// return false +// } + +// else if(type == "null") { +// return null +// } +// } + +// private static String addSample ( +// String message +// ) { +// return "Entry ${this.getCount()}: ${message}".toString() +// } +// } diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index d55e4ed..f89637f 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -138,14 +138,14 @@ class SchemaValidator extends PluginExtensionPoint { Map options = null, String samplesheetParam ) { - def String baseDir = session.baseDir def Map params = session.params // Set defaults for optional inputs def String schemaFilename = options?.containsKey('parameters_schema') ? options.parameters_schema as String : 'nextflow_schema.json' + def Boolean header = options?.containsKey("header") ? options.header as Boolean : true def slurper = new JsonSlurper() - def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) + def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(schemaFilename)) ) def Map samplesheetValue = (Map) findDeep(parsed, samplesheetParam) def Path samplesheetFile = params[samplesheetParam] as Path if (samplesheetFile == null) { @@ -158,7 +158,7 @@ class SchemaValidator extends PluginExtensionPoint { throw new SchemaValidationException("", []) } else if (samplesheetValue.containsKey('schema')) { - schemaFile = Path.of(Utils.getSchemaPath(baseDir, samplesheetValue['schema'].toString())) + schemaFile = Path.of(Utils.getSchemaPath(samplesheetValue['schema'].toString())) } else { log.error "Parameter '--$samplesheetParam' does not contain a schema in the parameter schema ($schemaFilename). Unable to create a channel from it." throw new SchemaValidationException("", []) @@ -168,7 +168,7 @@ class SchemaValidator extends PluginExtensionPoint { final channel = CH.create() def List arrayChannel = [] try { - arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile) + arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile, header) } catch (Exception e) { log.error( """ Following error has been found during samplesheet conversion: @@ -252,7 +252,6 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ ) { def Map params = initialiseExpectedParams(session.params) - def String baseDir = session.baseDir def Boolean s3PathCheck = params.validationS3PathCheck ? params.validationS3PathCheck : false def Boolean useMonochromeLogs = options?.containsKey('monochrome_logs') ? options.monochrome_logs as Boolean : params.monochrome_logs ? params.monochrome_logs as Boolean : @@ -269,7 +268,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ //=====================================================================// // Check for nextflow core params and unexpected params def slurper = new JsonSlurper() - def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) + def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(schemaFilename)) ) def Map schemaParams = (Map) parsed.get('defs') def specifiedParamKeys = params.keySet() @@ -312,7 +311,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ //=====================================================================// // Validate parameters against the schema - def String schema_string = Files.readString( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) + def String schema_string = Files.readString( Path.of(Utils.getSchemaPath(schemaFilename)) ) def validator = new JsonSchemaValidator(schema_string) // check for warnings @@ -384,7 +383,6 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ String command ) { def Map params = initialiseExpectedParams(session.params) - def String baseDir = session.baseDir def String schemaFilename = options?.containsKey('parameters_schema') ? options.parameters_schema as String : 'nextflow_schema.json' def Boolean useMonochromeLogs = options?.containsKey('monochrome_logs') ? options.monochrome_logs as Boolean : @@ -397,7 +395,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ String output = '' output += 'Typical pipeline command:\n\n' output += " ${colors.cyan}${command}${colors.reset}\n\n" - Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) + Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(schemaFilename)) ) Integer max_chars = paramsMaxChars(params_map) + 1 Integer desc_indent = max_chars + 14 Integer dec_linewidth = 160 - desc_indent @@ -487,7 +485,6 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ ) { def String schemaFilename = options?.containsKey('parameters_schema') ? options.parameters_schema as String : 'nextflow_schema.json' - def String baseDir = session.baseDir def Map params = session.params // Get a selection of core Nextflow workflow options @@ -512,7 +509,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ // Get pipeline parameters defined in JSON Schema def Map params_summary = [:] - def Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) + def Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(schemaFilename)) ) for (group in params_map.keySet()) { def sub_params = new LinkedHashMap() def Map group_params = params_map.get(group) as Map // This gets the parameters of that particular group @@ -565,7 +562,6 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ WorkflowMetadata workflow ) { - def String baseDir = session.baseDir def Map params = session.params def String schemaFilename = options?.containsKey('parameters_schema') ? options.parameters_schema as String : 'nextflow_schema.json' diff --git a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy index 9d6e5ab..234470a 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy @@ -1,8 +1,14 @@ package nextflow.validation import org.yaml.snakeyaml.Yaml -import groovy.json.JsonSlurper +import org.json.JSONArray +import org.json.JSONObject +import org.json.JSONPointer +import org.json.JSONPointerException +import nextflow.Global +import groovy.json.JsonGenerator +import groovy.json.JsonSlurper import groovy.util.logging.Slf4j import java.nio.file.Path @@ -45,13 +51,11 @@ public class Utils { return header } - // Converts a given file to a map - public static List fileToMaps(Path file, String schemaName, String baseDir) { + // Converts a given file to a List + public static List fileToList(Path file, Path schema) { def String fileType = Utils.getFileType(file) def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null - def List> fileContent - def Map types = variableTypes(schemaName, baseDir) - def Boolean containsHeader = !(types.keySet().size() == 1 && types.keySet()[0] == "") + def Map types = variableTypes(schema) if (types.find{ it.value == "array" } as Boolean && fileType in ["csv", "tsv"]){ def msg = "Using \"type\": \"array\" in schema with a \".$fileType\" samplesheet is not supported\n" @@ -59,91 +63,82 @@ public class Utils { throw new SchemaValidationException(msg, []) } - if(!containsHeader){ - types = ["empty": types[""]] - } if(fileType == "yaml"){ - fileContent = new Yaml().load((file.text)).collect { - if(containsHeader) { - return it as Map - } - return ["empty": it] as Map - } + return new Yaml().load((file.text)) } else if(fileType == "json"){ - fileContent = new JsonSlurper().parseText(file.text).collect { - if(containsHeader) { - return it as Map - } - return ["empty": it] as Map - } + return new JsonSlurper().parseText(file.text) as List } else { - fileContent = file.splitCsv(header:containsHeader ?: ["empty"], strip:true, sep:delimiter, quote:'\"') + def Boolean header = getValueFromJson("#/items/properties", new JSONObject(schema.text)) ? true : false + def List fileContent = file.splitCsv(header:header, strip:true, sep:delimiter, quote:'\"') + if (!header) { + // Flatten no header inputs if they contain one value + fileContent = fileContent.collect { it instanceof List && it.size() == 1 ? it[0] : it } + } + + return castToType(fileContent) } - def List> fileContentCasted = castToType(fileContent, types) - return fileContentCasted + } + + // Converts a given file to a JSONArray + public static JSONArray fileToJsonArray(Path file, Path schema) { + // Remove all null values from JSON object + // and convert the groovy object to a JSONArray + def jsonGenerator = new JsonGenerator.Options() + .excludeNulls() + .build() + return new JSONArray(jsonGenerator.toJson(fileToList(file, schema))) } // // Cast a value to the provided type in a Strict mode // - public static List castToType(List rows, Map types) { - def List casted = [] + public static Object castToType(Object input) { def Set validBooleanValues = ['true', 'false'] as Set - for( Map row in rows) { - def Map castedRow = [:] - - for (String key in row.keySet()) { - def String str = row[key] - def String type = types[key] - - try { - if( str == null || str == '' ) castedRow[key] = null - else if( type == null ) castedRow[key] = str - else if( type.toLowerCase() == 'boolean' && str.toLowerCase() in validBooleanValues ) castedRow[key] = str.toBoolean() - else if( type.toLowerCase() == 'character' ) castedRow[key] = str.toCharacter() - else if( type.toLowerCase() == 'short' && str.isNumber() ) castedRow[key] = str.toShort() - else if( type.toLowerCase() == 'integer' && str.isInteger() ) castedRow[key] = str.toInteger() - else if( type.toLowerCase() == 'long' && str.isLong() ) castedRow[key] = str.toLong() - else if( type.toLowerCase() == 'float' && str.isFloat() ) castedRow[key] = str.toFloat() - else if( type.toLowerCase() == 'double' && str.isDouble() ) castedRow[key] = str.toDouble() - else if( type.toLowerCase() == 'string' ) castedRow[key] = str - else { - castedRow[key] = str - } - } catch( Exception e ) { - log.warn "Unable to cast value $str to type $type: $e" - castedRow[key] = str - } - + if (input instanceof Map) { + // Cast all values in the map + def Map output = [:] + input.each { k, v -> + output[k] = castToType(v) } - - casted = casted + castedRow + return output + } + else if (input instanceof List) { + // Cast all values in the list + def List output = [] + for( entry : input ) { + output.add(castToType(entry)) + } + return output + } else if (input instanceof String) { + // Cast the string if there is one + if (input == "") { + return null + } + return JSONObject.stringToValue(input) } - - return casted } // Resolve Schema path relative to main workflow directory - public static String getSchemaPath(String baseDir, String schemaFilename='nextflow_schema.json') { + public static String getSchemaPath(String schemaFilename='nextflow_schema.json') { if (Path.of(schemaFilename).exists()) { return schemaFilename } else { - return "${baseDir}/${schemaFilename}" + return "${Global.getSession().baseDir.toString()}/${schemaFilename}" } } // Function to obtain the variable types of properties from a JSON Schema - public static Map variableTypes(String schemaFilename, String baseDir) { + public static Map variableTypes(Path schema) { def Map variableTypes = [:] def String type = '' // Read the schema def slurper = new JsonSlurper() - def Map parsed = (Map) slurper.parse( Path.of(getSchemaPath(baseDir, schemaFilename)) ) + def Map parsed = (Map) slurper.parse( schema ) // Obtain the type of each variable in the schema def Map properties = (Map) parsed['items']['properties'] @@ -160,13 +155,14 @@ public class Utils { variableTypes[key] = type } else { - variableTypes[key] = 'string' // If there isn't a type specifyed, return 'string' to avoid having a null value + variableTypes[key] = 'string' // If there isn't a type specified, return 'string' to avoid having a null value } } return variableTypes } + // Function to check if a String value is an Integer public static Boolean isInteger(String input) { try { input as Integer @@ -175,4 +171,34 @@ public class Utils { return false } } + + // Function to check if a String value is a Float + public static Boolean isFloat(String input) { + try { + input as Float + return true + } catch (NumberFormatException e) { + return false + } + } + + // Function to check if a String value is a Double + public static Boolean isDouble(String input) { + try { + input as Double + return true + } catch (NumberFormatException e) { + return false + } + } + + // Function to get the value from a JSON pointer + public static String getValueFromJson(String jsonPointer, Object json) { + def JSONPointer schemaPointer = new JSONPointer(jsonPointer) + try { + return schemaPointer.queryFrom(json) ?: "" + } catch (JSONPointerException e) { + return "" + } + } } \ No newline at end of file From 083cbad004e81883364d15cdeb66f780a893350d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 2 Feb 2024 13:02:24 +0100 Subject: [PATCH 52/96] improve the efficiency of util functions --- .../validation/CustomEvaluatorFactory.groovy | 8 +++- .../CustomEvaluators/SchemaEvaluator.groovy | 6 ++- .../validation/SchemaValidator.groovy | 40 ++++++++++--------- .../src/main/nextflow/validation/Utils.groovy | 6 +-- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy index 0c52cbc..b049a26 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy @@ -1,6 +1,7 @@ package nextflow.validation import nextflow.Global +import nextflow.Session import dev.harrel.jsonschema.EvaluatorFactory import dev.harrel.jsonschema.Evaluator import dev.harrel.jsonschema.SchemaParsingContext @@ -13,9 +14,12 @@ import dev.harrel.jsonschema.JsonNode class CustomEvaluatorFactory implements EvaluatorFactory { private Boolean lenientMode + private String baseDir CustomEvaluatorFactory() { - this.lenientMode = Global.getSession().params.validationLenientMode ?: false + def Session session = Global.getSession() + this.lenientMode = session.params.validationLenientMode ?: false + this.baseDir = session.baseDir.toString() } @Override @@ -35,7 +39,7 @@ class CustomEvaluatorFactory implements EvaluatorFactory { } else if (fieldName == "exists" && schemaNode.isBoolean()) { return Optional.of(new ExistsEvaluator(schemaNode.asBoolean())) } else if (fieldName == "schema" && schemaNode.isString()) { - return Optional.of(new SchemaEvaluator(schemaNode.asString())) + return Optional.of(new SchemaEvaluator(schemaNode.asString(), this.baseDir)) } else if (fieldName == "uniqueEntries" && schemaNode.isArray()) { return Optional.of(new UniqueEntriesEvaluator(schemaNode.asArray())) } else if (fieldName == "type" && (schemaNode.isString() || schemaNode.isArray()) && lenientMode) { diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index f41bd11..95dbe6f 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -21,8 +21,10 @@ class SchemaEvaluator implements Evaluator { // Evaluate the file using the given schema private final String schema + private final String baseDir - SchemaEvaluator(String schema) { + SchemaEvaluator(String schema, String baseDir) { + this.baseDir = baseDir this.schema = schema } @@ -45,7 +47,7 @@ class SchemaEvaluator implements Evaluator { log.debug("Started validating ${file.toString()}") - def String schemaFull = Utils.getSchemaPath(this.schema) + def String schemaFull = Utils.getSchemaPath(this.baseDir, this.schema) def JSONArray arrayJSON = Utils.fileToJsonArray(file, Path.of(schemaFull)) def String schemaContents = Files.readString( Path.of(schemaFull) ) def validator = new JsonSchemaValidator(schemaContents) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index f89637f..3c82428 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -144,8 +144,10 @@ class SchemaValidator extends PluginExtensionPoint { def String schemaFilename = options?.containsKey('parameters_schema') ? options.parameters_schema as String : 'nextflow_schema.json' def Boolean header = options?.containsKey("header") ? options.header as Boolean : true + def baseDir = session.baseDir.toString() + def slurper = new JsonSlurper() - def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(schemaFilename)) ) + def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) def Map samplesheetValue = (Map) findDeep(parsed, samplesheetParam) def Path samplesheetFile = params[samplesheetParam] as Path if (samplesheetFile == null) { @@ -158,7 +160,7 @@ class SchemaValidator extends PluginExtensionPoint { throw new SchemaValidationException("", []) } else if (samplesheetValue.containsKey('schema')) { - schemaFile = Path.of(Utils.getSchemaPath(samplesheetValue['schema'].toString())) + schemaFile = Path.of(Utils.getSchemaPath(baseDir, samplesheetValue['schema'].toString())) } else { log.error "Parameter '--$samplesheetParam' does not contain a schema in the parameter schema ($schemaFilename). Unable to create a channel from it." throw new SchemaValidationException("", []) @@ -167,18 +169,19 @@ class SchemaValidator extends PluginExtensionPoint { // Convert to channel final channel = CH.create() def List arrayChannel = [] - try { - arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile, header) - } catch (Exception e) { - log.error( - """ Following error has been found during samplesheet conversion: - ${e} - -Please run validateParameters() first before trying to convert a samplesheet to a channel. -Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ -""" as String - ) - } + // TODO turn this back on when done + // try { + arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile) +// } catch (Exception e) { +// log.error( +// """ Following error has been found during samplesheet conversion: +// ${e} + +// Please run validateParameters() first before trying to convert a samplesheet to a channel. +// Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ +// """ as String +// ) +// } session.addIgniter { arrayChannel.each { @@ -252,6 +255,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ ) { def Map params = initialiseExpectedParams(session.params) + def String baseDir = session.baseDir.toString() def Boolean s3PathCheck = params.validationS3PathCheck ? params.validationS3PathCheck : false def Boolean useMonochromeLogs = options?.containsKey('monochrome_logs') ? options.monochrome_logs as Boolean : params.monochrome_logs ? params.monochrome_logs as Boolean : @@ -268,7 +272,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ //=====================================================================// // Check for nextflow core params and unexpected params def slurper = new JsonSlurper() - def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(schemaFilename)) ) + def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) def Map schemaParams = (Map) parsed.get('defs') def specifiedParamKeys = params.keySet() @@ -311,7 +315,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ //=====================================================================// // Validate parameters against the schema - def String schema_string = Files.readString( Path.of(Utils.getSchemaPath(schemaFilename)) ) + def String schema_string = Files.readString( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) def validator = new JsonSchemaValidator(schema_string) // check for warnings @@ -395,7 +399,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ String output = '' output += 'Typical pipeline command:\n\n' output += " ${colors.cyan}${command}${colors.reset}\n\n" - Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(schemaFilename)) ) + Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(session.baseDir.toString(), schemaFilename)) ) Integer max_chars = paramsMaxChars(params_map) + 1 Integer desc_indent = max_chars + 14 Integer dec_linewidth = 160 - desc_indent @@ -509,7 +513,7 @@ Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ // Get pipeline parameters defined in JSON Schema def Map params_summary = [:] - def Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(schemaFilename)) ) + def Map params_map = paramsLoad( Path.of(Utils.getSchemaPath(session.baseDir.toString(), schemaFilename)) ) for (group in params_map.keySet()) { def sub_params = new LinkedHashMap() def Map group_params = params_map.get(group) as Map // This gets the parameters of that particular group diff --git a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy index 234470a..eea3300 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/Utils.groovy @@ -123,11 +123,11 @@ public class Utils { } // Resolve Schema path relative to main workflow directory - public static String getSchemaPath(String schemaFilename='nextflow_schema.json') { + public static String getSchemaPath(String baseDir, String schemaFilename='nextflow_schema.json') { if (Path.of(schemaFilename).exists()) { return schemaFilename } else { - return "${Global.getSession().baseDir.toString()}/${schemaFilename}" + return "${baseDir}/${schemaFilename}" } } @@ -193,7 +193,7 @@ public class Utils { } // Function to get the value from a JSON pointer - public static String getValueFromJson(String jsonPointer, Object json) { + public static Object getValueFromJson(String jsonPointer, Object json) { def JSONPointer schemaPointer = new JSONPointer(jsonPointer) try { return schemaPointer.queryFrom(json) ?: "" From b83d3801deae6bc59b90be0f4cdb776dc65651ef Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 2 Feb 2024 14:13:37 +0100 Subject: [PATCH 53/96] support deeply nested samplesheets --- .../validation/SamplesheetConverter.groovy | 173 +++++------------- 1 file changed, 48 insertions(+), 125 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index 50de35a..b05bfc7 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -33,148 +33,71 @@ import nextflow.Session class SamplesheetConverter { private static List rows = [] + private static Map meta = [:] - static List convertToList(Path samplesheetFile, Path schemaFile, Boolean header) { - - def JSONObject schemaMap = new JSONObject(schemaFile.text) - def JSONArray samplesheetList = Utils.fileToJsonArray(samplesheetFile, schemaFile) - - this.rows = [] - - def Iterator samplesheetIterator = samplesheetList.iterator() + private static Map getMeta() { + this.meta + } - while (samplesheetIterator.hasNext()) { - println(samplesheetIterator.next()) - } + private static Map resetMeta() { + this.meta = [:] + } - // def List outputs = samplesheetList.collect { fullRow -> - - // println(fullrow.getClass()) - - // def Map row = fullRow.findAll { it.value != "" } - // def Set rowKeys = header ? row.keySet() : ["empty"].toSet() - - // Check the header (CSV/TSV) or present fields (YAML) - // TODO reimplement warning for unused fields - - // this.rows.add(row) - - // def Map meta = [:] - // def ArrayList output = [] - - // for( Map.Entry field : schemaFields ){ - // def String key = header ? field.key : "empty" - // def Object input = row[key] - - // // Convert field to a meta field or add it as an input to the channel - // def List metaNames = field['value']['meta'] as List - // if(metaNames) { - // for(name : metaNames) { - // meta[name] = (input != '' && input != null) ? - // castToNFType(input, field) : - // field['value']['default'] != null ? - // castToNFType(field['value']['default'], field) : - // null - // } - // } - // else { - // def inputVal = (input != '' && input != null) ? - // castToNFType(input, field) : - // field['value']['default'] != null ? - // castToNFType(field['value']['default'], field) : - // [] - // output.add(inputVal) - // } - // } - // // Add meta to the output when a meta field has been created - // if(meta != [:]) { output.add(0, meta) } - // return [] - // } - - return [] + private static addMeta(Map newEntries) { + this.meta = this.meta + newEntries } - // Function to transform an input field from the samplesheet to its desired type - private static castToNFType( - Object input, - Map.Entry field - ) { - def String type = field['value']['type'] - def String key = field.key - - // Recursively call this function for each item in the array if the field is an array-type - // The returned values are collected into a single array - if (type == "array") { - def Map.Entry subfield = (Map.Entry) Map.entry(field.key, field['value']['items']) - log.debug "subfield = $subfield" - def ArrayList result = input.collect{ castToNFType(it, subfield) } as ArrayList - return result - } + private static Boolean isMeta() { + this.meta.size() > 0 + } - def String inputStr = input as String - // Convert string values - if(type == "string" || !type) { - def String result = inputStr as String - - // Check and convert to the desired format - def String format = field['value']['format'] - if(format) { - if(format == "file-path-pattern") { - def ArrayList inputFiles = Nextflow.file(inputStr) as ArrayList - return inputFiles - } - if(format.contains("path")) { - def Path inputFile = Nextflow.file(inputStr) as Path - return inputFile - } - } - + public static List convertToList(Path samplesheetFile, Path schemaFile) { - // Return the plain string value - return result - } + def LinkedHashMap schemaMap = new JsonSlurper().parseText(schemaFile.text) as LinkedHashMap + def List samplesheetList = Utils.fileToList(samplesheetFile, schemaFile) - // Convert number values - else if(type == "number") { - try { - def int result = inputStr as int - return result - } - catch (NumberFormatException e) { - log.debug("Could not convert ${input} to an integer. Trying to convert to a float.") - } + this.rows = [] - try { - def float result = inputStr as float - return result + def List channelFormat = samplesheetList.collect { entry -> + resetMeta() + def Object result = formatEntry(entry, schemaMap["items"] as LinkedHashMap) + if (result instanceof List) { + result.add(0,getMeta()) + } else if (isMeta()) { + result = [getMeta(), result] } - catch (NumberFormatException e) { - log.debug("Could not convert ${inputStr} to a float. Trying to convert to a double.") - } - - def double result = inputStr as double return result } + return channelFormat + } - // Convert integer values - else if(type == "integer") { - - def int result = inputStr as int + private static Object formatEntry(Object input, LinkedHashMap schema) { + + if (input instanceof Map) { + def List result = [] + def LinkedHashMap properties = schema["properties"] + properties.each { property, schemaValues -> + def value = input[property] ?: [] + def List metaIds = schemaValues["meta"] instanceof List ? schemaValues["meta"] as List : schemaValues["meta"] instanceof String ? [schemaValues["meta"]] : [] + if (metaIds) { + metaIds.each { + addMeta(["${it}":value]) + } + } else { + result += [formatEntry(value, schemaValues as LinkedHashMap)] + } + } return result - } - - // Convert boolean values - else if(type == "boolean") { - - if(inputStr.toLowerCase() == "true") { - return true + } else if (input instanceof List) { + def List result = [] + input.each { + result += [formatEntry(it, schema["items"] as LinkedHashMap)] } - return false + return result + } else { + return input } - else if(type == "null") { - return null - } } } From 69314b6a46b737aebf93c019555989cc9a010ece Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 2 Feb 2024 14:35:37 +0100 Subject: [PATCH 54/96] update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b808d78..c48a66e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,12 +20,15 @@ To migrate simple schemas, you can simply follow the next steps: - Removed all validation code from the `.fromSamplesheet()` channel factory. The validation is now solely done in the `validateParameters()` function. A custom error message will now be displayed if any error has been encountered during the conversion ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) - Removed the `unique` keyword from the samplesheet schema. You should now use [`uniqueItems`](https://json-schema.org/understanding-json-schema/reference/array#uniqueItems) or `uniqueEntries` instead ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) - Removed the `skip_duplicate_check` option from the `fromSamplesheet()` channel factory and the `--validationSkipDuplicateCheck` parameter. You should now use the `uniqueEntries` or [`uniqueItems`](https://json-schema.org/understanding-json-schema/reference/array#uniqueItems) keywords in the schema instead ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) +- `.fromSamplesheet()` now does dynamic typecasting instead of using the `type` fields in the JSON schema. This is done due to the complexity of `draft-2020-12` JSON schemas. This should not have that much impact but keep in mind that some types can be different between this and earlier versions because of this ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) +- `.fromSamplesheet()` will now set all missing values as `[]` instead of the type specific defaults (because of the changes in the previous point). This should not change that much as this will also result in `false` when used in conditions. ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) ## Improvements - Setting the `exists` keyword to `false` will now check if the path does not exist ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) - The `schema` keyword will now work in all schemas. ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) - Improved the error messages ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) +- `.fromSamplesheet()` now supports deeply nested samplesheets ([#141](https://github.com/nextflow-io/nf-validation/pull/141)) # Version 1.1.3 - Asahikawa From e840ffd360c71930eee151e637422313583f0bf5 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 2 Feb 2024 17:20:25 +0100 Subject: [PATCH 55/96] add format casting --- .../validation/SamplesheetConverter.groovy | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index b05bfc7..3d51ada 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -71,6 +71,7 @@ class SamplesheetConverter { return channelFormat } + // TODO add path casting private static Object formatEntry(Object input, LinkedHashMap schema) { if (input instanceof Map) { @@ -84,20 +85,40 @@ class SamplesheetConverter { addMeta(["${it}":value]) } } else { - result += [formatEntry(value, schemaValues as LinkedHashMap)] + result.add(formatEntry(value, schemaValues as LinkedHashMap)) } } return result } else if (input instanceof List) { def List result = [] input.each { - result += [formatEntry(it, schema["items"] as LinkedHashMap)] + result.add(formatEntry(it, schema["items"] as LinkedHashMap)) } return result } else { + def List formats = getPathFormats(schema) + if (formats && input instanceof String) { + return Nextflow.file(input) + } return input } } + private static List validPathFormats = ["file-path", "path", "directory-path", "file-path-pattern"] + + private static List getPathFormats(Map schemaEntry) { + def List formats = [] + formats.add(schemaEntry.format ?: []) + def List options = ["anyOf", "oneOf", "allOf"] + options.each { option -> + if(schemaEntry[option]) { + schemaEntry[option].each { + formats.add(it["format"] ?: []) + } + } + } + return formats.findAll { it != [] && this.validPathFormats.contains(it) } + } + } From 4f5575ad19e3a038a2a81d944fd89a998ca6a6ad Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 5 Feb 2024 13:22:09 +0100 Subject: [PATCH 56/96] add warnings for unidentified headers --- .../validation/SamplesheetConverter.groovy | 43 ++++++++++++++++--- .../validation/SchemaValidator.groovy | 3 +- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index 3d51ada..7d6a737 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -32,6 +32,14 @@ import nextflow.Session @CompileStatic class SamplesheetConverter { + private static Path samplesheetFile + private static Path schemaFile + + SamplesheetConverter(Path samplesheetFile, Path schemaFile) { + this.samplesheetFile = samplesheetFile + this.schemaFile = schemaFile + } + private static List rows = [] private static Map meta = [:] @@ -51,10 +59,24 @@ class SamplesheetConverter { this.meta.size() > 0 } - public static List convertToList(Path samplesheetFile, Path schemaFile) { + private static List unusedHeaders = [] + + private static addUnusedHeader (String header) { + this.unusedHeaders.add(header) + } + + private static logUnusedHeadersWarning(String fileName) { + def Set unusedHeaders = this.unusedHeaders as Set + if(unusedHeaders.size() > 0) { + def String processedHeaders = unusedHeaders.collect { "\t- ${it}" }.join("\n") + log.warn("Found the following unidentified headers in ${fileName}:\n${processedHeaders}" as String) + } + } + + public static List convertToList() { - def LinkedHashMap schemaMap = new JsonSlurper().parseText(schemaFile.text) as LinkedHashMap - def List samplesheetList = Utils.fileToList(samplesheetFile, schemaFile) + def LinkedHashMap schemaMap = new JsonSlurper().parseText(this.schemaFile.text) as LinkedHashMap + def List samplesheetList = Utils.fileToList(this.samplesheetFile, this.schemaFile) this.rows = [] @@ -68,15 +90,18 @@ class SamplesheetConverter { } return result } + logUnusedHeadersWarning(this.samplesheetFile.toString()) return channelFormat } - // TODO add path casting - private static Object formatEntry(Object input, LinkedHashMap schema) { + // TODO add warning for headers that aren't in the schema + private static Object formatEntry(Object input, LinkedHashMap schema, String headerPrefix = "") { if (input instanceof Map) { def List result = [] def LinkedHashMap properties = schema["properties"] + def Set unusedKeys = input.keySet() - properties.keySet() + unusedKeys.each{addUnusedHeader("${headerPrefix}${it}" as String)} properties.each { property, schemaValues -> def value = input[property] ?: [] def List metaIds = schemaValues["meta"] instanceof List ? schemaValues["meta"] as List : schemaValues["meta"] instanceof String ? [schemaValues["meta"]] : [] @@ -85,14 +110,18 @@ class SamplesheetConverter { addMeta(["${it}":value]) } } else { - result.add(formatEntry(value, schemaValues as LinkedHashMap)) + def String prefix = headerPrefix ? "${headerPrefix}${property}." : "${property}." + result.add(formatEntry(value, schemaValues as LinkedHashMap, prefix)) } } return result } else if (input instanceof List) { def List result = [] + def Integer count = 0 input.each { - result.add(formatEntry(it, schema["items"] as LinkedHashMap)) + def String prefix = headerPrefix ? "${headerPrefix}${count}." : "${count}." + result.add(formatEntry(it, schema["items"] as LinkedHashMap, prefix)) + count++ } return result } else { diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 3c82428..badab41 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -171,7 +171,8 @@ class SchemaValidator extends PluginExtensionPoint { def List arrayChannel = [] // TODO turn this back on when done // try { - arrayChannel = SamplesheetConverter.convertToList(samplesheetFile, schemaFile) + SamplesheetConverter converter = new SamplesheetConverter(samplesheetFile, schemaFile) + arrayChannel = converter.convertToList() // } catch (Exception e) { // log.error( // """ Following error has been found during samplesheet conversion: From 39322adbcaac00aff2b339172975d77cd98732ec Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 12 Feb 2024 15:24:36 +0100 Subject: [PATCH 57/96] Add some clarifying comments --- .../nextflow/validation/SamplesheetConverter.groovy | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index 7d6a737..f40302a 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -101,15 +101,23 @@ class SamplesheetConverter { def List result = [] def LinkedHashMap properties = schema["properties"] def Set unusedKeys = input.keySet() - properties.keySet() + + // Check for properties in the samplesheet that have not been defined in the schema unusedKeys.each{addUnusedHeader("${headerPrefix}${it}" as String)} + + // Loop over every property to maintain the correct order properties.each { property, schemaValues -> def value = input[property] ?: [] def List metaIds = schemaValues["meta"] instanceof List ? schemaValues["meta"] as List : schemaValues["meta"] instanceof String ? [schemaValues["meta"]] : [] + + // Add the value to the meta map if needed if (metaIds) { metaIds.each { addMeta(["${it}":value]) } - } else { + } + // return the correctly casted value + else { def String prefix = headerPrefix ? "${headerPrefix}${property}." : "${property}." result.add(formatEntry(value, schemaValues as LinkedHashMap, prefix)) } @@ -119,12 +127,14 @@ class SamplesheetConverter { def List result = [] def Integer count = 0 input.each { + // return the correctly casted value def String prefix = headerPrefix ? "${headerPrefix}${count}." : "${count}." result.add(formatEntry(it, schema["items"] as LinkedHashMap, prefix)) count++ } return result } else { + // Cast value to path type if needed and return the value def List formats = getPathFormats(schema) if (formats && input instanceof String) { return Nextflow.file(input) From 94716f2c8223c8db587fd3154a1882983eb24028 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 12 Feb 2024 16:17:26 +0100 Subject: [PATCH 58/96] made the format casting algorithm a bit smarter --- .../validation/SamplesheetConverter.groovy | 59 ++++++++++++++----- .../validation/SchemaValidator.groovy | 23 ++++---- 2 files changed, 56 insertions(+), 26 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index f40302a..373104d 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -135,29 +135,60 @@ class SamplesheetConverter { return result } else { // Cast value to path type if needed and return the value - def List formats = getPathFormats(schema) - if (formats && input instanceof String) { - return Nextflow.file(input) - } - return input + return castToFormat(input, schema) } } private static List validPathFormats = ["file-path", "path", "directory-path", "file-path-pattern"] + private static List schemaOptions = ["anyOf", "oneOf", "allOf"] + + private static Object castToFormat(Object value, Map schemaEntry) { + if(!(value instanceof String)) { + return value + } + + // A valid path format has been found in the schema + def Boolean foundStringFileFormat = false + + // Type string has been found without a valid path format + def Boolean foundStringNoFileFormat = false + + if ((schemaEntry.type ?: "") == "string") { + if (validPathFormats.contains(schemaEntry.format ?: "")) { + foundStringFileFormat = true + } else { + foundStringNoFileFormat = true + } + } - private static List getPathFormats(Map schemaEntry) { - def List formats = [] - formats.add(schemaEntry.format ?: []) - def List options = ["anyOf", "oneOf", "allOf"] - options.each { option -> - if(schemaEntry[option]) { - schemaEntry[option].each { - formats.add(it["format"] ?: []) + schemaOptions.each { option -> + schemaEntry[option]?.each { subSchema -> + if ((subSchema["type"] ?: "" ) == "string") { + if (validPathFormats.contains(subSchema["format"] ?: "")) { + foundStringFileFormat = true + } else { + foundStringNoFileFormat = true + } } } } - return formats.findAll { it != [] && this.validPathFormats.contains(it) } + + if(foundStringFileFormat && !foundStringNoFileFormat) { + return Nextflow.file(value) + } else if(foundStringFileFormat && foundStringNoFileFormat) { + // Do a simple check if the object could be a path + // This check looks for / in the filename or if a dot is + // present in the last 7 characters (possibly indicating an extension) + if( + value.contains("/") || + (value.size() >= 7 && value[-7..-1].contains(".")) || + (value.size() < 7 && value.contains(".")) + ) { + return Nextflow.file(value) + } + } + return value } } diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index badab41..da5ab9b 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -169,20 +169,19 @@ class SchemaValidator extends PluginExtensionPoint { // Convert to channel final channel = CH.create() def List arrayChannel = [] - // TODO turn this back on when done - // try { + try { SamplesheetConverter converter = new SamplesheetConverter(samplesheetFile, schemaFile) arrayChannel = converter.convertToList() -// } catch (Exception e) { -// log.error( -// """ Following error has been found during samplesheet conversion: -// ${e} - -// Please run validateParameters() first before trying to convert a samplesheet to a channel. -// Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ -// """ as String -// ) -// } + } catch (Exception e) { + log.error( + """ Following error has been found during samplesheet conversion: + ${e} + +Please run validateParameters() first before trying to convert a samplesheet to a channel. +Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ +""" as String + ) + } session.addIgniter { arrayChannel.each { From cb6992d3cb6e784a86fb7c1df69b6d929ebe1934 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 13 Feb 2024 09:43:02 +0100 Subject: [PATCH 59/96] remove old convertor code --- .../validation/SamplesheetConverterOld.groovy | 282 ------------------ 1 file changed, 282 deletions(-) delete mode 100644 plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverterOld.groovy diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverterOld.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverterOld.groovy deleted file mode 100644 index efda5c8..0000000 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverterOld.groovy +++ /dev/null @@ -1,282 +0,0 @@ -// package nextflow.validation - -// import groovy.json.JsonSlurper -// import groovy.json.JsonOutput -// import groovy.transform.CompileStatic -// import groovy.util.logging.Slf4j -// import groovyx.gpars.dataflow.DataflowReadChannel -// import groovyx.gpars.dataflow.DataflowWriteChannel - -// import java.nio.file.Path -// import java.util.concurrent.CompletableFuture -// import java.util.concurrent.CompletionException - -// import org.yaml.snakeyaml.Yaml -// import org.json.JSONArray -// import org.json.JSONObject -// import org.json.JSONTokener - -// import nextflow.Channel -// import nextflow.Global -// import nextflow.Nextflow -// import nextflow.plugin.extension.Function -// import nextflow.Session - -// /** -// * @author : mirpedrol -// * @author : nvnieuwk -// * @author : awgymer -// */ - -// @Slf4j -// @CompileStatic -// class SamplesheetConverterOld { - -// private static List errors = [] -// private static List schemaErrors = [] -// private static List warnings = [] - -// private static List rows = [] - -// static boolean hasErrors() { errors.size()>0 } -// static Set getErrors() { errors.sort().collect { "\t${it}".toString() } as Set } - -// static boolean hasSchemaErrors() { schemaErrors.size()>0 } -// static Set getSchemaErrors() { schemaErrors.sort().collect { "\t${it}".toString() } as Set } - -// static boolean hasWarnings() { warnings.size()>0 } -// static Set getWarnings() { warnings.sort().collect { "\t${it}".toString() } as Set } - -// private static Integer sampleCount = 0 - -// static resetCount(){ sampleCount = 0 } -// static increaseCount(){ sampleCount++ } -// static Integer getCount(){ sampleCount } - - -// static List convertToList( -// Path samplesheetFile, -// Path schemaFile -// ) { - -// def Map schemaMap = (Map) new JsonSlurper().parseText(schemaFile.text) -// def Map> schemaFields = (Map) schemaMap["items"]["properties"] -// def Set allFields = schemaFields.keySet() -// def List requiredFields = (List) schemaMap["items"]["required"] -// def Boolean containsHeader = !(allFields.size() == 1 && allFields[0] == "") - -// def String fileType = Utils.getFileType(samplesheetFile) -// def List> samplesheetList = Utils.fileToJsonArray(samplesheetFile, schemaFile.toString(), Global.getSession().baseDir.toString()) - -// // Field checks + returning the channels -// def Map> booleanUniques = [:] -// def Map>> listUniques = [:] -// def Boolean headerCheck = true -// this.rows = [] -// resetCount() - -// def List outputs = samplesheetList.collect { Map fullRow -> -// increaseCount() - -// Map row = fullRow.findAll { it.value != "" } -// def Set rowKeys = containsHeader ? row.keySet() : ["empty"].toSet() -// def String entryInfo = fileType in ["yaml", "json"] ? " for entry ${this.getCount()}." : "" - -// // Check the header (CSV/TSV) or present fields (YAML) -// if(headerCheck) { -// def unexpectedFields = containsHeader ? rowKeys - allFields : [] -// if(unexpectedFields.size() > 0) { -// this.warnings << "The samplesheet contains following unchecked field(s): ${unexpectedFields}${entryInfo}".toString() -// } - -// if(fileType != 'yaml'){ -// headerCheck = false -// } -// } - -// this.rows.add(row) - -// def Map meta = [:] -// def ArrayList output = [] - -// for( Map.Entry field : schemaFields ){ -// def String key = containsHeader ? field.key : "empty" -// def Object input = row[key] - -// // Check if the field is deprecated -// if(field['value']['deprecated']){ -// this.warnings << "The '${key}' field is deprecated and will no longer be used in the future. Please check the official documentation of the pipeline for more information.".toString() -// } - -// // Check required dependencies -// def List dependencies = field['value']["dependentRequired"] as List -// if(input && dependencies) { -// def List missingValues = [] -// for( dependency in dependencies ){ -// if(row[dependency] == "" || !(row[dependency])) { -// missingValues.add(dependency) -// } -// } -// if (missingValues) { -// this.errors << addSample("${dependencies} field(s) should be defined when '${key}' is specified, but the field(s) ${missingValues} is/are not defined.".toString()) -// } -// } - -// // Check if the field is unique -// def unique = field['value']['unique'] -// def Boolean uniqueIsList = unique instanceof ArrayList -// if(unique && !uniqueIsList){ -// if(!(key in booleanUniques)){ -// booleanUniques[key] = [] -// } -// if(input in booleanUniques[key] && input){ -// this.errors << addSample("The '${key}' value needs to be unique. '${input}' was found at least twice in the samplesheet.".toString()) -// } -// booleanUniques[key].add(input as String) -// } -// else if(unique && uniqueIsList) { -// def Map newMap = (Map) row.subMap((List) [key] + (List) unique) -// if(!(key in listUniques)){ -// listUniques[key] = [] -// } -// if(newMap in listUniques[key] && input){ -// this.errors << addSample("The combination of '${key}' with fields ${unique} needs to be unique. ${newMap} was found at least twice.".toString()) -// } -// listUniques[key].add(newMap) -// } - -// // Convert field to a meta field or add it as an input to the channel -// def List metaNames = field['value']['meta'] as List -// if(metaNames) { -// for(name : metaNames) { -// meta[name] = (input != '' && input != null) ? -// castToNFType(input, field) : -// field['value']['default'] != null ? -// castToNFType(field['value']['default'], field) : -// null -// } -// } -// else { -// def inputVal = (input != '' && input != null) ? -// castToNFType(input, field) : -// field['value']['default'] != null ? -// castToNFType(field['value']['default'], field) : -// [] -// output.add(inputVal) -// } -// } -// // Add meta to the output when a meta field has been created -// if(meta != [:]) { output.add(0, meta) } -// return output -// } - -// // check for samplesheet errors -// if (this.hasErrors()) { -// String message = "Samplesheet errors:\n" + this.getErrors().join("\n") -// throw new SchemaValidationException(message, this.getErrors() as List) -// } - -// // check for schema errors -// if (this.hasSchemaErrors()) { -// String message = "Samplesheet schema errors:\n" + this.getSchemaErrors().join("\n") -// throw new SchemaValidationException(message, this.getSchemaErrors() as List) -// } - -// // check for warnings -// if( this.hasWarnings() ) { -// def msg = "Samplesheet warnings:\n" + this.getWarnings().join('\n') -// log.warn(msg) -// } - -// return outputs -// } - -// // Function to transform an input field from the samplesheet to its desired type -// private static castToNFType( -// Object input, -// Map.Entry field -// ) { -// def String type = field['value']['type'] -// def String key = field.key - -// // Recursively call this function for each item in the array if the field is an array-type -// // The returned values are collected into a single array -// if (type == "array") { -// def Map.Entry subfield = (Map.Entry) Map.entry(field.key, field['value']['items']) -// log.debug "subfield = $subfield" -// def ArrayList result = input.collect{ castToNFType(it, subfield) } as ArrayList -// return result -// } - -// def String inputStr = input as String -// // Convert string values -// if(type == "string" || !type) { -// def String result = inputStr as String - -// // Check and convert to the desired format -// def String format = field['value']['format'] -// if(format) { -// if(format == "file-path-pattern") { -// def ArrayList inputFiles = Nextflow.file(inputStr) as ArrayList -// return inputFiles -// } -// if(format.contains("path")) { -// def Path inputFile = Nextflow.file(inputStr) as Path -// return inputFile -// } -// } - - -// // Return the plain string value -// return result -// } - -// // Convert number values -// else if(type == "number") { -// try { -// def int result = inputStr as int -// return result -// } -// catch (NumberFormatException e) { -// log.debug("Could not convert ${input} to an integer. Trying to convert to a float.") -// } - -// try { -// def float result = inputStr as float -// return result -// } -// catch (NumberFormatException e) { -// log.debug("Could not convert ${inputStr} to a float. Trying to convert to a double.") -// } - -// def double result = inputStr as double -// return result -// } - -// // Convert integer values -// else if(type == "integer") { - -// def int result = inputStr as int -// return result -// } - -// // Convert boolean values -// else if(type == "boolean") { - -// if(inputStr.toLowerCase() == "true") { -// return true -// } -// return false -// } - -// else if(type == "null") { -// return null -// } -// } - -// private static String addSample ( -// String message -// ) { -// return "Entry ${this.getCount()}: ${message}".toString() -// } -// } From b10c091959d7344729be63150d89660f2f4983b4 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 13 Feb 2024 09:44:35 +0100 Subject: [PATCH 60/96] remove fixed todo --- .../src/main/nextflow/validation/SamplesheetConverter.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index 373104d..0a36e67 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -94,7 +94,6 @@ class SamplesheetConverter { return channelFormat } - // TODO add warning for headers that aren't in the schema private static Object formatEntry(Object input, LinkedHashMap schema, String headerPrefix = "") { if (input instanceof Map) { From 318d512547f8cd851557f73176e265b5377f8179 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 13 Feb 2024 09:54:50 +0100 Subject: [PATCH 61/96] remove unused imports --- .../validation/SamplesheetConverter.groovy | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index 0a36e67..e95e70d 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -1,26 +1,12 @@ package nextflow.validation import groovy.json.JsonSlurper -import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowReadChannel -import groovyx.gpars.dataflow.DataflowWriteChannel import java.nio.file.Path -import java.util.concurrent.CompletableFuture -import java.util.concurrent.CompletionException -import org.yaml.snakeyaml.Yaml -import org.json.JSONArray -import org.json.JSONObject -import org.json.JSONTokener - -import nextflow.Channel -import nextflow.Global import nextflow.Nextflow -import nextflow.plugin.extension.Function -import nextflow.Session /** * @author : mirpedrol From 731d27f04a364103df5ed40ae54e08ee9935b4ca Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 13 Feb 2024 11:00:46 +0100 Subject: [PATCH 62/96] add defaults --- .../main/nextflow/validation/SamplesheetConverter.groovy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index e95e70d..bb8999b 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -82,6 +82,9 @@ class SamplesheetConverter { private static Object formatEntry(Object input, LinkedHashMap schema, String headerPrefix = "") { + // Add default values for missing entries + input = input != null ? input : schema.default != null ? schema.default : [] + if (input instanceof Map) { def List result = [] def LinkedHashMap properties = schema["properties"] @@ -92,7 +95,7 @@ class SamplesheetConverter { // Loop over every property to maintain the correct order properties.each { property, schemaValues -> - def value = input[property] ?: [] + def value = input[property] def List metaIds = schemaValues["meta"] instanceof List ? schemaValues["meta"] as List : schemaValues["meta"] instanceof String ? [schemaValues["meta"]] : [] // Add the value to the meta map if needed @@ -120,7 +123,7 @@ class SamplesheetConverter { return result } else { // Cast value to path type if needed and return the value - return castToFormat(input, schema) + return processValue(input, schema) } } @@ -128,7 +131,7 @@ class SamplesheetConverter { private static List validPathFormats = ["file-path", "path", "directory-path", "file-path-pattern"] private static List schemaOptions = ["anyOf", "oneOf", "allOf"] - private static Object castToFormat(Object value, Map schemaEntry) { + private static Object processValue(Object value, Map schemaEntry) { if(!(value instanceof String)) { return value } From 891adb7e8ab27756b02eef63ffcb80ee5d018d71 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 13 Feb 2024 12:50:58 +0100 Subject: [PATCH 63/96] fix for defaults in meta --- .../validation/SamplesheetConverter.groovy | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index bb8999b..afb5cb6 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -59,6 +59,9 @@ class SamplesheetConverter { } } + /* + Convert the samplesheet to a list of entries based on a schema + */ public static List convertToList() { def LinkedHashMap schemaMap = new JsonSlurper().parseText(this.schemaFile.text) as LinkedHashMap @@ -80,10 +83,14 @@ class SamplesheetConverter { return channelFormat } + /* + This function processes an input value based on a schema. + The output will be created for addition to the output channel. + */ private static Object formatEntry(Object input, LinkedHashMap schema, String headerPrefix = "") { // Add default values for missing entries - input = input != null ? input : schema.default != null ? schema.default : [] + input = input != null ? input : schema.containsKey("default") ? schema.default : [] if (input instanceof Map) { def List result = [] @@ -97,16 +104,16 @@ class SamplesheetConverter { properties.each { property, schemaValues -> def value = input[property] def List metaIds = schemaValues["meta"] instanceof List ? schemaValues["meta"] as List : schemaValues["meta"] instanceof String ? [schemaValues["meta"]] : [] + def String prefix = headerPrefix ? "${headerPrefix}${property}." : "${property}." // Add the value to the meta map if needed if (metaIds) { metaIds.each { - addMeta(["${it}":value]) + addMeta(["${it}":processMeta(value, schemaValues as LinkedHashMap, prefix)]) } } // return the correctly casted value else { - def String prefix = headerPrefix ? "${headerPrefix}${property}." : "${property}." result.add(formatEntry(value, schemaValues as LinkedHashMap, prefix)) } } @@ -131,6 +138,11 @@ class SamplesheetConverter { private static List validPathFormats = ["file-path", "path", "directory-path", "file-path-pattern"] private static List schemaOptions = ["anyOf", "oneOf", "allOf"] + /* + This function processes a value that's not a map or list and casts it to a file type if necessary. + When there is uncertainty if the value should be a path, some simple logic is applied that tries + to guess if it should be a file type + */ private static Object processValue(Object value, Map schemaEntry) { if(!(value instanceof String)) { return value @@ -179,4 +191,43 @@ class SamplesheetConverter { return value } + /* + This function processes an input value based on a schema. + The output will be created for addition to the meta map. + */ + private static Object processMeta(Object input, LinkedHashMap schema, String headerPrefix) { + // Add default values for missing entries + input = input != null ? input : schema.containsKey("default") ? schema.default : [] + + if (input instanceof Map) { + def Map result = [:] + def LinkedHashMap properties = schema["properties"] + def Set unusedKeys = input.keySet() - properties.keySet() + + // Check for properties in the samplesheet that have not been defined in the schema + unusedKeys.each{addUnusedHeader("${headerPrefix}${it}" as String)} + + // Loop over every property to maintain the correct order + properties.each { property, schemaValues -> + def value = input[property] + def String prefix = headerPrefix ? "${headerPrefix}${property}." : "${property}." + result[property] = processMeta(value, schemaValues as LinkedHashMap, prefix) + } + return result + } else if (input instanceof List) { + def List result = [] + def Integer count = 0 + input.each { + // return the correctly casted value + def String prefix = headerPrefix ? "${headerPrefix}${count}." : "${count}." + result.add(processMeta(it, schema["items"] as LinkedHashMap, prefix)) + count++ + } + return result + } else { + // Cast value to path type if needed and return the value + return processValue(input, schema) + } + } + } From 00ba8c60189f14557a103aa177aa7225456179b2 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 13 Feb 2024 13:18:22 +0100 Subject: [PATCH 64/96] fix meta addition when meta is missing --- .../nextflow/validation/SamplesheetConverter.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index afb5cb6..e010201 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -72,10 +72,12 @@ class SamplesheetConverter { def List channelFormat = samplesheetList.collect { entry -> resetMeta() def Object result = formatEntry(entry, schemaMap["items"] as LinkedHashMap) - if (result instanceof List) { - result.add(0,getMeta()) - } else if (isMeta()) { - result = [getMeta(), result] + if(isMeta()) { + if(result instanceof List) { + result.add(0,getMeta()) + } else { + result = [getMeta(), result] + } } return result } From a85fe66b6915952d81212333665be5587bbd9c50 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 13 Feb 2024 13:18:30 +0100 Subject: [PATCH 65/96] fix existing tests --- .../SamplesheetConverterTest.groovy | 122 +++--------------- .../src/testResources/no_header_schema.json | 7 +- .../schema_input_with_arrays.json | 1 + 3 files changed, 18 insertions(+), 112 deletions(-) diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy index 7aa9d1d..93d3c29 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy @@ -80,7 +80,7 @@ class SamplesheetConverterTest extends Dsl2Spec{ noExceptionThrown() stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.12, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/test.txt, unique1, 1, itDoesExist]" as String) stdout.contains("[[string1:value, string2:value, integer1:0, integer2:0, boolean1:true, boolean2:true], string1, 25.08, false, [], [], [], [], [], itDoesExist]") - stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25.0, false, [], [], [], unique2, 1, itDoesExist]") + stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, [], [], [], unique2, 1, itDoesExist]") stdout.contains("[[string1:extraField, string2:extraField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir, unique3, 1, itDoesExist]" as String) } @@ -213,9 +213,9 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() - stdout.contains("[[array_meta:null], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) + stdout.contains("[[array_meta:[]], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) stdout.contains("[[array_meta:[look, an, array, in, meta]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) - stdout.contains("[[array_meta:null], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String) + stdout.contains("[[array_meta:[]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String) } def 'arrays should work fine - JSON' () { @@ -239,102 +239,11 @@ class SamplesheetConverterTest extends Dsl2Spec{ then: noExceptionThrown() - stdout.contains("[[array_meta:null], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) + stdout.contains("[[array_meta:[]], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2], [${getRootString()}/src/testResources/testDir, ${getRootString()}/src/testResources/testDir2/testFile2.txt], [string1, string2], [25, 26], [25, 26.5], [false, true], [1, 2, 3], [true], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) stdout.contains("[[array_meta:[look, an, array, in, meta]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt]]]" as String) - stdout.contains("[[array_meta:null], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String) - } - - def 'array errors before channel conversion - YAML' () { - given: - def SCRIPT_TEXT = ''' - include { fromSamplesheet } from 'plugin/nf-validation' - - params.input = 'src/testResources/error_arrays.yaml' - - workflow { - Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json").view() - } - ''' - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.startsWith('[[') ? it : null } - - then: - def error = thrown(SchemaValidationException) - def errorMessages = error.message.readLines() - errorMessages[0] == "\033[0;31mThe following errors have been detected:" - errorMessages[2] == "* -- Entry 1 - field_3: the file or directory 'src/testResources/testDir3' does not exist." - errorMessages[3] == "* -- Entry 1 - field_3: the file or directory 'src/testResources/testDir2/testFile3.txt' does not exist." - errorMessages[4] == "* -- Entry 1 - field_2: the file or directory 'src/testResources/testDir3' does not exist." - errorMessages[5] == "* -- Entry 1 - field_1: the file or directory 'src/testResources/testDir/testFile.fasta' does not exist." - errorMessages[6] == "* -- Entry 1 - field_1: the file or directory 'src/testResources/testDir2/testFile3.txt' does not exist." - errorMessages[7] == '* -- Entry 1 - field_4: array items are not unique (["string2","string2","string1"])' - errorMessages[8] == '* -- Entry 1 - field_1: string [src/testResources/testDir/testFile.fasta] does not match pattern ^.*\\.txt$ (["src/testResources/testDir/testFile.fasta","src/testResources/testDir2/testFile3.txt"])' - errorMessages[9] == "* -- Entry 1 - field_5: expected maximum item count: 3, found: 4 ([25,25,27,28])" - errorMessages[10] == "* -- Entry 1 - field_6: array items are not unique ([25,25])" - errorMessages[11] == "* -- Entry 2: Missing required value: field_4" - errorMessages[12] == "* -- Entry 2 - field_5: expected minimum item count: 2, found: 1 ([25])" - errorMessages[13] == "* -- Entry 3 - field_4: expected type: JSONArray, found: String (abc)" - !stdout - } - - def 'array errors samplesheet format - CSV' () { - given: - def SCRIPT_TEXT = ''' - include { fromSamplesheet } from 'plugin/nf-validation' - - params.input = 'src/testResources/correct.csv' - - workflow { - Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json").view() - } - ''' - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.startsWith('[[') ? it : null } - - then: - def error = thrown(SchemaValidationException) - def errorMessages = error.message.readLines() - errorMessages[0] == 'Using "type": "array" in schema with a ".csv" samplesheet is not supported' - !stdout - } - - def 'array errors samplesheet format - TSV' () { - given: - def SCRIPT_TEXT = ''' - include { fromSamplesheet } from 'plugin/nf-validation' - - params.input = 'src/testResources/correct.tsv' - - workflow { - Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_samplesheet_converter_arrays.json").view() - } - ''' - - when: - dsl_eval(SCRIPT_TEXT) - def stdout = capture - .toString() - .readLines() - .findResults {it.startsWith('[[') ? it : null } - - then: - def error = thrown(SchemaValidationException) - def errorMessages = error.message.readLines() - errorMessages[0] == 'Using "type": "array" in schema with a ".tsv" samplesheet is not supported' - !stdout + stdout.contains("[[array_meta:[]], [], [], [], [string1, string2], [25, 26], [25, 26.5], [], [1, 2, 3], [false, true, false], [${getRootString()}/src/testResources/testDir/testFile.txt], [[${getRootString()}/src/testResources/testDir/testFile.txt], [${getRootString()}/src/testResources/testDir/testFile.txt, ${getRootString()}/src/testResources/testDir2/testFile2.txt]]]" as String) } - def 'no header - CSV' () { given: def SCRIPT_TEXT = ''' @@ -352,12 +261,11 @@ class SamplesheetConverterTest extends Dsl2Spec{ def stdout = capture .toString() .readLines() - .findResults {it.startsWith('[') ? it : null } then: noExceptionThrown() - stdout.contains("[test_1]") - stdout.contains("[test_2]") + stdout.contains("test_1") + stdout.contains("test_2") } def 'no header - YAML' () { @@ -377,12 +285,11 @@ class SamplesheetConverterTest extends Dsl2Spec{ def stdout = capture .toString() .readLines() - .findResults {it.startsWith('[') ? it : null } then: noExceptionThrown() - stdout.contains("[test_1]") - stdout.contains("[test_2]") + stdout.contains("test_1") + stdout.contains("test_2") } def 'no header - JSON' () { @@ -402,12 +309,11 @@ class SamplesheetConverterTest extends Dsl2Spec{ def stdout = capture .toString() .readLines() - .findResults {it.startsWith('[') ? it : null } then: noExceptionThrown() - stdout.contains("[test_1]") - stdout.contains("[test_2]") + stdout.contains("test_1") + stdout.contains("test_2") } def 'extra field' () { @@ -427,10 +333,14 @@ class SamplesheetConverterTest extends Dsl2Spec{ def stdout = capture .toString() .readLines() + .collect { + it.split("nextflow.validation.SamplesheetConverter - ")[-1] + } then: noExceptionThrown() - stdout.contains("\tThe samplesheet contains following unchecked field(s): [extraField]") + stdout.contains("Found the following unidentified headers in src/testResources/extraFields.csv:") + stdout.contains("\t- extraField") stdout.contains("[[string1:fullField, string2:fullField, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, ${getRootString()}/src/testResources/test.txt, ${getRootString()}/src/testResources/testDir, [], unique1, 1, itDoesExist]" as String) stdout.contains("[[string1:value, string2:value, integer1:0, integer2:0, boolean1:true, boolean2:true], string1, 25, false, [], [], [], [], [], itDoesExist]") stdout.contains("[[string1:dependentRequired, string2:dependentRequired, integer1:10, integer2:10, boolean1:true, boolean2:true], string1, 25, false, [], [], [], unique2, 1, itDoesExist]") diff --git a/plugins/nf-validation/src/testResources/no_header_schema.json b/plugins/nf-validation/src/testResources/no_header_schema.json index 83d1281..ccfd576 100644 --- a/plugins/nf-validation/src/testResources/no_header_schema.json +++ b/plugins/nf-validation/src/testResources/no_header_schema.json @@ -3,12 +3,7 @@ "description": "Schema for the file provided with params.input", "type": "array", "items": { - "type": "object", - "properties": { - "": { - "type": "string" - } - } + "type": "string" } } diff --git a/plugins/nf-validation/src/testResources/schema_input_with_arrays.json b/plugins/nf-validation/src/testResources/schema_input_with_arrays.json index aa7ac61..7e4dcfa 100644 --- a/plugins/nf-validation/src/testResources/schema_input_with_arrays.json +++ b/plugins/nf-validation/src/testResources/schema_input_with_arrays.json @@ -89,6 +89,7 @@ "items": { "type": "array", "items": { + "type": "string", "format": "file-path", "pattern": "^.*\\.txt$", "exists": true From 84faee2646f1b261ce6f78df989572ffba476af7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 13 Feb 2024 14:13:05 +0100 Subject: [PATCH 66/96] add deeply nested samplesheet tests --- .../SamplesheetConverterTest.groovy | 48 +++++++++++ .../src/testResources/deeply_nested.json | 34 ++++++++ .../src/testResources/deeply_nested.yaml | 25 ++++++ ...schema_with_deeply_nested_samplesheet.json | 28 +++++++ .../samplesheet_schema_deeply_nested.json | 81 +++++++++++++++++++ 5 files changed, 216 insertions(+) create mode 100644 plugins/nf-validation/src/testResources/deeply_nested.json create mode 100644 plugins/nf-validation/src/testResources/deeply_nested.yaml create mode 100644 plugins/nf-validation/src/testResources/nextflow_schema_with_deeply_nested_samplesheet.json create mode 100644 plugins/nf-validation/src/testResources/samplesheet_schema_deeply_nested.json diff --git a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy index 93d3c29..1627370 100644 --- a/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy +++ b/plugins/nf-validation/src/test/nextflow/validation/SamplesheetConverterTest.groovy @@ -370,4 +370,52 @@ class SamplesheetConverterTest extends Dsl2Spec{ noExceptionThrown() stdout.contains("[test1, test2]") } + + def 'deeply nested samplesheet - YAML' () { + given: + def SCRIPT_TEXT = ''' + include { fromSamplesheet } from 'plugin/nf-validation' + + params.input = 'src/testResources/deeply_nested.yaml' + + workflow { + Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_deeply_nested_samplesheet.json").view() + } + ''' + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.startsWith('[') ? it : null } + + then: + noExceptionThrown() + stdout.contains("[[mapMeta:this is in a map, arrayMeta:[metaString45, metaString478], otherArrayMeta:[metaString45, metaString478], meta:metaValue, metaMap:[entry1:entry1String, entry2:12.56]], [[string1, string2], string3, 1, 1, ${getRootString()}/file1.txt], [string4, string5, string6], [[string7, string8], [string9, string10]], test]" as String) + } + + def 'deeply nested samplesheet - JSON' () { + given: + def SCRIPT_TEXT = ''' + include { fromSamplesheet } from 'plugin/nf-validation' + + params.input = 'src/testResources/deeply_nested.json' + + workflow { + Channel.fromSamplesheet("input", parameters_schema:"src/testResources/nextflow_schema_with_deeply_nested_samplesheet.json").view() + } + ''' + + when: + dsl_eval(SCRIPT_TEXT) + def stdout = capture + .toString() + .readLines() + .findResults {it.startsWith('[') ? it : null } + + then: + noExceptionThrown() + stdout.contains("[[mapMeta:this is in a map, arrayMeta:[metaString45, metaString478], otherArrayMeta:[metaString45, metaString478], meta:metaValue, metaMap:[entry1:entry1String, entry2:12.56]], [[string1, string2], string3, 1, 1, ${getRootString()}/file1.txt], [string4, string5, string6], [[string7, string8], [string9, string10]], test]" as String) + } } diff --git a/plugins/nf-validation/src/testResources/deeply_nested.json b/plugins/nf-validation/src/testResources/deeply_nested.json new file mode 100644 index 0000000..0b141e4 --- /dev/null +++ b/plugins/nf-validation/src/testResources/deeply_nested.json @@ -0,0 +1,34 @@ +[ + { + "map": { + "arrayTest": [ + "string1", + "string2" + ], + "stringTest": "string3", + "numberTest": 1, + "integerTest": 1, + "fileTest": "file1.txt", + "mapMeta": "this is in a map" + }, + "array": [ + "string4", + "string5", + "string6" + ], + "arrayInArray": [ + [ "string7", "string8" ], + [ "string9", "string10" ] + ], + "arrayMeta": [ + "metaString45", + "metaString478" + ], + "value": "test", + "meta": "metaValue", + "metaMap": { + "entry1": "entry1String", + "entry2": 12.56 + } + } +] \ No newline at end of file diff --git a/plugins/nf-validation/src/testResources/deeply_nested.yaml b/plugins/nf-validation/src/testResources/deeply_nested.yaml new file mode 100644 index 0000000..2d5f873 --- /dev/null +++ b/plugins/nf-validation/src/testResources/deeply_nested.yaml @@ -0,0 +1,25 @@ +- map: + arrayTest: + - string1 + - string2 + stringTest: string3 + numberTest: 1 + integerTest: 1 + fileTest: file1.txt + mapMeta: this is in a map + array: + - string4 + - string5 + - string6 + arrayInArray: [ + [ "string7", "string8" ], + [ "string9", "string10" ] + ] + arrayMeta: + - "metaString45" + - "metaString478" + value: test + meta: metaValue + metaMap: + entry1: entry1String + entry2: 12.56 \ No newline at end of file diff --git a/plugins/nf-validation/src/testResources/nextflow_schema_with_deeply_nested_samplesheet.json b/plugins/nf-validation/src/testResources/nextflow_schema_with_deeply_nested_samplesheet.json new file mode 100644 index 0000000..8377896 --- /dev/null +++ b/plugins/nf-validation/src/testResources/nextflow_schema_with_deeply_nested_samplesheet.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json", + "title": "nf-core/testpipeline pipeline parameters", + "description": "this is a test", + "type": "object", + "defs": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["input"], + "properties": { + "input": { + "type": "string", + "format": "file-path", + "mimetype": "text/csv", + "pattern": "^\\S+\\.csv$", + "schema": "src/testResources/samplesheet_schema_deeply_nested.json", + "description": "Path to comma-separated file containing information about the samples in the experiment.", + "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/testpipeline/usage#samplesheet-input).", + "fa_icon": "fas fa-file-csv" + } + } + } + } +} diff --git a/plugins/nf-validation/src/testResources/samplesheet_schema_deeply_nested.json b/plugins/nf-validation/src/testResources/samplesheet_schema_deeply_nested.json new file mode 100644 index 0000000..89d2496 --- /dev/null +++ b/plugins/nf-validation/src/testResources/samplesheet_schema_deeply_nested.json @@ -0,0 +1,81 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/test/test/master/assets/schema_input.json", + "title": "Test schema for samplesheets", + "description": "Schema for the file provided with params.input", + "type": "array", + "items": { + "type": "object", + "properties": { + "map": { + "type": "object", + "properties": { + "arrayTest": { + "type": "array", + "items": { + "type": "string" + } + }, + "stringTest": { + "type": "string" + }, + "numberTest": { + "type": "number" + }, + "integerTest": { + "type": "integer" + }, + "fileTest": { + "type": "string", + "format": "file-path" + }, + "mapMeta": { + "type": "string", + "meta": "mapMeta" + } + } + }, + "array": { + "type": "array", + "items": { + "type": "string" + } + }, + "arrayInArray": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "arrayMeta": { + "type": "array", + "meta": ["arrayMeta", "otherArrayMeta"], + "items": { + "type": "string" + } + }, + "value": { + "type": "string" + }, + "meta": { + "type": "string", + "meta": "meta" + }, + "metaMap": { + "type": "object", + "meta": "metaMap", + "properties": { + "entry1": { + "type": "string" + }, + "entry2": { + "type": "number" + } + } + } + } + } +} From 825219cc480e6423b1bd0b1732a0f224f8664a21 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 13 Feb 2024 14:32:13 +0100 Subject: [PATCH 67/96] update the error message for fromSamplesheet --- .../src/main/nextflow/validation/SchemaValidator.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index da5ab9b..6a45153 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -176,9 +176,12 @@ class SchemaValidator extends PluginExtensionPoint { log.error( """ Following error has been found during samplesheet conversion: ${e} + ${e.getStackTrace().join("\n\t")} Please run validateParameters() first before trying to convert a samplesheet to a channel. Reference: https://nextflow-io.github.io/nf-validation/parameters/validation/ + +Also make sure that the same schema is used for validation and conversion of the samplesheet """ as String ) } From 77f3128c01c7f3541053e91f2044d54a7bd76082 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Feb 2024 13:58:26 +0100 Subject: [PATCH 68/96] revert last commit + add clear error --- .../validation/SchemaValidator.groovy | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 6a45153..09d3c70 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -142,35 +142,51 @@ class SchemaValidator extends PluginExtensionPoint { // Set defaults for optional inputs def String schemaFilename = options?.containsKey('parameters_schema') ? options.parameters_schema as String : 'nextflow_schema.json' - def Boolean header = options?.containsKey("header") ? options.header as Boolean : true - - def baseDir = session.baseDir.toString() + def String baseDir = session.baseDir.toString() + // Get the samplesheet schema from the parameters schema def slurper = new JsonSlurper() def Map parsed = (Map) slurper.parse( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) def Map samplesheetValue = (Map) findDeep(parsed, samplesheetParam) def Path samplesheetFile = params[samplesheetParam] as Path - if (samplesheetFile == null) { - log.error "Parameter '--$samplesheetParam' was not provided. Unable to create a channel from it." + + // Some safeguard to make sure the channel factory runs correctly + if (samplesheetValue == null) { + log.error """ +Parameter '--$samplesheetParam' was not found in the schema ($schemaFilename). +Unable to create a channel from it. + +Please make sure you correctly specified the inputs to `.fromSamplesheet`: + +-------------------------------------------------------------------------------------- +Channel.fromSamplesheet("input") +-------------------------------------------------------------------------------------- + +This would create a channel from params.input using the schema specified in the parameters JSON schema for this parameter. + +Alternatively if you used another parameters schema than the default (\${projectDir}/nextflow_schema.json), make sure you specified the new schema like this: + +-------------------------------------------------------------------------------------- +Channel.fromSamplesheet("input", parameters_schema:"path/to/other/json/schema.json") +-------------------------------------------------------------------------------------- +""" throw new SchemaValidationException("", []) } - def Path schemaFile = null - if (samplesheetValue == null) { - log.error "Parameter '--$samplesheetParam' was not found in the schema ($schemaFilename). Unable to create a channel from it." + else if (samplesheetFile == null) { + log.error "Parameter '--$samplesheetParam' was not provided. Unable to create a channel from it." throw new SchemaValidationException("", []) } - else if (samplesheetValue.containsKey('schema')) { - schemaFile = Path.of(Utils.getSchemaPath(baseDir, samplesheetValue['schema'].toString())) - } else { + else if (!samplesheetValue.containsKey('schema')) { log.error "Parameter '--$samplesheetParam' does not contain a schema in the parameter schema ($schemaFilename). Unable to create a channel from it." throw new SchemaValidationException("", []) } - + // Convert to channel final channel = CH.create() def List arrayChannel = [] try { - SamplesheetConverter converter = new SamplesheetConverter(samplesheetFile, schemaFile) + def Path schemaFile = Path.of(Utils.getSchemaPath(baseDir, samplesheetValue['schema'].toString())) + def SamplesheetConverter converter = new SamplesheetConverter(samplesheetFile, schemaFile) arrayChannel = converter.convertToList() } catch (Exception e) { log.error( From d682362b19404b81b7a829f67d7ca5bcc8d78e28 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Feb 2024 16:47:48 +0100 Subject: [PATCH 69/96] first batch of docs fixes --- README.md | 11 +++--- docs/background.md | 6 +-- docs/migration_guide.md | 0 docs/nextflow_schema/create_schema.md | 8 ++++ .../nextflow_schema_specification.md | 38 ++++++------------- mkdocs.yml | 3 +- 6 files changed, 30 insertions(+), 36 deletions(-) create mode 100644 docs/migration_guide.md diff --git a/README.md b/README.md index f2d611e..88a6d8c 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ This [Nextflow plugin](https://www.nextflow.io/docs/latest/plugins.html#plugins) - 📖 Print usage instructions to the terminal (for use with `--help`) - ✍️ Print log output showing parameters with non-default values - ✅ Validate supplied parameters against the pipeline schema -- 📋 Validate the contents of supplied sample sheet files -- 🛠️ Create a Nextflow channel with a parsed sample sheet +- 📋 Validate the contents of supplied samplesheet files +- 🛠️ Create a Nextflow channel with a parsed samplesheet -Supported sample sheet formats are CSV, TSV and YAML (simple). +Supported samplesheet formats are CSV, TSV, JSON and YAML. ## Quick Start @@ -31,7 +31,7 @@ This is all that is needed - Nextflow will automatically fetch the plugin code a > [!NOTE] > The snippet above will always try to install the latest version, good to make sure > that the latest bug fixes are included! However, this can cause difficulties if running -> offline. You can pin a specific release using the syntax `nf-validation@0.3.2` +> offline. You can pin a specific release using the syntax `nf-validation@2.0.0` You can now include the plugin helper functions into your Nextflow pipeline: @@ -58,7 +58,7 @@ ch_input = Channel.fromSamplesheet("input") ## Dependencies - Java 11 or later -- +- ## Slack channel @@ -75,3 +75,4 @@ We would like to thank the key contributors who include (but are not limited to) - Nicolas Vannieuwkerke ([@nvnieuwk](https://github.com/nvnieuwk)) - Kevin Menden ([@KevinMenden](https://github.com/KevinMenden)) - Phil Ewels ([@ewels](https://github.com/ewels)) +- Arthur ([@awgymer](https://github.com/awgymer)) diff --git a/docs/background.md b/docs/background.md index 0691f55..0714a14 100644 --- a/docs/background.md +++ b/docs/background.md @@ -10,8 +10,8 @@ hide: The [Nextflow](https://nextflow.io/) workflow manager is a powerful tool for scientific workflows. In order for end users to launch a given workflow with different input data and varying settings, pipelines are developed using a special variable type called parameters (`params`). Defaults are hardcoded into scripts and config files but can be overwritten by user config files and command-line flags (see the [Nextflow docs](https://nextflow.io/docs/latest/config.html)). -In addition to config params, a common best-practice for pipelines is to use a "sample sheet" file containing required input information. For example: a sample identifier, filenames and other sample-level metadata. +In addition to config params, a common best-practice for pipelines is to use a "samplesheet" file containing required input information. For example: a sample identifier, filenames and other sample-level metadata. -Nextflow itself does not provide functionality to validate config parameters or parsed sample sheets. To bridge this gap, we developed code within the [nf-core community](https://nf-co.re/) to allow pipelines to work with a standard `nextflow_schema.json` file, written using the [JSON Schema](https://json-schema.org/) format. The file allows strict typing of parameter variables and inclusion of validation rules. +Nextflow itself does not provide functionality to validate config parameters or parsed samplesheets. To bridge this gap, we developed code within the [nf-core community](https://nf-co.re/) to allow pipelines to work with a standard `nextflow_schema.json` file, written using the [JSON Schema](https://json-schema.org/) format. The file allows strict typing of parameter variables and inclusion of validation rules. -The nf-validation plugin moves this code out of the nf-core template into a stand-alone package, to make it easier to use for the wider Nextflow community. It also incorporates a number of new features, such as native Groovy sample sheet validation. +The nf-validation plugin moves this code out of the nf-core template into a stand-alone package, to make it easier to use for the wider Nextflow community. It also incorporates a number of new features, such as native Groovy samplesheet validation. diff --git a/docs/migration_guide.md b/docs/migration_guide.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/nextflow_schema/create_schema.md b/docs/nextflow_schema/create_schema.md index 44a311f..f869e03 100644 --- a/docs/nextflow_schema/create_schema.md +++ b/docs/nextflow_schema/create_schema.md @@ -46,6 +46,14 @@ go to the pipeline root and run the following: nf-core schema build ``` +!!! warning + + The current version of `nf-core` tools (v2.12.1) does not support the new schema draft used in `nf-validation`. Running this command after building the schema will convert the schema to the right draft: + + ```bash + sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.org\/draft\/2020-12\/schema/g' -e 's/definitions/defs/g' nextflow_schema.json + ``` + The tool will run the `nextflow config` command to extract your pipeline's configuration and compare the output to your `nextflow_schema.json` file (if it exists). It will prompt you to update the schema file with any changes, then it will ask if you diff --git a/docs/nextflow_schema/nextflow_schema_specification.md b/docs/nextflow_schema/nextflow_schema_specification.md index 07a90aa..7252b74 100644 --- a/docs/nextflow_schema/nextflow_schema_specification.md +++ b/docs/nextflow_schema/nextflow_schema_specification.md @@ -36,7 +36,7 @@ JSON schema can group variables together in an `object`, but then the validation In reality, we have a very long "flat" list of parameters, all at the top level of `params.foo`. In order to give some structure to log outputs, documentation and so on, we group parameters into `defs`. -Each `definition` is an object with a title, description and so on. +Each `def` is an object with a title, description and so on. However, as they are under `defs` scope they are effectively ignored by the validation and so their nested nature is not a problem. We then bring the contents of each definition object back to the "flat" top level for validation using a series of `allOf` statements at the end of the schema, which reference the specific definition keys. @@ -115,8 +115,7 @@ Any parameters that _must_ be specified should be set as `required` in the schem !!! tip - Make sure you do not set a default value for the parameter, as then it will have - a value even if not supplied by the pipeline user and the required property will have no effect. + Make sure you do set `null` as a default value for the parameter, otherwise it will have a value even if not supplied by the pipeline user and the required property will have no effect. This is not done with a property key like other things described below, but rather by naming the parameter in the `required` array in the definition object / top-level object. @@ -164,13 +163,13 @@ Variable type, taken from the [JSON schema keyword vocabulary](https://json-sche - `number` (float) - `integer` - `boolean` (true / false) +- `object` (currently only supported for file validation, see [Nested paramters](#nested-parameters)) +- `array` (currently only supported for file validation, see [Nested paramters](#nested-parameters)) Validation checks that the supplied parameter matches the expected type, and will fail with an error if not. -These JSON schema types are _not_ supported (see [Nested paramters](#nested-parameters)): +This JSON schema type is _not_ supported: -- `object` -- `array` - `null` ### `default` @@ -223,7 +222,7 @@ If validation fails, this `errorMessage` is printed instead, and the raw JSON sc For example, instead of printing: ``` -ERROR ~ * --input: string [samples.yml] does not match pattern ^\S+\.csv$ (samples.yml) +* --input (samples.yml): "samples.yml" does not match regular expression [^\S+\.csv$] ``` We can set @@ -239,7 +238,7 @@ We can set and get: ``` -ERROR ~ * --input: File name must end in '.csv' cannot contain spaces (samples.yml) +* --input (samples.yml): File name must end in '.csv' cannot contain spaces ``` ### `enum` @@ -325,11 +324,6 @@ Formats can be used to give additional validation checks against `string` values The `format` key is a [standard JSON schema key](https://json-schema.org/understanding-json-schema/reference/string.html#format), however we primarily use it for validating file / directory path operations with non-standard schema values. -!!! note - - In addition to _validating_ the strings as the provided format type, nf-validation also _coerces_ the parameter variable type. - That is: if the schema defines `params.input` as a `file-path`, nf-validation will convert the parameter from a `String` into a `Nextflow.File`. - Example usage is as follows: ```json @@ -342,7 +336,7 @@ Example usage is as follows: The available `format` types are below: `file-path` -: States that the provided value is a file. Does not check its existence, but it does check that the path is not a directory. +: States that the provided value is a file. Does not check its existence, but it does check if the path is not a directory. `directory-path` : States that the provided value is a directory. Does not check its existence, but if it exists, it does check that the path is not a file. @@ -351,11 +345,11 @@ The available `format` types are below: : States that the provided value is a path (file or directory). Does not check its existence. `file-path-pattern` -: States that the provided value is a globbing pattern that will be used to fetch files. Checks that the pattern is valid and that at least one file is found. +: States that the provided value is a glob pattern that will be used to fetch files. Checks that the pattern is valid and that at least one file is found. ### `exists` -When a format is specified for a value, you can provide the key `exists` set to true in order to validate that the provided path exists. +When a format is specified for a value, you can provide the key `exists` set to true in order to validate that the provided path exists. Set this to `false` to validate that the path does not exist. Example usage is as follows: @@ -367,18 +361,9 @@ Example usage is as follows: } ``` -!!! note - - If `exists` is set to `false`, this validation is ignored. Does not check if the path exists. - -!!! note - - If the parameter is set to `null`, `false` or an empty string, this validation is ignored. It does not check if the path exists. - !!! note If the parameter is an S3 URL path, this validation is ignored. - Use `--validationS3PathCheck` or set `params.validationS3PathCheck = true` to validate them. ### `mimetype` @@ -404,8 +389,7 @@ Should only be set when `format` is `file-path`. !!! tip - Setting this field is key to working with sample sheet validation and channel generation, - as described in the next section of the nf-validation docs. + Setting this field is key to working with samplesheet validation and channel generation, as described in the next section of the nf-validation docs. These schema files are typically stored in the pipeline `assets` directory, but can be anywhere. diff --git a/mkdocs.yml b/mkdocs.yml index 98ec5a2..ae6a4a7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,6 +8,7 @@ nav: - Home: - index.md - background.md + - migration_guide.md - Schema: - nextflow_schema/index.md - nextflow_schema/create_schema.md @@ -19,7 +20,7 @@ nav: - parameters/validation.md - parameters/help_text.md - parameters/summary_log.md - - Sample sheets: + - Samplesheets: - samplesheets/validate_sample_sheet.md - samplesheets/fromSamplesheet.md - samplesheets/examples.md From dcdb2671854386e2a2dc57f80bc60ef14f378f1a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 10:16:58 +0100 Subject: [PATCH 70/96] shorten the fromsamplesheet error a bit --- .../src/main/nextflow/validation/SchemaValidator.groovy | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 09d3c70..0fcb25b 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -163,12 +163,6 @@ Channel.fromSamplesheet("input") -------------------------------------------------------------------------------------- This would create a channel from params.input using the schema specified in the parameters JSON schema for this parameter. - -Alternatively if you used another parameters schema than the default (\${projectDir}/nextflow_schema.json), make sure you specified the new schema like this: - --------------------------------------------------------------------------------------- -Channel.fromSamplesheet("input", parameters_schema:"path/to/other/json/schema.json") --------------------------------------------------------------------------------------- """ throw new SchemaValidationException("", []) } From a371036ec89853ad6e8d78c36a818679242febe8 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 10:56:54 +0100 Subject: [PATCH 71/96] fix for null meta attributes --- .../src/main/nextflow/validation/SamplesheetConverter.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index e010201..c4e58e4 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -111,7 +111,7 @@ class SamplesheetConverter { // Add the value to the meta map if needed if (metaIds) { metaIds.each { - addMeta(["${it}":processMeta(value, schemaValues as LinkedHashMap, prefix)]) + meta["${it}"] = processMeta(value, schemaValues as LinkedHashMap, prefix) } } // return the correctly casted value From d3c199c63fcd2a23fb32439de0a350b700c58dfa Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 12:46:51 +0100 Subject: [PATCH 72/96] second batch of docs fixes --- README.md | 6 +- docs/background.md | 4 +- .../nextflow_schema_specification.md | 40 +++++- .../sample_sheet_schema_examples.md | 14 +-- .../sample_sheet_schema_specification.md | 119 ++---------------- docs/parameters/summary_log.md | 2 +- docs/parameters/validation.md | 4 +- docs/samplesheets/examples.md | 14 +-- docs/samplesheets/fromSamplesheet.md | 20 +-- docs/samplesheets/validate_sample_sheet.md | 4 +- .../pipeline/nextflow.config | 2 +- .../pipeline/nextflow.config | 2 +- .../pipeline/nextflow.config | 2 +- examples/paramsHelp/pipeline/nextflow.config | 2 +- .../paramsSummaryLog/pipeline/nextflow.config | 2 +- .../paramsSummaryMap/pipeline/nextflow.config | 2 +- examples/validateParameters/log.txt | 6 +- .../pipeline/nextflow.config | 2 +- .../pipeline/nextflow_schema.json | 1 + .../pipeline/nextflow.config | 2 +- .../pipeline/nextflow.config | 2 +- 21 files changed, 91 insertions(+), 161 deletions(-) diff --git a/README.md b/README.md index 88a6d8c..6fb3aad 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ This [Nextflow plugin](https://www.nextflow.io/docs/latest/plugins.html#plugins) - 📖 Print usage instructions to the terminal (for use with `--help`) - ✍️ Print log output showing parameters with non-default values - ✅ Validate supplied parameters against the pipeline schema -- 📋 Validate the contents of supplied samplesheet files -- 🛠️ Create a Nextflow channel with a parsed samplesheet +- 📋 Validate the contents of supplied sample sheet files +- 🛠️ Create a Nextflow channel with a parsed sample sheet -Supported samplesheet formats are CSV, TSV, JSON and YAML. +Supported sample sheet formats are CSV, TSV, JSON and YAML. ## Quick Start diff --git a/docs/background.md b/docs/background.md index 0714a14..6b51571 100644 --- a/docs/background.md +++ b/docs/background.md @@ -10,8 +10,8 @@ hide: The [Nextflow](https://nextflow.io/) workflow manager is a powerful tool for scientific workflows. In order for end users to launch a given workflow with different input data and varying settings, pipelines are developed using a special variable type called parameters (`params`). Defaults are hardcoded into scripts and config files but can be overwritten by user config files and command-line flags (see the [Nextflow docs](https://nextflow.io/docs/latest/config.html)). -In addition to config params, a common best-practice for pipelines is to use a "samplesheet" file containing required input information. For example: a sample identifier, filenames and other sample-level metadata. +In addition to config params, a common best-practice for pipelines is to use a "sample sheet" file containing required input information. For example: a sample identifier, filenames and other sample-level metadata. Nextflow itself does not provide functionality to validate config parameters or parsed samplesheets. To bridge this gap, we developed code within the [nf-core community](https://nf-co.re/) to allow pipelines to work with a standard `nextflow_schema.json` file, written using the [JSON Schema](https://json-schema.org/) format. The file allows strict typing of parameter variables and inclusion of validation rules. -The nf-validation plugin moves this code out of the nf-core template into a stand-alone package, to make it easier to use for the wider Nextflow community. It also incorporates a number of new features, such as native Groovy samplesheet validation. +The nf-validation plugin moves this code out of the nf-core template into a stand-alone package, to make it easier to use for the wider Nextflow community. It also incorporates a number of new features, such as native Groovy sample sheet validation. diff --git a/docs/nextflow_schema/nextflow_schema_specification.md b/docs/nextflow_schema/nextflow_schema_specification.md index 7252b74..4770efb 100644 --- a/docs/nextflow_schema/nextflow_schema_specification.md +++ b/docs/nextflow_schema/nextflow_schema_specification.md @@ -389,7 +389,7 @@ Should only be set when `format` is `file-path`. !!! tip - Setting this field is key to working with samplesheet validation and channel generation, as described in the next section of the nf-validation docs. + Setting this field is key to working with sample sheet validation and channel generation, as described in the next section of the nf-validation docs. These schema files are typically stored in the pipeline `assets` directory, but can be anywhere. @@ -432,3 +432,41 @@ Specify a minimum / maximum value for an integer or float number length with `mi The JSON schema doc also mention `exclusiveMinimum`, `exclusiveMaximum` and `multipleOf` keys. Because nf-validation uses stock JSON schema validation libraries, these _should_ work for validating keys. However, they are not officially supported within the Nextflow schema ecosystem and so some interfaces may not recognise them. + +## Array-specific keys + +### `uniqueItems` + +All items in the array should be unique. + +- See the [JSON schema docs](https://json-schema.org/understanding-json-schema/reference/array#uniqueItems) + for details. + +```json +{ + "type": "array", + "uniqueItems": true +} +``` + +### `uniqueEntries` + +!!! example "Non-standard key" + +The combination of all values in the given keys should be unique. For this key to work you need to make sure the array items are of type `object` and contains the keys in the `uniqueEntries` list. + +```json +{ + "type": "array", + "items": { + "type": "object", + "uniqueEntries": ["foo", "bar"], + "properties": { + "foo": { "type": "string" }, + "bar": { "type": "string" } + } + } +} +``` + +This schema tells `nf-validation` that the combination of `foo` and `bar` should be unique across all objects in the array. diff --git a/docs/nextflow_schema/sample_sheet_schema_examples.md b/docs/nextflow_schema/sample_sheet_schema_examples.md index 14a4f30..a28ce44 100644 --- a/docs/nextflow_schema/sample_sheet_schema_examples.md +++ b/docs/nextflow_schema/sample_sheet_schema_examples.md @@ -42,17 +42,9 @@ You can see this, used for validating sample sheets with `--input` here: [`asset }, "fastq_2": { "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", - "anyOf": [ - { - "type": "string", - "pattern": "^\\S+\\.f(ast)?q\\.gz$", - "format": "file-path" - }, - { - "type": "string", - "maxLength": 0 - } - ] + "type": "string", + "pattern": "^\\S+\\.f(ast)?q\\.gz$", + "format": "file-path" }, "strandedness": { "type": "string", diff --git a/docs/nextflow_schema/sample_sheet_schema_specification.md b/docs/nextflow_schema/sample_sheet_schema_specification.md index bb7dcd8..7865b8e 100644 --- a/docs/nextflow_schema/sample_sheet_schema_specification.md +++ b/docs/nextflow_schema/sample_sheet_schema_specification.md @@ -2,7 +2,7 @@ description: Schema specification for sample sheet validation --- -# Sample sheet schema specification +# Sample sheetschema specification Sample sheet schema files are used by the nf-validation plugin for validation of sample sheet contents and type conversion / channel generation. @@ -18,17 +18,13 @@ Validation by the plugin works by parsing the supplied file contents into a groo then passing this to the JSON schema validation library. As such, the structure of the schema must match the structure of the parsed file. -Typically, sample sheets are CSV files, with fields represented as columns and samples as rows. TSV and simple unnested YAML files are also supported by the plugin. - -!!! warning - - Nested YAML files can be validated with the `validateParameters()` function, but cannot be converted to a channel with `.fromSamplesheet()`. +Typically, samplesheets are CSV files, with fields represented as columns and samples as rows. TSV, JSON and YAML samplesheets are also supported by this plugin In this case, the parsed object will be an `array` (see [JSON schema docs](https://json-schema.org/understanding-json-schema/reference/array.html#items)). The array type is associated with an `items` key which in our case contains a single `object`. The object has `properties`, where the keys must match the headers of the CSV file. -So, for CSV sample sheets, the top-level schema should look something like this: +So, for CSV samplesheets, the top-level schema should look something like this: ```json { @@ -44,7 +40,7 @@ So, for CSV sample sheets, the top-level schema should look something like this: } ``` -If your sample sheet has a different format (for example, a simple YAML file), +If your sample sheet has a different format (for example, a nested YAML file), you will need to build your schema to match the parsed structure. ## Properties @@ -75,10 +71,8 @@ Please refer to the [Nextflow schema specification](../nextflow_schema/nextflow_ !!! tip Sample sheets are commonly used to define input file paths. - Be sure to set `"type": "string"` and `"format": "file-path"` for these properties, - so that nf-validation correctly returns this sample sheet field as a `Nextflow.file` object. - -When using the `file-path-pattern` format for a globbing pattern, a list will be created with all files found by the globbing pattern. See [here](../nextflow_schema/nextflow_schema_specification.md#file-path-pattern) for more information. + Be sure to set `"type": "string"`, `exists: true` and `"format": "file-path"`, `"schema":"path/to/samplesheet/schema.json"` for these properties, + so that `fromSamplesheet` will not result in any errors. ## Sample sheet keys @@ -87,111 +81,18 @@ These exist in addition to those described in the [Nextflow schema specification ### `meta` -Type: `List` +Type: `List` or `String` -The current field will be considered a meta value when this parameter is present. This parameter should contain a list of the meta fields to assign this value to. The default is no meta for each field. +The current field will be considered a meta value when this parameter is present. This parameter should contain a list of the meta fields or a string stating a single meta field to assign this value to. The default is no meta for each field. For example: ```json { - "meta": ["id", "sample"] + "meta": "id" } ``` -will convert the `field` value to a meta value, resulting in the channel `[[id:value, sample:value]...]` +will convert the `field` value to a meta value, resulting in the channel `[[id:value]...]` See [here](https://github.com/nextflow-io/nf-validation/blob/ce3aef60e5103ea4798375fe6c59bae41b7d2a25/plugins/nf-validation/src/testResources/schema_input.json#L10-L25) for an example in the sample sheet. -### `unique` - -Type: `Boolean` or `List` - -Whether or not the field should contain a unique value over the entire sample sheet. - -Default: `false` - -- Can be `true`, in which case the value for this field should be unique for all samples in the sample sheet. -- Can be supplied with a list of field names, containing _other field names_ that should be unique _in combination with_ the current field. - -!!! example - - Consider the following example: - - ```json - "properties": { - "field1": { "unique": true }, - "field2": { "unique": ["field1"] } - } - ``` - - `field1` needs to be unique in this example. `field2` needs to be unique in combination with `field1`. So for a sample sheet like this: - - ```csv linenums="1" - field1,field2 - value1,value2 - value1,value3 - value1,value2 - ``` - - ..both checks will fail. - - * `field1` isn't unique since `value1` has been found more than once. - * `field2` isn't unique in combination with `field1` because the `value1,value2` combination has been found more than once. - - See [`schema_input.json#L48-L55`](https://github.com/nextflow-io/nf-validation/blob/ce3aef60e5103ea4798375fe6c59bae41b7d2a25/plugins/nf-validation/src/testResources/schema_input.json#L48-L55) - for an example in one of the plugin test-fixture sample sheets. - -### `deprecated` - -Type: `Boolean` - -A boolean variable stating that the field is deprecated and will be removed in the nearby future. This will throw a warning to the user that the current field is deprecated. The default value is `false`. - -Example: - -```json -"field": { - "deprecated": true -} -``` - -will show a warning stating that the use of `field` is deprecated: - -```console -The 'field' field is deprecated and -will no longer be used in the future. -Please check the official documentation -of the pipeline for more information. -``` - -### `dependentRequired` - -Type: `List` - -- See [JSON Schema docs](https://json-schema.org/understanding-json-schema/reference/conditionals.html#dependentrequired) - -A list containing names of other fields. The validator will check if these fields are filled in and throw an error if they aren't, but only when the field `dependentRequired` belongs to is filled in. - -!!! example - - ```json - "field1": { - "dependentRequired": ["field2"] - }, - "field2": {} - ``` - - will check if `field2` is given when `field1` has a value. So for example: - - ```csv linenums="1" - field1,field2 - value1,value2 - value1, - ,value2 - ``` - - - [x] The first row will pass the check because both fields are set. - - [ ] The second row will fail because `field1` is set, but `field2` isn't and `field1` is dependent on `field2`. - - [x] The third row will pass the check because `field1` isn't set. - - See [here](https://github.com/nextflow-io/nf-validation/blob/ce3aef60e5103ea4798375fe6c59bae41b7d2a25/plugins/nf-validation/src/testResources/schema_input.json#L10-L25) for an example in the sample sheet. diff --git a/docs/parameters/summary_log.md b/docs/parameters/summary_log.md index 98cc213..d9e91a2 100644 --- a/docs/parameters/summary_log.md +++ b/docs/parameters/summary_log.md @@ -10,7 +10,7 @@ This function returns a string that can be logged to the terminal, summarizing t !!! note - The summary prioritizes displaying only the parameters that are **different** the default schema values. + The summary prioritizes displaying only the parameters that are **different** than the default schema values. Parameters which don't have a default in the JSON Schema and which have a value of `null`, `""`, `false` or `'false'` won't be returned in the map. This is to streamline the extensive parameter lists often associated with pipelines, and highlight the customized elements. This feature is essential for users to verify their configurations, like checking for typos or confirming proper resolution, diff --git a/docs/parameters/validation.md b/docs/parameters/validation.md index 25a275f..1627ce0 100644 --- a/docs/parameters/validation.md +++ b/docs/parameters/validation.md @@ -129,7 +129,7 @@ Sometimes, a parameter that you want to set may not be described in the pipeline Maybe it's something you're using in your Nextflow configuration setup for your compute environment, or it's a complex parameter that cannot be handled in the schema, such as [nested parameters](../nextflow_schema/nextflow_schema_specification.md#nested-parameters). -In these cases, to avoid getting warnings when that unrecognised parameter is set, +In these cases, to avoid getting warnings when an unrecognised parameter is set, you can use `--validationSchemaIgnoreParams` / `params.validationSchemaIgnoreParams`. This should be a comma-separated list of strings that correspond to parameter names. @@ -147,8 +147,6 @@ For example, providing an integer as a string will no longer fail validation. The validation does not affect the parameter variable types in your pipeline. It attempts to cast a temporary copy of the params only, during the validation step. - You can find more information about how this works in the [JSON schema validation library docs](https://github.com/everit-org/json-schema#lenient-mode). - To enable lenient validation mode, set `params.validationLenientMode`: ```bash diff --git a/docs/samplesheets/examples.md b/docs/samplesheets/examples.md index 434e830..51b3d83 100644 --- a/docs/samplesheets/examples.md +++ b/docs/samplesheets/examples.md @@ -12,12 +12,12 @@ Understanding channel structure and manipulation is critical for getting the mos ### Glossary - A channel is the Nextflow object, referenced in the code -- An item is each thing passing through the channel, equivalent to one row in the samplesheet +- An item is each thing passing through the channel, equivalent to one row in the sample sheet - An element is each thing in the item, e.g., the meta value, fastq_1 etc. It may be a file or value ## Default mode -Each item in the channel emitted by `.fromSamplesheet()` is a flat tuple, corresponding with each row of the samplesheet. Each item will be composed of a meta value (if present) and any additional elements from columns in the samplesheet, e.g.: +Each item in the channel emitted by `.fromSamplesheet()` is a flat tuple, corresponding with each row of the sample sheet. Each item will be composed of a meta value (if present) and any additional elements from columns in the sample sheet, e.g.: ```csv sample,fastq_1,fastq_2,bed @@ -33,7 +33,7 @@ Might create a channel where each element consists of 4 items, a map value follo // Resulting in: [ [ id: "sample" ], fastq1.R1.fq.gz, fastq1.R2.fq.gz, sample1.bed] -[ [ id: "sample2" ], fastq2.R1.fq.gz, fastq2.R2.fq.gz, [] ] // A missing value from the samplesheet is an empty list +[ [ id: "sample2" ], fastq2.R1.fq.gz, fastq2.R2.fq.gz, [] ] // A missing value from the sample sheet is an empty list ``` This channel can be used as input of a process where the input declaration is: @@ -44,9 +44,9 @@ tuple val(meta), path(fastq_1), path(fastq_2), path(bed) It may be necessary to manipulate this channel to fit your process inputs. For more documentation, check out the [Nextflow operator docs](https://www.nextflow.io/docs/latest/operator.html), however here are some common use cases with `.fromSamplesheet()`. -## Using a samplesheet with no headers +## Using a sample sheet with no headers -Sometimes you only have one possible input in the pipeline samplesheet. In this case it doesn't make sense to have a header in the samplesheet. This can be done by creating a samplesheet with an empty string as input key: +Sometimes you only have one possible input in the pipeline sample sheet. In this case it doesn't make sense to have a header in the sample sheet. This can be done by creating a sample sheet with an empty string as input key: ```json { @@ -166,7 +166,7 @@ This example shows a channel which can have entries for WES or WGS data. WES dat ```nextflow // Channel with four elements - see docs for examples -params.input = "samplesheet.csv" +params.input = "sample sheet.csv" Channel.fromSamplesheet("input") .branch { meta, fastq_1, fastq_2, bed -> @@ -214,7 +214,7 @@ This example contains a channel where multiple samples can be in the same family // [[id:example2, family:family1], example2.txt] // [[id:example3, family:family2], example3.txt] -params.input = "samplesheet.csv" +params.input = "sample sheet.csv" Channel.fromSamplesheet("input") .tap { ch_raw } // Create a copy of the original channel diff --git a/docs/samplesheets/fromSamplesheet.md b/docs/samplesheets/fromSamplesheet.md index c11f187..5e42f5a 100644 --- a/docs/samplesheets/fromSamplesheet.md +++ b/docs/samplesheets/fromSamplesheet.md @@ -7,9 +7,9 @@ description: Channel factory to create a channel from a sample sheet. ## `fromSamplesheet` -This function validates and converts a samplesheet to a ready-to-use Nextflow channel. This is done using information encoded within a sample sheet schema (see the [docs](../nextflow_schema/sample_sheet_schema_specification.md)). +This function validates and converts a sample sheet to a ready-to-use Nextflow channel. This is done using information encoded within a sample sheet schema (see the [docs](../nextflow_schema/sample_sheet_schema_specification.md)). -The function has one mandatory argument: the name of the parameter which specifies the input samplesheet. The parameter specified must have the format `file-path` and include additional field `schema`: +The function has one mandatory argument: the name of the parameter which specifies the input sample sheet. The parameter specified must have the format `file-path` and include additional field `schema`: ```json hl_lines="4" { @@ -19,7 +19,7 @@ The function has one mandatory argument: the name of the parameter which specifi } ``` -The path specified in the `schema` key determines the JSON used for validation of the samplesheet. +The path specified in the `schema` key determines the JSON used for validation of the sample sheet. When using the `.fromSamplesheet` channel factory, some additional optional arguments can be used: @@ -40,7 +40,7 @@ Channel.fromSamplesheet( ## Basic example -In [this example](../../examples/fromSamplesheetBasic/), we create a simple channel from a CSV samplesheet. +In [this example](../../examples/fromSamplesheetBasic/), we create a simple channel from a CSV sample sheet. ``` --8<-- "examples/fromSamplesheetBasic/log.txt" @@ -52,10 +52,10 @@ In [this example](../../examples/fromSamplesheetBasic/), we create a simple chan --8<-- "examples/fromSamplesheetBasic/pipeline/main.nf" ``` -=== "samplesheet.csv" +=== "sample sheet.csv" ```csv - --8<-- "examples/fromSamplesheetBasic/samplesheet.csv" + --8<-- "examples/fromSamplesheetBasic/sample sheet.csv" ``` === "nextflow.config" @@ -88,10 +88,10 @@ In [this example](../../examples/fromSamplesheetBasic/), we create a simple chan --8<-- "examples/fromSamplesheetOrder/log.txt" ``` -=== "samplesheet.csv" +=== "sample sheet.csv" ```csv - --8<-- "examples/fromSamplesheetOrder/samplesheet.csv" + --8<-- "examples/fromSamplesheetOrder/sample sheet.csv" ``` === "assets/schema_input.json" @@ -139,10 +139,10 @@ This returns a channel with a meta map. --8<-- "examples/fromSamplesheetMeta/pipeline/main.nf" ``` -=== "samplesheet.csv" +=== "sample sheet.csv" ```csv - --8<-- "examples/fromSamplesheetMeta/samplesheet.csv" + --8<-- "examples/fromSamplesheetMeta/sample sheet.csv" ``` === "nextflow.config" diff --git a/docs/samplesheets/validate_sample_sheet.md b/docs/samplesheets/validate_sample_sheet.md index fdad579..8a94ff2 100644 --- a/docs/samplesheets/validate_sample_sheet.md +++ b/docs/samplesheets/validate_sample_sheet.md @@ -7,7 +7,7 @@ description: Validate the contents of a sample sheet file. When a parameter provides the `schema` field, the `validateParameters()` function will automatically parse and validate the provided file contents using this JSON schema. -It can validate CSV, TSV and simple YAML files. +It can validate CSV, TSV, JSON and YAML files. The path of the schema file must be relative to the root of the pipeline directory. See an example in the `input` field from the [example schema.json](https://raw.githubusercontent.com/nextflow-io/nf-validation/master/plugins/nf-validation/src/testResources/nextflow_schema_with_samplesheet.json#L20). @@ -26,4 +26,4 @@ See an example in the `input` field from the [example schema.json](https://raw.g } ``` -For more information about the samplesheet JSON schema refer to [samplesheet docs](../nextflow_schema/nextflow_schema_specification.md). +For more information about the sample sheet JSON schema refer to [sample sheet docs](../nextflow_schema/nextflow_schema_specification.md). diff --git a/examples/fromSamplesheetBasic/pipeline/nextflow.config b/examples/fromSamplesheetBasic/pipeline/nextflow.config index c06ab6b..efbc97a 100644 --- a/examples/fromSamplesheetBasic/pipeline/nextflow.config +++ b/examples/fromSamplesheetBasic/pipeline/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-validation@0.2.1' + id 'nf-validation@2.0.0' } params { diff --git a/examples/fromSamplesheetMeta/pipeline/nextflow.config b/examples/fromSamplesheetMeta/pipeline/nextflow.config index c06ab6b..efbc97a 100644 --- a/examples/fromSamplesheetMeta/pipeline/nextflow.config +++ b/examples/fromSamplesheetMeta/pipeline/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-validation@0.2.1' + id 'nf-validation@2.0.0' } params { diff --git a/examples/fromSamplesheetOrder/pipeline/nextflow.config b/examples/fromSamplesheetOrder/pipeline/nextflow.config index c06ab6b..efbc97a 100644 --- a/examples/fromSamplesheetOrder/pipeline/nextflow.config +++ b/examples/fromSamplesheetOrder/pipeline/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-validation@0.2.1' + id 'nf-validation@2.0.0' } params { diff --git a/examples/paramsHelp/pipeline/nextflow.config b/examples/paramsHelp/pipeline/nextflow.config index 0e1bf1f..c907af1 100644 --- a/examples/paramsHelp/pipeline/nextflow.config +++ b/examples/paramsHelp/pipeline/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-validation@0.2.1' + id 'nf-validation@2.0.0' } params { diff --git a/examples/paramsSummaryLog/pipeline/nextflow.config b/examples/paramsSummaryLog/pipeline/nextflow.config index 0e1bf1f..c907af1 100644 --- a/examples/paramsSummaryLog/pipeline/nextflow.config +++ b/examples/paramsSummaryLog/pipeline/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-validation@0.2.1' + id 'nf-validation@2.0.0' } params { diff --git a/examples/paramsSummaryMap/pipeline/nextflow.config b/examples/paramsSummaryMap/pipeline/nextflow.config index 0e1bf1f..c907af1 100644 --- a/examples/paramsSummaryMap/pipeline/nextflow.config +++ b/examples/paramsSummaryMap/pipeline/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-validation@0.2.1' + id 'nf-validation@2.0.0' } params { diff --git a/examples/validateParameters/log.txt b/examples/validateParameters/log.txt index a294ef4..65f59cf 100644 --- a/examples/validateParameters/log.txt +++ b/examples/validateParameters/log.txt @@ -1,10 +1,10 @@ N E X T F L O W ~ version 23.04.1 Launching `pipeline/main.nf` [amazing_crick] DSL2 - revision: 53bd9eac20 -ERROR ~ ERROR: Validation of pipeline parameters failed! +ERROR ~ Validation of pipeline parameters failed! -- Check '.nextflow.log' file for details The following invalid input values have been detected: -* --input: string [samplesheet.txt] does not match pattern ^\S+\.(csv|tsv|yaml|json)$ (samplesheet.txt) -* --input: the file 'samplesheet.txt' does not exist (samplesheet.txt) +* --input (samplesheet.txt): "samplesheet.txt" does not match regular expression [^\S+\.(csv|tsv|yml|yaml)$] +* --input (samplesheet.txt): the file or directory 'samplesheet.txt' does not exist diff --git a/examples/validateParameters/pipeline/nextflow.config b/examples/validateParameters/pipeline/nextflow.config index 7227d6a..da71bcc 100644 --- a/examples/validateParameters/pipeline/nextflow.config +++ b/examples/validateParameters/pipeline/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-validation@0.2.1' + id 'nf-validation@2.0.0' } params { diff --git a/examples/validateParameters/pipeline/nextflow_schema.json b/examples/validateParameters/pipeline/nextflow_schema.json index 6096ceb..c0df520 100644 --- a/examples/validateParameters/pipeline/nextflow_schema.json +++ b/examples/validateParameters/pipeline/nextflow_schema.json @@ -18,6 +18,7 @@ "mimetype": "text/csv", "schema": "assets/schema_input.json", "pattern": "^\\S+\\.(csv|tsv|yaml|json)$", + "exists": true, "description": "Path to comma-separated file containing information about the samples in the experiment.", "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row. See [usage docs](https://nf-co.re/testpipeline/usage#samplesheet-input).", "fa_icon": "fas fa-file-csv" diff --git a/examples/validationFailUnrecognisedParams/pipeline/nextflow.config b/examples/validationFailUnrecognisedParams/pipeline/nextflow.config index 9a0e564..0a1816c 100644 --- a/examples/validationFailUnrecognisedParams/pipeline/nextflow.config +++ b/examples/validationFailUnrecognisedParams/pipeline/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-validation@0.2.1' + id 'nf-validation@2.0.0' } params { diff --git a/examples/validationWarnUnrecognisedParams/pipeline/nextflow.config b/examples/validationWarnUnrecognisedParams/pipeline/nextflow.config index e30a871..d4c7f3d 100644 --- a/examples/validationWarnUnrecognisedParams/pipeline/nextflow.config +++ b/examples/validationWarnUnrecognisedParams/pipeline/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-validation@0.2.1' + id 'nf-validation@2.0.0' } params { From 8dba44ed9dd50b9a5053ecd5259ec071919a5435 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 13:08:01 +0100 Subject: [PATCH 73/96] third batch of docs fixes --- docs/samplesheets/fromSamplesheet.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/samplesheets/fromSamplesheet.md b/docs/samplesheets/fromSamplesheet.md index 5e42f5a..13ec8c3 100644 --- a/docs/samplesheets/fromSamplesheet.md +++ b/docs/samplesheets/fromSamplesheet.md @@ -21,21 +21,16 @@ The function has one mandatory argument: the name of the parameter which specifi The path specified in the `schema` key determines the JSON used for validation of the sample sheet. -When using the `.fromSamplesheet` channel factory, some additional optional arguments can be used: +When using the `.fromSamplesheet` channel factory, some one optional arguments can be used: - `parameters_schema`: File name for the pipeline parameters schema. (Default: `nextflow_schema.json`) -- `skip_duplicate_check`: Skip the checking for duplicates. Can also be skipped with the `--validationSkipDuplicateCheck` parameter. (Default: `false`) ```groovy Channel.fromSamplesheet('input') ``` ```groovy -Channel.fromSamplesheet( - 'input', - parameters_schema: 'custom_nextflow_schema.json', - skip_duplicate_check: false -) +Channel.fromSamplesheet('input', parameters_schema: 'custom_nextflow_schema.json') ``` ## Basic example From eadf4f21d29bd3ef8fa5b87c73bc252cd0c0d704 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 13:15:44 +0100 Subject: [PATCH 74/96] check for default formats when using anyOf, allOf and oneOf --- .../main/nextflow/validation/SamplesheetConverter.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy index c4e58e4..5fe8b21 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SamplesheetConverter.groovy @@ -150,6 +150,8 @@ class SamplesheetConverter { return value } + def String defaultFormat = schemaEntry.format ?: "" + // A valid path format has been found in the schema def Boolean foundStringFileFormat = false @@ -157,7 +159,7 @@ class SamplesheetConverter { def Boolean foundStringNoFileFormat = false if ((schemaEntry.type ?: "") == "string") { - if (validPathFormats.contains(schemaEntry.format ?: "")) { + if (validPathFormats.contains(schemaEntry.format ?: defaultFormat)) { foundStringFileFormat = true } else { foundStringNoFileFormat = true @@ -167,7 +169,7 @@ class SamplesheetConverter { schemaOptions.each { option -> schemaEntry[option]?.each { subSchema -> if ((subSchema["type"] ?: "" ) == "string") { - if (validPathFormats.contains(subSchema["format"] ?: "")) { + if (validPathFormats.contains(subSchema["format"] ?: defaultFormat)) { foundStringFileFormat = true } else { foundStringNoFileFormat = true From 59363d5c49c2e51c54d05da3b71e6d031f6896ad Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 14:58:03 +0100 Subject: [PATCH 75/96] wrote the migration docs --- docs/migration_guide.md | 160 +++++++++++++++++++++++++++ docs/samplesheets/examples.md | 35 +++--- docs/samplesheets/fromSamplesheet.md | 2 +- 3 files changed, 176 insertions(+), 21 deletions(-) diff --git a/docs/migration_guide.md b/docs/migration_guide.md index e69de29..3c748ae 100644 --- a/docs/migration_guide.md +++ b/docs/migration_guide.md @@ -0,0 +1,160 @@ +--- +title: Migration guide +description: Guide to migrate pipelines using nf-validation pre v2.0.0 to after v2.0.0 +hide: + - toc +--- + +# Migration guide + +This guide is intended to help you migrate your pipeline from older versions of the plugin to version 2.0.0 and later. + +## Major changes in the plugin + +Following list shows the major breaking changes introduced in version 2.0.0: + +1. The JSON schema draft has been updated from `draft-07` to `draft-2020-12`. See [JSON Schema draft 2020-12 release notes](https://json-schema.org/draft/2020-12/release-notes) and [JSON schema draft 2019-09 release notes](https://json-schema.org/draft/2019-09/release-notes) for more information. +2. The `unique` keyword for samplesheet schemas has been removed. Please use [`uniqueItems`](https://json-schema.org/understanding-json-schema/reference/array#uniqueItems) or [`uniqueEntries`](nextflow_schema/nextflow_schema_specification.md#uniqueentries) now instead. +3. The `dependentRequired` keyword now works as it's supposed to work in JSON schema. See [`dependentRequired`](https://json-schema.org/understanding-json-schema/reference/conditionals#dependentRequired) for more information + +A full list of changes can be found in the [changelog](../CHANGELOG.md). + +## Updating your pipeline + +If you aren't using any special features in your schemas, you can simply update your `nextflow_schema.json` file using the following command: + +```bash +sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.org\/draft\/2020-12\/schema/g' -e 's/definitions/defs/g' nextflow_schema.json +``` + +!!! note + Repeat this command for every JSON schema you use in your pipeline. e.g. for the default samplesheet schema: + ```bash + sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.org\/draft\/2020-12\/schema/g' -e 's/definitions/defs/g' assets/schema_input.json + ``` + +If you are using any special features in your schemas, you will need to update your schemas manually. Please refer to the [JSON Schema draft 2020-12 release notes](https://json-schema.org/draft/2020-12/release-notes) and [JSON schema draft 2019-09 release notes](https://json-schema.org/draft/2019-09/release-notes) for more information. + +However here are some guides to the more common migration patterns: + +### Updating `unique` keyword + +When you use `unique` in your schemas, you should update it to use `uniqueItems` or `uniqueEntries` instead. + +If you used the `unique:true` field, you should update it to use `uniqueItems` like this: + + +=== "Before v2.0" + ```json hl_lines="9" + { + "$schema": "http://json-schema.org/draft-07/schema", + "type": "array", + "items": { + "type": "object", + "properties": { + "sample": { + "type": "string", + "unique": true + } + } + } + } + ``` + +=== "After v2.0" + ```json hl_lines="12" + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "items": { + "type": "object", + "properties": { + "sample": { + "type": "string" + } + } + }, + "uniqueItems": true + } + ``` + + +If you used the `unique: ["field1", "field2"]` field, you should update it to use `uniqueEntries` like this: + +=== "Before v2.0" + ```json hl_lines="9" + { + "$schema": "http://json-schema.org/draft-07/schema", + "type": "array", + "items": { + "type": "object", + "properties": { + "sample": { + "type": "string", + "unique": ["sample"] + } + } + } + } + ``` + +=== "After v2.0" + ```json hl_lines="12" + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "items": { + "type": "object", + "properties": { + "sample": { + "type": "string" + } + } + }, + "uniqueEntries": ["sample"] + } + ``` + +### Updating `dependentRequired` keyword + +When you use `dependentRequired` in your schemas, you should update it like this: + +=== "Before v2.0" + ```json hl_lines="12" + { + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "fastq_1": { + "type": "string", + "format": "file-path" + }, + "fastq_2": { + "type": "string", + "format": "file-path" + "dependentRequired": ["fastq_1"] + } + } + } + ``` + +=== "After v2.0" + ```json hl_lines="14 15 16" + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "fastq_1": { + "type": "string", + "format": "file-path" + }, + "fastq_2": { + "type": "string", + "format": "file-path" + } + }, + "dependentRequired": { + "fastq_2": ["fastq_1"] + } + } + ``` \ No newline at end of file diff --git a/docs/samplesheets/examples.md b/docs/samplesheets/examples.md index 51b3d83..0e95313 100644 --- a/docs/samplesheets/examples.md +++ b/docs/samplesheets/examples.md @@ -17,7 +17,7 @@ Understanding channel structure and manipulation is critical for getting the mos ## Default mode -Each item in the channel emitted by `.fromSamplesheet()` is a flat tuple, corresponding with each row of the sample sheet. Each item will be composed of a meta value (if present) and any additional elements from columns in the sample sheet, e.g.: +Each item in the channel emitted by `.fromSamplesheet()` is a tuple, corresponding with each row of the sample sheet. Each item will be composed of a meta value (if present) and any additional elements from columns in the sample sheet, e.g.: ```csv sample,fastq_1,fastq_2,bed @@ -46,7 +46,7 @@ It may be necessary to manipulate this channel to fit your process inputs. For m ## Using a sample sheet with no headers -Sometimes you only have one possible input in the pipeline sample sheet. In this case it doesn't make sense to have a header in the sample sheet. This can be done by creating a sample sheet with an empty string as input key: +Sometimes you only have one possible input in the pipeline sample sheet. In this case it doesn't make sense to have a header in the sample sheet. This can be done by removing the `properties` section fro m the samplesheet and changing the type of the element from `object` to whatever type you want: ```json { @@ -54,12 +54,7 @@ Sometimes you only have one possible input in the pipeline sample sheet. In this "description": "Schema for the file provided with params.input", "type": "array", "items": { - "type": "object", - "properties": { - "": { - "type": "string" - } - } + "type": "string" } } ``` @@ -81,15 +76,15 @@ or this YAML file: The output of `.fromSamplesheet()` will look like this: ```bash -[test_1] -[test_2] +test_1 +test_2 ``` ## Changing the structure of channel items -Each item in the channel will be a flat tuple, but some processes will use multiple files as a list in their input channel, this is common in nf-core modules. For example, consider the following input declaration in a process, where FASTQ could be > 1 file: +Each item in the channel will be a tuple, but some processes will use multiple files as a list in their input channel, this is common in nf-core modules. For example, consider the following input declaration in a process, where FASTQ could be > 1 file: -```nextflow +```groovy process ZCAT_FASTQS { input: tuple val(meta), path(fastq) @@ -102,13 +97,13 @@ process ZCAT_FASTQS { The output of `.fromSamplesheet()` can be used by default with a process with the following input declaration: -```nextflow +```groovy val(meta), path(fastq_1), path(fastq_2) ``` To manipulate each item within a channel, you should use the [Nextflow `.map()` operator](https://www.nextflow.io/docs/latest/operator.html#map). This will apply a function to each element of the channel in turn. Here, we convert the flat tuple into a tuple composed of a meta and a list of FASTQ files: -```nextflow +```groovy Channel.fromSamplesheet("input") .map { meta, fastq_1, fastq_2 -> tuple(meta, [ fastq_1, fastq_2 ]) } .set { input } @@ -118,7 +113,7 @@ input.view() // Channel has 2 elements: meta, fastqs This is now compatible with the process defined above and will not raise a warning about input cardinality: -```nextflow +```groovy ZCAT_FASTQS(input) ``` @@ -126,7 +121,7 @@ ZCAT_FASTQS(input) For example, to remove the BED file from the channel created above, we could not return it from the map. Note the absence of the `bed` item in the return of the closure below: -```nextflow +```groovy Channel.fromSamplesheet("input") .map { meta, fastq_1, fastq_2, bed -> tuple(meta, fastq_1, fastq_2) } .set { input } @@ -140,7 +135,7 @@ In this way you can drop items from a channel. We could perform this twice to create one channel containing the FASTQs and one containing the BED files, however Nextflow has a native operator to separate channels called [`.multiMap()`](https://www.nextflow.io/docs/latest/operator.html#multimap). Here, we separate the FASTQs and BEDs into two separate channels using `multiMap`. Note, the channels are both contained in `input` and accessed as an attribute using dot notation: -```nextflow +```groovy Channel.fromSamplesheet("input") .multiMap { meta, fastq_1, fastq_2, bed -> fastq: tuple(meta, fastq_1, fastq_2) @@ -151,7 +146,7 @@ Channel.fromSamplesheet("input") The channel has two attributes, `fastq` and `bed`, which can be accessed separately. -```nextflow +```groovy input.fastq.view() // Channel has 3 elements: meta, fastq_1, fastq_2 input.bed.view() // Channel has 2 elements: meta, bed ``` @@ -164,7 +159,7 @@ You can use the [`.branch()` operator](https://www.nextflow.io/docs/latest/opera This example shows a channel which can have entries for WES or WGS data. WES data includes a BED file denoting the target regions, but WGS data does not. These analysis are different so we want to separate the WES and WGS entries from each other. We can separate the two using `.branch` based on the presence of the BED file: -```nextflow +```groovy // Channel with four elements - see docs for examples params.input = "sample sheet.csv" @@ -208,7 +203,7 @@ It's useful to determine the count of channel entries with similar values when y This example contains a channel where multiple samples can be in the same family. Later on in the pipeline we want to merge the analyzed files so one file gets created for each family. The result will be a channel with an extra meta field containing the count of channel entries with the same family name. -```nextflow +```groovy // channel created by fromSamplesheet() previous to modification: // [[id:example1, family:family1], example1.txt] // [[id:example2, family:family1], example2.txt] diff --git a/docs/samplesheets/fromSamplesheet.md b/docs/samplesheets/fromSamplesheet.md index 13ec8c3..d288a20 100644 --- a/docs/samplesheets/fromSamplesheet.md +++ b/docs/samplesheets/fromSamplesheet.md @@ -77,7 +77,7 @@ In [this example](../../examples/fromSamplesheetBasic/), we create a simple chan !!! danger - It is the order of fields **in the sample sheet JSON schema** which defines the order of items in the channel returned by `fromSamplesheet()`, _not_ the order of fields in the CSV file. + It is the order of fields **in the sample sheet JSON schema** which defines the order of items in the channel returned by `fromSamplesheet()`, _not_ the order of fields in the sample sheet file. ``` --8<-- "examples/fromSamplesheetOrder/log.txt" From 2fed4eb9fadbd2996a067b7a6253abb760e2e7dc Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 14:59:15 +0100 Subject: [PATCH 76/96] prettier --- docs/migration_guide.md | 110 ++---------------- .../sample_sheet_schema_specification.md | 1 - 2 files changed, 8 insertions(+), 103 deletions(-) diff --git a/docs/migration_guide.md b/docs/migration_guide.md index 3c748ae..6f4e188 100644 --- a/docs/migration_guide.md +++ b/docs/migration_guide.md @@ -28,10 +28,8 @@ sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.or ``` !!! note - Repeat this command for every JSON schema you use in your pipeline. e.g. for the default samplesheet schema: - ```bash - sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.org\/draft\/2020-12\/schema/g' -e 's/definitions/defs/g' assets/schema_input.json - ``` +Repeat this command for every JSON schema you use in your pipeline. e.g. for the default samplesheet schema: +`bash sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.org\/draft\/2020-12\/schema/g' -e 's/definitions/defs/g' assets/schema_input.json ` If you are using any special features in your schemas, you will need to update your schemas manually. Please refer to the [JSON Schema draft 2020-12 release notes](https://json-schema.org/draft/2020-12/release-notes) and [JSON schema draft 2019-09 release notes](https://json-schema.org/draft/2019-09/release-notes) for more information. @@ -43,118 +41,26 @@ When you use `unique` in your schemas, you should update it to use `uniqueItems` If you used the `unique:true` field, you should update it to use `uniqueItems` like this: - === "Before v2.0" - ```json hl_lines="9" - { - "$schema": "http://json-schema.org/draft-07/schema", - "type": "array", - "items": { - "type": "object", - "properties": { - "sample": { - "type": "string", - "unique": true - } - } - } - } - ``` +`json hl_lines="9" { "$schema": "http://json-schema.org/draft-07/schema", "type": "array", "items": { "type": "object", "properties": { "sample": { "type": "string", "unique": true } } } } ` === "After v2.0" - ```json hl_lines="12" - { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "array", - "items": { - "type": "object", - "properties": { - "sample": { - "type": "string" - } - } - }, - "uniqueItems": true - } - ``` - +`json hl_lines="12" { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "array", "items": { "type": "object", "properties": { "sample": { "type": "string" } } }, "uniqueItems": true } ` If you used the `unique: ["field1", "field2"]` field, you should update it to use `uniqueEntries` like this: === "Before v2.0" - ```json hl_lines="9" - { - "$schema": "http://json-schema.org/draft-07/schema", - "type": "array", - "items": { - "type": "object", - "properties": { - "sample": { - "type": "string", - "unique": ["sample"] - } - } - } - } - ``` +`json hl_lines="9" { "$schema": "http://json-schema.org/draft-07/schema", "type": "array", "items": { "type": "object", "properties": { "sample": { "type": "string", "unique": ["sample"] } } } } ` === "After v2.0" - ```json hl_lines="12" - { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "array", - "items": { - "type": "object", - "properties": { - "sample": { - "type": "string" - } - } - }, - "uniqueEntries": ["sample"] - } - ``` +`json hl_lines="12" { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "array", "items": { "type": "object", "properties": { "sample": { "type": "string" } } }, "uniqueEntries": ["sample"] } ` ### Updating `dependentRequired` keyword When you use `dependentRequired` in your schemas, you should update it like this: === "Before v2.0" - ```json hl_lines="12" - { - "$schema": "http://json-schema.org/draft-07/schema", - "type": "object", - "properties": { - "fastq_1": { - "type": "string", - "format": "file-path" - }, - "fastq_2": { - "type": "string", - "format": "file-path" - "dependentRequired": ["fastq_1"] - } - } - } - ``` +`json hl_lines="12" { "$schema": "http://json-schema.org/draft-07/schema", "type": "object", "properties": { "fastq_1": { "type": "string", "format": "file-path" }, "fastq_2": { "type": "string", "format": "file-path" "dependentRequired": ["fastq_1"] } } } ` === "After v2.0" - ```json hl_lines="14 15 16" - { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "fastq_1": { - "type": "string", - "format": "file-path" - }, - "fastq_2": { - "type": "string", - "format": "file-path" - } - }, - "dependentRequired": { - "fastq_2": ["fastq_1"] - } - } - ``` \ No newline at end of file +`json hl_lines="14 15 16" { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "fastq_1": { "type": "string", "format": "file-path" }, "fastq_2": { "type": "string", "format": "file-path" } }, "dependentRequired": { "fastq_2": ["fastq_1"] } } ` diff --git a/docs/nextflow_schema/sample_sheet_schema_specification.md b/docs/nextflow_schema/sample_sheet_schema_specification.md index 7865b8e..3028743 100644 --- a/docs/nextflow_schema/sample_sheet_schema_specification.md +++ b/docs/nextflow_schema/sample_sheet_schema_specification.md @@ -95,4 +95,3 @@ For example: will convert the `field` value to a meta value, resulting in the channel `[[id:value]...]` See [here](https://github.com/nextflow-io/nf-validation/blob/ce3aef60e5103ea4798375fe6c59bae41b7d2a25/plugins/nf-validation/src/testResources/schema_input.json#L10-L25) for an example in the sample sheet. - From a250de6d134b7773487c817af156fc7f907c4e68 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 15:47:40 +0100 Subject: [PATCH 77/96] add migration guide to changelog --- CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c48a66e..58e4225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,7 @@ :warning: This version contains a number of breaking changes. Please read the changelog carefully before upgrading. :warning: -To migrate simple schemas, you can simply follow the next steps: - -1. Change the `$schema` to `https://json-schema.org/draft/2020-12/schema` -2. Change the `definitions` keyword to `defs`. See [reason](https://json-schema.org/draft/2019-09/release-notes#semi-incompatible-changes). -3. Change the reference links in all `$ref` keywords to use the new `defs` keyword instead of `definitions` +To migrate your schemas please follow the [migration guide](docs/migration_guide.md) ## New features From 25a2445c586cc6d87028918209ae111a7a5818fc Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 15:50:11 +0100 Subject: [PATCH 78/96] fix migration guide url --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58e4225..58a60ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ :warning: This version contains a number of breaking changes. Please read the changelog carefully before upgrading. :warning: -To migrate your schemas please follow the [migration guide](docs/migration_guide.md) +To migrate your schemas please follow the [migration guide](https://nextflow-io.github.io/nf-validation/latest/migration_guide/) ## New features From 1deb52b5fe38e1526e713242993c89b2f952295f Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 15 Feb 2024 16:27:53 +0100 Subject: [PATCH 79/96] fix some custom error messages not showing up --- .../CustomEvaluators/SchemaEvaluator.groovy | 4 +- .../validation/JsonSchemaValidator.groovy | 39 ++++++++++--------- .../validation/SchemaValidator.groovy | 4 +- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy index 95dbe6f..ad9cfb5 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy @@ -50,9 +50,9 @@ class SchemaEvaluator implements Evaluator { def String schemaFull = Utils.getSchemaPath(this.baseDir, this.schema) def JSONArray arrayJSON = Utils.fileToJsonArray(file, Path.of(schemaFull)) def String schemaContents = Files.readString( Path.of(schemaFull) ) - def validator = new JsonSchemaValidator(schemaContents) + def validator = new JsonSchemaValidator() - def List validationErrors = validator.validate(arrayJSON) + def List validationErrors = validator.validate(arrayJSON, schemaContents) if (validationErrors) { def List errors = ["Validation of file failed:"] + validationErrors.collect { "\t${it}" as String} return Evaluator.Result.failure(errors.join("\n")) diff --git a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy index 9ba657a..7ca6406 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/JsonSchemaValidator.groovy @@ -23,12 +23,18 @@ import java.util.regex.Matcher public class JsonSchemaValidator { private static ValidatorFactory validator - private static JSONObject schema private static Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$') - JsonSchemaValidator(String schemaString) { - this.schema = new JSONObject(schemaString) - def String draft = Utils.getValueFromJson("#/\$schema", this.schema) + JsonSchemaValidator() { + this.validator = new ValidatorFactory() + .withJsonNodeFactory(new OrgJsonNode.Factory()) + // .withDialect() // TODO define the dialect + .withEvaluatorFactory(EvaluatorFactory.compose(new CustomEvaluatorFactory(), new FormatEvaluatorFactory())) + } + + private static List validateObject(JsonNode input, String validationType, Object rawJson, String schemaString) { + def JSONObject schema = new JSONObject(schemaString) + def String draft = Utils.getValueFromJson("#/\$schema", schema) if(draft != "https://json-schema.org/draft/2020-12/schema") { log.error("""Failed to load the meta schema: The used schema draft (${draft}) is not correct, please use \"https://json-schema.org/draft/2020-12/schema\" instead. @@ -36,14 +42,8 @@ See here for more information: https://json-schema.org/specification#migrating-f """) throw new SchemaValidationException("", []) } - this.validator = new ValidatorFactory() - .withJsonNodeFactory(new OrgJsonNode.Factory()) - // .withDialect() // TODO define the dialect - .withEvaluatorFactory(EvaluatorFactory.compose(new CustomEvaluatorFactory(), new FormatEvaluatorFactory())) - } - - private static List validateObject(JsonNode input, String validationType, Object rawJson) { - def Validator.Result result = this.validator.validate(this.schema, input) + + def Validator.Result result = this.validator.validate(schema, input) def List errors = [] for (error : result.getErrors()) { def String errorString = error.getError() @@ -55,9 +55,12 @@ See here for more information: https://json-schema.org/specification#migrating-f def String instanceLocation = error.getInstanceLocation() def String value = Utils.getValueFromJson(instanceLocation, rawJson) - // Get the errorMessage if there is one + // Get the custom errorMessage if there is one and the validation errors are not about the content of the file def String schemaLocation = error.getSchemaLocation().replaceFirst(/^[^#]+/, "") - def String customError = Utils.getValueFromJson("${schemaLocation}/errorMessage", this.schema) + def String customError = "" + if (!errorString.startsWith("Validation of file failed:")) { + customError = Utils.getValueFromJson("${schemaLocation}/errorMessage", schema) as String + } // Change some error messages to make them more clear if (customError == "") { @@ -96,13 +99,13 @@ See here for more information: https://json-schema.org/specification#migrating-f return errors } - public static List validate(JSONArray input) { + public static List validate(JSONArray input, String schemaString) { def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) - return this.validateObject(jsonInput, "field", input) + return this.validateObject(jsonInput, "field", input, schemaString) } - public static List validate(JSONObject input) { + public static List validate(JSONObject input, String schemaString) { def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) - return this.validateObject(jsonInput, "parameter", input) + return this.validateObject(jsonInput, "parameter", input, schemaString) } } \ No newline at end of file diff --git a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy index 0fcb25b..81573b2 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/SchemaValidator.groovy @@ -329,7 +329,7 @@ Also make sure that the same schema is used for validation and conversion of the //=====================================================================// // Validate parameters against the schema def String schema_string = Files.readString( Path.of(Utils.getSchemaPath(baseDir, schemaFilename)) ) - def validator = new JsonSchemaValidator(schema_string) + def validator = new JsonSchemaValidator() // check for warnings if( this.hasWarnings() ) { @@ -341,7 +341,7 @@ Also make sure that the same schema is used for validation and conversion of the def colors = logColours(useMonochromeLogs) // Validate - List validationErrors = validator.validate(paramsJSON) + List validationErrors = validator.validate(paramsJSON, schema_string) this.errors.addAll(validationErrors) if (this.hasErrors()) { def msg = "${colors.red}The following invalid input values have been detected:\n\n" + errors.join('\n').trim() + "\n${colors.reset}\n" From 3cdd4e9b1c99313732334318566712df8538ff11 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Feb 2024 14:12:58 +0100 Subject: [PATCH 80/96] Added deprecated error messages --- .../validation/CustomEvaluatorFactory.groovy | 2 ++ .../DeprecatedEvaluator.groovy | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/DeprecatedEvaluator.groovy diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy index b049a26..c3bc08a 100644 --- a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluatorFactory.groovy @@ -44,6 +44,8 @@ class CustomEvaluatorFactory implements EvaluatorFactory { return Optional.of(new UniqueEntriesEvaluator(schemaNode.asArray())) } else if (fieldName == "type" && (schemaNode.isString() || schemaNode.isArray()) && lenientMode) { return Optional.of(new LenientTypeEvaluator(schemaNode)) + } else if (fieldName == "deprecated" && schemaNode.isBoolean()) { + return Optional.of(new DeprecatedEvaluator(schemaNode.asBoolean())) } return Optional.empty() diff --git a/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/DeprecatedEvaluator.groovy b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/DeprecatedEvaluator.groovy new file mode 100644 index 0000000..ff9c18e --- /dev/null +++ b/plugins/nf-validation/src/main/nextflow/validation/CustomEvaluators/DeprecatedEvaluator.groovy @@ -0,0 +1,35 @@ +package nextflow.validation + +import dev.harrel.jsonschema.Evaluator +import dev.harrel.jsonschema.EvaluationContext +import dev.harrel.jsonschema.JsonNode +import nextflow.Nextflow + +import groovy.util.logging.Slf4j +import java.nio.file.Path + +/** + * @author : nvnieuwk + */ + +@Slf4j +class DeprecatedEvaluator implements Evaluator { + // Checks if the use of this value is deprecated + + private final Boolean deprecated + + DeprecatedEvaluator(Boolean deprecated) { + this.deprecated = deprecated + } + + @Override + public Evaluator.Result evaluate(EvaluationContext ctx, JsonNode node) { + // Checks if the value should be deprecated + if (!this.deprecated) { + return Evaluator.Result.success() + } + + return Evaluator.Result.failure("This option is deprecated") + + } +} \ No newline at end of file From 949a54bd07b595187ff010bb3db94a2e1f2a8ab1 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Feb 2024 14:23:00 +0100 Subject: [PATCH 81/96] Added deprecated to the docs --- .../nextflow_schema/nextflow_schema_specification.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/nextflow_schema/nextflow_schema_specification.md b/docs/nextflow_schema/nextflow_schema_specification.md index 4770efb..d1778cd 100644 --- a/docs/nextflow_schema/nextflow_schema_specification.md +++ b/docs/nextflow_schema/nextflow_schema_specification.md @@ -241,6 +241,18 @@ and get: * --input (samples.yml): File name must end in '.csv' cannot contain spaces ``` +### `deprecated` + +!!! example "Extended key" + +A boolean JSON flag that instructs anything using the schema that this parameter/field is deprecated and should not be used. This can be useful to generate messages telling the user that a parameter has changed between versions. + +JSON schema states that this is an informative key only, but in `nf-validation` this will cause a validation error if the parameter/field is used. + +!!! tip + + Using the [`errorMessage`](#errormessage) keyword can be useful to provide more information about the deprecation and what to use instead. + ### `enum` An array of enumerated values: the parameter must match one of these values exactly to pass validation. From 9cedf638064c26b8e7d665e5c3b25546f4985c14 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Feb 2024 14:25:07 +0100 Subject: [PATCH 82/96] prettier --- docs/nextflow_schema/nextflow_schema_specification.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/nextflow_schema/nextflow_schema_specification.md b/docs/nextflow_schema/nextflow_schema_specification.md index d1778cd..6375679 100644 --- a/docs/nextflow_schema/nextflow_schema_specification.md +++ b/docs/nextflow_schema/nextflow_schema_specification.md @@ -245,7 +245,7 @@ and get: !!! example "Extended key" -A boolean JSON flag that instructs anything using the schema that this parameter/field is deprecated and should not be used. This can be useful to generate messages telling the user that a parameter has changed between versions. +A boolean JSON flag that instructs anything using the schema that this parameter/field is deprecated and should not be used. This can be useful to generate messages telling the user that a parameter has changed between versions. JSON schema states that this is an informative key only, but in `nf-validation` this will cause a validation error if the parameter/field is used. From 960ee355d1c67f4dba6496ae8784d54b22e7f0fe Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:15:16 +0100 Subject: [PATCH 83/96] Update docs/migration_guide.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/migration_guide.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/migration_guide.md b/docs/migration_guide.md index 6f4e188..01864c2 100644 --- a/docs/migration_guide.md +++ b/docs/migration_guide.md @@ -27,6 +27,8 @@ If you aren't using any special features in your schemas, you can simply update sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.org\/draft\/2020-12\/schema/g' -e 's/definitions/defs/g' nextflow_schema.json ``` +This will replace the old schema draft specification (`draft-07`) by the new one (`2020-12`), and the old keyword `definitions` by the new notation `defs`. + !!! note Repeat this command for every JSON schema you use in your pipeline. e.g. for the default samplesheet schema: `bash sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.org\/draft\/2020-12\/schema/g' -e 's/definitions/defs/g' assets/schema_input.json ` From a9c1dfc1969138dc2de25bb670ea2862c4251a17 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:15:27 +0100 Subject: [PATCH 84/96] Update docs/nextflow_schema/create_schema.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/nextflow_schema/create_schema.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/nextflow_schema/create_schema.md b/docs/nextflow_schema/create_schema.md index f869e03..1d034da 100644 --- a/docs/nextflow_schema/create_schema.md +++ b/docs/nextflow_schema/create_schema.md @@ -53,7 +53,7 @@ nf-core schema build ```bash sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.org\/draft\/2020-12\/schema/g' -e 's/definitions/defs/g' nextflow_schema.json ``` - + A new version of the nf-core schema builder will be available soon. Keep an eye out! The tool will run the `nextflow config` command to extract your pipeline's configuration and compare the output to your `nextflow_schema.json` file (if it exists). It will prompt you to update the schema file with any changes, then it will ask if you From ad7463b9959b5633cc2502649dcd6354d20a3f77 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:19:10 +0100 Subject: [PATCH 85/96] Update docs/nextflow_schema/sample_sheet_schema_specification.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/nextflow_schema/sample_sheet_schema_specification.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/nextflow_schema/sample_sheet_schema_specification.md b/docs/nextflow_schema/sample_sheet_schema_specification.md index 3028743..899c540 100644 --- a/docs/nextflow_schema/sample_sheet_schema_specification.md +++ b/docs/nextflow_schema/sample_sheet_schema_specification.md @@ -2,7 +2,7 @@ description: Schema specification for sample sheet validation --- -# Sample sheetschema specification +# Sample sheet schema specification Sample sheet schema files are used by the nf-validation plugin for validation of sample sheet contents and type conversion / channel generation. From f127f1fd3059226b58ab7144310001240c56ab83 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:19:34 +0100 Subject: [PATCH 86/96] Update docs/nextflow_schema/sample_sheet_schema_specification.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/nextflow_schema/sample_sheet_schema_specification.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/nextflow_schema/sample_sheet_schema_specification.md b/docs/nextflow_schema/sample_sheet_schema_specification.md index 899c540..3e8fb0c 100644 --- a/docs/nextflow_schema/sample_sheet_schema_specification.md +++ b/docs/nextflow_schema/sample_sheet_schema_specification.md @@ -71,7 +71,7 @@ Please refer to the [Nextflow schema specification](../nextflow_schema/nextflow_ !!! tip Sample sheets are commonly used to define input file paths. - Be sure to set `"type": "string"`, `exists: true` and `"format": "file-path"`, `"schema":"path/to/samplesheet/schema.json"` for these properties, + Be sure to set `"type": "string"`, `exists: true`, `"format": "file-path"` and `"schema":"path/to/samplesheet/schema.json"` for these properties, so that `fromSamplesheet` will not result in any errors. ## Sample sheet keys From 7520cad74fe499879481ff56a6299b446486aee5 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:19:56 +0100 Subject: [PATCH 87/96] Update docs/nextflow_schema/sample_sheet_schema_specification.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/nextflow_schema/sample_sheet_schema_specification.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/nextflow_schema/sample_sheet_schema_specification.md b/docs/nextflow_schema/sample_sheet_schema_specification.md index 3e8fb0c..db0afd1 100644 --- a/docs/nextflow_schema/sample_sheet_schema_specification.md +++ b/docs/nextflow_schema/sample_sheet_schema_specification.md @@ -72,7 +72,7 @@ Please refer to the [Nextflow schema specification](../nextflow_schema/nextflow_ Sample sheets are commonly used to define input file paths. Be sure to set `"type": "string"`, `exists: true`, `"format": "file-path"` and `"schema":"path/to/samplesheet/schema.json"` for these properties, - so that `fromSamplesheet` will not result in any errors. + so that samplesheets are correctly validated and `fromSamplesheet` does not result in any errors. ## Sample sheet keys From e163089d89dbdd12d2eb628507ffb2f5aebff806 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:20:17 +0100 Subject: [PATCH 88/96] Update docs/background.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/background.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/background.md b/docs/background.md index 6b51571..0691f55 100644 --- a/docs/background.md +++ b/docs/background.md @@ -12,6 +12,6 @@ In order for end users to launch a given workflow with different input data and In addition to config params, a common best-practice for pipelines is to use a "sample sheet" file containing required input information. For example: a sample identifier, filenames and other sample-level metadata. -Nextflow itself does not provide functionality to validate config parameters or parsed samplesheets. To bridge this gap, we developed code within the [nf-core community](https://nf-co.re/) to allow pipelines to work with a standard `nextflow_schema.json` file, written using the [JSON Schema](https://json-schema.org/) format. The file allows strict typing of parameter variables and inclusion of validation rules. +Nextflow itself does not provide functionality to validate config parameters or parsed sample sheets. To bridge this gap, we developed code within the [nf-core community](https://nf-co.re/) to allow pipelines to work with a standard `nextflow_schema.json` file, written using the [JSON Schema](https://json-schema.org/) format. The file allows strict typing of parameter variables and inclusion of validation rules. The nf-validation plugin moves this code out of the nf-core template into a stand-alone package, to make it easier to use for the wider Nextflow community. It also incorporates a number of new features, such as native Groovy sample sheet validation. From 44a3404f90b9589679e1fa9e65082e3c1e9c5622 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:20:41 +0100 Subject: [PATCH 89/96] Update docs/samplesheets/examples.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/samplesheets/examples.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/samplesheets/examples.md b/docs/samplesheets/examples.md index 0e95313..1f97d46 100644 --- a/docs/samplesheets/examples.md +++ b/docs/samplesheets/examples.md @@ -46,7 +46,7 @@ It may be necessary to manipulate this channel to fit your process inputs. For m ## Using a sample sheet with no headers -Sometimes you only have one possible input in the pipeline sample sheet. In this case it doesn't make sense to have a header in the sample sheet. This can be done by removing the `properties` section fro m the samplesheet and changing the type of the element from `object` to whatever type you want: +Sometimes you only have one possible input in the pipeline sample sheet. In this case it doesn't make sense to have a header in the sample sheet. This can be done by removing the `properties` section from the sample sheet and changing the type of the element from `object` the desired type: ```json { From e1f343b69d40b1311aa76dc13f55ece73e79c9ba Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:21:04 +0100 Subject: [PATCH 90/96] Update docs/samplesheets/examples.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/samplesheets/examples.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/samplesheets/examples.md b/docs/samplesheets/examples.md index 1f97d46..5c11ce4 100644 --- a/docs/samplesheets/examples.md +++ b/docs/samplesheets/examples.md @@ -161,7 +161,7 @@ This example shows a channel which can have entries for WES or WGS data. WES dat ```groovy // Channel with four elements - see docs for examples -params.input = "sample sheet.csv" +params.input = "samplesheet.csv" Channel.fromSamplesheet("input") .branch { meta, fastq_1, fastq_2, bed -> From efa83ceea11252bbf3105ab2f21a548e7f77c116 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:21:21 +0100 Subject: [PATCH 91/96] Update docs/samplesheets/fromSamplesheet.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/samplesheets/fromSamplesheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/samplesheets/fromSamplesheet.md b/docs/samplesheets/fromSamplesheet.md index d288a20..338cdb9 100644 --- a/docs/samplesheets/fromSamplesheet.md +++ b/docs/samplesheets/fromSamplesheet.md @@ -21,7 +21,7 @@ The function has one mandatory argument: the name of the parameter which specifi The path specified in the `schema` key determines the JSON used for validation of the sample sheet. -When using the `.fromSamplesheet` channel factory, some one optional arguments can be used: +When using the `.fromSamplesheet` channel factory, one optional arguments can be used: - `parameters_schema`: File name for the pipeline parameters schema. (Default: `nextflow_schema.json`) From 6423d5a2a01cdd544a5000026e10ca473d3ea5fd Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:21:33 +0100 Subject: [PATCH 92/96] Update docs/samplesheets/fromSamplesheet.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/samplesheets/fromSamplesheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/samplesheets/fromSamplesheet.md b/docs/samplesheets/fromSamplesheet.md index 338cdb9..b46ea76 100644 --- a/docs/samplesheets/fromSamplesheet.md +++ b/docs/samplesheets/fromSamplesheet.md @@ -83,7 +83,7 @@ In [this example](../../examples/fromSamplesheetBasic/), we create a simple chan --8<-- "examples/fromSamplesheetOrder/log.txt" ``` -=== "sample sheet.csv" +=== "samplesheet.csv" ```csv --8<-- "examples/fromSamplesheetOrder/sample sheet.csv" From 674423a6278cfd567b8151e3eafe29f674f5e44d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:21:39 +0100 Subject: [PATCH 93/96] Update docs/samplesheets/fromSamplesheet.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/samplesheets/fromSamplesheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/samplesheets/fromSamplesheet.md b/docs/samplesheets/fromSamplesheet.md index b46ea76..6ba2b02 100644 --- a/docs/samplesheets/fromSamplesheet.md +++ b/docs/samplesheets/fromSamplesheet.md @@ -86,7 +86,7 @@ In [this example](../../examples/fromSamplesheetBasic/), we create a simple chan === "samplesheet.csv" ```csv - --8<-- "examples/fromSamplesheetOrder/sample sheet.csv" + --8<-- "examples/fromSamplesheetOrder/samplesheet.csv" ``` === "assets/schema_input.json" From ee712e6d8723100303a4ec532917bd854df72e79 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:21:47 +0100 Subject: [PATCH 94/96] Update docs/samplesheets/fromSamplesheet.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/samplesheets/fromSamplesheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/samplesheets/fromSamplesheet.md b/docs/samplesheets/fromSamplesheet.md index 6ba2b02..9fb6912 100644 --- a/docs/samplesheets/fromSamplesheet.md +++ b/docs/samplesheets/fromSamplesheet.md @@ -137,7 +137,7 @@ This returns a channel with a meta map. === "sample sheet.csv" ```csv - --8<-- "examples/fromSamplesheetMeta/sample sheet.csv" + --8<-- "examples/fromSamplesheetMeta/samplesheet.csv" ``` === "nextflow.config" From 11666e5fe6ceca0ac41ba44ebe5ff2bb97e3fd94 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:21:54 +0100 Subject: [PATCH 95/96] Update docs/samplesheets/fromSamplesheet.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- docs/samplesheets/fromSamplesheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/samplesheets/fromSamplesheet.md b/docs/samplesheets/fromSamplesheet.md index 9fb6912..eec3904 100644 --- a/docs/samplesheets/fromSamplesheet.md +++ b/docs/samplesheets/fromSamplesheet.md @@ -134,7 +134,7 @@ This returns a channel with a meta map. --8<-- "examples/fromSamplesheetMeta/pipeline/main.nf" ``` -=== "sample sheet.csv" +=== "samplesheet.csv" ```csv --8<-- "examples/fromSamplesheetMeta/samplesheet.csv" From 65853cabafceb33f1f5db4246413987fc60fa490 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Feb 2024 15:24:07 +0100 Subject: [PATCH 96/96] prettify suggestions --- docs/nextflow_schema/create_schema.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/nextflow_schema/create_schema.md b/docs/nextflow_schema/create_schema.md index 1d034da..14316f0 100644 --- a/docs/nextflow_schema/create_schema.md +++ b/docs/nextflow_schema/create_schema.md @@ -54,6 +54,7 @@ nf-core schema build sed -i -e 's/http:\/\/json-schema.org\/draft-07\/schema/https:\/\/json-schema.org\/draft\/2020-12\/schema/g' -e 's/definitions/defs/g' nextflow_schema.json ``` A new version of the nf-core schema builder will be available soon. Keep an eye out! + The tool will run the `nextflow config` command to extract your pipeline's configuration and compare the output to your `nextflow_schema.json` file (if it exists). It will prompt you to update the schema file with any changes, then it will ask if you