diff --git a/stroom-app/src/test/java/stroom/app/docs/GenerateAllDocumentation.java b/stroom-app/src/test/java/stroom/app/docs/GenerateAllDocumentation.java index 3624f8d0fe..1efc3d3d0d 100644 --- a/stroom-app/src/test/java/stroom/app/docs/GenerateAllDocumentation.java +++ b/stroom-app/src/test/java/stroom/app/docs/GenerateAllDocumentation.java @@ -24,7 +24,8 @@ public static void main(final String[] args) { final List documentationGenerators = List.of( new GenerateDocumentReferenceDoc(), new GeneratePipelineElementsDoc(), - new GenerateSnippetsDoc()); + new GenerateSnippetsDoc(), + new GenerateXsltFunctionDefinitions()); StroomDocsUtil.doWithClassScanResult(scanResult -> { documentationGenerators.forEach(documentationGenerator -> { diff --git a/stroom-app/src/test/java/stroom/app/docs/GenerateXsltFunctionDefinitions.java b/stroom-app/src/test/java/stroom/app/docs/GenerateXsltFunctionDefinitions.java new file mode 100644 index 0000000000..0e904623b6 --- /dev/null +++ b/stroom-app/src/test/java/stroom/app/docs/GenerateXsltFunctionDefinitions.java @@ -0,0 +1,354 @@ +package stroom.app.docs; + + +import stroom.docs.shared.NotDocumented; +import stroom.pipeline.xsltfunctions.XsltFunctionCategory; +import stroom.pipeline.xsltfunctions.XsltFunctionDef; +import stroom.test.common.docs.StroomDocsUtil; +import stroom.test.common.docs.StroomDocsUtil.GeneratesDocumentation; +import stroom.util.exception.ThrowingConsumer; +import stroom.util.json.JsonUtil; +import stroom.util.logging.LambdaLogger; +import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.logging.LogUtil; +import stroom.util.shared.NullSafe; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.github.classgraph.ClassInfo; +import io.github.classgraph.ScanResult; + +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class GenerateXsltFunctionDefinitions implements DocumentationGenerator { + + private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(GenerateXsltFunctionDefinitions.class); + + private static final Path DOCS_SUB_PATH = Paths.get( + "content/en/docs/reference-section/xslt-functions"); + private static final Path DATA_SUB_PATH = Paths.get( + "assets/data/xslt-functions"); + private static final String INDEX_DATA_FILENAME = "_index.json"; + private static final String INDEX_DOC_FILENAME = "_index.md"; + + private final ObjectMapper objectMapper; + + @GeneratesDocumentation + public static void main(String[] args) { + new GenerateXsltFunctionDefinitions().generate(); + } + + public GenerateXsltFunctionDefinitions() { + this.objectMapper = JsonUtil.getMapper(); + } + + @Override + @GeneratesDocumentation + public void generateAll(final ScanResult scanResult) { + try { + final Path outputPath = StroomDocsUtil.resolveStroomDocsFile(DATA_SUB_PATH, false); + Files.createDirectories(outputPath); + LOGGER.info("Clearing dir {}", outputPath.toAbsolutePath().normalize()); + + // Remove existing function files + try (final Stream pathStream = Files.list(outputPath)) { + pathStream + .filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(".json")) + .forEach(ThrowingConsumer.unchecked(path -> { + LOGGER.info("Deleting file {}", path.toAbsolutePath().normalize()); + Files.delete(path); + })); + } + + final List> annotatedClasses = getAllFunctionDefs(scanResult); + annotatedClasses.forEach(this::processFunction); + produceIndexFile(annotatedClasses); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + private void produceIndexFile(final List> annotatedClasses) { + final Map>> groups = annotatedClasses.stream() + .collect(Collectors.groupingBy(annotatedClass -> { + final XsltFunctionCategory[] categories = annotatedClass.annotation().commonCategory(); + Objects.requireNonNull(categories, () -> LogUtil.message( + "functions {} should have a commonCategory", + annotatedClass.clazz().getName())); + return categories[0]; + })); + + final Map map = new HashMap<>(); + final AtomicInteger errorCounter = new AtomicInteger(); + groups.forEach((category, classesGroup) -> { + final String docFilename = category.name() + .toLowerCase() + .replace("[^a-zA-Z0-9-]", "-") + ".md"; + final XsltFunctionCategoryIndex index = map.computeIfAbsent(category, + k -> new XsltFunctionCategoryIndex(null, k, docFilename)); + classesGroup.forEach(annotatedClass -> { + final String functionName = annotatedClass.annotation().name(); + index.addFunction(functionName); + }); + final int errorCount = checkDocPage(index); + errorCounter.addAndGet(errorCount); + }); + + try { + final String json = JsonUtil.getMapper().writeValueAsString(map); + LOGGER.info("Index:\n{}", json); + + final Path outputFile = buildDataFilePath(INDEX_DATA_FILENAME); + Files.writeString(outputFile, json, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); + } catch (final IOException e) { + throw new RuntimeException(e); + } + + if (errorCounter.get() > 0) { + throw new RuntimeException(LogUtil.message("There were {} errors, check the logs", errorCounter.get())); + } + } + + private int checkDocPage(final XsltFunctionCategoryIndex index) { + try { + int errorCount = 0; + // Check the _index.md file contains a link for each func with the appropriate category + final Path indexDocFilePath = buildDocsFilePath(INDEX_DOC_FILENAME); + if (!Files.isRegularFile(indexDocFilePath)) { + throw new RuntimeException(LogUtil.message("File {} does not exist", + indexDocFilePath.toAbsolutePath())); + } + final String docFileNameWithoutExtension = index.getDocFilename() + .replaceAll("\\.md$", ""); + final String indexDocContent = Files.readString(indexDocFilePath); + Set stringsToFind = index.functionNames + .stream() + .map(functionName -> + "[" + functionName + "](" + + docFileNameWithoutExtension + + "#" + functionName + ")") + .collect(Collectors.toCollection(HashSet::new)); + + stringsToFind.removeIf(indexDocContent::contains); + + if (!stringsToFind.isEmpty()) { + LOGGER.error("File {} is missing content for the following functions in category {}. " + + "Each function should have a link a bit like '[hash](conversion#hash)'.\n{}", + indexDocFilePath.toAbsolutePath(), + index.category, + String.join("\n", stringsToFind)); + } + errorCount += stringsToFind.size(); + + // Check the appropriate category file (e.g. conversion.md) contains a shortcode + // for each of the funcs in that category. + final Path categoryDocFilePath = buildDocsFilePath(index.getDocFilename()); + if (!Files.isRegularFile(categoryDocFilePath)) { + throw new RuntimeException(LogUtil.message("File {} does not exist", + indexDocFilePath.toAbsolutePath())); + } + + final String categoryDocContent = Files.readString(categoryDocFilePath); + + stringsToFind = index.functionNames + .stream() + .map(functionName -> "{{< xslt-func \"" + functionName + "\" >}}") + .collect(Collectors.toCollection(HashSet::new)); + + stringsToFind.removeIf(categoryDocContent::contains); + + if (!stringsToFind.isEmpty()) { + LOGGER.error("File {} is missing content for the following functions. " + + "Each function should have a shortcode call like '{{< xslt-func \"hash\" >}}'.\n{}", + categoryDocFilePath.toAbsolutePath(), + String.join("\n", stringsToFind)); + } + errorCount += stringsToFind.size(); + return errorCount; + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + private void generate() { + StroomDocsUtil.doWithClassScanResult(this::generateAll); +// final ObjectMapper mapper = JsonUtil.getMapper(); +// final ListXsltFunctionDef functionDefinitions = XsltFunctionFactory.getFunctionDefinitions; + } + + private List> getAllFunctionDefs(final ScanResult scanResult) { + try { + // Ideally we would look for all subclasses of StroomExtensionFunctionCall but that is not + // visible from here. However, TestXsltFunctions will check that all subclasses of that + // have the annotation + return scanResult.getAllClasses() + .parallelStream() + .filter(classInfo -> classInfo.hasAnnotation(XsltFunctionDef.class)) + .filter(classInfo -> !classInfo.hasAnnotation(NotDocumented.class)) + .filter(Predicate.not(ClassInfo::isInterface)) + .filter(Predicate.not(ClassInfo::isAbstract)) + .map(ClassInfo::loadClass) + .map(clazz -> { + final XsltFunctionDef anno = clazz.getAnnotation(XsltFunctionDef.class); + if (anno == null) { + LOGGER.error("XSLT Function {} is missing annotation {}", + clazz.getName(), + XsltFunctionDef.class.getName()); + return null; + } else { + return new AnnotatedClass<>(clazz, anno); + } + }) + .filter(Objects::nonNull) + .toList(); + } catch (final Exception e) { + LOGGER.error("Error {}", e.getMessage(), e); + throw new RuntimeException(e); + } + } + + private Path buildDataFilePath(final String filename) { + return StroomDocsUtil.resolveStroomDocsFile(DATA_SUB_PATH.resolve(filename), false); + } + + private Path buildDocsFilePath(final String filename) { + return StroomDocsUtil.resolveStroomDocsFile(DOCS_SUB_PATH.resolve(filename), false); + } + + private void processFunction(final AnnotatedClass annotatedClass) { + try { + final XsltFunctionDef functionDef = annotatedClass.annotation(); + final String json = objectMapper.writeValueAsString(functionDef); + final String filename = annotatedClass.annotation().name() + ".json"; + final Path filePath = buildDataFilePath(filename); + LOGGER.info("{} - {} - {}\n{}", + annotatedClass.clazz().getName(), + filename, + filePath.toAbsolutePath(), + json); + Files.writeString(filePath, json, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + + // -------------------------------------------------------------------------------- + + + private record AnnotatedClass(Class clazz, T annotation) { + + } + + + // -------------------------------------------------------------------------------- + + +// @JsonPropertyOrder(alphabetic = true) +// @JsonInclude(Include.NON_NULL) +// private static class XsltFunctionIndex { +// +// @JsonProperty +// private final Map> functions; +// +// private XsltFunctionIndex( +// @JsonProperty("functions") final Map> functions) { +// this.functions = functions; +// } +// +// public Map> getFunctions() { +// return functions; +// } +// } + + + // -------------------------------------------------------------------------------- + + + @JsonInclude(Include.NON_NULL) + private static class XsltFunctionIndexItem { + + @JsonProperty + private final String name; + @JsonProperty + private final XsltFunctionCategory category; + private final String docFilename; + + private XsltFunctionIndexItem(@JsonProperty("name") final String name, + @JsonProperty("category") final XsltFunctionCategory category, + @JsonProperty("docFilename") final String docFilename) { + this.name = name; + this.category = category; + this.docFilename = docFilename; + } + + public String getName() { + return name; + } + + public XsltFunctionCategory getCategory() { + return category; + } + + public String getDocFilename() { + return docFilename; + } + } + + + // -------------------------------------------------------------------------------- + + + @JsonInclude(Include.NON_NULL) + private static class XsltFunctionCategoryIndex { + + @JsonProperty + private final List functionNames; + @JsonProperty + private final XsltFunctionCategory category; + @JsonProperty + private final String docFilename; + + private XsltFunctionCategoryIndex(@JsonProperty("name") final List functionNames, + @JsonProperty("category") final XsltFunctionCategory category, + @JsonProperty("docFilename") final String docFilename) { + this.functionNames = NullSafe.mutableList(functionNames); + this.category = category; + this.docFilename = docFilename; + } + + public List getFunctionNames() { + return functionNames; + } + + private void addFunction(final String functionName) { + this.functionNames.add(functionName); + } + + public XsltFunctionCategory getCategory() { + return category; + } + + public String getDocFilename() { + return docFilename; + } + } +} diff --git a/stroom-pipeline/build.gradle b/stroom-pipeline/build.gradle index 3ce8ded303..857f256777 100644 --- a/stroom-pipeline/build.gradle +++ b/stroom-pipeline/build.gradle @@ -35,6 +35,7 @@ dependencies { implementation project(':stroom-job:stroom-job-api') implementation project(':stroom-search:stroom-searchable-api') + implementation libs.classgraph implementation libs.commons.compress implementation libs.commons.io implementation libs.commons.lang @@ -82,9 +83,10 @@ dependencies { testImplementation project(':stroom-task:stroom-task-mock') testImplementation project(':stroom-test-common') + testImplementation libs.bundles.common.test.implementation testImplementation libs.commons.io + testImplementation libs.guice.extension - testImplementation libs.bundles.common.test.implementation testRuntimeOnly libs.bundles.common.test.runtime } diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/BitmapLookup.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/BitmapLookup.java index 0debcc5950..6bb3ef648c 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/BitmapLookup.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/BitmapLookup.java @@ -41,6 +41,95 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; + +@XsltFunctionDef( + name = BitmapLookup.FUNCTION_NAME, + commonCategory = XsltFunctionCategory.PIPELINE, + commonDescription = """ + The bitmap-lookup() function looks up a bitmap key from reference or context data a value \ + (which can be an XML node set) for each set bit position and adds it to the resultant XML. + + If the look up fails no result will be returned. + + The key is a bitmap expressed as either a decimal integer or a hexidecimal value, \ + e.g. `14`/`0xE` is `1110` as a binary bitmap. + For each bit position that is set, (i.e. has a binary value of `1`) a lookup will be performed \ + using that bit position as the key. + In this example, positions `1`, `2` & `3` are set so a lookup would be performed for these \ + bit positions. + The result of each lookup for the bitmap are concatenated together in bit position order, \ + separated by a space. + + If `ignoreWarnings` is true then any lookup failures will be ignored and it will return the \ + value(s) for the bit positions it was able to lookup. + + This function can be useful when you have a set of values that can be represented as a bitmap \ + and you need them to be converted back to individual values. + For example if you have a set of additive account permissions (e.g Admin, ManageUsers, \ + PerformExport, etc.), each of which is associated with a bit position, then a user's \ + permissions could be defined as a single decimal/hex bitmap value. + Thus a bitmap lookup with this value would return all the permissions held by the user. + + For example the reference data store may contain: + + | Key (Bit position) | Value | + |--------------------|----------------| + | 0 | Administrator | + | 1 | Manage_Users | + | 2 | Perform_Export | + | 3 | View_Data | + | 4 | Manage_Jobs | + | 5 | Delete_Data | + | 6 | Manage_Volumes | + + The following are example lookups using the above reference data: + + | Lookup Key (decimal) | Lookup Key (Hex) | Bitmap | Result | + |----------------------|------------------|-----------|-----------------------------------------| + | `0` | `0x0` | `0000000` | - | + | `1` | `0x1` | `0000001` | `Administrator` | + | `74` | `0x4A` | `1001010` | `Manage_Users View_Data Manage_Volumes` | + | `2` | `0x2` | `0000010` | `Manage_Users` | + | `96` | `0x60` | `1100000` | `Delete_Data Manage_Volumes` | + """, + commonReturnType = XsltDataType.SEQUENCE, + commonReturnDescription = "The hash of the value", + signatures = { + @XsltFunctionSignature( + args = { + @XsltFunctionArg( + name = "map", + description = "The name of the map to perform the lookup in.", + argType = XsltDataType.STRING), + @XsltFunctionArg( + name = "key", + description = "The key to lookup in the named map.", + argType = XsltDataType.STRING), + @XsltFunctionArg( + name = "time", + description = """ + Determines which set of reference data was effective at the \ + requested time. + If no reference data exists with an effective time before the \ + requested time then the lookup will fail. + Time is in the format `yyyy-MM-dd'T'HH:mm:ss.SSSXX`, e.g. \ + `2010-01-01T00:00:00.000Z`.""", + isOptional = true, + argType = XsltDataType.STRING), + @XsltFunctionArg( + name = "ignoreWarnings", + description = "If true, any lookup failures will be ignored, else they " + + "will be reported as warnings.", + argType = XsltDataType.BOOLEAN, + isOptional = true), + @XsltFunctionArg( + name = "trace", + description = "If true, additional trace information is output as " + + "INFO messages.", + argType = XsltDataType.BOOLEAN, + isOptional = true), + }) + }) class BitmapLookup extends AbstractLookup { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(BitmapLookup.class); diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/CidrToNumericIPRange.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/CidrToNumericIPRange.java index cdeeb4dae2..59ac47b33d 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/CidrToNumericIPRange.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/CidrToNumericIPRange.java @@ -38,6 +38,7 @@ protected ArrayItem call(final String functionName, final XPathContext context, final long networkAddress = IpAddressUtil.toNumericIpAddress(cidrAddress) & subnetMask; final long broadcastAddress = networkAddress | (~subnetMask); + // TODO can we return immutable lists, e.g. List.of(...) ? return new SimpleArrayItem(new ArrayList<>(Arrays.asList( StringValue.makeStringValue(Long.toString(networkAddress)), StringValue.makeStringValue(Long.toString(broadcastAddress))))); diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/CommonXsltFunctionModule.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/CommonXsltFunctionModule.java index faa876fb7c..1e13c5b1f6 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/CommonXsltFunctionModule.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/CommonXsltFunctionModule.java @@ -616,7 +616,8 @@ private static class MetaStreamForIdFunction extends StroomExtensionFunctionDefi MetaStream.FUNCTION_NAME_FOR_ID, 2, 2, - new SequenceType[]{SequenceType.SINGLE_STRING, + new SequenceType[]{ + SequenceType.SINGLE_STRING, SequenceType.SINGLE_INTEGER}, SequenceType.NODE_SEQUENCE, functionCallProvider); @@ -724,7 +725,7 @@ private static class RandomFunction extends StroomExtensionFunctionDefinition functionCallProvider) { super( - "random", + Random.FUNCTION_NAME, 0, 0, new SequenceType[]{}, diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/Hash.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/Hash.java index 7fe6c74980..7327a621ad 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/Hash.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/Hash.java @@ -16,6 +16,7 @@ package stroom.pipeline.xsltfunctions; +import stroom.util.shared.NullSafe; import stroom.util.shared.Severity; import com.google.common.io.BaseEncoding; @@ -28,6 +29,47 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +@XsltFunctionDef( + name = "hash", + commonCategory = XsltFunctionCategory.CONVERSION, + commonDescription = "Generates a hash of the passed value.", + commonReturnType = XsltDataType.STRING, + commonReturnDescription = "The hash of the value", + signatures = { + @XsltFunctionSignature( + description = """ + Generates a hash of the supplied value using the named hash algorithm. + The SHA-256 algorithm will be used if not supplied. + + You can optionally supply a salt value to prepend to the hash.""", + args = { + @XsltFunctionArg( + name = "value", + description = "The value to hash.", + argType = XsltDataType.STRING), + @XsltFunctionArg( + name = "algorithm", + description = "The name of the hash algorithm to use.", + argType = XsltDataType.STRING, + isOptional = true, + defaultValue = "SHA-256", + allowedValues = { + "MD5", + "SHA", + "SHA-224", + "SHA-256", + "SHA-384", + "SHA-512", + }), + @XsltFunctionArg( + name = "salt", + description = """ + The salt to use as input to the hashing function. + The salt will be hashed first followed by the value.""", + isOptional = true, + argType = XsltDataType.STRING), + }) + }) class Hash extends StroomExtensionFunctionCall { public static final String FUNCTION_NAME = "hash"; @@ -41,12 +83,12 @@ protected Sequence call(final String functionName, final XPathContext context, f try { final String value = getSafeString(functionName, context, arguments, 0); - if (value != null && value.length() > 0) { + if (NullSafe.isNonEmptyString(value)) { String algorithm = null; if (arguments.length > 1) { algorithm = getSafeString(functionName, context, arguments, 1); } - if (algorithm == null || algorithm.trim().length() == 0) { + if (NullSafe.isBlankString(algorithm)) { algorithm = DEFAULT_ALGORITHM; } diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/Random.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/Random.java index 5d3f46a351..cb9533bc40 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/Random.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/Random.java @@ -21,7 +21,20 @@ import net.sf.saxon.om.Sequence; import net.sf.saxon.value.DoubleValue; +@XsltFunctionDef( + name = Random.FUNCTION_NAME, + commonCategory = XsltFunctionCategory.VALUE, + commonDescription = "Generates a random number greater than 0.0 and less than 1.0.", + commonReturnType = XsltDataType.DECIMAL, + commonReturnDescription = "The random number.", + signatures = { + @XsltFunctionSignature( + args = {}) + }) class Random extends StroomExtensionFunctionCall { + + public static final String FUNCTION_NAME = "random"; + @Override protected Sequence call(final String functionName, final XPathContext context, final Sequence[] arguments) { try { diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltDataType.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltDataType.java new file mode 100644 index 0000000000..124f28f407 --- /dev/null +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltDataType.java @@ -0,0 +1,36 @@ +package stroom.pipeline.xsltfunctions; + +import stroom.docref.HasDisplayValue; + +import com.fasterxml.jackson.annotation.JsonValue; + +public enum XsltDataType implements HasDisplayValue { + BOOLEAN("Boolean"), + DATE("Date"), + DATE_TIME("Date-Time"), + DECIMAL("Decimal"), + /** + * A sequence with nothing in it. + */ + EMPTY_SEQUENCE("Empty Sequence"), + INTEGER("Integer"), + /** + * Any sequence of nodes or atomic values, e.g. a single node, + * a list of nodes or a list of strings. + */ + SEQUENCE("Sequence"), + STRING("String"), + ; + + private final String displayValue; + + XsltDataType(final String displayValue) { + this.displayValue = displayValue; + } + + @JsonValue + @Override + public String getDisplayValue() { + return displayValue; + } +} diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionArg.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionArg.java new file mode 100644 index 0000000000..cc25734a7f --- /dev/null +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionArg.java @@ -0,0 +1,74 @@ +package stroom.pipeline.xsltfunctions; + +import stroom.query.language.functions.Val; +import stroom.query.language.functions.ValNumber; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface XsltFunctionArg { + + /** + * The name of the argument, typically lower camel case, e.g. inputValue. If the arg is + * varargs then the name should be singular as numbers will be appended to it automatically + * in the expression editor menus/snippets. + */ + @JsonProperty("name") + String name(); + + /** + * The type of the argument. Use the {@link Val} or {@link ValNumber} or interfaces + * if multiple types are supported. + */ + @JsonProperty("argType") + XsltDataType argType(); + + /** + * @return True if the argument is optional. As arguments are positional rather than named + * all arguments after an optional argument must also be optional. You can either make arguments + * optional or define multiple overloaded signatures. + */ + @JsonProperty("isOptional") + boolean isOptional() default false; + + /** + * @return True if this argument is a varargs parameter, i.e. arg... + */ + @JsonProperty("isVarargs") + boolean isVarargs() default false; + + /** + * If the argument is a varargs argument then this specifies the minimum number of arguments + * required. + */ + @JsonProperty("minVarargsCount") + int minVarargsCount() default 0; + + /** + * A description of the argument. + *

The description is assumed to be Markdown

+ */ + @JsonProperty("description") + String description() default ""; + + /** + * If the argument takes a finite set of values then specify them here. + */ + @JsonProperty("allowedValues") + String[] allowedValues() default {}; + + /** + * If the argument has a default value set it here. It can then be used as a default value + * for completion snippets and displayed in the menu help. + * Default value is a string as it may be another expression, e.g. 'null()' or a field '${EventId}'. + */ + @JsonProperty("defaultValue") + String defaultValue() default ""; + +} diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionCategory.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionCategory.java new file mode 100644 index 0000000000..3564be9801 --- /dev/null +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionCategory.java @@ -0,0 +1,58 @@ +package stroom.pipeline.xsltfunctions; + +import stroom.docref.HasDisplayValue; + +import com.fasterxml.jackson.annotation.JsonValue; + +public enum XsltFunctionCategory implements HasDisplayValue { + + CONVERSION( + "Conversion", + "Functions for converting a value from one form to another, e.g. hexToString."), + DATE( + "Date", + "Functions for date/time parsing and manipulation."), + STRING( + "String", + "Functions for string parsing and manipulation."), + URI( + "URI", + "Functions relating to parsing and manipulating URI/URLs."), + VALUE( + "Value", + "Functions that simply supply a value."), + NETWORK( + "Functions relating to networking (host names, IP addresses, etc.) or making remote calls.", + ""), + PIPELINE( + "Pipeline", + "Functions for obtaining information about the current pipeline process."), + OTHER( + "Other", + "Functions that don't fit into any other category."), + ; + + private final String displayValue; + private final String description; + + XsltFunctionCategory(final String displayValue, + final String description) { + this.displayValue = displayValue; + this.description = description; + } + + XsltFunctionCategory(final String displayValue) { + this.displayValue = displayValue; + this.description = null; + } + + @JsonValue + @Override + public String getDisplayValue() { + return displayValue; + } + + public String getDescription() { + return description; + } +} diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionDef.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionDef.java new file mode 100644 index 0000000000..648f7763ad --- /dev/null +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionDef.java @@ -0,0 +1,82 @@ +package stroom.pipeline.xsltfunctions; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface XsltFunctionDef { + + String UNDEFINED = "[UNDEFINED]"; + + @JsonProperty("name") + String name(); + + /** + * The html link anchor to the section of the documentation page for this function. + * It should only need to be set if the name when converted into anchor format differs + * from the actual anchor. + */ + @JsonProperty("helpAnchor") + String helpAnchor() default ""; + + /** + * Any alias names for the function + */ + @JsonProperty("aliases") + String[] aliases() default {}; + + /** + * The single category of functions that this function signature belongs to unless overridden at the + * signature level. + * Defined as an array to allow us to not have one by default. + */ + @JsonProperty("commonCategory") + XsltFunctionCategory[] commonCategory() default {}; + + /** + * An array of sub-categories that this function belongs to. The sub-categories represent a path + * in a tree of categories from root to leaf. E.g. if the main category is String the sub categories + * could be [Conversion, Case], i.e. String -> Conversion -> Case. + * Can be overridden at the signature level. + */ + @JsonProperty("commonSubCategories") + String[] commonSubCategories() default {}; + + /** + * A description of what the function does that is common to all signatures unless overridden + * at the signature level. + *

The description is assumed to be Markdown

+ */ + @JsonProperty("commonDescription") + String commonDescription() default ""; + + /** + * A single return type that is common to all signatures unless overridden at the signature level. + * You must specify either this or {@link XsltFunctionSignature#returnType()} + * Defined as an array to allow to be optional. + */ + @JsonProperty("commonReturnType") + XsltDataType[] commonReturnType() default {}; + + /** + * A return description that is common to all signatures unless overridden at the signature level + * You must specify either this or {@link XsltFunctionSignature#returnDescription()} + *

The description is assumed to be Markdown

+ */ + @JsonProperty("commonReturnDescription") + String commonReturnDescription() default ""; + + /** + * All the overloaded function signatures for the method, + * e.g. parseDate(dateStr) & parseDate(dateStr, format). + * Must have at least one signature. + */ + @JsonProperty("signatures") + XsltFunctionSignature[] signatures(); + +} diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionFactory.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionFactory.java new file mode 100644 index 0000000000..a3ab7be8a8 --- /dev/null +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionFactory.java @@ -0,0 +1,143 @@ +/* + * Copyright 2017 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.pipeline.xsltfunctions; + + +import io.github.classgraph.ClassGraph; +import io.github.classgraph.ScanResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; + +public class XsltFunctionFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(XsltFunctionFactory.class); + + // Hold them statically as we only want to scan the class path once + private static final Map> ALIAS_MAP = new HashMap<>(); + private static final Map, XsltFunctionDef> FUNCTION_DEF_MAP = + new HashMap<>(); + + static { +// scanClassPathForFunctions(); + } + + private static void scanClassPathForFunctions() { +// final StringBuilder functions = new StringBuilder(); + + // Scan the class path to find all the classes with @XsltFunctionDef + try (final ScanResult result = new ClassGraph() + .acceptPackages(StroomExtensionFunctionCall.class.getPackageName()) + .enableAnnotationInfo() + .enableClassInfo() + .ignoreClassVisibility() + .scan()) { + + result.getClassesWithAnnotation(XsltFunctionDef.class.getName()) + .forEach(classInfo -> { + + final Class clazz = classInfo.loadClass(); + if (StroomExtensionFunctionCall.class.isAssignableFrom(clazz)) { + final Class functionClazz = + (Class) clazz; + + final XsltFunctionDef functionDef = clazz.getAnnotation(XsltFunctionDef.class); + + FUNCTION_DEF_MAP.put(functionClazz, functionDef); + + // Add the class to our alias map for each name it has + Stream.concat(Stream.of(functionDef.name()), Stream.of(functionDef.aliases())) + .filter(Objects::nonNull) + .map(String::toLowerCase) + .forEach(name -> { +// functions.append(name); +// functions.append("|"); + + if (ALIAS_MAP.containsKey(name)) { + final Class existingClass = + ALIAS_MAP.get(name); + throw new RuntimeException(("Name/alias [" + name + + "] for class " + clazz.getName() + + " already exists for class " + + existingClass.getName())); + } + ALIAS_MAP.put(name, functionClazz); + }); + + + LOGGER.debug("Adding function {}", functionClazz.getName()); + } + }); + } + +// System.out.println(functions); + } + +// public static Function create(final ExpressionContext expressionContext, +// final String functionName) { +// final Class clazz = ALIAS_MAP.get(functionName.toLowerCase()); +// if (clazz != null) { +// try { +// final Constructor constructor = clazz +// .getConstructor(ExpressionContext.class, String.class); +// try { +// return constructor.newInstance(expressionContext, functionName); +// } catch (final InvocationTargetException +// | InstantiationException +// | IllegalAccessException e) { +// throw new RuntimeException(e.getMessage(), e); +// } +// } catch (final NoSuchMethodException e) { +// LOGGER.trace(e.getMessage(), e); +// } +// +// try { +// return clazz +// .getConstructor(String.class) +// .newInstance(functionName); +// } catch (final InvocationTargetException +// | InstantiationException +// | IllegalAccessException e) { +// throw new RuntimeException(e.getMessage(), e); +// } catch (final NoSuchMethodException e) { +// throw new RuntimeException(LogUtil.message( +// "Expecting to find a constructor like {}(expressionContext, functionName) " + +// "of {}(functionName). {}", +// clazz.getSimpleName(), +// clazz.getSimpleName(), +// LogUtil.exceptionMessage(e)), e); +// } +// } +// +// return null; +// } + +// public static Optional getFunctionDefinition( +// final Class clazz) { +// return Optional.ofNullable(FUNCTION_DEF_MAP.get(clazz)); +// } + + public static List getFunctionDefinitions() { + return new ArrayList<>(FUNCTION_DEF_MAP.values()); + } +} diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionSignature.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionSignature.java new file mode 100644 index 0000000000..23326f2d7a --- /dev/null +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/XsltFunctionSignature.java @@ -0,0 +1,56 @@ +package stroom.pipeline.xsltfunctions; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface XsltFunctionSignature { + + /** + * The single category of functions that this function signature belongs to. + * Defined as an array to allow us to not have one by default. + */ + @JsonProperty("category") + XsltFunctionCategory[] category() default {}; + + /** + * An array of sub-categories that this function belongs to. The sub-categories represent a path + * in a tree of categories from root to leaf. E.g. if the main category is String the sub categories + * could be [Conversion, Case], i.e. String -> Conversion -> Case. + */ + @JsonProperty("subCategories") + String[] subCategories() default {}; + + /** + * The description of what this signature of the function does. + * You must specify either this or {@link XsltFunctionDef#commonDescription()} + *

The description is assumed to be Markdown

+ */ + @JsonProperty("description") + String description() default ""; + + @JsonProperty("args") + XsltFunctionArg[] args(); + + /** + * The single return type of the function. + * You must specify either this or {@link XsltFunctionDef#commonReturnType()} + * Defined as an array to allow to be optional. + */ + @JsonProperty("returnType") + XsltDataType[] returnType() default {}; + + /** + * The description of what this signature of the function returns. + * You should specify either this or {@link XsltFunctionDef#commonReturnDescription()} + *

The description is assumed to be Markdown

+ */ + @JsonProperty("returnDescription") + String returnDescription() default ""; + +} diff --git a/stroom-pipeline/src/test/java/stroom/pipeline/xsltfunctions/TestXsltFunctions.java b/stroom-pipeline/src/test/java/stroom/pipeline/xsltfunctions/TestXsltFunctions.java new file mode 100644 index 0000000000..215456a3b8 --- /dev/null +++ b/stroom-pipeline/src/test/java/stroom/pipeline/xsltfunctions/TestXsltFunctions.java @@ -0,0 +1,86 @@ +package stroom.pipeline.xsltfunctions; + +import stroom.test.common.docs.StroomDocsUtil; +import stroom.util.logging.LambdaLogger; +import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.logging.LogUtil; + +import io.github.classgraph.ClassInfo; +import io.github.classgraph.ScanResult; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Not a test as such, more of a QA to ensure our XSLT functions have annotations on them so + * we can document them. + */ +public class TestXsltFunctions { + + private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(TestXsltFunctions.class); + + @Test + void findClassesWithNoAnnotation() { + StroomDocsUtil.doWithClassScanResult(scanResult -> { + final long count = streamFunctionClasses(scanResult) + .filter(classInfo -> !classInfo.hasAnnotation(XsltFunctionDef.class)) + .peek(classInfo -> { + final Class clazz = classInfo.loadClass(); + LOGGER.error("XSLT Function {} is missing annotation {}. Please add it", + clazz.getName(), + XsltFunctionDef.class.getName()); + }) + .count(); + + // TODO un-comment once all the annotations are added +// assertThat(count) +// .isZero(); + }); + } + + @Test + void checkFunctionAnnotations() { + StroomDocsUtil.doWithClassScanResult(scanResult -> { + final Map> nameToClassMap = new HashMap<>(); + final long count = streamFunctionClasses(scanResult) + .filter(classInfo -> classInfo.hasAnnotation(XsltFunctionDef.class)) + .peek(classInfo -> { + final Class clazz = classInfo.loadClass(); + final XsltFunctionDef anno = clazz.getAnnotation(XsltFunctionDef.class); + final String funcName = anno.name(); + assertThat(funcName) + .withFailMessage(() -> LogUtil.message( + "Function {} does not have a name in its {} annotation", + clazz.getName(), XsltFunctionDef.class)) + .isNotBlank(); + + assertThat(anno.signatures()) + .withFailMessage(() -> LogUtil.message( + "Function {} does not have any signatures in its {} annotation", + clazz.getName(), XsltFunctionDef.class)) + .isNotEmpty(); + + final Class prevVal = nameToClassMap.put(funcName, clazz); + if (prevVal != null) { + Assertions.fail( + "Function name '{}' used by at least two different classes {} and {}", + prevVal.getName(), clazz.getName()); + } + }) + .count(); + LOGGER.info("Found {} annotated XSLT functions", count); + }); + } + + private Stream streamFunctionClasses(final ScanResult scanResult) { + return scanResult.getSubclasses(StroomExtensionFunctionCall.class) + .stream() + .filter(classInfo -> !classInfo.isInterface()) + .filter(classInfo -> !classInfo.isAbstract()); + } +} diff --git a/stroom-test-common/src/main/java/stroom/test/common/docs/StroomDocsUtil.java b/stroom-test-common/src/main/java/stroom/test/common/docs/StroomDocsUtil.java index f83fc43426..a6af9da178 100644 --- a/stroom-test-common/src/main/java/stroom/test/common/docs/StroomDocsUtil.java +++ b/stroom-test-common/src/main/java/stroom/test/common/docs/StroomDocsUtil.java @@ -1,6 +1,7 @@ package stroom.test.common.docs; import stroom.test.common.ProjectPathUtil; +import stroom.util.concurrent.LazyValue; import stroom.util.io.DiffUtil; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; @@ -40,6 +41,11 @@ public class StroomDocsUtil { private static final String MARKER_START_HTML = ""; private static final String MARKER_END_HTML = ""; + private static final LazyValue STROOM_DOCS_REPO_DIR = LazyValue.initialisedBy( + StroomDocsUtil::getStroomDocsRepoDir); + +// private static volatile ClassInfoList classInfoList = null; + private StroomDocsUtil() { } @@ -161,6 +167,31 @@ private static void checkFileIsWritable(final Path file) { * is a regular file. */ public static Path resolveStroomDocsFile(final Path subPath) { + return resolveStroomDocsFile(subPath, true); + } + + /** + * @param subPath A path to a file in the stroom-docs repo that is relative to the + * stroom-docs repo root. + * @return An absolute path to the file which (if checkExists is true) has been + * tested to see if it exists and is a regular file + */ + public static Path resolveStroomDocsFile(final Path subPath, final boolean checkExists) { + final Path stroomDocsRepoDir = STROOM_DOCS_REPO_DIR.getValueWithLocks(); + final Path file = stroomDocsRepoDir.resolve(subPath) + .toAbsolutePath() + .normalize(); + + + if (checkExists && !Files.isRegularFile(file)) { + throw new RuntimeException(LogUtil.message("stroom-docs file '{}' does not exist", + stroomDocsRepoDir.toAbsolutePath() + .normalize())); + } + return file; + } + + private static Path getStroomDocsRepoDir() { final String stroomDocsRepoDirStr = System.getProperty(STROOM_DOCS_REPO_DIR_PROP_KEY); final Path stroomDocsRepoDir; @@ -179,14 +210,7 @@ public static Path resolveStroomDocsFile(final Path subPath) { stroomDocsRepoDir.toAbsolutePath().normalize(), STROOM_DOCS_REPO_DIR_PROP_KEY)); } - - final Path file = stroomDocsRepoDir.resolve(subPath).toAbsolutePath().normalize(); - - if (!Files.isRegularFile(file)) { - throw new RuntimeException(LogUtil.message("stroom-docs file '{}' does not exist", - stroomDocsRepoDir.toAbsolutePath().normalize())); - } - return file; + return stroomDocsRepoDir; } public static void doWithClassScanResult(final Consumer scanResultConsumer) { @@ -200,6 +224,24 @@ public static void doWithClassScanResult(final Consumer scanResultCo } } +// public static ClassInfoList getAllStroomClasses() { +// if (classInfoList == null) { +// synchronized (StroomDocsUtil.class) { +// if (classInfoList == null) { +// try (final ScanResult scanResult = +// new ClassGraph() +// .enableAllInfo() // Scan classes, methods, fields, annotations +// .acceptPackages(STROOM_PACKAGE_NAME) +// .scan()) { +// classInfoList = scanResult.getAllClasses(); +// } +// LOGGER.info("Found {} stroom classes", classInfoList.size()); +// } +// } +// } +// return classInfoList; +// } + // --------------------------------------------------------------------------------