Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Creating processes using the Active MQ interface #6183

Merged
merged 20 commits into from
Nov 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Kitodo/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,12 @@
<artifactId>Saxon-HE</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-annotations</artifactId>
<version>${spotbugs-maven-plugin.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>se.jiderhamn.classloader-leak-prevention</groupId>
<artifactId>classloader-leak-prevention-servlet3</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,8 @@ public enum ParameterCore implements ParameterInterface {

ACTIVE_MQ_AUTH_PASSWORD(new Parameter<>("activeMQ.authPassword", "")),

ACTIVE_MQ_CREATE_NEW_PROCESSES_QUEUE(new Parameter<UndefinedParameter>("activeMQ.createNewProcesses.queue")),

ACTIVE_MQ_FINALIZE_STEP_QUEUE(new Parameter<UndefinedParameter>("activeMQ.finalizeStep.queue")),

ACTIVE_MQ_KITODO_SCRIPT_ALLOW(new Parameter<UndefinedParameter>("activeMQ.kitodoScript.allow")),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ public void verifyDocType() throws IOException, ProcessGenerationException {
if (docTypeMetadata.isPresent() && docTypeMetadata.get() instanceof MetadataEntry) {
String docType = ((MetadataEntry)docTypeMetadata.get()).getValue();
if (StringUtils.isNotBlank(docType)
&& !this.getWorkpiece().getLogicalStructure().getType().equals(docType)) {
&& !Objects.equals(this.getWorkpiece().getLogicalStructure().getType(), docType)) {
this.getWorkpiece().getLogicalStructure().setType(docType);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,11 @@ public class ActiveMQDirector implements Runnable, ServletContextListener {
private static final Logger logger = LogManager.getLogger(ActiveMQDirector.class);

// When implementing new services, add them to this list
private static Collection<? extends ActiveMQProcessor> services;
private static Collection<ActiveMQProcessor> services;

static {
services = Arrays.asList(new FinalizeStepProcessor(), new TaskActionProcessor(), new KitodoScriptProcessor());
services = Arrays.asList(new FinalizeStepProcessor(), new TaskActionProcessor(),
new CreateNewProcessesProcessor(), new KitodoScriptProcessor());
}

private static Connection connection = null;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,310 @@
/*
* (c) Kitodo. Key to digital objects e. V. <[email protected]>
*
* This file is part of the Kitodo project.
*
* It is licensed under GNU General Public License version 3 or later.
*
* For the full copyright and license information, please read the
* GPL3-License.txt file that was distributed with this source code.
*/

package org.kitodo.production.interfaces.activemq;

import edu.umd.cs.findbugs.annotations.CheckForNull;
import edu.umd.cs.findbugs.annotations.NonNull;
import edu.umd.cs.findbugs.annotations.Nullable;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;

import javax.jms.JMSException;

import org.apache.commons.lang3.tuple.Pair;
import org.kitodo.api.MdSec;
import org.kitodo.api.Metadata;
import org.kitodo.api.MetadataEntry;
import org.kitodo.api.MetadataGroup;
import org.kitodo.data.database.beans.ImportConfiguration;
import org.kitodo.data.database.beans.Process;
import org.kitodo.data.database.beans.Template;
import org.kitodo.data.database.exceptions.DAOException;
import org.kitodo.data.exceptions.DataException;
import org.kitodo.exceptions.ProcessorException;
import org.kitodo.production.dto.ProcessDTO;
import org.kitodo.production.services.ServiceManager;
import org.kitodo.production.services.data.ImportConfigurationService;

/**
* Order to create a new process. This contains all the necessary data.
*/
public class CreateNewProcessOrder {

/* Catalog imports can be specified (none, one or more). An
* "importconfiguration" and a search "value" must be specified. The search
* is carried out in the default search field. If no hit is found, or more
* than one, the search aborts with an error message. In the case of
* multiple imports, a repeated import is carried out according to the
* procedure specified in the rule set. */
private static final String FIELD_IMPORT = "import";
private static final String FIELD_IMPORT_CONFIG = "importconfiguration";
private static final String FIELD_IMPORT_VALUE = "value";

/* Additionally metadata can be passed. Passing multiple metadata or passing
* grouped metadata is also possible. */
private static final String FIELD_METADATA = "metadata";

/* A parent process can optionally be specified. The process ID or the
* process title can be specified. (If the value is all digits, it is
* considered the process ID, else it is considered the process title.) The
* process must be found in the client’s processes. If no parent process is
* specified, but a metadata entry with a use="higherLevelIdentifier" is
* included in the data from the catalog, the parent process is searched for
Comment on lines +67 to +69
Copy link
Collaborator

@BartChris BartChris Nov 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Edit:

I think i read the code wrong. We are probably always searching for a parent (even if important level is set to 1), sorry.

This is happening here:

// always try to find a parent for last imported process (e.g. level ==
// importDepth) in the database!
if (Objects.nonNull(parentID) && level == importDepth) {
checkForParent(parentID, template.getRuleset(), projectId);
}
.

However i am still wondering if we actually making the link to the parent process during the ActiveMQ import if we only provide metadata with a higherlevelidentifier. The parent process is searched while calling the importProcessHierarchy logic but i am not seeing the place in the ActiveMQ code, where we actually use the identified parent to establish the connection between child and the existing parent. I think this is missing from the code.

---outdated:
Is it really enough to provide a metadata import with a higherLevelIdentifier? In case a import configuration is provided, the import is explicitely called with an import level of 1 (IMPORT_WITHOUT_ANY_HIERARCHY):

 List<TempProcess> processHierarchy = importService.importProcessHierarchy(
                order.getImports().get(which).getValue(), order.getImports().get(which).getKey(),
                order.getProjectId(), order.getTemplateId(), IMPORT_WITHOUT_ANY_HIERARCHY,
                rulesetManagement.getFunctionalKeys(FunctionalMetadata.HIGHERLEVEL_IDENTIFIER));

This leads to the code for loading the parent to actually never be called.
https://github.com/kitodo/kitodo-production/blob/master/Kitodo/src/main/java/org/kitodo/production/services/data/ImportService.java#L552-L557

So we are not creating any parents as described in the spec, but we are also not searching for any parent by the recordIdentifier.

* using the metadata entry with use="recordIdentifier". It must already
* exist for the client. No parent process is implicitly created. The child
* process is added at the last position in the parent process. */
private static final String FIELD_PARENT = "parent";

// Mandatory information is the project ID.
private static final String FIELD_PROJECT = "project";

// Mandatory information is the process template.
private static final String FIELD_TEMPLATE = "template";

/* A process title can optionally be specified. If it is specified
* explicitly, exactly this process title is used, otherwise the system
* creates the process title according to the configured rule. The process
* title must still be unused for the client who owns the project. */
private static final String FIELD_TITLE = "title";

private final Integer projectId;
private final Integer templateId;
private final List<Pair<ImportConfiguration, String>> imports;
private final Optional<String> title;
private final Optional<Integer> parentId;
private final Collection<Metadata> metadata;

/**
* Creates a new CreateNewProcessOrder from an Active MQ message.
*
* @param ticket
* Active MQ message with (hopefully) all the data
* @throws DAOException
* if the ImportConfiguartionDAO is unable to find an import
* configuration with the given ID
* @throws DataException
* if there is an error accessing the search service
* @throws IllegalArgumentException
* If a required field is missing in the Active MQ message
* message, or contains inappropriate values.
* @throws JMSException
* Defined by the JMS API. I have not seen any cases where this
* would actually be thrown in the calls used here.
* @throws ProcessorException
* if the process count for the title is not exactly one
*/
CreateNewProcessOrder(MapMessageObjectReader ticket) throws DAOException, DataException, JMSException,
ProcessorException {
this.projectId = ticket.getMandatoryInteger(FIELD_PROJECT);
this.templateId = ticket.getMandatoryInteger(FIELD_TEMPLATE);
this.imports = convertImports(ticket.getList(FIELD_IMPORT));
this.title = Optional.ofNullable(ticket.getString(FIELD_TITLE));
this.parentId = Optional.ofNullable(convertProcessId(ticket.getString(FIELD_PARENT)));
this.metadata = convertMetadata(ticket.getMapOfString(FIELD_METADATA), MdSec.DMD_SEC);
}

/**
* Converts import details into safe data objects. For {@code null}, it will
* return an empty list, never {@code null}.
*
* @throws IllegalArgumentException
* if a list member is not a map, or one of the mandatory map
* entries is missing or of a wrong type
* @throws DAOException
* if the ImportConfiguartionDAO is unable to find an import
* configuration with that ID
*/
private static final List<Pair<ImportConfiguration, String>> convertImports(@Nullable List<?> imports)
throws DAOException {

if (Objects.isNull(imports) || imports.isEmpty()) {
return Collections.emptyList();
}

final ImportConfigurationService importConfigurationService = ServiceManager.getImportConfigurationService();
List<Pair<ImportConfiguration, String>> result = new ArrayList<>();
for (Object dubious : imports) {
if (!(dubious instanceof Map)) {
throw new IllegalArgumentException("Entry of \"imports\" is not a map");
}
Map<?, ?> map = (Map<?, ?>) dubious;
ImportConfiguration importconfiguration = importConfigurationService.getById(MapMessageObjectReader
.getMandatoryInteger(map, FIELD_IMPORT_CONFIG));
String value = MapMessageObjectReader.getMandatoryString(map, FIELD_IMPORT_VALUE);
result.add(Pair.of(importconfiguration, value));
}
return result;
}

/**
* Gets the process ID. If the string is an integer, it is used as the
* process ID. Otherwise it is considered a title and searched for. If it is
* a title, there must be exactly one process for it to be converted to an
* ID.
*
* @param processId
* parent process reference
* @return ID of the parent process
* @throws DataException
* if there is an error accessing the search service
* @throws ProcessorException
* if the process count for the title is not exactly one
*/
@CheckForNull
private static final Integer convertProcessId(String processId) throws DataException, ProcessorException {
if (Objects.isNull(processId)) {
return null;
}
if (processId.matches("\\d+")) {
return Integer.valueOf(processId);
matthias-ronge marked this conversation as resolved.
Show resolved Hide resolved
} else {
List<ProcessDTO> parents = ServiceManager.getProcessService().findByTitle(processId);
if (parents.size() == 0) {
throw new ProcessorException("Parent process not found");
} else if (parents.size() > 1) {
throw new ProcessorException("Parent process exists more than one");
} else {
return parents.get(0).getId();
}
}
}

/**
* Converts metadata details into safe data objects. For {@code null}, it
* will return an empty collection, never {@code null}.
*/
private static final HashSet<Metadata> convertMetadata(@Nullable Map<?, ?> metadata, @Nullable MdSec domain) {

HashSet<Metadata> result = new HashSet<>();
if (Objects.isNull(metadata)) {
return result;
}

for (Entry<?, ?> entry : metadata.entrySet()) {
Object dubiousKey = entry.getKey();
if (!(dubiousKey instanceof String) || ((String) dubiousKey).isEmpty()) {
throw new IllegalArgumentException("Invalid metadata key");
}
String key = (String) dubiousKey;

Object dubiousValuesList = entry.getValue();
if (!(dubiousValuesList instanceof List)) {
dubiousValuesList = Collections.singletonList(dubiousValuesList);
}
for (Object dubiousValue : (List<?>) dubiousValuesList) {
if (dubiousValue instanceof Map) {
MetadataGroup metadataGroup = new MetadataGroup();
metadataGroup.setKey(key);
metadataGroup.setMetadata(convertMetadata((Map<?, ?>) dubiousValue, null));
metadataGroup.setDomain(domain);
result.add(metadataGroup);
} else {
MetadataEntry metadataEntry = new MetadataEntry();
metadataEntry.setKey(key);
metadataEntry.setValue(dubiousValue.toString());
metadataEntry.setDomain(domain);
result.add(metadataEntry);
}
}
}
return result;
}

/**
* Returns the project ID. This is a mandatory field and can never be
* {@code null}.
*
* @return the project ID
*/
@NonNull
Integer getProjectId() {
return projectId;
}

/**
* Returns the production template.
*
* @return the template
* @throws DAOException
* if the template cannot be loaded
*/
Template getTemplate() throws DAOException {
return ServiceManager.getTemplateService().getById(templateId);
}

/**
* Returns the production template ID. This is a mandatory field and can
* never be {@code null}.
*
* @return the template ID
*/
@NonNull
Integer getTemplateId() {
return templateId;
}

/**
* Returns import instructions. Each instruction consists of an import
* configuration and a search value to be searched for in the default search
* field. Subsequent search statements must be executed as additive imports.
* Can be empty, but never {@code null}.
*
* @return import instructions
*/
@NonNull
List<Pair<ImportConfiguration, String>> getImports() {
return imports;
}

/**
* Returns an (optional) predefined title. If specified, this title must be
* used. Otherwise, the title must be formed using the formation rule. Can
* be {@code Optional.empty()}, but never {@code null}.
*
* @return the title, if any
*/
@NonNull
Optional<String> getTitle() {
return title;
}

/**
* Returns the parent process, if any.
*
* @return the parent process, or {@code null}
* @throws DAOException
* if the process cannot be loaded
*/
@CheckForNull
Process getParent() throws DAOException {
return parentId.isPresent() ? ServiceManager.getProcessService().getById(parentId.get()) : null;
}

/**
* Specifies the metadata for the logical structure root of the process to
* be created. Can be empty, but never {@code null}.
*
* @return the metadata
*/
@NonNull
Collection<Metadata> getMetadata() {
return metadata;
}
}
Loading
Loading