FEMS Backports 2024-06 (2) (OpenEMS#2692)

- UI: prepare build for native Android App
- SymmetricEss: add Min-/MaxCellVoltage-/Temperature channels to modbus slave table
  - Added the following register to the Modbus slave table
  - MIN_CELL_VOLTAGE
  - MAX_CELL_VOLTAGE
  - MIN_CELL_TEMPERATURE
  - MAX_CELL_TEMPERATURE
- UI: add footer subnavigation
  - New subnavigation in /history/production
- HeatingElement: added JUnit test
- UI: allow SNAPSHOT update for OWNER/INSTALLER
  - Remove special handling for SNAPSHOT versions in update handler
- Tools: add internal tool to convert between JSONRPC and XLSX
- UI: add loading screen during authentication
  - Introduction of loading screen, shown during authentification process
  - Skip "Login"-screen if a valid authentication token exists
- EdgeConfig: remove unused factories & properties
  - Removed unused factories & factory properties from EdgeConfig
  - Added separate request to get all factories and properties for component update / installation
- UI: fix error messages shown in status.component
  - fix display of error message in status.component, was showing property channels as well
- Apps for DiscovergyMeter and  TimeInfluxDb
- SunSpec: set persistence priority VERY_LOW
- Backend improvements
  - Add logging for certain commands
  - Improve Metadata Odoo performance
  - Improve Postgres performance
    - reduce update time from 120 to 30 seconds
    - fix Sum-State update bug
  - InfluxDB:
    - extract channel filter to separate class; add unit tests
    - add configurable channel blacklist
    - improve FieldTypeConflictHandler; add unit tests
  - StringUtils: drop matchesIntegerPattern and matchesFloatPattern; replace with Guava
  - ThreadPoolUtils: fix possible NullPointerException
- ComponentManagerImpl: remove Thread Sleep
  - Remove test thread sleep in "GetChannel" handler
- UI: Channels fix setting values
  - Bug: "Cannot read properties of undefined (reading 'componentId')" when setting a value in the channels view
- Debian package: implement apt purge
- CI: Improve java build
- Debian: fix build
        ```bash
        + dpkg-deb -Zxz --build debian /woodpecker/src/git.intranet.fenecon.de/FENECON/fems/build/target/fems.debdpkg-deb: error: maintainer script 'postrm' has bad permissions 644 (must be >=0555 and <=0775)
        ```
- CI: split build image
  - Split android sdk into a seperate image.
  - add bash image to git
- Replace StringUtils.matches*Pattern

---------

Co-authored-by: Johann Kaufmann <[email protected]>
Co-authored-by: Kai Jeschek <[email protected]>
Co-authored-by: Lukas Rieger <[email protected]>
Co-authored-by: Manoj-Kumar Varikela <[email protected]>
Co-authored-by: Maximilian Lang <[email protected]>
Co-authored-by: Michael Grill <[email protected]>
Co-authored-by: Pooran Chandrashekaraiah <[email protected]>
Co-authored-by: Sebastian Asen <[email protected]>
Co-authored-by: Stefan Feilmeier <[email protected]>
Co-authored-by: Hueseyin Sahutoglu <[email protected]>

build.gradle

@@ -80,8 +80,8 @@ subprojects {
 				html.required = false
 				sarif.required = false
 			}
-			minHeapSize = "256m"
-			maxHeapSize = "1g"
+			minHeapSize = "512m"
+			maxHeapSize = "2048m"
 			// Exclude com.dalsemi.onewire
 			exclude '**/com/dalsemi/onewire/*'
 		}

cnf/pom.xml

@@ -9,8 +9,8 @@
 		<dependency>
 			<!-- Bnd Gradle Plugin For The Bnd Workspace -->
 			<!-- On update: also update gradle.properties file -->
-			<groupId>biz.aQute.bnd.workspace</groupId>
-			<artifactId>biz.aQute.bnd.workspace.gradle.plugin</artifactId>
+			<groupId>biz.aQute.bnd</groupId>
+			<artifactId>biz.aQute.bnd.gradle</artifactId>
 			<version>7.0.0</version>
 		</dependency>
 		<!-- com -->
@@ -262,6 +262,11 @@
 			<artifactId>fastexcel</artifactId>
 			<version>0.18.0</version>
 		</dependency>
+		<dependency>
+			<groupId>org.dhatim</groupId>
+			<artifactId>fastexcel-reader</artifactId>
+			<version>0.18.0</version>
+		</dependency>
 		<dependency>
 			<!-- Eclipse Paho MQTTv5 Client -->
 			<groupId>org.eclipse.paho</groupId>

codecov.yml

@@ -4,12 +4,12 @@ coverage:
   status:
     project:
       default:
-        target: '60%'
+        target: auto #default
 
 comment:
   layout: "condensed_header, diff"
   behavior: default
   require_changes: false
   require_base: false
   require_head: true
-  hide_project_coverage: true
+  hide_project_coverage: true

gradle.properties

@@ -10,4 +10,4 @@ maven.repo.local=cnf
 org.gradle.caching=true
 org.gradle.parallel=false
 org.gradle.workers.max=1
-org.gradle.jvmargs=-Xms256m -Xmx512m "-XX:MaxMetaspaceSize=256m"
+org.gradle.jvmargs=-Xms512m -Xmx2048m "-XX:MaxMetaspaceSize=512m"

io.openems.backend.application/BackendApp.bndrun

@@ -60,6 +60,7 @@
 
 -runbundles: \
 	Java-WebSocket;version='[1.5.4,1.5.5)',\
+	com.fasterxml.aalto-xml;version='[1.3.2,1.3.3)',\
 	com.google.gson;version='[2.11.0,2.11.1)',\
 	com.google.guava;version='[33.2.1,33.2.2)',\
 	com.google.guava.failureaccess;version='[1.0.2,1.0.3)',\
@@ -100,7 +101,9 @@
 	io.openems.wrapper.retrofit-converter-scalars;version=snapshot,\
 	io.openems.wrapper.retrofit2;version=snapshot,\
 	io.reactivex.rxjava3.rxjava;version='[3.1.8,3.1.9)',\
+	org.apache.commons.commons-compress;version='[1.26.1,1.26.2)',\
 	org.apache.commons.commons-csv;version='[1.10.0,1.10.1)',\
+	org.apache.commons.commons-io;version='[2.15.1,2.15.2)',\
 	org.apache.felix.configadmin;version='[1.9.26,1.9.27)',\
 	org.apache.felix.eventadmin;version='[1.6.4,1.6.5)',\
 	org.apache.felix.fileinstall;version='[3.7.4,3.7.5)',\
@@ -121,4 +124,5 @@
 	org.osgi.util.promise;version='[1.3.0,1.3.1)',\
 	org.owasp.encoder;version='[1.2.3,1.2.4)',\
 	org.postgresql.jdbc;version='[42.7.3,42.7.4)',\
-	reactive-streams;version='[1.0.4,1.0.5)'
+	reactive-streams;version='[1.0.4,1.0.5)',\
+	stax2-api;version='[4.2.0,4.2.1)'

io.openems.backend.core/src/io/openems/backend/core/jsonrpcrequesthandler/EdgeRpcRequestHandler.java

@@ -1,14 +1,23 @@
 package io.openems.backend.core.jsonrpcrequesthandler;
 
+import static io.openems.common.utils.JsonUtils.getAsOptionalBoolean;
+import static io.openems.common.utils.JsonUtils.getAsOptionalString;
+import static io.openems.common.utils.JsonUtils.getAsString;
+import static java.util.Collections.emptyMap;
+
+import java.util.Map;
+import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 
 import io.openems.backend.common.metadata.AppCenterHandler;
+import io.openems.backend.common.metadata.Metadata.GenericSystemLog;
 import io.openems.backend.common.metadata.User;
 import io.openems.common.exceptions.OpenemsError;
 import io.openems.common.exceptions.OpenemsError.OpenemsNamedException;
 import io.openems.common.jsonrpc.base.JsonrpcResponseSuccess;
 import io.openems.common.jsonrpc.request.AppCenterRequest;
+import io.openems.common.jsonrpc.request.ComponentJsonApiRequest;
 import io.openems.common.jsonrpc.request.EdgeRpcRequest;
 import io.openems.common.jsonrpc.request.GetEdgeConfigRequest;
 import io.openems.common.jsonrpc.request.QueryHistoricTimeseriesDataRequest;
@@ -49,39 +58,62 @@ protected CompletableFuture<EdgeRpcResponse> handleRequest(User user, UUID messa
 		}
 		user.assertEdgeRoleIsAtLeast(EdgeRpcRequest.METHOD, edgeRpcRequest.getEdgeId(), Role.GUEST);
 
-		CompletableFuture<? extends JsonrpcResponseSuccess> resultFuture;
-		switch (request.getMethod()) {
-		case AppCenterRequest.METHOD:
-			resultFuture = AppCenterHandler.handleUserRequest(this.parent.appCenterMetadata, //
-					t -> this.handleRequest(user, messageId, t), //
-					AppCenterRequest.from(request), user, edgeId);
-			break;
-
-		case QueryHistoricTimeseriesDataRequest.METHOD:
-			resultFuture = this.handleQueryHistoricDataRequest(edgeId, user,
-					QueryHistoricTimeseriesDataRequest.from(request));
-			break;
-
-		case QueryHistoricTimeseriesEnergyRequest.METHOD:
-			resultFuture = this.handleQueryHistoricEnergyRequest(edgeId, user,
-					QueryHistoricTimeseriesEnergyRequest.from(request));
-			break;
-
-		case QueryHistoricTimeseriesEnergyPerPeriodRequest.METHOD:
-			resultFuture = this.handleQueryHistoricEnergyPerPeriodRequest(edgeId, user,
-					QueryHistoricTimeseriesEnergyPerPeriodRequest.from(request));
-			break;
-
-		case QueryHistoricTimeseriesExportXlxsRequest.METHOD:
-			resultFuture = this.handleQueryHistoricTimeseriesExportXlxsRequest(edgeId, user,
-					QueryHistoricTimeseriesExportXlxsRequest.from(request));
-			break;
-
-		case GetEdgeConfigRequest.METHOD:
-			resultFuture = this.handleGetEdgeConfigRequest(edgeId, user, GetEdgeConfigRequest.from(request));
-			break;
-
-		default:
+		var resultFuture = switch (request.getMethod()) {
+		case AppCenterRequest.METHOD -> AppCenterHandler.handleUserRequest(this.parent.appCenterMetadata, //
+				t -> this.handleRequest(user, messageId, t), //
+				AppCenterRequest.from(request), user, edgeId);
+
+		case QueryHistoricTimeseriesDataRequest.METHOD ->
+			this.handleQueryHistoricDataRequest(edgeId, user, QueryHistoricTimeseriesDataRequest.from(request));
+
+		case QueryHistoricTimeseriesEnergyRequest.METHOD ->
+			this.handleQueryHistoricEnergyRequest(edgeId, user, QueryHistoricTimeseriesEnergyRequest.from(request));
+
+		case QueryHistoricTimeseriesEnergyPerPeriodRequest.METHOD -> this.handleQueryHistoricEnergyPerPeriodRequest(
+				edgeId, user, QueryHistoricTimeseriesEnergyPerPeriodRequest.from(request));
+
+		case QueryHistoricTimeseriesExportXlxsRequest.METHOD -> this.handleQueryHistoricTimeseriesExportXlxsRequest(
+				edgeId, user, QueryHistoricTimeseriesExportXlxsRequest.from(request));
+
+		case GetEdgeConfigRequest.METHOD ->
+			this.handleGetEdgeConfigRequest(edgeId, user, GetEdgeConfigRequest.from(request));
+
+		case ComponentJsonApiRequest.METHOD -> {
+			final var componentRequest = ComponentJsonApiRequest.from(request);
+			if (!"_host".equals(componentRequest.getComponentId())) {
+				yield null;
+			}
+			switch (componentRequest.getPayload().getMethod()) {
+			case "executeSystemCommand" -> {
+				final var executeSystemCommandRequest = componentRequest.getPayload();
+				final var p = executeSystemCommandRequest.getParams();
+				this.parent.metadata.logGenericSystemLog(new LogSystemExecuteCommend(edgeId, user, //
+						getAsString(p, "command"), //
+						getAsOptionalBoolean(p, "sudo").orElse(null), //
+						getAsOptionalString(p, "username").orElse(null), //
+						getAsOptionalString(p, "password").orElse(null), //
+						getAsOptionalBoolean(p, "runInBackground").orElse(null) //
+				));
+			}
+			case "executeSystemUpdate" -> {
+				this.parent.metadata.logGenericSystemLog(new LogUpdateSystem(edgeId, user));
+			}
+			case "executeSystemRestart" -> {
+				final var executeSystemCommandRequest = componentRequest.getPayload();
+				final var p = executeSystemCommandRequest.getParams();
+				this.parent.metadata.logGenericSystemLog(new LogRestartSystem(edgeId, user, //
+						getAsOptionalString(p, "type").orElse(null) //
+				));
+			}
+			}
+			yield null;
+		}
+
+		default -> null;
+		};
+
+		if (resultFuture == null) {
+			// Request not handled delegate to edge
 			resultFuture = this.parent.edgeWebsocket.send(edgeId, user, request);
 		}
 
@@ -100,6 +132,74 @@ protected CompletableFuture<EdgeRpcResponse> handleRequest(User user, UUID messa
 		return result;
 	}
 
+	private record LogSystemExecuteCommend(//
+			String edgeId, // non-null
+			User user, // non-null
+			String command, // non-null
+			Boolean sudo, // null-able
+			String username, // null-able
+			String password, // null-able
+			Boolean runInBackground // null-able
+	) implements GenericSystemLog {
+
+		@Override
+		public String teaser() {
+			return "Systemcommand: " + this.command;
+		}
+
+		@Override
+		public Map<String, String> getValues() {
+			return Map.of(//
+					"Sudo", Boolean.toString(Optional.ofNullable(this.sudo()).orElse(false)), //
+					"Command", this.command(), //
+					"Username", this.username(), //
+					"Password", this.password() == null || this.password().isEmpty() ? "[NOT_SET]" : "[SET]", //
+					"Run in Background", Boolean.toString(Optional.ofNullable(this.runInBackground()).orElse(false)) //
+			);
+		}
+
+	}
+
+	private record LogUpdateSystem(//
+			String edgeId, // non-null
+			User user // non-null
+	) implements GenericSystemLog {
+
+		@Override
+		public String teaser() {
+			return "Systemupdate";
+		}
+
+		@Override
+		public Map<String, String> getValues() {
+			return emptyMap();
+		}
+
+	}
+
+	private record LogRestartSystem(//
+			String edgeId, // non-null
+			User user, // non-null
+			String type // null-able
+	) implements GenericSystemLog {
+
+		@Override
+		public String teaser() {
+			return "Systemrestart";
+		}
+
+		@Override
+		public Map<String, String> getValues() {
+			if (this.type == null) {
+				return emptyMap();
+			}
+			return Map.of(//
+					"type", this.type //
+			);
+		}
+
+	}
+
 	/**
 	 * Handles a {@link QueryHistoricTimeseriesDataRequest}.
 	 *