merge to release

.github/workflows/codeql.yml

@@ -52,7 +52,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3
+      uses: github/codeql-action/init@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5
       with:        
         languages: ${{ matrix.language }}
         config-file: ${{ github.workspace }}/.github/codeql-config.yml        
@@ -83,6 +83,6 @@ jobs:
       run: cmake --build --preset linux-release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3
+      uses: github/codeql-action/analyze@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/docker-publish.yml

@@ -56,7 +56,7 @@ jobs:
 
       # Workaround: https://github.com/docker/build-push-action/issues/461
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
+        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c
 
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
@@ -84,7 +84,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
         env:
-          #DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: index
           CURRENT_DATETIME: ${{ env.CURRENT_DATETIME }}
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
@@ -119,10 +119,9 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
-          cache-to: type=gha,mode=max
-          #platforms: linux/amd64
+          cache-to: type=gha,mode=max          
           file: Dockerfile
-          #annotations: ${{ steps.meta.outputs.annotations }}                      
+          annotations: ${{ steps.meta.outputs.annotations }}                      
           build-args: |
             OWNER=nam20485
             GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}

.github/workflows/docker-scout-scan.yml

@@ -64,17 +64,7 @@ jobs:
 
       # Workaround: https://github.com/docker/build-push-action/issues/461
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-
-      # # Login against GHCR Docker registry except on PR
-      # # https://github.com/docker/login-action
-      # - name: Log into GHCR registry ${{ env.REGISTRY }}
-      #   if: github.event_name != 'pull_request'
-      #   uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
-      #   with:
-      #     registry: ${{ env.REGISTRY }}
-      #     username: ${{ github.actor }}
-      #     password: ${{ secrets.GITHUB_TOKEN }}
+        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c
 
       # Login against Docker Hub to allow running Docker Scout
       # https://github.com/docker/login-action
@@ -114,8 +104,8 @@ jobs:
             GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
             VCPKG_BINARY_SOURCES=clear;nuget,GitHub,readwrite
 
-      - name: List Local Images
-        run: docker image ls
+      # - name: List Local Images
+      #   run: docker image ls
 
     #   # Sign the resulting Docker image digest except on PRs.
     #   # This will only write to the public Rekor transparency log when the Docker
@@ -143,7 +133,7 @@ jobs:
 
       - name: Upload SARIF result
         id: upload-sarif
-        uses: github/codeql-action/upload-sarif@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3
+        uses: github/codeql-action/upload-sarif@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5
         with:
           sarif_file: sarif.output.json
 
@@ -159,3 +149,4 @@ jobs:
           only-severities: critical,high
           write-comment: true
           github-token: ${{ secrets.GITHUB_TOKEN }} # to be able to write the comment
+

.github/workflows/scorecard.yml

@@ -75,6 +75,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3
+        uses: github/codeql-action/upload-sarif@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5
         with:
           sarif_file: results.sarif

.gitignore

@@ -47,3 +47,4 @@ OdbDesignServer/PyOdbDesignLib/_PyOdbDesignLib.pyd
 deploy/kubeconfig
 Dockerfile.commentedOut
 scripts/create-release-invoker.js
+/compose-designs

Dockerfile

@@ -80,25 +80,26 @@ LABEL org.opencontainers.image.source=https://github.com/nam20485/OdbDesign \
 EXPOSE 8888
 
 RUN mkdir --parents /OdbDesign/bin
-WORKDIR /OdbDesign/bin
+WORKDIR /OdbDesign
 
 # copy binaries
-COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignLib/*.so .
-COPY --from=build /src/OdbDesign/out/build/linux-release/Utils/*.so .
-COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/OdbDesignServer .
-COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/*.so .
-COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignTests/OdbDesignTests .
+COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignLib/*.so ./bin/
+COPY --from=build /src/OdbDesign/out/build/linux-release/Utils/*.so ./bin/
+COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/OdbDesignServer ./bin/
+COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/*.so ./bin/
+COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignTests/OdbDesignTests ./bin/
 
 # copy templates directory
-RUN mkdir ./templates
+RUN mkdir -p ./templates
 COPY --from=build /src/OdbDesign/OdbDesignServer/templates/* ./templates
 
 # create designs directory
-RUN mkdir ./designs
+# required to be volume mounted!
+#RUN mkdir ./designs
 
 # run
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/OdbDesign/bin
 # ENV ODBDESIGN_SERVER_REQUEST_USERNAME=${ODBDESIGN_SERVER_REQUEST_USERNAME}
 # ENV ODBDESIGN_SERVER_REQUEST_PASSWORD=${ODBDESIGN_SERVER_REQUEST_PASSWORD}
-RUN chmod +x ./OdbDesignServer
-ENTRYPOINT [ "./OdbDesignServer", "--designs-dir", "./designs", "--templates-dir", "./templates" ]
+RUN chmod +x ./bin/OdbDesignServer
+ENTRYPOINT [ "./bin/OdbDesignServer", "--designs-dir", "./designs", "--templates-dir", "./templates" ]

OdbDesignServer/Controllers/FileUploadController.cpp

@@ -1,7 +1,9 @@
 #include "FileUploadController.h"
+#include "fastcopy.h"
 
 using namespace std::filesystem;
 using namespace Odb::Lib::App;
+using namespace Utils;
 
 namespace Odb::App::Server
 {
@@ -94,7 +96,7 @@
 
     crow::response FileUploadController::handleOctetStreamUpload(const std::string& filename, const crow::request& req)
     {
         const auto tempPath = temp_directory_path() / std::tmpnam(nullptr);
         std::ofstream outfile(tempPath, std::ofstream::binary);
         outfile << req.body;
         outfile.close();
@@ -104,7 +106,8 @@
 
         path finalPath(m_serverApp.args().designsDir());
         finalPath /= safeName;
-        rename(tempPath, finalPath);
+        //rename(tempPath, finalPath);        
+        auto ec = fastcopy(tempPath, finalPath, false);
 
         std::string responseBody = "{ \"filename\": \"" + safeName + "\" }";
 
@@ -124,11 +127,8 @@
             {
                 // log to debug and skip rest of the loop
                 CROW_LOG_DEBUG << " Value: " << part_value.body << '\n';
-                continue;
-            }
-            else
-            {
                 CROW_LOG_ERROR << "multipart/form-data POST failed! Part name was: [" << part_name << "], which is not supported. Part name should be [" << MULTIPART_FORMDATA_PART_NAME << "].";
+                continue;
             }
 
             // Extract the file name
@@ -160,7 +160,7 @@
             }
 
             // Create a new file with the extracted file name and write file contents to it
             const auto tempPath = temp_directory_path() / std::tmpnam(nullptr);
             std::ofstream out_file(tempPath);
             if (!out_file)
             {
@@ -173,7 +173,8 @@
             auto safeName = sanitizeFilename(outfile_name);
             path finalPath(m_serverApp.args().designsDir());
             finalPath /= safeName;
-            rename(tempPath, finalPath);
+            //rename(tempPath, finalPath);
+            auto ec = fastcopy(tempPath, finalPath, false);
 
             CROW_LOG_INFO << " Contents written to " << outfile_name << '\n';
         }

OdbDesignServer/Controllers/HealthCheckController.cpp

@@ -26,16 +26,16 @@ namespace Odb::App::Server
 
 	crow::response HealthCheckController::health_check_live(const crow::request& req)
 	{
-		return crow::response(crow::status::OK, "healthy: live");
+		return crow::response(crow::status::OK, "txt", "healthy: live");
 	}
 
 	crow::response HealthCheckController::health_check_ready(const crow::request& req)
 	{
-		return crow::response(crow::status::OK, "healthy: ready");
+		return crow::response(crow::status::OK, "txt", "healthy: ready");
 	}
 
 	crow::response HealthCheckController::health_check_started(const crow::request& req)
-	{
-		return crow::response(crow::status::OK, "healthy: started");
+	{		
+		return crow::response(crow::status::OK, "txt", "healthy: started");
 	}
 }

Utils/CMakeLists.txt

@@ -1,7 +1,7 @@
 # CMakeList.txt : CMake project for OdbDesignServer
 #
 
-add_library(Utils SHARED "utils_export.h"   "ExitCode.h" "ThreadSafeQueue.h" "WorkQueueLoopThread.h" "Logger.h" "Logger.cpp" "CommandLineArgs.h" "CommandLineArgs.cpp" "bin2ascii.h" "ArchiveExtractor.cpp" "ArchiveExtractor.h" "libarchive_extract.cpp" "libarchive_extract.h" "str_utils.cpp" "str_utils.h"  "IJsonable.h" "IJsonable.cpp" "CrowReturnable.h" "JsonCrowReturnable.h" "timestamp.h" "timestamp.cpp" "StopWatch.h" "StopWatch.cpp" "UrlEncoding.h" "UrlEncoding.cpp" "StringVector.h" "equals_within.h" "equals_within.cpp" "crow_win.h")
+add_library(Utils SHARED "utils_export.h"   "ExitCode.h" "ThreadSafeQueue.h" "WorkQueueLoopThread.h" "Logger.h" "Logger.cpp" "CommandLineArgs.h" "CommandLineArgs.cpp" "bin2ascii.h" "ArchiveExtractor.cpp" "ArchiveExtractor.h" "libarchive_extract.cpp" "libarchive_extract.h" "str_utils.cpp" "str_utils.h"  "IJsonable.h" "IJsonable.cpp" "CrowReturnable.h" "JsonCrowReturnable.h" "timestamp.h" "timestamp.cpp" "StopWatch.h" "StopWatch.cpp" "UrlEncoding.h" "UrlEncoding.cpp" "StringVector.h" "equals_within.h" "equals_within.cpp" "crow_win.h" "fastcopy.h" "fastcopy.cpp")
 
 # state that anybody linking to us needs to include the current source dir,
 # while we don't.

Utils/fastcopy.cpp

@@ -0,0 +1,59 @@
+#include "fastcopy.h"
+
+using namespace std;
+using namespace std::filesystem;
+
+namespace Utils
+{
+	error_code copy(const path& source, const path& dest, bool overwriteExisting)
+	{
+		error_code ec;
+
+		auto options = copy_options::none;
+		if (overwriteExisting)
+		{
+			options = copy_options::overwrite_existing;
+		}
+
+		if (copy_file(source, dest, options, ec))
+		{
+			remove(source, ec);
+		}
+
+		return ec;
+	}
+
+	error_code copy(const string& source, const string& dest, bool overwriteExisting)
+	{
+		return copy(path(source), path(dest), overwriteExisting);
+	}
+
+	error_code fastcopy(const string& source, const string& dest, bool overwriteExisting)
+	{
+		return fastcopy(path(source), path(dest), overwriteExisting);
+	}
+
+	error_code fastcopy(const path& source, const path& dest, bool overwriteExisting)
+	{
+		error_code ec;
+
+		try
+		{
+			rename(source, dest);
+		}
+		catch (filesystem_error& fe)
+		{
+			// can't rename across devices- try standard copy and remove
+			if (fe.code() == std::errc::cross_device_link)
+			{
+				ec = copy(source, dest, overwriteExisting);
+			}
+			else
+			{
+				throw fe;
+			}
+		}
+
+		return ec;
+	}
+}

Utils/fastcopy.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <filesystem>
+#include <string>
+#include "utils_export.h"
+
+namespace Utils
+{
+	UTILS_EXPORT std::error_code copy(const std::filesystem::path& source, const std::filesystem::path& dest, bool overwriteExisting);
+	UTILS_EXPORT std::error_code copy(const std::string& source, const std::string& dest, bool overwriteExisting);
+
+	UTILS_EXPORT std::error_code fastcopy(const std::string& source, const std::string& dest, bool overwriteExisting);
+	UTILS_EXPORT std::error_code fastcopy(const std::filesystem::path& source, const std::filesystem::path& dest, bool overwriteExisting);
+}

compose.yml

@@ -1,8 +1,8 @@
-name: odbdesignserver-swaggerui
+name: odbdesign
 
 services:
 
-  odbdesign-server:
+  server:
     ## enable for passing in branch name as an environment variable
     #environment:
     #  - BRANCH=nam20485 
@@ -12,7 +12,9 @@ services:
     # build:
     #   context: .
     #   dockerfile: Dockerfile
-    container_name: odbdesign-server    
+    #container_name: odbdesign-server   
+    volumes:
+      - ./compose-designs:/OdbDesign/designs
     ports:
       - 8888:8888
     environment:
@@ -26,9 +28,9 @@ services:
     #   - BRANCH=nam20485 
     # image: ghcr.io/nam20485/odbdesignserver-swaggerui:${BRANCH}-latest
     image: ghcr.io/nam20485/odbdesignserver-swaggerui:nam20485-latest
-    container_name: swagger-ui
+    #container_name: swagger-ui
     depends_on:
-      - odbdesign-server
+      - server
     ports:
       - 8080:8080    
 

deploy/kube/OdbDesignServer/deployment.yaml

@@ -19,14 +19,30 @@ spec:
       labels:
         app: odbdesign-server
         version: v1
-    spec:
+    spec:    
+      volumes:
+        - name: odbdesign-server-storage
+          persistentVolumeClaim:
+            claimName: k3d-volume-claim
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: NotIn
+                values:
+                  - "true"
       containers:
         - name: odbdesign-server
           image: ghcr.io/nam20485/odbdesign:nam20485-latest
           imagePullPolicy: Always
           ports:
             - containerPort: 8888
               name: ods-dep-port
+          volumeMounts:
+            - mountPath: /OdbDesign/designs
+              name: odbdesign-server-storage
           env:
             - name: ODBDESIGN_SERVER_REQUEST_USERNAME
               valueFrom: 
@@ -40,11 +56,11 @@ spec:
                   key: ODBDESIGN_SERVER_REQUEST_PASSWORD           
           resources:
             limits:
+              cpu: "2"
+              memory: 2Gi
+            requests:
               cpu: "1"
               memory: 1Gi
-            requests:
-              cpu: 500m
-              memory: 500Mi
           livenessProbe:
             httpGet:
               path: /healthz/live
@@ -64,4 +80,3 @@ spec:
             failureThreshold: 10
             periodSeconds: 1
             #initialDelaySeconds: 1          
-

deploy/kube/k3d-volume-pv.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: k3d-volume
+  #name: task-pv-volume
+  labels:
+    type: local
+spec:
+  storageClassName: manual
+  capacity:
+    storage: 1Ti
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /mnt/d/k3dvolume