diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b4c0d1ef..c7c26d68 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -52,7 +52,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3 + uses: github/codeql-action/init@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: languages: ${{ matrix.language }} config-file: ${{ github.workspace }}/.github/codeql-config.yml @@ -83,6 +83,6 @@ jobs: run: cmake --build --preset linux-release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3 + uses: github/codeql-action/analyze@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 90bee9e2..d96d89e3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -56,7 +56,7 @@ jobs: # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # Login against a Docker registry except on PR # https://github.com/docker/login-action @@ -84,7 +84,7 @@ jobs: id: meta uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 env: - #DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + DOCKER_METADATA_ANNOTATIONS_LEVELS: index CURRENT_DATETIME: ${{ env.CURRENT_DATETIME }} with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} @@ -119,10 +119,9 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha - cache-to: type=gha,mode=max - #platforms: linux/amd64 + cache-to: type=gha,mode=max file: Dockerfile - #annotations: ${{ steps.meta.outputs.annotations }} + annotations: ${{ steps.meta.outputs.annotations }} build-args: | OWNER=nam20485 GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docker-scout-scan.yml b/.github/workflows/docker-scout-scan.yml index 206ece88..c685f879 100644 --- a/.github/workflows/docker-scout-scan.yml +++ b/.github/workflows/docker-scout-scan.yml @@ -64,17 +64,7 @@ jobs: # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx - uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - - # # Login against GHCR Docker registry except on PR - # # https://github.com/docker/login-action - # - name: Log into GHCR registry ${{ env.REGISTRY }} - # if: github.event_name != 'pull_request' - # uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - # with: - # registry: ${{ env.REGISTRY }} - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # Login against Docker Hub to allow running Docker Scout # https://github.com/docker/login-action @@ -114,8 +104,8 @@ jobs: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} VCPKG_BINARY_SOURCES=clear;nuget,GitHub,readwrite - - name: List Local Images - run: docker image ls + # - name: List Local Images + # run: docker image ls # # Sign the resulting Docker image digest except on PRs. # # This will only write to the public Rekor transparency log when the Docker @@ -143,7 +133,7 @@ jobs: - name: Upload SARIF result id: upload-sarif - uses: github/codeql-action/upload-sarif@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3 + uses: github/codeql-action/upload-sarif@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: sarif_file: sarif.output.json @@ -159,3 +149,4 @@ jobs: only-severities: critical,high write-comment: true github-token: ${{ secrets.GITHUB_TOKEN }} # to be able to write the comment + \ No newline at end of file diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index e99065c0..f2af71e4 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -75,6 +75,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@379614612a29c9e28f31f39a59013eb8012a51f0 # v3.24.3 + uses: github/codeql-action/upload-sarif@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5 with: sarif_file: results.sarif diff --git a/.gitignore b/.gitignore index 8326ea80..1d795bf2 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,4 @@ OdbDesignServer/PyOdbDesignLib/_PyOdbDesignLib.pyd deploy/kubeconfig Dockerfile.commentedOut scripts/create-release-invoker.js +/compose-designs diff --git a/Dockerfile b/Dockerfile index 70093daa..6fae52b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,25 +80,26 @@ LABEL org.opencontainers.image.source=https://github.com/nam20485/OdbDesign \ EXPOSE 8888 RUN mkdir --parents /OdbDesign/bin -WORKDIR /OdbDesign/bin +WORKDIR /OdbDesign # copy binaries -COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignLib/*.so . -COPY --from=build /src/OdbDesign/out/build/linux-release/Utils/*.so . -COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/OdbDesignServer . -COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/*.so . -COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignTests/OdbDesignTests . +COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignLib/*.so ./bin/ +COPY --from=build /src/OdbDesign/out/build/linux-release/Utils/*.so ./bin/ +COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/OdbDesignServer ./bin/ +COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignServer/*.so ./bin/ +COPY --from=build /src/OdbDesign/out/build/linux-release/OdbDesignTests/OdbDesignTests ./bin/ # copy templates directory -RUN mkdir ./templates +RUN mkdir -p ./templates COPY --from=build /src/OdbDesign/OdbDesignServer/templates/* ./templates # create designs directory -RUN mkdir ./designs +# required to be volume mounted! +#RUN mkdir ./designs # run ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/OdbDesign/bin # ENV ODBDESIGN_SERVER_REQUEST_USERNAME=${ODBDESIGN_SERVER_REQUEST_USERNAME} # ENV ODBDESIGN_SERVER_REQUEST_PASSWORD=${ODBDESIGN_SERVER_REQUEST_PASSWORD} -RUN chmod +x ./OdbDesignServer -ENTRYPOINT [ "./OdbDesignServer", "--designs-dir", "./designs", "--templates-dir", "./templates" ] +RUN chmod +x ./bin/OdbDesignServer +ENTRYPOINT [ "./bin/OdbDesignServer", "--designs-dir", "./designs", "--templates-dir", "./templates" ] diff --git a/OdbDesignServer/Controllers/FileUploadController.cpp b/OdbDesignServer/Controllers/FileUploadController.cpp index 1d97cbe7..d2b133b7 100644 --- a/OdbDesignServer/Controllers/FileUploadController.cpp +++ b/OdbDesignServer/Controllers/FileUploadController.cpp @@ -1,7 +1,9 @@ #include "FileUploadController.h" +#include "fastcopy.h" using namespace std::filesystem; using namespace Odb::Lib::App; +using namespace Utils; namespace Odb::App::Server { @@ -104,7 +106,8 @@ namespace Odb::App::Server path finalPath(m_serverApp.args().designsDir()); finalPath /= safeName; - rename(tempPath, finalPath); + //rename(tempPath, finalPath); + auto ec = fastcopy(tempPath, finalPath, false); std::string responseBody = "{ \"filename\": \"" + safeName + "\" }"; @@ -124,11 +127,8 @@ namespace Odb::App::Server { // log to debug and skip rest of the loop CROW_LOG_DEBUG << " Value: " << part_value.body << '\n'; - continue; - } - else - { CROW_LOG_ERROR << "multipart/form-data POST failed! Part name was: [" << part_name << "], which is not supported. Part name should be [" << MULTIPART_FORMDATA_PART_NAME << "]."; + continue; } // Extract the file name @@ -173,7 +173,8 @@ namespace Odb::App::Server auto safeName = sanitizeFilename(outfile_name); path finalPath(m_serverApp.args().designsDir()); finalPath /= safeName; - rename(tempPath, finalPath); + //rename(tempPath, finalPath); + auto ec = fastcopy(tempPath, finalPath, false); CROW_LOG_INFO << " Contents written to " << outfile_name << '\n'; } diff --git a/OdbDesignServer/Controllers/HealthCheckController.cpp b/OdbDesignServer/Controllers/HealthCheckController.cpp index 4ba60b26..875ca269 100644 --- a/OdbDesignServer/Controllers/HealthCheckController.cpp +++ b/OdbDesignServer/Controllers/HealthCheckController.cpp @@ -26,16 +26,16 @@ namespace Odb::App::Server crow::response HealthCheckController::health_check_live(const crow::request& req) { - return crow::response(crow::status::OK, "healthy: live"); + return crow::response(crow::status::OK, "txt", "healthy: live"); } crow::response HealthCheckController::health_check_ready(const crow::request& req) { - return crow::response(crow::status::OK, "healthy: ready"); + return crow::response(crow::status::OK, "txt", "healthy: ready"); } crow::response HealthCheckController::health_check_started(const crow::request& req) - { - return crow::response(crow::status::OK, "healthy: started"); + { + return crow::response(crow::status::OK, "txt", "healthy: started"); } } diff --git a/Utils/CMakeLists.txt b/Utils/CMakeLists.txt index 64987dbe..04432a2d 100644 --- a/Utils/CMakeLists.txt +++ b/Utils/CMakeLists.txt @@ -1,7 +1,7 @@ # CMakeList.txt : CMake project for OdbDesignServer # -add_library(Utils SHARED "utils_export.h" "ExitCode.h" "ThreadSafeQueue.h" "WorkQueueLoopThread.h" "Logger.h" "Logger.cpp" "CommandLineArgs.h" "CommandLineArgs.cpp" "bin2ascii.h" "ArchiveExtractor.cpp" "ArchiveExtractor.h" "libarchive_extract.cpp" "libarchive_extract.h" "str_utils.cpp" "str_utils.h" "IJsonable.h" "IJsonable.cpp" "CrowReturnable.h" "JsonCrowReturnable.h" "timestamp.h" "timestamp.cpp" "StopWatch.h" "StopWatch.cpp" "UrlEncoding.h" "UrlEncoding.cpp" "StringVector.h" "equals_within.h" "equals_within.cpp" "crow_win.h") +add_library(Utils SHARED "utils_export.h" "ExitCode.h" "ThreadSafeQueue.h" "WorkQueueLoopThread.h" "Logger.h" "Logger.cpp" "CommandLineArgs.h" "CommandLineArgs.cpp" "bin2ascii.h" "ArchiveExtractor.cpp" "ArchiveExtractor.h" "libarchive_extract.cpp" "libarchive_extract.h" "str_utils.cpp" "str_utils.h" "IJsonable.h" "IJsonable.cpp" "CrowReturnable.h" "JsonCrowReturnable.h" "timestamp.h" "timestamp.cpp" "StopWatch.h" "StopWatch.cpp" "UrlEncoding.h" "UrlEncoding.cpp" "StringVector.h" "equals_within.h" "equals_within.cpp" "crow_win.h" "fastcopy.h" "fastcopy.cpp") # state that anybody linking to us needs to include the current source dir, # while we don't. diff --git a/Utils/fastcopy.cpp b/Utils/fastcopy.cpp new file mode 100644 index 00000000..f6fab078 --- /dev/null +++ b/Utils/fastcopy.cpp @@ -0,0 +1,59 @@ +#include "fastcopy.h" + +using namespace std; +using namespace std::filesystem; + +namespace Utils +{ + error_code copy(const path& source, const path& dest, bool overwriteExisting) + { + error_code ec; + + auto options = copy_options::none; + if (overwriteExisting) + { + options = copy_options::overwrite_existing; + } + + if (copy_file(source, dest, options, ec)) + { + remove(source, ec); + } + + return ec; + } + + error_code copy(const string& source, const string& dest, bool overwriteExisting) + { + return copy(path(source), path(dest), overwriteExisting); + } + + error_code fastcopy(const string& source, const string& dest, bool overwriteExisting) + { + return fastcopy(path(source), path(dest), overwriteExisting); + } + + error_code fastcopy(const path& source, const path& dest, bool overwriteExisting) + { + error_code ec; + + try + { + rename(source, dest); + } + catch (filesystem_error& fe) + { + // can't rename across devices- try standard copy and remove + if (fe.code() == std::errc::cross_device_link) + { + ec = copy(source, dest, overwriteExisting); + } + else + { + throw fe; + } + } + + return ec; + } +} \ No newline at end of file diff --git a/Utils/fastcopy.h b/Utils/fastcopy.h new file mode 100644 index 00000000..05d2a373 --- /dev/null +++ b/Utils/fastcopy.h @@ -0,0 +1,14 @@ +#pragma once + +#include +#include +#include "utils_export.h" + +namespace Utils +{ + UTILS_EXPORT std::error_code copy(const std::filesystem::path& source, const std::filesystem::path& dest, bool overwriteExisting); + UTILS_EXPORT std::error_code copy(const std::string& source, const std::string& dest, bool overwriteExisting); + + UTILS_EXPORT std::error_code fastcopy(const std::string& source, const std::string& dest, bool overwriteExisting); + UTILS_EXPORT std::error_code fastcopy(const std::filesystem::path& source, const std::filesystem::path& dest, bool overwriteExisting); +} diff --git a/compose.yml b/compose.yml index 0d1165e5..8fc03db3 100644 --- a/compose.yml +++ b/compose.yml @@ -1,8 +1,8 @@ -name: odbdesignserver-swaggerui +name: odbdesign services: - odbdesign-server: + server: ## enable for passing in branch name as an environment variable #environment: # - BRANCH=nam20485 @@ -12,7 +12,9 @@ services: # build: # context: . # dockerfile: Dockerfile - container_name: odbdesign-server + #container_name: odbdesign-server + volumes: + - ./compose-designs:/OdbDesign/designs ports: - 8888:8888 environment: @@ -26,9 +28,9 @@ services: # - BRANCH=nam20485 # image: ghcr.io/nam20485/odbdesignserver-swaggerui:${BRANCH}-latest image: ghcr.io/nam20485/odbdesignserver-swaggerui:nam20485-latest - container_name: swagger-ui + #container_name: swagger-ui depends_on: - - odbdesign-server + - server ports: - 8080:8080 \ No newline at end of file diff --git a/deploy/kube/OdbDesignServer/deployment.yaml b/deploy/kube/OdbDesignServer/deployment.yaml index e6cb55cf..08a2c691 100644 --- a/deploy/kube/OdbDesignServer/deployment.yaml +++ b/deploy/kube/OdbDesignServer/deployment.yaml @@ -19,7 +19,20 @@ spec: labels: app: odbdesign-server version: v1 - spec: + spec: + volumes: + - name: odbdesign-server-storage + persistentVolumeClaim: + claimName: k3d-volume-claim + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: NotIn + values: + - "true" containers: - name: odbdesign-server image: ghcr.io/nam20485/odbdesign:nam20485-latest @@ -27,6 +40,9 @@ spec: ports: - containerPort: 8888 name: ods-dep-port + volumeMounts: + - mountPath: /OdbDesign/designs + name: odbdesign-server-storage env: - name: ODBDESIGN_SERVER_REQUEST_USERNAME valueFrom: @@ -40,11 +56,11 @@ spec: key: ODBDESIGN_SERVER_REQUEST_PASSWORD resources: limits: + cpu: "2" + memory: 2Gi + requests: cpu: "1" memory: 1Gi - requests: - cpu: 500m - memory: 500Mi livenessProbe: httpGet: path: /healthz/live @@ -64,4 +80,3 @@ spec: failureThreshold: 10 periodSeconds: 1 #initialDelaySeconds: 1 - diff --git a/deploy/kube/k3d-volume-pv.yaml b/deploy/kube/k3d-volume-pv.yaml new file mode 100644 index 00000000..00b8cb2d --- /dev/null +++ b/deploy/kube/k3d-volume-pv.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: k3d-volume + #name: task-pv-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 1Ti + accessModes: + - ReadWriteOnce + hostPath: + path: /mnt/d/k3dvolume \ No newline at end of file diff --git a/deploy/kube/k3d-volume-pvc.yaml b/deploy/kube/k3d-volume-pvc.yaml new file mode 100644 index 00000000..853fa124 --- /dev/null +++ b/deploy/kube/k3d-volume-pvc.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + #name: task-pv-claim + name: k3d-volume-claim +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/deploy/kube/local-ingress.yaml b/deploy/kube/local-ingress.yaml index fa1b4d14..c0e59d4e 100644 --- a/deploy/kube/local-ingress.yaml +++ b/deploy/kube/local-ingress.yaml @@ -3,17 +3,7 @@ kind: Ingress metadata: name: odbdesign-server-ingress spec: - rules: - - host: ods.local - http: - paths: - - pathType: Prefix - path: "/" - backend: - service: - name: odbdesign-server-service - port: - name: ods-svc-port + rules: - host: precision5820 http: paths: diff --git a/scripts/create-k3d-cluster.ps1 b/scripts/create-k3d-cluster.ps1 index e0fea396..e988133c 100644 --- a/scripts/create-k3d-cluster.ps1 +++ b/scripts/create-k3d-cluster.ps1 @@ -11,7 +11,11 @@ param( # When set to true, the cluster will be deleted first [switch]$DeleteClusterFirst = $false, # When set to true, the cluster will be deleted without asking for confirmation - [switch]$ForceDelete = $false + [switch]$ForceDelete = $false, + # Host Volume Path for PersistentVolume + [Parameter(Mandatory=$true)] + [string]$HostVolumePath = "D:/k3dvolume" + ) # $clusterName="k3dcluster" @@ -46,6 +50,9 @@ k3d cluster create $ClusterName ` --agents $NumAgents ` --k3s-arg="--tls-san=${hostIp}@server:0" ` --k3s-arg="--tls-san=precision5820@server:0" ` - -p "${IngressHostPort}:80@loadbalancer" + --port "${IngressHostPort}:80@loadbalancer" ` + --port "8443:443@loadbalancer" ` + --volume ${HostVolumePath}:/k3dvolume@all + #--volume ${HostVolumePath}:/tmp/k3dvolume@all Write-Host "Cluster '$ClusterName' created." diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1 index 28d3e592..8a2bb299 100644 --- a/scripts/deploy.ps1 +++ b/scripts/deploy.ps1 @@ -14,10 +14,21 @@ if ($LASTEXITCODE -ne 0) { } # -# odbdesign-server +# Common (pre) # -# apply manifests +# persistent volume +kubectl apply -f deploy/kube/k3d-volume-pv.yaml +kubectl apply -f deploy/kube/k3d-volume-pvc.yaml + +# +# OdbDesignServer +# + +# secrets +& "$PSScriptRoot\odbdesign-server-request-secret.ps1" + +# apply deployment/service manifests kubectl apply -f deploy/kube/OdbDesignServer/deployment.yaml kubectl apply -f deploy/kube/OdbDesignServer/service.yaml @@ -30,7 +41,7 @@ kubectl rollout status deployment/$DeploymentName # Swagger UI # -# apply manifests +# apply deployment/service manifests kubectl apply -f deploy/kube/OdbDesignServer-SwaggerUI/deployment.yaml kubectl apply -f deploy/kube/OdbDesignServer-SwaggerUI/service.yaml @@ -39,7 +50,7 @@ kubectl rollout restart deployment/odbdesign-server-swaggerui-v1 kubectl rollout status deployment/odbdesign-server-swaggerui-v1 # -# common +# Common (post) # # apply ingress manifest diff --git a/scripts/odbdesign-server-request-secret.ps1 b/scripts/odbdesign-server-request-secret.ps1 new file mode 100644 index 00000000..7ab764ad --- /dev/null +++ b/scripts/odbdesign-server-request-secret.ps1 @@ -0,0 +1 @@ +kubectl create secret generic 'odbdesign-server-request-secret' --from-literal=ODBDESIGN_SERVER_REQUEST_USERNAME=$env:ODBDESIGN_SERVER_REQUEST_USERNAME --from-literal=ODBDESIGN_SERVER_REQUEST_PASSWORD=$env:ODBDESIGN_SERVER_REQUEST_PASSWORD \ No newline at end of file diff --git a/swagger/odbdesign-server-0.9-swagger.yaml b/swagger/odbdesign-server-0.9-swagger.yaml index 30e40eb7..e403b8d9 100644 --- a/swagger/odbdesign-server-0.9-swagger.yaml +++ b/swagger/odbdesign-server-0.9-swagger.yaml @@ -11,11 +11,12 @@ info: servers: - url: http://default-ingress-1165108808.us-west-2.elb.amazonaws.com - url: http://localhost:8888 + - url: http://precision5820:8081 tags: - name: "file upload" - name: "filemodel" #- name: "hello world" - #- name: "health check" + - name: "health check" - name: "steps" - name: "layers" - name: "symbols" @@ -564,15 +565,36 @@ paths: description: "" security: - BasicAuth: [] + /healthz/live: + get: + tags: ["health check"] + parameters: [] + responses: + "200": + description: "" + /healthz/started: + get: + tags: ["health check"] + parameters: [] + responses: + "200": + description: "" + /healthz/ready: + get: + tags: ["health check"] + parameters: [] + responses: + "200": + description: "" components: securitySchemes: BasicAuth: type: http scheme: basic - schemas: - odbdesign-server-schema: - type: object - properties: - id: - type: integer - format: int64 + # schemas: + # odbdesign-server-schema: + # type: object + # properties: + # id: + # type: integer + # format: int64