From c5ecff03adeaf6b42dfbe3e395069c5363f6fed9 Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Thu, 6 Jun 2024 11:07:11 -0500 Subject: [PATCH] Add s3 storage artifact route and ui integration of it chore: Update ArtifactPreview component to show loading spinner during fetch chore: Update ArtifactPreview component to show loading spinner during fetch --- backend/package-lock.json | 275 +++++++++++++++++- backend/package.json | 1 + backend/src/routes/api/storage/index.ts | 105 +++++++ .../src/routes/api/storage/storageUtils.ts | 185 ++++++++++++ backend/src/types.ts | 75 +++++ backend/src/utils/constants.ts | 1 + docs/dashboard-config.md | 4 +- frontend/src/__mocks__/mockDashboardConfig.ts | 3 + frontend/src/concepts/areas/const.ts | 4 + frontend/src/concepts/areas/types.ts | 7 +- .../useGetEventsByExecutionId.spec.ts | 4 +- .../mlmd/useGetEventsByExecutionId.ts | 21 +- .../content/artifacts/ArtifactUriLink.tsx | 94 ++++++ .../content/artifacts/__tests__/utils.spec.ts | 55 ++++ .../pipelines/content/artifacts/utils.ts | 37 +++ .../PipelineRunArtifactSelect.tsx | 2 +- .../markdown/MarkdownCompare.tsx | 52 +++- .../pipeline/PipelineTaskDetails.tsx | 4 +- .../pipelineRun/PipelineRunDetails.tsx | 7 +- .../SelectedNodeInputOutputTab.tsx | 4 +- .../artifacts/ArtifactNodeDetails.tsx | 26 +- .../artifacts/ArtifactNodeDrawerContent.tsx | 5 +- .../artifacts/ArtifactVisualization.tsx | 91 +++++- .../taskDetails/ArtifactPreview.tsx | 74 +++++ .../taskDetails/TaskDetailsInputOutput.tsx | 30 +- .../taskDetails/TaskDetailsPrintKeyValues.tsx | 3 +- .../topology/__tests__/parseUtils.spec.ts | 47 ++- .../concepts/pipelines/topology/parseUtils.ts | 90 ++++-- .../pipelines/topology/pipelineTaskTypes.ts | 1 + .../topology/usePipelineTaskTopology.tsx | 40 ++- frontend/src/k8sTypes.ts | 1 + .../ArtifactOverviewDetails.tsx | 5 +- .../experiments/artifacts/ArtifactsTable.tsx | 5 +- .../__tests__/ArtifactDetails.spec.tsx | 31 +- .../__tests__/ArtifactsTable.spec.tsx | 9 + .../compareRuns/CompareRunsMetricsSection.tsx | 21 +- .../executions/details/ExecutionDetails.tsx | 6 +- .../global/experiments/executions/utils.ts | 13 +- frontend/src/services/storageService.ts | 36 +++ ...dhdashboardconfigs.opendatahub.io.crd.yaml | 2 + .../odh-dashboard-config.yaml | 1 + 41 files changed, 1346 insertions(+), 131 deletions(-) create mode 100644 backend/src/routes/api/storage/index.ts create mode 100644 backend/src/routes/api/storage/storageUtils.ts create mode 100644 frontend/src/concepts/pipelines/content/artifacts/ArtifactUriLink.tsx create mode 100644 frontend/src/concepts/pipelines/content/artifacts/__tests__/utils.spec.ts create mode 100644 frontend/src/concepts/pipelines/content/artifacts/utils.ts create mode 100644 frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/ArtifactPreview.tsx create mode 100644 frontend/src/services/storageService.ts diff --git a/backend/package-lock.json b/backend/package-lock.json index 80d86d65cc..4731f4ce4d 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -22,6 +22,7 @@ "http-errors": "^1.8.0", "js-yaml": "^4.0.0", "lodash": "^4.17.21", + "minio": "^7.1.3", "pino": "^8.11.0", "prom-client": "^14.0.1", "ts-node": "^10.9.1" @@ -2450,6 +2451,12 @@ "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", "devOptional": true }, + "node_modules/@zxing/text-encoding": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@zxing/text-encoding/-/text-encoding-0.9.0.tgz", + "integrity": "sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==", + "optional": true + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -2741,6 +2748,11 @@ "node": ">=4" } }, + "node_modules/async": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -2758,7 +2770,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", - "optional": true, "engines": { "node": ">= 0.4" }, @@ -3026,6 +3037,27 @@ "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" }, + "node_modules/block-stream2": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/block-stream2/-/block-stream2-2.1.0.tgz", + "integrity": "sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==", + "dependencies": { + "readable-stream": "^3.4.0" + } + }, + "node_modules/block-stream2/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -3047,6 +3079,11 @@ "node": ">=8" } }, + "node_modules/browser-or-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", + "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==" + }, "node_modules/browserslist": { "version": "4.21.9", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.9.tgz", @@ -3123,6 +3160,14 @@ "ieee754": "^1.2.1" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -3180,7 +3225,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "optional": true, "dependencies": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -3535,6 +3579,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/decode-uri-component": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/decompress-response": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", @@ -5379,6 +5431,27 @@ "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-2.2.0.tgz", "integrity": "sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg==" }, + "node_modules/fast-xml-parser": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", + "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastify": { "version": "4.22.0", "resolved": "https://registry.npmjs.org/fastify/-/fastify-4.22.0.tgz", @@ -5570,6 +5643,14 @@ "node": ">=8" } }, + "node_modules/filter-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/find-my-way": { "version": "7.6.0", "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-7.6.0.tgz", @@ -5618,7 +5699,6 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "optional": true, "dependencies": { "is-callable": "^1.1.3" } @@ -5790,7 +5870,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", - "optional": true, "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -5956,7 +6035,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "optional": true, "dependencies": { "get-intrinsic": "^1.1.3" }, @@ -6078,7 +6156,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "optional": true, "engines": { "node": ">= 0.4" }, @@ -6090,7 +6167,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "optional": true, "dependencies": { "has-symbols": "^1.0.2" }, @@ -6431,6 +6507,21 @@ "node": ">= 0.10" } }, + "node_modules/is-arguments": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", + "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-array-buffer": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", @@ -6495,7 +6586,6 @@ "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "optional": true, "engines": { "node": ">= 0.4" }, @@ -6571,6 +6661,20 @@ "node": ">=6" } }, + "node_modules/is-generator-function": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", + "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -6698,7 +6802,6 @@ "version": "1.1.10", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.10.tgz", "integrity": "sha512-PJqgEHiWZvMpaFZ3uTc8kHPM4+4ADTlDniuQL7cU/UDA0Ql7F70yGfHph3cLNe+c9toaigv+DFzTJKhc2CtO6A==", - "optional": true, "dependencies": { "available-typed-arrays": "^1.0.5", "call-bind": "^1.0.2", @@ -8652,6 +8755,11 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "devOptional": true }, + "node_modules/json-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-stream/-/json-stream-1.0.0.tgz", + "integrity": "sha512-H/ZGY0nIAg3QcOwE1QN/rK/Fa7gJn7Ii5obwp6zyPO4xiPNwpIMjqy2gwjBEGqzkF/vSWEIBQCBuN19hYiL6Qg==" + }, "node_modules/json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", @@ -8941,6 +9049,38 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minio": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minio/-/minio-7.1.3.tgz", + "integrity": "sha512-xPrLjWkTT5E7H7VnzOjF//xBp9I40jYB4aWhb2xTFopXXfw+Wo82DDWngdUju7Doy3Wk7R8C4LAgwhLHHnf0wA==", + "dependencies": { + "async": "^3.2.4", + "block-stream2": "^2.1.0", + "browser-or-node": "^2.1.1", + "buffer-crc32": "^0.2.13", + "fast-xml-parser": "^4.2.2", + "ipaddr.js": "^2.0.1", + "json-stream": "^1.0.0", + "lodash": "^4.17.21", + "mime-types": "^2.1.35", + "query-string": "^7.1.3", + "through2": "^4.0.2", + "web-encoding": "^1.1.5", + "xml": "^1.0.1", + "xml2js": "^0.5.0" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, + "node_modules/minio/node_modules/ipaddr.js": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", + "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==", + "engines": { + "node": ">= 10" + } + }, "node_modules/minipass": { "version": "4.2.5", "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.5.tgz", @@ -9715,6 +9855,23 @@ "node": ">=0.6" } }, + "node_modules/query-string": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", + "integrity": "sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==", + "dependencies": { + "decode-uri-component": "^0.2.2", + "filter-obj": "^1.1.0", + "split-on-first": "^1.0.0", + "strict-uri-encode": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -10131,6 +10288,11 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/sax": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz", + "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==" + }, "node_modules/secure-json-parse": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", @@ -10310,6 +10472,14 @@ "source-map": "^0.6.0" } }, + "node_modules/split-on-first": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", + "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==", + "engines": { + "node": ">=6" + } + }, "node_modules/split2": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", @@ -10384,11 +10554,18 @@ "node": ">= 0.10.0" } }, + "node_modules/strict-uri-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", + "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==", + "engines": { + "node": ">=4" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -10515,6 +10692,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "node_modules/supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -10692,6 +10874,27 @@ "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", "optional": true }, + "node_modules/through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dependencies": { + "readable-stream": "3" + } + }, + "node_modules/through2/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/tiny-lru": { "version": "11.0.1", "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-11.0.1.tgz", @@ -11106,11 +11309,22 @@ "punycode": "^2.1.0" } }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "node_modules/uuid": { "version": "3.4.0", @@ -11182,6 +11396,17 @@ "makeerror": "1.0.12" } }, + "node_modules/web-encoding": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", + "integrity": "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==", + "dependencies": { + "util": "^0.12.3" + }, + "optionalDependencies": { + "@zxing/text-encoding": "0.9.0" + } + }, "node_modules/which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -11213,7 +11438,6 @@ "version": "1.1.9", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.9.tgz", "integrity": "sha512-w9c4xkx6mPidwp7180ckYWfMmvxpjlZuIudNtDf4N/tTAUB8VJbX25qZoAsrtGuYNnGw3pa0AXgbGKRB8/EceA==", - "optional": true, "dependencies": { "available-typed-arrays": "^1.0.5", "call-bind": "^1.0.2", @@ -11338,6 +11562,31 @@ } } }, + "node_modules/xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==" + }, + "node_modules/xml2js": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", + "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/backend/package.json b/backend/package.json index 10fe6d13fe..5a0dc59e4e 100644 --- a/backend/package.json +++ b/backend/package.json @@ -50,6 +50,7 @@ "http-errors": "^1.8.0", "js-yaml": "^4.0.0", "lodash": "^4.17.21", + "minio": "^7.1.3", "pino": "^8.11.0", "prom-client": "^14.0.1", "ts-node": "^10.9.1" diff --git a/backend/src/routes/api/storage/index.ts b/backend/src/routes/api/storage/index.ts new file mode 100644 index 0000000000..7281e58aab --- /dev/null +++ b/backend/src/routes/api/storage/index.ts @@ -0,0 +1,105 @@ +import { FastifyInstance, FastifyReply } from 'fastify'; +import { getObjectSize, getObjectStream, setupMinioClient } from './storageUtils'; +import { getDashboardConfig } from '../../../utils/resourceUtils'; +import { getDirectCallOptions } from '../../../utils/directCallUtils'; +import { getAccessToken } from '../../../utils/directCallUtils'; +import { OauthFastifyRequest } from '../../../types'; + +export default async (fastify: FastifyInstance): Promise => { + fastify.get( + '/:namespace/size', + async ( + request: OauthFastifyRequest<{ + Querystring: { key: string }; + Params: { namespace: string }; + }>, + reply: FastifyReply, + ) => { + try { + const dashConfig = getDashboardConfig(); + if (dashConfig?.spec.dashboardConfig.disableS3Endpoint !== false) { + reply.code(404).send('Not found'); + return reply; + } + + const { namespace } = request.params; + const { key } = request.query; + + const requestOptions = await getDirectCallOptions(fastify, request, request.url); + const token = getAccessToken(requestOptions); + + const { client, bucket } = await setupMinioClient(fastify, token, namespace); + + const size = await getObjectSize({ + client, + key, + bucket, + }); + + reply.send(size); + } catch (err) { + reply.code(500).send(err.message); + return reply; + } + }, + ); + + fastify.get( + '/:namespace', + async ( + request: OauthFastifyRequest<{ + Querystring: { key: string; peek?: number }; + Params: { namespace: string }; + }>, + reply: FastifyReply, + ) => { + try { + const dashConfig = getDashboardConfig(); + if (dashConfig?.spec.dashboardConfig.disableS3Endpoint !== false) { + reply.code(404).send('Not found'); + return reply; + } + + const { namespace } = request.params; + const { key, peek } = request.query; + + const requestOptions = await getDirectCallOptions(fastify, request, request.url); + const token = getAccessToken(requestOptions); + + const { client, bucket } = await setupMinioClient(fastify, token, namespace); + + const stream = await getObjectStream({ + client, + key, + bucket, + peek, + }); + + reply.type('text/plain'); + + await new Promise((resolve, reject) => { + stream.on('data', (chunk) => { + reply.raw.write(chunk); + }); + + stream.on('end', () => { + reply.raw.end(); + resolve(); + }); + + stream.on('error', (err) => { + fastify.log.error('Stream error:', err); + reply.raw.statusCode = 500; + reply.raw.end(err.message); + reject(err); + }); + }); + + return; + } catch (err) { + reply.code(500).send(err.message); + return reply; + } + }, + ); +}; diff --git a/backend/src/routes/api/storage/storageUtils.ts b/backend/src/routes/api/storage/storageUtils.ts new file mode 100644 index 0000000000..d33e88c0ea --- /dev/null +++ b/backend/src/routes/api/storage/storageUtils.ts @@ -0,0 +1,185 @@ +import { Client as MinioClient } from 'minio'; +import { DSPipelineKind, KubeFastifyInstance } from '../../../types'; +import { Transform, TransformOptions } from 'stream'; + +export interface PreviewStreamOptions extends TransformOptions { + peek: number; +} + +/** + * Transform stream that only stream the first X number of bytes. + */ +export class PreviewStream extends Transform { + constructor({ peek, ...opts }: PreviewStreamOptions) { + // acts like passthrough + let transform: TransformOptions['transform'] = (chunk, _encoding, callback) => + callback(undefined, chunk); + // implements preview - peek must be positive number + if (peek && peek > 0) { + let size = 0; + transform = (chunk, _encoding, callback) => { + const delta = peek - size; + size += chunk.length; + if (size >= peek) { + callback(undefined, chunk.slice(0, delta)); + this.resume(); // do not handle any subsequent data + return; + } + callback(undefined, chunk); + }; + } + super({ ...opts, transform }); + } +} + +export async function getDspa( + fastify: KubeFastifyInstance, + token: string, + namespace: string, +): Promise { + const dspaResponse = await fastify.kube.customObjectsApi + .listNamespacedCustomObject( + 'datasciencepipelinesapplications.opendatahub.io', + 'v1alpha1', + namespace, + 'datasciencepipelinesapplications', + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + { + headers: { + authorization: `Bearer ${token}`, + }, + }, + ) + .catch((e) => { + throw `A ${e.statusCode} error occurred when trying to fetch dspa aws storage credentials: ${ + e.response?.body?.message || e?.response?.statusMessage + }`; + }); + + const dspas = ( + dspaResponse?.body as { + items: DSPipelineKind[]; + } + )?.items; + + if (!dspas || !dspas.length) { + throw 'No Data Science Pipeline Application found'; + } + + return dspas[0]; +} + +async function getDspaSecretKeys( + fastify: KubeFastifyInstance, + token: string, + namespace: string, + dspa: DSPipelineKind, +): Promise<{ accessKey: string; secretKey: string }> { + try { + const secret = await fastify.kube.coreV1Api.readNamespacedSecret( + dspa.spec.objectStorage.externalStorage.s3CredentialsSecret.secretName, + namespace, + undefined, + undefined, + undefined, + { + headers: { + authorization: `Bearer ${token}`, + }, + }, + ); + + const accessKey = atob( + secret.body.data[dspa.spec.objectStorage.externalStorage.s3CredentialsSecret.accessKey], + ); + const secretKey = atob( + secret.body.data[dspa.spec.objectStorage.externalStorage.s3CredentialsSecret.secretKey], + ); + + if (!accessKey || !secretKey) { + throw 'Access key or secret key is empty'; + } + + return { accessKey, secretKey }; + } catch (err) { + console.error('Unable to get dspa secret keys: ', err); + throw new Error('Unable to get dspa secret keys: ' + err); + } +} + +/** + * Create minio client with aws instance profile credentials if needed. + * @param config minio client options where `accessKey` and `secretKey` are optional. + */ +export async function setupMinioClient( + fastify: KubeFastifyInstance, + token: string, + namespace: string, +): Promise<{ client: MinioClient; bucket: string }> { + try { + const dspa = await getDspa(fastify, token, namespace); + + // check if object storage connection is available + if ( + !dspa.status?.conditions?.find((c) => c.type === 'APIServerReady' && c.status === 'True') || + !dspa.status?.conditions?.find( + (c) => c.type === 'ObjectStoreAvailable' && c.status === 'True', + ) + ) { + throw 'Object store is not available'; + } + + const externalStorage = dspa.spec.objectStorage.externalStorage; + if (externalStorage) { + const { region, host: endPoint, bucket } = externalStorage; + const { accessKey, secretKey } = await getDspaSecretKeys(fastify, token, namespace, dspa); + return { + client: new MinioClient({ accessKey, secretKey, endPoint, region }), + bucket, + }; + } + } catch (err) { + console.error('Unable to create minio client: ', err); + throw new Error('Unable to create minio client: ' + err); + } +} + +/** MinioRequestConfig describes the info required to retrieve an artifact. */ +export interface MinioRequestConfig { + bucket: string; + key: string; + client: MinioClient; + peek?: number; +} + +/** + * Returns a stream from an object in a s3 compatible object store (e.g. minio). + * + * @param param.bucket Bucket name to retrieve the object from. + * @param param.key Key of the object to retrieve. + * @param param.client Minio client. + * @param param.peek Number of bytes to preview. + * + */ +export async function getObjectStream({ + key, + client, + bucket, + peek = 1e8, // 100mb +}: MinioRequestConfig): Promise { + const safePeek = Math.min(peek, 1e8); // 100mb + const stream = await client.getObject(bucket, key); + return stream.pipe(new PreviewStream({ peek: safePeek })); +} + +export async function getObjectSize({ bucket, key, client }: MinioRequestConfig): Promise { + const stat = await client.statObject(bucket, key); + return stat.size; +} diff --git a/backend/src/types.ts b/backend/src/types.ts index 692cc9cf4d..586f412244 100644 --- a/backend/src/types.ts +++ b/backend/src/types.ts @@ -37,6 +37,7 @@ export type DashboardConfig = K8sResourceCommon & { disableModelMesh: boolean; disableAcceleratorProfiles: boolean; disablePipelineExperiments: boolean; + disableS3Endpoint: boolean; disableDistributedWorkloads: boolean; disableModelRegistry: boolean; }; @@ -1014,9 +1015,83 @@ export type K8sCondition = { lastHeartbeatTime?: string; }; +export type DSPipelineExternalStorageKind = { + bucket: string; + host: string; + port?: ''; + scheme: string; + region: string; + s3CredentialsSecret: { + accessKey: string; + secretKey: string; + secretName: string; + }; +}; + export type DSPipelineKind = K8sResourceCommon & { + metadata: { + name: string; + namespace: string; + }; spec: { dspVersion: string; + apiServer?: Partial<{ + apiServerImage: string; + artifactImage: string; + artifactScriptConfigMap: Partial<{ + key: string; + name: string; + }>; + enableSamplePipeline: boolean; + }>; + database?: Partial<{ + externalDB: Partial<{ + host: string; + passwordSecret: Partial<{ + key: string; + name: string; + }>; + pipelineDBName: string; + port: string; + username: string; + }>; + image: string; + mariaDB: Partial<{ + image: string; + passwordSecret: Partial<{ + key: string; + name: string; + }>; + pipelineDBName: string; + username: string; + }>; + }>; + mlpipelineUI?: { + configMap?: string; + image: string; + }; + persistentAgent?: Partial<{ + image: string; + pipelineAPIServerName: string; + }>; + scheduledWorkflow?: Partial<{ + image: string; + }>; + objectStorage: Partial<{ + externalStorage: DSPipelineExternalStorageKind; + minio: Partial<{ + bucket: string; + image: string; + s3CredentialsSecret: Partial<{ + accessKey: string; + secretKey: string; + secretName: string; + }>; + }>; + }>; + viewerCRD?: Partial<{ + image: string; + }>; }; status?: { conditions?: K8sCondition[]; diff --git a/backend/src/utils/constants.ts b/backend/src/utils/constants.ts index 5e2991db93..971f64ba0a 100644 --- a/backend/src/utils/constants.ts +++ b/backend/src/utils/constants.ts @@ -62,6 +62,7 @@ export const blankDashboardCR: DashboardConfig = { disableModelMesh: false, disableAcceleratorProfiles: false, disablePipelineExperiments: true, + disableS3Endpoint: true, disableDistributedWorkloads: false, disableModelRegistry: true, }, diff --git a/docs/dashboard-config.md b/docs/dashboard-config.md index a17994c6d7..01324560e3 100644 --- a/docs/dashboard-config.md +++ b/docs/dashboard-config.md @@ -61,7 +61,8 @@ spec: disableKServeMetrics: true disableBiasMetrics: false disablePerformanceMetrics: false - disablePipelineExperiments: false + disablePipelineExperiments: true + disableS3Endpoint: true disableDistributedWorkloads: false ``` @@ -157,6 +158,7 @@ spec: disableBiasMetrics: false disablePerformanceMetrics: false disablePipelineExperiments: true + disableS3Endpoint: true notebookController: enabled: true gpuSetting: autodetect diff --git a/frontend/src/__mocks__/mockDashboardConfig.ts b/frontend/src/__mocks__/mockDashboardConfig.ts index e90471c3f3..cdee3e82ae 100644 --- a/frontend/src/__mocks__/mockDashboardConfig.ts +++ b/frontend/src/__mocks__/mockDashboardConfig.ts @@ -22,6 +22,7 @@ type MockDashboardConfigType = { disablePerformanceMetrics?: boolean; disableBiasMetrics?: boolean; disablePipelineExperiments?: boolean; + disableS3Endpoint?: boolean; disableDistributedWorkloads?: boolean; disableModelRegistry?: boolean; disableNotebookController?: boolean; @@ -49,6 +50,7 @@ export const mockDashboardConfig = ({ disablePerformanceMetrics = false, disableBiasMetrics = false, disablePipelineExperiments = true, + disableS3Endpoint = true, disableDistributedWorkloads = false, disableModelRegistry = true, disableNotebookController = false, @@ -87,6 +89,7 @@ export const mockDashboardConfig = ({ disableModelMesh, disableAcceleratorProfiles, disablePipelineExperiments, + disableS3Endpoint, disableDistributedWorkloads, disableModelRegistry, }, diff --git a/frontend/src/concepts/areas/const.ts b/frontend/src/concepts/areas/const.ts index e386bf0829..10834bdb4a 100644 --- a/frontend/src/concepts/areas/const.ts +++ b/frontend/src/concepts/areas/const.ts @@ -73,6 +73,10 @@ export const SupportedAreasStateMap: SupportedAreasState = { featureFlags: ['disablePipelineExperiments'], reliantAreas: [SupportedArea.DS_PIPELINES], }, + [SupportedArea.S3_ENDPOINT]: { + featureFlags: ['disableS3Endpoint'], + reliantAreas: [SupportedArea.DS_PIPELINES], + }, [SupportedArea.DISTRIBUTED_WORKLOADS]: { featureFlags: ['disableDistributedWorkloads'], requiredComponents: [StackComponent.KUEUE], diff --git a/frontend/src/concepts/areas/types.ts b/frontend/src/concepts/areas/types.ts index b912330ec7..acb81d32d9 100644 --- a/frontend/src/concepts/areas/types.ts +++ b/frontend/src/concepts/areas/types.ts @@ -24,12 +24,16 @@ export enum SupportedArea { HOME = 'home', /* Standalone areas */ - DS_PIPELINES = 'ds-pipelines', // TODO: Jupyter Tile Support? (outside of feature flags today) WORKBENCHES = 'workbenches', // TODO: Support Applications/Tile area // TODO: Support resources area + /* Pipelines areas */ + DS_PIPELINES = 'ds-pipelines', + PIPELINE_EXPERIMENTS = 'pipeline-experiments', + S3_ENDPOINT = 's3-endpoint', + /* Admin areas */ BYON = 'bring-your-own-notebook', CLUSTER_SETTINGS = 'cluster-settings', @@ -50,7 +54,6 @@ export enum SupportedArea { BIAS_METRICS = 'bias-metrics', PERFORMANCE_METRICS = 'performance-metrics', TRUSTY_AI = 'trusty-ai', - PIPELINE_EXPERIMENTS = 'pipeline-experiments', /* Distributed Workloads areas */ DISTRIBUTED_WORKLOADS = 'distributed-workloads', diff --git a/frontend/src/concepts/pipelines/apiHooks/mlmd/__tests__/useGetEventsByExecutionId.spec.ts b/frontend/src/concepts/pipelines/apiHooks/mlmd/__tests__/useGetEventsByExecutionId.spec.ts index c674dcb2dd..284afecaaf 100644 --- a/frontend/src/concepts/pipelines/apiHooks/mlmd/__tests__/useGetEventsByExecutionId.spec.ts +++ b/frontend/src/concepts/pipelines/apiHooks/mlmd/__tests__/useGetEventsByExecutionId.spec.ts @@ -54,7 +54,7 @@ describe('useGetEventsByExecutionId', () => { await renderResult.waitForNextUpdate(); expect(renderResult.result.current).toStrictEqual( - standardUseFetchState(mockEventsResponse, true), + standardUseFetchState(mockEventsResponse.getEventsList(), true), ); expect(renderResult).hookToHaveUpdateCount(2); @@ -108,7 +108,7 @@ describe('useGetEventsByExecutionIds', () => { await renderResult.waitForNextUpdate(); expect(renderResult.result.current).toStrictEqual( - standardUseFetchState(mockEventsResponse, true), + standardUseFetchState(mockEventsResponse.getEventsList(), true), ); expect(renderResult).hookToHaveUpdateCount(2); diff --git a/frontend/src/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId.ts b/frontend/src/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId.ts index f1bfabd797..143d6cea0b 100644 --- a/frontend/src/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId.ts +++ b/frontend/src/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId.ts @@ -1,33 +1,24 @@ import React from 'react'; import { usePipelinesAPI } from '~/concepts/pipelines/context'; -import { - GetEventsByExecutionIDsRequest, - GetEventsByExecutionIDsResponse, -} from '~/third_party/mlmd'; +import { GetEventsByExecutionIDsRequest, Event } from '~/third_party/mlmd'; import useFetchState, { FetchState, FetchStateCallbackPromise } from '~/utilities/useFetchState'; -export const useGetEventsByExecutionId = ( - executionId: number, -): FetchState => { - const ids = React.useMemo(() => [executionId], [executionId]); +export const useGetEventsByExecutionId = (executionId?: number): FetchState => { + const ids = React.useMemo(() => (executionId !== undefined ? [executionId] : []), [executionId]); return useGetEventsByExecutionIds(ids); }; -export const useGetEventsByExecutionIds = ( - executionIds: number[], -): FetchState => { +export const useGetEventsByExecutionIds = (executionIds: number[]): FetchState => { const { metadataStoreServiceClient } = usePipelinesAPI(); - const call = React.useCallback< - FetchStateCallbackPromise - >(async () => { + const call = React.useCallback>(async () => { const request = new GetEventsByExecutionIDsRequest(); request.setExecutionIdsList(executionIds); const response = await metadataStoreServiceClient.getEventsByExecutionIDs(request); - return response; + return response.getEventsList(); }, [executionIds, metadataStoreServiceClient]); return useFetchState(call, null); diff --git a/frontend/src/concepts/pipelines/content/artifacts/ArtifactUriLink.tsx b/frontend/src/concepts/pipelines/content/artifacts/ArtifactUriLink.tsx new file mode 100644 index 0000000000..6a2c9b1fa4 --- /dev/null +++ b/frontend/src/concepts/pipelines/content/artifacts/ArtifactUriLink.tsx @@ -0,0 +1,94 @@ +import React from 'react'; +import { + Button, + Flex, + FlexItem, + Icon, + Popover, + Split, + SplitItem, + Truncate, +} from '@patternfly/react-core'; +import { ExclamationTriangleIcon, ExternalLinkAltIcon } from '@patternfly/react-icons'; +import { usePipelinesAPI } from '~/concepts/pipelines/context'; +import { useIsAreaAvailable, SupportedArea } from '~/concepts/areas'; +import { MAX_STORAGE_OBJECT_SIZE, fetchStorageObjectSize } from '~/services/storageService'; +import { bytesAsRoundedGiB } from '~/utilities/number'; +import { extractS3UriComponents, getArtifactUrlFromUri } from './utils'; + +interface ArtifactUriLinkProps { + uri: string; +} + +export const ArtifactUriLink: React.FC = ({ uri }) => { + const { namespace } = usePipelinesAPI(); + const isS3EndpointAvailable = useIsAreaAvailable(SupportedArea.S3_ENDPOINT).status; + const [size, setSize] = React.useState(null); + + const url = React.useMemo(() => { + if (!uri || !isS3EndpointAvailable) { + return; + } + + const uriComponents = extractS3UriComponents(uri); + + if (uriComponents) { + fetchStorageObjectSize(namespace, uriComponents.path) + .then((sizeBytes) => setSize(sizeBytes)) + .catch(() => null); + } + + return getArtifactUrlFromUri(uri, namespace); + }, [isS3EndpointAvailable, namespace, uri]); + + if (!url) { + return uri; + } + + // we do not fetch over 100MB + const isOversizedFile = size !== null && size > MAX_STORAGE_OBJECT_SIZE; + + return ( + + {isOversizedFile && ( + + + + + + )} + + + + + ); +}; diff --git a/frontend/src/concepts/pipelines/content/artifacts/__tests__/utils.spec.ts b/frontend/src/concepts/pipelines/content/artifacts/__tests__/utils.spec.ts new file mode 100644 index 0000000000..8af9828c4d --- /dev/null +++ b/frontend/src/concepts/pipelines/content/artifacts/__tests__/utils.spec.ts @@ -0,0 +1,55 @@ +import { + extractS3UriComponents, + getArtifactUrlFromUri, +} from '~/concepts/pipelines/content/artifacts/utils'; + +describe('getArtifactUrlFromUri', () => { + it('should return the uri as is if it starts with http://', () => { + const uri = 'http://example.com/artifact'; + const namespace = 'test-namespace'; + expect(getArtifactUrlFromUri(uri, namespace)).toBe(uri); + }); + + it('should return the uri as is if it starts with https://', () => { + const uri = 'https://example.com/artifact'; + const namespace = 'test-namespace'; + expect(getArtifactUrlFromUri(uri, namespace)).toBe(uri); + }); + + it('should return undefined if the uri is not supported', () => { + const uri = 'ftp://example.com/artifact'; + const namespace = 'test-namespace'; + expect(getArtifactUrlFromUri(uri, namespace)).toBeUndefined(); + }); + + it('should return the backend URL for S3 uri', () => { + const uri = 's3://my-bucket/my-artifact'; + const namespace = 'test-namespace'; + const expectedUrl = '/api/storage/test-namespace?key=my-artifact'; + expect(getArtifactUrlFromUri(uri, namespace)).toBe(expectedUrl); + }); +}); + +describe('extractS3UriComponents', () => { + it('should return undefined for non-S3 URIs', () => { + const uri = 'https://example.com'; + expect(extractS3UriComponents(uri)).toBeUndefined(); + }); + + it('should return undefined for URIs without the S3 prefix', () => { + const uri = 'my-bucket/my-object'; + expect(extractS3UriComponents(uri)).toBeUndefined(); + }); + + it('should return the bucket and path components for valid S3 URIs', () => { + const uri = 's3://my-bucket/my-object'; + const expectedComponents = { bucket: 'my-bucket', path: 'my-object' }; + expect(extractS3UriComponents(uri)).toEqual(expectedComponents); + }); + + it('should handle URIs with special characters in the path', () => { + const uri = 's3://my-bucket/my%20object'; + const expectedComponents = { bucket: 'my-bucket', path: 'my%20object' }; + expect(extractS3UriComponents(uri)).toEqual(expectedComponents); + }); +}); diff --git a/frontend/src/concepts/pipelines/content/artifacts/utils.ts b/frontend/src/concepts/pipelines/content/artifacts/utils.ts new file mode 100644 index 0000000000..d44a250b02 --- /dev/null +++ b/frontend/src/concepts/pipelines/content/artifacts/utils.ts @@ -0,0 +1,37 @@ +export function extractS3UriComponents(uri: string): { bucket: string; path: string } | undefined { + const s3Prefix = 's3://'; + if (!uri.startsWith(s3Prefix)) { + return; + } + + const s3UrlWithoutPrefix = uri.slice(s3Prefix.length); + const firstSlashIndex = s3UrlWithoutPrefix.indexOf('/'); + const bucket = s3UrlWithoutPrefix.substring(0, firstSlashIndex); + const path = s3UrlWithoutPrefix.substring(firstSlashIndex + 1); + + return { bucket, path }; +} + +/** + * Get the url to fetch the artifact from the backend or http/https url + * + * @param uri + * @returns url to fetch the artifact from the backend or http/https url or undefined if the uri is not supported + */ +export function getArtifactUrlFromUri(uri: string, namespace: string): string | undefined { + // Check if the uri starts with http or https return it as is + if (uri.startsWith('http://') || uri.startsWith('https://')) { + return uri; + } + + // Otherwise check if the uri is s3 + // If it is not s3, return undefined as we only support fetching from s3 buckets + const uriComponents = extractS3UriComponents(uri); + if (!uriComponents) { + return; + } + + const { path } = uriComponents; + + return `/api/storage/${namespace}?key=${encodeURIComponent(path)}`; +} diff --git a/frontend/src/concepts/pipelines/content/compareRuns/metricsSection/PipelineRunArtifactSelect.tsx b/frontend/src/concepts/pipelines/content/compareRuns/metricsSection/PipelineRunArtifactSelect.tsx index b016f14639..2b15849d79 100644 --- a/frontend/src/concepts/pipelines/content/compareRuns/metricsSection/PipelineRunArtifactSelect.tsx +++ b/frontend/src/concepts/pipelines/content/compareRuns/metricsSection/PipelineRunArtifactSelect.tsx @@ -15,7 +15,7 @@ import { import { CompressIcon, ExpandIcon } from '@patternfly/react-icons'; import { PipelineRunKFv2 } from '~/concepts/pipelines/kfTypes'; -type ArtifactDisplayConfig = { config: T; title: string }; +type ArtifactDisplayConfig = { config: T; title: string; fileSize?: number }; type PipelineRunArtifactSelectProps = { run?: PipelineRunKFv2; diff --git a/frontend/src/concepts/pipelines/content/compareRuns/metricsSection/markdown/MarkdownCompare.tsx b/frontend/src/concepts/pipelines/content/compareRuns/metricsSection/markdown/MarkdownCompare.tsx index 322a379fc5..abee3c8e65 100644 --- a/frontend/src/concepts/pipelines/content/compareRuns/metricsSection/markdown/MarkdownCompare.tsx +++ b/frontend/src/concepts/pipelines/content/compareRuns/metricsSection/markdown/MarkdownCompare.tsx @@ -1,5 +1,6 @@ import * as React from 'react'; import { + Alert, Bullseye, Divider, EmptyState, @@ -9,6 +10,8 @@ import { Flex, FlexItem, Spinner, + Stack, + StackItem, } from '@patternfly/react-core'; import { PipelineRunKFv2 } from '~/concepts/pipelines/kfTypes'; @@ -22,6 +25,14 @@ import { import { CompareRunsEmptyState } from '~/concepts/pipelines/content/compareRuns/CompareRunsEmptyState'; import { PipelineRunArtifactSelect } from '~/concepts/pipelines/content/compareRuns/metricsSection/PipelineRunArtifactSelect'; import MarkdownView from '~/components/MarkdownView'; +import { + MAX_STORAGE_OBJECT_SIZE, + fetchStorageObject, + fetchStorageObjectSize, +} from '~/services/storageService'; +import { usePipelinesAPI } from '~/concepts/pipelines/context'; +import { extractS3UriComponents } from '~/concepts/pipelines/content/artifacts/utils'; +import { bytesAsRoundedGiB } from '~/utilities/number'; type MarkdownCompareProps = { runArtifacts?: RunArtifact[]; @@ -31,10 +42,12 @@ type MarkdownCompareProps = { export type MarkdownAndTitle = { title: string; config: string; + fileSize?: number; }; const MarkdownCompare: React.FC = ({ runArtifacts, isLoaded }) => { const [expandedGraph, setExpandedGraph] = React.useState(undefined); + const { namespace } = usePipelinesAPI(); const fullArtifactPaths: FullArtifactPath[] = React.useMemo(() => { if (!runArtifacts) { @@ -56,13 +69,25 @@ const MarkdownCompare: React.FC = ({ runArtifacts, isLoade })) .filter((markdown) => !!markdown.uri) .forEach(async ({ uri, title, run }) => { - const data = uri; // TODO: fetch data from uri: https://issues.redhat.com/browse/RHOAIENG-7206 + const uriComponents = extractS3UriComponents(uri); + if (!uriComponents) { + return; + } + const sizeBytes = await fetchStorageObjectSize(namespace, uriComponents.path).catch( + () => undefined, + ); + const text = await fetchStorageObject(namespace, uriComponents.path).catch(() => null); + + if (text === null) { + return; + } runMapBuilder[run.run_id] = run; const config = { title, - config: data, + config: text, + fileSize: sizeBytes, }; if (run.run_id in configMapBuilder) { @@ -73,7 +98,7 @@ const MarkdownCompare: React.FC = ({ runArtifacts, isLoade }); return { configMap: configMapBuilder, runMap: runMapBuilder }; - }, [fullArtifactPaths]); + }, [fullArtifactPaths, namespace]); if (!isLoaded) { return ( @@ -97,6 +122,23 @@ const MarkdownCompare: React.FC = ({ runArtifacts, isLoade ); } + const renderMarkdownWithSize = (config: MarkdownAndTitle) => ( + + {config.fileSize && config.fileSize > MAX_STORAGE_OBJECT_SIZE && ( + + + {`This file is ${bytesAsRoundedGiB( + config.fileSize, + )} GiB in size but we do not fetch files over 100MB. To view the full file, please download it from your S3 bucket.`} + + + )} + + + + + ); + return (
{expandedGraph ? ( @@ -105,7 +147,7 @@ const MarkdownCompare: React.FC = ({ runArtifacts, isLoade data={[expandedGraph]} setExpandedGraph={(config) => setExpandedGraph(config)} expandedGraph={expandedGraph} - renderArtifact={(config) => } + renderArtifact={renderMarkdownWithSize} /> ) : ( @@ -118,7 +160,7 @@ const MarkdownCompare: React.FC = ({ runArtifacts, isLoade data={configs} setExpandedGraph={(config) => setExpandedGraph(config)} expandedGraph={expandedGraph} - renderArtifact={(config) => } + renderArtifact={renderMarkdownWithSize} /> = ({ task }) => { ({ label: a.label, value: a.type }))} + artifacts={task.inputs.artifacts} params={task.inputs.params?.map((p) => ({ label: p.label, value: p.value ?? p.type }))} /> @@ -41,7 +41,7 @@ const PipelineTaskDetails: React.FC = ({ task }) => { ({ label: a.label, value: a.type }))} + artifacts={task.outputs.artifacts} params={task.outputs.params?.map((p) => ({ label: p.label, value: p.value ?? p.type }))} /> diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDetails.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDetails.tsx index 1f241aed9d..0f7e1a4ab8 100644 --- a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDetails.tsx +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDetails.tsx @@ -39,8 +39,6 @@ import { routePipelineRunsNamespace } from '~/routes'; import PipelineJobReferenceName from '~/concepts/pipelines/content/PipelineJobReferenceName'; import useExecutionsForPipelineRun from '~/concepts/pipelines/content/pipelinesDetails/pipelineRun/useExecutionsForPipelineRun'; import { useGetEventsByExecutionIds } from '~/concepts/pipelines/apiHooks/mlmd/useGetEventsByExecutionId'; -import { parseEventsByType } from '~/pages/pipelines/global/experiments/executions/utils'; -import { Event } from '~/third_party/mlmd'; import { usePipelineRunArtifacts } from './artifacts'; const PipelineRunDetails: PipelineCoreDetailsPageComponent = ({ breadcrumbPath, contextPath }) => { @@ -63,15 +61,14 @@ const PipelineRunDetails: PipelineCoreDetailsPageComponent = ({ breadcrumbPath, const [executions, executionsLoaded, executionsError] = useExecutionsForPipelineRun(run); const [artifacts] = usePipelineRunArtifacts(run); - const [eventsResponse] = useGetEventsByExecutionIds( + const [events] = useGetEventsByExecutionIds( React.useMemo(() => executions.map((execution) => execution.getId()), [executions]), ); - const events = parseEventsByType(eventsResponse); const nodes = usePipelineTaskTopology( pipelineSpec, run?.run_details, executions, - events[Event.Type.OUTPUT], + events, artifacts, ); diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/SelectedNodeInputOutputTab.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/SelectedNodeInputOutputTab.tsx index fb4e744b9d..3bd6701fc4 100644 --- a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/SelectedNodeInputOutputTab.tsx +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/SelectedNodeInputOutputTab.tsx @@ -127,7 +127,7 @@ const SelectedNodeInputOutputTab: React.FC = ({ ({ label: a.label, value: a.type }))} + artifacts={task.inputs.artifacts} params={getParams(task.inputs.params, getExecutionFieldsMap('inputs'))} /> @@ -136,7 +136,7 @@ const SelectedNodeInputOutputTab: React.FC = ({ ({ label: a.label, value: a.type }))} + artifacts={task.outputs.artifacts} params={getParams(task.outputs.params, getExecutionFieldsMap('outputs'))} /> diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDetails.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDetails.tsx index 2b50456d9a..cda7ec68e1 100644 --- a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDetails.tsx +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDetails.tsx @@ -10,6 +10,7 @@ import { DescriptionListGroup, DescriptionListTerm, DescriptionListDescription, + StackItem, } from '@patternfly/react-core'; import { Artifact } from '~/third_party/mlmd'; @@ -19,12 +20,14 @@ import { getArtifactName } from '~/pages/pipelines/global/experiments/artifacts/ import PipelinesTableRowTime from '~/concepts/pipelines/content/tables/PipelinesTableRowTime'; import PipelineRunDrawerRightContent from '~/concepts/pipelines/content/pipelinesDetails/pipelineRun/PipelineRunDrawerRightContent'; import { SupportedArea, useIsAreaAvailable } from '~/concepts/areas'; +import { ArtifactUriLink } from '~/concepts/pipelines/content/artifacts/ArtifactUriLink'; +import ArtifactPreview from '~/concepts/pipelines/content/pipelinesDetails/taskDetails/ArtifactPreview'; type ArtifactNodeDetailsProps = Pick< React.ComponentProps, 'upstreamTaskName' > & { - artifact: Artifact.AsObject; + artifact: Artifact; }; export const ArtifactNodeDetails: React.FC = ({ @@ -32,7 +35,7 @@ export const ArtifactNodeDetails: React.FC = ({ upstreamTaskName, }) => { const { namespace } = usePipelinesAPI(); - const artifactName = getArtifactName(artifact); + const artifactName = getArtifactName(artifact.toObject()); const isExperimentsAvailable = useIsAreaAvailable(SupportedArea.PIPELINE_EXPERIMENTS).status; return ( @@ -56,18 +59,20 @@ export const ArtifactNodeDetails: React.FC = ({ Artifact name {isExperimentsAvailable ? ( - {artifactName} + + {artifactName} + ) : ( artifactName )} Artifact type - {artifact.type} + {artifact.getType()} Created at - + @@ -84,7 +89,16 @@ export const ArtifactNodeDetails: React.FC = ({ > {artifactName} - {artifact.uri} + + + + + + + + + + diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDrawerContent.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDrawerContent.tsx index 7c32cf7866..ff2a74bde5 100644 --- a/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDrawerContent.tsx +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/pipelineRun/artifacts/ArtifactNodeDrawerContent.tsx @@ -60,10 +60,7 @@ export const ArtifactNodeDrawerContent: React.FC title={Artifact details} aria-label="Artifact details" > - + = ({ artifact }) => { + const [downloadedArtifact, setDownloadedArtifact] = React.useState(null); + const [downloadedArtifactSize, setDownloadedArtifactSize] = React.useState(null); + const [loading, setLoading] = React.useState(false); + const { namespace } = usePipelinesAPI(); + const isS3EndpointAvailable = useIsAreaAvailable(SupportedArea.S3_ENDPOINT).status; + const artifactType = artifact.getType(); + React.useEffect(() => { + if (!isS3EndpointAvailable) { + return; + } + + if (artifactType === ArtifactType.MARKDOWN || artifactType === ArtifactType.HTML) { + const uri = artifact.getUri(); + if (uri) { + const uriComponents = extractS3UriComponents(uri); + if (uriComponents) { + const downloadArtifact = async (path: string) => { + await fetchStorageObjectSize(namespace, path) + .then((size) => setDownloadedArtifactSize(size)) + .catch(() => null); + await fetchStorageObject(namespace, path) + .then((text) => setDownloadedArtifact(text)) + .catch(() => null); + setLoading(false); + }; + setLoading(true); + setDownloadedArtifact(null); + setDownloadedArtifactSize(null); + downloadArtifact(uriComponents.path); + } + } + } + }, [artifact, artifactType, isS3EndpointAvailable, namespace]); + if (artifactType === ArtifactType.CLASSIFICATION_METRICS) { const confusionMatrix = artifact.getCustomPropertiesMap().get('confusionMatrix'); const confidenceMetrics = artifact.getCustomPropertiesMap().get('confidenceMetrics'); @@ -132,13 +179,43 @@ export const ArtifactVisualization: React.FC = ({ ar ); } - if (artifactType === ArtifactType.HTML || artifactType === ArtifactType.MARKDOWN) { - return ( - - - - ); + if (artifactType === ArtifactType.MARKDOWN || artifactType === ArtifactType.HTML) { + if (loading) { + return ( + + + + ); + } + if (downloadedArtifact) { + return ( + + {downloadedArtifactSize && downloadedArtifactSize > MAX_STORAGE_OBJECT_SIZE && ( + + + {`This file is ${bytesAsRoundedGiB( + downloadedArtifactSize, + )} GB in size but we do not fetch files over 100MB. To view the full file, please download it from your S3 bucket.`} + + + )} + + Artifact details + + + + + + ); + } } - return null; + return ( + + + + ); }; diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/ArtifactPreview.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/ArtifactPreview.tsx new file mode 100644 index 0000000000..2392ff8457 --- /dev/null +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/ArtifactPreview.tsx @@ -0,0 +1,74 @@ +import React from 'react'; +import { Bullseye, CodeBlock, CodeBlockCode, Spinner } from '@patternfly/react-core'; +import { useIsAreaAvailable, SupportedArea } from '~/concepts/areas'; +import { usePipelinesAPI } from '~/concepts/pipelines/context'; +import { Artifact } from '~/third_party/mlmd'; +import { extractS3UriComponents } from '~/concepts/pipelines/content/artifacts/utils'; +import { fetchStorageObject } from '~/services/storageService'; + +type ArtifactPreviewProps = { + artifact: Artifact; + maxBytes?: number; + maxLines?: number; +}; + +const ArtifactPreview: React.FC = ({ + artifact, + maxBytes = 255, + maxLines = 4, +}) => { + const isS3EndpointAvailable = useIsAreaAvailable(SupportedArea.S3_ENDPOINT).status; + const { namespace } = usePipelinesAPI(); + const [preview, setPreview] = React.useState(null); + const [isLoading, setIsLoading] = React.useState(false); + + React.useEffect(() => { + const uri = artifact.getUri(); + if (!uri || !isS3EndpointAvailable) { + return; + } + + setPreview(null); + const uriComponents = extractS3UriComponents(uri); + if (!uriComponents) { + return; + } + setIsLoading(true); + fetchStorageObject(namespace, uriComponents.path, maxBytes) + .catch(() => null) + .then((text) => setPreview(text)) + .finally(() => setIsLoading(false)); + }, [artifact, isS3EndpointAvailable, maxBytes, namespace]); + + if (isLoading) { + return ( + + + + ); + } + + if (!preview) { + return null; + } + + // Try to parse the preview as JSON + let code = preview; + try { + code = JSON.parse(preview); + code = JSON.stringify(code, null, 2); + } catch { + // ignore + } + + code = code.split('\n').slice(0, maxLines).join('\n').trim(); + code = `${code}...`; + + return ( + + {code} + + ); +}; + +export default ArtifactPreview; diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsInputOutput.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsInputOutput.tsx index 71f92fb775..911ae667aa 100644 --- a/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsInputOutput.tsx +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsInputOutput.tsx @@ -2,10 +2,13 @@ import * as React from 'react'; import { Stack, StackItem } from '@patternfly/react-core'; import TaskDetailsSection from '~/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsSection'; import TaskDetailsPrintKeyValues from '~/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsPrintKeyValues'; +import { PipelineTaskArtifact } from '~/concepts/pipelines/topology'; +import { ArtifactUriLink } from '~/concepts/pipelines/content/artifacts/ArtifactUriLink'; +import ArtifactPreview from './ArtifactPreview'; type TaskDetailsInputOutputProps = { type: 'Input' | 'Output'; - artifacts?: React.ComponentProps['items']; + artifacts?: PipelineTaskArtifact[]; params?: React.ComponentProps['items']; }; @@ -14,6 +17,29 @@ const TaskDetailsInputOutput: React.FC = ({ params, type, }) => { + const artifactKeyValues = React.useMemo(() => { + if (!artifacts) { + return []; + } + + return artifacts.map((artifactInputOutput) => { + const artifact = artifactInputOutput.value; + + if (artifact) { + return { + label: artifactInputOutput.label, + value: , + preview: , + }; + } + + return { + label: artifactInputOutput.label, + value: artifactInputOutput.type, + }; + }); + }, [artifacts]); + if (!params && !artifacts) { return null; } @@ -23,7 +49,7 @@ const TaskDetailsInputOutput: React.FC = ({ {artifacts && ( - + )} diff --git a/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsPrintKeyValues.tsx b/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsPrintKeyValues.tsx index 5ecd331b4b..b5935bc144 100644 --- a/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsPrintKeyValues.tsx +++ b/frontend/src/concepts/pipelines/content/pipelinesDetails/taskDetails/TaskDetailsPrintKeyValues.tsx @@ -2,7 +2,7 @@ import * as React from 'react'; import { Grid, GridItem, Truncate } from '@patternfly/react-core'; type TaskDetailsPrintKeyValuesProps = { - items: { label: string; value: React.ReactNode }[]; + items: { label: string; value: React.ReactNode; preview?: React.ReactNode }[]; }; const TaskDetailsPrintKeyValues: React.FC = ({ items }) => ( @@ -15,6 +15,7 @@ const TaskDetailsPrintKeyValues: React.FC = ({ i {result.value} + {result.preview && {result.preview}} ))} diff --git a/frontend/src/concepts/pipelines/topology/__tests__/parseUtils.spec.ts b/frontend/src/concepts/pipelines/topology/__tests__/parseUtils.spec.ts index 2ecb786ab4..74b68b181d 100644 --- a/frontend/src/concepts/pipelines/topology/__tests__/parseUtils.spec.ts +++ b/frontend/src/concepts/pipelines/topology/__tests__/parseUtils.spec.ts @@ -11,6 +11,7 @@ import { ResourceType, parseRuntimeInfoFromExecutions, parseVolumeMounts, + getExecutionLinkedArtifactMap, } from '~/concepts/pipelines/topology/parseUtils'; import { ArtifactStateKF, @@ -25,21 +26,16 @@ import { TaskKF, TriggerStrategy, } from '~/concepts/pipelines/kfTypes'; -import { Artifact, Execution, Value } from '~/third_party/mlmd'; +import { Artifact, Execution, Value, Event } from '~/third_party/mlmd'; describe('pipeline topology parseUtils', () => { describe('parseInputOutput', () => { - it('returns undefined when no definition is provided', () => { - const result = parseInputOutput(); - expect(result).toBeUndefined(); - }); - it('returns data with params when the definition includes parameters', () => { const testDefinition = { parameters: { 'some-string-param': { parameterType: InputDefinitionParameterType.STRING } }, }; - const result = parseInputOutput(testDefinition); + const result = parseInputOutput(testDefinition, []); expect(result).toEqual({ params: [{ label: 'some-string-param', type: 'STRING' }] }); }); @@ -55,7 +51,7 @@ describe('pipeline topology parseUtils', () => { }, }; - const result = parseInputOutput(testDefinition); + const result = parseInputOutput(testDefinition, []); expect(result).toEqual({ artifacts: [{ label: 'some-artifact', type: 'system.Artifact (v1)' }], }); @@ -614,3 +610,38 @@ describe('pipeline topology parseUtils', () => { }); }); }); + +describe('getExecutionLinkedArtifactMap', () => { + it('returns an empty object when artifacts or events are not provided', () => { + const result = getExecutionLinkedArtifactMap(undefined, undefined); + expect(result).toEqual({}); + }); + + it('returns an empty object when artifacts or events are empty', () => { + const result = getExecutionLinkedArtifactMap([], []); + expect(result).toEqual({}); + }); + + it('returns the correct linked artifact map', () => { + const artifacts = [new Artifact().setId(1), new Artifact().setId(2), new Artifact().setId(3)]; + const events = [ + new Event().setArtifactId(1).setExecutionId(1), + new Event().setArtifactId(2).setExecutionId(1), + new Event().setArtifactId(3).setExecutionId(2), + new Event().setArtifactId(1).setExecutionId(2), + ]; + + const result = getExecutionLinkedArtifactMap(artifacts, events); + + expect(result).toEqual({ + 1: [ + { event: events[0], artifact: artifacts[0] }, + { event: events[1], artifact: artifacts[1] }, + ], + 2: [ + { event: events[2], artifact: artifacts[2] }, + { event: events[3], artifact: artifacts[0] }, + ], + }); + }); +}); diff --git a/frontend/src/concepts/pipelines/topology/parseUtils.ts b/frontend/src/concepts/pipelines/topology/parseUtils.ts index 7ebcc55a57..775bd58b61 100644 --- a/frontend/src/concepts/pipelines/topology/parseUtils.ts +++ b/frontend/src/concepts/pipelines/topology/parseUtils.ts @@ -12,7 +12,9 @@ import { TaskDetailKF, } from '~/concepts/pipelines/kfTypes'; import { VolumeMount } from '~/types'; -import { Artifact, Execution } from '~/third_party/mlmd'; +import { Artifact, Event, Execution } from '~/third_party/mlmd'; +import { LinkedArtifact } from '~/concepts/pipelines/apiHooks/mlmd/types'; +import { getArtifactNameFromEvent } from '~/concepts/pipelines/content/compareRuns/metricsSection/utils'; import { PipelineTaskInputOutput, PipelineTaskRunStatus } from './pipelineTaskTypes'; export const composeArtifactType = (data: InputOutputArtifactType): string => @@ -88,35 +90,47 @@ export const parseTasksForArtifactRelationship = ( {}, ); +export function filterEventWithInputArtifact(linkedArtifact: LinkedArtifact[]): LinkedArtifact[] { + return linkedArtifact.filter((obj) => obj.event.getType() === Event.Type.INPUT); +} + +export function filterEventWithOutputArtifact(linkedArtifact: LinkedArtifact[]): LinkedArtifact[] { + return linkedArtifact.filter((obj) => obj.event.getType() === Event.Type.OUTPUT); +} + export const parseInputOutput = ( - definition?: InputOutputDefinition, + definition: InputOutputDefinition, + linkedArtifacts: LinkedArtifact[], ): PipelineTaskInputOutput | undefined => { let data: PipelineTaskInputOutput | undefined; - if (definition) { - const { artifacts, parameters } = definition; - data = {}; - - if (parameters) { - data = { - ...data, - params: Object.entries(parameters).map(([paramLabel, { parameterType }]) => ({ - label: paramLabel, - type: parameterType, - // TODO: support value - })), - }; - } + const { artifacts, parameters } = definition; + data = {}; + + if (parameters) { + data = { + ...data, + params: Object.entries(parameters).map(([paramLabel, { parameterType }]) => ({ + label: paramLabel, + type: parameterType, + // TODO: support value + })), + }; + } - if (artifacts) { - data = { - ...data, - artifacts: Object.entries(artifacts).map(([paramLabel, { artifactType }]) => ({ + if (artifacts) { + data = { + ...data, + artifacts: Object.entries(artifacts).map(([paramLabel, { artifactType }]) => { + const linkedArtifact = linkedArtifacts.find( + (obj) => getArtifactNameFromEvent(obj.event) === paramLabel, + ); + return { label: paramLabel, type: composeArtifactType(artifactType), - // TODO: support value - })), - }; - } + value: linkedArtifact?.artifact, + }; + }), + }; } return data; @@ -338,3 +352,31 @@ export const idForTaskArtifact = ( groupId ? `GROUP.${groupId}.ARTIFACT.${taskId}.${artifactId}` : `ARTIFACT.${taskId}.${artifactId}`; + +export const getExecutionLinkedArtifactMap = ( + artifacts?: Artifact[] | null, + events?: Event[] | null, +): Record => { + if (!artifacts || !events) { + return {}; + } + const executionMap: Record = {}; + + const artifactMap: Record = {}; + artifacts.forEach((artifact) => { + artifactMap[artifact.getId()] = artifact; + }); + + events.forEach((event) => { + const artifact = artifactMap[event.getArtifactId()]; + const executionId = event.getExecutionId(); + + if (!(executionId in executionMap)) { + executionMap[executionId] = []; + } + + executionMap[executionId].push({ event, artifact }); + }); + + return executionMap; +}; diff --git a/frontend/src/concepts/pipelines/topology/pipelineTaskTypes.ts b/frontend/src/concepts/pipelines/topology/pipelineTaskTypes.ts index bb94eb7717..e2b26e9afe 100644 --- a/frontend/src/concepts/pipelines/topology/pipelineTaskTypes.ts +++ b/frontend/src/concepts/pipelines/topology/pipelineTaskTypes.ts @@ -17,6 +17,7 @@ export type PipelineTaskParam = { export type PipelineTaskArtifact = { label: string; type: string; + value?: Artifact; }; export type PipelineTaskStep = { diff --git a/frontend/src/concepts/pipelines/topology/usePipelineTaskTopology.tsx b/frontend/src/concepts/pipelines/topology/usePipelineTaskTopology.tsx index bcbe911a2f..c656ded04c 100644 --- a/frontend/src/concepts/pipelines/topology/usePipelineTaskTopology.tsx +++ b/frontend/src/concepts/pipelines/topology/usePipelineTaskTopology.tsx @@ -13,9 +13,14 @@ import { createNode } from '~/concepts/topology'; import { PipelineNodeModelExpanded } from '~/concepts/topology/types'; import { createArtifactNode, createGroupNode } from '~/concepts/topology/utils'; import { Artifact, Execution, Event } from '~/third_party/mlmd'; +import { LinkedArtifact } from '~/concepts/pipelines/apiHooks/mlmd/types'; +import { parseEventsByType } from '~/pages/pipelines/global/experiments/executions/utils'; import { ComponentArtifactMap, composeArtifactType, + filterEventWithInputArtifact, + filterEventWithOutputArtifact, + getExecutionLinkedArtifactMap, idForTaskArtifact, parseComponentsForArtifactRelationship, parseInputOutput, @@ -100,6 +105,7 @@ const getNodesForTasks = ( executors: PipelineExecutorsKF, componentArtifactMap: ComponentArtifactMap, taskArtifactMap: TaskArtifactMap, + executionLinkedArtifactMap: Record, runDetails?: RunDetailsKF, executions?: Execution[] | null, inputArtifacts?: InputOutputDefinitionArtifacts, @@ -129,12 +135,33 @@ const getNodesForTasks = ( const executorLabel = component?.executorLabel; const executor = executorLabel ? executors[executorLabel] : undefined; + let linkedArtifacts: LinkedArtifact[] = []; + if (executions) { + const execution = executions.find( + (e) => + e.getCustomPropertiesMap().get('task_name')?.getStringValue() === (taskName || taskId), + ); + if (execution) { + linkedArtifacts = executionLinkedArtifactMap[execution.getId()] ?? []; + } + } + const pipelineTask: PipelineTask = { type: 'groupTask', name: taskName, steps: executor?.container ? [executor.container] : undefined, - inputs: parseInputOutput(component?.inputDefinitions), - outputs: parseInputOutput(component?.outputDefinitions), + inputs: component?.inputDefinitions + ? parseInputOutput( + component.inputDefinitions, + filterEventWithInputArtifact(linkedArtifacts), + ) + : undefined, + outputs: component?.outputDefinitions + ? parseInputOutput( + component.outputDefinitions, + filterEventWithOutputArtifact(linkedArtifacts), + ) + : undefined, status, volumeMounts: parseVolumeMounts(spec.platform_spec, executorLabel), }; @@ -187,6 +214,7 @@ const getNodesForTasks = ( executors, componentArtifactMap, subTasksArtifactMap, + executionLinkedArtifactMap, runDetails, executions, component?.inputDefinitions?.artifacts, @@ -214,7 +242,7 @@ export const usePipelineTaskTopology = ( spec?: PipelineSpecVariable, runDetails?: RunDetailsKF, executions?: Execution[] | null, - events?: Event[], + events?: Event[] | null, artifacts?: Artifact[] | undefined, ): PipelineNodeModelExpanded[] => React.useMemo(() => { @@ -232,8 +260,11 @@ export const usePipelineTaskTopology = ( }, } = pipelineSpec; + const outputEvents = parseEventsByType(events ?? [])[Event.Type.OUTPUT]; + const componentArtifactMap = parseComponentsForArtifactRelationship(components); const taskArtifactMap = parseTasksForArtifactRelationship('root', tasks); + const executionLinkedArtifactMap = getExecutionLinkedArtifactMap(artifacts, events); // There are some duplicated nodes, remove them return _.uniqBy( @@ -245,10 +276,11 @@ export const usePipelineTaskTopology = ( executors, componentArtifactMap, taskArtifactMap, + executionLinkedArtifactMap, runDetails, executions, inputDefinitions?.artifacts, - events, + outputEvents, artifacts, )[0], (node) => node.id, diff --git a/frontend/src/k8sTypes.ts b/frontend/src/k8sTypes.ts index cdf7d3df79..14f406d8bb 100644 --- a/frontend/src/k8sTypes.ts +++ b/frontend/src/k8sTypes.ts @@ -1212,6 +1212,7 @@ export type DashboardCommonConfig = { disableAcceleratorProfiles: boolean; // TODO Temp feature flag - remove with https://issues.redhat.com/browse/RHOAIENG-3826 disablePipelineExperiments: boolean; + disableS3Endpoint: boolean; disableDistributedWorkloads: boolean; disableModelRegistry: boolean; }; diff --git a/frontend/src/pages/pipelines/global/experiments/artifacts/ArtifactDetails/ArtifactOverviewDetails.tsx b/frontend/src/pages/pipelines/global/experiments/artifacts/ArtifactDetails/ArtifactOverviewDetails.tsx index 7697eaa293..4cc242a507 100644 --- a/frontend/src/pages/pipelines/global/experiments/artifacts/ArtifactDetails/ArtifactOverviewDetails.tsx +++ b/frontend/src/pages/pipelines/global/experiments/artifacts/ArtifactDetails/ArtifactOverviewDetails.tsx @@ -12,6 +12,7 @@ import { } from '@patternfly/react-core'; import { Artifact } from '~/third_party/mlmd'; +import { ArtifactUriLink } from '~/concepts/pipelines/content/artifacts/ArtifactUriLink'; import { ArtifactPropertyDescriptionList } from './ArtifactPropertyDescriptionList'; interface ArtifactOverviewDetailsProps { @@ -32,7 +33,9 @@ export const ArtifactOverviewDetails: React.FC = ( {artifact?.uri && ( <> URI - {artifact.uri} + + + )} diff --git a/frontend/src/pages/pipelines/global/experiments/artifacts/ArtifactsTable.tsx b/frontend/src/pages/pipelines/global/experiments/artifacts/ArtifactsTable.tsx index 1e3a65eed7..4285892466 100644 --- a/frontend/src/pages/pipelines/global/experiments/artifacts/ArtifactsTable.tsx +++ b/frontend/src/pages/pipelines/global/experiments/artifacts/ArtifactsTable.tsx @@ -13,6 +13,7 @@ import SimpleDropdownSelect from '~/components/SimpleDropdownSelect'; import { ArtifactType } from '~/concepts/pipelines/kfTypes'; import { useMlmdListContext, usePipelinesAPI } from '~/concepts/pipelines/context'; import { artifactsDetailsRoute } from '~/routes'; +import { ArtifactUriLink } from '~/concepts/pipelines/content/artifacts/ArtifactUriLink'; import { FilterOptions, columns, initialFilterData, options } from './constants'; import { getArtifactName } from './utils'; @@ -149,7 +150,9 @@ export const ArtifactsTable: React.FC = ({ {artifact.id} {artifact.type} - {artifact.uri} + + + diff --git a/frontend/src/pages/pipelines/global/experiments/artifacts/__tests__/ArtifactDetails.spec.tsx b/frontend/src/pages/pipelines/global/experiments/artifacts/__tests__/ArtifactDetails.spec.tsx index 9da0401ea1..da856b32e2 100644 --- a/frontend/src/pages/pipelines/global/experiments/artifacts/__tests__/ArtifactDetails.spec.tsx +++ b/frontend/src/pages/pipelines/global/experiments/artifacts/__tests__/ArtifactDetails.spec.tsx @@ -9,12 +9,22 @@ import { artifactsBaseRoute } from '~/routes'; import { ArtifactDetails } from '~/pages/pipelines/global/experiments/artifacts/ArtifactDetails'; import GlobalPipelineCoreDetails from '~/pages/pipelines/global/GlobalPipelineCoreDetails'; import * as useGetArtifactById from '~/concepts/pipelines/apiHooks/mlmd/useGetArtifactById'; +import * as fetchStorageObjectSize from '~/services/storageService'; jest.mock('~/redux/selectors', () => ({ ...jest.requireActual('~/redux/selectors'), useUser: jest.fn(() => ({ isAdmin: true })), })); +jest.mock('~/concepts/areas/useIsAreaAvailable', () => () => ({ + status: true, + featureFlags: {}, + reliantAreas: {}, + requiredComponents: {}, + requiredCapabilities: {}, + customCondition: jest.fn(), +})); + jest.mock('~/concepts/pipelines/context/PipelinesContext', () => ({ usePipelinesAPI: jest.fn(() => ({ pipelinesServer: { @@ -37,6 +47,7 @@ jest.mock('~/concepts/pipelines/context/PipelinesContext', () => ({ describe('ArtifactDetails', () => { const useGetArtifactByIdSpy = jest.spyOn(useGetArtifactById, 'useGetArtifactById'); + const fetchStorageObjectSizeSpy = jest.spyOn(fetchStorageObjectSize, 'fetchStorageObjectSize'); beforeEach(() => { useGetArtifactByIdSpy.mockReturnValue([ @@ -45,7 +56,7 @@ describe('ArtifactDetails', () => { id: 1, typeId: 14, type: 'system.Artifact', - uri: 'https://test-artifact!-aiplatform.googleapis.com/v1/12.15', + uri: 's3://namespace/bucket/path/to/artifact', propertiesMap: [], customPropertiesMap: [ [ @@ -109,6 +120,22 @@ describe('ArtifactDetails', () => { expect(overviewTab).toHaveAttribute('aria-selected', 'true'); }); + it('renders warning on oversized file', async () => { + fetchStorageObjectSizeSpy.mockResolvedValue(1e9); + + render( + + + , + ); + + expect(await screen.findByTestId('storage-file-oversized-warning')).toBeVisible(); + }); + it('renders Overview tab metadata contents', () => { render( @@ -126,7 +153,7 @@ describe('ArtifactDetails', () => { const datasetDescriptionList = screen.getByTestId('dataset-description-list'); expect(within(datasetDescriptionList).getByRole('term')).toHaveTextContent('URI'); expect(within(datasetDescriptionList).getByRole('definition')).toHaveTextContent( - 'https://test-artifact!-aiplatform.googleapis.com/v1/12.15', + 's3://namespace/bucket/path/to/artifact', ); const customPropsDescriptionList = screen.getByTestId('custom-props-description-list'); diff --git a/frontend/src/pages/pipelines/global/experiments/artifacts/__tests__/ArtifactsTable.spec.tsx b/frontend/src/pages/pipelines/global/experiments/artifacts/__tests__/ArtifactsTable.spec.tsx index 124ee72372..f9dcba5607 100644 --- a/frontend/src/pages/pipelines/global/experiments/artifacts/__tests__/ArtifactsTable.spec.tsx +++ b/frontend/src/pages/pipelines/global/experiments/artifacts/__tests__/ArtifactsTable.spec.tsx @@ -16,6 +16,15 @@ jest.mock('~/redux/selectors', () => ({ useUser: jest.fn(() => ({ isAdmin: true })), })); +jest.mock('~/concepts/areas/useIsAreaAvailable', () => () => ({ + status: true, + featureFlags: {}, + reliantAreas: {}, + requiredComponents: {}, + requiredCapabilities: {}, + customCondition: jest.fn(), +})); + jest.mock('~/concepts/pipelines/context/PipelinesContext', () => ({ usePipelinesAPI: jest.fn(() => ({ pipelinesServer: { diff --git a/frontend/src/pages/pipelines/global/experiments/compareRuns/CompareRunsMetricsSection.tsx b/frontend/src/pages/pipelines/global/experiments/compareRuns/CompareRunsMetricsSection.tsx index c6ecf5535e..2c0b8f71f3 100644 --- a/frontend/src/pages/pipelines/global/experiments/compareRuns/CompareRunsMetricsSection.tsx +++ b/frontend/src/pages/pipelines/global/experiments/compareRuns/CompareRunsMetricsSection.tsx @@ -17,6 +17,7 @@ import ScalarMetricTable from '~/concepts/pipelines/content/compareRuns/metricsS import RocCurveCompare from '~/concepts/pipelines/content/compareRuns/metricsSection/roc/RocCurveCompare'; import ConfusionMatrixCompare from '~/concepts/pipelines/content/compareRuns/metricsSection/confusionMatrix/ConfusionMatrixCompare'; import MarkdownCompare from '~/concepts/pipelines/content/compareRuns/metricsSection/markdown/MarkdownCompare'; +import { SupportedArea, useIsAreaAvailable } from '~/concepts/areas'; export const CompareRunMetricsSection: React.FunctionComponent = () => { const { runs, selectedRuns } = useCompareRuns(); @@ -26,6 +27,7 @@ export const CompareRunMetricsSection: React.FunctionComponent = () => { const [activeTabKey, setActiveTabKey] = React.useState( MetricSectionTabLabels.SCALAR, ); + const isS3EndpointAvailable = useIsAreaAvailable(SupportedArea.S3_ENDPOINT).status; const selectedMlmdPackages = React.useMemo( () => @@ -91,15 +93,16 @@ export const CompareRunMetricsSection: React.FunctionComponent = () => { - {MetricSectionTabLabels.MARKDOWN}} - isDisabled // TODO enable when markdown can be fetched from storage (s3): https://issues.redhat.com/browse/RHOAIENG-7206 - > - - - - + {isS3EndpointAvailable && ( + {MetricSectionTabLabels.MARKDOWN}} + > + + + + + )} ); diff --git a/frontend/src/pages/pipelines/global/experiments/executions/details/ExecutionDetails.tsx b/frontend/src/pages/pipelines/global/experiments/executions/details/ExecutionDetails.tsx index 8937264044..1090db9ce6 100644 --- a/frontend/src/pages/pipelines/global/experiments/executions/details/ExecutionDetails.tsx +++ b/frontend/src/pages/pipelines/global/experiments/executions/details/ExecutionDetails.tsx @@ -40,11 +40,9 @@ const ExecutionDetails: PipelineCoreDetailsPageComponent = ({ breadcrumbPath, co const navigate = useNavigate(); const { namespace } = usePipelinesAPI(); const [execution, executionLoaded, executionError] = useGetExecutionById(executionId); - const [eventsResponse, eventsLoaded, eventsError] = useGetEventsByExecutionId( - Number(executionId), - ); + const [events, eventsLoaded, eventsError] = useGetEventsByExecutionId(Number(executionId)); const [artifactTypes, artifactTypesLoaded] = useGetArtifactTypes(); - const allEvents = parseEventsByType(eventsResponse); + const allEvents = parseEventsByType(events); const artifactTypeMap = artifactTypes.reduce((acc, artifactType) => { acc[artifactType.getId()] = artifactType.getName(); diff --git a/frontend/src/pages/pipelines/global/experiments/executions/utils.ts b/frontend/src/pages/pipelines/global/experiments/executions/utils.ts index b191fc6b0d..7f37caf950 100644 --- a/frontend/src/pages/pipelines/global/experiments/executions/utils.ts +++ b/frontend/src/pages/pipelines/global/experiments/executions/utils.ts @@ -1,10 +1,5 @@ import { Struct } from 'google-protobuf/google/protobuf/struct_pb'; -import { - Event, - Execution, - GetEventsByExecutionIDsResponse, - Value as MlmdValue, -} from '~/third_party/mlmd'; +import { Event, Execution, Value as MlmdValue } from '~/third_party/mlmd'; export type MlmdMetadataValueType = string | number | Struct | undefined; @@ -30,9 +25,7 @@ export const getMlmdMetadataValue = (value?: MlmdValue): MlmdMetadataValueType = } }; -export const parseEventsByType = ( - response: GetEventsByExecutionIDsResponse | null, -): Record => { +export const parseEventsByType = (response: Event[] | null): Record => { const events: Record = { [Event.Type.UNKNOWN]: [], [Event.Type.DECLARED_INPUT]: [], @@ -48,7 +41,7 @@ export const parseEventsByType = ( return events; } - response.getEventsList().forEach((event) => { + response.forEach((event) => { const type = event.getType(); const id = event.getArtifactId(); if (type >= 0 && id > 0) { diff --git a/frontend/src/services/storageService.ts b/frontend/src/services/storageService.ts new file mode 100644 index 0000000000..690b205969 --- /dev/null +++ b/frontend/src/services/storageService.ts @@ -0,0 +1,36 @@ +import axios from 'axios'; + +export const MAX_STORAGE_OBJECT_SIZE = 1e8; + +export const fetchStorageObject = ( + namespace: string, + key: string, + peek?: number, +): Promise => { + const url = `/api/storage/${namespace}`; + return axios + .get(url, { + params: { + key, + peek, + }, + }) + .then((response) => response.data) + .catch((e) => { + throw new Error(e.response.data.message); + }); +}; + +export const fetchStorageObjectSize = (namespace: string, key: string): Promise => { + const url = `/api/storage/${namespace}/size`; + return axios + .get(url, { + params: { + key, + }, + }) + .then((response) => response.data) + .catch((e) => { + throw new Error(e.response.data.message); + }); +}; diff --git a/manifests/crd/odhdashboardconfigs.opendatahub.io.crd.yaml b/manifests/crd/odhdashboardconfigs.opendatahub.io.crd.yaml index e3cf6c4a65..fb49e77472 100644 --- a/manifests/crd/odhdashboardconfigs.opendatahub.io.crd.yaml +++ b/manifests/crd/odhdashboardconfigs.opendatahub.io.crd.yaml @@ -69,6 +69,8 @@ spec: type: boolean disablePipelineExperiments: type: boolean + disableS3Endpoint: + type: boolean disableDistributedWorkloads: type: boolean disableModelRegistry: diff --git a/manifests/overlays/odhdashboardconfig/odh-dashboard-config.yaml b/manifests/overlays/odhdashboardconfig/odh-dashboard-config.yaml index db4a69a0e3..3e682d042e 100644 --- a/manifests/overlays/odhdashboardconfig/odh-dashboard-config.yaml +++ b/manifests/overlays/odhdashboardconfig/odh-dashboard-config.yaml @@ -17,6 +17,7 @@ spec: disableProjects: true disablePipelines: true disablePipelineExperiments: true + disableS3Endpoint: true disableModelServing: true disableProjectSharing: true disableCustomServingRuntimes: true