diff --git a/e2e/benchmarks/browserstack-benchmark/package.json b/e2e/benchmarks/browserstack-benchmark/package.json index 3d33d4898cb..5f208b47bcd 100644 --- a/e2e/benchmarks/browserstack-benchmark/package.json +++ b/e2e/benchmarks/browserstack-benchmark/package.json @@ -10,7 +10,6 @@ "devDependencies": { "@tensorflow/tfjs": "link:../../../tfjs", "@tensorflow/tfjs-backend-wasm": "link:../../../link-package/node_modules/@tensorflow/tfjs-backend-wasm", - "@tensorflow/tfjs-vis": "link:../../../tfjs-vis", "argparse": "^2.0.1", "firebase-admin": "^11.0.1", "jasmine": "^3.7.0", @@ -45,6 +44,7 @@ "minimist": "1.2.6" }, "dependencies": { + "@tensorflow/tfjs-vis": "^1.5.1", "JSONStream": "^1.3.5" } } diff --git a/e2e/benchmarks/browserstack-benchmark/yarn.lock b/e2e/benchmarks/browserstack-benchmark/yarn.lock index 77070f2e9ff..43d9360ff97 100644 --- a/e2e/benchmarks/browserstack-benchmark/yarn.lock +++ b/e2e/benchmarks/browserstack-benchmark/yarn.lock @@ -216,41 +216,41 @@ resolved "https://registry.yarnpkg.com/@socket.io/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#568d9beae00b0d835f4f8c53fd55714986492e61" integrity sha512-dOlCBKnDw4iShaIsH/bxujKTM18+2TOAsYz+KSc11Am38H4q5Xw8Bbz97ZYdrVNM+um3p7w86Bvvmcn9q+5+eQ== -"@tensorflow/tfjs-backend-cpu@file:../../../link-package/node_modules/@tensorflow/tfjs-backend-cpu": +"@tensorflow/tfjs-backend-cpu@link:../../../link-package/node_modules/@tensorflow/link-package/node_modules/@tensorflow/tfjs-backend-cpu": version "0.0.0" - dependencies: - "@types/seedrandom" "2.4.27" - seedrandom "2.4.3" "@tensorflow/tfjs-backend-cpu@link:../../../link-package/node_modules/@tensorflow/tfjs-backend-cpu": version "0.0.0" - dependencies: - "@types/seedrandom" "2.4.27" - seedrandom "2.4.3" + uid "" "@tensorflow/tfjs-backend-wasm@link:../../../link-package/node_modules/@tensorflow/tfjs-backend-wasm": version "0.0.0" - dependencies: - "@tensorflow/tfjs-backend-cpu" "file:../../../link-package/node_modules/@tensorflow/tfjs-backend-cpu" - "@types/emscripten" "~0.0.34" + uid "" "@tensorflow/tfjs-backend-webgl@link:../../../link-package/node_modules/@tensorflow/tfjs-backend-webgl": version "0.0.0" + uid "" "@tensorflow/tfjs-converter@link:../../../link-package/node_modules/@tensorflow/tfjs-converter": version "0.0.0" + uid "" "@tensorflow/tfjs-core@link:../../../link-package/node_modules/@tensorflow/tfjs-core": version "0.0.0" + uid "" "@tensorflow/tfjs-data@link:../../../link-package/node_modules/@tensorflow/tfjs-data": version "0.0.0" + uid "" "@tensorflow/tfjs-layers@link:../../../link-package/node_modules/@tensorflow/tfjs-layers": version "0.0.0" + uid "" -"@tensorflow/tfjs-vis@link:../../../tfjs-vis": - version "1.4.3" +"@tensorflow/tfjs-vis@^1.5.1": + version "1.5.1" + resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-vis/-/tfjs-vis-1.5.1.tgz#e959832ee42ed99f71b073a3e6cf8bbfe136e589" + integrity sha512-oNithKiR7VZaE+xUvz6Leww4TYEPhKi8j5xnEYvT3j7brK2Njdvril7UgFtZ8EYZBdeX6XNim5Eu3/23gTQ1dA== dependencies: d3-format "~1.3.0" d3-selection "~1.3.0" @@ -262,18 +262,7 @@ "@tensorflow/tfjs@link:../../../tfjs": version "0.0.0" - dependencies: - "@tensorflow/tfjs-backend-cpu" "link:../../../link-package/node_modules/@tensorflow/tfjs-backend-cpu" - "@tensorflow/tfjs-backend-webgl" "link:../../../link-package/node_modules/@tensorflow/tfjs-backend-webgl" - "@tensorflow/tfjs-converter" "link:../../../link-package/node_modules/@tensorflow/tfjs-converter" - "@tensorflow/tfjs-core" "link:../../../link-package/node_modules/@tensorflow/tfjs-core" - "@tensorflow/tfjs-data" "link:../../../link-package/node_modules/@tensorflow/tfjs-data" - "@tensorflow/tfjs-layers" "link:../../../link-package/node_modules/@tensorflow/tfjs-layers" - argparse "^1.0.10" - chalk "^4.1.0" - core-js "3" - regenerator-runtime "^0.13.5" - yargs "^16.0.3" + uid "" "@tootallnate/once@2": version "2.0.0" @@ -386,6 +375,14 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-3.0.1.tgz#5f8f2bca0a5863cb69bc0b0acd88c96cb1d4ae10" integrity sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA== +"@types/node-fetch@^2.1.2": + version "2.6.3" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.3.tgz#175d977f5e24d93ad0f57602693c435c57ad7e80" + integrity sha512-ETTL1mOEdq/sxUtgtOhKjyB2Irra4cjxksvcMUR5Zr4n+PxVhsCD9WS46oPbHL3et9Zde7CNRr+WUNlcHvsX+w== + dependencies: + "@types/node" "*" + form-data "^3.0.0" + "@types/node@*": version "18.11.9" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.9.tgz#02d013de7058cea16d36168ef2fc653464cfbad4" @@ -396,6 +393,16 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.38.tgz#f8bb07c371ccb1903f3752872c89f44006132947" integrity sha512-5jY9RhV7c0Z4Jy09G+NIDTsCZ5G0L5n+Z+p+Y7t5VJHM30bgwzSjVtlcBxqAj+6L/swIlvtOSzr8rBk/aNyV2g== +"@types/offscreencanvas@~2019.3.0": + version "2019.3.0" + resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553" + integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q== + +"@types/offscreencanvas@~2019.7.0": + version "2019.7.0" + resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.7.0.tgz#e4a932069db47bb3eabeb0b305502d01586fa90d" + integrity sha512-PGcyveRIpL1XIqK8eBsmRBt76eFgtzuPiSTyKHZxnGemp2yzGzWpjYKAfK3wIMiU7eH+851yEpiuP8JZerTmWg== + "@types/qs@*": version "6.9.7" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" @@ -406,10 +413,10 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== -"@types/seedrandom@2.4.27": - version "2.4.27" - resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.27.tgz#9db563937dd86915f69092bc43259d2f48578e41" - integrity sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE= +"@types/seedrandom@^2.4.28": + version "2.4.30" + resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.30.tgz#d2efe425869b84163c2d56e779dddadb9372cbfa" + integrity sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ== "@types/serve-static@*": version "1.15.0" @@ -419,6 +426,16 @@ "@types/mime" "*" "@types/node" "*" +"@types/webgl-ext@0.0.30": + version "0.0.30" + resolved "https://registry.yarnpkg.com/@types/webgl-ext/-/webgl-ext-0.0.30.tgz#0ce498c16a41a23d15289e0b844d945b25f0fb9d" + integrity sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg== + +"@webgpu/types@0.1.21": + version "0.1.21" + resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.21.tgz#b181202daec30d66ccd67264de23814cfd176d3a" + integrity sha512-pUrWq3V5PiSGFLeLxoGqReTZmiiXwY3jRkIG5sLLKjyqNxrwm/04b4nw7LSmGWJcKk59XOM/YRTUwOzo4MMlow== + JSONStream@^1.3.5: version "1.3.5" resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0" @@ -525,6 +542,11 @@ async-retry@^1.3.3: dependencies: retry "0.13.1" +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" @@ -695,6 +717,13 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + commander@2: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" @@ -742,10 +771,10 @@ cookie@~0.4.1: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432" integrity "sha1-DkHyTeXs8xeUfIL8eJ4GqISCRDI= sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==" -core-js@3: - version "3.21.1" - resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.21.1.tgz#f2e0ddc1fc43da6f904706e8e955bc19d06a0d94" - integrity sha512-FRq5b/VMrWlrmCzwRrpDYNxyHP9BcAZC+xHJaqTgIE5091ZV1NTmyh0sGOg5XqpnHvR0svdy0sv1gWA1zmhxig== +core-js@3.29.1: + version "3.29.1" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.29.1.tgz#40ff3b41588b091aaed19ca1aa5cb111803fa9a6" + integrity sha512-+jwgnhg6cQxKYIIjGtAHq2nwUOolo9eoFZ4sHfUH09BLXBgxnH4gA0zEd+t+BO2cNB8idaBtZFcFTRjQJRJmAw== core-js@^1.0.0: version "1.2.7" @@ -1033,6 +1062,11 @@ delaunator@4: resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-4.0.1.tgz#3d779687f57919a7a418f8ab947d3bddb6846957" integrity sha512-WNPWi1IRKZfCt/qIDMfERkDp93+iZEmOxN2yy4Jg+Xhv8SLk2UTqqbe1sfiipn0and9QrE914/ihdx82Y/Giag== +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -1331,6 +1365,15 @@ follow-redirects@^1.0.0: resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7" integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w== +form-data@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f" + integrity sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + from@~0: version "0.1.7" resolved "https://registry.yarnpkg.com/from/-/from-0.1.7.tgz#83c60afc58b9c56997007ed1a768b3ab303a44fe" @@ -1978,7 +2021,7 @@ log4js@^6.4.1: rfdc "^1.3.0" streamroller "^3.0.5" -long@^4.0.0: +long@4.0.0, long@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== @@ -2059,7 +2102,7 @@ mime-db@1.52.0, "mime-db@>= 1.43.0 < 2": resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.0.8, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@^2.0.8, mime-types@^2.1.12, mime-types@~2.1.24, mime-types@~2.1.34: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== @@ -2127,7 +2170,7 @@ negotiator@0.6.3: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== -node-fetch@2.6.7, node-fetch@^1.0.1, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.7, node-fetch@^1.0.1, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@~2.6.1: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== @@ -2451,10 +2494,10 @@ safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@~5.2.0: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -seedrandom@2.4.3: - version "2.4.3" - resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-2.4.3.tgz#2438504dad33917314bff18ac4d794f16d6aaecc" - integrity sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw= +seedrandom@^3.0.5: + version "3.0.5" + resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.5.tgz#54edc85c95222525b0c7a6f6b3543d8e0b3aa0a7" + integrity sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg== semver@^5.6.0: version "5.7.1" @@ -2575,7 +2618,7 @@ string-width@^4.1.0, string-width@^4.2.0: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -string_decoder@^1.1.1: +string_decoder@^1.1.1, string_decoder@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== diff --git a/scripts/generate_cloudbuild.ts b/scripts/generate_cloudbuild.ts index de9699fa29f..d9450cf8bfb 100644 --- a/scripts/generate_cloudbuild.ts +++ b/scripts/generate_cloudbuild.ts @@ -17,146 +17,12 @@ import {printTable} from 'console-table-printer'; import * as fs from 'fs'; import * as yaml from 'js-yaml'; import * as path from 'path'; - import {BAZEL_PACKAGES} from './bazel_packages'; - - -const DEPENDENCY_GRAPH = JSON.parse( - fs.readFileSync(path.join(__dirname, 'package_dependencies.json'), 'utf8')); - -// This is a reverse dependencies graph. Each entry in the graph lists the -// packages that depend on it. -const REVERSE_DEPENDENCY_GRAPH = transposeGraph(DEPENDENCY_GRAPH); - -// Topologically sort the dependency tree and arrange -// steps in dependency order. -const DEPENDENCY_ORDER = topologicalSort(DEPENDENCY_GRAPH); +import {DEPENDENCY_GRAPH, DEPENDENCY_ORDER, findDeps, findReverseDeps} from './graph_utils'; // Steps to exclude from cloudbuild files. const EXCLUDE_STEPS = new Set(['build-deps', 'yarn-common']); -type Graph = Iterable> = { - [node: string]: V -} - -/** - * Verify that an object is a valid graph. - */ -function verifyGraph(graph: Graph) { - const nodes = new Set(Object.keys(graph)); - for (const [node, edges] of Object.entries(graph)) { - for (const edge of edges) { - if (!nodes.has(edge)) { - throw new Error( - `Graph edge ${edge} of node ${node} not found in the graph`); - } - } - } -} - -/** - * Transpose a directed graph i.e. reverse the direction of the edges. - */ -function transposeGraph(graph: Graph) { - verifyGraph(graph); - const transposed: Graph> = {}; - for (const [nodeName, connectedNodes] of Object.entries(graph)) { - for (const connectedNode of connectedNodes) { - if (!transposed[connectedNode]) { - transposed[connectedNode] = new Set(); - } - if (!transposed[nodeName]) { - // Make sure the node itself ends up in the transposed graph. - transposed[nodeName] = new Set(); - } - transposed[connectedNode].add(nodeName); - } - } - return transposed; -} - -/** - * Topologically sort a directed acyclic graph. - * - * Returns a list of graph nodes such that, by following edges, - * you can only move forward in the list, not backward. - */ -function topologicalSort(graph: Graph) { - // We can't use a standard sorting algorithm because - // often, two packages won't have any dependency relationship - // between each other, meaning they are incomparable. - verifyGraph(graph); - const sorted: string[] = []; - - while (sorted.length < Object.keys(graph).length) { - // Find nodes not yet in 'sorted' that have edges - // only to nodes already in 'sorted' - const emptyNodes = Object.entries(graph) - .filter(([node, edges]) => { - if (sorted.includes(node)) { - return false; - } - for (const edge of edges) { - if (!sorted.includes(edge)) { - return false; - } - } - return true; - }) - .map(([node, edges]) => node); - - // If there are no such nodes, then the graph has a cycle. - if (emptyNodes.length === 0) { - throw new Error('Dependency graph has a cycle.'); - } - - for (let node of emptyNodes) { - sorted.push(node); - } - } - return sorted; -} - -/** - * Find all subnodes in the subgraph generated by taking the transitive - * closure at `node`. - */ -function findSubgraph(node: string, graph: Graph, subnodes = new Set()) { - const directSubnodes = graph[node]; - if (directSubnodes) { - for (const directSubnode of directSubnodes) { - if (!subnodes.has(directSubnode)) { - subnodes.add(directSubnode); - findSubgraph(directSubnode, graph, subnodes); - } - } - } - - return subnodes; -} - -/** - * Find the transitive closure of dependencies of the given packages. - */ -function findDeps(packages: Iterable) { - return new Set( - [...packages] - .map(packageName => findSubgraph(packageName, DEPENDENCY_GRAPH)) - .reduce((a, b) => [...a, ...b], [])); -} - -/** - * Find the reverse dependencies of the given packages, i.e. find the - * set of packages that include at least one of the given packages in - * their transitive closure of dependencies. - */ -function findReverseDeps(packages: Iterable) { - return new Set([ - ...packages - ].map(packageName => findSubgraph(packageName, REVERSE_DEPENDENCY_GRAPH)) - .reduce((a, b) => [...a, ...b], [])); -} - interface CloudbuildStep { name: string, id: string, diff --git a/scripts/graph_utils.ts b/scripts/graph_utils.ts new file mode 100644 index 00000000000..f19b0263369 --- /dev/null +++ b/scripts/graph_utils.ts @@ -0,0 +1,150 @@ +// Copyright 2023 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +import * as fs from 'fs'; +import * as path from 'path'; + +export const DEPENDENCY_GRAPH = JSON.parse( + fs.readFileSync(path.join(__dirname, 'package_dependencies.json'), 'utf8')); + +// This is a reverse dependencies graph. Each entry in the graph lists the +// packages that depend on it. +export const REVERSE_DEPENDENCY_GRAPH = transposeGraph(DEPENDENCY_GRAPH); + +// Topologically sort the dependency tree and arrange +// steps in dependency order. +export const DEPENDENCY_ORDER = topologicalSort(DEPENDENCY_GRAPH); + +export type Graph = Iterable> = { + [node: string]: V +} + +/** + * Verify that an object is a valid graph. + */ +export function verifyGraph(graph: Graph) { + const nodes = new Set(Object.keys(graph)); + for (const [node, edges] of Object.entries(graph)) { + for (const edge of edges) { + if (!nodes.has(edge)) { + throw new Error( + `Graph edge ${edge} of node ${node} not found in the graph`); + } + } + } +} + +/** + * Transpose a directed graph i.e. reverse the direction of the edges. + */ +export function transposeGraph(graph: Graph) { + verifyGraph(graph); + const transposed: Graph> = {}; + for (const [nodeName, connectedNodes] of Object.entries(graph)) { + for (const connectedNode of connectedNodes) { + if (!transposed[connectedNode]) { + transposed[connectedNode] = new Set(); + } + if (!transposed[nodeName]) { + // Make sure the node itself ends up in the transposed graph. + transposed[nodeName] = new Set(); + } + transposed[connectedNode].add(nodeName); + } + } + return transposed; +} + +/** + * Topologically sort a directed acyclic graph. + * + * Returns a list of graph nodes such that, by following edges, + * you can only move forward in the list, not backward. + */ +export function topologicalSort(graph: Graph) { + // We can't use a standard sorting algorithm because + // often, two packages won't have any dependency relationship + // between each other, meaning they are incomparable. + verifyGraph(graph); + const sorted: string[] = []; + + while (sorted.length < Object.keys(graph).length) { + // Find nodes not yet in 'sorted' that have edges + // only to nodes already in 'sorted' + const emptyNodes = Object.entries(graph) + .filter(([node, edges]) => { + if (sorted.includes(node)) { + return false; + } + for (const edge of edges) { + if (!sorted.includes(edge)) { + return false; + } + } + return true; + }) + .map(([node, edges]) => node); + + // If there are no such nodes, then the graph has a cycle. + if (emptyNodes.length === 0) { + throw new Error('Dependency graph has a cycle.'); + } + + for (let node of emptyNodes) { + sorted.push(node); + } + } + return sorted; +} + +/** + * Find all subnodes in the subgraph generated by taking the transitive + * closure at `node`. + */ +export function findSubgraph(node: string, graph: Graph, subnodes = new Set()) { + const directSubnodes = graph[node]; + if (directSubnodes) { + for (const directSubnode of directSubnodes) { + if (!subnodes.has(directSubnode)) { + subnodes.add(directSubnode); + findSubgraph(directSubnode, graph, subnodes); + } + } + } + + return subnodes; +} + +/** + * Find the transitive closure of dependencies of the given packages. + */ +export function findDeps(packages: Iterable): Set { + return new Set( + [...packages] + .map(packageName => findSubgraph(packageName, DEPENDENCY_GRAPH)) + .reduce((a, b) => [...a, ...b], [])); +} + +/** + * Find the reverse dependencies of the given packages, i.e. find the + * set of packages that include at least one of the given packages in + * their transitive closure of dependencies. + */ +export function findReverseDeps(packages: Iterable): Set { + return new Set( + [...packages] + .map(packageName => findSubgraph(packageName, REVERSE_DEPENDENCY_GRAPH)) + .reduce((a, b) => [...a, ...b], [])); +} diff --git a/scripts/release-tfjs.ts b/scripts/release-tfjs.ts index e9c2a99620d..c4c37251f33 100644 --- a/scripts/release-tfjs.ts +++ b/scripts/release-tfjs.ts @@ -30,6 +30,7 @@ import * as fs from 'fs'; import * as shell from 'shelljs'; import {TMP_DIR, $, question, makeReleaseDir, createPR, TFJS_RELEASE_UNIT, updateTFJSDependencyVersions, ALPHA_RELEASE_UNIT, getMinorUpdateVersion, getPatchUpdateVersion, E2E_PHASE, getReleaseBlockers, getNightlyVersion} from './release-util'; import * as path from 'path'; +import {findDeps} from './graph_utils'; const parser = new argparse.ArgumentParser({ description: 'Create a release PR for the tfjs monorepo.', @@ -139,8 +140,9 @@ async function main() { } // Guess release version from tfjs-core's latest version, with a minor update. - const newVersion = await getNewVersion('tfjs-core', - incrementVersion ?? getMinorUpdateVersion, !args.guess_version); + const newVersion = await getNewVersion( + 'tfjs-core', incrementVersion ?? getMinorUpdateVersion, + !args.guess_version); // Populate the versions map with new versions for monorepo packages. const versions = new Map(); @@ -154,8 +156,9 @@ async function main() { // version as the other monorepo packages. for (const phase of ALPHA_RELEASE_UNIT.phases) { for (const packageName of phase.packages) { - const newVersion = await getNewVersion(packageName, - incrementVersion ?? getPatchUpdateVersion, !args.guess_version); + const newVersion = await getNewVersion( + packageName, incrementVersion ?? getPatchUpdateVersion, + !args.guess_version); versions.set(packageName, newVersion); } } @@ -203,6 +206,7 @@ async function main() { // Update versions in package.json files. const phases = [...TFJS_RELEASE_UNIT.phases, ...ALPHA_RELEASE_UNIT.phases, E2E_PHASE]; + const errors: Error[] = []; for (const phase of phases) { for (const packageName of phase.packages) { shell.cd(packageName); @@ -223,14 +227,39 @@ async function main() { // Update dependency versions of all package.json files found in the // package to use the new verison numbers (except ones in node_modules). const subpackages = - $(`find ${packagePath} -name package.json -not -path \'*/node_modules/*\'`) - .split('\n'); + $(`find ${ + packagePath} -name package.json -not -path \'*/node_modules/*\'`) + .split('\n'); for (const packageJsonPath of subpackages) { const pkg = fs.readFileSync(packageJsonPath, 'utf8'); console.log(chalk.magenta.bold( `~~~ Update dependency versions for ${packageJsonPath} ~~~`)); - const updated = updateTFJSDependencyVersions(pkg, versions, phase.deps || []); - fs.writeFileSync(packageJsonPath, updated); + + // Only update versions that are a (possibly transitive) dependency of + // the package and are listed in the phase deps (we throw an error + // if we find a dependency that doesn't satisfy these conditions). + const transitiveDeps = [...findDeps([packageName])].filter( + dep => phase.deps.includes(dep)); + + // Also add the package itself so subpackages can use it. + // Some packages, like e2e, are never published to npm, so check first. + if (versions.has(packageName)) { + transitiveDeps.push(packageName); + } + + const packageDependencyVersions = + new Map(transitiveDeps.map(dep => [dep, versions.get(dep)!])); + + try { + const updated = + updateTFJSDependencyVersions(pkg, packageDependencyVersions); + + fs.writeFileSync(packageJsonPath, updated); + } catch (e) { + e.message = `For ${packageJsonPath}, ${packageName} ${e.message}`; + console.error(e.stack); + errors.push(e); + } } shell.cd('..'); @@ -241,6 +270,10 @@ async function main() { } } } + if (errors.length > 0) { + throw new Error('Some package version updates had errors' + errors); + } + // Use dev prefix to avoid branch being locked. const devBranchName = `dev_${releaseBranch}`; diff --git a/scripts/release-util.ts b/scripts/release-util.ts index e945b2b9ffa..57914363e1e 100755 --- a/scripts/release-util.ts +++ b/scripts/release-util.ts @@ -355,40 +355,34 @@ export function updateTFJSDependencyVersions( const parsedPkg = JSON.parse(pkg); - for (const dep of depsToReplace) { - const newVersion = versions.get(dep); - if (!newVersion) { - throw new Error(`No new version found for ${dep}`); - } - // Get the current dependency package version. - let version = ''; - const depNpmName = `@tensorflow/${dep}`; - if (parsedPkg['dependencies'] != null && - parsedPkg['dependencies'][depNpmName] != null) { - version = parsedPkg['dependencies'][depNpmName]; - } else if ( - parsedPkg['peerDependencies'] != null && - parsedPkg['peerDependencies'][depNpmName] != null) { - version = parsedPkg['peerDependencies'][depNpmName]; - } else if ( - parsedPkg['devDependencies'] != null && - parsedPkg['devDependencies'][depNpmName] != null) { - version = parsedPkg['devDependencies'][depNpmName]; - } - if (version == null) { - throw new Error(`No dependency found for ${dep}.`); - } - - let relaxedVersionPrefix = ''; - if (version.startsWith('~') || version.startsWith('^')) { - relaxedVersionPrefix = version.slice(0, 1); + const dependencyMaps: Array<{[index: string]: string}> = [ + parsedPkg['dependencies'], + parsedPkg['peerDependencies'], + parsedPkg['devDependencies'], + ].filter(v => v != null); + + for (const dependencyMap of dependencyMaps) { + for (const [name, version] of Object.entries(dependencyMap)) { + const prefix = '@tensorflow/'; + if (name.startsWith(prefix) && version.startsWith('link:')) { + const tfjsName = name.slice(prefix.length); + const newVersion = versions.get(tfjsName); + if (newVersion == null) { + throw new Error(`Versions map does not include ${tfjsName}`); + } + + let relaxedVersionPrefix = ''; + if (version.startsWith('~') || version.startsWith('^')) { + relaxedVersionPrefix = version.slice(0, 1); + } + const versionLatest = relaxedVersionPrefix + newVersion; + pkg = `${pkg}`.replace( + new RegExp(`"${name}": "${version}"`, 'g'), + `"${name}": "${versionLatest}"`); + } } - const versionLatest = relaxedVersionPrefix + newVersion; - - pkg = `${pkg}`.replace( - new RegExp(`"${depNpmName}": "${version}"`, 'g'), - `"${depNpmName}": "${versionLatest}"`); } + return pkg; }