|
| 1 | +import fs from 'fs' |
| 2 | +import path from 'path' |
| 3 | +import basic from './basic' |
| 4 | +import { generatePropMapping } from '../../prop_mapping' |
| 5 | +import { Intrinsic } from '../../../intrinsics' |
| 6 | +import chalk from 'chalk' |
| 7 | + |
| 8 | +const PROP_MAPPING_TEST_SUITES = [ |
| 9 | + basic, |
| 10 | +] |
| 11 | + |
| 12 | +// Order important - worst to best |
| 13 | +enum PropResultType { |
| 14 | + FalsePositive, |
| 15 | + Miss, |
| 16 | + PartiallyCorrect, |
| 17 | + Correct, |
| 18 | +} |
| 19 | + |
| 20 | +type ComponentResult = { |
| 21 | + componentName: string |
| 22 | + results: PropResultType[] |
| 23 | +} |
| 24 | + |
| 25 | +type ResultTotals = { |
| 26 | + totalMappings: number |
| 27 | + correct: number |
| 28 | + partiallyCorrect: number |
| 29 | + falsePositives: number |
| 30 | +} |
| 31 | + |
| 32 | +export type BenchmarkingSuiteResult = { |
| 33 | + name: string |
| 34 | + results: ComponentResult[] |
| 35 | +} |
| 36 | + |
| 37 | +function areIntrinsicsEqual(a: Intrinsic, b: Intrinsic) { |
| 38 | + function replacer(key: string, value: boolean | string | string[] | Record<string, any>) { |
| 39 | + // TODO is order of children([]) significant? |
| 40 | + if (Array.isArray(value)) { |
| 41 | + return value.sort() |
| 42 | + } |
| 43 | + if (value && typeof value === 'object') { |
| 44 | + return Object.keys(value) |
| 45 | + .sort() |
| 46 | + .reduce( |
| 47 | + (acc, key) => { |
| 48 | + acc[key] = value[key] |
| 49 | + return acc |
| 50 | + }, |
| 51 | + {} as Record<string, any>, |
| 52 | + ) |
| 53 | + } |
| 54 | + return value |
| 55 | + } |
| 56 | + |
| 57 | + return JSON.stringify(a, replacer) === JSON.stringify(b, replacer) |
| 58 | +} |
| 59 | + |
| 60 | +function isASubsetOf(a: any, b: any): boolean { |
| 61 | + if (a === undefined) { |
| 62 | + return true |
| 63 | + } |
| 64 | + // TODO is order of children([]) significant? |
| 65 | + if (Array.isArray(a) && Array.isArray(b)) { |
| 66 | + return a.every((value) => b.includes(value)) |
| 67 | + } |
| 68 | + if (a && typeof a === 'object' && b && typeof b === 'object') { |
| 69 | + return Object.keys(a).every((subKey) => isASubsetOf(a[subKey], b[subKey])) |
| 70 | + } |
| 71 | + return a === b |
| 72 | +} |
| 73 | + |
| 74 | +function getPropMappingBenchmarkingResults( |
| 75 | + printDiffSinceLastRun = false, |
| 76 | + prevResults?: BenchmarkingSuiteResult[], |
| 77 | +) { |
| 78 | + /** |
| 79 | + * For each suite of components, get the total number of props that have a mapping |
| 80 | + * in the test data, as well as the count of mappings we've correctly generated. |
| 81 | + * We then divide correct / total to get the overall success rate |
| 82 | + */ |
| 83 | + return PROP_MAPPING_TEST_SUITES.map(({ testCases, name }) => { |
| 84 | + return testCases.reduce( |
| 85 | + (suiteResults, testCase, caseIndex) => { |
| 86 | + const componentResults: PropResultType[] = [] |
| 87 | + const actualResult = generatePropMapping({ |
| 88 | + componentPropertyDefinitions: testCase.componentPropertyDefinitions, |
| 89 | + signature: testCase.signature, |
| 90 | + }) |
| 91 | + |
| 92 | + // iterate expected individual prop mappings for current component |
| 93 | + Object.keys(testCase.perfectResult).forEach((prop, propIndex) => { |
| 94 | + let propResult = PropResultType.Miss |
| 95 | + if (prop in actualResult) { |
| 96 | + if (areIntrinsicsEqual(actualResult[prop], testCase.perfectResult[prop])) { |
| 97 | + propResult = PropResultType.Correct |
| 98 | + } else if (isASubsetOf(actualResult[prop], testCase.perfectResult[prop])) { |
| 99 | + // TODO would be good if this also reflected |
| 100 | + propResult = PropResultType.PartiallyCorrect |
| 101 | + } else { |
| 102 | + propResult = PropResultType.FalsePositive |
| 103 | + } |
| 104 | + } |
| 105 | + if (printDiffSinceLastRun && prevResults) { |
| 106 | + const prevPropResult = prevResults.find( |
| 107 | + (prevSuite) => prevSuite.name === suiteResults.name, |
| 108 | + )?.results[caseIndex]?.results[propIndex] |
| 109 | + if (propResult !== prevPropResult) { |
| 110 | + const text = |
| 111 | + `Prop mapping result changed: ${PropResultType[prevPropResult!]} -> ${PropResultType[propResult]}` + |
| 112 | + ` (${suiteResults.name} - ${testCase.exportName} - ${prop})` + |
| 113 | + (prop in actualResult |
| 114 | + ? `\nResult: ${JSON.stringify(actualResult[prop], null, 2)}` |
| 115 | + : '') |
| 116 | + if (propResult > prevPropResult!) { |
| 117 | + console.log(chalk.green(text)) |
| 118 | + } else { |
| 119 | + console.log(chalk.red(text)) |
| 120 | + } |
| 121 | + } |
| 122 | + } |
| 123 | + componentResults.push(propResult) |
| 124 | + }) |
| 125 | + |
| 126 | + suiteResults.results.push({ |
| 127 | + componentName: testCase.exportName, |
| 128 | + results: componentResults, |
| 129 | + }) |
| 130 | + |
| 131 | + return suiteResults |
| 132 | + }, |
| 133 | + { |
| 134 | + name, |
| 135 | + results: [], |
| 136 | + } as BenchmarkingSuiteResult, |
| 137 | + ) |
| 138 | + }) |
| 139 | +} |
| 140 | + |
| 141 | +export function runPropMappingBenchmarking() { |
| 142 | + const prevResults = JSON.parse( |
| 143 | + fs.readFileSync(path.join(__dirname, 'prop_mapping_benchmarking_snapshot.json'), 'utf8'), |
| 144 | + ) as BenchmarkingSuiteResult[] |
| 145 | + |
| 146 | + const results = getPropMappingBenchmarkingResults(true, prevResults) |
| 147 | + |
| 148 | + const hasChanged = JSON.stringify(results) !== JSON.stringify(prevResults) |
| 149 | + |
| 150 | + return { results, prevResults, hasChanged } |
| 151 | +} |
| 152 | + |
| 153 | +function getSuiteTotals(suiteResult: BenchmarkingSuiteResult) { |
| 154 | + const propResults = suiteResult.results.flatMap((r) => r.results) |
| 155 | + return propResults.reduce( |
| 156 | + (acc, propResult) => { |
| 157 | + if (propResult === PropResultType.Correct) { |
| 158 | + acc.correct++ |
| 159 | + } else if (propResult === PropResultType.PartiallyCorrect) { |
| 160 | + acc.partiallyCorrect++ |
| 161 | + } else if (propResult === PropResultType.FalsePositive) { |
| 162 | + acc.falsePositives++ |
| 163 | + } |
| 164 | + acc.totalMappings++ |
| 165 | + return acc |
| 166 | + }, |
| 167 | + { |
| 168 | + totalMappings: 0, |
| 169 | + correct: 0, |
| 170 | + partiallyCorrect: 0, |
| 171 | + falsePositives: 0, |
| 172 | + } as ResultTotals, |
| 173 | + ) |
| 174 | +} |
| 175 | + |
| 176 | +export function prettyPrintBenchmarkingResults( |
| 177 | + suiteResultsWithoutTotal: BenchmarkingSuiteResult[], |
| 178 | + prevSuiteResultsWithoutTotal: BenchmarkingSuiteResult[], |
| 179 | +) { |
| 180 | + const prettyResults: Record<string, Record<string, string | number>> = {} |
| 181 | + |
| 182 | + const addTotals = (allResults: BenchmarkingSuiteResult[]) => [ |
| 183 | + ...allResults, |
| 184 | + { |
| 185 | + name: 'TOTAL', |
| 186 | + results: allResults.flatMap((r) => r.results), |
| 187 | + }, |
| 188 | + ] |
| 189 | + |
| 190 | + const suiteResults = addTotals(suiteResultsWithoutTotal) |
| 191 | + const prevSuiteResults = addTotals(prevSuiteResultsWithoutTotal) |
| 192 | + |
| 193 | + suiteResults.forEach((suiteResult, index) => { |
| 194 | + const prevTotals = getSuiteTotals(prevSuiteResults[index]) |
| 195 | + const currenttotals = getSuiteTotals(suiteResult) |
| 196 | + |
| 197 | + const printDiff = (print: (totals: ResultTotals) => string | number) => { |
| 198 | + const prev = print(prevTotals) |
| 199 | + const current = print(currenttotals) |
| 200 | + return current === prev ? current : `${prev} -> ${current}` |
| 201 | + } |
| 202 | + |
| 203 | + prettyResults[suiteResult.name] = { |
| 204 | + 'Total Mappings': printDiff((totals) => totals.totalMappings), |
| 205 | + Correct: printDiff( |
| 206 | + (totals) => |
| 207 | + `${totals.correct} (${((totals.correct / totals.totalMappings) * 100).toFixed(1)}%)`, |
| 208 | + ), |
| 209 | + 'Partially correct': printDiff( |
| 210 | + (totals) => |
| 211 | + `${totals.partiallyCorrect} (${((totals.partiallyCorrect / totals.totalMappings) * 100).toFixed(1)}%)`, |
| 212 | + ), |
| 213 | + 'False positives': printDiff( |
| 214 | + (totals) => |
| 215 | + `${totals.falsePositives} (${((totals.falsePositives / totals.totalMappings) * 100).toFixed(1)}%)`, |
| 216 | + ), |
| 217 | + } |
| 218 | + }) |
| 219 | + |
| 220 | + console.table(prettyResults) |
| 221 | +} |
0 commit comments