From 50df190ba8b503f0bb298b1b42e12fbea0a02f6e Mon Sep 17 00:00:00 2001 From: Zhonglei Ma Date: Tue, 30 Sep 2025 14:54:52 +0800 Subject: [PATCH 1/5] syntax highlighting does not correctly recognize parameter name with a hyphen ('-') --- packages/compiler/src/core/charcode.ts | 18 ++++++++++ packages/compiler/src/core/scanner.ts | 47 ++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/core/charcode.ts b/packages/compiler/src/core/charcode.ts index 06306655c0d..9d682696d25 100644 --- a/packages/compiler/src/core/charcode.ts +++ b/packages/compiler/src/core/charcode.ts @@ -229,6 +229,17 @@ export function isAsciiIdentifierContinue(ch: number) { ); } +export function isAsciiDocIdentifierContinue(ch: number) { + return ( + (ch >= CharCode.A && ch <= CharCode.Z) || + (ch >= CharCode.a && ch <= CharCode.z) || + (ch >= CharCode._0 && ch <= CharCode._9) || + ch === CharCode.$ || + ch === CharCode._ || + ch === CharCode.Minus // Support hyphen + ); +} + export function isIdentifierStart(codePoint: number) { return ( isAsciiIdentifierStart(codePoint) || @@ -243,6 +254,13 @@ export function isIdentifierContinue(codePoint: number) { ); } +export function isDocIdentifierContinue(codePoint: number) { + return ( + isAsciiDocIdentifierContinue(codePoint) || + (codePoint > CharCode.MaxAscii && isNonAsciiIdentifierCharacter(codePoint)) + ); +} + export function isNonAsciiIdentifierCharacter(codePoint: number) { return lookupInNonAsciiMap(codePoint, nonAsciiIdentifierMap); } diff --git a/packages/compiler/src/core/scanner.ts b/packages/compiler/src/core/scanner.ts index 17800356258..de9f1e3521a 100644 --- a/packages/compiler/src/core/scanner.ts +++ b/packages/compiler/src/core/scanner.ts @@ -1,10 +1,12 @@ import { CharCode, codePointBefore, + isAsciiDocIdentifierContinue, isAsciiIdentifierContinue, isAsciiIdentifierStart, isBinaryDigit, isDigit, + isDocIdentifierContinue, isHexDigit, isIdentifierContinue, isIdentifierStart, @@ -536,6 +538,12 @@ export interface Scanner { * getTokenText(). */ getTokenValue(): string; + + /** + * Scan a documentation identifier that supports hyphens. + * This method should be used in doc comment parsing context. + */ + scanDocIdentifier(): Token.Identifier; } export enum TokenFlags { @@ -616,6 +624,7 @@ export function createScanner( eof, getTokenText, getTokenValue, + scanDocIdentifier, }; function eof() { @@ -873,7 +882,7 @@ export function createScanner( } if (isAsciiIdentifierStart(ch)) { - return scanIdentifier(); + return scanDocIdentifier(); } if (ch <= CharCode.MaxAscii) { @@ -882,7 +891,7 @@ export function createScanner( const cp = input.codePointAt(position)!; if (isIdentifierStart(cp)) { - return scanNonAsciiIdentifier(cp); + return scanNonAsciiDocIdentifier(cp); } return scanUnknown(Token.DocText); @@ -1532,6 +1541,40 @@ export function createScanner( return (token = Token.Identifier); } + function scanDocIdentifier(): Token.Identifier { + tokenPosition = position; + tokenFlags = TokenFlags.None; + let ch: number; + + do { + position++; + if (eof()) { + return (token = Token.Identifier); + } + } while (isAsciiDocIdentifierContinue((ch = input.charCodeAt(position)))); + + if (ch > CharCode.MaxAscii) { + const cp = input.codePointAt(position)!; + if (isNonAsciiIdentifierCharacter(cp)) { + return scanNonAsciiDocIdentifier(cp); + } + } + + return (token = Token.Identifier); + } + + function scanNonAsciiDocIdentifier(startCodePoint: number): Token.Identifier { + tokenFlags |= TokenFlags.NonAscii; + let cp = startCodePoint; + do { + position += utf16CodeUnits(cp); + if (eof()) break; + cp = input.codePointAt(position)!; + } while (isDocIdentifierContinue(cp)); + + return (token = Token.Identifier); + } + function scanBacktickedIdentifier(): Token.Identifier { position++; // consume '`' From 3f8264afff360437379df1c92f6f890d844d86f9 Mon Sep 17 00:00:00 2001 From: Zhonglei Ma Date: Tue, 30 Sep 2025 15:04:46 +0800 Subject: [PATCH 2/5] Create syntax-highlighting-not-correctly-recognize-hyphen-param-2025-8-30-6-59-7.md --- ...t-correctly-recognize-hyphen-param-2025-8-30-6-59-7.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .chronus/changes/syntax-highlighting-not-correctly-recognize-hyphen-param-2025-8-30-6-59-7.md diff --git a/.chronus/changes/syntax-highlighting-not-correctly-recognize-hyphen-param-2025-8-30-6-59-7.md b/.chronus/changes/syntax-highlighting-not-correctly-recognize-hyphen-param-2025-8-30-6-59-7.md new file mode 100644 index 00000000000..a91be199d9c --- /dev/null +++ b/.chronus/changes/syntax-highlighting-not-correctly-recognize-hyphen-param-2025-8-30-6-59-7.md @@ -0,0 +1,8 @@ +--- +# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking +changeKind: fix +packages: + - "@typespec/compiler" +--- + +Syntax highlighting does not correctly recognize parameter name with hyphen From 028b9ff864d3fccad39012ff786ded3f6ed1b4d5 Mon Sep 17 00:00:00 2001 From: Zhonglei Ma Date: Tue, 21 Oct 2025 17:30:01 +0800 Subject: [PATCH 3/5] updated --- packages/compiler/src/core/charcode.ts | 18 -- packages/compiler/src/core/scanner.ts | 250 +++++++++++++------ packages/compiler/src/server/type-details.ts | 2 +- 3 files changed, 180 insertions(+), 90 deletions(-) diff --git a/packages/compiler/src/core/charcode.ts b/packages/compiler/src/core/charcode.ts index 9d682696d25..06306655c0d 100644 --- a/packages/compiler/src/core/charcode.ts +++ b/packages/compiler/src/core/charcode.ts @@ -229,17 +229,6 @@ export function isAsciiIdentifierContinue(ch: number) { ); } -export function isAsciiDocIdentifierContinue(ch: number) { - return ( - (ch >= CharCode.A && ch <= CharCode.Z) || - (ch >= CharCode.a && ch <= CharCode.z) || - (ch >= CharCode._0 && ch <= CharCode._9) || - ch === CharCode.$ || - ch === CharCode._ || - ch === CharCode.Minus // Support hyphen - ); -} - export function isIdentifierStart(codePoint: number) { return ( isAsciiIdentifierStart(codePoint) || @@ -254,13 +243,6 @@ export function isIdentifierContinue(codePoint: number) { ); } -export function isDocIdentifierContinue(codePoint: number) { - return ( - isAsciiDocIdentifierContinue(codePoint) || - (codePoint > CharCode.MaxAscii && isNonAsciiIdentifierCharacter(codePoint)) - ); -} - export function isNonAsciiIdentifierCharacter(codePoint: number) { return lookupInNonAsciiMap(codePoint, nonAsciiIdentifierMap); } diff --git a/packages/compiler/src/core/scanner.ts b/packages/compiler/src/core/scanner.ts index de9f1e3521a..d6f98e739cf 100644 --- a/packages/compiler/src/core/scanner.ts +++ b/packages/compiler/src/core/scanner.ts @@ -1,12 +1,10 @@ import { CharCode, codePointBefore, - isAsciiDocIdentifierContinue, isAsciiIdentifierContinue, isAsciiIdentifierStart, isBinaryDigit, isDigit, - isDocIdentifierContinue, isHexDigit, isIdentifierContinue, isIdentifierStart, @@ -220,6 +218,7 @@ export type DocToken = | Token.CloseBrace | Token.Identifier | Token.Hyphen + | Token.Dot | Token.DocText | Token.DocCodeSpan | Token.DocCodeFenceDelimiter @@ -538,12 +537,6 @@ export interface Scanner { * getTokenText(). */ getTokenValue(): string; - - /** - * Scan a documentation identifier that supports hyphens. - * This method should be used in doc comment parsing context. - */ - scanDocIdentifier(): Token.Identifier; } export enum TokenFlags { @@ -624,7 +617,6 @@ export function createScanner( eof, getTokenText, getTokenValue, - scanDocIdentifier, }; function eof() { @@ -868,7 +860,10 @@ export function createScanner( case CharCode.Backtick: return lookAhead(1) === CharCode.Backtick && lookAhead(2) === CharCode.Backtick ? next(Token.DocCodeFenceDelimiter, 3) - : scanDocCodeSpan(); + : scanDocMemberAccessOrIdentifier(); + + case CharCode.Dot: + return next(Token.Dot); case CharCode.LessThan: case CharCode.GreaterThan: @@ -882,7 +877,7 @@ export function createScanner( } if (isAsciiIdentifierStart(ch)) { - return scanDocIdentifier(); + return scanDocIdentifierOrMemberAccess(); } if (ch <= CharCode.MaxAscii) { @@ -891,7 +886,7 @@ export function createScanner( const cp = input.codePointAt(position)!; if (isIdentifierStart(cp)) { - return scanNonAsciiDocIdentifier(cp); + return scanDocNonAsciiIdentifierOrMemberAccess(cp); } return scanUnknown(Token.DocText); @@ -900,6 +895,174 @@ export function createScanner( return (token = Token.EndOfFile); } + function scanDocIdentifierOrMemberAccess(): Token.Identifier { + if (!scanNormalIdentifierPart()) { + return (token = Token.Identifier); + } + + return scanMemberAccessContinuation(); + } + + function scanDocNonAsciiIdentifierOrMemberAccess(startCodePoint: number): Token.Identifier { + if (!scanNonAsciiIdentifierPart(startCodePoint)) { + return (token = Token.Identifier); + } + + return scanMemberAccessContinuation(); + } + + function scanDocMemberAccessOrIdentifier(): Token.Identifier { + if (!scanSingleBacktickedPart()) { + return unterminated(Token.Identifier); + } + + return scanMemberAccessContinuation(); + } + + function scanMemberAccessContinuation(): Token.Identifier { + while (position < endPosition) { + let tempPos = position; + + // Skip whitespace + while (tempPos < endPosition && isWhiteSpaceSingleLine(input.charCodeAt(tempPos))) { + tempPos++; + } + + // Check for . + if (tempPos >= endPosition || input.charCodeAt(tempPos) !== CharCode.Dot) { + break; + } + tempPos++; // Skip . + + // Skip whitespace after . + while (tempPos < endPosition && isWhiteSpaceSingleLine(input.charCodeAt(tempPos))) { + tempPos++; + } + + // Check what comes after . + if (tempPos >= endPosition) { + break; + } + + const nextChar = input.charCodeAt(tempPos); + position = tempPos; + + if (nextChar === CharCode.Backtick) { + // backticked identifier + if (!scanSingleBacktickedPart()) { + return unterminated(Token.Identifier); + } + } else if (isAsciiIdentifierStart(nextChar)) { + // Ordinary ASCII identifier + if (!scanNormalIdentifierPart()) { + break; + } + } else if (nextChar > CharCode.MaxAscii) { + // Non-ASCII identifier + const cp = input.codePointAt(position)!; + if (isIdentifierStart(cp)) { + if (!scanNonAsciiIdentifierPart(cp)) { + break; + } + } else { + break; + } + } else { + // Not a valid identifier, stop + break; + } + } + + return (token = Token.Identifier); + } + + function scanSingleBacktickedPart(): boolean { + if (eof() || input.charCodeAt(position) !== CharCode.Backtick) { + return false; + } + + position++; // Consume ` + tokenFlags |= TokenFlags.Backticked; + + while (!eof()) { + const ch = input.charCodeAt(position); + switch (ch) { + case CharCode.Backslash: + position++; + tokenFlags |= TokenFlags.Escaped; + if (!eof()) position++; // Consume escaped characters + continue; + case CharCode.Backtick: + position++; // Consume ending ` + return true; + case CharCode.CarriageReturn: + case CharCode.LineFeed: + return false; // Unterminated + default: + if (ch > CharCode.MaxAscii) { + tokenFlags |= TokenFlags.NonAscii; + } + position++; + } + } + + return false; // Unterminated + } + + function scanNormalIdentifierPart(): boolean { + if (eof()) { + return false; + } + + const ch = input.charCodeAt(position); + if (!isAsciiIdentifierStart(ch)) { + return false; + } + + // Scan for common identifiers + do { + position++; + if (eof()) { + return true; + } + } while (isAsciiIdentifierContinue(input.charCodeAt(position))); + + // Check if there are non-ascii characters + if (!eof()) { + const nextChar = input.charCodeAt(position); + if (nextChar > CharCode.MaxAscii) { + let cp = input.codePointAt(position)!; + if (isNonAsciiIdentifierCharacter(cp)) { + // Contains non-ASCII identifier characters, continue scanning + tokenFlags |= TokenFlags.NonAscii; + do { + position += utf16CodeUnits(cp); + if (eof()) break; + cp = input.codePointAt(position)!; + } while (isIdentifierContinue(cp)); + } + } + } + + return true; + } + + function scanNonAsciiIdentifierPart(startCodePoint: number): boolean { + if (eof()) { + return false; + } + + tokenFlags |= TokenFlags.NonAscii; + let cp = startCodePoint; + do { + position += utf16CodeUnits(cp); + if (eof()) break; + cp = input.codePointAt(position)!; + } while (isIdentifierContinue(cp)); + + return true; + } + function reScanStringTemplate(lastTokenFlags: TokenFlags): StringTemplateToken { position = tokenPosition; tokenFlags = TokenFlags.None; @@ -1070,24 +1233,6 @@ export function createScanner( return terminated ? token : unterminated(token); } - function scanDocCodeSpan(): Token.DocCodeSpan { - position++; // consume '`' - - loop: for (; !eof(); position++) { - const ch = input.charCodeAt(position); - switch (ch) { - case CharCode.Backtick: - position++; - return (token = Token.DocCodeSpan); - case CharCode.CarriageReturn: - case CharCode.LineFeed: - break loop; - } - } - - return unterminated(Token.DocCodeSpan); - } - function scanString(tokenFlags: TokenFlags): Token.StringLiteral | Token.StringTemplateHead { if (tokenFlags & TokenFlags.TripleQuoted) { position += 3; // consume '"""' @@ -1214,14 +1359,11 @@ export function createScanner( } function getIdentifierTokenValue(): string { - const start = tokenFlags & TokenFlags.Backticked ? tokenPosition + 1 : tokenPosition; - const end = - tokenFlags & TokenFlags.Backticked && !(tokenFlags & TokenFlags.Unterminated) - ? position - 1 - : position; - + // For mixed member access expressions, the original text is returned directly without backtick processing. const text = - tokenFlags & TokenFlags.Escaped ? unescapeString(start, end) : input.substring(start, end); + tokenFlags & TokenFlags.Escaped + ? unescapeString(tokenPosition, position) + : input.substring(tokenPosition, position); if (tokenFlags & TokenFlags.NonAscii) { return text.normalize("NFC"); @@ -1541,40 +1683,6 @@ export function createScanner( return (token = Token.Identifier); } - function scanDocIdentifier(): Token.Identifier { - tokenPosition = position; - tokenFlags = TokenFlags.None; - let ch: number; - - do { - position++; - if (eof()) { - return (token = Token.Identifier); - } - } while (isAsciiDocIdentifierContinue((ch = input.charCodeAt(position)))); - - if (ch > CharCode.MaxAscii) { - const cp = input.codePointAt(position)!; - if (isNonAsciiIdentifierCharacter(cp)) { - return scanNonAsciiDocIdentifier(cp); - } - } - - return (token = Token.Identifier); - } - - function scanNonAsciiDocIdentifier(startCodePoint: number): Token.Identifier { - tokenFlags |= TokenFlags.NonAscii; - let cp = startCodePoint; - do { - position += utf16CodeUnits(cp); - if (eof()) break; - cp = input.codePointAt(position)!; - } while (isDocIdentifierContinue(cp)); - - return (token = Token.Identifier); - } - function scanBacktickedIdentifier(): Token.Identifier { position++; // consume '`' diff --git a/packages/compiler/src/server/type-details.ts b/packages/compiler/src/server/type-details.ts index 5b27e2c26a7..ec2c4a2eaac 100644 --- a/packages/compiler/src/server/type-details.ts +++ b/packages/compiler/src/server/type-details.ts @@ -50,7 +50,7 @@ export async function getSymbolDetails( } lines.push( //prettier-ignore - `_@${tag.tagName.sv}_${"paramName" in tag ? ` \`${tag.paramName.sv}\`` : ""} —\n${getDocContent(tag.content)}`, + `_@${tag.tagName.sv}_${"paramName" in tag ? ` \`${tag.paramName.sv}\`` : ("propName" in tag ? ` ${tag.propName.sv}` : "")} —\n${getDocContent(tag.content)}`, ); } } From ae11644ac1c192635a8f21d85d1abebd0b940ea9 Mon Sep 17 00:00:00 2001 From: Zhonglei Ma Date: Wed, 22 Oct 2025 11:36:23 +0800 Subject: [PATCH 4/5] fixed pipline issue --- packages/compiler/src/core/scanner.ts | 17 +++++++++++++---- packages/compiler/src/server/type-details.ts | 2 +- .../generated-defs/TypeSpec.Protobuf.ts | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/compiler/src/core/scanner.ts b/packages/compiler/src/core/scanner.ts index 731c4b1f822..6fc1f4d4d81 100644 --- a/packages/compiler/src/core/scanner.ts +++ b/packages/compiler/src/core/scanner.ts @@ -1359,11 +1359,20 @@ export function createScanner( } function getIdentifierTokenValue(): string { - // For mixed member access expressions, the original text is returned directly without backtick processing. + // Check if it is a pure backticked identifier (the entire token is surrounded by a single backtick pair) + const isPureBackticked = + tokenFlags & TokenFlags.Backticked && + !(tokenFlags & TokenFlags.Unterminated) && + input.charCodeAt(tokenPosition) === CharCode.Backtick && + input.charCodeAt(position - 1) === CharCode.Backtick && + // Make sure there are no other backticks in the middle (excluding mixed member access) + input.substring(tokenPosition + 1, position - 1).indexOf("`") === -1; + + const start = isPureBackticked ? tokenPosition + 1 : tokenPosition; + const end = isPureBackticked ? position - 1 : position; + const text = - tokenFlags & TokenFlags.Escaped - ? unescapeString(tokenPosition, position) - : input.substring(tokenPosition, position); + tokenFlags & TokenFlags.Escaped ? unescapeString(start, end) : input.substring(start, end); if (tokenFlags & TokenFlags.NonAscii) { return text.normalize("NFC"); diff --git a/packages/compiler/src/server/type-details.ts b/packages/compiler/src/server/type-details.ts index ec2c4a2eaac..6d87ac68cd5 100644 --- a/packages/compiler/src/server/type-details.ts +++ b/packages/compiler/src/server/type-details.ts @@ -50,7 +50,7 @@ export async function getSymbolDetails( } lines.push( //prettier-ignore - `_@${tag.tagName.sv}_${"paramName" in tag ? ` \`${tag.paramName.sv}\`` : ("propName" in tag ? ` ${tag.propName.sv}` : "")} —\n${getDocContent(tag.content)}`, + `_@${tag.tagName.sv}_${"paramName" in tag ? ` \`${tag.paramName.sv}\`` : ("propName" in tag ? ` \`${tag.propName.sv}\`` : "")} —\n${getDocContent(tag.content)}`, ); } } diff --git a/packages/protobuf/generated-defs/TypeSpec.Protobuf.ts b/packages/protobuf/generated-defs/TypeSpec.Protobuf.ts index 0e59e7977b2..968cbc9abe9 100644 --- a/packages/protobuf/generated-defs/TypeSpec.Protobuf.ts +++ b/packages/protobuf/generated-defs/TypeSpec.Protobuf.ts @@ -28,7 +28,7 @@ export type MessageDecorator = (context: DecoratorContext, target: Type) => void * - not fall within the implementation reserved range of 19000 to 19999, inclusive. * - not fall within any range that was [marked reserved](# * - * @TypeSpec .Protobuf.reserve). + * @TypeSpec.Protobuf.reserve ). * * #### API Compatibility Note * From 05803dc5b9c9e430a990f6008c67ba1daeb10498 Mon Sep 17 00:00:00 2001 From: Zhonglei Ma Date: Wed, 22 Oct 2025 13:33:02 +0800 Subject: [PATCH 5/5] fixed test --- packages/compiler/test/scanner.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/compiler/test/scanner.test.ts b/packages/compiler/test/scanner.test.ts index e2cb213801c..34beb6b9e1c 100644 --- a/packages/compiler/test/scanner.test.ts +++ b/packages/compiler/test/scanner.test.ts @@ -339,7 +339,7 @@ describe("compiler: scanner", () => { [Token.Identifier, "`1!=2`", { pos: 28, value: "1!=2", line: 2, character: 8 }], [Token.Whitespace, " ", { pos: 34, value: " ", line: 2, character: 14 }], - [Token.Identifier, "`x\\`x`", { pos: 35, value: "x`x", line: 2, character: 15 }], + [Token.Identifier, "`x\\`x`", { pos: 35, value: "`x`x`", line: 2, character: 15 }], [Token.Whitespace, " ", { pos: 41, value: " ", line: 2, character: 21 }], [Token.Identifier, "`\\\\x`", { pos: 42, value: "\\x", line: 2, character: 22 }], @@ -349,7 +349,7 @@ describe("compiler: scanner", () => { [Token.Identifier, "`import`", { pos: 55, value: "import", line: 2, character: 35 }], [Token.Whitespace, " ", { pos: 63, value: " ", line: 2, character: 43 }], - [Token.Identifier, "`a\\n\\t\\`b`", { pos: 64, value: "a\n\t`b", line: 2, character: 44 }], + [Token.Identifier, "`a\\n\\t\\`b`", { pos: 64, value: "`a\n\t`b`", line: 2, character: 44 }], ]); });