Skip to content

Commit dd0af13

Browse files
gitKrystangitKrystan
authored
Merge pull request #253 from patricklx/fix-byte-issues
2 parents 844a082 + b3e3bbf commit dd0af13

File tree

8 files changed

+361
-87
lines changed

8 files changed

+361
-87
lines changed

Diff for: ‎src/parse/index.ts

+34-14
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,42 @@
11
import { traverse } from '@babel/core';
22
import type {
33
BlockStatement,
4+
File,
45
Node,
56
ObjectExpression,
67
StaticBlock,
78
} from '@babel/types';
8-
import type { Parsed as RawGlimmerTemplate } from 'content-tag';
99
import { Preprocessor } from 'content-tag';
1010
import type { Parser } from 'prettier';
1111
import { parsers as babelParsers } from 'prettier/plugins/babel.js';
1212

1313
import { PRINTER_NAME } from '../config.js';
1414
import type { Options } from '../options.js';
1515
import { assert } from '../utils/index.js';
16-
import { preprocessTemplateRange } from './preprocess.js';
16+
import {
17+
byteToCharIndex,
18+
preprocessTemplateRange,
19+
type Template,
20+
} from './preprocess.js';
1721

1822
const typescript = babelParsers['babel-ts'] as Parser<Node | undefined>;
1923
const p = new Preprocessor();
2024

2125
/** Converts a node into a GlimmerTemplate node */
2226
function convertNode(
2327
node: BlockStatement | ObjectExpression | StaticBlock,
24-
rawTemplate: RawGlimmerTemplate,
28+
rawTemplate: Template,
2529
): void {
30+
node.innerComments = [];
2631
node.extra = Object.assign(node.extra ?? {}, {
2732
isGlimmerTemplate: true as const,
2833
template: rawTemplate,
2934
});
3035
}
3136

3237
/** Traverses the AST and replaces the transformed template parts with other AST */
33-
function convertAst(ast: Node, rawTemplates: RawGlimmerTemplate[]): void {
34-
const unprocessedTemplates = [...rawTemplates];
38+
function convertAst(ast: File, templates: Template[]): void {
39+
const unprocessedTemplates = [...templates];
3540

3641
traverse(ast, {
3742
enter(path) {
@@ -47,11 +52,10 @@ function convertAst(ast: Node, rawTemplates: RawGlimmerTemplate[]): void {
4752

4853
const templateIndex = unprocessedTemplates.findIndex(
4954
(t) =>
50-
(t.range.start === start && t.range.end === end) ||
55+
(t.utf16Range.start === start && t.utf16Range.end === end) ||
5156
(node.type === 'ObjectExpression' &&
52-
node.extra?.['parenthesized'] === true &&
53-
t.range.start === start - 1 &&
54-
t.range.end === end + 1),
57+
t.utf16Range.start === start - 1 &&
58+
t.utf16Range.end === end + 1),
5559
);
5660
if (templateIndex > -1) {
5761
const rawTemplate = unprocessedTemplates.splice(templateIndex, 1)[0];
@@ -60,6 +64,12 @@ function convertAst(ast: Node, rawTemplates: RawGlimmerTemplate[]): void {
6064
'expected raw template because splice index came from findIndex',
6165
);
6266
}
67+
const index =
68+
node.innerComments?.[0] &&
69+
ast.comments?.indexOf(node.innerComments[0]);
70+
if (ast.comments && index !== undefined && index >= 0) {
71+
ast.comments.splice(index, 1);
72+
}
6373
convertNode(node, rawTemplate);
6474
} else {
6575
return null;
@@ -87,15 +97,25 @@ function preprocess(
8797
fileName: string,
8898
): {
8999
code: string;
90-
rawTemplates: RawGlimmerTemplate[];
100+
templates: Template[];
91101
} {
92102
const rawTemplates = p.parse(code, fileName);
103+
const templates: Template[] = rawTemplates.map((r) => ({
104+
type: r.type,
105+
range: r.range,
106+
contentRange: r.contentRange,
107+
contents: r.contents,
108+
utf16Range: {
109+
start: byteToCharIndex(code, r.range.start),
110+
end: byteToCharIndex(code, r.range.end),
111+
},
112+
}));
93113

94-
for (const rawTemplate of rawTemplates) {
95-
code = preprocessTemplateRange(rawTemplate, code);
114+
for (const template of templates) {
115+
code = preprocessTemplateRange(template, code);
96116
}
97117

98-
return { rawTemplates, code };
118+
return { templates, code };
99119
}
100120

101121
export const parser: Parser<Node | undefined> = {
@@ -106,7 +126,7 @@ export const parser: Parser<Node | undefined> = {
106126
const preprocessed = preprocess(code, options.filepath);
107127
const ast = await typescript.parse(preprocessed.code, options);
108128
assert('expected ast', ast);
109-
convertAst(ast, preprocessed.rawTemplates);
129+
convertAst(ast as File, preprocessed.templates);
110130
return ast;
111131
},
112132
};

Diff for: ‎src/parse/preprocess.ts

+54-73
Original file line numberDiff line numberDiff line change
@@ -1,104 +1,85 @@
1-
import type { Parsed as RawGlimmerTemplate } from 'content-tag';
2-
3-
const EMPTY_SPACE = ' ';
1+
export interface Template {
2+
contents: string;
3+
type: string;
4+
range: {
5+
start: number;
6+
end: number;
7+
};
8+
utf16Range: {
9+
start: number;
10+
end: number;
11+
};
12+
}
413

5-
/**
6-
* Given a string (`original`), replaces the bytes in the given `range` with
7-
* equivalent bytes of empty space (' ') surrounded by the given prefix and
8-
* suffix. The total byte length will not change.
9-
*
10-
* Returns the resulting string.
11-
*/
12-
function replaceByteRange(
13-
originalBuffer: Buffer,
14-
range: { start: number; end: number },
15-
options: { prefix: string; suffix: string },
16-
): string {
17-
const prefixBuffer = Buffer.from(options.prefix);
18-
const suffixBuffer = Buffer.from(options.suffix);
14+
const BufferMap: Map<string, Buffer> = new Map();
1915

20-
// Validate range
21-
if (
22-
range.start < 0 ||
23-
range.end > originalBuffer.length ||
24-
range.start > range.end ||
25-
prefixBuffer.length + suffixBuffer.length > range.end - range.start
26-
) {
27-
throw new Error(
28-
`Invalid byte range:\n\tstart=${range.start}\n\tend=${
29-
range.end
30-
}\n\tprefix=${options.prefix}\n\tsuffix=${
31-
options.suffix
32-
}\n\tstring=\n\t${originalBuffer.toString()}`,
33-
);
16+
function getBuffer(s: string): Buffer {
17+
let buf = BufferMap.get(s);
18+
if (!buf) {
19+
buf = Buffer.from(s);
20+
BufferMap.set(s, buf);
3421
}
22+
return buf;
23+
}
3524

36-
// Adjust the space length to account for the prefix and suffix lengths
37-
const totalReplacementLength = range.end - range.start;
38-
const spaceLength =
39-
totalReplacementLength - prefixBuffer.length - suffixBuffer.length;
40-
41-
// Create a buffer for the replacement
42-
const spaceBuffer = Buffer.alloc(spaceLength, EMPTY_SPACE);
43-
44-
// Concatenate prefix, space, and suffix buffers
45-
const replacementBuffer = Buffer.concat([
46-
prefixBuffer,
47-
spaceBuffer,
48-
suffixBuffer,
49-
]);
50-
51-
// Create buffers for before and after the range using subarray
52-
const beforeRange = originalBuffer.subarray(0, range.start);
53-
const afterRange = originalBuffer.subarray(range.end);
25+
/** Slice string using byte range */
26+
export function sliceByteRange(s: string, a: number, b?: number): string {
27+
const buf = getBuffer(s);
28+
return buf.subarray(a, b).toString();
29+
}
5430

55-
// Concatenate all parts and convert back to a string
56-
const result = Buffer.concat([beforeRange, replacementBuffer, afterRange]);
31+
/** Converts byte index to js char index (utf16) */
32+
export function byteToCharIndex(s: string, byteOffset: number): number {
33+
const buf = getBuffer(s);
34+
return buf.subarray(0, byteOffset).toString().length;
35+
}
5736

58-
if (result.length !== originalBuffer.length) {
59-
throw new Error(
60-
`Result length (${result.length}) does not match original length (${originalBuffer.length})`,
61-
);
62-
}
37+
/** Calculate byte length */
38+
export function byteLength(s: string): number {
39+
return getBuffer(s).length;
40+
}
6341

64-
return result.toString('utf8');
42+
function replaceRange(
43+
s: string,
44+
start: number,
45+
end: number,
46+
substitute: string,
47+
): string {
48+
return sliceByteRange(s, 0, start) + substitute + sliceByteRange(s, end);
6549
}
6650

6751
/**
6852
* Replace the template with a parsable placeholder that takes up the same
6953
* range.
7054
*/
7155
export function preprocessTemplateRange(
72-
rawTemplate: RawGlimmerTemplate,
56+
template: Template,
7357
code: string,
7458
): string {
75-
const codeBuffer = Buffer.from(code);
76-
7759
let prefix: string;
7860
let suffix: string;
7961

80-
if (rawTemplate.type === 'class-member') {
62+
if (template.type === 'class-member') {
8163
// Replace with StaticBlock
82-
prefix = 'static{';
83-
suffix = '}';
64+
prefix = 'static{/*';
65+
suffix = '*/}';
8466
} else {
8567
// Replace with BlockStatement or ObjectExpression
86-
prefix = '{';
87-
suffix = '}';
68+
prefix = '{/*';
69+
suffix = '*/}';
8870

89-
const nextToken = codeBuffer
90-
.subarray(rawTemplate.range.end)
91-
.toString()
92-
.match(/\S+/);
71+
const nextToken = code.slice(template.range.end).toString().match(/\S+/);
9372
if (nextToken && nextToken[0] === 'as') {
9473
// Replace with parenthesized ObjectExpression
9574
prefix = '(' + prefix;
9675
suffix = suffix + ')';
9776
}
9877
}
9978

100-
return replaceByteRange(codeBuffer, rawTemplate.range, {
101-
prefix,
102-
suffix,
103-
});
79+
const content = template.contents.replaceAll('/', '\\/');
80+
const tplLength = template.range.end - template.range.start;
81+
const spaces =
82+
tplLength - byteLength(content) - prefix.length - suffix.length;
83+
const total = prefix + content + ' '.repeat(spaces) + suffix;
84+
return replaceRange(code, template.range.start, template.range.end, total);
10485
}

Diff for: ‎tests/cases/gts/issue-191-d.gts

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import Component from '@glimmer/component';
2+
import { on } from '@ember/modifier';
3+
4+
import { getSnippetElement, toClipboard, withExtraStyles } from './copy-utils';
5+
import Menu from './menu';
6+
7+
/**
8+
* This component is injected via the markdown rendering
9+
*/
10+
export default class CopyMenu extends Component {
11+
copyAsText = (event: Event) => {
12+
let code = getSnippetElement(event);
13+
14+
navigator.clipboard.writeText(code.innerText);
15+
};
16+
17+
copyAsImage = async (event: Event) => {
18+
let code = getSnippetElement(event);
19+
20+
await withExtraStyles(code, () => toClipboard(code));
21+
};
22+
23+
<template>
24+
<Menu data-test-copy-menu>
25+
<:trigger as |t|>
26+
<t.Default class="absolute top-3 right-4 z-10" data-test-copy-menu>
27+
📋
28+
</t.Default>
29+
</:trigger>
30+
31+
<:options as |Item|>
32+
<Item {{on "click" this.copyAsText}}>
33+
Copy as text
34+
</Item>
35+
<Item {{on "click" this.copyAsImage}}>
36+
Copy as image
37+
</Item>
38+
</:options>
39+
</Menu>
40+
</template>
41+
}

Diff for: ‎tests/cases/gts/issue-191-e.gts

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import Component from '@glimmer/component';
2+
3+
/**
4+
* This component contains a multi-byte character
5+
*/
6+
export default class MultiByteCharComponent extends Component {
7+
get rows() {
8+
console.log('abc다윤6')
9+
return []
10+
}
11+
<template>
12+
{{#each this.rows as |row|}}
13+
{{row.id}}
14+
{{/each}}
15+
</template>
16+
}

0 commit comments

Comments
 (0)