Skip to content

Commit

Permalink
Merge pull request #253 from patricklx/fix-byte-issues
Browse files Browse the repository at this point in the history
  • Loading branch information
gitKrystan authored Mar 15, 2024
2 parents 844a082 + b3e3bbf commit dd0af13
Show file tree
Hide file tree
Showing 8 changed files with 361 additions and 87 deletions.
48 changes: 34 additions & 14 deletions src/parse/index.ts
Original file line number Diff line number Diff line change
@@ -1,37 +1,42 @@
import { traverse } from '@babel/core';
import type {
BlockStatement,
File,
Node,
ObjectExpression,
StaticBlock,
} from '@babel/types';
import type { Parsed as RawGlimmerTemplate } from 'content-tag';
import { Preprocessor } from 'content-tag';
import type { Parser } from 'prettier';
import { parsers as babelParsers } from 'prettier/plugins/babel.js';

import { PRINTER_NAME } from '../config.js';
import type { Options } from '../options.js';
import { assert } from '../utils/index.js';
import { preprocessTemplateRange } from './preprocess.js';
import {
byteToCharIndex,
preprocessTemplateRange,
type Template,
} from './preprocess.js';

const typescript = babelParsers['babel-ts'] as Parser<Node | undefined>;
const p = new Preprocessor();

/** Converts a node into a GlimmerTemplate node */
function convertNode(
node: BlockStatement | ObjectExpression | StaticBlock,
rawTemplate: RawGlimmerTemplate,
rawTemplate: Template,
): void {
node.innerComments = [];
node.extra = Object.assign(node.extra ?? {}, {
isGlimmerTemplate: true as const,
template: rawTemplate,
});
}

/** Traverses the AST and replaces the transformed template parts with other AST */
function convertAst(ast: Node, rawTemplates: RawGlimmerTemplate[]): void {
const unprocessedTemplates = [...rawTemplates];
function convertAst(ast: File, templates: Template[]): void {
const unprocessedTemplates = [...templates];

traverse(ast, {
enter(path) {
Expand All @@ -47,11 +52,10 @@ function convertAst(ast: Node, rawTemplates: RawGlimmerTemplate[]): void {

const templateIndex = unprocessedTemplates.findIndex(
(t) =>
(t.range.start === start && t.range.end === end) ||
(t.utf16Range.start === start && t.utf16Range.end === end) ||
(node.type === 'ObjectExpression' &&
node.extra?.['parenthesized'] === true &&
t.range.start === start - 1 &&
t.range.end === end + 1),
t.utf16Range.start === start - 1 &&
t.utf16Range.end === end + 1),
);
if (templateIndex > -1) {
const rawTemplate = unprocessedTemplates.splice(templateIndex, 1)[0];
Expand All @@ -60,6 +64,12 @@ function convertAst(ast: Node, rawTemplates: RawGlimmerTemplate[]): void {
'expected raw template because splice index came from findIndex',
);
}
const index =
node.innerComments?.[0] &&
ast.comments?.indexOf(node.innerComments[0]);
if (ast.comments && index !== undefined && index >= 0) {
ast.comments.splice(index, 1);
}
convertNode(node, rawTemplate);
} else {
return null;
Expand Down Expand Up @@ -87,15 +97,25 @@ function preprocess(
fileName: string,
): {
code: string;
rawTemplates: RawGlimmerTemplate[];
templates: Template[];
} {
const rawTemplates = p.parse(code, fileName);
const templates: Template[] = rawTemplates.map((r) => ({
type: r.type,
range: r.range,
contentRange: r.contentRange,
contents: r.contents,
utf16Range: {
start: byteToCharIndex(code, r.range.start),
end: byteToCharIndex(code, r.range.end),
},
}));

for (const rawTemplate of rawTemplates) {
code = preprocessTemplateRange(rawTemplate, code);
for (const template of templates) {
code = preprocessTemplateRange(template, code);
}

return { rawTemplates, code };
return { templates, code };
}

export const parser: Parser<Node | undefined> = {
Expand All @@ -106,7 +126,7 @@ export const parser: Parser<Node | undefined> = {
const preprocessed = preprocess(code, options.filepath);
const ast = await typescript.parse(preprocessed.code, options);
assert('expected ast', ast);
convertAst(ast, preprocessed.rawTemplates);
convertAst(ast as File, preprocessed.templates);
return ast;
},
};
127 changes: 54 additions & 73 deletions src/parse/preprocess.ts
Original file line number Diff line number Diff line change
@@ -1,104 +1,85 @@
import type { Parsed as RawGlimmerTemplate } from 'content-tag';

const EMPTY_SPACE = ' ';
export interface Template {
contents: string;
type: string;
range: {
start: number;
end: number;
};
utf16Range: {
start: number;
end: number;
};
}

/**
* Given a string (`original`), replaces the bytes in the given `range` with
* equivalent bytes of empty space (' ') surrounded by the given prefix and
* suffix. The total byte length will not change.
*
* Returns the resulting string.
*/
function replaceByteRange(
originalBuffer: Buffer,
range: { start: number; end: number },
options: { prefix: string; suffix: string },
): string {
const prefixBuffer = Buffer.from(options.prefix);
const suffixBuffer = Buffer.from(options.suffix);
const BufferMap: Map<string, Buffer> = new Map();

// Validate range
if (
range.start < 0 ||
range.end > originalBuffer.length ||
range.start > range.end ||
prefixBuffer.length + suffixBuffer.length > range.end - range.start
) {
throw new Error(
`Invalid byte range:\n\tstart=${range.start}\n\tend=${
range.end
}\n\tprefix=${options.prefix}\n\tsuffix=${
options.suffix
}\n\tstring=\n\t${originalBuffer.toString()}`,
);
function getBuffer(s: string): Buffer {
let buf = BufferMap.get(s);
if (!buf) {
buf = Buffer.from(s);
BufferMap.set(s, buf);
}
return buf;
}

// Adjust the space length to account for the prefix and suffix lengths
const totalReplacementLength = range.end - range.start;
const spaceLength =
totalReplacementLength - prefixBuffer.length - suffixBuffer.length;

// Create a buffer for the replacement
const spaceBuffer = Buffer.alloc(spaceLength, EMPTY_SPACE);

// Concatenate prefix, space, and suffix buffers
const replacementBuffer = Buffer.concat([
prefixBuffer,
spaceBuffer,
suffixBuffer,
]);

// Create buffers for before and after the range using subarray
const beforeRange = originalBuffer.subarray(0, range.start);
const afterRange = originalBuffer.subarray(range.end);
/** Slice string using byte range */
export function sliceByteRange(s: string, a: number, b?: number): string {
const buf = getBuffer(s);
return buf.subarray(a, b).toString();
}

// Concatenate all parts and convert back to a string
const result = Buffer.concat([beforeRange, replacementBuffer, afterRange]);
/** Converts byte index to js char index (utf16) */
export function byteToCharIndex(s: string, byteOffset: number): number {
const buf = getBuffer(s);
return buf.subarray(0, byteOffset).toString().length;
}

if (result.length !== originalBuffer.length) {
throw new Error(
`Result length (${result.length}) does not match original length (${originalBuffer.length})`,
);
}
/** Calculate byte length */
export function byteLength(s: string): number {
return getBuffer(s).length;
}

return result.toString('utf8');
function replaceRange(
s: string,
start: number,
end: number,
substitute: string,
): string {
return sliceByteRange(s, 0, start) + substitute + sliceByteRange(s, end);
}

/**
* Replace the template with a parsable placeholder that takes up the same
* range.
*/
export function preprocessTemplateRange(
rawTemplate: RawGlimmerTemplate,
template: Template,
code: string,
): string {
const codeBuffer = Buffer.from(code);

let prefix: string;
let suffix: string;

if (rawTemplate.type === 'class-member') {
if (template.type === 'class-member') {
// Replace with StaticBlock
prefix = 'static{';
suffix = '}';
prefix = 'static{/*';
suffix = '*/}';
} else {
// Replace with BlockStatement or ObjectExpression
prefix = '{';
suffix = '}';
prefix = '{/*';
suffix = '*/}';

const nextToken = codeBuffer
.subarray(rawTemplate.range.end)
.toString()
.match(/\S+/);
const nextToken = code.slice(template.range.end).toString().match(/\S+/);
if (nextToken && nextToken[0] === 'as') {
// Replace with parenthesized ObjectExpression
prefix = '(' + prefix;
suffix = suffix + ')';
}
}

return replaceByteRange(codeBuffer, rawTemplate.range, {
prefix,
suffix,
});
const content = template.contents.replaceAll('/', '\\/');
const tplLength = template.range.end - template.range.start;
const spaces =
tplLength - byteLength(content) - prefix.length - suffix.length;
const total = prefix + content + ' '.repeat(spaces) + suffix;
return replaceRange(code, template.range.start, template.range.end, total);
}
41 changes: 41 additions & 0 deletions tests/cases/gts/issue-191-d.gts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import Component from '@glimmer/component';
import { on } from '@ember/modifier';

import { getSnippetElement, toClipboard, withExtraStyles } from './copy-utils';
import Menu from './menu';

/**
* This component is injected via the markdown rendering
*/
export default class CopyMenu extends Component {
copyAsText = (event: Event) => {
let code = getSnippetElement(event);

navigator.clipboard.writeText(code.innerText);
};

copyAsImage = async (event: Event) => {
let code = getSnippetElement(event);

await withExtraStyles(code, () => toClipboard(code));
};

<template>
<Menu data-test-copy-menu>
<:trigger as |t|>
<t.Default class="absolute top-3 right-4 z-10" data-test-copy-menu>
📋
</t.Default>
</:trigger>

<:options as |Item|>
<Item {{on "click" this.copyAsText}}>
Copy as text
</Item>
<Item {{on "click" this.copyAsImage}}>
Copy as image
</Item>
</:options>
</Menu>
</template>
}
16 changes: 16 additions & 0 deletions tests/cases/gts/issue-191-e.gts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import Component from '@glimmer/component';

/**
* This component contains a multi-byte character
*/
export default class MultiByteCharComponent extends Component {
get rows() {
console.log('abc다윤6')
return []
}
<template>
{{#each this.rows as |row|}}
{{row.id}}
{{/each}}
</template>
}
Loading

0 comments on commit dd0af13

Please sign in to comment.