Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tokenize: grid-template-areas #1523

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+709 −2
Open

tokenize: grid-template-areas #1523

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6698242
tokenize: grid-template-areas
romainmenke Nov 25, 2024
6136836
Merge branch 'main' into tokenize-grid-template-areas--courteous-impa…
romainmenke Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions packages/css-tokenizer/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Changes to CSS Tokenizer

### Unreleased (minor)

- Add `tokenizeGridTemplateAreas` for `grid-template-areas` strings.

### 3.0.3

_October 25, 2024_
2 changes: 1 addition & 1 deletion packages/css-tokenizer/dist/index.cjs
View file

Large diffs are not rendered by default.

42 changes: 42 additions & 0 deletions packages/css-tokenizer/dist/index.d.ts
View file
Original file line number Diff line number Diff line change
@@ -41,6 +41,30 @@ export declare function cloneTokens(tokens: Array<CSSToken>): Array<CSSToken>;
*/
export declare type CSSToken = TokenAtKeyword | TokenBadString | TokenBadURL | TokenCDC | TokenCDO | TokenColon | TokenComma | TokenComment | TokenDelim | TokenDimension | TokenEOF | TokenFunction | TokenHash | TokenIdent | TokenNumber | TokenPercentage | TokenSemicolon | TokenString | TokenURL | TokenWhitespace | TokenOpenParen | TokenCloseParen | TokenOpenSquare | TokenCloseSquare | TokenOpenCurly | TokenCloseCurly | TokenUnicodeRange;

export declare interface GridTemplateAreasTokenNamedCell {
type: TokenTypeGridTemplateAreas.NamedCell;
/**
* The name of the cell
*/
value: string;
}

export declare interface GridTemplateAreasTokenNullCell {
type: TokenTypeGridTemplateAreas.NullCell;
/**
* The dots representing the null cell
*/
value: string;
}

export declare interface GridTemplateAreasTokenTrash {
type: TokenTypeGridTemplateAreas.Trash;
/**
* The incorrect cell value
*/
value: string;
}

/**
* The type of hash token
*/
@@ -380,6 +404,13 @@ export declare function tokenize(input: {
onParseError?: (error: ParseError) => void;
}): Array<CSSToken>;

/**
* Tokenize a CSS string describing grid template areas into a list of tokens.
*/
export declare function tokenizeGridTemplateAreas(input: {
valueOf(): string;
}): Array<GridTemplateAreasTokenNamedCell | GridTemplateAreasTokenNullCell | GridTemplateAreasTokenTrash>;

/**
* Create a tokenizer for a CSS string.
*/
@@ -573,6 +604,17 @@ export declare enum TokenType {
UnicodeRange = "unicode-range-token"
}

/**
* All possible CSS token types for grid template areas
*
* @see {@link https://drafts.csswg.org/css-grid/#valdef-grid-template-areas-string}
*/
export declare enum TokenTypeGridTemplateAreas {
NamedCell = "named-cell-token",
NullCell = "null-cell-token",
Trash = "trash-token"
}

export declare interface TokenUnicodeRange extends Token<TokenType.UnicodeRange, {
startOfRange: number;
endOfRange: number;
2 changes: 1 addition & 1 deletion packages/css-tokenizer/dist/index.mjs
View file

Large diffs are not rendered by default.

376 changes: 376 additions & 0 deletions packages/css-tokenizer/docs/css-tokenizer.api.json
View file
Original file line number Diff line number Diff line change
@@ -505,6 +505,225 @@
"endIndex": 54
}
},
{
"kind": "Interface",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenNamedCell:interface",
"docComment": "",
"excerptTokens": [
{
"kind": "Content",
"text": "export interface GridTemplateAreasTokenNamedCell "
}
],
"fileUrlPath": "dist/_types/interfaces/token-grid-template-areas.d.ts",
"releaseTag": "Public",
"name": "GridTemplateAreasTokenNamedCell",
"preserveMemberOrder": false,
"members": [
{
"kind": "PropertySignature",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenNamedCell#type:member",
"docComment": "",
"excerptTokens": [
{
"kind": "Content",
"text": "type: "
},
{
"kind": "Reference",
"text": "TokenTypeGridTemplateAreas.NamedCell",
"canonicalReference": "@csstools/css-tokenizer!TokenTypeGridTemplateAreas.NamedCell:member"
},
{
"kind": "Content",
"text": ";"
}
],
"isReadonly": false,
"isOptional": false,
"releaseTag": "Public",
"name": "type",
"propertyTypeTokenRange": {
"startIndex": 1,
"endIndex": 2
}
},
{
"kind": "PropertySignature",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenNamedCell#value:member",
"docComment": "/**\n * The name of the cell\n */\n",
"excerptTokens": [
{
"kind": "Content",
"text": "value: "
},
{
"kind": "Content",
"text": "string"
},
{
"kind": "Content",
"text": ";"
}
],
"isReadonly": false,
"isOptional": false,
"releaseTag": "Public",
"name": "value",
"propertyTypeTokenRange": {
"startIndex": 1,
"endIndex": 2
}
}
],
"extendsTokenRanges": []
},
{
"kind": "Interface",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenNullCell:interface",
"docComment": "",
"excerptTokens": [
{
"kind": "Content",
"text": "export interface GridTemplateAreasTokenNullCell "
}
],
"fileUrlPath": "dist/_types/interfaces/token-grid-template-areas.d.ts",
"releaseTag": "Public",
"name": "GridTemplateAreasTokenNullCell",
"preserveMemberOrder": false,
"members": [
{
"kind": "PropertySignature",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenNullCell#type:member",
"docComment": "",
"excerptTokens": [
{
"kind": "Content",
"text": "type: "
},
{
"kind": "Reference",
"text": "TokenTypeGridTemplateAreas.NullCell",
"canonicalReference": "@csstools/css-tokenizer!TokenTypeGridTemplateAreas.NullCell:member"
},
{
"kind": "Content",
"text": ";"
}
],
"isReadonly": false,
"isOptional": false,
"releaseTag": "Public",
"name": "type",
"propertyTypeTokenRange": {
"startIndex": 1,
"endIndex": 2
}
},
{
"kind": "PropertySignature",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenNullCell#value:member",
"docComment": "/**\n * The dots representing the null cell\n */\n",
"excerptTokens": [
{
"kind": "Content",
"text": "value: "
},
{
"kind": "Content",
"text": "string"
},
{
"kind": "Content",
"text": ";"
}
],
"isReadonly": false,
"isOptional": false,
"releaseTag": "Public",
"name": "value",
"propertyTypeTokenRange": {
"startIndex": 1,
"endIndex": 2
}
}
],
"extendsTokenRanges": []
},
{
"kind": "Interface",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenTrash:interface",
"docComment": "",
"excerptTokens": [
{
"kind": "Content",
"text": "export interface GridTemplateAreasTokenTrash "
}
],
"fileUrlPath": "dist/_types/interfaces/token-grid-template-areas.d.ts",
"releaseTag": "Public",
"name": "GridTemplateAreasTokenTrash",
"preserveMemberOrder": false,
"members": [
{
"kind": "PropertySignature",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenTrash#type:member",
"docComment": "",
"excerptTokens": [
{
"kind": "Content",
"text": "type: "
},
{
"kind": "Reference",
"text": "TokenTypeGridTemplateAreas.Trash",
"canonicalReference": "@csstools/css-tokenizer!TokenTypeGridTemplateAreas.Trash:member"
},
{
"kind": "Content",
"text": ";"
}
],
"isReadonly": false,
"isOptional": false,
"releaseTag": "Public",
"name": "type",
"propertyTypeTokenRange": {
"startIndex": 1,
"endIndex": 2
}
},
{
"kind": "PropertySignature",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenTrash#value:member",
"docComment": "/**\n * The incorrect cell value\n */\n",
"excerptTokens": [
{
"kind": "Content",
"text": "value: "
},
{
"kind": "Content",
"text": "string"
},
{
"kind": "Content",
"text": ";"
}
],
"isReadonly": false,
"isOptional": false,
"releaseTag": "Public",
"name": "value",
"propertyTypeTokenRange": {
"startIndex": 1,
"endIndex": 2
}
}
],
"extendsTokenRanges": []
},
{
"kind": "Enum",
"canonicalReference": "@csstools/css-tokenizer!HashType:enum",
@@ -4489,6 +4708,83 @@
],
"name": "tokenize"
},
{
"kind": "Function",
"canonicalReference": "@csstools/css-tokenizer!tokenizeGridTemplateAreas:function(1)",
"docComment": "/**\n * Tokenize a CSS string describing grid template areas into a list of tokens.\n */\n",
"excerptTokens": [
{
"kind": "Content",
"text": "export declare function tokenizeGridTemplateAreas(input: "
},
{
"kind": "Content",
"text": "{\n valueOf(): string;\n}"
},
{
"kind": "Content",
"text": "): "
},
{
"kind": "Reference",
"text": "Array",
"canonicalReference": "!Array:interface"
},
{
"kind": "Content",
"text": "<"
},
{
"kind": "Reference",
"text": "GridTemplateAreasTokenNamedCell",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenNamedCell:interface"
},
{
"kind": "Content",
"text": " | "
},
{
"kind": "Reference",
"text": "GridTemplateAreasTokenNullCell",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenNullCell:interface"
},
{
"kind": "Content",
"text": " | "
},
{
"kind": "Reference",
"text": "GridTemplateAreasTokenTrash",
"canonicalReference": "@csstools/css-tokenizer!GridTemplateAreasTokenTrash:interface"
},
{
"kind": "Content",
"text": ">"
},
{
"kind": "Content",
"text": ";"
}
],
"fileUrlPath": "dist/_types/tokenize-grid-template-areas.d.ts",
"returnTypeTokenRange": {
"startIndex": 3,
"endIndex": 11
},
"releaseTag": "Public",
"overloadIndex": 1,
"parameters": [
{
"parameterName": "input",
"parameterTypeTokenRange": {
"startIndex": 1,
"endIndex": 2
},
"isOptional": false
}
],
"name": "tokenizeGridTemplateAreas"
},
{
"kind": "Function",
"canonicalReference": "@csstools/css-tokenizer!tokenizer:function(1)",
@@ -5635,6 +5931,86 @@
}
]
},
{
"kind": "Enum",
"canonicalReference": "@csstools/css-tokenizer!TokenTypeGridTemplateAreas:enum",
"docComment": "/**\n * All possible CSS token types for grid template areas\n *\n * @see\n *\n * {@link https://drafts.csswg.org/css-grid/#valdef-grid-template-areas-string}\n */\n",
"excerptTokens": [
{
"kind": "Content",
"text": "export declare enum TokenTypeGridTemplateAreas "
}
],
"fileUrlPath": "dist/_types/interfaces/token-grid-template-areas.d.ts",
"releaseTag": "Public",
"name": "TokenTypeGridTemplateAreas",
"preserveMemberOrder": false,
"members": [
{
"kind": "EnumMember",
"canonicalReference": "@csstools/css-tokenizer!TokenTypeGridTemplateAreas.NamedCell:member",
"docComment": "",
"excerptTokens": [
{
"kind": "Content",
"text": "NamedCell = "
},
{
"kind": "Content",
"text": "\"named-cell-token\""
}
],
"initializerTokenRange": {
"startIndex": 1,
"endIndex": 2
},
"releaseTag": "Public",
"name": "NamedCell"
},
{
"kind": "EnumMember",
"canonicalReference": "@csstools/css-tokenizer!TokenTypeGridTemplateAreas.NullCell:member",
"docComment": "",
"excerptTokens": [
{
"kind": "Content",
"text": "NullCell = "
},
{
"kind": "Content",
"text": "\"null-cell-token\""
}
],
"initializerTokenRange": {
"startIndex": 1,
"endIndex": 2
},
"releaseTag": "Public",
"name": "NullCell"
},
{
"kind": "EnumMember",
"canonicalReference": "@csstools/css-tokenizer!TokenTypeGridTemplateAreas.Trash:member",
"docComment": "",
"excerptTokens": [
{
"kind": "Content",
"text": "Trash = "
},
{
"kind": "Content",
"text": "\"trash-token\""
}
],
"initializerTokenRange": {
"startIndex": 1,
"endIndex": 2
},
"releaseTag": "Public",
"name": "Trash"
}
]
},
{
"kind": "Interface",
"canonicalReference": "@csstools/css-tokenizer!TokenUnicodeRange:interface",
13 changes: 13 additions & 0 deletions packages/css-tokenizer/src/index.ts
View file
Original file line number Diff line number Diff line change
@@ -112,3 +112,16 @@ export {
isTokenWhitespace,
isTokenWhiteSpaceOrComment,
} from './util/type-predicates';

// Grid template areas
export { tokenizeGridTemplateAreas } from './tokenize-grid-template-areas';

export {
TokenTypeGridTemplateAreas,
} from './interfaces/token-grid-template-areas';

export type {
GridTemplateAreasTokenNamedCell,
GridTemplateAreasTokenNullCell,
GridTemplateAreasTokenTrash
} from './interfaces/token-grid-template-areas';
34 changes: 34 additions & 0 deletions packages/css-tokenizer/src/interfaces/token-grid-template-areas.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/**
* All possible CSS token types for grid template areas
*
* @see {@link https://drafts.csswg.org/css-grid/#valdef-grid-template-areas-string}
*/
export enum TokenTypeGridTemplateAreas {
NamedCell = 'named-cell-token',
NullCell = 'null-cell-token',
Trash = 'trash-token',
}

export interface GridTemplateAreasTokenNamedCell {
type: TokenTypeGridTemplateAreas.NamedCell
/**
* The name of the cell
*/
value: string
}

export interface GridTemplateAreasTokenNullCell {
type: TokenTypeGridTemplateAreas.NullCell
/**
* The dots representing the null cell
*/
value: string
}

export interface GridTemplateAreasTokenTrash {
type: TokenTypeGridTemplateAreas.Trash
/**
* The incorrect cell value
*/
value: string
}
127 changes: 127 additions & 0 deletions packages/css-tokenizer/src/tokenize-grid-template-areas.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import { CARRIAGE_RETURN, CHARACTER_TABULATION, FORM_FEED, FULL_STOP, LINE_FEED, NULL, REPLACEMENT_CHARACTER, SPACE } from "./code-points/code-points";
import { isIdentCodePoint, isSurrogate, isWhitespace } from "./code-points/ranges";
import type { CodePointReader } from "./interfaces/code-point-reader";
import type { GridTemplateAreasTokenNamedCell, GridTemplateAreasTokenNullCell, GridTemplateAreasTokenTrash } from "./interfaces/token-grid-template-areas";
import { TokenTypeGridTemplateAreas } from "./interfaces/token-grid-template-areas";
import { Reader } from "./reader";

/**
* Tokenize a CSS string describing grid template areas into a list of tokens.
*/
export function tokenizeGridTemplateAreas(input: { valueOf(): string }): Array<GridTemplateAreasTokenNamedCell | GridTemplateAreasTokenNullCell | GridTemplateAreasTokenTrash> {
const tokens: Array<GridTemplateAreasTokenNamedCell | GridTemplateAreasTokenNullCell | GridTemplateAreasTokenTrash> = [];
const reader = new Reader(input.valueOf());

TOKENIZER_LOOP:
while (true) {
reader.resetRepresentation();

const peeked = reader.source.codePointAt(reader.cursor);
if (typeof peeked === "undefined") {
break;
}

switch (peeked) {
case LINE_FEED:
case CARRIAGE_RETURN:
case FORM_FEED:
case CHARACTER_TABULATION:
case SPACE:
consumeWhiteSpace(reader);
continue TOKENIZER_LOOP;

case FULL_STOP:
tokens.push(consumeNullCell(reader));
continue TOKENIZER_LOOP;

default:
if (peeked === NULL || isSurrogate(peeked) || isIdentCodePoint(peeked)) {
tokens.push(consumeNamedCell(reader));
continue TOKENIZER_LOOP;
}

tokens.push(consumeTrash(reader));
continue TOKENIZER_LOOP;
}
}

return tokens;
}

export function consumeNamedCell(reader: CodePointReader): GridTemplateAreasTokenNamedCell {
const codePoints: Array<number> = [];

while (true) {
const codePoint = reader.source.codePointAt(reader.cursor) ?? -1;
if (codePoint === NULL || isSurrogate(codePoint)) {
codePoints.push(REPLACEMENT_CHARACTER);
reader.advanceCodePoint(1 + +(codePoint > 0xffff));
continue;
}

if (isIdentCodePoint(codePoint)) {
codePoints.push(codePoint);
reader.advanceCodePoint(1 + +(codePoint > 0xffff));
continue;
}

break;
}

return {
type: TokenTypeGridTemplateAreas.NamedCell,
value: String.fromCodePoint(...codePoints),
}
}

function consumeNullCell(reader: CodePointReader): GridTemplateAreasTokenNullCell {
while (reader.source.codePointAt(reader.cursor) === FULL_STOP) {
reader.advanceCodePoint();
}

return {
type: TokenTypeGridTemplateAreas.NullCell,
value: reader.source.slice(reader.representationStart, reader.representationEnd + 1),
};
}

function consumeWhiteSpace(reader: CodePointReader): void {
while (isWhitespace(reader.source.codePointAt(reader.cursor) ?? -1)) {
reader.advanceCodePoint();
}
}

export function consumeTrash(reader: CodePointReader): GridTemplateAreasTokenTrash {
reader.advanceCodePoint();

TOKENIZER_LOOP:
while (true) {
const peeked = reader.source.codePointAt(reader.cursor);
if (typeof peeked === "undefined") {
break TOKENIZER_LOOP;
}

switch (peeked) {
case LINE_FEED:
case CARRIAGE_RETURN:
case FORM_FEED:
case CHARACTER_TABULATION:
case SPACE:
case FULL_STOP:
case NULL:
break TOKENIZER_LOOP;
default:
if (isSurrogate(peeked) || isIdentCodePoint(peeked)) {
break TOKENIZER_LOOP;
}

reader.advanceCodePoint();
continue TOKENIZER_LOOP;
}
}

return {
type: TokenTypeGridTemplateAreas.Trash,
value: reader.source.slice(reader.representationStart, reader.representationEnd + 1),
};
}
3 changes: 3 additions & 0 deletions packages/css-tokenizer/test/test.mjs
View file
Original file line number Diff line number Diff line change
@@ -43,3 +43,6 @@ import './mutations/dimension.mjs';
// Keep this as the last test,
// it is only intended to increase test coverage by double checking more obscure cases.
import './css-tokenizer-tests/test.mjs';

// Tokens Grid Template Areas
import './token-grid-template-areas/basic.mjs';
108 changes: 108 additions & 0 deletions packages/css-tokenizer/test/token-grid-template-areas/basic.mjs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import { tokenizeGridTemplateAreas, tokenize, isTokenString } from '@csstools/css-tokenizer';
import assert from 'node:assert';

function tokenizeFromSource(source) {
const t = tokenize({ css: source });
const firstString = t.find(isTokenString);
if (!firstString) {
return [];
}

return tokenizeGridTemplateAreas(firstString[4].value);
}

{
assert.deepEqual(
tokenizeFromSource('"a a"'),
[
{ type: 'named-cell-token', value: 'a' },
{ type: 'named-cell-token', value: 'a' },
],
);
}

{
assert.deepEqual(
tokenizeFromSource('"a"'),
[{ type: 'named-cell-token', value: 'a' }],
);
}

{
assert.deepEqual(
tokenizeFromSource('"."'),
[{ type: 'null-cell-token', value: '.' }],
);
}

{
assert.deepEqual(
tokenizeFromSource('"..."'),
[{ type: 'null-cell-token', value: '...' }],
);
}

{
assert.deepEqual(
tokenizeFromSource('"@ a"'),
[
{ type: 'trash-token', value: '@' },
{ type: 'named-cell-token', value: 'a' },
],
);
}

{
assert.deepEqual(
// Unescaped, so bad string
tokenizeFromSource('"a\nb"'),
[],
);
}

{
assert.deepEqual(
// Escaped, so regular string
tokenizeFromSource('"a\\\nb"'),
[{ type: 'named-cell-token', value: 'ab' }],
);
}

{
assert.deepEqual(
// Escaped, so regular string
tokenizeFromSource('"a\\\n b"'),
[
{ type: 'named-cell-token', value: 'a' },
{ type: 'named-cell-token', value: 'b' },
],
);
}

{
assert.deepEqual(
tokenizeFromSource('"a\\\\"'),
[
{ type: 'named-cell-token', value: 'a' },
{ type: 'trash-token', value: '\\' },
],
);
}

{
assert.deepEqual(
tokenizeFromSource('"tu\\0U"'),
[{ type: 'named-cell-token', value: 'tu�U' }],
);
}

{
assert.deepEqual(
tokenizeFromSource('"tu\\\\0U"'),
[
{ type: 'named-cell-token', value: 'tu' },
{ type: 'trash-token', value: '\\' },
{ type: 'named-cell-token', value: '0U' },
],
);
}