From c25755d52bdaa97098edeba1682738246079643c Mon Sep 17 00:00:00 2001 From: Adam Snyder Date: Sun, 30 Jun 2024 18:31:06 -0700 Subject: [PATCH] feat: Initial implementation, focused on parsing type definitions --- .github/.release-please-manifest.json | 1 + .github/dependabot.yaml | 17 + .github/release-please-config.json | 10 + .github/workflows/ci.yaml | 42 + .golangci.yaml | 120 ++ LICENSE | 21 + README.md | 19 + ast/ast.go | 8 + ast/expression.go | 138 +++ ast/signature.go | 40 + ast/statement.go | 91 ++ ast/type.go | 72 ++ ast/walk.go | 142 +++ ast/walk_example_test.go | 31 + go.mod | 3 + .../testdata/additional_properties.ts.txt | 11 + internal/testdata/array.ts.txt | 6 + internal/testdata/basic_type.ts.txt | 4 + internal/testdata/enum.ts.txt | 12 + internal/testdata/generic.ts.txt | 11 + internal/testdata/inline_type.ts.txt | 4 + internal/testdata/interface.ts.txt | 17 + .../testdata/interface_complex_syntax.ts.txt | 40 + .../interface_with_union_array.ts.txt | 9 + .../interface_with_union_struct_field.ts.txt | 51 + internal/testdata/multiple_extends.ts.txt | 8 + internal/testdata/namespace.ts.txt | 18 + internal/testdata/namespace_with_type.ts.txt | 12 + internal/testdata/object.ts.txt | 6 + internal/testdata/qualified.ts.txt | 3 + internal/testdata/readonly.ts.txt | 8 + internal/testdata/tuple.ts.txt | 3 + internal/testdata/union_struct.ts.txt | 34 + internal/testdata/union_type.ts.txt | 7 + parser/lexer.go | 256 +++++ parser/lexer_test.go | 559 +++++++++ parser/parser.go | 431 +++++++ parser/parser_example_test.go | 25 + parser/parser_test.go | 1022 +++++++++++++++++ token/token.go | 102 ++ 40 files changed, 3414 insertions(+) create mode 100644 .github/.release-please-manifest.json create mode 100644 .github/dependabot.yaml create mode 100644 .github/release-please-config.json create mode 100644 .github/workflows/ci.yaml create mode 100644 .golangci.yaml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 ast/ast.go create mode 100644 ast/expression.go create mode 100644 ast/signature.go create mode 100644 ast/statement.go create mode 100644 ast/type.go create mode 100644 ast/walk.go create mode 100644 ast/walk_example_test.go create mode 100644 go.mod create mode 100644 internal/testdata/additional_properties.ts.txt create mode 100644 internal/testdata/array.ts.txt create mode 100644 internal/testdata/basic_type.ts.txt create mode 100644 internal/testdata/enum.ts.txt create mode 100644 internal/testdata/generic.ts.txt create mode 100644 internal/testdata/inline_type.ts.txt create mode 100644 internal/testdata/interface.ts.txt create mode 100644 internal/testdata/interface_complex_syntax.ts.txt create mode 100644 internal/testdata/interface_with_union_array.ts.txt create mode 100644 internal/testdata/interface_with_union_struct_field.ts.txt create mode 100644 internal/testdata/multiple_extends.ts.txt create mode 100644 internal/testdata/namespace.ts.txt create mode 100644 internal/testdata/namespace_with_type.ts.txt create mode 100644 internal/testdata/object.ts.txt create mode 100644 internal/testdata/qualified.ts.txt create mode 100644 internal/testdata/readonly.ts.txt create mode 100644 internal/testdata/tuple.ts.txt create mode 100644 internal/testdata/union_struct.ts.txt create mode 100644 internal/testdata/union_type.ts.txt create mode 100644 parser/lexer.go create mode 100644 parser/lexer_test.go create mode 100644 parser/parser.go create mode 100644 parser/parser_example_test.go create mode 100644 parser/parser_test.go create mode 100644 token/token.go diff --git a/.github/.release-please-manifest.json b/.github/.release-please-manifest.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/.github/.release-please-manifest.json @@ -0,0 +1 @@ +{} diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..57b9ebd --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/dependabot-2.0.json +version: 2 +updates: + - directory: / + package-ecosystem: gomod + schedule: + interval: monthly + commit-message: + prefix: chore + include: scope + - directory: / + package-ecosystem: github-actions + schedule: + interval: monthly + commit-message: + prefix: chore + include: scope diff --git a/.github/release-please-config.json b/.github/release-please-config.json new file mode 100644 index 0000000..078c292 --- /dev/null +++ b/.github/release-please-config.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "packages": { + ".": { + "release-type": "go", + "bump-minor-pre-major": true, + "include-v-in-tag": true + } + } +} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..33043e2 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,42 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow + +on: + push: + branches: + - main + pull_request: {} + workflow_dispatch: {} + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: ["1.20", "1.21", "1.22"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + - run: go test -cover ./... + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + - uses: golangci/golangci-lint-action@v6 + with: + version: v1.59.1 + + release-please: + runs-on: ubuntu-latest + needs: [test, lint] + if: github.ref == 'refs/heads/main' + steps: + - uses: googleapis/release-please-action@v4 + id: release-please + with: + token: ${{ secrets.PAT }} + config-file: .github/release-please-config.json + manifest-file: .github/.release-please-manifest.json diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..32bee51 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,120 @@ +# See https://golangci-lint.run/usage/configuration/ + +linters: + disable-all: true + enable: + # See https://golangci-lint.run/usage/linters/ + - asasalint # Check for pass []any as any in variadic func(...any). + - bodyclose # Checks whether HTTP response body is closed successfully. + - contextcheck # Check whether the function uses a non-inherited context. + - durationcheck # Check for two durations multiplied together. + - errcheck # Checks whether Rows.Err of rows is checked successfully. + - errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and reports occations, where the check for the returned error can be omitted. + - errorlint # Errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13. + - forbidigo # Forbids identifiers. + - gci # Gci controls Go package import order and makes it always deterministic. + - gocritic # Provides diagnostics that check for bugs, performance and style issues. Extensible without recompilation through dynamic rules. Dynamic rules are written declaratively with AST patterns, filters, report message and optional suggestion. + - godot # Check if comments end in a period. + - gosec # Inspects source code for security problems. + - gosimple # Linter for Go source code that specializes in simplifying code. + - govet # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string. + - inamedparam # Reports interfaces with unnamed method parameters. + - ineffassign # Detects when assignments to existing variables are not used. + - mirror # Reports wrong mirror patterns of bytes/strings usage. + - misspell # Finds commonly misspelled English words. + - musttag # Enforce field tags in (un)marshaled structs. + - nilerr # Finds the code that returns nil even if it checks that the error is not nil. + - nilnil # Checks that there is no simultaneous return of nil error and an invalid value. + - noctx # Finds sending http request without context.Context. + - nolintlint # Reports ill-formed or insufficient nolint directives. + - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL. + - perfsprint # Checks that fmt.Sprintf can be replaced with a faster alternative. + - protogetter # Reports direct reads from proto message fields when getters should be used. + - reassign # Checks that package variables are not reassigned. + - revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. + - staticcheck # It's a set of rules from staticcheck. It's not the same thing as the staticcheck binary. The author of staticcheck doesn't support or approve the use of staticcheck as a library inside golangci-lint. + - tenv # # Tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17. + - unconvert # Remove unnecessary type conversions. + - unused # Checks Go code for unused constants, variables, functions and types. + +linters-settings: + # See https://golangci-lint.run/usage/linters/#linters-configuration + forbidigo: + forbid: + - 'fmt\.Print.*' # Should be using a logger + gci: + sections: + - standard + - default + - prefix(github.com/armsnyder) + gocritic: + enabled-tags: + - performance + - opinionated + - experimental + disabled-checks: + - whyNoLint # False positives, use nolintlint instead + govet: + enable-all: true + disable: + - fieldalignment # Too struct + nolintlint: + require-specific: true + revive: + enable-all-rules: true + rules: + # See https://revive.run/r + - name: add-constant # too strict + disabled: true + - name: argument-limit # too strict + disabled: true + - name: cognitive-complexity + arguments: + - 30 + - name: cyclomatic + arguments: + - 30 + - name: file-header # too strict + disabled: true + - name: function-length + arguments: + - 50 # statements + - 0 # lines (0 to disable) + - name: function-result-limit # too strict + disabled: true + - name: import-shadowing # too strict, results in uglier code + disabled: true + - name: line-length-limit # too strict + disabled: true + - name: max-public-structs # too strict + disabled: true + - name: modifies-parameter # too strict + disabled: true + - name: modifies-value-receiver # too strict + disabled: true + - name: nested-structs # too strict + disabled: true + - name: package-comments # too strict + disabled: true + - name: unhandled-error + disabled: true # not as good as errcheck + +issues: + exclude-rules: + - path: _test\.go$ + linters: + - gosec # too strict + - noctx # too strict + - path: _test\.go$ + text: (cognitive-complexity|function-length|dot-imports|import-alias-naming) # too strict + linters: + - revive + # Shadowing err is common. + - text: 'shadow: declaration of "err"' + linters: + - govet + - text: "^exported:.+stutters" # too strict and gets in the way of combining types like handlers + linters: + - revive + - path: _test\.go$ + text: "unused-parameter" # too strict diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..916a904 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Adam Snyder + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1d8d084 --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# TypeScript AST + +This library provides a way to parse TypeScript source code into an abstract +syntax tree (AST) in Golang. The packages are laid out similar to the standard +[go](https://pkg.go.dev/go) library. + +[Package Documentation](https://pkg.go.dev/github.com/armsnyder/typescript-ast-go) + +The main two packages are: + +- [parser](https://pkg.go.dev/github.com/armsnyder/typescript-ast-go/parser): + Parse TypeScript source code into an AST. +- [ast](https://pkg.go.dev/github.com/armsnyder/typescript-ast-go/ast): The + AST nodes and visitor for TypeScript source code. + +This library was originally created in order to parse TypeScript type +definitions specifically for the Language Server Protocol Specification. As a +result, it is not feature complete and may not work for all TypeScript source +code. diff --git a/ast/ast.go b/ast/ast.go new file mode 100644 index 0000000..13c1757 --- /dev/null +++ b/ast/ast.go @@ -0,0 +1,8 @@ +// Package ast defines the abstract syntax tree (AST) for the TypeScript +// programming language and provides functionality for traversing the AST. +package ast + +// Node is a common interface that all nodes in the AST implement. +type Node interface { + node() +} diff --git a/ast/expression.go b/ast/expression.go new file mode 100644 index 0000000..888a5cf --- /dev/null +++ b/ast/expression.go @@ -0,0 +1,138 @@ +package ast + +import "github.com/armsnyder/typescript-ast-go/token" + +// Expr is a [Node] that represents an expression. An expression produces a +// value. +type Expr interface { + Node + expr() +} + +// NumericLiteral is a numeric literal expression. +type NumericLiteral struct { + Text string +} + +func (n *NumericLiteral) String() string { + return n.Text +} + +func (*NumericLiteral) node() {} +func (*NumericLiteral) expr() {} + +// StringLiteral is a string literal expression. +type StringLiteral struct { + Text string +} + +func (n *StringLiteral) String() string { + return n.Text +} + +func (*StringLiteral) node() {} +func (*StringLiteral) expr() {} + +// ArrayLiteralExpression is an array literal expression. +type ArrayLiteralExpression struct { + Elements []Expr +} + +func (*ArrayLiteralExpression) node() {} +func (*ArrayLiteralExpression) expr() {} + +// Identifier is an identifier literal expression. +type Identifier struct { + Text string +} + +func (n *Identifier) String() string { + return n.Text +} + +func (*Identifier) node() {} +func (*Identifier) expr() {} + +// QualifiedName is a qualified name expression. +type QualifiedName struct { + Left *Identifier + Right *Identifier +} + +func (*QualifiedName) node() {} +func (*QualifiedName) expr() {} + +// EnumMember is an enum member expression. +type EnumMember struct { + Name *Identifier + Initializer Expr + LeadingComment string +} + +func (n *EnumMember) String() string { + return n.LeadingComment +} + +func (*EnumMember) node() {} +func (*EnumMember) expr() {} + +// TypeParameter is a type parameter expression. +type TypeParameter struct { + Name *Identifier +} + +func (*TypeParameter) node() {} +func (*TypeParameter) expr() {} + +// HeritageClause is a heritage clause expression. +type HeritageClause struct { + Types []*ExpressionWithTypeArguments +} + +func (*HeritageClause) node() {} +func (*HeritageClause) expr() {} + +// ExpressionWithTypeArguments is an expression with type arguments. +type ExpressionWithTypeArguments struct { + Expression *Identifier +} + +func (*ExpressionWithTypeArguments) node() {} +func (*ExpressionWithTypeArguments) expr() {} + +// Parameter is a parameter expression. +type Parameter struct { + Name *Identifier + Type Type +} + +func (*Parameter) node() {} +func (*Parameter) expr() {} + +// VariableDeclarationList is an expression that declares a list of variables. +type VariableDeclarationList struct { + Declarations []*VariableDeclaration +} + +func (*VariableDeclarationList) node() {} +func (*VariableDeclarationList) expr() {} + +// VariableDeclaration is an expression that declares a variable. +type VariableDeclaration struct { + Name *Identifier + Type Type + Initializer Expr +} + +func (*VariableDeclaration) node() {} +func (*VariableDeclaration) expr() {} + +// PrefixUnaryExpression is an expression that applies a unary operator to an +// operand. +type PrefixUnaryExpression struct { + Operator token.Kind + Operand Expr +} + +func (*PrefixUnaryExpression) node() {} +func (*PrefixUnaryExpression) expr() {} diff --git a/ast/signature.go b/ast/signature.go new file mode 100644 index 0000000..6138c43 --- /dev/null +++ b/ast/signature.go @@ -0,0 +1,40 @@ +package ast + +// Signature is a [Node] that represents a signature. A signature defines a +// property. +type Signature interface { + Expr + signature() +} + +// PropertySignature is an expression that defines an object property. +type PropertySignature struct { + Name *Identifier + QuestionToken bool + Type Type + LeadingComment string + TrailingComment string +} + +func (n *PropertySignature) String() string { + return n.LeadingComment + " / " + n.TrailingComment +} + +func (*PropertySignature) node() {} +func (*PropertySignature) expr() {} +func (*PropertySignature) signature() {} + +// IndexSignature is an expression that defines an object index signature. +type IndexSignature struct { + Parameters []*Parameter + Type Type + LeadingComment string +} + +func (n *IndexSignature) String() string { + return n.LeadingComment +} + +func (*IndexSignature) node() {} +func (*IndexSignature) expr() {} +func (*IndexSignature) signature() {} diff --git a/ast/statement.go b/ast/statement.go new file mode 100644 index 0000000..14d467e --- /dev/null +++ b/ast/statement.go @@ -0,0 +1,91 @@ +package ast + +// Stmt is a [Node] that represents a statement. A statement performs an +// action. +type Stmt interface { + Node + stmt() +} + +// SourceFile is a statement that represents a source file. +type SourceFile struct { + Statements []Stmt +} + +func (*SourceFile) node() {} +func (*SourceFile) stmt() {} + +// ModuleBlock is a statement that represents a block of statements in a +// module. +type ModuleBlock struct { + Statements []Stmt +} + +func (*ModuleBlock) node() {} +func (*ModuleBlock) stmt() {} + +// VariableStatement is a statement that declares a variable. +type VariableStatement struct { + DeclarationList *VariableDeclarationList + LeadingComment string +} + +func (n *VariableStatement) String() string { + return n.LeadingComment +} + +func (*VariableStatement) node() {} +func (*VariableStatement) stmt() {} + +// TypeAliasDeclaration is a statement that introduces a new type alias. +type TypeAliasDeclaration struct { + Name *Identifier + Type Type + LeadingComment string +} + +func (n *TypeAliasDeclaration) String() string { + return n.LeadingComment +} + +func (*TypeAliasDeclaration) node() {} +func (*TypeAliasDeclaration) stmt() {} + +// EnumDeclaration is a statement that introduces a new enum. +type EnumDeclaration struct { + Name *Identifier + Members []*EnumMember + LeadingComment string +} + +func (n *EnumDeclaration) String() string { + return n.LeadingComment +} + +func (*EnumDeclaration) node() {} +func (*EnumDeclaration) stmt() {} + +// InterfaceDeclaration is a statement that introduces a new interface. +type InterfaceDeclaration struct { + Name *Identifier + TypeParameters []*TypeParameter + HeritageClauses []*HeritageClause + Members []Signature + LeadingComment string +} + +func (n *InterfaceDeclaration) String() string { + return n.LeadingComment +} + +func (*InterfaceDeclaration) node() {} +func (*InterfaceDeclaration) stmt() {} + +type ModuleDeclaration struct { + Name *Identifier + Body *ModuleBlock + LeadingComment string +} + +func (*ModuleDeclaration) node() {} +func (*ModuleDeclaration) stmt() {} diff --git a/ast/type.go b/ast/type.go new file mode 100644 index 0000000..9e4cd3a --- /dev/null +++ b/ast/type.go @@ -0,0 +1,72 @@ +package ast + +// Type is a [Node] that represents a type expression. A type expression is a +// specific [Expr] that represents a type. +type Type interface { + Expr + typ() +} + +// LiteralType is a literal type expression. +type LiteralType struct { + Literal Expr +} + +func (*LiteralType) node() {} +func (*LiteralType) expr() {} +func (*LiteralType) typ() {} + +// TypeLiteral is a type literal expression. +type TypeLiteral struct { + Members []Signature +} + +func (*TypeLiteral) node() {} +func (*TypeLiteral) expr() {} +func (*TypeLiteral) typ() {} + +// ArrayType is an array type expression. +type ArrayType struct { + ElementType Expr +} + +func (*ArrayType) node() {} +func (*ArrayType) expr() {} +func (*ArrayType) typ() {} + +// TypeReference is a type reference expression. +type TypeReference struct { + TypeName Expr +} + +func (*TypeReference) node() {} +func (*TypeReference) expr() {} +func (*TypeReference) typ() {} + +// UnionType is a union type expression. +type UnionType struct { + Types []Type +} + +func (*UnionType) node() {} +func (*UnionType) expr() {} +func (*UnionType) typ() {} + +// TupleType is a tuple type expression. +type TupleType struct { + Elements []Type +} + +func (*TupleType) node() {} +func (*TupleType) expr() {} +func (*TupleType) typ() {} + +// ParenthesizedType is an expression that wraps another expression in +// parentheses. +type ParenthesizedType struct { + Type Type +} + +func (*ParenthesizedType) node() {} +func (*ParenthesizedType) expr() {} +func (*ParenthesizedType) typ() {} diff --git a/ast/walk.go b/ast/walk.go new file mode 100644 index 0000000..94bf4ce --- /dev/null +++ b/ast/walk.go @@ -0,0 +1,142 @@ +package ast + +import "fmt" + +// Visitor is an interface for visiting nodes in the AST. +// +// The Visit method is called for each node in the AST. If the Visit method +// returns a non-nil Visitor, the children of the node are visited, followed by +// a call to w.Visit(nil). +type Visitor interface { + Visit(node Node) (w Visitor) +} + +// Walk traverses the AST rooted at node in depth-first order. +// +// It starts by calling v.Visit(node); node must not be nil. If the visitor +// returned by v.Visit(node) is not nil, Walk is called recursively with the +// visitor and each of the non-nil children of node, followed by a call to +// w.Visit(nil). +func Walk(v Visitor, node Node) { //nolint:revive // cyclomatic + w := v.Visit(node) + if w == nil { + return + } + + switch n := node.(type) { + // Expressions. + case *NumericLiteral, *StringLiteral, *Identifier: + case *QualifiedName: + Walk(w, n.Left) + Walk(w, n.Right) + case *ArrayLiteralExpression: + for _, elem := range n.Elements { + Walk(w, elem) + } + case *EnumMember: + Walk(w, n.Name) + Walk(w, n.Initializer) + case *TypeParameter: + Walk(w, n.Name) + case *HeritageClause: + for _, typ := range n.Types { + Walk(w, typ) + } + case *ExpressionWithTypeArguments: + Walk(w, n.Expression) + case *PropertySignature: + Walk(w, n.Name) + Walk(w, n.Type) + case *IndexSignature: + for _, param := range n.Parameters { + Walk(w, param) + } + Walk(w, n.Type) + case *Parameter: + Walk(w, n.Name) + Walk(w, n.Type) + case *VariableDeclarationList: + for _, decl := range n.Declarations { + Walk(w, decl) + } + case *VariableDeclaration: + Walk(w, n.Name) + Walk(w, n.Type) + Walk(w, n.Initializer) + case *PrefixUnaryExpression: + Walk(w, n.Operand) + + // Types. + case *LiteralType: + Walk(w, n.Literal) + case *TypeLiteral: + for _, member := range n.Members { + Walk(w, member) + } + case *ArrayType: + Walk(w, n.ElementType) + case *TypeReference: + Walk(w, n.TypeName) + case *UnionType: + for _, typ := range n.Types { + Walk(w, typ) + } + case *TupleType: + for _, elem := range n.Elements { + Walk(w, elem) + } + case *ParenthesizedType: + Walk(w, n.Type) + + // Statements. + case *SourceFile: + for _, stmt := range n.Statements { + Walk(w, stmt) + } + case *ModuleBlock: + for _, stmt := range n.Statements { + Walk(w, stmt) + } + case *VariableStatement: + Walk(w, n.DeclarationList) + case *TypeAliasDeclaration: + Walk(w, n.Name) + Walk(w, n.Type) + case *EnumDeclaration: + Walk(w, n.Name) + for _, member := range n.Members { + Walk(w, member) + } + case *InterfaceDeclaration: + Walk(w, n.Name) + for _, member := range n.Members { + Walk(w, member) + } + case *ModuleDeclaration: + Walk(w, n.Name) + Walk(w, n.Body) + + default: + panic(fmt.Sprintf("unknown node type %T", n)) + } + + w.Visit(nil) +} + +type inspector func(Node) bool + +func (f inspector) Visit(node Node) Visitor { + if f(node) { + return f + } + return nil +} + +// Inspect traverses the AST rooted at node in depth-first order. +// +// It starts by calling f(node); node must not be nil. If f returns true, +// Inspect invokes f recursively for each of the non-nil children of node, +// followed by a call to f(nil). +func Inspect(node Node, f func(Node) bool) { + Walk(inspector(f), node) +} diff --git a/ast/walk_example_test.go b/ast/walk_example_test.go new file mode 100644 index 0000000..17ada6e --- /dev/null +++ b/ast/walk_example_test.go @@ -0,0 +1,31 @@ +package ast_test + +import ( + "fmt" + + "github.com/armsnyder/typescript-ast-go/ast" + "github.com/armsnyder/typescript-ast-go/parser" +) + +func Example() { + sourceFile := parser.Parse([]byte(` + export interface ProgressParams { + token: ProgressToken; + }`)) + + ast.Inspect(sourceFile, func(node ast.Node) bool { + if node != nil { + fmt.Printf("Visited %T\n", node) + } + return true + }) + + // Output: + // Visited *ast.SourceFile + // Visited *ast.InterfaceDeclaration + // Visited *ast.Identifier + // Visited *ast.PropertySignature + // Visited *ast.Identifier + // Visited *ast.TypeReference + // Visited *ast.Identifier +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..94af130 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/armsnyder/typescript-ast-go + +go 1.20 diff --git a/internal/testdata/additional_properties.ts.txt b/internal/testdata/additional_properties.ts.txt new file mode 100644 index 0000000..368d8ba --- /dev/null +++ b/internal/testdata/additional_properties.ts.txt @@ -0,0 +1,11 @@ +interface FormattingOptions { + /** + * Size of a tab in spaces. + */ + tabSize: uinteger; + + /** + * Signature for further properties. + */ + [key: string]: boolean | integer | string; +} diff --git a/internal/testdata/array.ts.txt b/internal/testdata/array.ts.txt new file mode 100644 index 0000000..1a89823 --- /dev/null +++ b/internal/testdata/array.ts.txt @@ -0,0 +1,6 @@ +/** + * LSP arrays. + * + * @since 3.17.0 + */ +export type LSPArray = LSPAny[]; diff --git a/internal/testdata/basic_type.ts.txt b/internal/testdata/basic_type.ts.txt new file mode 100644 index 0000000..dd9f66c --- /dev/null +++ b/internal/testdata/basic_type.ts.txt @@ -0,0 +1,4 @@ +/** + * Defines an integer number in the range of -2^31 to 2^31 - 1. + */ +export type integer = number; diff --git a/internal/testdata/enum.ts.txt b/internal/testdata/enum.ts.txt new file mode 100644 index 0000000..1df98a2 --- /dev/null +++ b/internal/testdata/enum.ts.txt @@ -0,0 +1,12 @@ +export enum SemanticTokenTypes { + namespace = 'namespace', + /** + * Represents a generic type. Acts as a fallback for types which + * can't be mapped to a specific type like class or enum. + */ + type = 'type', + class = 'class', + enum = 'enum', + interface = 'interface', + string = 'string', +} diff --git a/internal/testdata/generic.ts.txt b/internal/testdata/generic.ts.txt new file mode 100644 index 0000000..4daa478 --- /dev/null +++ b/internal/testdata/generic.ts.txt @@ -0,0 +1,11 @@ +interface ProgressParams { + /** + * The progress token provided by the client or server. + */ + token: ProgressToken; + + /** + * The progress data. + */ + value: T; +} diff --git a/internal/testdata/inline_type.ts.txt b/internal/testdata/inline_type.ts.txt new file mode 100644 index 0000000..404461f --- /dev/null +++ b/internal/testdata/inline_type.ts.txt @@ -0,0 +1,4 @@ +interface HoverParams { + textDocument: string; /** The text document's URI in string form */ + position: { line: uinteger; character: uinteger; }; +} diff --git a/internal/testdata/interface.ts.txt b/internal/testdata/interface.ts.txt new file mode 100644 index 0000000..0b17f80 --- /dev/null +++ b/internal/testdata/interface.ts.txt @@ -0,0 +1,17 @@ +interface ResponseMessage extends Message { + /** + * The request id. + */ + id: integer | string | null; + + /** + * The result of a request. This member is REQUIRED on success. + * This member MUST NOT exist if there was an error invoking the method. + */ + result?: string | number | boolean | array | object | null; + + /** + * The error object in case a request fails. + */ + error?: ResponseError; +} diff --git a/internal/testdata/interface_complex_syntax.ts.txt b/internal/testdata/interface_complex_syntax.ts.txt new file mode 100644 index 0000000..019c1e1 --- /dev/null +++ b/internal/testdata/interface_complex_syntax.ts.txt @@ -0,0 +1,40 @@ +export interface WorkspaceEdit { + /** + * Holds changes to existing resources. + */ + changes?: { [uri: DocumentUri]: TextEdit[]; }; + + /** + * Depending on the client capability + * `workspace.workspaceEdit.resourceOperations` document changes are either + * an array of `TextDocumentEdit`s to express changes to n different text + * documents where each text document edit addresses a specific version of + * a text document. Or it can contain above `TextDocumentEdit`s mixed with + * create, rename and delete file / folder operations. + * + * Whether a client supports versioned document edits is expressed via + * `workspace.workspaceEdit.documentChanges` client capability. + * + * If a client neither supports `documentChanges` nor + * `workspace.workspaceEdit.resourceOperations` then only plain `TextEdit`s + * using the `changes` property are supported. + */ + documentChanges?: ( + TextDocumentEdit[] | + (TextDocumentEdit | CreateFile | RenameFile | DeleteFile)[] + ); + + /** + * A map of change annotations that can be referenced in + * `AnnotatedTextEdit`s or create, rename and delete file / folder + * operations. + * + * Whether clients honor this property depends on the client capability + * `workspace.changeAnnotationSupport`. + * + * @since 3.16.0 + */ + changeAnnotations?: { + [id: string /* ChangeAnnotationIdentifier */]: ChangeAnnotation; + }; +} diff --git a/internal/testdata/interface_with_union_array.ts.txt b/internal/testdata/interface_with_union_array.ts.txt new file mode 100644 index 0000000..53bf3f3 --- /dev/null +++ b/internal/testdata/interface_with_union_array.ts.txt @@ -0,0 +1,9 @@ +export interface TextDocumentEdit { + /** + * The edits to be applied. + * + * @since 3.16.0 - support for AnnotatedTextEdit. This is guarded by the + * client capability `workspace.workspaceEdit.changeAnnotationSupport` + */ + edits: (TextEdit | AnnotatedTextEdit)[]; +} diff --git a/internal/testdata/interface_with_union_struct_field.ts.txt b/internal/testdata/interface_with_union_struct_field.ts.txt new file mode 100644 index 0000000..96c9e6c --- /dev/null +++ b/internal/testdata/interface_with_union_struct_field.ts.txt @@ -0,0 +1,51 @@ +/** + * Options specific to a notebook plus its cells + * to be synced to the server. + * + * If a selector provides a notebook document + * filter but no cell selector all cells of a + * matching notebook document will be synced. + * + * If a selector provides no notebook document + * filter but only a cell selector all notebook + * documents that contain at least one matching + * cell will be synced. + * + * @since 3.17.0 + */ +export interface NotebookDocumentSyncOptions { + /** + * The notebooks to be synced + */ + notebookSelector: ({ + /** + * The notebook to be synced. If a string + * value is provided it matches against the + * notebook type. '*' matches every notebook. + */ + notebook: string | NotebookDocumentFilter; + + /** + * The cells of the matching notebook to be synced. + */ + cells?: { language: string }[]; + } | { + /** + * The notebook to be synced. If a string + * value is provided it matches against the + * notebook type. '*' matches every notebook. + */ + notebook?: string | NotebookDocumentFilter; + + /** + * The cells of the matching notebook to be synced. + */ + cells: { language: string }[]; + })[]; + + /** + * Whether save notification should be forwarded to + * the server. Will only be honored if mode === `notebook`. + */ + save?: boolean; +} diff --git a/internal/testdata/multiple_extends.ts.txt b/internal/testdata/multiple_extends.ts.txt new file mode 100644 index 0000000..8877f4d --- /dev/null +++ b/internal/testdata/multiple_extends.ts.txt @@ -0,0 +1,8 @@ +/** + * Registration options specific to a notebook. + * + * @since 3.17.0 + */ +export interface NotebookDocumentSyncRegistrationOptions + extends NotebookDocumentSyncOptions, + StaticRegistrationOptions {} diff --git a/internal/testdata/namespace.ts.txt b/internal/testdata/namespace.ts.txt new file mode 100644 index 0000000..8e03045 --- /dev/null +++ b/internal/testdata/namespace.ts.txt @@ -0,0 +1,18 @@ +export namespace ErrorCodes { + // Defined by JSON-RPC + export const ParseError: integer = -32700; + export const InvalidRequest: integer = -32600; + + /** + * This is the start range of JSON-RPC reserved error codes. + * It doesn't denote a real error code. No LSP error codes should + * be defined between the start and end range. For backwards + * compatibility the `ServerNotInitialized` and the `UnknownErrorCode` + * are left in the range. + * + * @since 3.16.0 + */ + export const jsonrpcReservedErrorRangeStart: integer = -32099; + /** @deprecated use jsonrpcReservedErrorRangeStart */ + export const serverErrorStart: integer = jsonrpcReservedErrorRangeStart; +} diff --git a/internal/testdata/namespace_with_type.ts.txt b/internal/testdata/namespace_with_type.ts.txt new file mode 100644 index 0000000..2f948e6 --- /dev/null +++ b/internal/testdata/namespace_with_type.ts.txt @@ -0,0 +1,12 @@ +export namespace DiagnosticSeverity { + /** + * Reports an error. + */ + export const Error: 1 = 1; + /** + * Reports a warning. + */ + export const Warning: 2 = 2; +} + +export type DiagnosticSeverity = 1 | 2; diff --git a/internal/testdata/object.ts.txt b/internal/testdata/object.ts.txt new file mode 100644 index 0000000..21f410c --- /dev/null +++ b/internal/testdata/object.ts.txt @@ -0,0 +1,6 @@ +/** + * LSP object definition. + * + * @since 3.17.0 + */ +export type LSPObject = { [key: string]: LSPAny }; diff --git a/internal/testdata/qualified.ts.txt b/internal/testdata/qualified.ts.txt new file mode 100644 index 0000000..a763a4f --- /dev/null +++ b/internal/testdata/qualified.ts.txt @@ -0,0 +1,3 @@ +export interface FullDocumentDiagnosticReport { + kind: DocumentDiagnosticReportKind.Full; +} diff --git a/internal/testdata/readonly.ts.txt b/internal/testdata/readonly.ts.txt new file mode 100644 index 0000000..3c266d6 --- /dev/null +++ b/internal/testdata/readonly.ts.txt @@ -0,0 +1,8 @@ +export interface SemanticTokensDelta { + readonly resultId?: string; + /** + * The semantic token edits to transform a previous result into a new + * result. + */ + edits: SemanticTokensEdit[]; +} diff --git a/internal/testdata/tuple.ts.txt b/internal/testdata/tuple.ts.txt new file mode 100644 index 0000000..5b89f77 --- /dev/null +++ b/internal/testdata/tuple.ts.txt @@ -0,0 +1,3 @@ +export interface ParameterInformation { + label: string | [uinteger, uinteger]; +} diff --git a/internal/testdata/union_struct.ts.txt b/internal/testdata/union_struct.ts.txt new file mode 100644 index 0000000..7802691 --- /dev/null +++ b/internal/testdata/union_struct.ts.txt @@ -0,0 +1,34 @@ +/** + * A notebook document filter denotes a notebook document by + * different properties. + * + * @since 3.17.0 + */ +export type NotebookDocumentFilter = { + /** The type of the enclosing notebook. */ + notebookType: string; + + /** A Uri [scheme](#Uri.scheme), like `file` or `untitled`. */ + scheme?: string; + + /** A glob pattern. */ + pattern?: string; +} | { + /** The type of the enclosing notebook. */ + notebookType?: string; + + /** A Uri [scheme](#Uri.scheme), like `file` or `untitled`.*/ + scheme: string; + + /** A glob pattern. */ + pattern?: string; +} | { + /** The type of the enclosing notebook. */ + notebookType?: string; + + /** A Uri [scheme](#Uri.scheme), like `file` or `untitled`. */ + scheme?: string; + + /** A glob pattern. */ + pattern: string; +}; diff --git a/internal/testdata/union_type.ts.txt b/internal/testdata/union_type.ts.txt new file mode 100644 index 0000000..b28ac30 --- /dev/null +++ b/internal/testdata/union_type.ts.txt @@ -0,0 +1,7 @@ +/** + * The LSP any type + * + * @since 3.17.0 + */ +export type LSPAny = LSPObject | LSPArray | string | integer | uinteger | + decimal | boolean | null; diff --git a/parser/lexer.go b/parser/lexer.go new file mode 100644 index 0000000..1d183ad --- /dev/null +++ b/parser/lexer.go @@ -0,0 +1,256 @@ +package parser + +import ( + "bytes" + + "github.com/armsnyder/typescript-ast-go/token" +) + +type lexer struct { + Source []byte + + offset int + isInsideBlock bool + willBeTrailingComment bool + nextToken token.Token +} + +func (x *lexer) Peek() token.Token { + if x.nextToken.Kind == 0 { + x.nextToken = x.next() + } + + return x.nextToken +} + +func (x *lexer) Pop() token.Token { + if x.nextToken.Kind == 0 { + return x.next() + } + + t := x.nextToken + x.nextToken = token.Token{} + return t +} + +func (x *lexer) next() token.Token { + for x.offset < len(x.Source) { + switch x.Source[x.offset] { + case ' ', '\t', '\r': + x.offset++ + + case '\n': + x.offset++ + x.willBeTrailingComment = false + + case '/': + return x.nextComment() + + default: + x.willBeTrailingComment = true + + switch x.Source[x.offset] { + case '|': + return x.char(token.Or) + + case '=': + return x.char(token.Assign) + + case '-': + return x.char(token.Minus) + + case '(': + return x.char(token.LParen) + + case ')': + return x.char(token.RParen) + + case '[': + return x.char(token.LBrack) + + case ']': + return x.char(token.RBrack) + + case '{': + x.isInsideBlock = true + return x.char(token.LBrace) + + case '}': + x.isInsideBlock = false + return x.char(token.RBrace) + + case '<': + return x.char(token.LAngle) + + case '>': + return x.char(token.RAngle) + + case ',': + return x.char(token.Comma) + + case '.': + return x.char(token.Dot) + + case ':': + return x.char(token.Colon) + + case ';': + return x.char(token.Semicolon) + + case '?': + return x.char(token.Question) + + case '\'': + return x.nextString() + + default: + if x.Source[x.offset] >= '0' && x.Source[x.offset] <= '9' { + return x.nextNumber() + } + + if (x.Source[x.offset] >= 'a' && x.Source[x.offset] <= 'z') || (x.Source[x.offset] >= 'A' && x.Source[x.offset] <= 'Z') { + return x.nextIdent() + } + + return token.Token{Kind: token.Illegal} + } + } + } + + return token.Token{Kind: token.EOF} +} + +func (x *lexer) nextComment() token.Token { + if x.offset+1 >= len(x.Source) { + return token.Token{Kind: token.Illegal} + } + + switch x.Source[x.offset+1] { + case '/': + return x.nextLineComment() + + case '*': + return x.nextBlockComment() + + default: + return token.Token{Kind: token.Illegal} + } +} + +func (x *lexer) nextLineComment() token.Token { + x.offset += 2 + for x.offset+1 < len(x.Source) && x.Source[x.offset] == ' ' { + x.offset++ + } + + commentStart := x.offset + + for x.offset < len(x.Source) && x.Source[x.offset] != '\n' { + x.offset++ + } + + kind := token.Comment + if x.willBeTrailingComment { + kind = token.LineComment + } + + return token.Token{Kind: kind, Text: string(x.Source[commentStart:x.offset])} +} + +func (x *lexer) nextBlockComment() token.Token { + x.offset += 2 + for x.offset+1 < len(x.Source) && x.Source[x.offset] == '*' { + x.offset++ + } + + innerEndIndex := bytes.Index(x.Source[x.offset:], []byte("*/")) + if innerEndIndex == -1 { + return token.Token{Kind: token.Illegal} + } + innerEndIndex += x.offset + endIndex := innerEndIndex + 2 + for innerEndIndex > 0 && x.Source[innerEndIndex-1] == '*' { + innerEndIndex-- + } + + var comment []byte + + for x.offset < innerEndIndex { + lineEnd := bytes.Index(x.Source[x.offset:innerEndIndex], []byte("\n")) + if lineEnd == -1 { + comment = append(comment, x.Source[x.offset:innerEndIndex]...) + break + } + + comment = append(comment, x.Source[x.offset:x.offset+lineEnd]...) + comment = append(comment, '\n') + x.offset += lineEnd + 1 + + i := bytes.IndexFunc(x.Source[x.offset:innerEndIndex], func(r rune) bool { + return r != ' ' && r != '\t' && r != '*' + }) + if i == -1 { + break + } + + x.offset += i + } + + x.offset = endIndex + + kind := token.Comment + if x.willBeTrailingComment { + kind = token.LineComment + } + + return token.Token{Kind: kind, Text: string(bytes.TrimSpace(comment))} +} + +func (x *lexer) nextString() token.Token { + x.offset++ + start := x.offset + + end := bytes.IndexByte(x.Source[x.offset:], '\'') + if end == -1 { + return token.Token{Kind: token.Illegal} + } + + x.offset += end + 1 + + return token.Token{Kind: token.String, Text: string(x.Source[start : x.offset-1])} +} + +func (x *lexer) char(kind token.Kind) token.Token { + x.offset++ + return token.Token{Kind: kind} +} + +func (x *lexer) nextNumber() token.Token { + start := x.offset + for x.offset < len(x.Source) && x.Source[x.offset] >= '0' && x.Source[x.offset] <= '9' { + x.offset++ + } + return token.Token{Kind: token.Number, Text: string(x.Source[start:x.offset])} +} + +func (x *lexer) nextIdent() (tok token.Token) { + start := x.offset + + end := bytes.IndexFunc(x.Source[x.offset:], func(r rune) bool { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_': + return false + default: + return true + } + }) + if end == -1 { + x.offset = len(x.Source) + } else { + x.offset += end + } + + value := string(x.Source[start:x.offset]) + + return token.Token{Kind: token.Ident, Text: value} +} diff --git a/parser/lexer_test.go b/parser/lexer_test.go new file mode 100644 index 0000000..46667c3 --- /dev/null +++ b/parser/lexer_test.go @@ -0,0 +1,559 @@ +package parser + +import ( + "fmt" + "os" + "testing" + + "github.com/armsnyder/typescript-ast-go/token" +) + +func TestLexer(t *testing.T) { + tests := []struct { + testdata string + want []token.Token + }{ + { + testdata: "array", + want: []token.Token{ + {Kind: token.Comment, Text: "LSP arrays.\n\n@since 3.17.0"}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "type"}, + {Kind: token.Ident, Text: "LSPArray"}, + {Kind: token.Assign}, + {Kind: token.Ident, Text: "LSPAny"}, + {Kind: token.LBrack}, + {Kind: token.RBrack}, + {Kind: token.Semicolon}, + }, + }, + { + testdata: "basic_type", + want: []token.Token{ + {Kind: token.Comment, Text: "Defines an integer number in the range of -2^31 to 2^31 - 1."}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "type"}, + {Kind: token.Ident, Text: "integer"}, + {Kind: token.Assign}, + {Kind: token.Ident, Text: "number"}, + {Kind: token.Semicolon}, + }, + }, + { + testdata: "enum", + want: []token.Token{ + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "enum"}, + {Kind: token.Ident, Text: "SemanticTokenTypes"}, + {Kind: token.LBrace}, + {Kind: token.Ident, Text: "namespace"}, + {Kind: token.Assign}, + {Kind: token.String, Text: "namespace"}, + {Kind: token.Comma}, + {Kind: token.Comment, Text: "Represents a generic type. Acts as a fallback for types which\ncan't be mapped to a specific type like class or enum."}, + {Kind: token.Ident, Text: "type"}, + {Kind: token.Assign}, + {Kind: token.String, Text: "type"}, + {Kind: token.Comma}, + {Kind: token.Ident, Text: "class"}, + {Kind: token.Assign}, + {Kind: token.String, Text: "class"}, + {Kind: token.Comma}, + {Kind: token.Ident, Text: "enum"}, + {Kind: token.Assign}, + {Kind: token.String, Text: "enum"}, + {Kind: token.Comma}, + {Kind: token.Ident, Text: "interface"}, + {Kind: token.Assign}, + {Kind: token.String, Text: "interface"}, + {Kind: token.Comma}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Assign}, + {Kind: token.String, Text: "string"}, + {Kind: token.Comma}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "generic", + want: []token.Token{ + {Kind: token.Ident, Text: "interface"}, + {Kind: token.Ident, Text: "ProgressParams"}, + {Kind: token.LAngle}, + {Kind: token.Ident, Text: "T"}, + {Kind: token.RAngle}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "The progress token provided by the client or server."}, + {Kind: token.Ident, Text: "token"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "ProgressToken"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "The progress data."}, + {Kind: token.Ident, Text: "value"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "T"}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "inline_type", + want: []token.Token{ + {Kind: token.Ident, Text: "interface"}, + {Kind: token.Ident, Text: "HoverParams"}, + {Kind: token.LBrace}, + {Kind: token.Ident, Text: "textDocument"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.LineComment, Text: "The text document's URI in string form"}, + {Kind: token.Ident, Text: "position"}, + {Kind: token.Colon}, + {Kind: token.LBrace}, + {Kind: token.Ident, Text: "line"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "uinteger"}, + {Kind: token.Semicolon}, + {Kind: token.Ident, Text: "character"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "uinteger"}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "interface", + want: []token.Token{ + {Kind: token.Ident, Text: "interface"}, + {Kind: token.Ident, Text: "ResponseMessage"}, + {Kind: token.Ident, Text: "extends"}, + {Kind: token.Ident, Text: "Message"}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "The request id."}, + {Kind: token.Ident, Text: "id"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "integer"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "null"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "The result of a request. This member is REQUIRED on success.\nThis member MUST NOT exist if there was an error invoking the method."}, + {Kind: token.Ident, Text: "result"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "number"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "boolean"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "array"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "object"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "null"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "The error object in case a request fails."}, + {Kind: token.Ident, Text: "error"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "ResponseError"}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "interface_complex_syntax", + want: []token.Token{ + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "interface"}, + {Kind: token.Ident, Text: "WorkspaceEdit"}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "Holds changes to existing resources."}, + {Kind: token.Ident, Text: "changes"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.LBrace}, + {Kind: token.LBrack}, + {Kind: token.Ident, Text: "uri"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "DocumentUri"}, + {Kind: token.RBrack}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "TextEdit"}, + {Kind: token.LBrack}, + {Kind: token.RBrack}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "Depending on the client capability\n`workspace.workspaceEdit.resourceOperations` document changes are either\nan array of `TextDocumentEdit`s to express changes to n different text\ndocuments where each text document edit addresses a specific version of\na text document. Or it can contain above `TextDocumentEdit`s mixed with\ncreate, rename and delete file / folder operations.\n\nWhether a client supports versioned document edits is expressed via\n`workspace.workspaceEdit.documentChanges` client capability.\n\nIf a client neither supports `documentChanges` nor\n`workspace.workspaceEdit.resourceOperations` then only plain `TextEdit`s\nusing the `changes` property are supported."}, + {Kind: token.Ident, Text: "documentChanges"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.LParen}, + {Kind: token.Ident, Text: "TextDocumentEdit"}, + {Kind: token.LBrack}, + {Kind: token.RBrack}, + {Kind: token.Or}, + {Kind: token.LParen}, + {Kind: token.Ident, Text: "TextDocumentEdit"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "CreateFile"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "RenameFile"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "DeleteFile"}, + {Kind: token.RParen}, + {Kind: token.LBrack}, + {Kind: token.RBrack}, + {Kind: token.RParen}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "A map of change annotations that can be referenced in\n`AnnotatedTextEdit`s or create, rename and delete file / folder\noperations.\n\nWhether clients honor this property depends on the client capability\n`workspace.changeAnnotationSupport`.\n\n@since 3.16.0"}, + {Kind: token.Ident, Text: "changeAnnotations"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.LBrace}, + {Kind: token.LBrack}, + {Kind: token.Ident, Text: "id"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.LineComment, Text: "ChangeAnnotationIdentifier"}, + {Kind: token.RBrack}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "ChangeAnnotation"}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "interface_with_union_array", + want: []token.Token{ + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "interface"}, + {Kind: token.Ident, Text: "TextDocumentEdit"}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "The edits to be applied.\n\n@since 3.16.0 - support for AnnotatedTextEdit. This is guarded by the\nclient capability `workspace.workspaceEdit.changeAnnotationSupport`"}, + {Kind: token.Ident, Text: "edits"}, + {Kind: token.Colon}, + {Kind: token.LParen}, + {Kind: token.Ident, Text: "TextEdit"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "AnnotatedTextEdit"}, + {Kind: token.RParen}, + {Kind: token.LBrack}, + {Kind: token.RBrack}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "interface_with_union_struct_field", + want: []token.Token{ + {Kind: token.Comment, Text: "Options specific to a notebook plus its cells\nto be synced to the server.\n\nIf a selector provides a notebook document\nfilter but no cell selector all cells of a\nmatching notebook document will be synced.\n\nIf a selector provides no notebook document\nfilter but only a cell selector all notebook\ndocuments that contain at least one matching\ncell will be synced.\n\n@since 3.17.0"}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "interface"}, + {Kind: token.Ident, Text: "NotebookDocumentSyncOptions"}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "The notebooks to be synced"}, + {Kind: token.Ident, Text: "notebookSelector"}, + {Kind: token.Colon}, + {Kind: token.LParen}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "The notebook to be synced. If a string\nvalue is provided it matches against the\nnotebook type. '*' matches every notebook."}, + {Kind: token.Ident, Text: "notebook"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "NotebookDocumentFilter"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "The cells of the matching notebook to be synced."}, + {Kind: token.Ident, Text: "cells"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.LBrace}, + {Kind: token.Ident, Text: "language"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.RBrace}, + {Kind: token.LBrack}, + {Kind: token.RBrack}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + {Kind: token.Or}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "The notebook to be synced. If a string\nvalue is provided it matches against the\nnotebook type. '*' matches every notebook."}, + {Kind: token.Ident, Text: "notebook"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "NotebookDocumentFilter"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "The cells of the matching notebook to be synced."}, + {Kind: token.Ident, Text: "cells"}, + {Kind: token.Colon}, + {Kind: token.LBrace}, + {Kind: token.Ident, Text: "language"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.RBrace}, + {Kind: token.LBrack}, + {Kind: token.RBrack}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + {Kind: token.RParen}, + {Kind: token.LBrack}, + {Kind: token.RBrack}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "Whether save notification should be forwarded to\nthe server. Will only be honored if mode === `notebook`."}, + {Kind: token.Ident, Text: "save"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "boolean"}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "multiple_extends", + want: []token.Token{ + {Kind: token.Comment, Text: "Registration options specific to a notebook.\n\n@since 3.17.0"}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "interface"}, + {Kind: token.Ident, Text: "NotebookDocumentSyncRegistrationOptions"}, + {Kind: token.Ident, Text: "extends"}, + {Kind: token.Ident, Text: "NotebookDocumentSyncOptions"}, + {Kind: token.Comma}, + {Kind: token.Ident, Text: "StaticRegistrationOptions"}, + {Kind: token.LBrace}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "namespace", + want: []token.Token{ + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "namespace"}, + {Kind: token.Ident, Text: "ErrorCodes"}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "Defined by JSON-RPC"}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "const"}, + {Kind: token.Ident, Text: "ParseError"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "integer"}, + {Kind: token.Assign}, + {Kind: token.Minus}, + {Kind: token.Number, Text: "32700"}, + {Kind: token.Semicolon}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "const"}, + {Kind: token.Ident, Text: "InvalidRequest"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "integer"}, + {Kind: token.Assign}, + {Kind: token.Minus}, + {Kind: token.Number, Text: "32600"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "This is the start range of JSON-RPC reserved error codes.\nIt doesn't denote a real error code. No LSP error codes should\nbe defined between the start and end range. For backwards\ncompatibility the `ServerNotInitialized` and the `UnknownErrorCode`\nare left in the range.\n\n@since 3.16.0"}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "const"}, + {Kind: token.Ident, Text: "jsonrpcReservedErrorRangeStart"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "integer"}, + {Kind: token.Assign}, + {Kind: token.Minus}, + {Kind: token.Number, Text: "32099"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "@deprecated use jsonrpcReservedErrorRangeStart"}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "const"}, + {Kind: token.Ident, Text: "serverErrorStart"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "integer"}, + {Kind: token.Assign}, + {Kind: token.Ident, Text: "jsonrpcReservedErrorRangeStart"}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "object", + want: []token.Token{ + {Kind: token.Comment, Text: "LSP object definition.\n\n@since 3.17.0"}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "type"}, + {Kind: token.Ident, Text: "LSPObject"}, + {Kind: token.Assign}, + {Kind: token.LBrace}, + {Kind: token.LBrack}, + {Kind: token.Ident, Text: "key"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.RBrack}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "LSPAny"}, + {Kind: token.RBrace}, + {Kind: token.Semicolon}, + }, + }, + { + testdata: "readonly", + want: []token.Token{ + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "interface"}, + {Kind: token.Ident, Text: "SemanticTokensDelta"}, + {Kind: token.LBrace}, + {Kind: token.Ident, Text: "readonly"}, + {Kind: token.Ident, Text: "resultId"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "The semantic token edits to transform a previous result into a new\nresult."}, + {Kind: token.Ident, Text: "edits"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "SemanticTokensEdit"}, + {Kind: token.LBrack}, + {Kind: token.RBrack}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + }, + }, + { + testdata: "union_struct", + want: []token.Token{ + {Kind: token.Comment, Text: "A notebook document filter denotes a notebook document by\ndifferent properties.\n\n@since 3.17.0"}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "type"}, + {Kind: token.Ident, Text: "NotebookDocumentFilter"}, + {Kind: token.Assign}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "The type of the enclosing notebook."}, + {Kind: token.Ident, Text: "notebookType"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "A Uri [scheme](#Uri.scheme), like `file` or `untitled`."}, + {Kind: token.Ident, Text: "scheme"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "A glob pattern."}, + {Kind: token.Ident, Text: "pattern"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + {Kind: token.Or}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "The type of the enclosing notebook."}, + {Kind: token.Ident, Text: "notebookType"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "A Uri [scheme](#Uri.scheme), like `file` or `untitled`."}, + {Kind: token.Ident, Text: "scheme"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "A glob pattern."}, + {Kind: token.Ident, Text: "pattern"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + {Kind: token.Or}, + {Kind: token.LBrace}, + {Kind: token.Comment, Text: "The type of the enclosing notebook."}, + {Kind: token.Ident, Text: "notebookType"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "A Uri [scheme](#Uri.scheme), like `file` or `untitled`."}, + {Kind: token.Ident, Text: "scheme"}, + {Kind: token.Question}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.Comment, Text: "A glob pattern."}, + {Kind: token.Ident, Text: "pattern"}, + {Kind: token.Colon}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Semicolon}, + {Kind: token.RBrace}, + {Kind: token.Semicolon}, + }, + }, + { + testdata: "union_type", + // [Comment("The LSP any type\n\n@since 3.17.0") Ident, Text: "export" Type Ident("LSPAny") Assign Ident("LSPObject") Or Ident("LSPArray") Or Ident("string") Or Ident("integer") Or Ident("uinteger") Or Ident("decimal") Or Ident("boolean") Or Ident("null") Semicolon] + want: []token.Token{ + {Kind: token.Comment, Text: "The LSP any type\n\n@since 3.17.0"}, + {Kind: token.Ident, Text: "export"}, + {Kind: token.Ident, Text: "type"}, + {Kind: token.Ident, Text: "LSPAny"}, + {Kind: token.Assign}, + {Kind: token.Ident, Text: "LSPObject"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "LSPArray"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "string"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "integer"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "uinteger"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "decimal"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "boolean"}, + {Kind: token.Or}, + {Kind: token.Ident, Text: "null"}, + {Kind: token.Semicolon}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.testdata, func(t *testing.T) { + source, err := os.ReadFile(fmt.Sprintf("../internal/testdata/%s.ts.txt", tt.testdata)) + if err != nil { + t.Fatal(err) + } + + lex := lexer{Source: source} + + var got []token.Token + for { + tok := lex.Pop() + if tok.Kind == token.EOF { + break + } + got = append(got, tok) + if tok.Kind == token.Illegal { + break + } + } + + if len(got) != len(tt.want) { + t.Fatalf("\ngot:\n%v\nwant:\n%v", got, tt.want) + } + + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("\ngot:\n%v\nwant:\n%v", got, tt.want) + } + } + }) + } +} diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 0000000..d0271cc --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,431 @@ +// Package parser provides a [Parse] function for parsing TypeScript source +// files into an abstract syntax tree (AST). +package parser + +import ( + "fmt" + + "github.com/armsnyder/typescript-ast-go/ast" + "github.com/armsnyder/typescript-ast-go/token" +) + +func Parse(source []byte) *ast.SourceFile { + p := parser{lex: &lexer{Source: source}} + return p.parseSourceFile() +} + +type parser struct { + lex *lexer + tok token.Token + lastComment string + lastLineComment string +} + +func (p *parser) parseSourceFile() *ast.SourceFile { + sourceFile := &ast.SourceFile{} + + for { + p.advance() + if p.tok.Kind == token.EOF { + return sourceFile + } + sourceFile.Statements = append(sourceFile.Statements, p.parseStatement()) + } +} + +func (p *parser) parseStatement() ast.Stmt { + p.expect(token.Ident) + for { + switch p.tok.Text { + case "export": + p.advance() + case "const": + return p.parseVariableStatement() + case "type": + return p.parseTypeAliasDeclaration() + case "enum": + return p.parseEnumDeclaration() + case "interface": + return p.parseInterfaceDeclaration() + case "namespace": + return p.parseModuleDeclaration() + default: + panic(fmt.Sprintf("unexpected token %s", p.tok)) + } + } +} + +func (p *parser) parseVariableStatement() *ast.VariableStatement { + p.eat(token.Ident) + decl := &ast.VariableStatement{LeadingComment: p.consumeComment()} + decl.DeclarationList = p.parseVariableDeclarationList() + return decl +} + +func (p *parser) parseVariableDeclarationList() *ast.VariableDeclarationList { + decl := &ast.VariableDeclarationList{} + for { + decl.Declarations = append(decl.Declarations, p.parseVariableDeclaration()) + if p.tok.Kind != token.Comma { + p.eat(token.Semicolon) + return decl + } + p.advance() + } +} + +func (p *parser) parseVariableDeclaration() *ast.VariableDeclaration { + decl := &ast.VariableDeclaration{} + decl.Name = p.parseIdentifier() + if p.tok.Kind == token.Colon { + p.advance() + decl.Type = p.parseType() + } + if p.tok.Kind == token.Assign { + p.advance() + decl.Initializer = p.parseInitializer() + } + return decl +} + +func (p *parser) parseModuleDeclaration() *ast.ModuleDeclaration { + p.eat(token.Ident) + decl := &ast.ModuleDeclaration{LeadingComment: p.consumeComment()} + decl.Name = p.parseIdentifier() + p.eat(token.LBrace) + decl.Body = &ast.ModuleBlock{} + for { + if p.tok.Kind == token.RBrace { + p.eat(token.RBrace) + return decl + } + decl.Body.Statements = append(decl.Body.Statements, p.parseStatement()) + } +} + +func (p *parser) parseTypeAliasDeclaration() *ast.TypeAliasDeclaration { + p.eat(token.Ident) + decl := &ast.TypeAliasDeclaration{LeadingComment: p.consumeComment()} + decl.Name = p.parseIdentifier() + p.eat(token.Assign) + decl.Type = p.parseType() + return decl +} + +func (p *parser) parseEnumDeclaration() *ast.EnumDeclaration { + p.eat(token.Ident) + decl := &ast.EnumDeclaration{LeadingComment: p.consumeComment()} + decl.Name = p.parseIdentifier() + p.eat(token.LBrace) + for { + if p.tok.Kind == token.RBrace { + break + } + decl.Members = append(decl.Members, p.parseEnumMember()) + switch p.tok.Kind { + case token.Comma: + p.advance() + case token.RBrace: + default: + panic(fmt.Sprintf("unexpected token %s", p.tok)) + } + } + p.eat(token.RBrace) + return decl +} + +func (p *parser) parseInterfaceDeclaration() *ast.InterfaceDeclaration { + p.eat(token.Ident) + decl := &ast.InterfaceDeclaration{LeadingComment: p.consumeComment()} + decl.Name = p.parseIdentifier() + if p.tok.Kind == token.LAngle { + decl.TypeParameters = p.parseTypeParameters() + } + if p.tok.Kind == token.Ident && p.tok.Text == "extends" { + decl.HeritageClauses = p.parseHeritageClauses() + } + p.eat(token.LBrace) + for { + if p.tok.Kind == token.RBrace { + break + } + decl.Members = append(decl.Members, p.parseSignature()) + } + p.eat(token.RBrace) + return decl +} + +func (p *parser) parseSignature() ast.Signature { + switch p.tok.Kind { + case token.Ident: + return p.parsePropertySignature() + case token.LBrack: + return p.parseIndexSignature() + default: + panic(fmt.Sprintf("unexpected token %s", p.tok)) + } +} + +func (p *parser) parseHeritageClauses() []*ast.HeritageClause { + p.eat(token.Ident) + var heritageClauses []*ast.HeritageClause + for { + heritageClauses = append(heritageClauses, p.parseHeritageClause()) + if p.tok.Kind != token.Comma { + break + } + p.advance() + } + return heritageClauses +} + +func (p *parser) parseHeritageClause() *ast.HeritageClause { + return &ast.HeritageClause{ + Types: []*ast.ExpressionWithTypeArguments{p.parseExpressionWithTypeArguments()}, + } +} + +func (p *parser) parseExpressionWithTypeArguments() *ast.ExpressionWithTypeArguments { + return &ast.ExpressionWithTypeArguments{Expression: p.parseIdentifier()} +} + +func (p *parser) parseTypeParameters() []*ast.TypeParameter { + p.eat(token.LAngle) + var typeParameters []*ast.TypeParameter + for { + typeParameters = append(typeParameters, p.parseTypeParameter()) + if p.tok.Kind != token.Comma { + break + } + p.advance() + } + p.eat(token.RAngle) + return typeParameters +} + +func (p *parser) parseTypeParameter() *ast.TypeParameter { + return &ast.TypeParameter{Name: p.parseIdentifier()} +} + +func (p *parser) parsePropertySignature() *ast.PropertySignature { + signature := &ast.PropertySignature{LeadingComment: p.consumeComment()} + if p.tok.Kind == token.Ident && p.tok.Text == "readonly" { + p.advance() + } + signature.Name = p.parseIdentifier() + if p.tok.Kind == token.Question { + signature.QuestionToken = true + p.advance() + } + p.eat(token.Colon) + signature.Type = p.parseType() + if p.tok.Kind == token.Semicolon { + p.advance() + } + signature.TrailingComment = p.consumeLineComment() + return signature +} + +func (p *parser) parseEnumMember() *ast.EnumMember { + member := &ast.EnumMember{ + Name: p.parseIdentifier(), + LeadingComment: p.consumeComment(), + } + if p.tok.Kind != token.Assign { + return member + } + p.advance() + member.Initializer = p.parseInitializer() + return member +} + +func (p *parser) parseInitializer() ast.Expr { + switch p.tok.Kind { + case token.Number: + return &ast.NumericLiteral{Text: p.eat(token.Number).Text} + case token.String: + return &ast.StringLiteral{Text: p.eat(token.String).Text} + case token.Minus: + p.advance() + return &ast.PrefixUnaryExpression{ + Operator: token.Minus, + Operand: p.parseInitializer(), + } + case token.Ident: + return &ast.TypeReference{TypeName: p.parseIdentifier()} + case token.LBrack: + return p.parseArrayLiteralExpression() + default: + panic(fmt.Sprintf("unexpected token %s", p.tok)) + } +} + +func (p *parser) parseArrayLiteralExpression() *ast.ArrayLiteralExpression { + p.eat(token.LBrack) + expr := &ast.ArrayLiteralExpression{} + for p.tok.Kind != token.RBrack { + expr.Elements = append(expr.Elements, p.parseInitializer()) + if p.tok.Kind == token.Comma { + p.advance() + } + } + p.eat(token.RBrack) + return expr +} + +func (p *parser) parseIdentifier() *ast.Identifier { + return &ast.Identifier{Text: p.eat(token.Ident).Text} +} + +func (p *parser) parseType() ast.Type { + return p.parseTypeCheckUnion() +} + +func (p *parser) parseTypeCheckUnion() ast.Type { + typ := p.parseTypeCheckArray() + if p.tok.Kind != token.Or { + return typ + } + + types := []ast.Type{typ} + for { + p.eat(token.Or) + types = append(types, p.parseTypeCheckArray()) + if p.tok.Kind != token.Or { + return &ast.UnionType{Types: types} + } + } +} + +func (p *parser) parseTypeCheckArray() ast.Type { + typ := p.parseTypeInner() + if p.tok.Kind != token.LBrack { + return typ + } + + p.eat(token.LBrack) + p.eat(token.RBrack) + return &ast.ArrayType{ElementType: typ} +} + +func (p *parser) parseTypeInner() ast.Type { + switch p.tok.Kind { + case token.Ident: + return p.parseTypeReference() + case token.LBrace: + return p.parseTypeLiteral() + case token.LParen: + return p.parseParenthesizedType() + case token.LBrack: + return p.parseTupleType() + case token.String: + return &ast.LiteralType{Literal: &ast.StringLiteral{Text: p.eat(token.String).Text}} + case token.Number: + return &ast.LiteralType{Literal: &ast.NumericLiteral{Text: p.eat(token.Number).Text}} + default: + panic(fmt.Sprintf("unexpected token %s", p.tok)) + } +} + +func (p *parser) parseTupleType() *ast.TupleType { + p.eat(token.LBrack) + els := []ast.Type{} + for { + els = append(els, p.parseType()) + if p.tok.Kind != token.Comma { + p.eat(token.RBrack) + return &ast.TupleType{Elements: els} + } + p.advance() + } +} + +func (p *parser) parseTypeReference() *ast.TypeReference { + first := p.parseIdentifier() + if p.tok.Kind != token.Dot { + return &ast.TypeReference{TypeName: first} + } + p.advance() + return &ast.TypeReference{TypeName: &ast.QualifiedName{Left: first, Right: p.parseIdentifier()}} +} + +func (p *parser) parseParenthesizedType() *ast.ParenthesizedType { + p.eat(token.LParen) + typ := p.parseType() + p.eat(token.RParen) + return &ast.ParenthesizedType{Type: typ} +} + +func (p *parser) parseTypeLiteral() *ast.TypeLiteral { + p.eat(token.LBrace) + literal := &ast.TypeLiteral{} + for { + if p.tok.Kind == token.RBrace { + p.eat(token.RBrace) + return literal + } + literal.Members = append(literal.Members, p.parseSignature()) + } +} + +func (p *parser) parseIndexSignature() *ast.IndexSignature { + signature := &ast.IndexSignature{LeadingComment: p.consumeComment()} + p.eat(token.LBrack) + for p.tok.Kind != token.RBrack { + signature.Parameters = append(signature.Parameters, p.parseParameter()) + } + p.consumeLineComment() // Throw away + p.eat(token.RBrack) + p.eat(token.Colon) + signature.Type = p.parseType() + if p.tok.Kind == token.Semicolon { + p.advance() + } + return signature +} + +func (p *parser) parseParameter() *ast.Parameter { + name := p.parseIdentifier() + p.eat(token.Colon) + typ := p.parseType() + return &ast.Parameter{Name: name, Type: typ} +} + +func (p *parser) eat(kind token.Kind) token.Token { + p.expect(kind) + tok := p.tok + p.advance() + return tok +} + +func (p *parser) expect(kind token.Kind) { + if p.tok.Kind != kind { + panic(fmt.Sprintf("expected kind %s, got %s", kind, p.tok)) + } +} + +func (p *parser) advance() { + for { + p.tok = p.lex.Pop() + switch p.tok.Kind { + case token.Comment: + p.lastComment = p.tok.Text + case token.LineComment: + p.lastLineComment = p.tok.Text + default: + return + } + } +} + +func (p *parser) consumeComment() string { + comment := p.lastComment + p.lastComment = "" + return comment +} + +func (p *parser) consumeLineComment() string { + comment := p.lastLineComment + p.lastLineComment = "" + return comment +} diff --git a/parser/parser_example_test.go b/parser/parser_example_test.go new file mode 100644 index 0000000..db7467e --- /dev/null +++ b/parser/parser_example_test.go @@ -0,0 +1,25 @@ +package parser_test + +import ( + "fmt" + + "github.com/armsnyder/typescript-ast-go/parser" +) + +func Example() { + src := []byte(` + export interface ProgressParams { + /** + * The progress token provided by the client or server. + */ + token: ProgressToken; + + /** + * The progress data. + */ + value: T; + }`) + sourceFile := parser.Parse(src) + fmt.Printf("Parsed %T", sourceFile.Statements[0]) + // Output: Parsed *ast.InterfaceDeclaration +} diff --git a/parser/parser_test.go b/parser/parser_test.go new file mode 100644 index 0000000..c1f06b9 --- /dev/null +++ b/parser/parser_test.go @@ -0,0 +1,1022 @@ +package parser_test + +import ( + "fmt" + "os" + "reflect" + "strings" + "testing" + + "github.com/armsnyder/typescript-ast-go/ast" + "github.com/armsnyder/typescript-ast-go/parser" + "github.com/armsnyder/typescript-ast-go/token" +) + +func TestParser_Testdata(t *testing.T) { + tests := []struct { + testdata string + want *ast.SourceFile + }{ + { + testdata: "additional_properties", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "FormattingOptions"}, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "tabSize"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "uinteger"}, + }, + LeadingComment: "Size of a tab in spaces.", + }, + &ast.IndexSignature{ + Parameters: []*ast.Parameter{{ + Name: &ast.Identifier{Text: "key"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + }}, + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "boolean"}, + }, + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "integer"}, + }, + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + }, + }, + LeadingComment: "Signature for further properties.", + }, + }, + }, + }, + }, + }, + { + testdata: "array", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.TypeAliasDeclaration{ + Name: &ast.Identifier{Text: "LSPArray"}, + Type: &ast.ArrayType{ + ElementType: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "LSPAny"}, + }, + }, + LeadingComment: "LSP arrays.\n\n@since 3.17.0", + }, + }, + }, + }, + { + testdata: "basic_type", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.TypeAliasDeclaration{ + Name: &ast.Identifier{Text: "integer"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "number"}, + }, + LeadingComment: "Defines an integer number in the range of -2^31 to 2^31 - 1.", + }, + }, + }, + }, + { + testdata: "enum", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.EnumDeclaration{ + Name: &ast.Identifier{Text: "SemanticTokenTypes"}, + Members: []*ast.EnumMember{ + { + Name: &ast.Identifier{Text: "namespace"}, + Initializer: &ast.StringLiteral{Text: "namespace"}, + }, + { + Name: &ast.Identifier{Text: "type"}, + Initializer: &ast.StringLiteral{Text: "type"}, + LeadingComment: "Represents a generic type. Acts as a fallback for types which\ncan't be mapped to a specific type like class or enum.", + }, + { + Name: &ast.Identifier{Text: "class"}, + Initializer: &ast.StringLiteral{Text: "class"}, + }, + { + Name: &ast.Identifier{Text: "enum"}, + Initializer: &ast.StringLiteral{Text: "enum"}, + }, + { + Name: &ast.Identifier{Text: "interface"}, + Initializer: &ast.StringLiteral{Text: "interface"}, + }, + { + Name: &ast.Identifier{Text: "string"}, + Initializer: &ast.StringLiteral{Text: "string"}, + }, + }, + }, + }, + }, + }, + { + testdata: "generic", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "ProgressParams"}, + TypeParameters: []*ast.TypeParameter{ + {Name: &ast.Identifier{Text: "T"}}, + }, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "token"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "ProgressToken"}, + }, + LeadingComment: "The progress token provided by the client or server.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "value"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "T"}, + }, + LeadingComment: "The progress data.", + }, + }, + }, + }, + }, + }, + { + testdata: "inline_type", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "HoverParams"}, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "textDocument"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + TrailingComment: "The text document's URI in string form", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "position"}, + Type: &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "line"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "uinteger"}, + }, + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "character"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "uinteger"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + testdata: "interface", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "ResponseMessage"}, + HeritageClauses: []*ast.HeritageClause{ + { + Types: []*ast.ExpressionWithTypeArguments{ + {Expression: &ast.Identifier{Text: "Message"}}, + }, + }, + }, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "id"}, + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeReference{TypeName: &ast.Identifier{Text: "integer"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "string"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "null"}}, + }, + }, + LeadingComment: "The request id.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "result"}, + QuestionToken: true, + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeReference{TypeName: &ast.Identifier{Text: "string"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "number"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "boolean"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "array"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "object"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "null"}}, + }, + }, + LeadingComment: "The result of a request. This member is REQUIRED on success.\nThis member MUST NOT exist if there was an error invoking the method.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "error"}, + QuestionToken: true, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "ResponseError"}, + }, + LeadingComment: "The error object in case a request fails.", + }, + }, + }, + }, + }, + }, + { + testdata: "interface_complex_syntax", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "WorkspaceEdit"}, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "changes"}, + QuestionToken: true, + Type: &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.IndexSignature{ + Parameters: []*ast.Parameter{{ + Name: &ast.Identifier{Text: "uri"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "DocumentUri"}, + }, + }}, + Type: &ast.ArrayType{ + ElementType: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "TextEdit"}, + }, + }, + }, + }, + }, + LeadingComment: "Holds changes to existing resources.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "documentChanges"}, + QuestionToken: true, + Type: &ast.ParenthesizedType{ + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.ArrayType{ + ElementType: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "TextDocumentEdit"}, + }, + }, + &ast.ArrayType{ + ElementType: &ast.ParenthesizedType{ + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "TextDocumentEdit"}, + }, + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "CreateFile"}, + }, + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "RenameFile"}, + }, + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "DeleteFile"}, + }, + }, + }, + }, + }, + }, + }, + }, + LeadingComment: strings.TrimSpace(` +Depending on the client capability +` + "`workspace.workspaceEdit.resourceOperations`" + ` document changes are either +an array of ` + "`TextDocumentEdit`" + `s to express changes to n different text +documents where each text document edit addresses a specific version of +a text document. Or it can contain above ` + "`TextDocumentEdit`" + `s mixed with +create, rename and delete file / folder operations. + +Whether a client supports versioned document edits is expressed via +` + "`workspace.workspaceEdit.documentChanges`" + ` client capability. + +If a client neither supports ` + "`documentChanges`" + ` nor +` + "`workspace.workspaceEdit.resourceOperations`" + ` then only plain ` + "`TextEdit`" + `s +using the ` + "`changes`" + ` property are supported. +`), + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "changeAnnotations"}, + QuestionToken: true, + Type: &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.IndexSignature{ + Parameters: []*ast.Parameter{{ + Name: &ast.Identifier{Text: "id"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + }}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "ChangeAnnotation"}, + }, + }, + }, + }, + LeadingComment: strings.TrimSpace(` +A map of change annotations that can be referenced in +` + "`AnnotatedTextEdit`" + `s or create, rename and delete file / folder +operations. + +Whether clients honor this property depends on the client capability +` + "`workspace.changeAnnotationSupport`" + `. + +@since 3.16.0 +`), + }, + }, + }, + }, + }, + }, + { + testdata: "interface_with_union_array", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "TextDocumentEdit"}, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "edits"}, + Type: &ast.ArrayType{ + ElementType: &ast.ParenthesizedType{ + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "TextEdit"}, + }, + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "AnnotatedTextEdit"}, + }, + }, + }, + }, + }, + LeadingComment: strings.TrimSpace(` +The edits to be applied. + +@since 3.16.0 - support for AnnotatedTextEdit. This is guarded by the +client capability ` + "`workspace.workspaceEdit.changeAnnotationSupport`"), + }, + }, + }, + }, + }, + }, + { + testdata: "interface_with_union_struct_field", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "NotebookDocumentSyncOptions"}, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "notebookSelector"}, + Type: &ast.ArrayType{ + ElementType: &ast.ParenthesizedType{ + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "notebook"}, + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "NotebookDocumentFilter"}, + }, + }, + }, + LeadingComment: "The notebook to be synced. If a string\nvalue is provided it matches against the\nnotebook type. '*' matches every notebook.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "cells"}, + QuestionToken: true, + Type: &ast.ArrayType{ + ElementType: &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "language"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + }, + }, + }, + }, + LeadingComment: "The cells of the matching notebook to be synced.", + }, + }, + }, + &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "notebook"}, + QuestionToken: true, + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "NotebookDocumentFilter"}, + }, + }, + }, + LeadingComment: "The notebook to be synced. If a string\nvalue is provided it matches against the\nnotebook type. '*' matches every notebook.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "cells"}, + Type: &ast.ArrayType{ + ElementType: &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "language"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + }, + }, + }, + }, + LeadingComment: "The cells of the matching notebook to be synced.", + }, + }, + }, + }, + }, + }, + }, + LeadingComment: "The notebooks to be synced", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "save"}, + QuestionToken: true, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "boolean"}, + }, + LeadingComment: "Whether save notification should be forwarded to\nthe server. Will only be honored if mode === `notebook`.", + }, + }, + LeadingComment: strings.TrimSpace(` +Options specific to a notebook plus its cells +to be synced to the server. + +If a selector provides a notebook document +filter but no cell selector all cells of a +matching notebook document will be synced. + +If a selector provides no notebook document +filter but only a cell selector all notebook +documents that contain at least one matching +cell will be synced. + +@since 3.17.0`), + }, + }, + }, + }, + { + testdata: "multiple_extends", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "NotebookDocumentSyncRegistrationOptions"}, + HeritageClauses: []*ast.HeritageClause{ + { + Types: []*ast.ExpressionWithTypeArguments{ + {Expression: &ast.Identifier{Text: "NotebookDocumentSyncOptions"}}, + }, + }, + { + Types: []*ast.ExpressionWithTypeArguments{ + {Expression: &ast.Identifier{Text: "StaticRegistrationOptions"}}, + }, + }, + }, + LeadingComment: "Registration options specific to a notebook.\n\n@since 3.17.0", + }, + }, + }, + }, + { + testdata: "namespace", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.ModuleDeclaration{ + Name: &ast.Identifier{Text: "ErrorCodes"}, + Body: &ast.ModuleBlock{ + Statements: []ast.Stmt{ + &ast.VariableStatement{ + DeclarationList: &ast.VariableDeclarationList{ + Declarations: []*ast.VariableDeclaration{{ + Name: &ast.Identifier{Text: "ParseError"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "integer"}, + }, + Initializer: &ast.PrefixUnaryExpression{ + Operator: token.Minus, + Operand: &ast.NumericLiteral{Text: "32700"}, + }, + }}, + }, + LeadingComment: "Defined by JSON-RPC", + }, + &ast.VariableStatement{ + DeclarationList: &ast.VariableDeclarationList{ + Declarations: []*ast.VariableDeclaration{{ + Name: &ast.Identifier{Text: "InvalidRequest"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "integer"}, + }, + Initializer: &ast.PrefixUnaryExpression{ + Operator: token.Minus, + Operand: &ast.NumericLiteral{Text: "32600"}, + }, + }}, + }, + }, + &ast.VariableStatement{ + DeclarationList: &ast.VariableDeclarationList{ + Declarations: []*ast.VariableDeclaration{{ + Name: &ast.Identifier{Text: "jsonrpcReservedErrorRangeStart"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "integer"}, + }, + Initializer: &ast.PrefixUnaryExpression{ + Operator: token.Minus, + Operand: &ast.NumericLiteral{Text: "32099"}, + }, + }}, + }, + LeadingComment: strings.TrimSpace(` +This is the start range of JSON-RPC reserved error codes. +It doesn't denote a real error code. No LSP error codes should +be defined between the start and end range. For backwards +compatibility the ` + "`ServerNotInitialized`" + ` and the ` + "`UnknownErrorCode`" + ` +are left in the range. + +@since 3.16.0 +`), + }, + &ast.VariableStatement{ + DeclarationList: &ast.VariableDeclarationList{ + Declarations: []*ast.VariableDeclaration{{ + Name: &ast.Identifier{Text: "serverErrorStart"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "integer"}, + }, + Initializer: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "jsonrpcReservedErrorRangeStart"}, + }, + }}, + }, + LeadingComment: "@deprecated use jsonrpcReservedErrorRangeStart", + }, + }, + }, + }, + }, + }, + }, + { + testdata: "namespace_with_type", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.ModuleDeclaration{ + Name: &ast.Identifier{Text: "DiagnosticSeverity"}, + Body: &ast.ModuleBlock{ + Statements: []ast.Stmt{ + &ast.VariableStatement{ + DeclarationList: &ast.VariableDeclarationList{ + Declarations: []*ast.VariableDeclaration{{ + Name: &ast.Identifier{Text: "Error"}, + Type: &ast.LiteralType{ + Literal: &ast.NumericLiteral{Text: "1"}, + }, + Initializer: &ast.NumericLiteral{Text: "1"}, + }}, + }, + LeadingComment: "Reports an error.", + }, + &ast.VariableStatement{ + DeclarationList: &ast.VariableDeclarationList{ + Declarations: []*ast.VariableDeclaration{{ + Name: &ast.Identifier{Text: "Warning"}, + Type: &ast.LiteralType{ + Literal: &ast.NumericLiteral{Text: "2"}, + }, + Initializer: &ast.NumericLiteral{Text: "2"}, + }}, + }, + LeadingComment: "Reports a warning.", + }, + }, + }, + }, + &ast.TypeAliasDeclaration{ + Name: &ast.Identifier{Text: "DiagnosticSeverity"}, + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.LiteralType{ + Literal: &ast.NumericLiteral{Text: "1"}, + }, + &ast.LiteralType{ + Literal: &ast.NumericLiteral{Text: "2"}, + }, + }, + }, + }, + }, + }, + }, + { + testdata: "object", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.TypeAliasDeclaration{ + Name: &ast.Identifier{Text: "LSPObject"}, + Type: &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.IndexSignature{ + Parameters: []*ast.Parameter{{ + Name: &ast.Identifier{Text: "key"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + }}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "LSPAny"}, + }, + }, + }, + }, + LeadingComment: "LSP object definition.\n\n@since 3.17.0", + }, + }, + }, + }, + { + testdata: "qualified", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "FullDocumentDiagnosticReport"}, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "kind"}, + Type: &ast.TypeReference{ + TypeName: &ast.QualifiedName{ + Left: &ast.Identifier{Text: "DocumentDiagnosticReportKind"}, + Right: &ast.Identifier{Text: "Full"}, + }, + }, + }, + }, + }, + }, + }, + }, + { + testdata: "readonly", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "SemanticTokensDelta"}, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "resultId"}, + QuestionToken: true, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "edits"}, + Type: &ast.ArrayType{ + ElementType: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "SemanticTokensEdit"}, + }, + }, + LeadingComment: "The semantic token edits to transform a previous result into a new\nresult.", + }, + }, + }, + }, + }, + }, + { + testdata: "tuple", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "ParameterInformation"}, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "label"}, + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + &ast.TupleType{ + Elements: []ast.Type{ + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "uinteger"}, + }, + &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "uinteger"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + testdata: "union_struct", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.TypeAliasDeclaration{ + Name: &ast.Identifier{Text: "NotebookDocumentFilter"}, + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "notebookType"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + LeadingComment: "The type of the enclosing notebook.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "scheme"}, + QuestionToken: true, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + LeadingComment: "A Uri [scheme](#Uri.scheme), like `file` or `untitled`.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "pattern"}, + QuestionToken: true, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + LeadingComment: "A glob pattern.", + }, + }, + }, + &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "notebookType"}, + QuestionToken: true, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + LeadingComment: "The type of the enclosing notebook.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "scheme"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + LeadingComment: "A Uri [scheme](#Uri.scheme), like `file` or `untitled`.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "pattern"}, + QuestionToken: true, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + LeadingComment: "A glob pattern.", + }, + }, + }, + &ast.TypeLiteral{ + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "notebookType"}, + QuestionToken: true, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + LeadingComment: "The type of the enclosing notebook.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "scheme"}, + QuestionToken: true, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + LeadingComment: "A Uri [scheme](#Uri.scheme), like `file` or `untitled`.", + }, + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "pattern"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + LeadingComment: "A glob pattern.", + }, + }, + }, + }, + }, + LeadingComment: "A notebook document filter denotes a notebook document by\ndifferent properties.\n\n@since 3.17.0", + }, + }, + }, + }, + { + testdata: "union_type", + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.TypeAliasDeclaration{ + Name: &ast.Identifier{Text: "LSPAny"}, + Type: &ast.UnionType{ + Types: []ast.Type{ + &ast.TypeReference{TypeName: &ast.Identifier{Text: "LSPObject"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "LSPArray"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "string"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "integer"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "uinteger"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "decimal"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "boolean"}}, + &ast.TypeReference{TypeName: &ast.Identifier{Text: "null"}}, + }, + }, + LeadingComment: "The LSP any type\n\n@since 3.17.0", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.testdata, func(t *testing.T) { + source, err := os.ReadFile(fmt.Sprintf("../internal/testdata/%s.ts.txt", tt.testdata)) + if err != nil { + t.Fatal(err) + } + + if got := parser.Parse(source); !reflect.DeepEqual(got, tt.want) { + t.Errorf("\ngot:\n%s\n\nwant:\n%s", printTreeStructure(got), printTreeStructure(tt.want)) + } + }) + } +} + +func TestParser_Inline(t *testing.T) { + tests := []struct { + name string + src string + want *ast.SourceFile + }{ + { + name: "array", + src: `export const EOL: string[] = ['\n', '\r\n', '\r'];`, + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.VariableStatement{ + DeclarationList: &ast.VariableDeclarationList{ + Declarations: []*ast.VariableDeclaration{{ + Name: &ast.Identifier{Text: "EOL"}, + Type: &ast.ArrayType{ + ElementType: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + }, + Initializer: &ast.ArrayLiteralExpression{ + Elements: []ast.Expr{ + &ast.StringLiteral{Text: `\n`}, + &ast.StringLiteral{Text: `\r\n`}, + &ast.StringLiteral{Text: `\r`}, + }, + }, + }}, + }, + }, + }, + }, + }, + { + name: "identifier with numbers", + src: `export const UTF8: string = 'utf-8';`, + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.VariableStatement{ + DeclarationList: &ast.VariableDeclarationList{ + Declarations: []*ast.VariableDeclaration{{ + Name: &ast.Identifier{Text: "UTF8"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "string"}, + }, + Initializer: &ast.StringLiteral{Text: "utf-8"}, + }}, + }, + }, + }, + }, + }, + { + name: "keyword property", + src: ` +interface FileEvent { + type: FileChangeType; +}`, + want: &ast.SourceFile{ + Statements: []ast.Stmt{ + &ast.InterfaceDeclaration{ + Name: &ast.Identifier{Text: "FileEvent"}, + Members: []ast.Signature{ + &ast.PropertySignature{ + Name: &ast.Identifier{Text: "type"}, + Type: &ast.TypeReference{ + TypeName: &ast.Identifier{Text: "FileChangeType"}, + }, + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := parser.Parse([]byte(tt.src)); !reflect.DeepEqual(got, tt.want) { + t.Errorf("\ngot:\n%s\n\nwant:\n%s", printTreeStructure(got), printTreeStructure(tt.want)) + } + }) + } +} + +func printTreeStructure(node ast.Node) string { + if node == nil { + return "" + } + + sb := &strings.Builder{} + indent := 0 + + ast.Inspect(node, func(node ast.Node) bool { + if node == nil { + indent-- + return false + } + + for i := 0; i < indent; i++ { + sb.WriteString(" ") + } + + sb.WriteString(reflect.TypeOf(node).String()) + if stringer, ok := node.(fmt.Stringer); ok { + sb.WriteString(" ") + sb.WriteString(stringer.String()) + } + sb.WriteString("\n") + + indent++ + return true + }) + + return sb.String() +} diff --git a/token/token.go b/token/token.go new file mode 100644 index 0000000..ab23af5 --- /dev/null +++ b/token/token.go @@ -0,0 +1,102 @@ +// Package token contains the definition of the lexical tokens of the +// TypeScript programming language. +package token + +import "strconv" + +// Kind is the set of lexical tokens of the TypeScript programming +// language. +type Kind int + +const ( + // Special tokens. + Illegal Kind = iota + EOF + + // Comments. + Comment // // or /* */ at the beginning of a line + LineComment // // or /* */ at the end of a line + + // Identifiers and literals. + Ident // main, const, extends, etc. + Number // 12345 + String // "abc" + + // Operators. + Or // | + Assign // = + Minus // - + + // Delimiters and punctuation. + LParen // ( + RParen // ) + LBrack // [ + RBrack // ] + LBrace // { + RBrace // } + LAngle // < + RAngle // > + Comma // , + Dot // . + Colon // : + Semicolon // ; + Question // ? +) + +var tokens = [...]string{ + // Special tokens. + Illegal: "Illegal", + EOF: "EOF", + + // Comments. + Comment: "Comment", + LineComment: "LineComment", + + // Identifiers and literals. + Ident: "Ident", + Number: "Number", + String: "String", + + // Operators. + Or: "|", + Assign: "=", + Minus: "-", + + // Delimiters and punctuation. + LParen: "(", + RParen: ")", + LBrack: "[", + RBrack: "]", + LBrace: "{", + RBrace: "}", + LAngle: "<", + RAngle: ">", + Comma: ",", + Dot: ".", + Colon: ":", + Semicolon: ";", + Question: "?", +} + +func (k Kind) String() string { + if 0 <= k && k < Kind(len(tokens)) { + return tokens[k] + } + return "token(" + strconv.Itoa(int(k)) + ")" +} + +func (k Kind) IsLiteral() bool { + switch k { + case Ident, Number, String, Comment, LineComment: + return true + default: + return false + } +} + +// Token represents a lexical token of the TypeScript programming language. +// Identifiers and literals are accompanied by the corresponding text. +type Token struct { + Kind Kind + Text string +}