Merge pull request #1010 from savetheclocktower/tree-sitter-june

Tree-sitter rolling fixes, 1.118 edition

packages/language-javascript/grammars/tree-sitter/highlights.scm

@@ -1,3 +1,25 @@
+; MISC
+; ====
+
+; In the JSX construct `<FOO.Bar>`, `FOO` should not be marked as
+; `constant.other.js`. Block off identifiers within complex JSX tag names early
+; to prevent this.
+
+(jsx_opening_element
+  (member_expression
+    (identifier) @_IGNORE_
+      (#set! capture.final)))
+
+(jsx_closing_element
+  (member_expression
+    (identifier) @_IGNORE_
+      (#set! capture.final)))
+
+(jsx_self_closing_element
+  (member_expression
+    (identifier) @_IGNORE_
+      (#set! capture.final)))
+
 
 ; STRINGS
 ; =======
@@ -775,16 +797,16 @@
 
 ; The "Foo" in `<Foo />`.
 (jsx_self_closing_element
-  name: (identifier) @entity.name.tag.js
+  name: (_) @entity.name.tag.js
   ) @meta.tag.jsx.js
 
 ; The "Foo" in `<Foo>`.
 (jsx_opening_element
-  name: (identifier) @entity.name.tag.js)
+  name: (_) @entity.name.tag.js)
 
 ; The "Foo" in `</Foo>`.
 (jsx_closing_element
-  name: (identifier) @entity.name.tag.js)
+  name: (_) @entity.name.tag.js)
 
 ; The "bar" in `<Foo bar={true} />`.
 (jsx_attribute

packages/language-php/grammars/modern-tree-sitter-php-html.cson

@@ -5,8 +5,17 @@ parser: 'tree-sitter-php'
 
 injectionRegex: 'php|PHP'
 
+# The TextMate grammar has an elaborate `firstLineMatch` pattern defined with
+# Oniguruma; it catches hashbang syntax and complex vim/emacs modelines along
+# with the typical `<?php` prefix.
+#
+# We can't compete with all that, but we can catch obvious instances of
+# shebangs and `<?php`s. In fact, we must, or else the TextMate grammar will be
+# ranked higher for certain files.
+firstLineRegex: '^\\s*<\\?(?:php|PHP|=|\\s|$)|^\\#!.*(?:\\s|\\/)php\\d?'
+
 treeSitter:
-  parserSource: 'github:tree-sitter/tree-sitter-php#b569a5f2c0d592e67430520d1a0e1f765d83ceb0'
+  parserSource: 'github:tree-sitter/tree-sitter-php#27afeb02e49ff30acd17b67897b1c0114561a38c'
   grammar: 'tree-sitter/tree-sitter-php.wasm'
 
 fileTypes: [

packages/language-php/grammars/modern-tree-sitter-php.cson

@@ -7,7 +7,7 @@ parser: 'tree-sitter-php'
 injectionRegex: '^(internal-php)$'
 
 treeSitter:
-  parserSource: 'github:tree-sitter/tree-sitter-php#b569a5f2c0d592e67430520d1a0e1f765d83ceb0'
+  parserSource: 'github:tree-sitter/tree-sitter-php#27afeb02e49ff30acd17b67897b1c0114561a38c'
   grammar: 'tree-sitter/tree-sitter-php.wasm'
   highlightsQuery: 'tree-sitter/queries/highlights.scm'
   tagsQuery: 'tree-sitter/queries/tags.scm'

packages/language-php/grammars/tree-sitter/queries/highlights.scm

@@ -419,12 +419,12 @@
 
 (string
   "'" @punctuation.definition.string.begin.php
-  (string_value)?
+  (string_content)?
   "'" @punctuation.definition.string.end.php) @string.quoted.single.php
 
 (encapsed_string
   "\"" @punctuation.definition.string.begin.php
-  (string_value)?
+  (string_content)?
   "\"" @punctuation.definition.string.end.php) @string.quoted.double.php
 
 (encapsed_string
@@ -558,7 +558,7 @@
 (conditional_expression
   ["?" ":"] @keyword.operator.ternary.php)
 
-(unary_op_expression "@" @keyword.operator.error-control.php)
+(error_suppression_expression "@" @keyword.operator.error-control.php)
 
 [
   "=="

packages/language-ruby/grammars/tree-sitter-ruby/folds.scm

@@ -1,6 +1,5 @@
 [
   (method)
-  (comment)
   (singleton_method)
   (class)
   (module)
@@ -16,6 +15,10 @@
   (string_array)
 ] @fold
 
+; Multi-line comment syntax (=begin…=end) is obscure and out of favor, but we
+; might as well make it foldable.
+((comment) @fold
+  (#set! fold.endAt endPosition))
 
 ; Fold from `if` to the next `elsif` or `else` in the chain.
 ((if

packages/language-typescript/grammars/common/highlights.scm

@@ -71,6 +71,11 @@
     (shorthand_property_identifier_pattern) @variable.parameter.destructuring._LANG_)
     (#set! capture.final))
 
+(required_parameter
+  pattern: (object_pattern
+    (rest_pattern (identifier) @variable.parameter.destructuring.rest._LANG_))
+    (#set! capture.final))
+
 (required_parameter
   pattern: (object_pattern
     (object_assignment_pattern
@@ -860,7 +865,7 @@
 (binary_expression
   ["/" "+" "-" "*" "**" "%"] @keyword.operator.arithmetic._LANG_)
 
-(unary_expression ["+" "-"] @keyword.operator.unary._LANG_)
+(unary_expression ["+" "-" "void"] @keyword.operator.unary._LANG_)
 
 (binary_expression
   [

packages/language-typescript/grammars/common/tags.scm

@@ -7,11 +7,11 @@
   (#set! symbol.tag "function")) @definition.function
 
 (method_definition
-  name: (property_identifier) @name
+  name: [(property_identifier) (private_property_identifier)] @name
   (#set! symbol.tag "method")) @definition.method
 
 (abstract_method_signature
-  name: (property_identifier) @name
+  name: [(property_identifier) (private_property_identifier)] @name
   (#set! symbol.tag "method")) @definition.method
 
 (class_declaration

packages/language-typescript/grammars/modern-tree-sitter-tsx.cson

@@ -14,8 +14,8 @@ treeSitter:
   languageSegment: 'ts.tsx'
   grammar: 'tree-sitter-tsx/tree-sitter-tsx.wasm'
   highlightsQuery: [
-    'common/highlights.scm'
     'tree-sitter-tsx/highlights.scm'
+    'common/highlights.scm'
   ]
   indentsQuery: [
     'common/indents.scm'

packages/language-typescript/grammars/tree-sitter-tsx/highlights.scm

@@ -1,10 +1,31 @@
+; MISC
+; ====
+
+; In the JSX construct `<FOO.Bar>`, `FOO` should not be marked as
+; `constant.other.ts.tsx`. Block off identifiers within complex JSX tag names
+; early to prevent this.
+
+(jsx_opening_element
+  (nested_identifier
+    (identifier) @_IGNORE_
+      (#set! capture.final)))
+
+(jsx_closing_element
+  (nested_identifier
+    (identifier) @_IGNORE_
+      (#set! capture.final)))
+
+(jsx_self_closing_element
+  (nested_identifier
+    (identifier) @_IGNORE_
+      (#set! capture.final)))
 
 ; JSX
 ; ===
 
 ; The "Foo" in `<Foo />`.
 (jsx_self_closing_element
-  name: (identifier) @entity.name.tag.ts.tsx
+  name: (_) @entity.name.tag.ts.tsx
   ) @meta.tag.ts.tsx
 
 ; The "Foo" in `<Foo>`.

src/decoration.js

@@ -3,7 +3,7 @@ const { Emitter } = require('event-kit');
 let idCounter = 0;
 const nextId = () => idCounter++;
 
-const normalizeDecorationProperties = function(decoration, decorationParams) {
+const normalizeDecorationProperties = function (decoration, decorationParams) {
   decorationParams.id = decoration.id;
 
   if (
@@ -117,7 +117,7 @@ module.exports = class Decoration {
   Section: Event Subscription
   */
 
-  // Essential: When the {Decoration} is updated via {Decoration::update}.
+  // Essential: When the {Decoration} is updated via {Decoration::setProperties}.
   //
   // * `callback` {Function}
   //   * `event` {Object}

src/grammar-registry.js

@@ -259,8 +259,20 @@ module.exports = class GrammarRegistry {
     return useLegacyTreeSitter ? 'node-tree-sitter' : 'wasm-tree-sitter';
   }
 
-  // Extended: Returns a {Number} representing how well the grammar matches the
-  // `filePath` and `contents`.
+  // Extended: Evaluates a grammar's fitness for use for a certain file.
+  //
+  // By analyzing the file's extension and contents — plus other criteria, like
+  // the user's configuration — Pulsar will assign a score to this grammar that
+  // represents how suitable it is for the given file.
+  //
+  // Ultimately, whichever grammar scores highest for this file will be used
+  // to highlight it.
+  //
+  // * `grammar`: A given {Grammar}.
+  // * `filePath`: A {String} path to the file.
+  // * `contents`: The {String} contents of the file.
+  //
+  // Returns a {Number}.
   getGrammarScore(grammar, filePath, contents) {
     if (contents == null && fs.isFileSync(filePath)) {
       contents = fs.readFileSync(filePath, 'utf8');
@@ -305,8 +317,8 @@ module.exports = class GrammarRegistry {
         }
       }
 
-      // Prefer grammars with matching content regexes. Prefer a grammar with no content regex
-      // over one with a non-matching content regex.
+      // Prefer grammars with matching content regexes. Prefer a grammar with
+      // no content regex over one with a non-matching content regex.
       if (grammar.contentRegex) {
         const contentMatch = isTreeSitter
           ? grammar.contentRegex.test(contents)
@@ -318,7 +330,8 @@ module.exports = class GrammarRegistry {
         }
       }
 
-      // Prefer grammars that the user has manually installed over bundled grammars.
+      // Prefer grammars that the user has manually installed over bundled
+      // grammars.
       if (!grammar.bundledPackage) score += 0.01;
     }
 
@@ -544,7 +557,7 @@ module.exports = class GrammarRegistry {
     return disposable;
   }
 
-  // Experimental: Specify a type of syntax node that may embed other languages.
+  // Public: Specify a type of syntax node that may embed other languages.
   //
   // * `grammarId` The {String} id of the parent language
   // * `injectionPoint` An {Object} with the following keys:

src/pane.js

@@ -191,14 +191,14 @@ module.exports = class Pane {
   Section: Event Subscription
   */
 
-  // Public: Invoke the given callback when the pane resizes
+  // Public: Invoke the given callback when the pane resizes.
   //
-  // The callback will be invoked when pane's flexScale property changes.
+  // The callback will be invoked when pane's `flexScale` property changes.
   // Use {::getFlexScale} to get the current value.
   //
-  // * `callback` {Function} to be called when the pane is resized
-  //   * `flexScale` {Number} representing the panes `flex-grow`; ability for a
-  //     flex item to grow if necessary.
+  // * `callback` {Function} to be called when the pane is resized.
+  //   * `flexScale` {Number} representing the pane's `flex-grow`; ability for
+  //     a flex item to grow if necessary.
   //
   // Returns a {Disposable} on which '.dispose()' can be called to unsubscribe.
   onDidChangeFlexScale(callback) {
@@ -508,7 +508,7 @@ module.exports = class Pane {
     return this.items[index];
   }
 
-  // Makes the next item in the itemStack active.
+  // Public: Makes the next item in the itemStack active.
   activateNextRecentlyUsedItem() {
     if (this.items.length > 1) {
       if (this.itemStackIndex == null)
@@ -522,7 +522,7 @@ module.exports = class Pane {
     }
   }
 
-  // Makes the previous item in the itemStack active.
+  // Public: Makes the previous item in the itemStack active.
   activatePreviousRecentlyUsedItem() {
     if (this.items.length > 1) {
       if (
@@ -538,7 +538,8 @@ module.exports = class Pane {
     }
   }
 
-  // Moves the active item to the end of the itemStack once the ctrl key is lifted
+  // Public: Moves the active item to the end of the item stack once a modifier
+  // key (typically <kbd>Ctrl</kbd>) is lifted.
   moveActiveItemToTopOfStack() {
     delete this.itemStackIndex;
     this.addItemToStack(this.activeItem);

src/wasm-tree-sitter-grammar.js

@@ -121,15 +121,18 @@ module.exports = class WASMTreeSitterGrammar {
     return this.scopeNamesById.get(id);
   }
 
-  // Returns the Tree-sitter language instance associated with this grammar
-  // _if_ it has already loaded. Call this only when you can be certain that
-  // it's present.
+  // Extended: Retrieves the Tree-sitter `Language` instance associated with
+  // this grammar _if_ it has already been loaded.
+  //
+  // Language instances cannot be retrieved synchrously, so this will return
+  // `undefined` if the instance has not yet been loaded. In that case, going
+  // async will be unavoidable, and you’ll need to call {::getLanguage}.
   getLanguageSync() {
     return this._language;
   }
 
-  // Extended: Returns the Tree-sitter language instance associated with this
-  // grammar once it loads.
+  // Extended: Retrieves the Tree-sitter language instance associated with this
+  // grammar.
   //
   // Returns a {Promise} that will resolve with a Tree-sitter `Language`
   // instance. Once it resolves, the grammar is ready to perform parsing and to
@@ -355,6 +358,11 @@ module.exports = class WASMTreeSitterGrammar {
 
   // Extended: Calls `callback` when any of this grammar's query files change.
   //
+  // Since a grammar’s query files won’t change during ordinary operation, this
+  // method’s main purpose is to aid the development of grammars by applying
+  // changes to query files in real time. This happens automatically when
+  // Pulsar is running in dev mode.
+  //
   // The callback is invoked with an object argument with two keys:
   //
   // * `callback`: The callback to be invoked. Takes one argument:
@@ -389,6 +397,10 @@ module.exports = class WASMTreeSitterGrammar {
   // This differs from TextMate-style injections, which operate at the scope
   // level and are currently incompatible with Tree-sitter grammars.
   //
+  // You should typically not call this method directly; instead, call
+  // {GrammarRegistry::addInjectionPoint} and pass a given grammar’s root
+  // language scope as the first argument.
+  //
   // NOTE: Packages will call {::addInjectionPoint} with a given scope name,
   // and that call will be delegated to any Tree-sitter grammars that match
   // that scope name, whether they're legacy Tree-sitter or modern Tree-sitter.
@@ -439,7 +451,6 @@ module.exports = class WASMTreeSitterGrammar {
   //     applied varies based on the grammar; the function will be called with
   //     a grammar instance as its only argument.
   //
-  //
   addInjectionPoint(injectionPoint) {
     let { type } = injectionPoint;
     let injectionPoints = this.injectionPointsByType[type];
@@ -468,15 +479,15 @@ module.exports = class WASMTreeSitterGrammar {
   Section - Backward compatibility shims
   */
   /* eslint-disable no-unused-vars */
-  onDidUpdate(callback) {
+  onDidUpdate(_callback) {
     // do nothing
   }
 
-  tokenizeLines(text, compatibilityMode = true) {
+  tokenizeLines(text, _compatibilityMode = true) {
     return text.split('\n').map(line => this.tokenizeLine(line, null, false));
   }
 
-  tokenizeLine(line, ruleStack, firstLine) {
+  tokenizeLine(line, _ruleStack, _firstLine) {
     return {
       value: line,
       scopes: [this.scopeName]