Tree-sitter rolling fixes, 1.120 edition

packages/language-hyperlink/grammars/modern-tree-sitter-hyperlink.cson

@@ -5,5 +5,6 @@ parser: 'tree-sitter-hyperlink'
 
 injectionRegex: 'hyperlink'
 treeSitter:
+  parserSource: 'github:savetheclocktower/tree-sitter-hyperlink#04c3a667ba432236578ac99bbacd0412f88d6fac'
   grammar: 'ts/tree-sitter-hyperlink.wasm'
   highlightsQuery: 'ts/highlights.scm'

packages/language-javascript/grammars/tree-sitter/indents.scm

@@ -16,16 +16,32 @@
     (#is? test.last true))
     (#set! indent.matchIndentOf parent.startPosition))
 
-; By default, `case` and `default` need to be indented one level more than their containing
-; `switch`.
-([(switch_case "case" @match) (switch_default "default" @match)]
+; By default, `case` and `default` need to be indented one level more than
+; their containing `switch`.
+([
+  (switch_case "case" @match)
+  (switch_default "default" @match)
+  ; We include this one (and check for a switch_statment ancestor) to handle
+  ; a commonly encountered error state when the user is in the middle of typing
+  ; a switch statement.
+  (ERROR "case" @match)
+]
+  (#is? test.descendantOfType switch_statement)
   (#set! indent.matchIndentOf parent.parent.startPosition)
   (#set! indent.offsetIndent 1)
   (#is-not? test.config "language-javascript.indentation.alignCaseWithSwitch"))
 
 ; When this config setting is enabled, `case` and `default` need to be indented
 ; to match their containing `switch`.
-([(switch_case "case" @match) (switch_default "default" @match)]
+([
+  (switch_case "case" @match)
+  (switch_default "default" @match)
+  ; We include this one (and check for a switch_statment ancestor) to handle
+  ; a commonly encountered error state when the user is in the middle of typing
+  ; a switch statement.
+  (ERROR "case" @match)
+]
+  (#is? test.descendantOfType switch_statement)
   (#set! indent.matchIndentOf parent.parent.startPosition)
   (#set! indent.offsetIndent 0)
   (#is? test.config "language-javascript.indentation.alignCaseWithSwitch"))

packages/language-typescript/grammars/common/highlights.scm

@@ -841,6 +841,9 @@
 "new" @keyword.operator.new._LANG_
 
 "=" @keyword.operator.assignment._LANG_
+
+["&" "|" "<<" ">>" ">>>" "~" "^"] @keyword.operator.bitwise.js
+
 (non_null_expression "!" @keyword.operator.non-null._LANG_)
 (unary_expression "!" @keyword.operator.unary._LANG_)
 

packages/language-typescript/grammars/common/indents.scm

@@ -18,15 +18,31 @@
 
 ; By default, `case` and `default` need to be indented one level more than their containing
 ; `switch`.
-([(switch_case "case" @match) (switch_default "default" @match)]
+([
+  (switch_case "case" @match)
+  (switch_default "default" @match)
+  ; We include this one (and check for a switch_statment ancestor) to handle
+  ; a commonly encountered error state when the user is in the middle of typing
+  ; a switch statement.
+  (ERROR "case" @match)
+]
+  (#is? test.descendantOfType switch_statement)
   (#set! indent.matchIndentOf parent.parent.startPosition)
   (#set! indent.offsetIndent 1)
   (#is-not? test.config "language-typescript.indentation.alignCaseWithSwitch"))
 
 ; When this config setting is enabled, `case` and `default` need to be indented
 ; to match their containing `switch`.
 
-([(switch_case "case" @match) (switch_default "default" @match)]
+([
+  (switch_case "case" @match)
+  (switch_default "default" @match)
+  ; We include this one (and check for a switch_statment ancestor) to handle
+  ; a commonly encountered error state when the user is in the middle of typing
+  ; a switch statement.
+  (ERROR "case" @match)
+]
+  (#is? test.descendantOfType switch_statement)
   (#set! indent.matchIndentOf parent.parent.startPosition)
   (#set! indent.offsetIndent 0)
   (#is? test.config "language-typescript.indentation.alignCaseWithSwitch"))

spec/wasm-tree-sitter-language-mode-spec.js

@@ -561,6 +561,59 @@ describe('WASMTreeSitterLanguageMode', () => {
       ]);
     });
 
+    describe('when a highlighting query changes after load', () => {
+      it('updates the highlighting to reflect the new content', async () => {
+        jasmine.useRealClock();
+        const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
+
+        await grammar.setQueryForTest('highlightsQuery', scm`
+          (identifier) @variable
+        `);
+
+        buffer.setText('abc;');
+
+        const languageMode = new WASMTreeSitterLanguageMode({
+          buffer,
+          grammar
+        });
+        buffer.setLanguageMode(languageMode);
+        await languageMode.ready;
+        await wait(0);
+
+        expectTokensToEqual(editor, [
+          [
+            { text: 'abc', scopes: ['variable'] },
+            { text: ';', scopes: [] }
+          ]
+        ]);
+
+        // Set up a promise that resolves when highlighting updates after a
+        // query change.
+        let highlightingDidUpdate = new Promise((resolve) => {
+          let disposable = languageMode.onDidChangeHighlighting(
+            () => {
+              disposable.dispose();
+              resolve();
+            }
+          )
+        });
+
+        // Change the highlighting query.
+        await grammar.setQueryForTest('highlightsQuery', scm`
+          (identifier) @constant
+        `);
+        await highlightingDidUpdate;
+
+        // The language mode should automatically reload the query.
+        expectTokensToEqual(editor, [
+          [
+            { text: 'abc', scopes: ['constant'] },
+            { text: ';', scopes: [] }
+          ]
+        ]);
+      });
+    });
+
     // TODO: Ignoring these specs because web-tree-sitter doesn't seem to do
     // async. We can rehabilitate them if we ever figure it out.
     xdescribe('when the buffer changes during a parse', () => {
@@ -1750,6 +1803,53 @@ describe('WASMTreeSitterLanguageMode', () => {
 
       expect(Array.from(map.values())).toEqual([0, 1, 1, 2, 1, 0]);
     });
+
+    it('works correctly when straddling an injection boundary, even in the presence of whitespace', async () => {
+      const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
+
+      jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT);
+
+      const htmlGrammar = new WASMTreeSitterGrammar(
+        atom.grammars,
+        htmlGrammarPath,
+        htmlConfig
+      );
+
+      htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT);
+
+      atom.grammars.addGrammar(jsGrammar);
+      atom.grammars.addGrammar(htmlGrammar);
+
+      // This is just like the test above, except that we're indented a bit.
+      // Now the edge of the injection isn't at the beginning of the line; it's
+      // at the beginning of the first _text_ on the line.
+      buffer.setText(dedent`
+        <html>
+          <head>
+            <script>
+              let foo;
+              if (foo) {
+                debug(true);
+              }
+            </script>
+          </head>
+        </html>
+      `);
+
+      const languageMode = new WASMTreeSitterLanguageMode({
+        grammar: htmlGrammar,
+        buffer,
+        config: atom.config,
+        grammars: atom.grammars
+      });
+
+      buffer.setLanguageMode(languageMode);
+      await languageMode.ready;
+
+      let map = languageMode.suggestedIndentForBufferRows(0, 9, editor.getTabLength());
+
+      expect(Array.from(map.values())).toEqual([0, 1, 2, 3, 3, 4, 3, 2, 1, 0]);
+    })
   });
 
   describe('folding', () => {
@@ -2208,6 +2308,101 @@ describe('WASMTreeSitterLanguageMode', () => {
       `);
     });
 
+    it('does not enumerate redundant folds', async () => {
+      const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
+
+      await grammar.setQueryForTest('foldsQuery', `
+        (statement_block) @fold
+        (object) @fold
+      `);
+
+      // This odd way of formatting code produces a scenario where two folds
+      // would start on the same line. The second of the two folds would never
+      // be seen when toggling the fold on that line, so we shouldn't treat it
+      // as a valid fold for any other purpose.
+      buffer.setText(dedent`
+        if (foo) {results.push({
+          bar: 'baz'
+        })}
+      `);
+
+      const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
+      buffer.setLanguageMode(languageMode);
+      await languageMode.ready;
+
+      let ranges = languageMode.getFoldableRanges();
+      expect(ranges.length).toBe(1);
+    })
+
+    it('is not flummoxed by redundant folds when performing foldAllAtIndentLevel', async () => {
+      const grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
+
+      await grammar.setQueryForTest('foldsQuery', `
+        (statement_block) @fold
+        (object) @fold
+      `);
+
+      buffer.setText(dedent`
+        function foo() {
+          if (true) {
+            if (foo) {results.push({
+              bar: 'baz'
+            })}
+          }
+        }
+
+        function bar() {
+          if (false) {
+            // TODO
+          }
+        }
+      `);
+
+      const languageMode = new WASMTreeSitterLanguageMode({ grammar, buffer });
+      buffer.setLanguageMode(languageMode);
+      await languageMode.ready;
+
+      editor.foldAllAtIndentLevel(1);
+      expect(getDisplayText(editor)).toBe(dedent`
+        function foo() {
+          if (true) {…}
+        }
+
+        function bar() {
+          if (false) {…}
+        }
+      `);
+
+      buffer.setText(dedent`
+        function foo() {
+          if (true) {
+            if (foo) {
+            results.push({
+              bar: 'baz'
+            })}
+          }
+        }
+
+        function bar() {
+          if (false) {
+            // TODO
+          }
+        }
+      `);
+      await languageMode.atTransactionEnd();
+
+      editor.foldAllAtIndentLevel(1);
+      expect(getDisplayText(editor)).toBe(dedent`
+        function foo() {
+          if (true) {…}
+        }
+
+        function bar() {
+          if (false) {…}
+        }
+      `);
+    })
+
     it('can target named vs anonymous nodes as fold boundaries', async () => {
       const grammar = new WASMTreeSitterGrammar(atom.grammars, rubyGrammarPath, rubyConfig);
 

src/wasm-tree-sitter-grammar.js

@@ -182,6 +182,7 @@ module.exports = class WASMTreeSitterGrammar {
     }).then(() => {
       this._queryFilesLoaded = true;
       this._loadQueryFilesPromise = null;
+      this.emitter.emit('did-load-query-files', this);
     });
 
     return this._loadQueryFilesPromise;
@@ -324,7 +325,7 @@ module.exports = class WASMTreeSitterGrammar {
     this.queryCache.delete(queryType);
     this[queryType] = contents;
     let query = await this.getQuery(queryType);
-    this.emitter.emit('did-change-query-file', { filePath: '', queryType });
+    this.emitter.emit('did-change-query', { filePath: '', queryType });
     return query;
   }
 
@@ -350,29 +351,49 @@ module.exports = class WASMTreeSitterGrammar {
             this.queryCache.delete(queryType);
             return;
           }
-          this.emitter.emit('did-change-query-file', { filePath, queryType });
+          this.emitter.emit('did-change-query', { filePath, queryType });
         });
       }));
     }
   }
 
-  // Extended: Calls `callback` when any of this grammar's query files change.
+  // Extended: Calls `callback` when any of this grammar's queries change.
   //
-  // Since a grammar’s query files won’t change during ordinary operation, this
-  // method’s main purpose is to aid the development of grammars by applying
-  // changes to query files in real time. This happens automatically when
-  // Pulsar is running in dev mode.
+  // A grammar's queries typically will not change after initial load. When
+  // they do, it may mean:
   //
-  // The callback is invoked with an object argument with two keys:
+  // - The user is editing query files in dev mode; Pulsar will automatically
+  //   reload queries in dev mode after changes.
+  // - A community package is altering a query file via an API like
+  //   {::setQueryForTest}.
   //
-  // * `callback`: The callback to be invoked. Takes one argument:
-  //   * `data`: An object with keys:
-  //     * `filePath`: The path to the query file on disk.
-  //     * `queryType`: The type of query file, as denoted by its
+  // * `callback` {Function}
+  //   * `data` {Object}
+  //     * `filePath` {String} The path to the query file on disk.
+  //     * `queryType` {String} The type of query file, as denoted by its
   //         configuration key in the grammar file. Usually one of
   //         `highlightsQuery`, `indentsQuery`, `foldsQuery`, or `tagsQuery`.
+  onDidChangeQuery(callback) {
+    return this.emitter.on('did-change-query', callback);
+  }
+
+  // Extended: Calls `callback` when any of this grammar's queries change.
+  //
+  // Alias of {::onDidChangeQuery}.
   onDidChangeQueryFile(callback) {
-    return this.emitter.on('did-change-query-file', callback);
+    return this.onDidChangeQuery(callback);
+  }
+
+  // Extended: Calls `callback` when this grammar first loads its query files.
+  //
+  // Since a grammar may not load immedately on startup, this method makes it
+  // easier to hook into the query life cycle in order to modify or augment a
+  // grammar's default queries.
+  //
+  // * callback A function with the following argument:
+  //   * grammar The {WASMTreeSitterGrammar} whose queries have loaded.
+  onDidLoadQueryFiles(callback) {
+    return this.emitter.on('did-load-query-files', callback);
   }
 
   activate() {