[WIP] Feature/token color trie

browser/src/Editor/BufferHighlights.ts

@@ -16,8 +16,13 @@ export interface IBufferHighlightsUpdater {
     clearHighlightsForLine(line: number): void
 }
 
-// Helper class to efficiently update
-// buffer highlights in a batch.
+/**
+ * Helper class to efficiently update
+ * buffer highlights in a batch
+ *
+ * @name BufferHighlightsUpdater
+ * @class
+ */
 export class BufferHighlightsUpdater implements IBufferHighlightsUpdater {
     private _calls: any[] = []
 

browser/src/Editor/NeovimEditor/NeovimEditor.tsx

@@ -315,19 +315,15 @@ export class NeovimEditor extends Editor implements Oni.Editor {
             this._actions.setColors(updatedColors)
         }
 
-        this._colors.onColorsChanged.subscribe(() => onColorsChanged())
+        this._colors.onColorsChanged.subscribe(onColorsChanged)
         onColorsChanged()
 
-        const onTokenColorschanged = () => {
-            if (this._neovimInstance.isInitialized) {
-                this._neovimInstance.tokenColorSynchronizer.synchronizeTokenColors(
-                    this._tokenColors.tokenColors,
-                )
-            }
-        }
-
         this.trackDisposable(
-            this._tokenColors.onTokenColorsChanged.subscribe(() => onTokenColorschanged()),
+            this._tokenColors.onTokenColorsChanged.subscribe(() => {
+                if (this._neovimInstance.isInitialized) {
+                    this._syntaxHighlighter.notifyColorschemeRedraw(`${this.activeBuffer.id}`)
+                }
+            }),
         )
 
         // Overlays
@@ -1292,6 +1288,8 @@ export class NeovimEditor extends Editor implements Oni.Editor {
 
     private async _onColorsChanged(): Promise<void> {
         const newColorScheme = await this._neovimInstance.eval<string>("g:colors_name")
+        const { bufferNumber } = await this._neovimInstance.getContext()
+        this._syntaxHighlighter.notifyColorschemeRedraw(`${bufferNumber}`)
 
         // In error cases, the neovim API layer returns an array
         if (typeof newColorScheme !== "string") {

browser/src/Services/SyntaxHighlighting/ISyntaxHighlighter.ts

@@ -12,6 +12,7 @@ import { ISyntaxHighlightTokenInfo } from "./SyntaxHighlightingStore"
 export interface ISyntaxHighlighter extends IDisposable {
     notifyBufferUpdate(evt: Oni.EditorBufferChangedEventArgs): Promise<void>
     notifyViewportChanged(bufferId: string, topLineInView: number, bottomLineInView: number): void
+    notifyColorschemeRedraw(id: string): void
 
     getHighlightTokenAt(bufferId: string, position: types.Position): ISyntaxHighlightTokenInfo
 }

browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts

@@ -4,11 +4,11 @@
  * Handles enhanced syntax highlighting
  */
 
+import { Buffer, Editor } from "oni-api"
 import * as Log from "oni-core-logging"
 
 import { TokenColor, TokenColors } from "./../TokenColors"
-
-import { NeovimEditor } from "./../../Editor/NeovimEditor"
+import { prettyPrint } from "./../../Utility"
 
 import { HighlightInfo } from "./Definitions"
 import {
@@ -18,21 +18,37 @@ import {
 } from "./SyntaxHighlightingStore"
 import { TokenScorer } from "./TokenScorer"
 
+import { IBufferHighlightsUpdater } from "../../Editor/BufferHighlights"
 import * as Selectors from "./SyntaxHighlightSelectors"
 
-// SyntaxHighlightReconciler
-//
-// Essentially a renderer / reconciler, that will push
-// highlight calls to the active buffer based on the active
-// window and viewport
+interface IBufferWithSyntaxHighlighter extends Buffer {
+    updateHighlights?: (
+        tokenColors: TokenColor[],
+        highlightCallback: (args: IBufferHighlightsUpdater) => void,
+    ) => void
+}
+
+export interface IEditorWithSyntaxHighlighter extends Editor {
+    activeBuffer: IBufferWithSyntaxHighlighter
+}
+
+/**
+ * SyntaxHighlightReconciler
+ *
+ * Essentially a renderer / reconciler, that will push
+ * highlight calls to the active buffer based on the active
+ * window and viewport
+ * @name SyntaxHighlightReconciler
+ * @class
+ */
 export class SyntaxHighlightReconciler {
     private _previousState: { [line: number]: ISyntaxHighlightLineInfo } = {}
     private _tokenScorer = new TokenScorer()
 
-    constructor(private _editor: NeovimEditor, private _tokenColors: TokenColors) {}
+    constructor(private _editor: IEditorWithSyntaxHighlighter, private _tokenColors: TokenColors) {}
 
     public update(state: ISyntaxHighlightState) {
-        const activeBuffer: any = this._editor.activeBuffer
+        const { activeBuffer } = this._editor
 
         if (!activeBuffer) {
             return
@@ -77,6 +93,15 @@ export class SyntaxHighlightReconciler {
                 }
             })
 
+            // Get only the token colors that apply to the visible section of the buffer
+            const visibleTokens = tokens.reduce((accumulator, { highlights }) => {
+                if (highlights) {
+                    const tokenColors = highlights.map(({ tokenColor }) => tokenColor)
+                    accumulator.push(...tokenColors)
+                }
+                return accumulator
+            }, [])
+
             filteredLines.forEach(line => {
                 const lineNumber = parseInt(line, 10)
                 this._previousState[line] = Selectors.getLineFromBuffer(
@@ -87,26 +112,22 @@ export class SyntaxHighlightReconciler {
 
             if (tokens.length) {
                 Log.verbose(
-                    "[SyntaxHighlightReconciler] Applying changes to " + tokens.length + " lines.",
-                )
-                activeBuffer.updateHighlights(
-                    this._tokenColors.tokenColors,
-                    (highlightUpdater: any) => {
-                        tokens.forEach(token => {
-                            const { line, highlights } = token
-                            if (Log.isDebugLoggingEnabled()) {
-                                Log.debug(
-                                    "[SyntaxHighlightingReconciler] Updating tokens for line: " +
-                                        token.line +
-                                        " | " +
-                                        JSON.stringify(highlights),
-                                )
-                            }
-
-                            highlightUpdater.setHighlightsForLine(line, highlights)
-                        })
-                    },
+                    `[SyntaxHighlightReconciler] Applying changes to ${tokens.length} lines.`,
                 )
+                activeBuffer.updateHighlights(visibleTokens, highlightUpdater => {
+                    tokens.forEach(token => {
+                        const { line, highlights } = token
+                        if (Log.isDebugLoggingEnabled()) {
+                            Log.debug(
+                                `[SyntaxHighlightingReconciler] Updating tokens for line: ${line} | ${prettyPrint(
+                                    highlights,
+                                )}`,
+                            )
+                        }
+
+                        highlightUpdater.setHighlightsForLine(line, highlights)
+                    })
+                })
             }
         }
     }
@@ -121,8 +142,7 @@ export class SyntaxHighlightReconciler {
     }
 
     private _getHighlightGroupFromScope(scopes: string[]): TokenColor {
-        const configurationColors = this._tokenColors.tokenColors
-        const highestRanked = this._tokenScorer.rankTokenScopes(scopes, configurationColors)
+        const highestRanked = this._tokenScorer.rankTokenScopes(scopes, this._tokenColors.tokenTree)
         return highestRanked
     }
 }

browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts

@@ -28,7 +28,10 @@ import {
 } from "./SyntaxHighlightingStore"
 
 import { ISyntaxHighlighter } from "./ISyntaxHighlighter"
-import { SyntaxHighlightReconciler } from "./SyntaxHighlightReconciler"
+import {
+    IEditorWithSyntaxHighlighter,
+    SyntaxHighlightReconciler,
+} from "./SyntaxHighlightReconciler"
 import { getLineFromBuffer } from "./SyntaxHighlightSelectors"
 
 import * as Utility from "./../../Utility"
@@ -45,7 +48,10 @@ export class SyntaxHighlighter implements ISyntaxHighlighter {
     constructor(private _editor: NeovimEditor, private _tokenColors: TokenColors) {
         this._store = createSyntaxHighlightStore()
 
-        this._reconciler = new SyntaxHighlightReconciler(this._editor, this._tokenColors)
+        this._reconciler = new SyntaxHighlightReconciler(
+            this._editor as IEditorWithSyntaxHighlighter,
+            this._tokenColors,
+        )
         this._unsubscribe = this._store.subscribe(() => {
             const state = this._store.getState()
             this._reconciler.update(state)
@@ -62,12 +68,10 @@ export class SyntaxHighlighter implements ISyntaxHighlighter {
         bottomLineInView: number,
     ): void {
         Log.verbose(
-            "[SyntaxHighlighting.notifyViewportChanged] - bufferId: " +
-                bufferId +
-                " topLineInView: " +
-                topLineInView +
-                " bottomLineInView: " +
-                bottomLineInView,
+            `[SyntaxHighlighting.notifyViewportChanged] -
+             bufferId: ${bufferId} 
+             topLineInView: ${topLineInView}
+             bottomLineInView:  ${bottomLineInView}`,
         )
 
         const state = this._store.getState()
@@ -89,6 +93,10 @@ export class SyntaxHighlighter implements ISyntaxHighlighter {
         })
     }
 
+    public async notifyColorschemeRedraw(bufferId: string) {
+        this._store.dispatch({ type: "SYNTAX_RESET_BUFFER", bufferId })
+    }
+
     public async notifyBufferUpdate(evt: Oni.EditorBufferChangedEventArgs): Promise<void> {
         const firstChange = evt.contentChanges[0]
         if (!firstChange.range && !firstChange.rangeLength) {
@@ -157,6 +165,10 @@ export class NullSyntaxHighlighter implements ISyntaxHighlighter {
         return null
     }
 
+    public notifyColorschemeRedraw(id: string): void {
+        return null
+    }
+
     public notifyViewportChanged(
         bufferId: string,
         topLineInView: number,

browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts

@@ -51,6 +51,11 @@ export const bufferReducer: Reducer<IBufferSyntaxHighlightState> = (
     action: ISyntaxHighlightAction,
 ) => {
     switch (action.type) {
+        case "SYNTAX_RESET_BUFFER":
+            return {
+                ...state,
+                lines: linesReducer(state.lines, action),
+            }
         case "SYNTAX_UPDATE_BUFFER":
             return {
                 ...state,
@@ -125,6 +130,21 @@ export const linesReducer: Reducer<SyntaxHighlightLines> = (
             }
             return newState
         }
+        case "SYNTAX_RESET_BUFFER":
+            const resetState = Object.entries(state).reduce<SyntaxHighlightLines>(
+                (newResetState, [lineNumber, line]) => {
+                    newResetState[lineNumber] = {
+                        tokens: [],
+                        ruleStack: null,
+                        ...line,
+                        dirty: true,
+                    }
+                    return newResetState
+                },
+                {},
+            )
+            return resetState
+
         case "SYNTAX_UPDATE_BUFFER":
             const updatedBufferState: SyntaxHighlightLines = {
                 ...state,

browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts

@@ -73,6 +73,10 @@ export const DefaultSyntaxHighlightState: ISyntaxHighlightState = {
 }
 
 export type ISyntaxHighlightAction =
+    | {
+          type: "SYNTAX_RESET_BUFFER"
+          bufferId: string
+      }
     | {
           type: "SYNTAX_UPDATE_BUFFER"
           language: string
@@ -150,14 +154,14 @@ const updateBufferLineMiddleware = (store: any) => (next: any) => (action: any)
                 action.lineNumber === 0 ? null : buffer.lines[action.lineNumber - 1].ruleStack
             const tokenizeResult = grammar.tokenizeLine(action.line, previousRuleStack)
 
-            const tokens = tokenizeResult.tokens.map((t: any) => ({
+            const tokens = tokenizeResult.tokens.map(token => ({
                 range: types.Range.create(
                     action.lineNumber,
-                    t.startIndex,
+                    token.startIndex,
                     action.lineNumber,
-                    t.endIndex,
+                    token.endIndex,
                 ),
-                scopes: t.scopes,
+                scopes: token.scopes,
             }))
 
             const updateInsertLineAction: ISyntaxHighlightAction = {
@@ -180,7 +184,11 @@ const updateBufferLineMiddleware = (store: any) => (next: any) => (action: any)
 const updateTokenMiddleware = (store: any) => (next: any) => (action: any) => {
     const result: ISyntaxHighlightAction = next(action)
 
-    if (action.type === "SYNTAX_UPDATE_BUFFER" || action.type === "SYNTAX_UPDATE_BUFFER_VIEWPORT") {
+    if (
+        action.type === "SYNTAX_UPDATE_BUFFER" ||
+        action.type === "SYNTAX_UPDATE_BUFFER_VIEWPORT" ||
+        action.type === "SYNTAX_RESET_BUFFER"
+    ) {
         const state: ISyntaxHighlightState = store.getState()
         const bufferId = action.bufferId
 
@@ -212,7 +220,7 @@ const updateTokenMiddleware = (store: any) => (next: any) => (action: any) => {
 
             syntaxHighlightingJobs.startJob(
                 new SyntaxHighlightingPeriodicJob(
-                    store as any,
+                    store,
                     action.bufferId,
                     grammar,
                     relevantRange.top,