diff --git a/semantic-tokens-sample/package-lock.json b/semantic-tokens-sample/package-lock.json index 5c2a593e..7f109c70 100644 --- a/semantic-tokens-sample/package-lock.json +++ b/semantic-tokens-sample/package-lock.json @@ -1,5 +1,5 @@ { - "name": "base-sample", + "name": "semantic-tokens-sample", "version": "0.0.1", "lockfileVersion": 1, "requires": true, diff --git a/semantic-tokens-sample/src/extension.ts b/semantic-tokens-sample/src/extension.ts index eeb387bf..007f36ac 100644 --- a/semantic-tokens-sample/src/extension.ts +++ b/semantic-tokens-sample/src/extension.ts @@ -21,7 +21,7 @@ const legend = (function () { })(); export function activate(context: vscode.ExtensionContext) { - context.subscriptions.push(vscode.languages.registerSemanticTokensProvider({ language: 'semanticLanguage'}, new SemanticTokensProvider(), legend)); + context.subscriptions.push(vscode.languages.registerDocumentSemanticTokensProvider({ language: 'semanticLanguage'}, new DocumentSemanticTokensProvider(), legend)); } interface IParsedToken { @@ -32,8 +32,8 @@ interface IParsedToken { tokenModifiers: string[]; } -class SemanticTokensProvider implements vscode.SemanticTokensProvider { - async provideSemanticTokens(document: vscode.TextDocument, options: vscode.SemanticTokensRequestOptions, token: vscode.CancellationToken): Promise { +class DocumentSemanticTokensProvider implements vscode.DocumentSemanticTokensProvider { + async provideDocumentSemanticTokens(document: vscode.TextDocument, token: vscode.CancellationToken): Promise { const allTokens = this._parseText(document.getText()); const builder = new vscode.SemanticTokensBuilder(); allTokens.forEach((token) => { diff --git a/semantic-tokens-sample/vscode.proposed.d.ts b/semantic-tokens-sample/vscode.proposed.d.ts index 29bcbbda..7c358e7e 100644 --- a/semantic-tokens-sample/vscode.proposed.d.ts +++ b/semantic-tokens-sample/vscode.proposed.d.ts @@ -35,8 +35,7 @@ declare module 'vscode' { /** * The result id of the tokens. * - * On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result, - * the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`. + * This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented). */ readonly resultId?: string; readonly data: Uint32Array; @@ -48,8 +47,7 @@ declare module 'vscode' { /** * The result id of the tokens. * - * On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result, - * the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`. + * This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented). */ readonly resultId?: string; readonly edits: SemanticTokensEdit[]; @@ -65,21 +63,11 @@ declare module 'vscode' { constructor(start: number, deleteCount: number, data?: Uint32Array); } - export interface SemanticTokensRequestOptions { - readonly ranges?: readonly Range[]; - /** - * The previous result id that the editor still holds in memory. - * - * Only when this is set it is safe for a `SemanticTokensProvider` to return `SemanticTokensEdits`. - */ - readonly previousResultId?: string; - } - /** - * The semantic tokens provider interface defines the contract between extensions and + * The document semantic tokens provider interface defines the contract between extensions and * semantic tokens. */ - export interface SemanticTokensProvider { + export interface DocumentSemanticTokensProvider { /** * A file can contain many tokens, perhaps even hundreds of thousands of tokens. Therefore, to improve * the memory consumption around describing semantic tokens, we have decided to avoid allocating an object @@ -87,21 +75,18 @@ declare module 'vscode' { * of each token is expressed relative to the token before it because most tokens remain stable relative to * each other when edits are made in a file. * - * * --- - * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following fields: + * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following array indices: * - at index `5*i` - `deltaLine`: token line number, relative to the previous token * - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line) * - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline. * - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes` * - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers` * - * - * * --- * ### How to encode tokens * - * Here is an example for encoding a file with 3 tokens: + * Here is an example for encoding a file with 3 tokens in a uint32 array: * ``` * { line: 2, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] }, * { line: 2, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] }, @@ -140,8 +125,12 @@ declare module 'vscode' { * // 1st token, 2nd token, 3rd token * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] * ``` - * - * + */ + provideDocumentSemanticTokens(document: TextDocument, token: CancellationToken): ProviderResult; + + /** + * Instead of always returning all the tokens in a file, it is possible for a `DocumentSemanticTokensProvider` to implement + * this method (`updateSemanticTokens`) and then return incremental updates to the previously provided semantic tokens. * * --- * ### How tokens change when the document changes @@ -162,8 +151,8 @@ declare module 'vscode' { * ``` * It is possible to express these new tokens in terms of an edit applied to the previous tokens: * ``` - * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] - * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] + * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // old tokens + * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // new tokens * * edit: { start: 0, deleteCount: 1, data: [3] } // replace integer at offset 0 with 3 * ``` @@ -182,51 +171,56 @@ declare module 'vscode' { * ``` * Again, it is possible to express these new tokens in terms of an edit applied to the previous tokens: * ``` - * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] - * [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ] + * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // old tokens + * [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ] // new tokens * * edit: { start: 10, deleteCount: 1, data: [1,3,5,0,2,2] } // replace integer at offset 10 with [1,3,5,0,2,2] * ``` * - * - * - * --- - * ### When to return `SemanticTokensEdits` - * - * When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider. - * In principle, each call to `provideSemanticTokens` can return a full representations of the semantic tokens, and that would - * be a perfectly reasonable semantic tokens provider implementation. - * - * However, when having a language server running in a separate process, transferring all the tokens between processes - * might be slow, so VS Code allows to return the new tokens expressed in terms of multiple edits applied to the previous - * tokens. - * - * To clearly define what "previous tokens" means, it is possible to return a `resultId` with the semantic tokens. If the - * editor still has in memory the previous result, the editor will pass in options the previous `resultId` at - * `SemanticTokensRequestOptions.previousResultId`. Only when the editor passes in the previous `resultId`, it is allowed - * that a semantic tokens provider returns the new tokens expressed as edits to be applied to the previous result. Even in this - * case, the semantic tokens provider needs to return a new `resultId` that will identify these new tokens as a basis - * for the next request. - * - * *NOTE 1*: It is illegal to return `SemanticTokensEdits` if `options.previousResultId` is not set. - * *NOTE 2*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state. + * *NOTE*: When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider. + * *NOTE*: If the provider cannot compute `SemanticTokensEdits`, it can "give up" and return all the tokens in the document again. + * *NOTE*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state. */ - provideSemanticTokens(document: TextDocument, options: SemanticTokensRequestOptions, token: CancellationToken): ProviderResult; + provideDocumentSemanticTokensEdits?(document: TextDocument, previousResultId: string, token: CancellationToken): ProviderResult; + } + + /** + * The document range semantic tokens provider interface defines the contract between extensions and + * semantic tokens. + */ + export interface DocumentRangeSemanticTokensProvider { + /** + * See [provideDocumentSemanticTokens](#DocumentSemanticTokensProvider.provideDocumentSemanticTokens). + */ + provideDocumentRangeSemanticTokens(document: TextDocument, range: Range, token: CancellationToken): ProviderResult; } export namespace languages { /** - * Register a semantic tokens provider. + * Register a semantic tokens provider for a whole document. * * Multiple providers can be registered for a language. In that case providers are sorted * by their [score](#languages.match) and the best-matching provider is used. Failure * of the selected provider will cause a failure of the whole operation. * * @param selector A selector that defines the documents this provider is applicable to. - * @param provider A semantic tokens provider. + * @param provider A document semantic tokens provider. * @return A [disposable](#Disposable) that unregisters this provider when being disposed. */ - export function registerSemanticTokensProvider(selector: DocumentSelector, provider: SemanticTokensProvider, legend: SemanticTokensLegend): Disposable; + export function registerDocumentSemanticTokensProvider(selector: DocumentSelector, provider: DocumentSemanticTokensProvider, legend: SemanticTokensLegend): Disposable; + + /** + * Register a semantic tokens provider for a document range. + * + * Multiple providers can be registered for a language. In that case providers are sorted + * by their [score](#languages.match) and the best-matching provider is used. Failure + * of the selected provider will cause a failure of the whole operation. + * + * @param selector A selector that defines the documents this provider is applicable to. + * @param provider A document range semantic tokens provider. + * @return A [disposable](#Disposable) that unregisters this provider when being disposed. + */ + export function registerDocumentRangeSemanticTokensProvider(selector: DocumentSelector, provider: DocumentRangeSemanticTokensProvider, legend: SemanticTokensLegend): Disposable; } //#endregion