Add sample

This commit is contained in:
Alexandru Dima
2019-12-09 15:39:21 +01:00
parent 9c2110873e
commit 0a4a2d44d0
14 changed files with 821 additions and 0 deletions

4
semantic-tokens-sample/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
out
node_modules
.vscode-test/
*.vsix

View File

@ -0,0 +1,35 @@
// A launch configuration that compiles the extension and then opens it inside a new window
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
{
"version": "0.2.0",
"configurations": [{
"name": "Run Extension",
"type": "extensionHost",
"request": "launch",
"runtimeExecutable": "${execPath}",
"args": [
"--extensionDevelopmentPath=${workspaceFolder}"
],
"outFiles": [
"${workspaceFolder}/out/**/*.js"
],
"preLaunchTask": "npm: watch"
},
{
"name": "Run Extension Tests",
"type": "extensionHost",
"request": "launch",
"runtimeExecutable": "${execPath}",
"args": [
"--extensionDevelopmentPath=${workspaceFolder}",
"--extensionTestsPath=${workspaceFolder}/out/test"
],
"outFiles": [
"${workspaceFolder}/out/test/**/*.js"
],
"preLaunchTask": "npm: watch"
}
]
}

View File

@ -0,0 +1,3 @@
{
"editor.insertSpaces": false
}

View File

@ -0,0 +1,20 @@
// See https://go.microsoft.com/fwlink/?LinkId=733558
// for the documentation about the tasks.json format
{
"version": "2.0.0",
"tasks": [
{
"type": "npm",
"script": "watch",
"problemMatcher": "$tsc-watch",
"isBackground": true,
"presentation": {
"reveal": "never"
},
"group": {
"kind": "build",
"isDefault": true
}
}
]
}

View File

@ -0,0 +1,9 @@
.vscode/**
.vscode-test/**
out/test/**
out/**/*.map
src/**
.gitignore
tsconfig.json
vsc-extension-quickstart.md
tslint.json

View File

@ -0,0 +1,21 @@
# Semantic tokens sample
This is an extension sample showing a very simple semantic tokens provider. This semantic tokens provider always returns all the tokens in a file.
![Screenshot](demo.png)
## How to run
Launch the extension and open the file `sample/sample.semanticLanguage` and use the following settings:
```json
"editor.tokenColorCustomizationsExperimental": {
"*.static": {
"foreground": "#ff0000",
"fontStyle": "bold"
},
"type": {
"foreground": "#00aa00"
}
}
```

Binary file not shown.

After

Width:  |  Height:  |  Size: 81 KiB

323
semantic-tokens-sample/package-lock.json generated Normal file
View File

@ -0,0 +1,323 @@
{
"name": "base-sample",
"version": "0.0.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@babel/code-frame": {
"version": "7.5.5",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.5.5.tgz",
"integrity": "sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==",
"dev": true,
"requires": {
"@babel/highlight": "^7.0.0"
}
},
"@babel/highlight": {
"version": "7.5.0",
"resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.5.0.tgz",
"integrity": "sha512-7dV4eu9gBxoM0dAnj/BCFDW9LFU0zvTrkq0ugM7pnHEgguOEeOz1so2ZghEdzviYzQEED0r4EAgpsBChKy1TRQ==",
"dev": true,
"requires": {
"chalk": "^2.0.0",
"esutils": "^2.0.2",
"js-tokens": "^4.0.0"
}
},
"@types/node": {
"version": "10.14.17",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.17.tgz",
"integrity": "sha512-p/sGgiPaathCfOtqu2fx5Mu1bcjuP8ALFg4xpGgNkcin7LwRyzUKniEHBKdcE1RPsenq5JVPIpMTJSygLboygQ==",
"dev": true
},
"@types/vscode": {
"version": "1.33.0",
"resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.33.0.tgz",
"integrity": "sha512-JSmGiValbrcG5g20jjCfKakLiuWyrcjVezj+SEAEZ4klXQktE5EtowuGlkLVqbkiBK4iY5wy/4yW8OjecuHnjQ==",
"dev": true
},
"ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"dev": true,
"requires": {
"color-convert": "^1.9.0"
}
},
"argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
"integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
"dev": true,
"requires": {
"sprintf-js": "~1.0.2"
}
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"builtin-modules": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
"integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=",
"dev": true
},
"chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
"dev": true,
"requires": {
"ansi-styles": "^3.2.1",
"escape-string-regexp": "^1.0.5",
"supports-color": "^5.3.0"
}
},
"color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"dev": true,
"requires": {
"color-name": "1.1.3"
}
},
"color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=",
"dev": true
},
"commander": {
"version": "2.20.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz",
"integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==",
"dev": true
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
},
"diff": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
"integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
"dev": true
},
"escape-string-regexp": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
"dev": true
},
"esprima": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
"integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
"dev": true
},
"esutils": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
"integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
"dev": true
},
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
"dev": true
},
"glob": {
"version": "7.1.4",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
"integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
"dev": true,
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.0.4",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
"dev": true
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"dev": true,
"requires": {
"once": "^1.3.0",
"wrappy": "1"
}
},
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"dev": true
},
"js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"dev": true
},
"js-yaml": {
"version": "3.13.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
"integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
"dev": true,
"requires": {
"argparse": "^1.0.7",
"esprima": "^4.0.0"
}
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}
},
"minimist": {
"version": "0.0.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
"dev": true
},
"mkdirp": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
"dev": true,
"requires": {
"minimist": "0.0.8"
}
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dev": true,
"requires": {
"wrappy": "1"
}
},
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
"dev": true
},
"path-parse": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
"integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
"dev": true
},
"resolve": {
"version": "1.12.0",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.12.0.tgz",
"integrity": "sha512-B/dOmuoAik5bKcD6s6nXDCjzUKnaDvdkRyAk6rsmsKLipWj4797iothd7jmmUhWTfinVMU+wc56rYKsit2Qy4w==",
"dev": true,
"requires": {
"path-parse": "^1.0.6"
}
},
"semver": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
"dev": true
},
"sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
"dev": true
},
"supports-color": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
"dev": true,
"requires": {
"has-flag": "^3.0.0"
}
},
"tslib": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz",
"integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==",
"dev": true
},
"tslint": {
"version": "5.19.0",
"resolved": "https://registry.npmjs.org/tslint/-/tslint-5.19.0.tgz",
"integrity": "sha512-1LwwtBxfRJZnUvoS9c0uj8XQtAnyhWr9KlNvDIdB+oXyT+VpsOAaEhEgKi1HrZ8rq0ki/AAnbGSv4KM6/AfVZw==",
"dev": true,
"requires": {
"@babel/code-frame": "^7.0.0",
"builtin-modules": "^1.1.1",
"chalk": "^2.3.0",
"commander": "^2.12.1",
"diff": "^3.2.0",
"glob": "^7.1.1",
"js-yaml": "^3.13.1",
"minimatch": "^3.0.4",
"mkdirp": "^0.5.1",
"resolve": "^1.3.2",
"semver": "^5.3.0",
"tslib": "^1.8.0",
"tsutils": "^2.29.0"
}
},
"tsutils": {
"version": "2.29.0",
"resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz",
"integrity": "sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==",
"dev": true,
"requires": {
"tslib": "^1.8.1"
}
},
"typescript": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.2.tgz",
"integrity": "sha512-lmQ4L+J6mnu3xweP8+rOrUwzmN+MRAj7TgtJtDaXE5PMyX2kCrklhg3rvOsOIfNeAWMQWO2F1GPc1kMD2vLAfw==",
"dev": true
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
"dev": true
}
}
}

View File

@ -0,0 +1,40 @@
{
"name": "semantic-tokens-sample",
"displayName": "semantic-tokens-sample",
"description": "Sample showing the Semantic Tokens Provider API",
"version": "0.0.1",
"publisher": "vscode-samples",
"engines": {
"vscode": "^1.41.0"
},
"categories": [
"Other"
],
"activationEvents": [
"onLanguage:semanticLanguage"
],
"enableProposedApi": true,
"main": "./out/extension.js",
"contributes": {
"languages": [
{
"id": "semanticLanguage",
"extensions": [
".semanticLanguage"
]
}
]
},
"scripts": {
"vscode:prepublish": "npm run compile",
"compile": "tsc -p ./",
"lint": "tslint -p ./",
"watch": "tsc -watch -p ./"
},
"devDependencies": {
"@types/node": "^10.14.17",
"@types/vscode": "^1.32.0",
"tslint": "^5.16.0",
"typescript": "^3.5.1"
}
}

View File

@ -0,0 +1,16 @@
Available token types:
[comment] [string] [keyword] [number] [regexp] [operator] [namespace]
[type] [struct] [class] [interface] [enum] [parameterType] [function]
[macro] [variable] [constant] [parameter] [property] [label]
Available token modifiers:
[type.declaration] [type.documentation] [type.member] [type.static]
[type.abstract] [type.deprecated] [type.modification] [type.async]
Some examples:
[class.static.token] [type.static.abstract]
[class.static.token] [type.static]
[struct]
[function.private]

View File

@ -0,0 +1,99 @@
import * as vscode from 'vscode';
const tokenTypes = new Map<string, number>();
const tokenModifiers = new Map<string, number>();
const legend = (function () {
const tokenTypesLegend = [
'comment', 'string', 'keyword', 'number', 'regexp', 'operator', 'namespace',
'type', 'struct', 'class', 'interface', 'enum', 'parameterType', 'function',
'macro', 'variable', 'constant', 'parameter', 'property', 'label'
];
tokenTypesLegend.forEach((tokenType, index) => tokenTypes.set(tokenType, index));
const tokenModifiersLegend = [
'declaration', 'documentation', 'member', 'static', 'abstract', 'deprecated',
'modification', 'async'
];
tokenModifiersLegend.forEach((tokenModifier, index) => tokenModifiers.set(tokenModifier, index));
return new vscode.SemanticTokensLegend(tokenTypesLegend, tokenModifiersLegend);
})();
export function activate(context: vscode.ExtensionContext) {
context.subscriptions.push(vscode.languages.registerSemanticTokensProvider({ language: 'semanticLanguage'}, new SemanticTokensProvider(), legend));
}
interface IParsedToken {
line: number;
startCharacter: number;
length: number;
tokenType: string;
tokenModifiers: string[];
}
class SemanticTokensProvider implements vscode.SemanticTokensProvider {
async provideSemanticTokens(document: vscode.TextDocument, options: vscode.SemanticTokensRequestOptions, token: vscode.CancellationToken): Promise<vscode.SemanticTokens> {
const allTokens = this._parseText(document.getText());
const builder = new vscode.SemanticTokensBuilder();
allTokens.forEach((token) => {
builder.push(token.line, token.startCharacter, token.length, this._encodeTokenType(token.tokenType), this._encodeTokenModifiers(token.tokenModifiers));
});
return new vscode.SemanticTokens(builder.build());
}
private _encodeTokenType(tokenType: string): number {
if (!tokenTypes.has(tokenType)) {
return 0;
}
return tokenTypes.get(tokenType)!;
}
private _encodeTokenModifiers(strTokenModifiers: string[]): number {
let result = 0;
for (let i = 0; i < strTokenModifiers.length; i++) {
const tokenModifier = strTokenModifiers[i];
if (tokenModifiers.has(tokenModifier)) {
result = result | (1 << tokenModifiers.get(tokenModifier)!);
}
}
return result;
}
private _parseText(text: string): IParsedToken[] {
let r: IParsedToken[] = [];
let lines = text.split(/\r\n|\r|\n/);
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
let currentOffset = 0;
do {
const openOffset = line.indexOf('[', currentOffset);
if (openOffset === -1) {
break;
}
const closeOffset = line.indexOf(']', openOffset);
if (closeOffset === -1) {
break;
}
let tokenData = this._parseTextToken(line.substring(openOffset + 1, closeOffset));
r.push({
line: i,
startCharacter: openOffset + 1,
length: closeOffset - openOffset - 1,
tokenType: tokenData.tokenType,
tokenModifiers: tokenData.tokenModifiers
});
currentOffset = closeOffset;
} while (true);
}
return r;
}
private _parseTextToken(text: string): { tokenType: string; tokenModifiers: string[]; } {
let parts = text.split('.');
return {
tokenType: parts[0],
tokenModifiers: parts.slice(1)
};
}
}

View File

@ -0,0 +1,11 @@
{
"compilerOptions": {
"module": "commonjs",
"target": "es6",
"outDir": "out",
"sourceMap": true,
"rootDir": "src",
"strict": true
},
"exclude": ["node_modules", ".vscode-test"]
}

View File

@ -0,0 +1,6 @@
{
"rules": {
"indent": [true, "tabs"],
"semicolon": [true, "always"]
}
}

View File

@ -0,0 +1,234 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
/**
* This is the place for API experiments and proposals.
* These API are NOT stable and subject to change. They are only available in the Insiders
* distribution and CANNOT be used in published extensions.
*
* To test these API in local environment:
* - Use Insiders release of VS Code.
* - Add `"enableProposedApi": true` to your package.json.
* - Copy this file to your project.
*/
declare module 'vscode' {
//#region Semantic tokens: https://github.com/microsoft/vscode/issues/86415
export class SemanticTokensLegend {
public readonly tokenTypes: string[];
public readonly tokenModifiers: string[];
constructor(tokenTypes: string[], tokenModifiers: string[]);
}
export class SemanticTokensBuilder {
constructor();
push(line: number, char: number, length: number, tokenType: number, tokenModifiers: number): void;
build(): Uint32Array;
}
export class SemanticTokens {
/**
* The result id of the tokens.
*
* On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result,
* the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`.
*/
readonly resultId?: string;
readonly data: Uint32Array;
constructor(data: Uint32Array, resultId?: string);
}
export class SemanticTokensEdits {
/**
* The result id of the tokens.
*
* On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result,
* the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`.
*/
readonly resultId?: string;
readonly edits: SemanticTokensEdit[];
constructor(edits: SemanticTokensEdit[], resultId?: string);
}
export class SemanticTokensEdit {
readonly start: number;
readonly deleteCount: number;
readonly data?: Uint32Array;
constructor(start: number, deleteCount: number, data?: Uint32Array);
}
export interface SemanticTokensRequestOptions {
readonly ranges?: readonly Range[];
/**
* The previous result id that the editor still holds in memory.
*
* Only when this is set it is safe for a `SemanticTokensProvider` to return `SemanticTokensEdits`.
*/
readonly previousResultId?: string;
}
/**
* The semantic tokens provider interface defines the contract between extensions and
* semantic tokens.
*/
export interface SemanticTokensProvider {
/**
* A file can contain many tokens, perhaps even hundreds of thousands of tokens. Therefore, to improve
* the memory consumption around describing semantic tokens, we have decided to avoid allocating an object
* for each token and we represent tokens from a file as an array of integers. Furthermore, the position
* of each token is expressed relative to the token before it because most tokens remain stable relative to
* each other when edits are made in a file.
*
*
* ---
* In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following fields:
* - at index `5*i` - `deltaLine`: token line number, relative to the previous token
* - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line)
* - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline.
* - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes`
* - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers`
*
*
*
* ---
* ### How to encode tokens
*
* Here is an example for encoding a file with 3 tokens:
* ```
* { line: 2, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] },
* { line: 2, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] },
* { line: 5, startChar: 2, length: 7, tokenType: "classes", tokenModifiers: [] }
* ```
*
* 1. First of all, a legend must be devised. This legend must be provided up-front and capture all possible token types.
* For this example, we will choose the following legend which must be passed in when registering the provider:
* ```
* tokenTypes: ['properties', 'types', 'classes'],
* tokenModifiers: ['private', 'static']
* ```
*
* 2. The first transformation step is to encode `tokenType` and `tokenModifiers` as integers using the legend. Token types are looked
* up by index, so a `tokenType` value of `1` means `tokenTypes[1]`. Multiple token modifiers can be set by using bit flags,
* so a `tokenModifier` value of `3` is first viewed as binary `0b00000011`, which means `[tokenModifiers[0], tokenModifiers[1]]` because
* bits 0 and 1 are set. Using this legend, the tokens now are:
* ```
* { line: 2, startChar: 5, length: 3, tokenType: 0, tokenModifiers: 3 },
* { line: 2, startChar: 10, length: 4, tokenType: 1, tokenModifiers: 0 },
* { line: 5, startChar: 2, length: 7, tokenType: 2, tokenModifiers: 0 }
* ```
*
* 3. The next steps is to encode each token relative to the previous token in the file. In this case, the second token
* is on the same line as the first token, so the `startChar` of the second token is made relative to the `startChar`
* of the first token, so it will be `10 - 5`. The third token is on a different line than the second token, so the
* `startChar` of the third token will not be altered:
* ```
* { deltaLine: 2, deltaStartChar: 5, length: 3, tokenType: 0, tokenModifiers: 3 },
* { deltaLine: 0, deltaStartChar: 5, length: 4, tokenType: 1, tokenModifiers: 0 },
* { deltaLine: 3, deltaStartChar: 2, length: 7, tokenType: 2, tokenModifiers: 0 }
* ```
*
* 4. Finally, the last step is to inline each of the 5 fields for a token in a single array, which is a memory friendly representation:
* ```
* // 1st token, 2nd token, 3rd token
* [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
* ```
*
*
*
* ---
* ### How tokens change when the document changes
*
* Let's look at how tokens might change.
*
* Continuing with the above example, suppose a new line was inserted at the top of the file.
* That would make all the tokens move down by one line (notice how the line has changed for each one):
* ```
* { line: 3, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] },
* { line: 3, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] },
* { line: 6, startChar: 2, length: 7, tokenType: "classes", tokenModifiers: [] }
* ```
* The integer encoding of the tokens does not change substantially because of the delta-encoding of positions:
* ```
* // 1st token, 2nd token, 3rd token
* [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
* ```
* It is possible to express these new tokens in terms of an edit applied to the previous tokens:
* ```
* [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
* [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
*
* edit: { start: 0, deleteCount: 1, data: [3] } // replace integer at offset 0 with 3
* ```
*
* Furthermore, let's assume that a new token has appeared on line 4:
* ```
* { line: 3, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] },
* { line: 3, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] },
* { line: 4, startChar: 3, length: 5, tokenType: "properties", tokenModifiers: ["static"] },
* { line: 6, startChar: 2, length: 7, tokenType: "classes", tokenModifiers: [] }
* ```
* The integer encoding of the tokens is:
* ```
* // 1st token, 2nd token, 3rd token, 4th token
* [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ]
* ```
* Again, it is possible to express these new tokens in terms of an edit applied to the previous tokens:
* ```
* [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
* [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ]
*
* edit: { start: 10, deleteCount: 1, data: [1,3,5,0,2,2] } // replace integer at offset 10 with [1,3,5,0,2,2]
* ```
*
*
*
* ---
* ### When to return `SemanticTokensEdits`
*
* When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider.
* In principle, each call to `provideSemanticTokens` can return a full representations of the semantic tokens, and that would
* be a perfectly reasonable semantic tokens provider implementation.
*
* However, when having a language server running in a separate process, transferring all the tokens between processes
* might be slow, so VS Code allows to return the new tokens expressed in terms of multiple edits applied to the previous
* tokens.
*
* To clearly define what "previous tokens" means, it is possible to return a `resultId` with the semantic tokens. If the
* editor still has in memory the previous result, the editor will pass in options the previous `resultId` at
* `SemanticTokensRequestOptions.previousResultId`. Only when the editor passes in the previous `resultId`, it is allowed
* that a semantic tokens provider returns the new tokens expressed as edits to be applied to the previous result. Even in this
* case, the semantic tokens provider needs to return a new `resultId` that will identify these new tokens as a basis
* for the next request.
*
* *NOTE 1*: It is illegal to return `SemanticTokensEdits` if `options.previousResultId` is not set.
* *NOTE 2*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state.
*/
provideSemanticTokens(document: TextDocument, options: SemanticTokensRequestOptions, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
}
export namespace languages {
/**
* Register a semantic tokens provider.
*
* Multiple providers can be registered for a language. In that case providers are sorted
* by their [score](#languages.match) and the best-matching provider is used. Failure
* of the selected provider will cause a failure of the whole operation.
*
* @param selector A selector that defines the documents this provider is applicable to.
* @param provider A semantic tokens provider.
* @return A [disposable](#Disposable) that unregisters this provider when being disposed.
*/
export function registerSemanticTokensProvider(selector: DocumentSelector, provider: SemanticTokensProvider, legend: SemanticTokensLegend): Disposable;
}
//#endregion
}