diff --git a/semantic-tokens-sample/.gitignore b/semantic-tokens-sample/.gitignore new file mode 100644 index 00000000..5fe00fea --- /dev/null +++ b/semantic-tokens-sample/.gitignore @@ -0,0 +1,4 @@ +out +node_modules +.vscode-test/ +*.vsix diff --git a/semantic-tokens-sample/.vscode/launch.json b/semantic-tokens-sample/.vscode/launch.json new file mode 100644 index 00000000..527cbf4b --- /dev/null +++ b/semantic-tokens-sample/.vscode/launch.json @@ -0,0 +1,35 @@ +// A launch configuration that compiles the extension and then opens it inside a new window +// Use IntelliSense to learn about possible attributes. +// Hover to view descriptions of existing attributes. +// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 +{ + "version": "0.2.0", + "configurations": [{ + "name": "Run Extension", + "type": "extensionHost", + "request": "launch", + "runtimeExecutable": "${execPath}", + "args": [ + "--extensionDevelopmentPath=${workspaceFolder}" + ], + "outFiles": [ + "${workspaceFolder}/out/**/*.js" + ], + "preLaunchTask": "npm: watch" + }, + { + "name": "Run Extension Tests", + "type": "extensionHost", + "request": "launch", + "runtimeExecutable": "${execPath}", + "args": [ + "--extensionDevelopmentPath=${workspaceFolder}", + "--extensionTestsPath=${workspaceFolder}/out/test" + ], + "outFiles": [ + "${workspaceFolder}/out/test/**/*.js" + ], + "preLaunchTask": "npm: watch" + } + ] +} diff --git a/semantic-tokens-sample/.vscode/settings.json b/semantic-tokens-sample/.vscode/settings.json new file mode 100644 index 00000000..e46111f1 --- /dev/null +++ b/semantic-tokens-sample/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "editor.insertSpaces": false +} \ No newline at end of file diff --git a/semantic-tokens-sample/.vscode/tasks.json b/semantic-tokens-sample/.vscode/tasks.json new file mode 100644 index 00000000..241aa6d9 --- /dev/null +++ b/semantic-tokens-sample/.vscode/tasks.json @@ -0,0 +1,20 @@ +// See https://go.microsoft.com/fwlink/?LinkId=733558 +// for the documentation about the tasks.json format +{ + "version": "2.0.0", + "tasks": [ + { + "type": "npm", + "script": "watch", + "problemMatcher": "$tsc-watch", + "isBackground": true, + "presentation": { + "reveal": "never" + }, + "group": { + "kind": "build", + "isDefault": true + } + } + ] +} \ No newline at end of file diff --git a/semantic-tokens-sample/.vscodeignore b/semantic-tokens-sample/.vscodeignore new file mode 100644 index 00000000..85571789 --- /dev/null +++ b/semantic-tokens-sample/.vscodeignore @@ -0,0 +1,9 @@ +.vscode/** +.vscode-test/** +out/test/** +out/**/*.map +src/** +.gitignore +tsconfig.json +vsc-extension-quickstart.md +tslint.json \ No newline at end of file diff --git a/semantic-tokens-sample/README.md b/semantic-tokens-sample/README.md new file mode 100644 index 00000000..193858e1 --- /dev/null +++ b/semantic-tokens-sample/README.md @@ -0,0 +1,21 @@ +# Semantic tokens sample + +This is an extension sample showing a very simple semantic tokens provider. This semantic tokens provider always returns all the tokens in a file. + +![Screenshot](demo.png) + +## How to run + +Launch the extension and open the file `sample/sample.semanticLanguage` and use the following settings: + +```json +"editor.tokenColorCustomizationsExperimental": { + "*.static": { + "foreground": "#ff0000", + "fontStyle": "bold" + }, + "type": { + "foreground": "#00aa00" + } +} +``` diff --git a/semantic-tokens-sample/demo.png b/semantic-tokens-sample/demo.png new file mode 100644 index 00000000..38289529 Binary files /dev/null and b/semantic-tokens-sample/demo.png differ diff --git a/semantic-tokens-sample/package-lock.json b/semantic-tokens-sample/package-lock.json new file mode 100644 index 00000000..5c2a593e --- /dev/null +++ b/semantic-tokens-sample/package-lock.json @@ -0,0 +1,323 @@ +{ + "name": "base-sample", + "version": "0.0.1", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@babel/code-frame": { + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.5.5.tgz", + "integrity": "sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==", + "dev": true, + "requires": { + "@babel/highlight": "^7.0.0" + } + }, + "@babel/highlight": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.5.0.tgz", + "integrity": "sha512-7dV4eu9gBxoM0dAnj/BCFDW9LFU0zvTrkq0ugM7pnHEgguOEeOz1so2ZghEdzviYzQEED0r4EAgpsBChKy1TRQ==", + "dev": true, + "requires": { + "chalk": "^2.0.0", + "esutils": "^2.0.2", + "js-tokens": "^4.0.0" + } + }, + "@types/node": { + "version": "10.14.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.17.tgz", + "integrity": "sha512-p/sGgiPaathCfOtqu2fx5Mu1bcjuP8ALFg4xpGgNkcin7LwRyzUKniEHBKdcE1RPsenq5JVPIpMTJSygLboygQ==", + "dev": true + }, + "@types/vscode": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.33.0.tgz", + "integrity": "sha512-JSmGiValbrcG5g20jjCfKakLiuWyrcjVezj+SEAEZ4klXQktE5EtowuGlkLVqbkiBK4iY5wy/4yW8OjecuHnjQ==", + "dev": true + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "builtin-modules": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", + "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", + "dev": true + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "commander": { + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", + "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "diff": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "glob": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", + "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "path-parse": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true + }, + "resolve": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.12.0.tgz", + "integrity": "sha512-B/dOmuoAik5bKcD6s6nXDCjzUKnaDvdkRyAk6rsmsKLipWj4797iothd7jmmUhWTfinVMU+wc56rYKsit2Qy4w==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "tslib": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz", + "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==", + "dev": true + }, + "tslint": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.19.0.tgz", + "integrity": "sha512-1LwwtBxfRJZnUvoS9c0uj8XQtAnyhWr9KlNvDIdB+oXyT+VpsOAaEhEgKi1HrZ8rq0ki/AAnbGSv4KM6/AfVZw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "builtin-modules": "^1.1.1", + "chalk": "^2.3.0", + "commander": "^2.12.1", + "diff": "^3.2.0", + "glob": "^7.1.1", + "js-yaml": "^3.13.1", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "resolve": "^1.3.2", + "semver": "^5.3.0", + "tslib": "^1.8.0", + "tsutils": "^2.29.0" + } + }, + "tsutils": { + "version": "2.29.0", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz", + "integrity": "sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==", + "dev": true, + "requires": { + "tslib": "^1.8.1" + } + }, + "typescript": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.6.2.tgz", + "integrity": "sha512-lmQ4L+J6mnu3xweP8+rOrUwzmN+MRAj7TgtJtDaXE5PMyX2kCrklhg3rvOsOIfNeAWMQWO2F1GPc1kMD2vLAfw==", + "dev": true + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + } + } +} diff --git a/semantic-tokens-sample/package.json b/semantic-tokens-sample/package.json new file mode 100644 index 00000000..191b7c30 --- /dev/null +++ b/semantic-tokens-sample/package.json @@ -0,0 +1,40 @@ +{ + "name": "semantic-tokens-sample", + "displayName": "semantic-tokens-sample", + "description": "Sample showing the Semantic Tokens Provider API", + "version": "0.0.1", + "publisher": "vscode-samples", + "engines": { + "vscode": "^1.41.0" + }, + "categories": [ + "Other" + ], + "activationEvents": [ + "onLanguage:semanticLanguage" + ], + "enableProposedApi": true, + "main": "./out/extension.js", + "contributes": { + "languages": [ + { + "id": "semanticLanguage", + "extensions": [ + ".semanticLanguage" + ] + } + ] + }, + "scripts": { + "vscode:prepublish": "npm run compile", + "compile": "tsc -p ./", + "lint": "tslint -p ./", + "watch": "tsc -watch -p ./" + }, + "devDependencies": { + "@types/node": "^10.14.17", + "@types/vscode": "^1.32.0", + "tslint": "^5.16.0", + "typescript": "^3.5.1" + } +} diff --git a/semantic-tokens-sample/sample/sample.semanticLanguage b/semantic-tokens-sample/sample/sample.semanticLanguage new file mode 100644 index 00000000..1c7b1265 --- /dev/null +++ b/semantic-tokens-sample/sample/sample.semanticLanguage @@ -0,0 +1,16 @@ +Available token types: + [comment] [string] [keyword] [number] [regexp] [operator] [namespace] + [type] [struct] [class] [interface] [enum] [parameterType] [function] + [macro] [variable] [constant] [parameter] [property] [label] + +Available token modifiers: + [type.declaration] [type.documentation] [type.member] [type.static] + [type.abstract] [type.deprecated] [type.modification] [type.async] + +Some examples: + [class.static.token] [type.static.abstract] + [class.static.token] [type.static] + + [struct] + + [function.private] diff --git a/semantic-tokens-sample/src/extension.ts b/semantic-tokens-sample/src/extension.ts new file mode 100644 index 00000000..eeb387bf --- /dev/null +++ b/semantic-tokens-sample/src/extension.ts @@ -0,0 +1,99 @@ +import * as vscode from 'vscode'; + +const tokenTypes = new Map(); +const tokenModifiers = new Map(); + +const legend = (function () { + const tokenTypesLegend = [ + 'comment', 'string', 'keyword', 'number', 'regexp', 'operator', 'namespace', + 'type', 'struct', 'class', 'interface', 'enum', 'parameterType', 'function', + 'macro', 'variable', 'constant', 'parameter', 'property', 'label' + ]; + tokenTypesLegend.forEach((tokenType, index) => tokenTypes.set(tokenType, index)); + + const tokenModifiersLegend = [ + 'declaration', 'documentation', 'member', 'static', 'abstract', 'deprecated', + 'modification', 'async' + ]; + tokenModifiersLegend.forEach((tokenModifier, index) => tokenModifiers.set(tokenModifier, index)); + + return new vscode.SemanticTokensLegend(tokenTypesLegend, tokenModifiersLegend); +})(); + +export function activate(context: vscode.ExtensionContext) { + context.subscriptions.push(vscode.languages.registerSemanticTokensProvider({ language: 'semanticLanguage'}, new SemanticTokensProvider(), legend)); +} + +interface IParsedToken { + line: number; + startCharacter: number; + length: number; + tokenType: string; + tokenModifiers: string[]; +} + +class SemanticTokensProvider implements vscode.SemanticTokensProvider { + async provideSemanticTokens(document: vscode.TextDocument, options: vscode.SemanticTokensRequestOptions, token: vscode.CancellationToken): Promise { + const allTokens = this._parseText(document.getText()); + const builder = new vscode.SemanticTokensBuilder(); + allTokens.forEach((token) => { + builder.push(token.line, token.startCharacter, token.length, this._encodeTokenType(token.tokenType), this._encodeTokenModifiers(token.tokenModifiers)); + }); + return new vscode.SemanticTokens(builder.build()); + } + + private _encodeTokenType(tokenType: string): number { + if (!tokenTypes.has(tokenType)) { + return 0; + } + return tokenTypes.get(tokenType)!; + } + + private _encodeTokenModifiers(strTokenModifiers: string[]): number { + let result = 0; + for (let i = 0; i < strTokenModifiers.length; i++) { + const tokenModifier = strTokenModifiers[i]; + if (tokenModifiers.has(tokenModifier)) { + result = result | (1 << tokenModifiers.get(tokenModifier)!); + } + } + return result; + } + + private _parseText(text: string): IParsedToken[] { + let r: IParsedToken[] = []; + let lines = text.split(/\r\n|\r|\n/); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + let currentOffset = 0; + do { + const openOffset = line.indexOf('[', currentOffset); + if (openOffset === -1) { + break; + } + const closeOffset = line.indexOf(']', openOffset); + if (closeOffset === -1) { + break; + } + let tokenData = this._parseTextToken(line.substring(openOffset + 1, closeOffset)); + r.push({ + line: i, + startCharacter: openOffset + 1, + length: closeOffset - openOffset - 1, + tokenType: tokenData.tokenType, + tokenModifiers: tokenData.tokenModifiers + }); + currentOffset = closeOffset; + } while (true); + } + return r; + } + + private _parseTextToken(text: string): { tokenType: string; tokenModifiers: string[]; } { + let parts = text.split('.'); + return { + tokenType: parts[0], + tokenModifiers: parts.slice(1) + }; + } +} diff --git a/semantic-tokens-sample/tsconfig.json b/semantic-tokens-sample/tsconfig.json new file mode 100644 index 00000000..aa034c3f --- /dev/null +++ b/semantic-tokens-sample/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es6", + "outDir": "out", + "sourceMap": true, + "rootDir": "src", + "strict": true + }, + "exclude": ["node_modules", ".vscode-test"] +} diff --git a/semantic-tokens-sample/tslint.json b/semantic-tokens-sample/tslint.json new file mode 100644 index 00000000..0ab0ca6e --- /dev/null +++ b/semantic-tokens-sample/tslint.json @@ -0,0 +1,6 @@ +{ + "rules": { + "indent": [true, "tabs"], + "semicolon": [true, "always"] + } +} \ No newline at end of file diff --git a/semantic-tokens-sample/vscode.proposed.d.ts b/semantic-tokens-sample/vscode.proposed.d.ts new file mode 100644 index 00000000..29bcbbda --- /dev/null +++ b/semantic-tokens-sample/vscode.proposed.d.ts @@ -0,0 +1,234 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * This is the place for API experiments and proposals. + * These API are NOT stable and subject to change. They are only available in the Insiders + * distribution and CANNOT be used in published extensions. + * + * To test these API in local environment: + * - Use Insiders release of VS Code. + * - Add `"enableProposedApi": true` to your package.json. + * - Copy this file to your project. + */ + +declare module 'vscode' { + + //#region Semantic tokens: https://github.com/microsoft/vscode/issues/86415 + + export class SemanticTokensLegend { + public readonly tokenTypes: string[]; + public readonly tokenModifiers: string[]; + + constructor(tokenTypes: string[], tokenModifiers: string[]); + } + + export class SemanticTokensBuilder { + constructor(); + push(line: number, char: number, length: number, tokenType: number, tokenModifiers: number): void; + build(): Uint32Array; + } + + export class SemanticTokens { + /** + * The result id of the tokens. + * + * On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result, + * the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`. + */ + readonly resultId?: string; + readonly data: Uint32Array; + + constructor(data: Uint32Array, resultId?: string); + } + + export class SemanticTokensEdits { + /** + * The result id of the tokens. + * + * On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result, + * the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`. + */ + readonly resultId?: string; + readonly edits: SemanticTokensEdit[]; + + constructor(edits: SemanticTokensEdit[], resultId?: string); + } + + export class SemanticTokensEdit { + readonly start: number; + readonly deleteCount: number; + readonly data?: Uint32Array; + + constructor(start: number, deleteCount: number, data?: Uint32Array); + } + + export interface SemanticTokensRequestOptions { + readonly ranges?: readonly Range[]; + /** + * The previous result id that the editor still holds in memory. + * + * Only when this is set it is safe for a `SemanticTokensProvider` to return `SemanticTokensEdits`. + */ + readonly previousResultId?: string; + } + + /** + * The semantic tokens provider interface defines the contract between extensions and + * semantic tokens. + */ + export interface SemanticTokensProvider { + /** + * A file can contain many tokens, perhaps even hundreds of thousands of tokens. Therefore, to improve + * the memory consumption around describing semantic tokens, we have decided to avoid allocating an object + * for each token and we represent tokens from a file as an array of integers. Furthermore, the position + * of each token is expressed relative to the token before it because most tokens remain stable relative to + * each other when edits are made in a file. + * + * + * --- + * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following fields: + * - at index `5*i` - `deltaLine`: token line number, relative to the previous token + * - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line) + * - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline. + * - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes` + * - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers` + * + * + * + * --- + * ### How to encode tokens + * + * Here is an example for encoding a file with 3 tokens: + * ``` + * { line: 2, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] }, + * { line: 2, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] }, + * { line: 5, startChar: 2, length: 7, tokenType: "classes", tokenModifiers: [] } + * ``` + * + * 1. First of all, a legend must be devised. This legend must be provided up-front and capture all possible token types. + * For this example, we will choose the following legend which must be passed in when registering the provider: + * ``` + * tokenTypes: ['properties', 'types', 'classes'], + * tokenModifiers: ['private', 'static'] + * ``` + * + * 2. The first transformation step is to encode `tokenType` and `tokenModifiers` as integers using the legend. Token types are looked + * up by index, so a `tokenType` value of `1` means `tokenTypes[1]`. Multiple token modifiers can be set by using bit flags, + * so a `tokenModifier` value of `3` is first viewed as binary `0b00000011`, which means `[tokenModifiers[0], tokenModifiers[1]]` because + * bits 0 and 1 are set. Using this legend, the tokens now are: + * ``` + * { line: 2, startChar: 5, length: 3, tokenType: 0, tokenModifiers: 3 }, + * { line: 2, startChar: 10, length: 4, tokenType: 1, tokenModifiers: 0 }, + * { line: 5, startChar: 2, length: 7, tokenType: 2, tokenModifiers: 0 } + * ``` + * + * 3. The next steps is to encode each token relative to the previous token in the file. In this case, the second token + * is on the same line as the first token, so the `startChar` of the second token is made relative to the `startChar` + * of the first token, so it will be `10 - 5`. The third token is on a different line than the second token, so the + * `startChar` of the third token will not be altered: + * ``` + * { deltaLine: 2, deltaStartChar: 5, length: 3, tokenType: 0, tokenModifiers: 3 }, + * { deltaLine: 0, deltaStartChar: 5, length: 4, tokenType: 1, tokenModifiers: 0 }, + * { deltaLine: 3, deltaStartChar: 2, length: 7, tokenType: 2, tokenModifiers: 0 } + * ``` + * + * 4. Finally, the last step is to inline each of the 5 fields for a token in a single array, which is a memory friendly representation: + * ``` + * // 1st token, 2nd token, 3rd token + * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] + * ``` + * + * + * + * --- + * ### How tokens change when the document changes + * + * Let's look at how tokens might change. + * + * Continuing with the above example, suppose a new line was inserted at the top of the file. + * That would make all the tokens move down by one line (notice how the line has changed for each one): + * ``` + * { line: 3, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] }, + * { line: 3, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] }, + * { line: 6, startChar: 2, length: 7, tokenType: "classes", tokenModifiers: [] } + * ``` + * The integer encoding of the tokens does not change substantially because of the delta-encoding of positions: + * ``` + * // 1st token, 2nd token, 3rd token + * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] + * ``` + * It is possible to express these new tokens in terms of an edit applied to the previous tokens: + * ``` + * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] + * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] + * + * edit: { start: 0, deleteCount: 1, data: [3] } // replace integer at offset 0 with 3 + * ``` + * + * Furthermore, let's assume that a new token has appeared on line 4: + * ``` + * { line: 3, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] }, + * { line: 3, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] }, + * { line: 4, startChar: 3, length: 5, tokenType: "properties", tokenModifiers: ["static"] }, + * { line: 6, startChar: 2, length: 7, tokenType: "classes", tokenModifiers: [] } + * ``` + * The integer encoding of the tokens is: + * ``` + * // 1st token, 2nd token, 3rd token, 4th token + * [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ] + * ``` + * Again, it is possible to express these new tokens in terms of an edit applied to the previous tokens: + * ``` + * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] + * [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ] + * + * edit: { start: 10, deleteCount: 1, data: [1,3,5,0,2,2] } // replace integer at offset 10 with [1,3,5,0,2,2] + * ``` + * + * + * + * --- + * ### When to return `SemanticTokensEdits` + * + * When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider. + * In principle, each call to `provideSemanticTokens` can return a full representations of the semantic tokens, and that would + * be a perfectly reasonable semantic tokens provider implementation. + * + * However, when having a language server running in a separate process, transferring all the tokens between processes + * might be slow, so VS Code allows to return the new tokens expressed in terms of multiple edits applied to the previous + * tokens. + * + * To clearly define what "previous tokens" means, it is possible to return a `resultId` with the semantic tokens. If the + * editor still has in memory the previous result, the editor will pass in options the previous `resultId` at + * `SemanticTokensRequestOptions.previousResultId`. Only when the editor passes in the previous `resultId`, it is allowed + * that a semantic tokens provider returns the new tokens expressed as edits to be applied to the previous result. Even in this + * case, the semantic tokens provider needs to return a new `resultId` that will identify these new tokens as a basis + * for the next request. + * + * *NOTE 1*: It is illegal to return `SemanticTokensEdits` if `options.previousResultId` is not set. + * *NOTE 2*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state. + */ + provideSemanticTokens(document: TextDocument, options: SemanticTokensRequestOptions, token: CancellationToken): ProviderResult; + } + + export namespace languages { + /** + * Register a semantic tokens provider. + * + * Multiple providers can be registered for a language. In that case providers are sorted + * by their [score](#languages.match) and the best-matching provider is used. Failure + * of the selected provider will cause a failure of the whole operation. + * + * @param selector A selector that defines the documents this provider is applicable to. + * @param provider A semantic tokens provider. + * @return A [disposable](#Disposable) that unregisters this provider when being disposed. + */ + export function registerSemanticTokensProvider(selector: DocumentSelector, provider: SemanticTokensProvider, legend: SemanticTokensLegend): Disposable; + } + + //#endregion + +}