Skip to content

Commit 5fb6fc3

Browse files
DanTupDanTup
committed
Add support for running skipped tests in Flutter projects
Fixes #3151.
1 parent 50e93a5 commit 5fb6fc3

8 files changed

Lines changed: 156 additions & 31 deletions

File tree

‎package.json‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,7 @@
181181
},
182182
{
183183
"command": "dart.runAllSkippedTestsWithoutDebugging",
184-
"title": "Run All Skipped Tests (Standard Dart projects only)",
185-
"enablement": "dart-code:anyStandardDartProjectLoaded",
184+
"title": "Run All Skipped Tests",
186185
"category": "Dart",
187186
"icon": {
188187
"dark": "./media/commands/test-run-skipped-dark.svg",
@@ -1272,12 +1271,12 @@
12721271
"group": "4.5_exec@4"
12731272
},
12741273
{
1275-
"when": "dart-code:anyProjectLoaded && viewItem =~ /dart-code:testContainerNodeWithSkips/ && viewItem =~ /dart-code:isInDartTestSuite/",
1274+
"when": "dart-code:anyProjectLoaded && viewItem =~ /dart-code:testContainerNodeWithSkips/ && viewItem =~ /dart-code:canRunSkipped/",
12761275
"command": "dart.startDebuggingSkippedTests",
12771276
"group": "4.5_exec@5"
12781277
},
12791278
{
1280-
"when": "dart-code:anyProjectLoaded && viewItem =~ /dart-code:testContainerNodeWithSkips/ && viewItem =~ /dart-code:isInDartTestSuite/",
1279+
"when": "dart-code:anyProjectLoaded && viewItem =~ /dart-code:testContainerNodeWithSkips/ && viewItem =~ /dart-code:canRunSkipped/",
12811280
"command": "dart.startWithoutDebuggingSkippedTests",
12821281
"group": "4.5_exec@6"
12831282
},

‎src/extension/commands/test.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as fs from "fs";
22
import * as path from "path";
33
import * as vs from "vscode";
4+
import { FlutterCapabilities } from "../../shared/capabilities/flutter";
45
import { noAction } from "../../shared/constants";
56
import { Logger } from "../../shared/interfaces";
67
import { escapeDartString, generateTestNameFromFileName } from "../../shared/utils";
@@ -24,7 +25,7 @@ export let isInImplementationFileThatCanHaveTest = false;
2425
abstract class TestCommands implements vs.Disposable {
2526
private disposables: vs.Disposable[] = [];
2627

27-
constructor(protected readonly logger: Logger, protected readonly wsContext: WorkspaceContext) {
28+
constructor(protected readonly logger: Logger, protected readonly wsContext: WorkspaceContext, protected readonly flutterCapabilities: FlutterCapabilities) {
2829
this.disposables.push(
2930
vs.commands.registerCommand("dart.runTestAtCursor", () => this.runTestAtCursor(false), this),
3031
vs.commands.registerCommand("dart.goToTests", (resource: vs.Uri | undefined) => this.goToTestOrImplementationFile(resource), this),
@@ -43,7 +44,7 @@ abstract class TestCommands implements vs.Disposable {
4344
}
4445

4546
private startTestFromOutline(noDebug: boolean, test: TestOutlineInfo, launchTemplate: any | undefined) {
46-
const canRunSkippedTest = !test.isGroup && !isInsideFlutterProject(vs.Uri.file(test.file));
47+
const canRunSkippedTest = !test.isGroup && (this.flutterCapabilities.supportsRunSkippedTests || !isInsideFlutterProject(vs.Uri.file(test.file)));
4748

4849
return vs.debug.startDebugging(
4950
vs.workspace.getWorkspaceFolder(vs.Uri.file(test.file)),
@@ -177,8 +178,8 @@ abstract class TestCommands implements vs.Disposable {
177178
}
178179

179180
export class DasTestCommands extends TestCommands {
180-
constructor(logger: Logger, wsContext: WorkspaceContext, private readonly fileTracker: DasFileTracker) {
181-
super(logger, wsContext);
181+
constructor(logger: Logger, wsContext: WorkspaceContext, private readonly fileTracker: DasFileTracker, flutterCapabilities: FlutterCapabilities) {
182+
super(logger, wsContext, flutterCapabilities);
182183
}
183184

184185
protected testForCursor(editor: vs.TextEditor): TestOutlineInfo | undefined {
@@ -208,8 +209,8 @@ export class DasTestCommands extends TestCommands {
208209
}
209210

210211
export class LspTestCommands extends TestCommands {
211-
constructor(logger: Logger, wsContext: WorkspaceContext, private readonly fileTracker: LspFileTracker) {
212-
super(logger, wsContext);
212+
constructor(logger: Logger, wsContext: WorkspaceContext, private readonly fileTracker: LspFileTracker, flutterCapabilities: FlutterCapabilities) {
213+
super(logger, wsContext, flutterCapabilities);
213214
}
214215

215216
protected testForCursor(editor: vs.TextEditor): LspTestOutlineInfo | undefined {

‎src/extension/extension.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -510,9 +510,9 @@ export async function activate(context: vs.ExtensionContext, isRestart: boolean
510510
context.subscriptions.push(new LoggingCommands(logger, context.logPath));
511511
context.subscriptions.push(new OpenInOtherEditorCommands(logger, sdks));
512512
if (dasAnalyzer)
513-
context.subscriptions.push(new DasTestCommands(logger, workspaceContext, dasAnalyzer.fileTracker));
513+
context.subscriptions.push(new DasTestCommands(logger, workspaceContext, dasAnalyzer.fileTracker, flutterCapabilities));
514514
if (lspAnalyzer)
515-
context.subscriptions.push(new LspTestCommands(logger, workspaceContext, lspAnalyzer.fileTracker));
515+
context.subscriptions.push(new LspTestCommands(logger, workspaceContext, lspAnalyzer.fileTracker, flutterCapabilities));
516516

517517
if (lspClient && lspAnalyzer) {
518518
// TODO: LSP equivs of the others...
@@ -542,7 +542,7 @@ export async function activate(context: vs.ExtensionContext, isRestart: boolean
542542
);
543543
if (lspAnalyzer)
544544
context.subscriptions.push(new TestDiscoverer(logger, lspAnalyzer.fileTracker, testTreeModel));
545-
const testTreeProvider = new TestResultsProvider(testTreeModel, testCoordinator);
545+
const testTreeProvider = new TestResultsProvider(testTreeModel, testCoordinator, flutterCapabilities);
546546
const testTreeView = vs.window.createTreeView("dartTestTree", { treeDataProvider: testTreeProvider });
547547
const tryReveal = async (node: TreeNode) => {
548548
try {

‎src/extension/views/test_view.ts‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as path from "path";
22
import * as vs from "vscode";
3-
import { DART_TEST_CONTAINER_NODE_WITH_FAILURES_CONTEXT, DART_TEST_CONTAINER_NODE_WITH_SKIPS_CONTEXT, DART_TEST_GROUP_NODE_CONTEXT, DART_TEST_SUITE_NODE_CONTEXT, DART_TEST_TEST_NODE_CONTEXT, IS_IN_DART_TEST_SUITE_CONTEXT, IS_IN_FLUTTER_TEST_SUITE_CONTEXT } from "../../shared/constants";
3+
import { FlutterCapabilities } from "../../shared/capabilities/flutter";
4+
import { DART_TEST_CAN_RUN_SKIPPED_CONTEXT, DART_TEST_CONTAINER_NODE_WITH_FAILURES_CONTEXT, DART_TEST_CONTAINER_NODE_WITH_SKIPS_CONTEXT, DART_TEST_GROUP_NODE_CONTEXT, DART_TEST_SUITE_NODE_CONTEXT, DART_TEST_TEST_NODE_CONTEXT } from "../../shared/constants";
45
import { TestStatus } from "../../shared/enums";
56
import { TestSessionCoordinator } from "../../shared/test/coordinator";
67
import { GroupNode, SuiteNode, TestContainerNode, TestNode, TestTreeModel, TreeNode } from "../../shared/test/test_model";
@@ -21,8 +22,10 @@ export class TestResultsProvider implements vs.Disposable, vs.TreeDataProvider<T
2122
private onDidChangeTreeDataEmitter: vs.EventEmitter<TreeNode | undefined> = new vs.EventEmitter<TreeNode | undefined>();
2223
public readonly onDidChangeTreeData: vs.Event<TreeNode | undefined> = this.onDidChangeTreeDataEmitter.event;
2324
private currentTestTerminal: [vs.Terminal, vs.EventEmitter<string>] | undefined;
25+
private readonly treeItemBuilder: TreeItemBuilder;
2426

25-
constructor(private readonly data: TestTreeModel, private readonly coordinator: TestSessionCoordinator) {
27+
constructor(private readonly data: TestTreeModel, private readonly coordinator: TestSessionCoordinator, private readonly flutterCapabilities: FlutterCapabilities) {
28+
this.treeItemBuilder = new TreeItemBuilder(flutterCapabilities);
2629
this.disposables.push(data.onDidChangeTreeData.listen((node) => this.onDidChangeTreeDataEmitter.fire(node)));
2730
this.disposables.push(vs.workspace.onDidChangeConfiguration((e) => this.handleConfigChange(e)));
2831

@@ -34,8 +37,8 @@ export class TestResultsProvider implements vs.Disposable, vs.TreeDataProvider<T
3437
this.disposables.push(vs.commands.registerCommand("dart.startWithoutDebuggingTest", (treeNode: SuiteNode | GroupNode | TestNode) => this.runTests(treeNode, this.getTestNames(treeNode), false, false, treeNode instanceof TestNode)));
3538
this.disposables.push(vs.commands.registerCommand("dart.startDebuggingSkippedTests", (treeNode: SuiteNode | GroupNode | TestNode) => this.runTests(treeNode, this.getTestNames(treeNode, TestStatus.Skipped), true, false, true)));
3639
this.disposables.push(vs.commands.registerCommand("dart.startWithoutDebuggingSkippedTests", (treeNode: SuiteNode | GroupNode | TestNode) => this.runTests(treeNode, this.getTestNames(treeNode, TestStatus.Skipped), false, false, true)));
37-
this.disposables.push(vs.commands.registerCommand("dart.startDebuggingFailedTests", (treeNode: SuiteNode | GroupNode | TestNode) => this.runTests(treeNode, this.getTestNames(treeNode, TestStatus.Failed), true, false)));
38-
this.disposables.push(vs.commands.registerCommand("dart.startWithoutDebuggingFailedTests", (treeNode: SuiteNode | GroupNode | TestNode) => this.runTests(treeNode, this.getTestNames(treeNode, TestStatus.Failed), false, false)));
40+
this.disposables.push(vs.commands.registerCommand("dart.startDebuggingFailedTests", (treeNode: SuiteNode | GroupNode | TestNode) => this.runTests(treeNode, this.getTestNames(treeNode, TestStatus.Failed), true, false, false)));
41+
this.disposables.push(vs.commands.registerCommand("dart.startWithoutDebuggingFailedTests", (treeNode: SuiteNode | GroupNode | TestNode) => this.runTests(treeNode, this.getTestNames(treeNode, TestStatus.Failed), false, false, false)));
3942
this.disposables.push(vs.commands.registerCommand("dart.runAllSkippedTestsWithoutDebugging", () => this.runAllSkippedTests()));
4043
this.disposables.push(vs.commands.registerCommand("dart.runAllFailedTestsWithoutDebugging", () => this.runAllFailedTests()));
4144

@@ -123,7 +126,7 @@ export class TestResultsProvider implements vs.Disposable, vs.TreeDataProvider<T
123126
);
124127
}
125128

126-
private async runTests(treeNode: GroupNode | SuiteNode | TestNode, testNames: string[] | undefined, debug: boolean, suppressPromptOnErrors: boolean, runSkippedTests?: boolean, token?: vs.CancellationToken) {
129+
private async runTests(treeNode: GroupNode | SuiteNode | TestNode, testNames: string[] | undefined, debug: boolean, suppressPromptOnErrors: boolean, runSkippedTests: boolean, token?: vs.CancellationToken) {
127130
const subs: vs.Disposable[] = [];
128131
return new Promise<void>(async (resolve, reject) => {
129132
// Construct a unique ID for this session so we can track when it completes.
@@ -139,7 +142,8 @@ export class TestResultsProvider implements vs.Disposable, vs.TreeDataProvider<T
139142
resolve();
140143
}));
141144
const programPath = fsPath(treeNode.suiteData.path);
142-
const shouldRunSkippedTests = runSkippedTests && !isInsideFlutterProject(vs.Uri.file(treeNode.suiteData.path));
145+
const canRunSkippedTest = this.flutterCapabilities.supportsRunSkippedTests || !isInsideFlutterProject(vs.Uri.file(treeNode.suiteData.path));
146+
const shouldRunSkippedTests = runSkippedTests && canRunSkippedTest;
143147
const didStart = await vs.debug.startDebugging(
144148
vs.workspace.getWorkspaceFolder(vs.Uri.file(treeNode.suiteData.path)),
145149
{
@@ -221,11 +225,11 @@ export class TestResultsProvider implements vs.Disposable, vs.TreeDataProvider<T
221225

222226
public getTreeItem(element: TreeNode): vs.TreeItem {
223227
if (element instanceof SuiteNode) {
224-
return treeItemBuilder.createSuiteNode(element);
228+
return this.treeItemBuilder.createSuiteNode(element);
225229
} else if (element instanceof GroupNode) {
226-
return treeItemBuilder.createGroupNode(element);
230+
return this.treeItemBuilder.createGroupNode(element);
227231
} else if (element instanceof TestNode) {
228-
return treeItemBuilder.createTestNode(element);
232+
return this.treeItemBuilder.createTestNode(element);
229233
} else {
230234
throw new Error(`Unrecognised tree node type: ${element}`);
231235
}
@@ -317,6 +321,8 @@ function getIconPath(status: TestStatus, isStale: boolean): vs.Uri | undefined {
317321
}
318322

319323
class TreeItemBuilder {
324+
constructor(private readonly flutterCapabilities: FlutterCapabilities) { }
325+
320326
public createSuiteNode(node: SuiteNode): vs.TreeItem {
321327
// TODO: children is quite expensive, we should add a faster way.
322328
const collapseState = node.children?.length || 0 > 0 ? vs.TreeItemCollapsibleState.Collapsed : vs.TreeItemCollapsibleState.None;
@@ -371,13 +377,11 @@ class TreeItemBuilder {
371377
contexts += `${DART_TEST_CONTAINER_NODE_WITH_SKIPS_CONTEXT} `;
372378
}
373379

374-
if (node.suiteData.isFlutterSuite)
375-
contexts += `${IS_IN_FLUTTER_TEST_SUITE_CONTEXT} `;
376-
else
377-
contexts += `${IS_IN_DART_TEST_SUITE_CONTEXT} `;
380+
// Don't mark a node as skipable unless it's non-Flutter or the current Flutter version
381+
// supports run-skipped.
382+
if (!node.suiteData.isFlutterSuite || this.flutterCapabilities.supportsRunSkippedTests)
383+
contexts += `${DART_TEST_CAN_RUN_SKIPPED_CONTEXT} `;
378384

379385
return contexts.trimEnd();
380386
}
381387
}
382-
383-
const treeItemBuilder = new TreeItemBuilder();

‎src/shared/capabilities/flutter.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ export class FlutterCapabilities {
1717
get supportsExposeUrl() { return versionIsAtLeast(this.version, "1.18.0-5"); }
1818
get supportsDartDefine() { return versionIsAtLeast(this.version, "1.17.0"); }
1919
get supportsRestartDebounce() { return versionIsAtLeast(this.version, "1.21.0-0"); }
20+
// TODO: Set this
21+
get supportsRunSkippedTests() { return false; }
2022
get supportsShowWebServerDevice() { return versionIsAtLeast(this.version, "1.26.0-0"); }
2123
get supportsWebRendererOption() { return versionIsAtLeast(this.version, "1.25.0-0"); }
2224
get supportsDevToolsServerAddress() { return versionIsAtLeast(this.version, "1.26.0-12"); }

‎src/shared/constants.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,7 @@ export const DART_TEST_CONTAINER_NODE_WITH_SKIPS_CONTEXT = "dart-code:testContai
4242
export const DART_TEST_CONTAINER_NODE_WITH_FAILURES_CONTEXT = "dart-code:testContainerNodeWithFailures";
4343
export const DART_TEST_GROUP_NODE_CONTEXT = "dart-code:testGroupNode";
4444
export const DART_TEST_TEST_NODE_CONTEXT = "dart-code:testTestNode";
45-
export const IS_IN_FLUTTER_TEST_SUITE_CONTEXT = "dart-code:isInFlutterTestSuite";
46-
export const IS_IN_DART_TEST_SUITE_CONTEXT = "dart-code:isInDartTestSuite";
45+
export const DART_TEST_CAN_RUN_SKIPPED_CONTEXT = "dart-code:canRunSkipped";
4746

4847
export const DART_DEP_PROJECT_NODE_CONTEXT = "dart-code:depProjectNode";
4948
export const DART_DEP_PACKAGE_NODE_CONTEXT = "dart-code:depPackageNode";

‎src/test/flutter_test_debug/debug/flutter_test.test.ts‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import * as assert from "assert";
22
import * as path from "path";
33
import * as vs from "vscode";
44
import { DebuggerType, VmServiceExtension } from "../../../shared/enums";
5+
import { TestDoneNotification } from "../../../shared/test_protocol";
56
import { fsPath } from "../../../shared/utils/fs";
67
import { waitFor } from "../../../shared/utils/promises";
78
import { DartDebugClient } from "../../dart_debug_client";
@@ -61,6 +62,89 @@ describe("flutter test debugger", () => {
6162
assert.ok(testDoneNotification, JSON.stringify(customEvents.map((e) => e.body), undefined, 4));
6263
});
6364

65+
function isTestDoneNotification(e: vs.DebugSessionCustomEvent) {
66+
if (e.event !== "dart.testRunNotification")
67+
return false;
68+
const notification = e.body.notification as TestDoneNotification;
69+
return notification.type === "testDone" && !notification.hidden;
70+
}
71+
72+
it("does not attempt to run skipped tests from codelens if not supported", async function () {
73+
if (extApi.flutterCapabilities.supportsRunSkippedTests)
74+
this.skip();
75+
76+
const editor = await openFile(flutterTestMainFile);
77+
await waitForResult(() => !!extApi.fileTracker.getOutlineFor(flutterTestMainFile));
78+
79+
const fileCodeLens = await getCodeLens(editor.document);
80+
const testPos = positionOf(`test^Widgets('Skipped test`);
81+
82+
const codeLensForTest = fileCodeLens.filter((cl) => cl.range.start.line === testPos.line);
83+
assert.equal(codeLensForTest.length, 2);
84+
85+
if (!codeLensForTest[0].command) {
86+
// If there's no command, skip the test. This happens very infrequently and appears to be a VS Code
87+
// race condition. Rather than failing our test runs, skip.
88+
// TODO: Remove this if https://github.com/microsoft/vscode/issues/79805 gets a reliable fix.
89+
this.skip();
90+
return;
91+
}
92+
93+
const runAction = codeLensForTest.find((cl) => cl.command!.title === "Run")!;
94+
assert.equal(runAction.command!.command, "_dart.startWithoutDebuggingTestFromOutline");
95+
assert.equal(runAction.command!.arguments![0].fullName, "Skipped test");
96+
assert.equal(runAction.command!.arguments![0].isGroup, false);
97+
98+
const customEvents = await captureDebugSessionCustomEvents(async () => {
99+
const didStart = await vs.commands.executeCommand(runAction.command!.command, ...(runAction.command!.arguments ?? []));
100+
assert.ok(didStart);
101+
});
102+
103+
const testDoneNotification = customEvents.find(isTestDoneNotification);
104+
assert.ok(testDoneNotification, JSON.stringify(customEvents.map((e) => e.body), undefined, 4));
105+
106+
const testDone = testDoneNotification.body.notification as TestDoneNotification;
107+
assert.strictEqual(testDone.skipped, true);
108+
});
109+
110+
it("can run skipped tests from codelens if supported", async function () {
111+
if (!extApi.flutterCapabilities.supportsRunSkippedTests)
112+
this.skip();
113+
114+
const editor = await openFile(flutterTestMainFile);
115+
await waitForResult(() => !!extApi.fileTracker.getOutlineFor(flutterTestMainFile));
116+
117+
const fileCodeLens = await getCodeLens(editor.document);
118+
const testPos = positionOf(`test^Widgets('Skipped test`);
119+
120+
const codeLensForTest = fileCodeLens.filter((cl) => cl.range.start.line === testPos.line);
121+
assert.equal(codeLensForTest.length, 2);
122+
123+
if (!codeLensForTest[0].command) {
124+
// If there's no command, skip the test. This happens very infrequently and appears to be a VS Code
125+
// race condition. Rather than failing our test runs, skip.
126+
// TODO: Remove this if https://github.com/microsoft/vscode/issues/79805 gets a reliable fix.
127+
this.skip();
128+
return;
129+
}
130+
131+
const runAction = codeLensForTest.find((cl) => cl.command!.title === "Run")!;
132+
assert.equal(runAction.command!.command, "_dart.startWithoutDebuggingTestFromOutline");
133+
assert.equal(runAction.command!.arguments![0].fullName, "Skipped test");
134+
assert.equal(runAction.command!.arguments![0].isGroup, false);
135+
136+
const customEvents = await captureDebugSessionCustomEvents(async () => {
137+
const didStart = await vs.commands.executeCommand(runAction.command!.command, ...(runAction.command!.arguments ?? []));
138+
assert.ok(didStart);
139+
});
140+
141+
const testDoneNotification = customEvents.find(isTestDoneNotification);
142+
assert.ok(testDoneNotification, JSON.stringify(customEvents.map((e) => e.body), undefined, 4));
143+
144+
const testDone = testDoneNotification.body.notification as TestDoneNotification;
145+
assert.strictEqual(testDone.skipped, false); // Test should have run.
146+
});
147+
64148
it("receives the expected events from a Flutter test script", async () => {
65149
const config = await startDebugger(dc, flutterTestMainFile);
66150
await waitAllThrowIfTerminates(dc,
@@ -226,4 +310,34 @@ describe("flutter test debugger", () => {
226310
dc.launch(config),
227311
);
228312
});
313+
314+
it("can rerun only skipped tests", async function () {
315+
if (!extApi.flutterCapabilities.supportsRunSkippedTests)
316+
this.skip();
317+
318+
await openFile(flutterTestMainFile);
319+
const config = await startDebugger(dc, flutterTestMainFile);
320+
config.noDebug = true;
321+
await waitAllThrowIfTerminates(dc,
322+
dc.configurationSequence(),
323+
dc.waitForEvent("terminated"),
324+
dc.launch(config),
325+
);
326+
327+
// Now run only skipped tests.
328+
await vs.commands.executeCommand("dart.runAllSkippedTestsWithoutDebugging");
329+
330+
await openFile(flutterTestMainFile);
331+
// Expected results differ from what's in the file as the skipped tests will be run
332+
// and also the parent groups/suite status will be recomputed so they will be not-stale
333+
// in the new results (so we can't just filter to skipped, like we do in the failed test).
334+
const expectedResults = `
335+
test/widget_test.dart [2/2 passed, {duration}ms] (pass.svg)
336+
Skipped test [{duration}ms] (pass.svg)
337+
`.trim();
338+
339+
// Get the actual tree, filtered only to those that ran in the last run.
340+
const actualResults = (await makeTextTree(flutterTestMainFile, extApi.testTreeProvider, { onlyActive: true })).join("\n");
341+
assert.strictEqual(actualResults, expectedResults);
342+
});
229343
});

0 commit comments

Comments
 (0)