Skip to content

Commit c8a6ad5

Browse files
DanTupDanTup
committed
Merge some Flutter debug tests to speed up test runs
Spawning a real Flutter app is very slow, so combining some of these tests to check multiple things speeds up the whole run significantly. See #6165
1 parent 3c54102 commit c8a6ad5

2 files changed

Lines changed: 122 additions & 171 deletions

File tree

‎src/test/flutter_debug/flutter_run.test_device.test.ts‎

Lines changed: 50 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { versionIsAtLeast } from "../../shared/utils";
66
import { fsPath } from "../../shared/utils/fs";
77
import { DartDebugClient } from "../dart_debug_client";
88
import { createDebugClient, ensureServiceExtensionValue, flutterTestDeviceId, flutterTestDeviceIsWeb, killFlutterTester, startDebugger, waitAllThrowIfTerminates } from "../debug_helpers";
9-
import { activateWithoutAnalysis, closeAllOpenFiles, customScriptExt, defer, deferUntilLast, delay, ensureHasRunWithArgsStarting, flutterHelloWorldFolder, flutterHelloWorldMainFile, flutterHelloWorldNavigateFromFile, flutterHelloWorldNavigateToFile, flutterHelloWorldOverflowFile, flutterHelloWorldReadmeFile, flutterHelloWorldStack60File, getLaunchConfiguration, getPackages, makeTrivialChangeToFileDirectly, openFile, positionOf, prepareHasRunFile, privateApi, saveTrivialChangeToFile, sb, setConfigForTest, waitForResult, watchPromise } from "../helpers";
9+
import { activateWithoutAnalysis, closeAllOpenFiles, customScriptExt, defer, deferUntilLast, delay, flutterHelloWorldFolder, flutterHelloWorldMainFile, flutterHelloWorldNavigateFromFile, flutterHelloWorldNavigateToFile, flutterHelloWorldOverflowFile, flutterHelloWorldReadmeFile, flutterHelloWorldStack60File, getLaunchConfiguration, getPackages, makeTrivialChangeToFileDirectly, openFile, positionOf, privateApi, saveTrivialChangeToFile, sb, setConfigForTest, waitForResult, watchPromise } from "../helpers";
1010

1111
describe(`flutter run debugger (only test device)`, () => {
1212
beforeEach("Skip test-device tests on web", function () {
@@ -81,7 +81,7 @@ describe(`flutter run debugger (only test device)`, () => {
8181
});
8282
});
8383

84-
it("can override platform, toggle brightness/theme", async () => {
84+
it("can override platform/brightness and re-sends theme on hot restart only if set by us", async () => {
8585
const config = await startDebugger(dc, flutterHelloWorldMainFile);
8686
await waitAllThrowIfTerminates(dc,
8787
dc.flutterAppStarted(),
@@ -104,28 +104,6 @@ describe(`flutter run debugger (only test device)`, () => {
104104
// Wait for Brightness extension before trying to call it.
105105
await waitForResult(() => privateApi.debugCommands.vmServices.serviceExtensionIsLoaded(VmServiceExtension.BrightnessOverride), "Brightness override loaded");
106106

107-
await ensureServiceExtensionValue(VmServiceExtension.BrightnessOverride, "Brightness.light", dc);
108-
await vs.commands.executeCommand("flutter.toggleBrightness");
109-
await ensureServiceExtensionValue(VmServiceExtension.BrightnessOverride, "Brightness.dark", dc);
110-
await vs.commands.executeCommand("flutter.toggleBrightness");
111-
await ensureServiceExtensionValue(VmServiceExtension.BrightnessOverride, "Brightness.light", dc);
112-
113-
await waitAllThrowIfTerminates(dc,
114-
dc.waitForEvent("terminated"),
115-
dc.terminateRequest(),
116-
);
117-
});
118-
119-
it("re-sends theme on hot restart only if set by us, not if set by someone else", async () => {
120-
const config = await startDebugger(dc, flutterHelloWorldMainFile);
121-
await waitAllThrowIfTerminates(dc,
122-
dc.flutterAppStarted(),
123-
dc.configurationSequence(),
124-
dc.launch(config),
125-
);
126-
127-
await waitForResult(() => privateApi.debugCommands.vmServices.serviceExtensionIsLoaded(VmServiceExtension.BrightnessOverride), "Waiting for BrightnessOverride extension", 60000);
128-
129107
// Re-sends theme on hot restart if set by us.
130108
{
131109
// Set the brightness to Dark through our toggle. This leaves us in control so we should
@@ -299,52 +277,41 @@ describe(`flutter run debugger (only test device)`, () => {
299277
);
300278
});
301279

302-
it("can run using a custom tool", async () => {
280+
it("passes custom tool in the launch config", async () => {
281+
// This only verifies the launch config (without spawning the app, which is slow).
282+
// The debug adapter tests cover testing that the custom tool is invoked.
303283
const root = fsPath(flutterHelloWorldFolder);
304-
const hasRunFile = prepareHasRunFile(root, "flutter");
305-
306-
const config = await startDebugger(dc, flutterHelloWorldMainFile, {
307-
customTool: path.join(root, `scripts/custom_flutter.${customScriptExt}`),
284+
const customTool = path.join(root, `scripts/custom_flutter.${customScriptExt}`);
285+
const config = await getLaunchConfiguration(flutterHelloWorldMainFile, {
286+
customTool,
308287
customToolReplacesArgs: 0,
288+
deviceId: flutterTestDeviceId,
309289
});
310-
await waitAllThrowIfTerminates(dc,
311-
dc.assertOutputContains("console", `Launching lib${path.sep}main.dart on Flutter test device in debug mode...\n`),
312-
dc.flutterAppStarted(),
313-
dc.configurationSequence(),
314-
dc.launch(config),
315-
);
316290

317-
await waitAllThrowIfTerminates(dc,
318-
dc.waitForEvent("terminated"),
319-
dc.terminateRequest(),
320-
);
321-
322-
ensureHasRunWithArgsStarting(root, hasRunFile, "run --machine --start-paused");
291+
assert.equal(config!.customTool, customTool);
292+
assert.equal(config!.customToolReplacesArgs, 0);
293+
assert.ok(config!.toolArgs!.includes("-d"));
294+
assert.ok(config!.toolArgs!.includes(flutterTestDeviceId));
323295
});
324296

325-
it("can replace all args using custom tool", async () => {
297+
it("passes custom tool args in the launch config", async () => {
298+
// This only verifies the config the extension produces (without spawning the
299+
// app, which is slow). The debug adapter replacing its args with these is
300+
// covered by the adapter's own tests.
326301
const root = fsPath(flutterHelloWorldFolder);
327-
const hasRunFile = prepareHasRunFile(root, "flutter");
328-
329-
const config = await startDebugger(dc, flutterHelloWorldMainFile, {
330-
customTool: path.join(root, `scripts/custom_flutter.${customScriptExt}`),
302+
const customTool = path.join(root, `scripts/custom_flutter.${customScriptExt}`);
303+
// These differ to the usual ones so we can detect they replaced them.
304+
const toolArgs = ["run", "--ignore-deprecation", "--start-paused", "--machine"];
305+
const config = await getLaunchConfiguration(flutterHelloWorldMainFile, {
306+
customTool,
331307
customToolReplacesArgs: 999999,
332-
// These differ to the usual ones so we can detect they replaced them.
333-
toolArgs: ["run", "--ignore-deprecation", "--start-paused", "--machine"],
308+
deviceId: flutterTestDeviceId,
309+
toolArgs,
334310
});
335-
await waitAllThrowIfTerminates(dc,
336-
dc.assertOutputContains("console", `Launching lib${path.sep}main.dart on Flutter test device in debug mode...\n`),
337-
dc.flutterAppStarted(),
338-
dc.configurationSequence(),
339-
dc.launch(config),
340-
);
341-
342-
await waitAllThrowIfTerminates(dc,
343-
dc.waitForEvent("terminated"),
344-
dc.terminateRequest(),
345-
);
346311

347-
ensureHasRunWithArgsStarting(root, hasRunFile, "run --ignore-deprecation --start-paused --machine");
312+
assert.equal(config!.customTool, customTool);
313+
assert.equal(config!.customToolReplacesArgs, 999999);
314+
assert.deepStrictEqual(config!.toolArgs!.slice(0, toolArgs.length), toolArgs);
348315
});
349316

350317
it("can fetch slices of stack frames", async () => {
@@ -445,7 +412,7 @@ describe(`flutter run debugger (only test device)`, () => {
445412
});
446413

447414
describe("can evaluate at breakpoint", () => {
448-
it("simple expressions", async () => {
415+
it("simple expressions, complex expressions, and expressions that return variables", async () => {
449416
await openFile(flutterHelloWorldMainFile);
450417
const config = await startDebugger(dc, flutterHelloWorldMainFile);
451418
await waitAllThrowIfTerminates(dc,
@@ -455,53 +422,30 @@ describe(`flutter run debugger (only test device)`, () => {
455422
}),
456423
);
457424

458-
const evaluateResult = await dc.evaluateForFrame(`"test"`);
459-
assert.ok(evaluateResult);
460-
assert.equal(evaluateResult.result, `"test"`);
461-
assert.equal(evaluateResult.variablesReference, 0);
462-
463-
await waitAllThrowIfTerminates(dc,
464-
dc.waitForEvent("terminated"),
465-
dc.terminateRequest(),
466-
);
467-
});
468-
469-
it("complex expressions", async () => {
470-
await openFile(flutterHelloWorldMainFile);
471-
const config = await startDebugger(dc, flutterHelloWorldMainFile);
472-
await waitAllThrowIfTerminates(dc,
473-
dc.hitBreakpoint(config, {
474-
line: positionOf("^// BREAKPOINT1").line,
475-
path: fsPath(flutterHelloWorldMainFile),
476-
}),
477-
);
478-
479-
const evaluateResult = await dc.evaluateForFrame(`(new DateTime.now()).year`);
480-
assert.ok(evaluateResult);
481-
assert.equal(evaluateResult.result, (new Date()).getFullYear().toString());
482-
assert.equal(evaluateResult.variablesReference, 0);
483-
484-
await waitAllThrowIfTerminates(dc,
485-
dc.waitForEvent("terminated"),
486-
dc.terminateRequest(),
487-
);
488-
});
425+
// Simple expressions.
426+
{
427+
const evaluateResult = await dc.evaluateForFrame(`"test"`);
428+
assert.ok(evaluateResult);
429+
assert.equal(evaluateResult.result, `"test"`);
430+
assert.equal(evaluateResult.variablesReference, 0);
431+
}
489432

490-
it("an expression that returns a variable", async () => {
491-
await openFile(flutterHelloWorldMainFile);
492-
const config = await startDebugger(dc, flutterHelloWorldMainFile);
493-
await waitAllThrowIfTerminates(dc,
494-
dc.hitBreakpoint(config, {
495-
line: positionOf("^// BREAKPOINT1").line,
496-
path: fsPath(flutterHelloWorldMainFile),
497-
}),
498-
);
433+
// Complex expressions.
434+
{
435+
const evaluateResult = await dc.evaluateForFrame(`(new DateTime.now()).year`);
436+
assert.ok(evaluateResult);
437+
assert.equal(evaluateResult.result, (new Date()).getFullYear().toString());
438+
assert.equal(evaluateResult.variablesReference, 0);
439+
}
499440

500-
const evaluateResult = await dc.evaluateForFrame(`new DateTime.now()`);
501-
const thisYear = new Date().getFullYear().toString();
502-
assert.ok(evaluateResult);
503-
assert.ok(evaluateResult.result.startsWith("DateTime (" + thisYear), `Result '${evaluateResult.result}' did not start with ${thisYear}`);
504-
assert.ok(evaluateResult.variablesReference);
441+
// An expression that returns a variable.
442+
{
443+
const evaluateResult = await dc.evaluateForFrame(`new DateTime.now()`);
444+
const thisYear = new Date().getFullYear().toString();
445+
assert.ok(evaluateResult);
446+
assert.ok(evaluateResult.result.startsWith("DateTime (" + thisYear), `Result '${evaluateResult.result}' did not start with ${thisYear}`);
447+
assert.ok(evaluateResult.variablesReference);
448+
}
505449

506450
await waitAllThrowIfTerminates(dc,
507451
dc.waitForEvent("terminated"),

‎src/test/flutter_debug/flutter_run.test_device_and_web.test.ts‎

Lines changed: 72 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -68,28 +68,44 @@ describe(`flutter run debugger (launch on ${flutterTestDeviceId})`, () => {
6868
);
6969
});
7070

71-
it("expected debugger services/extensions are available in noDebug mode", async () => {
71+
it("does not stop at breakpoints and exposes only expected services in noDebug mode", async () => {
72+
await openFile(flutterHelloWorldMainFile);
7273
const config = await startDebugger(dc, flutterHelloWorldMainFile);
7374
config.noDebug = true;
74-
await waitAllThrowIfTerminates(dc,
75-
dc.waitForCustomEvent("flutter.appStarted"),
76-
dc.flutterAppStarted(),
77-
dc.configurationSequence(),
78-
dc.launch(config),
79-
);
75+
76+
let didStop = false;
77+
78+
dc.waitForEvent("stopped")
79+
.then(() => didStop = true)
80+
.catch(() => {
81+
// Swallow errors, as we don't care if this times out, we're only using it
82+
// to tell if we stopped by the time we hit the end of this test.
83+
});
8084

8185
const expectHotReload = true;
8286
const expectOtherServices = false;
8387

84-
await waitForResult(() => privateApi.debugCommands.vmServices.serviceIsRegistered(VmService.HotReload) === expectHotReload, "Hot reload registered");
85-
await waitForResult(() => privateApi.debugCommands.vmServices.serviceExtensionIsLoaded(VmServiceExtension.DebugPaint) === expectOtherServices, "Debug paint loaded");
86-
await waitForResult(() => privateApi.debugCommands.vmServices.serviceExtensionIsLoaded(VmServiceExtension.DebugBanner) === expectOtherServices, "Debug banner loaded");
87-
8888
await waitAllThrowIfTerminates(dc,
8989
dc.waitForEvent("terminated"),
90-
dc.terminateRequest(),
90+
dc.setBreakpointWithoutHitting(config, {
91+
line: positionOf("^// BREAKPOINT1").line + 1, // positionOf is 0-based, but seems to want 1-based
92+
path: fsPath(flutterHelloWorldMainFile),
93+
// TODO: This should be false in noDebug mode in SDK DAPs too.
94+
// verified: false,
95+
}),
96+
dc.flutterAppStarted()
97+
// These waitForResults also ensure the app has started (and therefore passed
98+
// the breakpoint location) before we check that we didn't stop.
99+
.then(() => waitForResult(() => privateApi.debugCommands.vmServices.serviceIsRegistered(VmService.HotReload) === expectHotReload, "Hot reload registered"))
100+
.then(() => waitForResult(() => privateApi.debugCommands.vmServices.serviceExtensionIsLoaded(VmServiceExtension.DebugPaint) === expectOtherServices, "Debug paint loaded"))
101+
.then(() => waitForResult(() => privateApi.debugCommands.vmServices.serviceExtensionIsLoaded(VmServiceExtension.DebugBanner) === expectOtherServices, "Debug banner loaded"))
102+
// Keep running for a short period to ensure we didn't stop unexpectedly.
103+
.then(() => delay(1000))
104+
.then(() => dc.terminateRequest()),
91105
);
92106

107+
assert.equal(didStop, false);
108+
93109
await waitForResult(() => privateApi.debugCommands.vmServices.serviceIsRegistered(VmService.HotReload) === false, "Hot reload unregistered");
94110
await waitForResult(() => privateApi.debugCommands.vmServices.serviceExtensionIsLoaded(VmServiceExtension.DebugPaint) === false, "Debug paint unloaded");
95111
await waitForResult(() => privateApi.debugCommands.vmServices.serviceExtensionIsLoaded(VmServiceExtension.DebugBanner) === false, "Debug banner unloaded");
@@ -209,34 +225,6 @@ describe(`flutter run debugger (launch on ${flutterTestDeviceId})`, () => {
209225
);
210226
});
211227

212-
it("does not stop at a breakpoint in noDebug mode", async () => {
213-
await openFile(flutterHelloWorldMainFile);
214-
const config = await startDebugger(dc, flutterHelloWorldMainFile);
215-
config.noDebug = true;
216-
217-
let didStop = false;
218-
219-
dc.waitForEvent("stopped")
220-
.then(() => didStop = true)
221-
.catch(() => {
222-
// Swallow errors, as we don't care if this times out, we're only using it
223-
// to tell if we stopped by the time we hit the end of this test.
224-
});
225-
await waitAllThrowIfTerminates(dc,
226-
dc.waitForEvent("terminated"),
227-
dc.setBreakpointWithoutHitting(config, {
228-
line: positionOf("^// BREAKPOINT1").line + 1, // positionOf is 0-based, but seems to want 1-based
229-
path: fsPath(flutterHelloWorldMainFile),
230-
// TODO: This should be false in noDebug mode in SDK DAPs too.
231-
// verified: false,
232-
})
233-
.then(() => delay(1000))
234-
.then(() => dc.terminateRequest()),
235-
);
236-
237-
assert.equal(didStop, false);
238-
});
239-
240228
it("stops at a breakpoint in a part file");
241229

242230
it("stops at a breakpoint in a deferred file");
@@ -569,11 +557,44 @@ describe(`flutter run debugger (launch on ${flutterTestDeviceId})`, () => {
569557
};
570558
}
571559

572-
it("stops at a breakpoint with a condition returning true", testBreakpointCondition("1 == 1", true));
573-
it("does not stop at a breakpoint with a condition returning false", testBreakpointCondition("1 == 0", false));
560+
it("stops at a breakpoint with a condition returning true, but not one returning false", async () => {
561+
await openFile(flutterHelloWorldMainFile);
562+
const config = await startDebugger(dc, flutterHelloWorldMainFile);
563+
564+
// genericMethod (BREAKPOINT2) runs before BREAKPOINT1 during the build, so the
565+
// false condition is evaluated first and must not stop, then the true condition
566+
// stops. If the false condition incorrectly stopped, the location assertion below
567+
// would fail because we'd be stopped at BREAKPOINT2 instead of BREAKPOINT1.
568+
const breakpoint1Line = positionOf("^// BREAKPOINT1").line;
569+
const breakpoint2Line = positionOf("^// BREAKPOINT2").line;
570+
571+
await waitAllThrowIfTerminates(
572+
dc,
573+
dc.waitForEvent("initialized")
574+
.then(() => dc.setBreakpointsRequest({
575+
// positionOf is 0-based, but seems to want 1-based
576+
breakpoints: [
577+
{ condition: "1 == 0", line: breakpoint2Line },
578+
{ condition: "1 == 1", line: breakpoint1Line },
579+
],
580+
source: { path: fsPath(flutterHelloWorldMainFile) },
581+
}))
582+
.then(() => dc.configurationDoneRequest()),
583+
dc.assertStoppedLocation("breakpoint", {
584+
line: breakpoint1Line,
585+
path: fsPath(flutterHelloWorldMainFile),
586+
}),
587+
dc.launch(config),
588+
);
589+
590+
await waitAllThrowIfTerminates(dc,
591+
dc.waitForEvent("terminated"),
592+
dc.terminateRequest(),
593+
);
594+
});
574595
it("reports errors evaluating breakpoint conditions", testBreakpointCondition("1 + '1'", false, `Debugger failed to evaluate breakpoint condition "1 + '1'"`));
575596

576-
it("provides local variables when stopped at a breakpoint", async () => {
597+
it("provides local variables and evaluateNames when stopped at a breakpoint", async () => {
577598
await openFile(flutterHelloWorldMainFile);
578599
const debugConfig = await startDebugger(dc, flutterHelloWorldMainFile);
579600
await dc.hitBreakpoint(debugConfig, {
@@ -653,6 +674,14 @@ describe(`flutter run debugger (launch on ${flutterTestDeviceId})`, () => {
653674
value: { evaluateName: `m[1.1]`, name: "value", value: `"one-point-one"` },
654675
}, dc);
655676

677+
// The evaluateNames of the variables above should evaluate to the same values.
678+
// Re-use the children already fetched above, except for the long strings whose
679+
// children are skipped above on web because of truncated-value differences.
680+
const listLongstringVariables = await dc.getVariables(variables.find((v) => v.name === "longStrings")!.variablesReference);
681+
const allVariables = listVariables.concat(listLongstringVariables).concat(mapVariables);
682+
683+
await Promise.all(allVariables.map((v) => ensureVariableEvaluateName(dc, v)));
684+
656685
await waitAllThrowIfTerminates(dc,
657686
dc.waitForEvent("terminated"),
658687
dc.terminateRequest(),
@@ -771,28 +800,6 @@ describe(`flutter run debugger (launch on ${flutterTestDeviceId})`, () => {
771800
);
772801
});
773802

774-
it("evaluateName evaluates to the expected value", async () => {
775-
await openFile(flutterHelloWorldMainFile);
776-
const config = await startDebugger(dc, flutterHelloWorldMainFile);
777-
await dc.hitBreakpoint(config, {
778-
line: positionOf("^// BREAKPOINT1").line,
779-
path: fsPath(flutterHelloWorldMainFile),
780-
});
781-
782-
const variables = await dc.getTopFrameVariables("Locals");
783-
const listVariables = await dc.getVariables(variables.find((v) => v.name === "l")!.variablesReference);
784-
const listLongstringVariables = await dc.getVariables(variables.find((v) => v.name === "longStrings")!.variablesReference);
785-
const mapVariables = await dc.getVariables(variables.find((v) => v.name === "m")!.variablesReference);
786-
const allVariables = listVariables.concat(listLongstringVariables).concat(mapVariables);
787-
788-
await Promise.all(allVariables.map((v) => ensureVariableEvaluateName(dc, v)));
789-
790-
await waitAllThrowIfTerminates(dc,
791-
dc.waitForEvent("terminated"),
792-
dc.terminateRequest(),
793-
);
794-
});
795-
796803
it("stops on exception", async function () {
797804
if (flutterTestDeviceIsWeb)
798805
return this.skip(); // Currently fails on web as it doesn't pause on the exception.

0 commit comments

Comments
 (0)