Skip to content

Rework the integration test timeout logic #9322

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Rework the integration test timeout logic #9322

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions packages/devtools_app/integration_test/test/live_connection/app_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,18 @@ void main() {
await resetHistory();
});

testWidgets('connect to app and switch tabs', (tester) async {
await pumpAndConnectDevTools(tester, testApp);

// For the sake of this test, do not show extension screens by default.
preferences.devToolsExtensions.showOnlyEnabledExtensions.value = true;
await tester.pumpAndSettle(shortPumpDuration);

logStatus('verify that we can load each DevTools screen');
await navigateThroughDevToolsScreens(tester, connectedToApp: true);
});
testWidgets(
'connect to app and switch tabs',
timeout: const Timeout(Duration(minutes: 2)),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just for consistency, can we add consts shortTimeout, mediumTimeout and longTimeout and put them in integration_test/test_infra/_utils.dart?

(tester) async {
await pumpAndConnectDevTools(tester, testApp);

// For the sake of this test, do not show extension screens by default.
preferences.devToolsExtensions.showOnlyEnabledExtensions.value = true;
await tester.pumpAndSettle(shortPumpDuration);

logStatus('verify that we can load each DevTools screen');
await navigateThroughDevToolsScreens(tester, connectedToApp: true);
},
);
}
4 changes: 3 additions & 1 deletion packages/devtools_app/integration_test/test/live_connection/debugger_panel_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ void main() {
expect(testApp.vmServiceUri, isNotNull);
});

testWidgets('Debugger panel', (tester) async {
testWidgets('Debugger panel', timeout: const Timeout(Duration(minutes: 4)), (
tester,
) async {
await pumpAndConnectDevTools(tester, testApp);
await switchToScreen(
tester,
Expand Down
242 changes: 123 additions & 119 deletions packages/devtools_app/integration_test/test/live_connection/devtools_extensions_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,125 +41,129 @@ void main() {
resetDevToolsExtensionEnabledStates();
});

testWidgets('end to end extensions flow', (tester) async {
await pumpDevTools(tester);

// TODO(https://github.com/flutter/devtools/issues/9196): re-enable this
// test verification once DTD can be started from the integration test
// harness.
// logStatus(
// 'verify static extensions are available before connecting to an app',
// );
// expect(extensionService.availableExtensions.length, 2);
// expect(extensionService.visibleExtensions.length, 2);
// await _verifyExtensionsSettingsMenu(tester, [
// ExtensionEnabledState.none, // dart_foo
// ExtensionEnabledState.none, // standalone_extension
// ]);

await connectToTestApp(tester, testApp);

expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 3);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.none, // dart_foo
ExtensionEnabledState.none, // foo
ExtensionEnabledState.none, // standalone_extension
], closeMenuWhenDone: false);

await _verifyExtensionVisibilitySetting(tester);

// dart_foo extension.
// Enable, test context menu actions, then disable from context menu.
await _switchToExtensionScreen(
tester,
extensionIndex: 0,
initialLoad: true,
);
await _answerEnableExtensionPrompt(tester, enable: true);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.enabled, // dart_foo
ExtensionEnabledState.none, // foo
ExtensionEnabledState.none, // standalone_extension
]);

await _verifyContextMenuActionsAndDisable(tester);

expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 2);
await _verifyExtensionTabVisibility(
tester,
extensionIndex: 0,
visible: false,
);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.disabled, // dart_foo
ExtensionEnabledState.none, // foo
ExtensionEnabledState.none, // standalone_extension
]);

// foo extension. Hide immediately.
await _switchToExtensionScreen(
tester,
extensionIndex: 1,
initialLoad: true,
);
await _answerEnableExtensionPrompt(tester, enable: false);

expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 1);
await _verifyExtensionTabVisibility(
tester,
extensionIndex: 1,
visible: false,
);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.disabled, // dart_foo
ExtensionEnabledState.disabled, // foo
ExtensionEnabledState.none, // standalone_extension
]);

// Re-enable foo extension from the extensions settings menu.
logStatus('verify we can re-enable an extension from the settings menu');
await _changeExtensionSetting(tester, extensionIndex: 1, enable: true);

expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 2);
await _switchToExtensionScreen(tester, extensionIndex: 1);
expect(find.byType(EnableExtensionPrompt), findsNothing);
expect(find.byType(EmbeddedExtensionView), findsOneWidget);
expect(find.byType(HtmlElementView), findsOneWidget);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.disabled, // dart_foo
ExtensionEnabledState.enabled, // foo
ExtensionEnabledState.none, // standalone_extension
]);

// standalone_extension. Disable directly from settings menu.
logStatus(
'verify we can disable an extension screen directly from the settings menu',
);
await _verifyExtensionTabVisibility(
tester,
extensionIndex: 2,
visible: true,
);

logStatus('disable the extension from the settings menu');
await _changeExtensionSetting(tester, extensionIndex: 2, enable: false);
expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 1);
await _verifyExtensionTabVisibility(
tester,
extensionIndex: 2,
visible: false,
);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.disabled, // dart_foo
ExtensionEnabledState.enabled, // foo
ExtensionEnabledState.disabled, // standalone_extension
]);
});
testWidgets(
'end to end extensions flow',
timeout: const Timeout(Duration(minutes: 3)),
(tester) async {
await pumpDevTools(tester);

// TODO(https://github.com/flutter/devtools/issues/9196): re-enable this
// test verification once DTD can be started from the integration test
// harness.
// logStatus(
// 'verify static extensions are available before connecting to an app',
// );
// expect(extensionService.availableExtensions.length, 2);
// expect(extensionService.visibleExtensions.length, 2);
// await _verifyExtensionsSettingsMenu(tester, [
// ExtensionEnabledState.none, // dart_foo
// ExtensionEnabledState.none, // standalone_extension
// ]);

await connectToTestApp(tester, testApp);

expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 3);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.none, // dart_foo
ExtensionEnabledState.none, // foo
ExtensionEnabledState.none, // standalone_extension
], closeMenuWhenDone: false);

await _verifyExtensionVisibilitySetting(tester);

// dart_foo extension.
// Enable, test context menu actions, then disable from context menu.
await _switchToExtensionScreen(
tester,
extensionIndex: 0,
initialLoad: true,
);
await _answerEnableExtensionPrompt(tester, enable: true);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.enabled, // dart_foo
ExtensionEnabledState.none, // foo
ExtensionEnabledState.none, // standalone_extension
]);

await _verifyContextMenuActionsAndDisable(tester);

expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 2);
await _verifyExtensionTabVisibility(
tester,
extensionIndex: 0,
visible: false,
);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.disabled, // dart_foo
ExtensionEnabledState.none, // foo
ExtensionEnabledState.none, // standalone_extension
]);

// foo extension. Hide immediately.
await _switchToExtensionScreen(
tester,
extensionIndex: 1,
initialLoad: true,
);
await _answerEnableExtensionPrompt(tester, enable: false);

expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 1);
await _verifyExtensionTabVisibility(
tester,
extensionIndex: 1,
visible: false,
);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.disabled, // dart_foo
ExtensionEnabledState.disabled, // foo
ExtensionEnabledState.none, // standalone_extension
]);

// Re-enable foo extension from the extensions settings menu.
logStatus('verify we can re-enable an extension from the settings menu');
await _changeExtensionSetting(tester, extensionIndex: 1, enable: true);

expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 2);
await _switchToExtensionScreen(tester, extensionIndex: 1);
expect(find.byType(EnableExtensionPrompt), findsNothing);
expect(find.byType(EmbeddedExtensionView), findsOneWidget);
expect(find.byType(HtmlElementView), findsOneWidget);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.disabled, // dart_foo
ExtensionEnabledState.enabled, // foo
ExtensionEnabledState.none, // standalone_extension
]);

// standalone_extension. Disable directly from settings menu.
logStatus(
'verify we can disable an extension screen directly from the settings menu',
);
await _verifyExtensionTabVisibility(
tester,
extensionIndex: 2,
visible: true,
);

logStatus('disable the extension from the settings menu');
await _changeExtensionSetting(tester, extensionIndex: 2, enable: false);
expect(extensionService.availableExtensions.length, 3);
expect(extensionService.visibleExtensions.length, 1);
await _verifyExtensionTabVisibility(
tester,
extensionIndex: 2,
visible: false,
);
await _verifyExtensionsSettingsMenu(tester, [
ExtensionEnabledState.disabled, // dart_foo
ExtensionEnabledState.enabled, // foo
ExtensionEnabledState.disabled, // standalone_extension
]);
},
);
}

Future<void> _switchToExtensionScreen(
Expand Down
38 changes: 21 additions & 17 deletions packages/devtools_app/integration_test/test/live_connection/eval_and_browse_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,30 +30,34 @@ void main() {
await resetHistory();
});

testWidgets('memory eval and browse', (tester) async {
await pumpAndConnectDevTools(tester, testApp);
testWidgets(
'memory eval and browse',
timeout: const Timeout(Duration(minutes: 3)),
(tester) async {
await pumpAndConnectDevTools(tester, testApp);

final evalTester = EvalTester(tester);
await prepareMemoryUI(tester, makeConsoleWider: true);
final evalTester = EvalTester(tester);
await prepareMemoryUI(tester, makeConsoleWider: true);

logStatus('test basic evaluation');
await testBasicEval(evalTester);
logStatus('test basic evaluation');
await testBasicEval(evalTester);

logStatus('test variable assignment');
await testAssignment(evalTester);
logStatus('test variable assignment');
await testAssignment(evalTester);

logStatus('test dump one instance to console');
await _profileOneInstance(evalTester);
logStatus('test dump one instance to console');
await _profileOneInstance(evalTester);

logStatus('test dump all instances to console');
await _profileAllInstances(evalTester);
logStatus('test dump all instances to console');
await _profileAllInstances(evalTester);

logStatus('test take a snapshot');
await evalTester.takeSnapshot();
logStatus('test take a snapshot');
await evalTester.takeSnapshot();

logStatus('test inbound references are listed on console instance');
await _inboundReferencesAreListed(evalTester);
});
logStatus('test inbound references are listed on console instance');
await _inboundReferencesAreListed(evalTester);
},
);
}

Future<void> _profileOneInstance(EvalTester tester) async {
Expand Down
23 changes: 14 additions & 9 deletions packages/devtools_app/integration_test/test/live_connection/eval_and_inspect_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,26 @@ void main() {
await resetHistory();
});

testWidgets('eval with scope in inspector window', (tester) async {
await pumpAndConnectDevTools(tester, testApp);
testWidgets(
'eval with scope in inspector window',
timeout: const Timeout(Duration(minutes: 3)),
(tester) async {
await pumpAndConnectDevTools(tester, testApp);

final evalTester = EvalTester(tester);
await evalTester.prepareInspectorUI();
final evalTester = EvalTester(tester);
await evalTester.prepareInspectorUI();

logStatus('testing basic evaluation');
await testBasicEval(evalTester);
logStatus('testing basic evaluation');
await testBasicEval(evalTester);

logStatus('testing variable assignment');
await testAssignment(evalTester);
});
logStatus('testing variable assignment');
await testAssignment(evalTester);
},
);

testWidgets(
'eval with scope on widget tree node',
timeout: const Timeout(Duration(minutes: 3)),
(tester) async {
await pumpAndConnectDevTools(tester, testApp);

Expand Down
4 changes: 3 additions & 1 deletion packages/devtools_app/integration_test/test/live_connection/export_snapshot_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ void main() {
await resetHistory();
});

testWidgets('Export snapshot', (tester) async {
testWidgets('Export snapshot', timeout: const Timeout(Duration(minutes: 2)), (
tester,
) async {
await pumpAndConnectDevTools(tester, testApp);
await prepareMemoryUI(tester);
await takeHeapSnapshot(tester);
Expand Down
Loading
Loading