Introduce an analyzer plugin for the test package.

pkgs/test_analyzer_plugin/README.md

@@ -0,0 +1,13 @@
+# test_analyzer_plugin
+
+This package is an analyzer plugin that provides additional static analysis for
+usage of the test package.
+
+This analyzer plugin provides the following additional analysis:
+
+* Report a warning when a `test` or a `group` is declared inside a `test`
+  declaration. This can _sometimes_ be detected at runtime. This warning is
+  reported statically.
+
+* Offer a quick fix in the IDE for the above warning, which moves the violating
+  `test` or `group` declaration below the containing `test` declaration.

pkgs/test_analyzer_plugin/lib/main.dart

@@ -0,0 +1,35 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server_plugin/plugin.dart';
+import 'package:analysis_server_plugin/registry.dart';
+
+import 'src/fixes.dart';
+import 'src/rules/non_nullable_is_not_null_rule.dart';
+import 'src/rules/test_in_test_rule.dart';
+
+final plugin = TestPackagePlugin();
+
+class TestPackagePlugin extends Plugin {
+  @override
+  void register(PluginRegistry registry) {
+    registry.registerWarningRule(TestInTestRule());
+    registry.registerFixForRule(
+        TestInTestRule.code, MoveBelowEnclosingTestCall.new);
+
+    registry.registerWarningRule(NonNullableIsNotNullRule());
+    // Should we register a fix for this rule? The only automatic fix I can
+    // think of would be to delete the entire statement:
+    // `expect(a, isNotNull);` or `expect(a, isNull);`.
+
+    // TODO(srawlins): More rules to catch:
+    // * `expect(7, contains(6))` - should only use `hasLength` with `Iterable`
+    //   or `String`.
+    // * `expect(7, hasLength(3))` - should only use `hasLength` with `Iterable`
+    //   or `String`.
+    // * `expect([].isEmpty, isFalse)` - should use `isEmpty` matcher.
+    // * `expect([].isNotEmpty, isTrue)` - should use `isNotEmpty` matcher.
+    // * `expect([].contains(7), isFalse)` - should use `contains` matcher.
+  }
+}

pkgs/test_analyzer_plugin/lib/src/fixes.dart

@@ -0,0 +1,58 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server_plugin/edit/dart/correction_producer.dart';
+import 'package:analysis_server_plugin/edit/dart/dart_fix_kind_priority.dart';
+import 'package:analyzer/dart/ast/ast.dart';
+import 'package:analyzer_plugin/utilities/change_builder/change_builder_core.dart';
+import 'package:analyzer_plugin/utilities/fixes/fixes.dart';
+import 'package:analyzer_plugin/utilities/range_factory.dart';
+
+import 'utilities.dart';
+
+class MoveBelowEnclosingTestCall extends ResolvedCorrectionProducer {
+  static const _moveBelowEnclosingTestCallKind = FixKind(
+      'dart.fix.moveBelowEnclosingTestCall',
+      DartFixKindPriority.standard,
+      "Move below the enclosing 'test' call");
+
+  MoveBelowEnclosingTestCall({required super.context});
+
+  @override
+  CorrectionApplicability get applicability =>
+      // This fix may break code by moving references to variables away from the
+      // scope in which they are declared.
+      CorrectionApplicability.singleLocation;
+
+  @override
+  FixKind get fixKind => _moveBelowEnclosingTestCallKind;
+
+  @override
+  Future<void> compute(ChangeBuilder builder) async {
+    var methodCall = node;
+    if (methodCall is! MethodInvocation) return;
+    AstNode? enclosingTestCall = findEnclosingTestCall(methodCall);
+    if (enclosingTestCall == null) return;
+
+    if (enclosingTestCall.parent is ExpressionStatement) {
+      // Move the 'test' call to below the outer 'test' call _statement_.
+      enclosingTestCall = enclosingTestCall.parent!;
+    }
+
+    if (methodCall.parent is ExpressionStatement) {
+      // Move the whole statement (don't leave the semicolon dangling).
+      methodCall = methodCall.parent!;
+    }
+
+    await builder.addDartFileEdit(file, (builder) {
+      var indent = utils.getLinePrefix(enclosingTestCall!.offset);
+      var source = utils.getRangeText(range.node(methodCall));
+
+      // Move the source for `methodCall` wholsale to be just after `enclosingTestCall`.
+      builder.addDeletion(range.deletionRange(methodCall));
+      builder.addSimpleInsertion(
+          enclosingTestCall.end, '$defaultEol$defaultEol$indent$source');
+    });
+  }
+}

pkgs/test_analyzer_plugin/lib/src/rules/non_nullable_is_not_null_rule.dart

@@ -0,0 +1,81 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analyzer/analysis_rule/analysis_rule.dart';
+import 'package:analyzer/analysis_rule/rule_context.dart';
+import 'package:analyzer/analysis_rule/rule_visitor_registry.dart';
+import 'package:analyzer/dart/ast/ast.dart';
+import 'package:analyzer/dart/ast/visitor.dart';
+import 'package:analyzer/dart/element/type_system.dart';
+import 'package:analyzer/error/error.dart';
+
+import '../utilities.dart';
+
+// TODO(srawlins): Several others; use same name or just different codes?
+// * `expect(null, isNotNull)` - always false
+// * `expect(null, isNull)`    - always true
+// * `expect(7, isNull)`       - always false
+class NonNullableIsNotNullRule extends MultiAnalysisRule {
+  static const LintCode nonNullableIsNotNullCode = LintCode(
+    'non_nullable_is_not_null',
+    'Do not check whether a non-nullable value isNotNull',
+    correctionMessage: 'Try changing the expectation, or removing it',
+  );
+
+  static const LintCode nonNullableIsNullCode = LintCode(
+    'non_nullable_is_null',
+    'Do not check whether a non-nullable value isNull',
+    correctionMessage: 'Try changing the expectation, or removing it',
+  );
+
+  NonNullableIsNotNullRule()
+      : super(
+          name: 'non_nullable_is_not_null',
+          description: "Non-nullable values will always pass an 'isNotNull' "
+              "expectation and never pass an 'isNull' expectation.",
+        );
+
+  @override
+  List<LintCode> get diagnosticCodes => [nonNullableIsNotNullCode];
+
+  @override
+  void registerNodeProcessors(
+      RuleVisitorRegistry registry, RuleContext context) {
+    var visitor = _Visitor(this, context.typeSystem);
+    registry.addMethodInvocation(this, visitor);
+  }
+}
+
+class _Visitor extends SimpleAstVisitor<void> {
+  final MultiAnalysisRule rule;
+
+  final TypeSystem typeSystem;
+
+  _Visitor(this.rule, this.typeSystem);
+
+  @override
+  void visitMethodInvocation(MethodInvocation node) {
+    if (!node.methodName.isExpect) {
+      return;
+    }
+
+    if (node.argumentList.arguments
+        case [var actual, SimpleIdentifier matcher]) {
+      var actualType = actual.staticType;
+      if (actualType == null) return;
+      if (typeSystem.isNonNullable(actualType)) {
+        if (matcher.isNotNull) {
+          // The actual value will always match this matcher.
+          rule.reportAtNode(matcher,
+              diagnosticCode:
+                  NonNullableIsNotNullRule.nonNullableIsNotNullCode);
+        } else if (matcher.isNull) {
+          // The actual value will never match this matcher.
+          rule.reportAtNode(matcher,
+              diagnosticCode: NonNullableIsNotNullRule.nonNullableIsNullCode);
+        }
+      }
+    }
+  }
+}

pkgs/test_analyzer_plugin/lib/src/rules/test_in_test_rule.dart

@@ -0,0 +1,55 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analyzer/analysis_rule/analysis_rule.dart';
+import 'package:analyzer/analysis_rule/rule_context.dart';
+import 'package:analyzer/analysis_rule/rule_visitor_registry.dart';
+import 'package:analyzer/dart/ast/ast.dart';
+import 'package:analyzer/dart/ast/visitor.dart';
+import 'package:analyzer/error/error.dart';
+
+import '../utilities.dart';
+
+class TestInTestRule extends AnalysisRule {
+  static const LintCode code = LintCode(
+    'test_in_test',
+    "Do not declare a 'test' or a 'group' inside a 'test'",
+    correctionMessage: "Try moving 'test' or 'group' outside of 'test'",
+  );
+
+  TestInTestRule()
+      : super(
+          name: 'test_in_test',
+          description:
+              'Tests and groups declared inside of a test are not properly '
+              'registered in the test framework.',
+        );
+
+  @override
+  LintCode get diagnosticCode => code;
+
+  @override
+  void registerNodeProcessors(
+      RuleVisitorRegistry registry, RuleContext context) {
+    var visitor = _Visitor(this);
+    registry.addMethodInvocation(this, visitor);
+  }
+}
+
+class _Visitor extends SimpleAstVisitor<void> {
+  final AnalysisRule rule;
+
+  _Visitor(this.rule);
+
+  @override
+  void visitMethodInvocation(MethodInvocation node) {
+    if (!node.methodName.isTest && !node.methodName.isGroup) {
+      return;
+    }
+    var enclosingTestCall = findEnclosingTestCall(node);
+    if (enclosingTestCall != null) {
+      rule.reportAtNode(node);
+    }
+  }
+}

pkgs/test_analyzer_plugin/lib/src/utilities.dart

@@ -0,0 +1,64 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analyzer/dart/ast/ast.dart';
+
+/// Finds an enclosing call to the 'test' function, if there is one.
+MethodInvocation? findEnclosingTestCall(MethodInvocation node) {
+  var ancestor = node.parent?.thisOrAncestorOfType<MethodInvocation>();
+  while (ancestor != null) {
+    if (ancestor.methodName.isTest) {
+      return ancestor;
+    }
+    ancestor = ancestor.parent?.thisOrAncestorOfType<MethodInvocation>();
+  }
+  return null;
+}
+
+extension SimpleIdentifierExtension on SimpleIdentifier {
+  /// Whether this identifier represents the 'test' function from the
+  /// 'test_core' package.
+  bool get isTest {
+    final element = this.element;
+    if (element == null) return false;
+    if (element.name != 'test') return false;
+    return element.library?.uri.path.startsWith('test_core/') ?? false;
+  }
+
+  /// Whether this identifier represents the 'group' function from the
+  /// 'test_core' package.
+  bool get isGroup {
+    final element = this.element;
+    if (element == null) return false;
+    if (element.name != 'group') return false;
+    return element.library?.uri.path.startsWith('test_core/') ?? false;
+  }
+
+  /// Whether this identifier represents the 'expect' function from the
+  /// 'matcher' package.
+  bool get isExpect {
+    final element = this.element;
+    if (element == null) return false;
+    if (element.name != 'expect') return false;
+    return element.library?.uri.path.startsWith('matcher/') ?? false;
+  }
+
+  /// Whether this identifier represents the 'isNotNull' constant from the
+  /// 'matcher' package.
+  bool get isNotNull {
+    final element = this.element;
+    if (element == null) return false;
+    if (element.name != 'isNotNull') return false;
+    return element.library?.uri.path.startsWith('matcher/') ?? false;
+  }
+
+  /// Whether this identifier represents the 'isNull' constant from the
+  /// 'matcher' package.
+  bool get isNull {
+    final element = this.element;
+    if (element == null) return false;
+    if (element.name != 'isNull') return false;
+    return element.library?.uri.path.startsWith('matcher/') ?? false;
+  }
+}

pkgs/test_analyzer_plugin/pubspec.yaml

@@ -0,0 +1,29 @@
+name: test_analyzer_plugin
+description: An analyzer plugin to report improper usage of the test package.
+version: 1.0.0
+publish_to: none
+
+environment:
+  sdk: '>=3.6.0 <4.0.0'
+
+dependencies:
+  analysis_server_plugin: ^0.2.2
+  analyzer: ^8.0.0
+  analyzer_plugin: ^0.13.6
+  analyzer_testing: ^0.1.2
+
+dev_dependencies:
+  test: any
+  test_reflective_loader: any
+
+  #dependency_overrides:
+  #  _fe_analyzer_shared:
+  #    path: /Users/srawlins/code/dart-sdk/sdk/pkg/_fe_analyzer_shared
+  #  analysis_server_plugin:
+  #    path: /Users/srawlins/code/dart-sdk/sdk/pkg/analysis_server_plugin
+  #  analyzer:
+  #    path: /Users/srawlins/code/dart-sdk/sdk/pkg/analyzer
+  #  analyzer_plugin:
+  #    path: /Users/srawlins/code/dart-sdk/sdk/pkg/analyzer_plugin
+  #  analyzer_testing:
+  #    path: /Users/srawlins/code/dart-sdk/sdk/pkg/analyzer_testing