identify failing tests

Author: slicklash

tests/test_java.py

@@ -118,7 +118,7 @@ def read_ap_version(self: AsyncProfiledProcess) -> str:
         return version
 
 
-def test_async_profiler_already_running(
+def xtest_async_profiler_already_running(
     application_pid: int,
     profiler_state: ProfilerState,
     assert_collapsed: AssertInCollapsed,
@@ -163,7 +163,7 @@ def test_async_profiler_already_running(
 
 
 @pytest.mark.parametrize("in_container", [True])
-def test_java_async_profiler_cpu_mode(
+def xtest_java_async_profiler_cpu_mode(
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
     profiler_state: ProfilerState,
@@ -186,7 +186,7 @@ def test_java_async_profiler_cpu_mode(
 
 @pytest.mark.parametrize("in_container", [True])
 @pytest.mark.parametrize("application_image_tag", ["musl"])
-def test_java_async_profiler_musl_and_cpu(
+def xtest_java_async_profiler_musl_and_cpu(
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
     profiler_state: ProfilerState,
@@ -215,7 +215,7 @@ def test_java_safemode_parameters(profiler_state: ProfilerState) -> None:
     assert "Java version checks are mandatory in --java-safemode" in str(excinfo.value)
 
 
-def test_java_safemode_version_check(
+def xtest_java_safemode_version_check(
     monkeypatch: MonkeyPatch,
     caplog: LogCaptureFixture,
     application_pid: int,
@@ -236,7 +236,7 @@ def test_java_safemode_version_check(
     assert log_record_extra(log_record)["jvm_version"] == repr(jvm_version)
 
 
-def test_java_safemode_build_number_check(
+def xtest_java_safemode_build_number_check(
     monkeypatch: MonkeyPatch,
     caplog: LogCaptureFixture,
     application_pid: int,
@@ -261,10 +261,10 @@ def test_java_safemode_build_number_check(
     [
         (False, (), False),  # default
         (False, ("-XX:ErrorFile=/tmp/my_custom_error_file.log",), False),  # custom error file
-        (True, (), False),  # containerized (other params are ignored)
+        #   (True, (), False),  # containerized (other params are ignored)
     ],
 )
-def test_hotspot_error_file(
+def xtest_hotspot_error_file(
     application_pid: int,
     monkeypatch: MonkeyPatch,
     caplog: LogCaptureFixture,
@@ -297,7 +297,7 @@ def start_async_profiler_and_crash(self: AsyncProfiledProcess, *args: Any, **kwa
     assert profiler._safemode_disable_reason is not None
 
 
-def test_disable_java_profiling(
+def xtest_disable_java_profiling(
     application_pid: int,
     monkeypatch: MonkeyPatch,
     caplog: LogCaptureFixture,
@@ -315,7 +315,7 @@ def test_disable_java_profiling(
     assert "Java profiling has been disabled, skipping profiling of all java process" in caplog.text
 
 
-def test_already_loaded_async_profiler_profiling_failure(
+def xtest_already_loaded_async_profiler_profiling_failure(
     monkeypatch: MonkeyPatch,
     caplog: LogCaptureFixture,
     application_pid: int,
@@ -339,7 +339,7 @@ def test_already_loaded_async_profiler_profiling_failure(
 # test only once; and don't test in container - as it will go down once we kill the Java app.
 @pytest.mark.parametrize("in_container", [False])
 @pytest.mark.parametrize("check_app_exited", [False])  # we're killing it, the exit check will raise.
-def test_async_profiler_output_written_upon_jvm_exit(
+def xtest_async_profiler_output_written_upon_jvm_exit(
     tmp_path_world_accessible: Path,
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
@@ -367,7 +367,7 @@ def delayed_kill() -> None:
 
 # test only once
 @pytest.mark.parametrize("in_container", [False])
-def test_async_profiler_stops_after_given_timeout(
+def xtest_async_profiler_stops_after_given_timeout(
     tmp_path_world_accessible: Path,
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
@@ -400,7 +400,7 @@ def test_async_profiler_stops_after_given_timeout(
 
 @pytest.mark.parametrize("in_container", [True])
 @pytest.mark.parametrize("application_image_tag,search_for", [("j9", "OpenJ9"), ("zing", "Zing")])
-def test_sanity_other_jvms(
+def xtest_sanity_other_jvms(
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
     search_for: str,
@@ -425,7 +425,7 @@ def test_sanity_other_jvms(
 
 @pytest.mark.parametrize("in_container", [True])
 @pytest.mark.parametrize("application_image_tag,search_for", [("eclipse-temurin-latest", "Temurin")])
-def test_sanity_latest_jvms(
+def xtest_sanity_latest_jvms(
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
     search_for: str,
@@ -462,7 +462,7 @@ def simulate_libjvm_delete(application_pid: int) -> None:
 
 # test only once. in a container, so that we don't mess up the environment :)
 @pytest.mark.parametrize("in_container", [True])
-def test_java_deleted_libjvm(
+def xtest_java_deleted_libjvm(
     application_pid: int,
     application_docker_container: Container,
     assert_collapsed: AssertInCollapsed,
@@ -501,7 +501,7 @@ def _filter_record(r: LogRecord) -> bool:
         pytest.param("ro", [docker.types.Mount(target="/tmpfs", source="", type="tmpfs", read_only=True)], id="ro"),
     ],
 )
-def test_java_noexec_or_ro_dirs(
+def xtest_java_noexec_or_ro_dirs(
     tmp_path_world_accessible: Path,  # will be used by AP for logs & outputs
     application_pid: int,
     extra_application_docker_mounts: List[docker.types.Mount],
@@ -569,7 +569,7 @@ def test_java_noexec_or_ro_dirs(
 
 
 @pytest.mark.parametrize("in_container", [True])
-def test_java_symlinks_in_paths(
+def xtest_java_symlinks_in_paths(
     application_pid: int,
     application_docker_container: Container,
     assert_collapsed: AssertInCollapsed,
@@ -616,7 +616,7 @@ def test_java_symlinks_in_paths(
 
 
 @pytest.mark.parametrize("in_container", [True])  # only in container is enough
-def test_java_appid_and_metadata_before_process_exits(
+def xtest_java_appid_and_metadata_before_process_exits(
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
     monkeypatch: MonkeyPatch,
@@ -657,7 +657,7 @@ def start_async_profiler_and_interrupt(self: AsyncProfiledProcess, *args: Any, *
 
 
 @pytest.mark.parametrize("in_container", [True])  # only in container is enough
-def test_java_attach_socket_missing(
+def xtest_java_attach_socket_missing(
     application_pid: int,
     profiler_state: ProfilerState,
 ) -> None:
@@ -680,7 +680,7 @@ def test_java_attach_socket_missing(
 
 # we know what messages to expect when in container, not on the host Java
 @pytest.mark.parametrize("in_container", [True])
-def test_java_jattach_async_profiler_log_output(
+def xtest_java_jattach_async_profiler_log_output(
     application_pid: int,
     caplog: LogCaptureFixture,
     profiler_state: ProfilerState,
@@ -817,7 +817,7 @@ def test_non_java_basename_version(
 
 @pytest.mark.parametrize("in_container", [True])
 @pytest.mark.parametrize("insert_dso_name", [False, True])
-def test_dso_name_in_ap_profile(
+def xtest_dso_name_in_ap_profile(
     application_pid: int,
     insert_dso_name: bool,
     profiler_state: ProfilerState,
@@ -836,7 +836,7 @@ def test_dso_name_in_ap_profile(
 @pytest.mark.parametrize("in_container", [True])
 @pytest.mark.parametrize("insert_dso_name", [False, True])
 @pytest.mark.parametrize("libc_pattern", [r"(^|;)\(/.*/libc-.*\.so\)($|;)"])
-def test_handling_missing_symbol_in_profile(
+def xtest_handling_missing_symbol_in_profile(
     application_pid: int,
     insert_dso_name: bool,
     libc_pattern: str,
@@ -852,7 +852,7 @@ def test_handling_missing_symbol_in_profile(
 
 
 @pytest.mark.parametrize("in_container", [True])
-def test_meminfo_logged(
+def xtest_meminfo_logged(
     application_pid: int,
     caplog: LogCaptureFixture,
     profiler_state: ProfilerState,
@@ -869,7 +869,7 @@ def test_meminfo_logged(
 
 # test that java frames include no semicolon but use a pipe '|' character instead, as implemented by AP
 @pytest.mark.parametrize("in_container", [True])
-def test_java_frames_include_no_semicolons(
+def xtest_java_frames_include_no_semicolons(
     application_pid: int,
     profiler_state: ProfilerState,
 ) -> None:
@@ -896,7 +896,7 @@ def test_java_frames_include_no_semicolons(
 
 # test that async profiler doesn't print anything to applications stdout, stderr streams
 @pytest.mark.parametrize("in_container", [True])
-def test_no_stray_output_in_stdout_stderr(
+def xtest_no_stray_output_in_stdout_stderr(
     application_pid: int,
     application_docker_container: Container,
     monkeypatch: MonkeyPatch,
@@ -1094,7 +1094,7 @@ def flush_output_and_stop_async_profiler(self: AsyncProfiledProcess, *args: Any,
         ),
     ],
 )
-def test_collect_default_jvm_flags(
+def xtest_collect_default_jvm_flags(
     profiler_state: ProfilerState,
     tmp_path: Path,
     application_pid: int,
@@ -1178,7 +1178,7 @@ def test_collect_default_jvm_flags(
         ),
     ],
 )
-def test_collect_cmdline_and_env_jvm_flags(
+def xtest_collect_cmdline_and_env_jvm_flags(
     docker_client: DockerClient,
     application_docker_image: Image,
     assert_collapsed: AssertInCollapsed,
@@ -1218,7 +1218,7 @@ def test_collect_cmdline_and_env_jvm_flags(
 @pytest.mark.parametrize("java_cli_flags", ["-XX:MinHeapFreeRatio=5 -XX:MaxHeapFreeRatio=95"])
 @pytest.mark.parametrize("in_container", [True])
 @pytest.mark.parametrize("expected_flags", [[]])
-def test_collect_flags_unsupported_filtered_out(
+def xtest_collect_flags_unsupported_filtered_out(
     docker_client: DockerClient,
     application_docker_image: Image,
     assert_collapsed: AssertInCollapsed,
@@ -1260,7 +1260,7 @@ def test_collect_flags_unsupported_filtered_out(
 
 @pytest.mark.parametrize("in_container", [True])
 @pytest.mark.parametrize("expected_flags", [[]])
-def test_collect_none_jvm_flags(
+def xtest_collect_none_jvm_flags(
     profiler_state: ProfilerState,
     tmp_path: Path,
     application_pid: int,
@@ -1272,7 +1272,7 @@ def test_collect_none_jvm_flags(
 
 @pytest.mark.parametrize("in_container", [True])
 @pytest.mark.parametrize("include_mmm", [True, False])
-def test_including_method_modifiers(
+def xtest_including_method_modifiers(
     application_pid: int,
     profiler_state: ProfilerState,
     include_mmm: bool,
@@ -1287,7 +1287,7 @@ def test_including_method_modifiers(
 
 @pytest.mark.parametrize("java_line_numbers", ["none", "line-of-function"])
 @pytest.mark.parametrize("in_container", [True])
-def test_including_line_numbers(
+def xtest_including_line_numbers(
     application_pid: int,
     profiler_state: ProfilerState,
     java_line_numbers: str,

tests/test_profiling_mode.py

@@ -56,7 +56,7 @@ def test_sanity(
         ("java", "ap", True, "java.lang.String[]"),
     ],
 )
-def test_allocation_being_profiled(
+def xtest_allocation_being_profiled(
     application_docker_container: Container,
     docker_client: DockerClient,
     gprofiler_docker_image: Image,

tests/test_python.py

@@ -39,7 +39,7 @@ def runtime() -> str:
 
 @pytest.mark.parametrize("in_container", [True])
 @pytest.mark.parametrize("application_image_tag", ["libpython"])
-def test_python_select_by_libpython(
+def xtest_python_select_by_libpython(
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
     profiler_state: ProfilerState,
@@ -83,7 +83,7 @@ def test_python_select_by_libpython(
     ],
 )
 @pytest.mark.parametrize("profiler_type", ["py-spy", "pyperf"])
-def test_python_matrix(
+def xtest_python_matrix(
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
     profiler_type: str,

tests/test_sanity.py

@@ -50,7 +50,7 @@
 
 
 @pytest.mark.parametrize("runtime", ["java"])
-def test_java_from_host(
+def xtest_java_from_host(
     tmp_path_world_accessible: Path,
     application_pid: int,
     assert_app_id: Callable,
@@ -68,7 +68,7 @@ def test_java_from_host(
 
 
 @pytest.mark.parametrize("runtime", ["python"])
-def test_pyspy(
+def xtest_pyspy(
     application_pid: int,
     assert_collapsed: AssertInCollapsed,
     assert_app_id: Callable,