From a0bded6199fb09b39585be2b225f7b9dfcedfe2b Mon Sep 17 00:00:00 2001 From: Pxl Date: Thu, 23 Oct 2025 20:20:53 +0800 Subject: [PATCH] [Bug](memtracker) fix wrong memtracker scope on PipelineTask destructor (#57276) http://43.132.222.7:8111/buildConfiguration/Doris_DorisRegression_P0Regression/819345?buildTab=log&linesState=452&logView=flowAware&focusLine=140513 ```cpp INFO: java_cmd /usr/lib/jvm/java-17-openjdk-amd64/bin/java INFO: jdk_version 17 StdoutLogger 2025-10-23 12:22:26,738 Start time: Thu 23 Oct 2025 12:22:26 PM CST INFO: java_cmd /usr/lib/jvm/java-17-openjdk-amd64/bin/java INFO: jdk_version 17 OpenJDK 64-Bit Server VM warning: Option CriticalJNINatives was deprecated in version 16.0 and will likely be removed in a future release. WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance. start BE in local mode F20251023 12:33:34.873679 673898 thread_mem_tracker_mgr.h:141] Check failed: doris::k_doris_exit || !doris::config::enable_memory_orphan_check || limiter_mem_tracker()->label() != "Orphan" The ThreadContext of the current thread not attach a valid MemoryTracker. after the thread is started, the ResourceContext in SCOPED_ATTACH_TASK macro should contain a valid MemoryTracker, or a valid MemoryTracker should be passed in later using SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER macro. *** Check failure stack trace: *** @ 0x56146ef6d2cf google::LogMessage::SendToLog() @ 0x56146ef638e0 google::LogMessage::Flush() @ 0x56146ef66fd9 google::LogMessageFatal::~LogMessageFatal() @ 0x5614564f4f76 doris::ThreadMemTrackerMgr::memory_orphan_check() @ 0x5614564f3dbf doris::ThreadMemTrackerMgr::consume() @ 0x561456515985 doris::vectorized::PODArrayBase<>::dealloc() @ 0x561456529239 doris::vectorized::ColumnVector<>::~ColumnVector() @ 0x5614566653c7 std::vector<>::~vector() @ 0x56145877b074 std::default_delete<>::operator()() @ 0x56146eb32053 doris::pipeline::PipelineTask::~PipelineTask() @ 0x561456152d78 std::_Sp_counted_base<>::_M_release_last_use() @ 0x56146eb9cac5 doris::pipeline::TaskScheduler::_do_work() @ 0x56145ab82e73 doris::ThreadPool::dispatch_thread() @ 0x56145ab5fd37 doris::Thread::supervise_thread() @ 0x5614560e8d27 asan_thread_start() @ 0x7f276c7a9609 start_thread @ 0x7f276c6bc133 clone @ (nil) (unknown) *** Query id: 0-0 *** *** is nereids: 1 *** *** tablet id: 0 *** *** Aborted at 1761194015 (unix time) try "date -d @1761194015" if you are using GNU date *** *** Current BE git commitID: a378de5546 *** *** SIGABRT unknown detail explain (@0xa4265) received by PID 672357 (TID 673898 OR 0x7b2429024700) from PID 672357; stack trace: *** 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /root/doris/be/src/common/signal_handler.h:420 1# 0x00007F276C7B5420 in /lib/x86_64-linux-gnu/libpthread.so.0 2# raise at ../sysdeps/unix/sysv/linux/raise.c:51 3# abort at /build/glibc-SzIz7B/glibc-2.31/stdlib/abort.c:81 4# 0x000056146EF72305 in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be 5# 0x000056146EF63BBA in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be 6# google::LogMessage::SendToLog() in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be 7# google::LogMessage::Flush() in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be 8# google::LogMessageFatal::~LogMessageFatal() in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be 9# doris::ThreadMemTrackerMgr::memory_orphan_check() at /root/doris/be/src/runtime/memory/thread_mem_tracker_mgr.h:140 10# doris::ThreadMemTrackerMgr::consume(long) at /root/doris/be/src/runtime/memory/thread_mem_tracker_mgr.h:214 11# doris::vectorized::PODArrayBase<8ul, 4096ul, doris::Allocator, 16ul, 15ul>::dealloc() in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be 12# doris::vectorized::ColumnVector<(doris::PrimitiveType)6>::~ColumnVector() at /root/doris/be/src/runtime/primitive_type.h:96 13# std::vector >::~vector() at /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/stl_vector.h:802 14# std::default_delete::operator()(doris::vectorized::Block*) const at /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/unique_ptr.h:93 15# doris::pipeline::PipelineTask::~PipelineTask() in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be 16# std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release_last_use() in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be 17# doris::pipeline::TaskScheduler::_do_work(int) at /root/doris/be/src/pipeline/task_scheduler.cpp:172 18# doris::ThreadPool::dispatch_thread() at /root/doris/be/src/util/threadpool.cpp:621 19# doris::Thread::supervise_thread(void*) at /root/doris/be/src/util/thread.cpp:461 20# asan_thread_start(void*) in /mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be 21# start_thread at /build/glibc-SzIz7B/glibc-2.31/nptl/pthread_create.c:478 22# __clone at ../sysdeps/unix/sysv/linux/x86_64/clone.S:97 INFO: java_cmd /usr/lib/jvm/java-17-openjdk-amd64/bin/java INFO: jdk_version 17 StdoutLogger 2025-10-23 12:46:12,802 Start time: Thu 23 Oct 2025 12:46:12 PM CST INFO: java_cmd /usr/lib/jvm/java-17-openjdk-amd64/bin/java INFO: jdk_version 17 OpenJDK 64-Bit Server VM warning: Option CriticalJNINatives was deprecated in version 16.0 and will likely be removed in a future release. WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance. start BE in local mode ----------------------------------------------------- Suppressions used: count bytes template 6268 1794634 libjvm 288 32256 libzip ---------------------------------------------------- ``` --- be/src/pipeline/pipeline_task.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp index 7fe201f3802347..0f3309da39583e 100644 --- a/be/src/pipeline/pipeline_task.cpp +++ b/be/src/pipeline/pipeline_task.cpp @@ -96,6 +96,15 @@ PipelineTask::PipelineTask(PipelinePtr& pipeline, uint32_t task_id, RuntimeState } PipelineTask::~PipelineTask() { + auto reset_member = [&]() { + _shared_state_map.clear(); + _sink_shared_state.reset(); + _op_shared_states.clear(); + _sink.reset(); + _operators.clear(); + _block.reset(); + _pipeline.reset(); + }; // PipelineTask is also hold by task queue( https://github.com/apache/doris/pull/49753), // so that it maybe the last one to be destructed. // But pipeline task hold some objects, like operators, shared state, etc. So that should release @@ -103,15 +112,11 @@ PipelineTask::~PipelineTask() { #ifndef BE_TEST if (_query_mem_tracker) { SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_query_mem_tracker); + reset_member(); + return; } #endif - _shared_state_map.clear(); - _sink_shared_state.reset(); - _op_shared_states.clear(); - _sink.reset(); - _operators.clear(); - _block.reset(); - _pipeline.reset(); + reset_member(); } Status PipelineTask::prepare(const std::vector& scan_range, const int sender_id,