Add remote symbolication support with build-id and PC offset (#324)

Author: jbachorik

.claude/commands/build-and-summarize

@@ -72,4 +72,4 @@ echo
 echo "Delegating to gradle-logs-analyst agent…"
 # If your CLI supports non-streaming, set it here to avoid verbose output.
 # Example (uncomment if supported): export CLAUDE_NO_STREAM=1
-claude "Act as the gradle-logs-analyst agent to parse the build log at: $LOG. Generate the required Gradle summary artifacts as specified in the gradle-logs-analyst agent definition."
+claude "Act as the gradle-logs-analyst agent to parse the build log at: $LOG. Generate the required Gradle summary artifacts as specified in the gradle-logs-analyst agent definition."

.claude/commands/build-and-summarize.md

@@ -1,7 +1,11 @@
 # build-and-summarize
 
-Runs `./gradlew` with full output captured to a timestamped log, shows minimal live progress (task starts + final build/test summary), then asks the `gradle-logs-analyst` agent to produce structured artifacts from the log.
+Execute the bash script `~/.claude/commands/build-and-summarize` with all provided arguments.
 
-## Usage
-```bash
-./.claude/commands/build-and-summarize [<gradle-args>...]
+This script will:
+- Run `./gradlew` with the specified arguments (defaults to 'build' if none provided)
+- Capture full output to a timestamped log in `build/logs/`
+- Show minimal live progress in the console
+- Delegate to the `gradle-logs-analyst` agent for structured analysis
+
+Pass through all arguments exactly as provided by the user.

README.md

@@ -402,6 +402,25 @@ Improved thread-local storage initialization to prevent race conditions:
 
 These architectural improvements focus on eliminating race conditions, improving performance in high-throughput scenarios, and providing better debugging capabilities for the native profiling engine.
 
+### Remote Symbolication Support (2025)
+
+Added support for remote symbolication to enable offloading symbol resolution from the agent to backend services:
+
+- **Build-ID extraction**: Automatically extracts GNU build-id from ELF binaries on Linux
+- **Raw addressing information**: Stores build-id and PC offset instead of resolved symbol names
+- **Remote symbolication mode**: Enable with `remotesym=true` profiler argument
+- **JFR integration**: Remote frames serialized with build-id and offset for backend resolution
+- **Zero encoding overhead**: Uses dedicated frame type (FRAME_NATIVE_REMOTE) for efficient serialization
+
+**Benefits**:
+- Reduces agent overhead by eliminating local symbol resolution
+- Enables centralized symbol resolution with better caching
+- Supports scenarios where debug symbols are not available locally
+
+**Key files**: `elfBuildId.h`, `elfBuildId.cpp`, `profiler.cpp`, `flightRecorder.cpp`
+
+For detailed documentation, see [doc/RemoteSymbolication.md](doc/RemoteSymbolication.md).
+
 ## Contributing
 1. Fork the repository
 2. Create a feature branch

ddprof-lib/src/main/cpp/arguments.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright 2017 Andrei Pangin
+ * Copyright 2026, Datadog, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -88,7 +89,10 @@ static const Multiplier UNIVERSAL[] = {
 //                          samples
 //     generations        - track surviving generations
 //     lightweight[=BOOL] - enable lightweight profiling - events without
-//     stacktraces (default: true) jfr                - dump events in Java
+//     stacktraces (default: true)
+//     remotesym[=BOOL]   - enable remote symbolication for native frames
+//                          (stores build-id and PC offset instead of symbol names)
+//     jfr                - dump events in Java
 //     Flight Recorder format interval=N         - sampling interval in ns
 //     (default: 10'000'000, i.e. 10 ms) jstackdepth=N      - maximum Java stack
 //     depth (default: 2048) safemode=BITS      - disable stack recovery
@@ -339,16 +343,35 @@ Error Arguments::parse(const char *args) {
         _enable_method_cleanup = true;
       }
 
-      CASE("wallsampler")
+      CASE("remotesym")
       if (value != NULL) {
         switch (value[0]) {
-        case 'j':
-          _wallclock_sampler = JVMTI;
+        case 'n': // no
+        case 'f': // false
+        case '0': // 0
+          _remote_symbolication = false;
           break;
-        case 'a':
+        case 'y': // yes
+        case 't': // true
+        case '1': // 1
         default:
-          _wallclock_sampler = ASGCT;
+          _remote_symbolication = true;
         }
+      } else {
+        // No value means enable
+        _remote_symbolication = true;
+      }
+
+      CASE("wallsampler")
+      if (value != NULL) {
+          switch (value[0]) {
+              case 'j':
+                  _wallclock_sampler = JVMTI;
+                  break;
+              case 'a':
+              default:
+                  _wallclock_sampler = ASGCT;
+          }
       }
 
       DEFAULT()

ddprof-lib/src/main/cpp/arguments.h

@@ -188,6 +188,7 @@ class Arguments {
   std::vector<std::string> _context_attributes;
   bool _lightweight;
   bool _enable_method_cleanup;
+  bool _remote_symbolication;  // Enable remote symbolication for native frames
 
   Arguments(bool persistent = false)
       : _buf(NULL),
@@ -221,7 +222,8 @@ class Arguments {
         _context_attributes({}),
         _wallclock_sampler(ASGCT),
         _lightweight(false),
-        _enable_method_cleanup(true) {}
+        _enable_method_cleanup(true),
+        _remote_symbolication(false) {}
 
   ~Arguments();
 

ddprof-lib/src/main/cpp/codeCache.cpp

@@ -37,6 +37,11 @@ CodeCache::CodeCache(const char *name, short lib_index,
   _plt_size = 0;
   _debug_symbols = false;
 
+  // Initialize build-id fields
+  _build_id = nullptr;
+  _build_id_len = 0;
+  _load_bias = 0;
+
   memset(_imports, 0, sizeof(_imports));
   _imports_patchable = imports_patchable;
 
@@ -48,16 +53,33 @@ CodeCache::CodeCache(const char *name, short lib_index,
   _blobs = new CodeBlob[_capacity];
 }
 
-CodeCache::CodeCache(const CodeCache &other) {
+void CodeCache::copyFrom(const CodeCache& other) {
   _name = NativeFunc::create(other._name, -1);
   _lib_index = other._lib_index;
   _min_address = other._min_address;
   _max_address = other._max_address;
   _text_base = other._text_base;
+  _image_base = other._image_base;
 
-  _imports_patchable = other._imports_patchable;
   _plt_offset = other._plt_offset;
   _plt_size = other._plt_size;
+  _debug_symbols = other._debug_symbols;
+
+  // Copy build-id information
+  _build_id_len = other._build_id_len;
+  if (other._build_id != nullptr && other._build_id_len > 0) {
+    size_t hex_str_len = strlen(other._build_id);
+    _build_id = static_cast<char*>(malloc(hex_str_len + 1));
+    if (_build_id != nullptr) {
+      strcpy(_build_id, other._build_id);
+    }
+  } else {
+    _build_id = nullptr;
+  }
+  _load_bias = other._load_bias;
+
+  memset(_imports, 0, sizeof(_imports));
+  _imports_patchable = other._imports_patchable;
 
   _dwarf_table_length = other._dwarf_table_length;
   _dwarf_table = new FrameDesc[_dwarf_table_length];
@@ -70,37 +92,23 @@ CodeCache::CodeCache(const CodeCache &other) {
   memcpy(_blobs, other._blobs, _count * sizeof(CodeBlob));
 }
 
+CodeCache::CodeCache(const CodeCache &other) {
+  copyFrom(other);
+}
+
 CodeCache &CodeCache::operator=(const CodeCache &other) {
   if (&other == this) {
     return *this;
-  } else {
-    delete _name;
-    delete _dwarf_table;
-    delete _blobs;
-
-    _name = NativeFunc::create(other._name, -1);
-    _lib_index = other._lib_index;
-    _min_address = other._min_address;
-    _max_address = other._max_address;
-    _text_base = other._text_base;
-
-    _imports_patchable = other._imports_patchable;
-
-    _plt_offset = other._plt_offset;
-    _plt_size = other._plt_size;
+  }
 
-    _dwarf_table_length = other._dwarf_table_length;
-    _dwarf_table = new FrameDesc[_dwarf_table_length];
-    memcpy(_dwarf_table, other._dwarf_table,
-           _dwarf_table_length * sizeof(FrameDesc));
+  NativeFunc::destroy(_name);
+  delete[] _dwarf_table;
+  delete[] _blobs;
+  free(_build_id);
 
-    _capacity = other._capacity;
-    _count = other._count;
-    _blobs = new CodeBlob[_capacity];
-    memcpy(_blobs, other._blobs, _count * sizeof(CodeBlob));
+  copyFrom(other);
 
-    return *this;
-  }
+  return *this;
 }
 
 CodeCache::~CodeCache() {
@@ -109,7 +117,8 @@ CodeCache::~CodeCache() {
   }
   NativeFunc::destroy(_name);
   delete[] _blobs;
-  delete _dwarf_table;
+  delete[] _dwarf_table;
+  free(_build_id);  // Free build-id memory
 }
 
 void CodeCache::expand() {
@@ -387,3 +396,24 @@ FrameDesc CodeCache::findFrameDesc(const void *pc) {
     return FrameDesc::default_frame;
   }
 }
+
+void CodeCache::setBuildId(const char* build_id, size_t build_id_len) {
+  // Free existing build-id if any
+  free(_build_id);
+  _build_id = nullptr;
+  _build_id_len = 0;
+
+  if (build_id != nullptr && build_id_len > 0) {
+    // build_id is a hex string, allocate based on actual string length
+    size_t hex_str_len = strlen(build_id);
+    _build_id = static_cast<char*>(malloc(hex_str_len + 1));
+
+    if (_build_id != nullptr) {
+      // Copy the hex string
+      strcpy(_build_id, build_id);
+      // Store the original byte length (not hex string length)
+      _build_id_len = build_id_len;
+    }
+  }
+}
+

ddprof-lib/src/main/cpp/codeCache.h

@@ -116,6 +116,11 @@ class CodeCache {
   unsigned int _plt_offset;
   unsigned int _plt_size;
 
+  // Build-ID and load bias for remote symbolication
+  char *_build_id;           // GNU build-id (hex string, null if not available)
+  size_t _build_id_len;      // Build-id length in bytes (raw, not hex string length)
+  uintptr_t _load_bias;      // Load bias (image_base - file_base address)
+
   void **_imports[NUM_IMPORTS][NUM_IMPORT_TYPES];
   bool _imports_patchable;
   bool _debug_symbols;
@@ -130,6 +135,7 @@ class CodeCache {
   void expand();
   void makeImportsPatchable();
   void saveImport(ImportId id, void** entry);
+  void copyFrom(const CodeCache& other);
 
 public:
   explicit CodeCache(const char *name, short lib_index = -1,
@@ -169,10 +175,30 @@ class CodeCache {
 
   void setDebugSymbols(bool debug_symbols) { _debug_symbols = debug_symbols; }
 
+  // Build-ID and remote symbolication support
+  const char* buildId() const { return _build_id; }
+  size_t buildIdLen() const { return _build_id_len; }
+  bool hasBuildId() const { return _build_id != nullptr; }
+  uintptr_t loadBias() const { return _load_bias; }
+  short libIndex() const { return _lib_index; }
+
+  // Sets the build-id (hex string) and stores the original byte length
+  // build_id: null-terminated hex string (e.g., "abc123..." for 40-char string)
+  // build_id_len: original byte length before hex conversion (e.g., 20 bytes)
+  void setBuildId(const char* build_id, size_t build_id_len);
+  void setLoadBias(uintptr_t load_bias) { _load_bias = load_bias; }
+
   void add(const void *start, int length, const char *name,
            bool update_bounds = false);
   void updateBounds(const void *start, const void *end);
   void sort();
+
+  /**
+   * Mark symbols matching the predicate with the given mark value.
+   *
+   * This is called during profiler initialization to mark JVM internal functions
+   * (MARK_VM_RUNTIME, MARK_INTERPRETER, MARK_COMPILER_ENTRY, MARK_ASYNC_PROFILER).
+   */
   template <typename NamePredicate>
   inline void mark(NamePredicate predicate, char value) {
       for (int i = 0; i < _count; i++) {
@@ -225,7 +251,7 @@ class CodeCacheArray {
     memset(_libs, 0, MAX_NATIVE_LIBS * sizeof(CodeCache *));
   }
 
-  CodeCache *operator[](int index) { return _libs[index]; }
+  CodeCache *operator[](int index) const { return __atomic_load_n(&_libs[index], __ATOMIC_ACQUIRE); }
 
   int count() const { return __atomic_load_n(&_count, __ATOMIC_RELAXED); }
 
@@ -247,7 +273,7 @@ class CodeCacheArray {
     return lib;
   }
 
-  size_t memoryUsage() {
+  size_t memoryUsage() const {
     return __atomic_load_n(&_used_memory, __ATOMIC_RELAXED);
   }
 };

ddprof-lib/src/main/cpp/counters.h

@@ -64,7 +64,10 @@
   X(UNWINDING_TIME_ASYNC, "unwinding_ticks_async")                             \
   X(UNWINDING_TIME_JVMTI, "unwinding_ticks_jvmti")                             \
   X(CALLTRACE_STORAGE_DROPPED, "calltrace_storage_dropped_traces")             \
-  X(LINE_NUMBER_TABLES, "line_number_tables")
+  X(LINE_NUMBER_TABLES, "line_number_tables")                                  \
+  X(REMOTE_SYMBOLICATION_FRAMES, "remote_symbolication_frames")                \
+  X(REMOTE_SYMBOLICATION_LIBS_WITH_BUILD_ID, "remote_symbolication_libs_with_build_id") \
+  X(REMOTE_SYMBOLICATION_BUILD_ID_CACHE_HITS, "remote_symbolication_build_id_cache_hits")
 #define X_ENUM(a, b) a,
 typedef enum CounterId : int {
   DD_COUNTER_TABLE(X_ENUM) DD_NUM_COUNTERS