[ABLD-430] Add options to dda test to use bazel for migrated tests

.bazelrc

@@ -17,9 +17,16 @@ common --@rules_python//python/config_settings:bootstrap_impl=script # https://g
 common --check_direct_dependencies=error # Escalate any bypassed `bazel_dep` to a resolution failure
 common --enable_platform_specific_config # Supported OS identifiers are linux, macos, windows, freebsd, and openbsd
 common --experimental_disk_cache_gc_max_size=5G # Cap applied whenever --disk_cache is also set, no-op otherwise
+common --experimental_strict_repo_env # Do not leak uncontrolled environment variables into repository rules
 common --experimental_ui_max_stdouterr_bytes=1073741819 # why?
 common --http_timeout_scaling=3.0 # At least one attempt reaches 30s (3,6,12,24,30,30,30,30) instead of only 10s (1,2,4,8,10,10,10,10)
-common --incompatible_strict_action_env # Do not leak local environment variables into the build context
+common --repo_env=DEPLOY_AGENT # Keep in sync with env_vars in MODULE.bazel
+common --repo_env=FORCED_PACKAGE_COMPRESSION_LEVEL # Keep in sync with env_vars in MODULE.bazel
+common --repo_env=GOCACHE # https://pkg.go.dev/cmd/go#hdr-Build_and_test_caching
+common --repo_env=GOMODCACHE # https://wiki.archlinux.org/title/XDG_Base_Directory#Partial
+common --repo_env=PACKAGE_VERSION # Keep in sync with env_vars in MODULE.bazel
+common --repo_env=SIGN_MAC # Keep in sync with env_vars in MODULE.bazel
+common --repo_env=XDG_CACHE_HOME # https://wiki.archlinux.org/title/XDG_Base_Directory
 common --skip_incompatible_explicit_targets # Let target_compatible_with skip rather than fail
 common --test_output=errors # Print test errors to console output instead of only capturing them in buried test.log
 common --verbose_failures
@@ -44,10 +51,12 @@ common:windows --strategy=standalone # Valid values are: [dynamic_worker, standa
 # rules_python 1.9.0 transitions enable_runfiles to `true` for every py_ target on Windows. Pre-setting it here makes
 # that transition a no-op, so Bazel deduplicates python_win and avoids 2 concurrent MSBuild racing on shared resources.
 common:windows --enable_runfiles
-# Neither repo_env nor shell_executable affect action keys. It can be both an advantage and a disadvantage.
-# For instance, if we need to have some special behavior in bash we can add it, however, in some cases
-# it may lead to cache poisoning.
 common:windows --repo_env=BAZEL_SH=C:/tools/msys64/usr/bin/bash.exe # for https://github.com/bazelbuild/bazel/pull/26927
+common:windows --repo_env=PIP_CACHE_DIR # https://pip.pypa.io/en/stable/topics/caching/#default-paths
+common:windows --repo_env=SYSTEMDRIVE # needed by vswhere to locate the VS installer instance database
+common:windows --repo_env=SYSTEMROOT # used by COM to load system DLLs, needed by vswhere
+common:windows --repo_env=USERPROFILE # used by MSYS2 bash to emulate HOME, needed by git to fetch repositories
+common:windows --repo_env=VSTUDIO_ROOT # visual_studio(path_variable) in MODULE.bazel
 common:windows --shell_executable=C:/tools/msys64/usr/bin/bash.exe
 
 # Remote cache config --------------------------------------------------------------------------------------------------

.github/CODEOWNERS

@@ -636,7 +636,6 @@
 /pkg/logs/client                        @DataDog/agent-log-pipelines
 /pkg/logs/diagnostic                    @DataDog/agent-log-pipelines
 /pkg/logs/message                       @DataDog/agent-log-pipelines
-/pkg/logs/pipeline                      @DataDog/agent-log-pipelines
 /pkg/logs/processor                     @DataDog/agent-log-pipelines
 /pkg/logs/sender                        @DataDog/agent-log-pipelines
 /pkg/process/                           @DataDog/container-experiences

.gitignore

@@ -88,7 +88,7 @@ auth_token
 /test/e2e/scripts/setup-instance/ignition.json
 /test/e2e/containers/fake_datadog/venv/
 /e2e-output
-/pkg/logs/pipeline/registry.json
+/comp/logs-library/pipeline/registry.json
 
 # local copy of device signing cert
 platform.pk8
@@ -215,7 +215,7 @@ test/new-e2e/system-probe/test-runner/test-runner
 test/new-e2e/start-microvms
 
 # trace-agent generates install.json files during tests
-pkg/serverless/trace/install.json
+pkg/serverless/trace/**/install.json
 
 # Files used for eBPF complexity analysis
 ebpf-calculator

.golangci.yml

@@ -151,6 +151,8 @@ linters:
             - "!**/comp/forwarder/defaultforwarder/blocked_endpoints.go"
             - "!**/comp/forwarder/defaultforwarder/default_forwarder.go"
             - "!**/comp/forwarder/eventplatform/eventplatformimpl/epforwarder.go"
+            - "!**/comp/logs-library/pipeline/provider.go"
+            - "!**/comp/logs-library/pipeline/provider_test.go"
             - "!**/comp/logs/agent/agentimpl/agent_test.go"
             - "!**/comp/logs/agent/agentimpl/analyze_logs_init.go"
             - "!**/comp/logs/agent/agentimpl/serverless.go"

BUILD.bazel

@@ -45,17 +45,17 @@ ALL_BUILD_TAGS = [
 # gazelle:exclude comp/autoscaling
 # gazelle:exclude comp/checks
 # gazelle:exclude comp/collector
-# gazelle:exclude comp/connectivitychecker
+# gazelle:exclude comp/connectivitychecker/checker
+# gazelle:exclude comp/connectivitychecker/fx
+# gazelle:exclude comp/connectivitychecker/impl
 # gazelle:exclude comp/core/autodiscovery
 # gazelle:exclude comp/core/agenttelemetry/fx
 # gazelle:exclude comp/core/agenttelemetry/impl
-# gazelle:exclude comp/core/config
 # gazelle:exclude comp/core/configstream
 # gazelle:exclude comp/core/configsync
 # gazelle:exclude comp/core/diagnose
 # gazelle:exclude comp/core/flare/flareimpl
 # gazelle:exclude comp/core/flare/helpers
-# gazelle:exclude comp/core/fxinstrumentation
 # gazelle:exclude comp/core/gui
 # gazelle:exclude comp/core/healthprobe
 # gazelle:exclude comp/core/hostname/hostnameimpl
@@ -67,22 +67,18 @@ ALL_BUILD_TAGS = [
 # gazelle:exclude comp/core/ipc/impl
 # gazelle:exclude comp/core/ipc/impl-none
 # gazelle:exclude comp/core/ipc/mock
-# gazelle:exclude comp/core/log
 # gazelle:exclude comp/core/log/fx
 # gazelle:exclude comp/core/log/fx-systemprobe
 # gazelle:exclude comp/core/log/fx-trace
 # gazelle:exclude comp/core/log/impl
 # gazelle:exclude comp/core/log/impl-systemprobe
 # gazelle:exclude comp/core/log/impl-trace
 # gazelle:exclude comp/core/log/rust
-# gazelle:exclude comp/core/lsof
-# gazelle:exclude comp/core/pid
 # gazelle:exclude comp/core/profiler
 # gazelle:exclude comp/core/remoteagent
 # gazelle:exclude comp/core/remoteagentregistry
 # gazelle:exclude comp/core/secrets/fx
 # gazelle:exclude comp/core/secrets/fx-noop
-# gazelle:exclude comp/core/secrets/impl
 # gazelle:exclude comp/core/settings
 # gazelle:exclude comp/core/status/statusimpl
 # gazelle:exclude comp/core/sysprobeconfig
@@ -104,21 +100,26 @@ ALL_BUILD_TAGS = [
 # gazelle:exclude comp/core/tagger/k8s_metadata
 # gazelle:exclude comp/core/tagger/kubetags
 # gazelle:exclude comp/core/tagger/mock
-# gazelle:exclude comp/core/tagger/proto
 # gazelle:exclude comp/core/tagger/server
-# gazelle:exclude comp/core/tagger/taglist
 # gazelle:exclude comp/core/tagger/tagstore
 # gazelle:exclude comp/core/workloadfilter
 # gazelle:exclude comp/core/workloadmeta
 # gazelle:exclude comp/core/workloadmeta
-# gazelle:exclude comp/dataobs
+# gazelle:exclude comp/dataobs/queryactions/fx
+# gazelle:exclude comp/dataobs/queryactions/impl
 # gazelle:exclude comp/dogstatsd
-# gazelle:exclude comp/etw
+# gazelle:exclude comp/etw/impl
 # gazelle:exclude comp/filterlist
-# gazelle:exclude comp/fleetstatus
+# gazelle:exclude comp/fleetstatus/fx
+# gazelle:exclude comp/fleetstatus/impl
 # gazelle:exclude comp/forwarder
-# gazelle:exclude comp/haagent
-# gazelle:exclude comp/healthplatform
+# gazelle:exclude comp/haagent/fx
+# gazelle:exclude comp/haagent/helpers
+# gazelle:exclude comp/haagent/impl
+# gazelle:exclude comp/haagent/mock
+# gazelle:exclude comp/healthplatform/fx
+# gazelle:exclude comp/healthplatform/impl
+# gazelle:exclude comp/healthplatform/mock
 # gazelle:exclude comp/host-profiler/collector
 # gazelle:exclude comp/host-profiler/flare
 # gazelle:exclude comp/host-profiler/oom
@@ -128,24 +129,27 @@ ALL_BUILD_TAGS = [
 # gazelle:exclude comp/host-profiler/symboluploader/symbol
 # gazelle:exclude comp/host-profiler/symboluploader/symbolcopier
 # gazelle:exclude comp/host-profiler/version
-# gazelle:exclude comp/languagedetection
-# gazelle:exclude comp/logonduration
+# gazelle:exclude comp/languagedetection/client/fx
+# gazelle:exclude comp/languagedetection/client/impl
+# gazelle:exclude comp/languagedetection/client/mock
+# gazelle:exclude comp/logonduration/fx
+# gazelle:exclude comp/logonduration/impl
 # gazelle:exclude comp/logs
 # gazelle:exclude comp/logs-library
 # gazelle:exclude comp/metadata
 # gazelle:exclude comp/ndmtmp
-# gazelle:exclude comp/netflow/common
 # gazelle:exclude comp/netflow/config
 # gazelle:exclude comp/netflow/flowaggregator
-# gazelle:exclude comp/netflow/format
 # gazelle:exclude comp/netflow/goflowlib
 # gazelle:exclude comp/netflow/portrollup
 # gazelle:exclude comp/netflow/server
 # gazelle:exclude comp/netflow/testutil
 # gazelle:exclude comp/netflow/topn
-# gazelle:exclude comp/networkdeviceconfig
+# gazelle:exclude comp/networkdeviceconfig/fx
+# gazelle:exclude comp/networkdeviceconfig/impl
 # gazelle:exclude comp/networkpath
-# gazelle:exclude comp/notableevents
+# gazelle:exclude comp/notableevents/fx
+# gazelle:exclude comp/notableevents/impl
 # gazelle:exclude comp/otelcol/collector
 # gazelle:exclude comp/otelcol/collector-contrib/fx
 # gazelle:exclude comp/otelcol/collector-contrib/impl
@@ -163,19 +167,30 @@ ALL_BUILD_TAGS = [
 # gazelle:exclude comp/otelcol/otlp
 # gazelle:exclude comp/otelcol/status/fx
 # gazelle:exclude comp/otelcol/status/impl
-# gazelle:exclude comp/privateactionrunner
+# gazelle:exclude comp/privateactionrunner/fx
+# gazelle:exclude comp/privateactionrunner/impl
+# gazelle:exclude comp/privateactionrunner/status
 # gazelle:exclude comp/process
 # gazelle:exclude comp/publishermetadatacache
-# gazelle:exclude comp/rdnsquerier
+# gazelle:exclude comp/rdnsquerier/fx
+# gazelle:exclude comp/rdnsquerier/fx-mock
+# gazelle:exclude comp/rdnsquerier/fx-none
+# gazelle:exclude comp/rdnsquerier/impl
+# gazelle:exclude comp/rdnsquerier/impl-none
+# gazelle:exclude comp/rdnsquerier/mock
 # gazelle:exclude comp/remote-config
 # gazelle:exclude comp/serializer
 # gazelle:exclude comp/snmpscan
-# gazelle:exclude comp/snmpscanmanager
+# gazelle:exclude comp/snmpscanmanager/fx
+# gazelle:exclude comp/snmpscanmanager/impl
+# gazelle:exclude comp/snmpscanmanager/mock
 # gazelle:exclude comp/snmptraps
-# gazelle:exclude comp/softwareinventory
+# gazelle:exclude comp/softwareinventory/fx
+# gazelle:exclude comp/softwareinventory/impl
 # gazelle:exclude comp/syntheticstestscheduler
 # gazelle:exclude comp/systray
-# gazelle:exclude comp/trace-telemetry
+# gazelle:exclude comp/trace-telemetry/fx
+# gazelle:exclude comp/trace-telemetry/impl
 # gazelle:exclude comp/trace/agent/fx
 # gazelle:exclude comp/trace/agent/fx-mock
 # gazelle:exclude comp/trace/agent/impl
@@ -185,7 +200,18 @@ ALL_BUILD_TAGS = [
 # gazelle:exclude comp/trace/etwtracer
 # gazelle:exclude comp/trace/payload-modifier
 # gazelle:exclude comp/trace/status
-# gazelle:exclude comp/updater
+# gazelle:exclude comp/workloadselection/fx
+# gazelle:exclude comp/workloadselection/impl
+# gazelle:exclude comp/updater/daemonchecker/fx
+# gazelle:exclude comp/updater/daemonchecker/mock
+# gazelle:exclude comp/updater/localapi
+# gazelle:exclude comp/updater/localapiclient
+# gazelle:exclude comp/updater/ssistatus/fx
+# gazelle:exclude comp/updater/ssistatus/impl
+# gazelle:exclude comp/updater/telemetry/fx
+# gazelle:exclude comp/updater/telemetry/impl
+# gazelle:exclude comp/updater/telemetry/mock
+# gazelle:exclude comp/updater/updater
 # gazelle:exclude comp/workloadselection
 # gazelle:exclude internal/remote-agent
 # gazelle:exclude internal/third_party
@@ -315,7 +341,6 @@ ALL_BUILD_TAGS = [
 # gazelle:exclude pkg/opentelemetry-mapping-go/inframetadata
 # gazelle:exclude pkg/orchestrator/config
 # gazelle:exclude pkg/persistentcache
-# gazelle:exclude pkg/pidfile
 # gazelle:exclude pkg/privateactionrunner
 # gazelle:exclude pkg/privileged-logs
 # gazelle:exclude pkg/process/checks
@@ -382,7 +407,6 @@ ALL_BUILD_TAGS = [
 # gazelle:exclude pkg/system-probe
 # gazelle:exclude pkg/trace/agent
 # gazelle:exclude pkg/trace/api
-# gazelle:exclude pkg/trace/config
 # gazelle:exclude pkg/trace/containertags
 # gazelle:exclude pkg/trace/event
 # gazelle:exclude pkg/trace/filters
@@ -391,16 +415,11 @@ ALL_BUILD_TAGS = [
 # gazelle:exclude pkg/trace/payload
 # gazelle:exclude pkg/trace/pb
 # gazelle:exclude pkg/trace/remoteconfighandler
-# gazelle:exclude pkg/trace/sampler
-# gazelle:exclude pkg/trace/semantics
-# gazelle:exclude pkg/trace/stats
 # gazelle:exclude pkg/trace/telemetry
 # gazelle:exclude pkg/trace/teststatsd
 # gazelle:exclude pkg/trace/testutil
 # gazelle:exclude pkg/trace/timing
 # gazelle:exclude pkg/trace/transform
-# gazelle:exclude pkg/trace/version
-# gazelle:exclude pkg/trace/watchdog
 # gazelle:exclude pkg/trace/writer
 # gazelle:exclude pkg/util/cloudproviders
 # gazelle:exclude pkg/util/clusteragent

MODULE.bazel

@@ -9,6 +9,7 @@ env_vars(
     name = "agent_volatile",
     # Each variable needs a justification. Every time you use one, the
     # target using it is potentially uncachable.
+    # ⚠️ Keep in sync with the `--repo_env=` entries in .bazelrc
     variables = [
         # This gets set by omnibus builds in many cases. It's used in the script
         # to set things like compression, so we'll mimic that for now.

bazel/rules/ebpf/ebpf.bzl

@@ -42,6 +42,7 @@ _PREBUILT_FLAGS = [
 _CORE_FLAGS = [
     "-DCOMPILE_CORE",
     "-g",
+    "-fdebug-compilation-dir=.",
 ]
 
 def _get_arch_flags(target_arch):

bazel/rules/ebpf/runtime_compilation.bzl

@@ -10,6 +10,8 @@ load("@bazel_lib//lib:run_binary.bzl", "run_binary")
 load("@bazel_lib//lib:write_source_files.bzl", "write_source_file")
 
 def _runtime_compilation_bundle_impl(name, visibility, src_c, out_name, include_dirs, header_deps, out_go_file):
+    _LINUX_ONLY = ["@platforms//os:linux"]
+
     flat_name = "{}_flat".format(name)
     run_binary(
         name = flat_name,
@@ -21,6 +23,7 @@ def _runtime_compilation_bundle_impl(name, visibility, src_c, out_name, include_
             "$(location {})".format(src_c),
             "$@",
         ] + include_dirs,
+        target_compatible_with = _LINUX_ONLY,
     )
 
     raw_name = "{}_raw".format(name)
@@ -34,6 +37,7 @@ def _runtime_compilation_bundle_impl(name, visibility, src_c, out_name, include_
             "$@",
             "runtime",
         ],
+        target_compatible_with = _LINUX_ONLY,
     )
 
     # The integrity tool uses runtime.Caller(0) to detect whether its output
@@ -48,6 +52,7 @@ def _runtime_compilation_bundle_impl(name, visibility, src_c, out_name, include_
         outs = ["{}/{}.go".format(name, out_name)],
         cmd = "sed -e '/^import \"github.com\\/DataDog\\/datadog-agent\\/pkg\\/ebpf\\/bytecode\\/runtime\"/d' -e 's/runtime\\.newAsset/newAsset/g' $< > $@",
         visibility = visibility,
+        target_compatible_with = _LINUX_ONLY,
     )
 
     if out_go_file:

bazel/rules/go_fast/repo.bzl

@@ -22,7 +22,7 @@ def _impl(rctx):
     repo_utils.execute_checked(
         rctx,
         arguments = [go, "build", "-overlay", "overlay.json", "-o", out, "cmd/go"],
-        environment = {"GOPATH": None, "GOROOT": str(sdk), "GOWORK": "off"},
+        environment = {"GOROOT": str(sdk), "GOWORK": "off"},
         op = "build {}".format(out),
     )
     rctx.file(

bazel/rules/go_sdk_overrides/repo.bzl

@@ -19,7 +19,7 @@ def _impl(rctx):
                 rctx,
                 op = "gopatch {}".format(p),
                 arguments = [go, "-C", rctx.path(rctx.attr._gopatch).dirname, "run", ".", "-p", p, rctx.path(".")],
-                environment = {"GOPATH": None, "GOROOT": None, "GOWORK": "off"},
+                environment = {"GOWORK": "off"},
             )
         else:
             rctx.patch(p, strip = 1)

bazel/rules/replace_prefix.sh

@@ -25,7 +25,7 @@ fi
 
 patch_text_file() {
     f="$1"
-    sed -ibak -e "s|^prefix=.*|prefix=$PREFIX|" -e "s|##PREFIX##|$PREFIX|" -e "s|\${EXT_BUILD_DEPS}|$PREFIX|" "$f" && rm -f "${f}bak"
+    sed -ibak -e "s|^prefix=.*|prefix=$PREFIX|" -e "s|##PREFIX##|$PREFIX|g" -e "s|\${EXT_BUILD_DEPS}|$PREFIX|g" "$f" && rm -f "${f}bak"
 }
 
 for f in "$@"; do
@@ -40,7 +40,7 @@ for f in "$@"; do
     fi
 
     case $f in
-        *.pc)
+        *.pc | *.py)
             patch_text_file "$f"
             ;;
         *)