[confgenerator] Add file offset storage to Otel Logging receivers.

cmd/ops_agent_uap_plugin/service_linux.go

@@ -43,6 +43,7 @@ const (
 	LogsDirectory               = "log/google-cloud-ops-agent"
 	FluentBitStateDiectory      = "state/fluent-bit"
 	FluentBitRuntimeDirectory   = "run/google-cloud-ops-agent-fluent-bit"
+	OtelStateDiectory           = "state/opentelemetry-collector"
 	OtelRuntimeDirectory        = "run/google-cloud-ops-agent-opentelemetry-collector"
 	DefaultPluginStateDirectory = "/var/lib/google-guest-agent/agent_state/plugins/ops-agent-plugin"
 )
@@ -209,7 +210,8 @@ func generateSubagentConfigs(ctx context.Context, runCommand RunCommandFunc, plu
 		"-service", "otel",
 		"-in", OpsAgentConfigLocationLinux,
 		"-out", path.Join(pluginStateDirectory, OtelRuntimeDirectory),
-		"-logs", path.Join(pluginStateDirectory, LogsDirectory))
+		"-logs", path.Join(pluginStateDirectory, LogsDirectory),
+		"-state", path.Join(pluginStateDirectory, OtelStateDiectory))
 
 	if output, err := runCommand(otelConfigGenerationCmd); err != nil {
 		return fmt.Errorf("failed to generate Otel config:\ncommand output: %s\ncommand error: %s", output, err)
@@ -220,7 +222,8 @@ func generateSubagentConfigs(ctx context.Context, runCommand RunCommandFunc, plu
 		"-service", "fluentbit",
 		"-in", OpsAgentConfigLocationLinux,
 		"-out", path.Join(pluginStateDirectory, FluentBitRuntimeDirectory),
-		"-logs", path.Join(pluginStateDirectory, LogsDirectory), "-state", path.Join(pluginStateDirectory, FluentBitStateDiectory))
+		"-logs", path.Join(pluginStateDirectory, LogsDirectory),
+		"-state", path.Join(pluginStateDirectory, FluentBitStateDiectory))
 
 	if output, err := runCommand(fluentBitConfigGenerationCmd); err != nil {
 		return fmt.Errorf("failed to generate Fluntbit config:\ncommand output: %s\ncommand error: %s", output, err)

cmd/ops_agent_windows/main_windows.go

@@ -106,6 +106,10 @@ func initServices() error {
 	if err := os.MkdirAll(fluentbitStoragePath, 0644); err != nil {
 		return err
 	}
+	otelStoragePath := filepath.Join(os.Getenv("PROGRAMDATA"), dataDirectory, "run", "file_storage")
+	if err := os.MkdirAll(otelStoragePath, 0644); err != nil {
+		return err
+	}
 	logDirectory := filepath.Join(os.Getenv("PROGRAMDATA"), dataDirectory, "log")
 	if err := os.MkdirAll(logDirectory, 0644); err != nil {
 		return err

confgenerator/confgenerator.go

@@ -139,7 +139,32 @@ func (uc *UnifiedConfig) getOTelLogLevel() string {
 	return logLevel
 }
 
-func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir string) (string, error) {
+// fileStorageExtensionID returns the file_storage extension used by all receivers and exporters.
+func fileStorageExtensionID() string {
+	return "file_storage"
+}
+
+// fileStorageExtensionConfig returns a configured file_storage extension to be used by all receivers and exporters.
+func fileStorageExtensionConfig(stateDir string) map[string]interface{} {
+	return map[string]interface{}{
+		"directory":        path.Join(stateDir, "file_storage"),
+		"create_directory": true,
+	}
+}
+
+func (uc *UnifiedConfig) getEnabledExtensions(ctx context.Context, stateDir string) map[string]interface{} {
+	extensions := map[string]interface{}{}
+	expOtlpExporter := experimentsFromContext(ctx)["otlp_exporter"]
+	if expOtlpExporter {
+		extensions["googleclientauth"] = map[string]interface{}{}
+	}
+	if uc.Logging.Service.OTelLogging {
+		extensions["file_storage"] = fileStorageExtensionConfig(stateDir)
+	}
+	return extensions
+}
+
+func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDir string) (string, error) {
 	p := platform.FromContext(ctx)
 	userAgent, _ := p.UserAgent("Google-Cloud-Ops-Agent-Metrics")
 	metricVersionLabel, _ := p.VersionLabel("google-cloud-ops-agent-metrics")
@@ -160,17 +185,11 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir string)
 	}
 	agentSelfMetrics.AddSelfMetricsPipelines(receiverPipelines, pipelines)
 
-	expOtlpExporter := experimentsFromContext(ctx)["otlp_exporter"]
-	extensions := map[string]interface{}{}
-	if expOtlpExporter {
-		extensions["googleclientauth"] = map[string]interface{}{}
-	}
-
 	otelConfig, err := otel.ModularConfig{
 		LogLevel:          uc.getOTelLogLevel(),
 		ReceiverPipelines: receiverPipelines,
 		Pipelines:         pipelines,
-		Extensions:        extensions,
+		Extensions:        uc.getEnabledExtensions(ctx, stateDir),
 		Exporters: map[otel.ExporterType]otel.Component{
 			otel.System: googleCloudExporter(userAgent, false, false),
 			otel.OTel:   googleCloudExporter(userAgent, true, true),

confgenerator/confgenerator_test.go

@@ -279,7 +279,7 @@ func generateConfigs(pc platformConfig, testDir string) (got map[string]string,
 	}
 
 	// Otel configs
-	otelGeneratedConfig, err := mergedUc.GenerateOtelConfig(ctx, "")
+	otelGeneratedConfig, err := mergedUc.GenerateOtelConfig(ctx, "", "")
 	if err != nil {
 		return
 	}

confgenerator/config.go

@@ -1179,7 +1179,7 @@ func (uc *UnifiedConfig) OTelLoggingSupported(ctx context.Context) bool {
 		ucLoggingCopy.Logging.Service = &LoggingService{}
 	}
 	ucLoggingCopy.Logging.Service.OTelLogging = true
-	_, err = ucLoggingCopy.GenerateOtelConfig(ctx, "")
+	_, err = ucLoggingCopy.GenerateOtelConfig(ctx, "", "")
 	return err == nil
 }
 

confgenerator/files.go

@@ -59,7 +59,7 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l
 			}
 		}
 	case "otel":
-		otelConfig, err := uc.GenerateOtelConfig(ctx, outDir)
+		otelConfig, err := uc.GenerateOtelConfig(ctx, outDir, stateDir)
 		if err != nil {
 			return fmt.Errorf("can't parse configuration: %w", err)
 		}

confgenerator/logging_receivers.go

@@ -207,6 +207,9 @@ func (r LoggingReceiverFilesMixin) Pipelines(ctx context.Context) ([]otel.Receiv
 		"preserve_leading_whitespaces":  true,
 		"preserve_trailing_whitespaces": true,
 	}
+	if !r.TransformationTest {
+		receiver_config["storage"] = fileStorageExtensionID()
+	}
 	if i := r.WildcardRefreshInterval; i != nil {
 		receiver_config["poll_interval"] = i.String()
 	}
@@ -606,7 +609,7 @@ func (r LoggingReceiverWindowsEventLog) Pipelines(ctx context.Context) ([]otel.R
 			"start_at":              "beginning",
 			"poll_interval":         "1s",
 			"ignore_channel_errors": true,
-			// TODO: Configure storage
+			"storage":               fileStorageExtensionID(),
 		}
 
 		var p []otel.Component
@@ -966,6 +969,7 @@ func (r LoggingReceiverSystemd) Pipelines(ctx context.Context) ([]otel.ReceiverP
 	receiver_config := map[string]any{
 		"start_at": "beginning",
 		"priority": "debug",
+		"storage":  fileStorageExtensionID(),
 	}
 
 	modify_fields_processors, err := LoggingProcessorModifyFields{

confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml

@@ -15,6 +15,10 @@ exporters:
       service_resource_labels: true
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+extensions:
+  file_storage:
+    create_directory: true
+    directory: file_storage
 processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
@@ -818,6 +822,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   filelog/syslog:
     exclude: []
     include:
@@ -832,6 +837,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   hostmetrics/hostmetrics:
     collection_interval: 60s
     scrapers:
@@ -870,6 +876,8 @@ receivers:
         - targets:
           - 0.0.0.0:20201
 service:
+  extensions:
+  - file_storage
   pipelines:
     logs/logs_default__pipeline_syslog:
       exporters:

confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml

@@ -15,6 +15,10 @@ exporters:
       service_resource_labels: true
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+extensions:
+  file_storage:
+    create_directory: true
+    directory: file_storage
 processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
@@ -789,6 +793,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   filelog/syslog:
     exclude: []
     include:
@@ -803,6 +808,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   hostmetrics/hostmetrics:
     collection_interval: 60s
     scrapers:
@@ -839,6 +845,8 @@ receivers:
         - targets:
           - 0.0.0.0:20201
 service:
+  extensions:
+  - file_storage
   pipelines:
     logs/logs_default__pipeline_syslog:
       exporters:

confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml

@@ -15,6 +15,10 @@ exporters:
       service_resource_labels: true
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
+extensions:
+  file_storage:
+    create_directory: true
+    directory: file_storage
 processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
@@ -1180,6 +1184,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   hostmetrics/hostmetrics:
     collection_interval: 60s
     scrapers:
@@ -1223,16 +1228,19 @@ receivers:
     ignore_channel_errors: true
     poll_interval: 1s
     start_at: beginning
+    storage: file_storage
   windowseventlog/windows__event__log_1:
     channel: Application
     ignore_channel_errors: true
     poll_interval: 1s
     start_at: beginning
+    storage: file_storage
   windowseventlog/windows__event__log_2:
     channel: Security
     ignore_channel_errors: true
     poll_interval: 1s
     start_at: beginning
+    storage: file_storage
   windowsperfcounters/iis:
     collection_interval: 60s
     perfcounters:
@@ -1266,6 +1274,8 @@ receivers:
       - _Total
       object: SQLServer:Databases
 service:
+  extensions:
+  - file_storage
   pipelines:
     logs/logs_default__pipeline_windows__event__log:
       exporters:

confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml

@@ -15,6 +15,10 @@ exporters:
       service_resource_labels: true
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version)
+extensions:
+  file_storage:
+    create_directory: true
+    directory: file_storage
 processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
@@ -1180,6 +1184,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   hostmetrics/hostmetrics:
     collection_interval: 60s
     scrapers:
@@ -1223,16 +1228,19 @@ receivers:
     ignore_channel_errors: true
     poll_interval: 1s
     start_at: beginning
+    storage: file_storage
   windowseventlog/windows__event__log_1:
     channel: Application
     ignore_channel_errors: true
     poll_interval: 1s
     start_at: beginning
+    storage: file_storage
   windowseventlog/windows__event__log_2:
     channel: Security
     ignore_channel_errors: true
     poll_interval: 1s
     start_at: beginning
+    storage: file_storage
   windowsperfcounters/iis:
     collection_interval: 60s
     perfcounters:
@@ -1266,6 +1274,8 @@ receivers:
       - _Total
       object: SQLServer:Databases
 service:
+  extensions:
+  - file_storage
   pipelines:
     logs/logs_default__pipeline_windows__event__log:
       exporters:

confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml

@@ -15,6 +15,10 @@ exporters:
       service_resource_labels: true
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+extensions:
+  file_storage:
+    create_directory: true
+    directory: file_storage
 processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
@@ -791,6 +795,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   filelog/syslog:
     exclude: []
     include:
@@ -805,6 +810,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   hostmetrics/hostmetrics:
     collection_interval: 60s
     scrapers:
@@ -843,6 +849,8 @@ receivers:
         - targets:
           - 0.0.0.0:20201
 service:
+  extensions:
+  - file_storage
   pipelines:
     logs/logs_default__pipeline_syslog:
       exporters:

confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml

@@ -15,6 +15,10 @@ exporters:
       service_resource_labels: true
       skip_create_descriptor: true
     user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+extensions:
+  file_storage:
+    create_directory: true
+    directory: file_storage
 processors:
   agentmetrics/hostmetrics_0:
     blank_label_metrics:
@@ -762,6 +766,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   filelog/syslog:
     exclude: []
     include:
@@ -776,6 +781,7 @@ receivers:
     preserve_leading_whitespaces: true
     preserve_trailing_whitespaces: true
     start_at: beginning
+    storage: file_storage
   hostmetrics/hostmetrics:
     collection_interval: 60s
     scrapers:
@@ -812,6 +818,8 @@ receivers:
         - targets:
           - 0.0.0.0:20201
 service:
+  extensions:
+  - file_storage
   pipelines:
     logs/logs_default__pipeline_syslog:
       exporters: