Update logging with new abstraction/configuration

16/umbraco-cms/fundamentals/backoffice/logviewer.md

@@ -30,13 +30,13 @@ Here are some example queries to help you get started. For more details on the s
 
 If you frequently use a custom query, you can save it for quick access. Type your query in the search box and click the heart icon to save it with a friendly name. Saved queries are stored in the `umbracoLogViewerQuery` table in the database.
 
-## Implementing Your Own Log Viewer
+## Implementing Your Own Log Viewer Source
 
-Umbraco allows you to implement a customn `ILogViewer` to fetch logs from alternative sources, such as **Azure Table Storage**.
+Umbraco allows you to implement a custom `ILogViewerRepository` and `ILogViewerService` to fetch logs from alternative sources, such as **Azure Table Storage**.
 
-### Creating a Custom Log Viewer
+### Creating a Custom Log Viewer Repository
 
-To fetch logs from Azure Table Storage, implement the `SerilogLogViewerSourceBase` class from `Umbraco.Cms.Core.Logging.Viewer`.
+To fetch logs from Azure Table Storage, extend the `LogViewerRepositoryBase` class from `Umbraco.Cms.Infrastructure.Services.Implement`.
 
 {% hint style="info" %}
 This implementation requires the `Azure.Data.Tables` NuGet package.
@@ -47,114 +47,185 @@ using Azure;
 using Azure.Data.Tables;
 using Serilog.Events;
 using Serilog.Formatting.Compact.Reader;
-using Serilog.Sinks.AzureTableStorage;
+using Umbraco.Cms.Core.Composing;
 using Umbraco.Cms.Core.Logging.Viewer;
-using ITableEntity = Azure.Data.Tables.ITableEntity;
+using Umbraco.Cms.Core.Serialization;
+using Umbraco.Cms.Core.Services;
+using Umbraco.Cms.Infrastructure.Logging.Serilog;
+using Umbraco.Cms.Infrastructure.Services.Implement;
 
 namespace My.Website;
 
-public class AzureTableLogViewer : SerilogLogViewerSourceBase
+public class AzureTableLogsRepository : LogViewerRepositoryBase
 {
-    public AzureTableLogViewer(ILogViewerConfig logViewerConfig, Serilog.ILogger serilogLog, ILogLevelLoader logLevelLoader)
-        : base(logViewerConfig, logLevelLoader, serilogLog)
+    private readonly IJsonSerializer _jsonSerializer;
+
+    public AzureTableLogsRepository(UmbracoFileConfiguration umbracoFileConfig, IJsonSerializer jsonSerializer) : base(
+        umbracoFileConfig)
     {
+        _jsonSerializer = jsonSerializer;
     }
 
-    public override bool CanHandleLargeLogs => true;
-
-    // This method will not be called - as we have indicated that this 'CanHandleLargeLogs'
-    public override bool CheckCanOpenLogs(LogTimePeriod logTimePeriod) => throw new NotImplementedException();
-
-    protected override IReadOnlyList<LogEvent> GetLogs(LogTimePeriod logTimePeriod, ILogFilter filter, int skip, int take)
+    protected override IEnumerable<ILogEntry> GetLogs(LogTimePeriod logTimePeriod, ILogFilter logFilter)
     {
-        //Replace ACCOUNT_NAME and KEY with your actual Azure Storage Account details. The "Logs" parameter refers to the table name where logs will be stored and retrieved from.
+        // This example uses a connection string compatible with the Azurite emulator
+        // https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite
         var client =
             new TableClient(
-                "DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=KEY;EndpointSuffix=core.windows.net",
-                "Logs");
-
-        // Table storage does not support skip, only take, so the best we can do is to not fetch more entities than we need in total.
-        // See: https://learn.microsoft.com/en-us/rest/api/storageservices/writing-linq-queries-against-the-table-service#returning-the-top-n-entities for more info.
-        var requiredEntities = skip + take;
-        IEnumerable<AzureTableLogEntity> results = client.Query<AzureTableLogEntity>().Take(requiredEntities);
-
-        return results
-            .Skip(skip)
-            .Take(take)
-            .Select(x => LogEventReader.ReadFromString(x.Data))
-            // Filter by timestamp to avoid retrieving all logs from the table, preventing memory and performance issues
-            .Where(evt => evt.Timestamp >= logTimePeriod.StartTime.Date &&
-                evt.Timestamp <= logTimePeriod.EndTime.Date.AddDays(1).AddSeconds(-1))
-            .Where(filter.TakeLogEvent)
-            .ToList();
+                "UseDevelopmentStorage=true",
+                "LogEventEntity");
+
+        // Filter by timestamp to avoid retrieving all logs from the table, preventing memory and performance issues
+        IEnumerable<AzureTableLogEntity> results = client.Query<AzureTableLogEntity>(
+            entity => entity.Timestamp >= logTimePeriod.StartTime.Date &&
+                      entity.Timestamp <= logTimePeriod.EndTime.Date.AddDays(1).AddSeconds(-1));
+
+        // Read the data and apply logfilters
+        IEnumerable<LogEvent> filteredData = results.Select(x => LogEventReader.ReadFromString(x.Data))
+            .Where(logFilter.TakeLogEvent);
+
+        return filteredData.Select(x => new LogEntry
+        {
+            Timestamp = x.Timestamp,
+            Level = Enum.Parse<Core.Logging.LogLevel>(x.Level.ToString()),
+            MessageTemplateText = x.MessageTemplate.Text,
+            Exception = x.Exception?.ToString(),
+            Properties = MapLogMessageProperties(x.Properties),
+            RenderedMessage = x.RenderMessage(),
+        });
     }
 
-    public override IReadOnlyList<SavedLogSearch>? GetSavedSearches()
+    private IReadOnlyDictionary<string, string?> MapLogMessageProperties(
+        IReadOnlyDictionary<string, LogEventPropertyValue>? properties)
     {
-        //This method is optional. If you store saved searches in Azure Table Storage, implement fetching logic here.
-        return base.GetSavedSearches();
+        var result = new Dictionary<string, string?>();
+
+        if (properties is not null)
+        {
+            foreach (KeyValuePair<string, LogEventPropertyValue> property in properties)
+            {
+                string? value;
+
+                if (property.Value is ScalarValue scalarValue)
+                {
+                    value = scalarValue.Value?.ToString();
+                }
+                else if (property.Value is StructureValue structureValue)
+                {
+                    var textWriter = new StringWriter();
+                    structureValue.Render(textWriter);
+                    value = textWriter.ToString();
+                }
+                else
+                {
+                    value = _jsonSerializer.Serialize(property.Value);
+                }
+
+                result.Add(property.Key, value);
+            }
+        }
+
+        return result;
     }
 
-    public override IReadOnlyList<SavedLogSearch>? AddSavedSearch(string? name, string? query)
+    public class AzureTableLogEntity : ITableEntity
     {
-        //This method is optional. If you store saved searches in Azure Table Storage, implement adding logic here.
-        return base.AddSavedSearch(name, query);
-    }
+        public required string Data { get; set; }
 
-    public override IReadOnlyList<SavedLogSearch>? DeleteSavedSearch(string? name, string? query)
-    {
-        //This method is optional. If you store saved searches in Azure Table Storage, implement deleting logic here.
-        return base.DeleteSavedSearch(name, query);
+        public required string PartitionKey { get; set; }
+
+        public required string RowKey { get; set; }
+
+        public DateTimeOffset? Timestamp { get; set; }
+
+        public ETag ETag { get; set; }
     }
 }
+```
+
+Azure Table Storage requires entities to implement the `ITableEntity` interface. Since Umbraco's default log entity does not implement this, a custom entity (`AzureTableLogEntity`) must be created to ensure logs are correctly fetched.
 
-public class AzureTableLogEntity : LogEventEntity, ITableEntity
+### Creating a custom log viewer service
+
+The next thing to do is create a new implementation of `ILogViewerService`. Amongst other things, this is responsible for figuring out whether a provided log query is allowed. Again a base class is available.
+
+```csharp
+public class AzureTableLogsService : LogViewerServiceBase
 {
-    public DateTimeOffset? Timestamp { get; set; }
+    public AzureTableLogsService(
+        ILogViewerQueryRepository logViewerQueryRepository,
+        ICoreScopeProvider provider,
+        ILogViewerRepository logViewerRepository)
+        : base(logViewerQueryRepository, provider, logViewerRepository)
+    {
+    }
+
+    // Change this to what you think is sensible.
+    // As an example, check whether more than 5 days off logs are requested.
+    public override Task<Attempt<bool, LogViewerOperationStatus>> CanViewLogsAsync(LogTimePeriod logTimePeriod)
+    {
+        return logTimePeriod.EndTime - logTimePeriod.StartTime < TimeSpan.FromDays(5)
+            ? Task.FromResult(Attempt.SucceedWithStatus(LogViewerOperationStatus.Success, true))
+            : Task.FromResult(Attempt.FailWithStatus(LogViewerOperationStatus.CancelledByLogsSizeValidation, false));
+    }
+
+    public override ReadOnlyDictionary<string, LogLevel> GetLogLevelsFromSinks()
+    {
+        var configuredLogLevels = new Dictionary<string, LogLevel>
+        {
+            { "Global", GetGlobalMinLogLevel() },
+            { "AzureTableStorage", LogViewerRepository.RestrictedToMinimumLevel() },
+        };
 
-    public ETag ETag { get; set; }
+        return configuredLogLevels.AsReadOnly();
+    }
 }
 ```
 
-Azure Table Storage requires entities to implement the `ITableEntity` interface. Since Umbraco’s default log entity does not implement this, a custom entity (`AzureTableLogEntity`) must be created to ensure logs are correctly fetched and stored.
-
-### Register implementation
+### Register implementations
 
-Umbraco needs to be made aware that there is a new implementation of an `ILogViewer` to register. We also need to replace the default JSON LogViewer that we ship in the core of Umbraco.
+Umbraco needs to be made aware that there is a new implementation of an `ILogViewerRepository` and an `ILogViewerService`. These need to replace the default ones that are shipped with Umbraco.
 
 ```csharp
 using Umbraco.Cms.Core.Composing;
 using Umbraco.Cms.Infrastructure.DependencyInjection;
+using Umbraco.Cms.Core.Services;
 
 namespace My.Website;
 
-public class LogViewerSavedSearches : IComposer
-{
-    public void Compose(IUmbracoBuilder builder) => builder.SetLogViewer<AzureTableLogViewer>();
+public class AzureTableLogsComposer : IComposer
+    {
+        public void Compose(IUmbracoBuilder builder)
+        {
+            builder.Services.AddUnique<ILogViewerRepository, AzureTableLogsRepository>();
+            builder.Services.AddUnique<ILogViewerService, AzureTableLogsService>();
+        }
+    }
 }
 ```
 
 ### Configuring Logging to Azure Table Storage
 
-With the above two classes, the setup is in place to view logs from an Azure Table. However, logs are not yet persisted into the Azure Table Storage account. To enable persistence, configure the Serilog logging pipeline to store logs in Azure Table Storage.
+With the above three classes, the setup is in place to view logs from an Azure Table. However, logs are not yet persisted into the Azure Table Storage account. To enable persistence, configure the Serilog logging pipeline to store logs in Azure Table Storage.
 
-* Install `Serilog.Sinks.AzureTableStorage` from NuGet.
-* Add a new sink to `appsettings.json` with credentials to persist logs to Azure.
+-   Install `Serilog.Sinks.AzureTableStorage` from NuGet.
+-   Add a new sink to `appsettings.json` with credentials to persist logs to Azure.
 
 The following sink needs to be added to the [`Serilog:WriteTo`](https://github.com/serilog/serilog-sinks-azuretablestorage#json-configuration) array.
 
 ```json
 {
-"Name": "AzureTableStorage",
-"Args": {
-  "storageTableName": "LogEventEntity",
-  "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact",
-  "connectionString": "DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=KEY;EndpointSuffix=core.windows.net"}
+    "Name": "AzureTableStorage",
+    "Args": {
+        "storageTableName": "LogEventEntity",
+        "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact",
+        "connectionString": "DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=KEY;EndpointSuffix=core.windows.net"
+    }
 }
 ```
 
 For more in-depth information about logging and how to configure it, see the [Logging](../code/debugging/logging.md) article.
 
 ### Compact Log Viewer - Desktop App
 
-[Compact Log Viewer](https://www.microsoft.com/store/apps/9N8RV8LKTXRJ?cid=storebadge\&ocid=badge). A desktop tool is available for viewing and querying JSON log files in the same way as the built-in Log Viewer in Umbraco.
+[Compact Log Viewer](https://www.microsoft.com/store/apps/9N8RV8LKTXRJ?cid=storebadge&ocid=badge). A desktop tool is available for viewing and querying JSON log files in the same way as the built-in Log Viewer in Umbraco.

16/umbraco-cms/fundamentals/code/debugging/logging.md

@@ -117,6 +117,10 @@ Serilog uses levels as the primary means for assigning importance to log events.
 
 Serilog can be configured and extended by using the .NET Core configuration such as the AppSetting.json files or environment variables. For more information, see the [Serilog config](../../../reference/configuration/serilog.md) article.
 
+## The UmbracoFile Sink
+
+Serilog uses the concept of Sinks to output the log messages to different places. Umbraco ships with a custom sink configuration called UmbracoFile that uses the [Serilog.Sinks.File](https://github.com/serilog/serilog-sinks-file) sink. This will save the logs to a rolling file on disk. You can disable this sink by setting its Enabled configuration flag to false, see [Serilog config](../../../reference/configuration/serilog.md) for more information.
+
 ## The logviewer dashboard
 
 Learn more about the [logviewer dashboard](../../backoffice/logviewer.md) in the backoffice and how it can be extended.

16/umbraco-cms/fundamentals/setup/server-setup/running-umbraco-in-docker.md

@@ -8,45 +8,49 @@ Docker is a platform for developing, shipping, and running applications in conta
 
 ## The Docker file system
 
-By default, files created inside a container are written to an ephemeral, writable container layer. 
+By default, files created inside a container are written to an ephemeral, writable container layer.
 This means that the files don't persist when the container is removed, and it's challenging to get files out of the container. Additionally, this writable layer is not suitable for performance-critical data processing.
 
 This has implications when running Umbraco in Docker.
 
 For more information, refer to the [Docker documentation on storage](https://docs.docker.com/engine/storage/).
 
-### General file system consideration 
+### General file system consideration
 
 In general, when working with files and Docker you work in a "push" fashion with read-only layers. When you build, you take all your files and "push" them into this read-only layer.
 
 This means that you should avoid making files on the fly, and instead rely on building your image.
 
-In an Umbraco context, this means you should not create or edit template, script or stylesheet files via the backoffice. These should be deployed as part of your web application and not managed via Umbraco. 
+In an Umbraco context, this means you should not create or edit template, script or stylesheet files via the backoffice. These should be deployed as part of your web application and not managed via Umbraco.
 
 Similarly, you shouldn't use InMemory modelsbuilder, since that also relies on creating files on the disk. While this is not a hard requirement, it doesn't provide any value unless you are live editing your site.
 
 Instead, configure models builder to use "source code" mode in development, and "none" in production, as [described when using runtime modes](https://docs.umbraco.com/umbraco-cms/fundamentals/setup/server-setup/runtime-modes).
 
-
 ### Logs
 
-Umbraco writes logs to the `/umbraco/Logs/` directory. Due to the performance implications of writing to a writable layer, 
-and the limited size, it is recommended to mount a volume to this directory.
+Umbraco writes logs to the `/umbraco/Logs/` directory. Due to the performance implications of writing to a writable layer, and the limited size, it is recommended to mount a volume to this directory.
+
+You may prefer to avoid writing to disk for logs when hosting in containers. If so, you can disable this default behavior and register a custom Serilog sink to alternative storage, such as Azure Table storage.
+
+You can also provide an alternative implementation of a common abstraction for the log viewer. In this way you can read logs from the location where you have configured them to be written.
+
+For more on this please read the article on Umbraco's [log viewer](../../backoffice/logviewer.md).
 
 ### Data
 
 The `/umbraco/Data/` directory is used to store temporary files, such as file uploads. Considering the limitations of the writable layer, you should also mount a volume to this directory.
 
 ### Media
 
-It's recommended to not store media in the writable layer. This is for similar performance reasons as logs, 
-but also for practical hosting reasons. You likely want to persist media files between containers. 
+It's recommended to not store media in the writable layer. This is for similar performance reasons as logs,
+but also for practical hosting reasons. You likely want to persist media files between containers.
 
 One solution is to use bind mounts. The ideal setup, though, is to store the media and ImageSharp cache externally. For more information, refer to the [Azure Blob Storage documentation](https://docs.umbraco.com/umbraco-cms/extending/filesystemproviders/azure-blob-storage).
 
 ### Required files
 
-Your solution may require some specific files to run, such as license files. You will need to pass these files into the container at build time, or mount them externally. 
+Your solution may require some specific files to run, such as license files. You will need to pass these files into the container at build time, or mount them externally.
 
 ## HTTPS
 

16/umbraco-cms/reference/configuration/serilog.md

@@ -105,6 +105,7 @@ By default, Umbraco uses a special Serilog 'sink' that is optimized for performa
       {
         "Name": "UmbracoFile",
         "Args": {
+          "Enabled": "True",
           "RestrictedToMinimumLevel": "Warning",
           "FileSizeLimitBytes": 1073741824,
           "RollingInterval" : "Day",
@@ -117,6 +118,8 @@ By default, Umbraco uses a special Serilog 'sink' that is optimized for performa
   }
 ```
 
+You can also disable this sink if you do not wish to write files to disk.
+
 ## Adding a custom log property to all log items
 
 You may wish to add a log property to all log messages. A good example could be a log property for the `environment` to determine if the log message came from `development` or `production`.