Custom Documentation

.gitignore

@@ -9,3 +9,4 @@ vendor/
 generated/
 .DS_Store
 *.swp
+*.pyc

custom_documentation/src/documentation_overrides.yaml

@@ -0,0 +1,211 @@
+- name: Endpoint.policy.applied.artifacts.global.channel
+  default:
+    description: The channel of the artifact.
+    example: stable
+    type: keyword
+  os:
+    linux:
+      description: The channel of the linux artifact.
+    windows:
+      description: The channel of the windows artifact.
+    macos:
+      description: The channel of the macos artifact.
+  # event:
+  #   linux_malicious_behavior_alert:
+  #     description: The channel of the artifact for linux malicious behavior alert.
+  #     example: stable
+
+- name: agent.type
+  default:
+    example: endpoint
+
+- name: Endpoint.policy.applied.artifacts.global.identifiers.name
+  default:
+    example: global-configuration-v1
+
+- name: Endpoint.policy.applied.artifacts.global.snapshot
+  default:
+    example: "latest"
+
+- name: Endpoint.policy.applied.artifacts.user.identifiers.name
+  os:
+    linux:
+      example: endpoint-trustlist-linux-v1
+    windows:
+      example: endpoint-trustlist-windows-v1
+    macos:
+      example: endpoint-trustlist-macos-v1
+
+- name: Endpoint.policy.applied.artifacts.user.version
+  default:
+    example: "1.0.0"
+
+- name: agent.build.original
+  default:
+    example: "version: 9.1.0, compiled: Mon Jun 2 15:00:00 2025, branch: main, commit: 3fd26249705c5a467960870702589686ef04da43"
+
+- name: data_stream.dataset
+  default:
+    example: endpoint.alerts
+
+- name: event.action
+  default:
+    description: |-
+      Possible values for Endpoint include:
+      - elastic_endpoint_telemetry
+      - endpoint_metadata
+      - endpoint_policy_response
+      - endpoint_metrics
+      - endpoint_heartbeat
+      - malicious_file
+      - endpoint_unquarantine
+      - lookup_result
+      - lookup_requested
+      - creation
+      - deletion
+      - demand
+      - clone
+      - link
+      - exchange
+      - execution
+      - modification
+      - open
+      - query
+      - save
+      - overwrite
+      - rename
+      - extended_attributes_delete
+      - mount
+      - unknown
+      - load
+      - connection_accepted
+      - connection_attempted
+      - disconnect_received
+      - http_request
+      - udp_datagram_outgoing
+      - udp_datagram_incoming
+      - icmp_outgoing
+      - icmp_incoming
+      - already_running
+      - fork
+      - end
+      - exec
+      - gid_change
+      - start
+      - session_id_change
+      - uid_change
+      - remote_thread
+      - process_open
+      - text_output
+      - memfd_create
+      - shmget
+      - ptrace
+      - load_module
+      - log_on
+      - log_off
+      - workstation_locked
+      - workstation_unlocked
+      - ssh_log_on
+      - rdp_log_on
+      - service-installed
+      - scheduled-task-created
+      - scheduled-task-updated
+      - added-user-account
+      - group-membership-enumerated
+      - user-member-enumerated
+      - token-right-adjusted
+      - network-share-object-added
+      - network-share-object-access-checked
+      - vault-credentials-were-read
+      - gatekeeper_override
+      - mbr-overwrite
+      - files-encrypted
+      - canary-activity
+      - rule_detection
+      - rule_prevention
+      - api
+      - launch_daemon
+      - mount
+      - unmount
+
+- name: event.category
+  default:
+    type: array of keyword
+    example: '["malware", "intrusion_detection"]'
+    description: |-
+      Possible values for Endpoint include:
+        - authentication
+        - configuration
+        - driver
+        - file
+        - host
+        - iam
+        - intrusion_detection
+        - library
+        - malware
+        - network
+        - process
+        - registry
+        - session
+        - rule
+        - credential_hardening
+        - api
+        - volume_device
+        - security
+
+- name: event.dataset
+  default:
+    example: endpoint.alerts
+
+- name: event.module
+  default:
+    example: endpoint
+    description: |-
+      The module for Endpoint is always `endpoint`
+
+- name: event.risk_score
+  default:
+    example: "99"
+    description: Endpoint risk score uses a scale of 0 to 100, where 100 is the highest risk.
+
+- name: event.severity
+  default:
+    example: "73"
+    description: Endpoint severity uses a scale of 0 to 100, where 100 is the highest risk.
+
+- name: event.type
+  default:
+    example: '["info", "allowed"]'
+    type: array of keyword
+    description: |-
+      Possible values for Endpoint include:
+      - allowed
+      - change
+      - creation
+      - deletion
+      - denied
+      - end
+      - info
+      - protocol
+      - start
+      - access
+      - admin
+      - user
+      - group
+
+- name: event.kind
+  default:
+    description: |-
+      Possible values for Endpoint include:
+        - alert
+        - event
+        - metric
+        - state
+
+- name: event.outcome
+  default:
+    description: |-
+      Possible values for Endpoint include:
+        - success
+        - failure
+        - unknown

scripts/generate-docs/pydocgen/Readme.md

@@ -0,0 +1,54 @@
+# Custom Documentation Generator
+
+## Description
+
+This module generates documentation for the custom endpoint fields defined in [custom_documentation](../../../custom_documentation/)
+
+### Background
+
+The fields defined in [custom_documentation](../../../custom_documentation/) do not have descriptions.  They are simply the possible fields
+of an event, including all the custom fields Endpoint uses but are not mapped.
+
+The fields defined in [package](../../../package/) are the fields that are mapped into Kibana.  These fields have descriptions and documentation.
+
+
+### Implementation
+
+This python module generates markdown for all of the fields in [custom_documentation](../../../custom_documentation/) by taking the following steps
+
+1. Parses all of the mapped fields defined in [package](../../../package/), collecting descriptions, examples, and other metadata
+
+2. Parses any override fields defined in [documentation_overrides.yaml](../../../custom_documentation/src/documentation_overrides.yaml)
+   - overrides can be set for any field.  They can be set at the event level, the os level, or a default override that applies to all
+    instances of that field.
+   - See [documentation_overrides.yaml](../../../custom_documentation/src/documentation_overrides.yaml) for the format
+   - If overrides are updated, the documentation must be regenerated
+
+3. Puts all of that data into an sqlite database
+
+4. Parses all of the endpoint fields defined in [custom_documentation](../../../custom_documentation/)
+
+5. Iterates over the custom_documentation data, filling out descriptions and examples pulled from the database that was just created.
+
+### Example Usage
+`python -m pydocgen --output-dir /path/to/output`
+
+#### Help statement
+```
+usage: __main__.py [-h] [--database DATABASE] [--no-cache] [--output-dir OUTPUT_DIR] [-v] [-l {DEBUG,INFO,WARNING,ERROR,CRITICAL}] [--csv CSV]
+
+Create markdown documentation for the fields defined in custom_documentation
+
+options:
+  -h, --help            show this help message and exit
+  --database DATABASE   path to the database
+  --no-cache            do not use cached database if it exists, always regenerate the database
+  --output-dir OUTPUT_DIR
+                        output directory for markdown documentation
+  -v, --verbose         Force maximum verbosity (DEBUG level + detailed output)
+  -l {DEBUG,INFO,WARNING,ERROR,CRITICAL}, --log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL}
+                        Set logging verbosity level
+  --csv CSV             Path to CSV file for missing documentation fields (optional)
+
+Example usage: python -m pydocgen --output-dir /path/to/output
+```

scripts/generate-docs/pydocgen/__main__.py

@@ -0,0 +1,110 @@
+import argparse
+import logging
+from logging import config
+import pathlib
+import traceback
+import sys
+import tempfile
+
+from .markdown import generate_custom_documentation_markdown
+
+from .models.custom_documentation import DocumentationOverrideMap
+
+from typing import Literal
+
+
+def configure_logging(
+    log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], 
+    verbose: bool
+) -> None:
+    """Configures the logging system with specified level and verbosity.
+
+    Args:
+        log_level: String representation of logging level (DEBUG, INFO, etc.)
+        verbose: Boolean flag to force maximum verbosity
+    """
+    level = getattr(logging, log_level)
+
+    # If verbose is specified, override to DEBUG level
+    if verbose:
+        level = logging.DEBUG
+
+    # Basic config with both handlers
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s - %(levelname)-8s %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Create markdown documentation for the fields defined in custom_documentation",
+        epilog="Example usage: python -m pydocgen --output-dir /path/to/output",
+    )
+
+    parser.add_argument(
+        "--database",
+        default=pathlib.Path(tempfile.gettempdir()) / "generate-docs.sqlite",
+        type=pathlib.Path,
+        help="path to the database",
+    )
+
+    parser.add_argument(
+        "--no-cache",
+        action="store_true",
+        help="do not use cached database if it exists, always regenerate the database",
+    )
+
+    parser.add_argument(
+        "--output-dir",
+        default=pathlib.Path.cwd().resolve() / "output",
+        type=pathlib.Path,
+        help="output directory for markdown documentation",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Force maximum verbosity (DEBUG level + detailed output)",
+    )
+
+    parser.add_argument(
+        "-l",
+        "--log-level",
+        type=str.upper,
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        default="INFO",
+        help="Set logging verbosity level",
+    )
+
+    parser.add_argument(
+        "--overrides",
+        type=pathlib.Path,
+        default=pathlib.Path.cwd().resolve() / "custom_documentation" / "src" / "documentation_overrides.yaml",
+    )
+
+    parser.add_argument(
+        "--csv",
+        type=pathlib.Path,
+        default=None,
+        help="Path to CSV file for missing documentation fields (optional)",
+    )
+
+    args = parser.parse_args()
+
+    configure_logging(args.log_level, args.verbose)
+
+    if args.no_cache and args.database.exists():
+        logging.info(f"Removing existing database {args.database} since --no-cache was specified")
+        args.database.unlink()
+
+    generate_custom_documentation_markdown(args.database, args.output_dir, args.csv)
+    logging.info(f"Generated markdown documentation to {args.output_dir}")
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        traceback.print_exc()
+        sys.exit(1)