[SecuritySolution][PrivMon] Rewrite dashboard queries to use FORK (#223212)

## Summary


### What is included?
* Improves the auth dashboard to display system events
* Add data view index patterns as visualisations index
* Move ESQL query generation to a shared folder
* Parse ESQL query and validate if fields exist in the dataview
* Rewrite the ESQL query if a FORK command has missing fields
* Add a visualisation warning message when there is no valid FORK branch

![Screenshot 2025-06-20 at 07 22
47](https://github.com/user-attachments/assets/3ff85561-33b6-4f40-8037-4e983d6e4057)


### Pros
* To be able to render parts of the query depending on whether indices
or fields exist in the cluster
* The queries become much easier to read, maintain and fix

### Cons
* We need to test the performance
* FORK is in tech preview
* The commands we can use in a fork are limited to “WHERE, LIMIT, SORT,
EVAL, STATS, DISSECT”

### How to test it?
* Open the dashboard without privmon data, some of the visualisations
should display the warning message
* Add privmon data, the visualisation should display the data
(elastic/security-documents-generator#163)
* Check if the visualisation displays the correct data.
* To test if the FORK rewrite logic is working, I update the queries on
my local environment to use a non-existent field and update the page.


### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

Author: machadoum

x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/privileged_user_monitoring/components/privileged_access_detection/pad_chart/hooks/pad_esql_source_query_hooks.ts

@@ -6,8 +6,8 @@
  */
 
 import { useIntervalForHeatmap } from './pad_heatmap_interval_hooks';
-import { getPrivilegedMonitorUsersJoin } from '../../../../helpers';
 import type { AnomalyBand } from '../pad_anomaly_bands';
+import { getPrivilegedMonitorUsersJoin } from '../../../../queries/helpers';
 
 const getHiddenBandsFilters = (anomalyBands: AnomalyBand[]) => {
   const hiddenBands = anomalyBands.filter((each) => each.hidden);

x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/privileged_user_monitoring/components/privileged_user_activity/columns.test.tsx

@@ -138,38 +138,14 @@ describe('columns', () => {
       expect(screen.getByText('Console')).toBeInTheDocument();
     });
 
-    it('renders type column as "Direct" for /api/v1/authn*', () => {
+    it('renders type column', () => {
       const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
-      render(<>{col.render?.('/api/v1/authn/something', baseRecord)}</>, {
+      render(<>{col.render?.('Direct', baseRecord)}</>, {
         wrapper: TestProviders,
       });
       expect(screen.getByText('Direct')).toBeInTheDocument();
     });
 
-    it('renders type column as "Federated" for /oauth2/v1/authorize', () => {
-      const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
-      render(<>{col.render?.('/oauth2/v1/authorize', baseRecord)}</>, { wrapper: TestProviders });
-      expect(screen.getByText('Federated')).toBeInTheDocument();
-    });
-
-    it('renders type column as "Federated" for /oauth2/v1/token', () => {
-      const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
-      render(<>{col.render?.('/oauth2/v1/token', baseRecord)}</>, { wrapper: TestProviders });
-      expect(screen.getByText('Federated')).toBeInTheDocument();
-    });
-
-    it('renders type column as "Federated" for string containing /sso/saml', () => {
-      const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
-      render(<>{col.render?.('/some/path/sso/saml', baseRecord)}</>, { wrapper: TestProviders });
-      expect(screen.getByText('Federated')).toBeInTheDocument();
-    });
-
-    it('renders type column as original value for unmatched string', () => {
-      const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
-      render(<>{col.render?.('/api/v1/authn', baseRecord)}</>, { wrapper: TestProviders });
-      expect(screen.getByText('Direct')).toBeInTheDocument();
-    });
-
     it('renders result column with badge', () => {
       const col = columns[5] as EuiTableFieldDataColumnType<TableItemType>;
       render(<>{col.render?.('success', baseRecord)}</>, { wrapper: TestProviders });

x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/privileged_user_monitoring/components/privileged_user_activity/columns.tsx

@@ -221,27 +221,20 @@ export const buildAuthenticationsColumns = (
     },
   },
   {
-    field: 'url',
+    field: 'type',
     name: (
       <FormattedMessage
         id="xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.columns.type"
         defaultMessage="Type"
       />
     ),
-    render: (url?: string) => {
-      if (!url) {
-        return getEmptyTagValue();
-      }
-
-      const type = getLoginTypeFromUrl(url);
-
+    render: (type?: string) => {
       if (!type) {
         return getEmptyTagValue();
       }
 
       return type;
     },
-    truncateText: true,
   },
   // TODO Add the column depending on this ticket output https://github.com/elastic/security-team/issues/12713
   // {
@@ -294,25 +287,3 @@ const getResultColor = (value: string) => {
   }
   return 'default';
 };
-
-// TODO Verify if we can improve this logic https://github.com/elastic/security-team/issues/12713
-const getLoginTypeFromUrl = (url: string) => {
-  if (url.startsWith('/api/v1/authn')) {
-    return i18n.translate(
-      'xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.columns.type.direct',
-      { defaultMessage: 'Direct' }
-    );
-  }
-
-  if (
-    url.startsWith('/oauth2/v1/authorize') ||
-    url.startsWith('/oauth2/v1/token') ||
-    url.includes('/sso/saml')
-  ) {
-    return i18n.translate(
-      'xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.columns.type.federated',
-      { defaultMessage: 'Federated' }
-    );
-  }
-  return undefined;
-};

x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/privileged_user_monitoring/components/privileged_user_activity/esql_source_query.ts

@@ -8,6 +8,7 @@
 import React, { useMemo } from 'react';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { useExpandableFlyoutApi } from '@kbn/expandable-flyout';
+import type { DataViewSpec } from '@kbn/data-views-plugin/public';
 import { useSpaceId } from '../../../../../common/hooks/use_space_id';
 import {
   generateListESQLQuery,
@@ -20,16 +21,14 @@ import {
   buildAuthenticationsColumns,
 } from './columns';
 import { getLensAttributes } from './get_lens_attributes';
-import {
-  getAccountSwitchesEsqlSource,
-  getAuthenticationsEsqlSource,
-  getGrantedRightsEsqlSource,
-} from './esql_source_query';
 import {
   ACCOUNT_SWITCH_STACK_BY,
   AUTHENTICATIONS_STACK_BY,
   GRANTED_RIGHTS_STACK_BY,
 } from './constants';
+import { getAuthenticationsEsqlSource } from '../../queries/authentications_esql_query';
+import { getAccountSwitchesEsqlSource } from '../../queries/account_switches_esql_query';
+import { getGrantedRightsEsqlSource } from '../../queries/granted_rights_esql_query';
 
 const toggleOptionsConfig = {
   [VisualizationToggleOptions.GRANTED_RIGHTS]: {
@@ -50,13 +49,24 @@ const toggleOptionsConfig = {
 };
 
 export const usePrivilegedUserActivityParams = (
-  selectedToggleOption: VisualizationToggleOptions
+  selectedToggleOption: VisualizationToggleOptions,
+  sourcererDataView: DataViewSpec
 ) => {
   const spaceId = useSpaceId();
+
+  const indexPattern = sourcererDataView?.title ?? '';
+  const fields = sourcererDataView?.fields;
+
   const esqlSource = useMemo(
     () =>
-      spaceId ? toggleOptionsConfig[selectedToggleOption].generateEsqlSource(spaceId) : undefined,
-    [selectedToggleOption, spaceId]
+      spaceId && indexPattern && fields
+        ? toggleOptionsConfig[selectedToggleOption].generateEsqlSource(
+            spaceId,
+            indexPattern,
+            fields
+          )
+        : undefined,
+    [selectedToggleOption, spaceId, indexPattern, fields]
   );
 
   const generateTableQuery = useMemo(
@@ -75,11 +85,17 @@ export const usePrivilegedUserActivityParams = (
     [selectedToggleOption, openRightPanel]
   );
 
+  const hasLoadedDependencies = useMemo(
+    () => Boolean(spaceId && indexPattern && fields),
+    [spaceId, indexPattern, fields]
+  );
+
   return {
     getLensAttributes,
     generateVisualizationQuery,
     generateTableQuery,
     columns,
+    hasLoadedDependencies,
   };
 };
 

x-pack/solutions/security/plugins/security_solution/public/entity_analytics/components/privileged_user_monitoring/components/privileged_user_activity/hooks.tsx

@@ -28,39 +28,64 @@ jest.mock('../../../../../common/hooks/use_space_id', () => ({
   useSpaceId: jest.fn().mockReturnValue('default'),
 }));
 
+const mockedSourcererDataView = {
+  title: 'test-*',
+  fields: {},
+};
+
 describe('UserActivityPrivilegedUsersPanel', () => {
   it('renders panel title', () => {
-    render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
+    render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
+      wrapper: TestProviders,
+    });
+
     expect(screen.getByText('Privileged user activity')).toBeInTheDocument();
   });
 
   it('renders the toggle button group', () => {
-    render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
-    expect(screen.getByRole('group', { name: /ABOUT_CONTROL_LEGEND/i })).toBeInTheDocument();
+    render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
+      wrapper: TestProviders,
+    });
+    expect(
+      screen.getByRole('group', { name: /Select a visualization to display/i })
+    ).toBeInTheDocument();
   });
 
   it('renders the stack by select with options', () => {
-    render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
+    render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
+      wrapper: TestProviders,
+    });
     expect(screen.getByText('Stack by')).toBeInTheDocument();
     expect(screen.getByRole('option', { name: 'Privileged user' })).toBeInTheDocument();
     expect(screen.getByRole('option', { name: 'Target user' })).toBeInTheDocument();
     expect(screen.getByRole('option', { name: 'Granted right' })).toBeInTheDocument();
   });
 
   it('renders the EsqlDashboardPanel', () => {
-    render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
+    render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
+      wrapper: TestProviders,
+    });
+    // select a visualization that doesn't require dataview fields
+    fireEvent.click(screen.getByTestId('account_switches'));
+
     expect(screen.getByTestId('esql-dashboard-panel')).toBeInTheDocument();
   });
 
   it('changes stack by option when select changes', () => {
-    render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
+    render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
+      wrapper: TestProviders,
+    });
     const select = screen.getByRole('combobox');
     fireEvent.change(select, { target: { value: 'group_name' } });
     expect((select as HTMLSelectElement).value).toBe('group_name');
   });
 
   it('renders the "View all events by privileged users" link', () => {
-    render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
+    render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
+      wrapper: TestProviders,
+    });
+    // select a visualization that doesn't require dataview fields
+    fireEvent.click(screen.getByTestId('account_switches'));
     expect(screen.getByText('View all events')).toBeInTheDocument();
   });
 });