diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
index 22aeeef0b31f89..c32fda96174477 100644
--- a/.github/workflows/airflow-plugin.yml
+++ b/.github/workflows/airflow-plugin.yml
@@ -61,7 +61,7 @@ jobs:
distribution: "zulu"
java-version: 17
- uses: gradle/actions/setup-gradle@v4
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index acf04cead04fa2..423200b0a2341e 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -39,7 +39,7 @@ jobs:
elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- uses: ./.github/actions/ci-optimization
id: ci-optimize
@@ -70,7 +70,9 @@ jobs:
with:
timezoneLinux: ${{ matrix.timezone }}
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
+ with:
+ checkout-head-only: false
- uses: actions/setup-python@v5
with:
python-version: "3.10"
@@ -207,7 +209,7 @@ jobs:
if: ${{ needs.setup.outputs.docker_change == 'true' }}
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
index c2ed45a0ae807a..ce508148050cf9 100644
--- a/.github/workflows/check-datahub-jars.yml
+++ b/.github/workflows/check-datahub-jars.yml
@@ -28,7 +28,7 @@ jobs:
command: ["datahub-client", "datahub-protobuf", "spark-lineage"]
runs-on: ubuntu-latest
steps:
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
index 0583f9ab3c4262..ee45d2ea402904 100644
--- a/.github/workflows/code-checks.yml
+++ b/.github/workflows/code-checks.yml
@@ -31,7 +31,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml
index 8717c969f99ecd..fcb17022b2834b 100644
--- a/.github/workflows/dagster-plugin.yml
+++ b/.github/workflows/dagster-plugin.yml
@@ -49,7 +49,7 @@ jobs:
distribution: "zulu"
java-version: 17
- uses: gradle/actions/setup-gradle@v4
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml
index 1e460e748cb101..b0d17da60131d4 100644
--- a/.github/workflows/docker-ingestion-smoke.yml
+++ b/.github/workflows/docker-ingestion-smoke.yml
@@ -25,7 +25,7 @@ jobs:
python_release_version: ${{ steps.python_release_version.outputs.release_version }}
steps:
- name: Checkout
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- name: Compute Tag
id: tag
env:
@@ -53,7 +53,7 @@ jobs:
if: ${{ needs.setup.outputs.publish == 'true' }}
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- name: Build and push
uses: ./.github/actions/docker-custom-build-and-push
with:
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
index 6f02db0b231a49..1d2c96dbc0e68d 100644
--- a/.github/workflows/docker-unified.yml
+++ b/.github/workflows/docker-unified.yml
@@ -75,7 +75,7 @@ jobs:
yarn_cache_key_prefix: ${{ steps.yarn-cache-key.outputs.yarn_cache_key_prefix }}
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- name: Compute Tag
id: tag
env:
@@ -176,7 +176,7 @@ jobs:
if: ${{ needs.setup.outputs.smoke_test_change == 'true' }}
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
@@ -241,7 +241,9 @@ jobs:
uses: depot/setup-action@v1
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
+ with:
+ checkout-head-only: false
- uses: actions/setup-python@v5
with:
@@ -303,7 +305,7 @@ jobs:
matrix: ${{ fromJson(needs.base_build.outputs.matrix) }}
steps:
- name: Checkout # adding checkout step just to make trivy upload happy
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- id: download_image
name: Download images from depot
if: ${{ needs.setup.outputs.use_depot_cache == 'true' }}
@@ -418,7 +420,9 @@ jobs:
${{ needs.setup.outputs.yarn_cache_key_prefix }}
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
+ with:
+ checkout-head-only: false
- name: Set up Depot CLI
if: ${{ needs.setup.outputs.use_depot_cache == 'true' }}
diff --git a/.github/workflows/github-actions-format.yml b/.github/workflows/github-actions-format.yml
index 76163574ba3614..04e225be52af6f 100644
--- a/.github/workflows/github-actions-format.yml
+++ b/.github/workflows/github-actions-format.yml
@@ -21,7 +21,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml
index b5190740820feb..0b091980fd4a95 100644
--- a/.github/workflows/gx-plugin.yml
+++ b/.github/workflows/gx-plugin.yml
@@ -49,7 +49,7 @@ jobs:
distribution: "zulu"
java-version: 17
- uses: gradle/actions/setup-gradle@v4
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml
index 8a1777522f416b..646a53007184fa 100644
--- a/.github/workflows/lint-actions.yml
+++ b/.github/workflows/lint-actions.yml
@@ -10,7 +10,7 @@ jobs:
actionlint:
runs-on: ubuntu-latest
steps:
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- uses: reviewdog/action-actionlint@v1
with:
reporter: github-pr-review
diff --git a/.github/workflows/markdown-format.yml b/.github/workflows/markdown-format.yml
index d1d84b26543988..3445e9cf420d0f 100644
--- a/.github/workflows/markdown-format.yml
+++ b/.github/workflows/markdown-format.yml
@@ -21,7 +21,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
index a434fc779f5603..f055fb6c3192e6 100644
--- a/.github/workflows/metadata-ingestion.yml
+++ b/.github/workflows/metadata-ingestion.yml
@@ -65,7 +65,9 @@ jobs:
distribution: "zulu"
java-version: 17
- uses: gradle/actions/setup-gradle@v4
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
+ with:
+ checkout-head-only: false
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml
index 9872d0c6a89b52..501be50829a3d1 100644
--- a/.github/workflows/metadata-io.yml
+++ b/.github/workflows/metadata-io.yml
@@ -46,7 +46,9 @@ jobs:
elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
+ with:
+ checkout-head-only: false
- uses: ./.github/actions/ci-optimization
id: ci-optimize
build:
@@ -61,7 +63,9 @@ jobs:
sudo docker image prune -a -f || true
- name: Disk Check
run: df -h . && docker images
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
+ with:
+ checkout-head-only: false
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml
index 1cf7ded9a8b804..84062ca5484506 100644
--- a/.github/workflows/metadata-model.yml
+++ b/.github/workflows/metadata-model.yml
@@ -33,7 +33,9 @@ jobs:
distribution: "zulu"
java-version: 17
- uses: gradle/actions/setup-gradle@v4
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
+ with:
+ checkout-head-only: false
- uses: actions/setup-python@v5
with:
python-version: "3.10"
diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml
index 3dce915883d9d4..ad21f7e2c87c6c 100644
--- a/.github/workflows/prefect-plugin.yml
+++ b/.github/workflows/prefect-plugin.yml
@@ -44,7 +44,7 @@ jobs:
distribution: "zulu"
java-version: 17
- uses: gradle/actions/setup-gradle@v4
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
index 4c9b3f375ea166..5f583ed5be9982 100644
--- a/.github/workflows/publish-datahub-jars.yml
+++ b/.github/workflows/publish-datahub-jars.yml
@@ -41,7 +41,7 @@ jobs:
tag: ${{ steps.tag.outputs.tag }}
steps:
- name: Checkout
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- name: Compute Tag
id: tag
env:
@@ -59,7 +59,7 @@ jobs:
needs: ["check-secret", "setup"]
if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
steps:
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
@@ -212,7 +212,7 @@ jobs:
needs: ["check-secret", "setup", "publish"]
if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
steps:
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
diff --git a/.github/workflows/python-build-pages.yml b/.github/workflows/python-build-pages.yml
index 51b9906f3d4598..d2a894430be637 100644
--- a/.github/workflows/python-build-pages.yml
+++ b/.github/workflows/python-build-pages.yml
@@ -42,7 +42,7 @@ jobs:
distribution: "zulu"
java-version: 17
- uses: gradle/actions/setup-gradle@v4
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
diff --git a/.github/workflows/react-cloudflare-pages.yml b/.github/workflows/react-cloudflare-pages.yml
index 033155f24580f6..e3f8c27464722c 100644
--- a/.github/workflows/react-cloudflare-pages.yml
+++ b/.github/workflows/react-cloudflare-pages.yml
@@ -26,7 +26,7 @@ jobs:
frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }}
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- uses: ./.github/actions/ci-optimization
id: ci-optimize
@@ -40,7 +40,7 @@ jobs:
if: ${{ github.event.pull_request.head.repo.fork != 'true' }}
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml
index ab22d9040e916a..cf7343f397bc4c 100644
--- a/.github/workflows/spark-smoke-test.yml
+++ b/.github/workflows/spark-smoke-test.yml
@@ -30,7 +30,7 @@ jobs:
spark-smoke-test:
runs-on: ubuntu-latest
steps:
- - uses: acryldata/sane-checkout-action@v3
+ - uses: acryldata/sane-checkout-action@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
diff --git a/.github/workflows/yaml-format.yml b/.github/workflows/yaml-format.yml
index f508672fbc4f07..75e92b7c71fbd5 100644
--- a/.github/workflows/yaml-format.yml
+++ b/.github/workflows/yaml-format.yml
@@ -21,7 +21,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out the repo
- uses: acryldata/sane-checkout-action@v3
+ uses: acryldata/sane-checkout-action@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
diff --git a/build.gradle b/build.gradle
index 57ea672dfdf120..397ed5aeb4c160 100644
--- a/build.gradle
+++ b/build.gradle
@@ -38,7 +38,7 @@ buildscript {
ext.pegasusVersion = '29.65.7'
ext.mavenVersion = '3.6.3'
ext.versionGradle = '8.14.3'
- ext.springVersion = '6.2.5'
+ ext.springVersion = '6.2.8'
ext.springBootVersion = '3.4.5'
ext.springKafkaVersion = '3.3.6'
ext.openTelemetryVersion = '1.49.0'
@@ -60,7 +60,7 @@ buildscript {
ext.hazelcastVersion = '5.3.6'
ext.ebeanVersion = '15.5.2'
ext.googleJavaFormatVersion = '1.18.1'
- ext.openLineageVersion = '1.25.0'
+ ext.openLineageVersion = '1.33.0'
ext.logbackClassicJava8 = '1.2.12'
ext.awsSdk2Version = '2.30.33'
ext.micrometerVersion = '1.15.1'
@@ -135,7 +135,7 @@ project.ext.externalDependency = [
'commonsCli': 'commons-cli:commons-cli:1.5.0',
'commonsIo': 'commons-io:commons-io:2.17.0',
'commonsLang': 'commons-lang:commons-lang:2.6',
- 'commonsText': 'org.apache.commons:commons-text:1.12.0',
+ 'commonsText': 'org.apache.commons:commons-text:1.14.0',
'commonsCollections': 'commons-collections:commons-collections:3.2.2',
'caffeine': 'com.github.ben-manes.caffeine:caffeine:3.1.8',
'datastaxOssNativeProtocol': 'com.datastax.oss:native-protocol:1.5.1',
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
index 37ef125cfd390e..0fda1205e502d0 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -2460,6 +2460,16 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) {
? dataJob.getDataPlatformInstance().getUrn()
: null;
}))
+ .dataFetcher(
+ "platform",
+ new LoadableTypeResolver<>(
+ dataPlatformType,
+ (env) -> {
+ final DataJob dataJob = env.getSource();
+ return dataJob != null && dataJob.getPlatform() != null
+ ? dataJob.getPlatform().getUrn()
+ : null;
+ }))
.dataFetcher(
"container",
new LoadableTypeResolver<>(
diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java
index 0158888ac2d733..2e440dd3040b7f 100644
--- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java
+++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java
@@ -113,8 +113,11 @@ public DataJob apply(
} else if (DEPRECATION_ASPECT_NAME.equals(name)) {
result.setDeprecation(DeprecationMapper.map(context, new Deprecation(data)));
} else if (DATA_PLATFORM_INSTANCE_ASPECT_NAME.equals(name)) {
- result.setDataPlatformInstance(
- DataPlatformInstanceAspectMapper.map(context, new DataPlatformInstance(data)));
+ DataPlatformInstance dataPlatformInstance = new DataPlatformInstance(data);
+ com.linkedin.datahub.graphql.generated.DataPlatformInstance value =
+ DataPlatformInstanceAspectMapper.map(context, dataPlatformInstance);
+ result.setPlatform(value.getPlatform());
+ result.setDataPlatformInstance(value);
} else if (CONTAINER_ASPECT_NAME.equals(name)) {
final com.linkedin.container.Container gmsContainer =
new com.linkedin.container.Container(data);
diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
index a6e681f9a49e76..ab0b4db879624e 100644
--- a/datahub-graphql-core/src/main/resources/entity.graphql
+++ b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -6798,6 +6798,11 @@ type DataJob implements EntityWithRelationships & Entity & BrowsableEntity {
"""
dataPlatformInstance: DataPlatformInstance
+ """
+ Standardized platform urn where the data job is defined
+ """
+ platform: DataPlatform
+
"""
The parent container in which the entity resides
"""
diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
index 42d9e7185a8483..43de8afd471c20 100644
--- a/datahub-web-react/package.json
+++ b/datahub-web-react/package.json
@@ -125,7 +125,7 @@
"type-check": "tsc --noEmit",
"type-watch": "tsc -w --noEmit",
"storybook": "storybook dev -p 6006",
- "build-storybook": "NODE_OPTIONS='--max-old-space-size=5120 --openssl-legacy-provider' storybook build"
+ "build-storybook": "NODE_OPTIONS='--max-old-space-size=8192 --openssl-legacy-provider' storybook build"
},
"browserslist": {
"production": [
diff --git a/datahub-web-react/src/alchemy-components/components/Carousel/Carousel.stories.tsx b/datahub-web-react/src/alchemy-components/components/Carousel/Carousel.stories.tsx
new file mode 100644
index 00000000000000..4b82403bd40710
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Carousel/Carousel.stories.tsx
@@ -0,0 +1,181 @@
+import { Carousel } from '@components';
+import type { Meta, StoryObj } from '@storybook/react';
+import React from 'react';
+
+const meta = {
+ title: 'Components / Carousel',
+ component: Carousel,
+ parameters: {
+ layout: 'centered',
+ docs: {
+ subtitle: 'A reusable carousel component with customizable styling and behavior.',
+ },
+ },
+ tags: ['autodocs'],
+ argTypes: {
+ autoplay: {
+ description: 'Whether the carousel should autoplay slides',
+ control: 'boolean',
+ },
+ autoplaySpeed: {
+ description: 'Speed of autoplay in milliseconds',
+ control: 'number',
+ },
+ arrows: {
+ description: 'Whether to show navigation arrows',
+ control: 'boolean',
+ },
+ dots: {
+ description: 'Whether to show dot indicators',
+ control: 'boolean',
+ },
+ animateDot: {
+ description: 'Whether to animate the active dot scaling from 0 to full size',
+ control: 'boolean',
+ },
+ dotDuration: {
+ description:
+ 'Duration in milliseconds for the dot scale animation. When autoplay is enabled, this automatically matches autoplaySpeed.',
+ control: 'number',
+ },
+ },
+} satisfies Meta;
+
+export default meta;
+type Story = StoryObj;
+
+// Sample slide content
+const SampleSlide = ({ title, color = '#f0f0f0' }: { title: string; color?: string }) => (
+
+ {title}
+
+);
+
+export const Basic: Story = {
+ args: {
+ autoplay: false,
+ arrows: true,
+ dots: true,
+ },
+ render: (args) => (
+
+
+
+
+
+
+
+ ),
+};
+
+export const Autoplay: Story = {
+ args: {
+ autoplay: true,
+ autoplaySpeed: 2000,
+ arrows: true,
+ dots: true,
+ },
+ render: (args) => (
+
+
+
+
+
+
+
+ ),
+};
+
+export const WithoutArrows: Story = {
+ args: {
+ autoplay: false,
+ arrows: false,
+ dots: true,
+ },
+ render: (args) => (
+
+
+
+
+
+
+
+ ),
+};
+
+export const WithoutDots: Story = {
+ args: {
+ autoplay: false,
+ arrows: true,
+ dots: false,
+ },
+ render: (args) => (
+
+
+
+
+
+
+
+ ),
+};
+
+export const WithAnimatedDot: Story = {
+ args: {
+ autoplay: false,
+ arrows: false,
+ dots: true,
+ animateDot: true,
+ },
+ render: (args) => (
+
+
+
+
+
+
+
+ 🖱️ Manual navigation: Click dots to see 3-second grow animation.
+
+ 💡 For autoplay, use the "AutoplayWithProgressDots" story.
+
+
+ ),
+};
+
+export const AutoplayWithProgressDots: Story = {
+ args: {
+ autoplay: true,
+ autoplaySpeed: 3000,
+ arrows: false,
+ dots: true,
+ infinite: false, // Stops on last slide instead of looping
+ animateDot: true,
+ },
+ render: (args) => (
+
+
+
+
+
+
+
+ 🎯 Autoplay: Dot animation duration automatically matches autoplay speed (3s).
+
+ ⏹️ Stops on last slide (no infinite loop).
+
+
+ ),
+};
diff --git a/datahub-web-react/src/alchemy-components/components/Carousel/Carousel.tsx b/datahub-web-react/src/alchemy-components/components/Carousel/Carousel.tsx
new file mode 100644
index 00000000000000..614098c7bd750e
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Carousel/Carousel.tsx
@@ -0,0 +1,173 @@
+import { Carousel as AntCarousel, CarouselProps as AntCarouselProps } from 'antd';
+import React, { ReactNode, forwardRef } from 'react';
+import styled, { css, keyframes } from 'styled-components';
+
+import colors from '@components/theme/foundations/colors';
+
+const scaleProgress = keyframes`
+ 0% {
+ transform: scale(0);
+ }
+ 100% {
+ transform: scale(1);
+ }
+`;
+
+const CarouselContainer = styled.div`
+ position: relative;
+`;
+
+const StyledCarousel = styled(AntCarousel)<{ $animateDot?: boolean; $dotDuration?: number }>`
+ .slick-dots {
+ display: flex !important;
+ justify-content: center;
+ align-items: center;
+ width: auto !important;
+ pointer-events: none; /* Allow clicks to pass through */
+
+ li {
+ pointer-events: auto; /* Re-enable clicks on individual dots */
+ margin: 0 4px !important;
+ width: 12px !important;
+ height: 12px !important;
+ display: flex !important;
+
+ button {
+ width: 12px !important;
+ height: 12px !important;
+ margin: 0 !important;
+ padding: 0 !important;
+ border-radius: 50%;
+ background: ${colors.gray[300]};
+ border: none;
+ opacity: 0.6;
+ transition:
+ background-color 0.2s ease,
+ opacity 0.2s ease;
+ transform-origin: center;
+
+ &:hover {
+ background: ${colors.gray[500]};
+ opacity: 1;
+ }
+ }
+
+ &.slick-active button {
+ background: ${({ $animateDot }) => ($animateDot ? colors.gray[300] : colors.primary[600])};
+ opacity: 1;
+ position: relative;
+
+ ${({ $animateDot, $dotDuration }) =>
+ $animateDot && $dotDuration
+ ? css`
+ &::before {
+ content: '';
+ position: absolute;
+ top: 0;
+ left: 0;
+ width: 100%;
+ height: 100%;
+ border-radius: 50%;
+ background: ${colors.primary[600]};
+ transform: scale(0);
+ animation: ${scaleProgress} ${$dotDuration}ms ease-out forwards;
+ }
+ `
+ : ''}
+ }
+ }
+ }
+
+ .slick-prev,
+ .slick-next {
+ width: 40px;
+ height: 40px;
+ z-index: 2;
+
+ &:before {
+ font-size: 20px;
+ color: ${colors.gray[600]};
+ transition: color 0.2s ease;
+ }
+
+ &:hover:before {
+ color: ${colors.primary[600]};
+ }
+ margin: 0 10px;
+ }
+
+ .slick-slide {
+ text-align: center;
+ min-height: 200px;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ border-radius: 8px;
+ }
+`;
+
+const RightComponentContainer = styled.div`
+ position: absolute;
+ bottom: -2px;
+ right: 14px;
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ z-index: 20; /* Higher than dots to ensure clicks work */
+ pointer-events: auto; /* Ensure this container captures clicks */
+`;
+
+const LeftComponentContainer = styled.div`
+ position: absolute;
+ bottom: -2px;
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ z-index: 20; /* Higher than dots to ensure clicks work */
+ pointer-events: auto; /* Ensure this container captures clicks */
+`;
+
+interface CarouselProps extends AntCarouselProps {
+ rightComponent?: ReactNode;
+ leftComponent?: ReactNode;
+ animateDot?: boolean;
+ dotDuration?: number;
+}
+
+export const Carousel = forwardRef(
+ (
+ {
+ autoplay = false,
+ autoplaySpeed = 5000,
+ arrows = false,
+ dots = true,
+ rightComponent,
+ leftComponent,
+ animateDot = false,
+ dotDuration,
+ ...props
+ },
+ ref,
+ ) => {
+ // When animateDot is enabled, use autoplaySpeed as the dot duration
+ // This ensures the visual progress matches the actual timing
+ const effectiveDotDuration = animateDot && autoplay ? autoplaySpeed : dotDuration;
+
+ return (
+
+
+ {leftComponent && {leftComponent} }
+ {rightComponent && {rightComponent} }
+
+ );
+ },
+);
diff --git a/datahub-web-react/src/alchemy-components/components/Carousel/__tests__/Carousel.test.tsx b/datahub-web-react/src/alchemy-components/components/Carousel/__tests__/Carousel.test.tsx
new file mode 100644
index 00000000000000..45aaea16c052f1
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Carousel/__tests__/Carousel.test.tsx
@@ -0,0 +1,156 @@
+import { render, screen } from '@testing-library/react';
+import React from 'react';
+import { describe, expect, it } from 'vitest';
+
+import { Carousel } from '@components/components/Carousel/Carousel';
+
+// Test slides component
+const TestSlide = ({ children }: { children: React.ReactNode }) => {children}
;
+
+describe('Carousel', () => {
+ describe('Basic functionality', () => {
+ it('renders with default props', () => {
+ const { container } = render(
+
+ Slide 1
+ ,
+ );
+
+ const carousel = container.querySelector('.slick-slider');
+ expect(carousel).toBeInTheDocument();
+ });
+
+ it('renders right component when provided', () => {
+ const RightComponent = () => (
+
+ Next
+
+ );
+
+ render(
+ }>
+ Slide 1
+ ,
+ );
+
+ expect(screen.getByTestId('right-component')).toBeInTheDocument();
+ });
+
+ it('renders left component when provided', () => {
+ const LeftComponent = () => (
+
+ Previous
+
+ );
+
+ render(
+ }>
+ Slide 1
+ ,
+ );
+
+ expect(screen.getByTestId('left-component')).toBeInTheDocument();
+ });
+ });
+
+ describe('Animation props without rendering styled errors', () => {
+ it('accepts animateDot and dotDuration props', () => {
+ const { container } = render(
+
+ Slide 1
+ ,
+ );
+
+ const carousel = container.querySelector('.slick-slider');
+ expect(carousel).toBeInTheDocument();
+ });
+
+ it('works without animation props', () => {
+ const { container } = render(
+
+ Slide 1
+ ,
+ );
+
+ const carousel = container.querySelector('.slick-slider');
+ expect(carousel).toBeInTheDocument();
+ });
+
+ it('passes through standard Ant Design Carousel props', () => {
+ const { container } = render(
+
+ Slide 1
+ ,
+ );
+
+ const carousel = container.querySelector('.slick-slider');
+ expect(carousel).toBeInTheDocument();
+ });
+
+ it('combines all props together', () => {
+ const { container } = render(
+
+ Slide 1
+ Slide 2
+ ,
+ );
+
+ const carousel = container.querySelector('.slick-slider');
+ expect(carousel).toBeInTheDocument();
+ });
+ });
+
+ describe('Interface coverage', () => {
+ it('handles all animateDot prop variations', () => {
+ // Test true value
+ const { container: container1 } = render(
+
+ Test
+ ,
+ );
+ expect(container1.querySelector('.slick-slider')).toBeInTheDocument();
+
+ // Test false value
+ const { container: container2 } = render(
+
+ Test
+ ,
+ );
+ expect(container2.querySelector('.slick-slider')).toBeInTheDocument();
+
+ // Test undefined (default)
+ const { container: container3 } = render(
+
+ Test
+ ,
+ );
+ expect(container3.querySelector('.slick-slider')).toBeInTheDocument();
+ });
+
+ it('handles all dotDuration prop variations', () => {
+ // Test positive number
+ const { container: container1 } = render(
+
+ Test
+ ,
+ );
+ expect(container1.querySelector('.slick-slider')).toBeInTheDocument();
+
+ // Test zero
+ const { container: container2 } = render(
+
+ Test
+ ,
+ );
+ expect(container2.querySelector('.slick-slider')).toBeInTheDocument();
+
+ // Test undefined (default)
+ const { container: container3 } = render(
+
+ Test
+ ,
+ );
+ expect(container3.querySelector('.slick-slider')).toBeInTheDocument();
+ });
+ });
+});
diff --git a/datahub-web-react/src/alchemy-components/components/Carousel/index.ts b/datahub-web-react/src/alchemy-components/components/Carousel/index.ts
new file mode 100644
index 00000000000000..c6fd6852b62a83
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/Carousel/index.ts
@@ -0,0 +1,2 @@
+export { Carousel } from './Carousel';
+export type { CarouselProps } from 'antd';
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedImage/LoadedImage.stories.tsx b/datahub-web-react/src/alchemy-components/components/LoadedImage/LoadedImage.stories.tsx
new file mode 100644
index 00000000000000..db617055287a4d
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedImage/LoadedImage.stories.tsx
@@ -0,0 +1,91 @@
+import { LoadedImage } from '@components';
+import type { Meta, StoryObj } from '@storybook/react';
+import React from 'react';
+
+const meta = {
+ title: 'Media / LoadedImage',
+ component: LoadedImage,
+ parameters: {
+ layout: 'centered',
+ docs: {
+ subtitle: 'A reusable image component with loading states, error handling, and skeleton placeholder.',
+ },
+ },
+ tags: ['autodocs'],
+ argTypes: {
+ src: {
+ description: 'Image source URL',
+ control: 'text',
+ },
+ alt: {
+ description: 'Alternative text for the image',
+ control: 'text',
+ },
+ width: {
+ description: 'Width of the image container',
+ control: 'text',
+ },
+ errorMessage: {
+ description: 'Custom error message when image fails to load',
+ control: 'text',
+ },
+ showErrorDetails: {
+ description: 'Whether to show the alt text in error message',
+ control: 'boolean',
+ },
+ },
+} satisfies Meta;
+
+export default meta;
+type Story = StoryObj;
+
+// Valid image URL for testing
+const SAMPLE_IMAGE_URL = 'https://picsum.photos/400/300';
+
+export const Basic: Story = {
+ args: {
+ src: SAMPLE_IMAGE_URL,
+ alt: 'Sample image',
+ width: '400px',
+ },
+};
+
+export const WithCustomWidth: Story = {
+ args: {
+ src: SAMPLE_IMAGE_URL,
+ alt: 'Sample image with custom width',
+ width: '600px',
+ },
+};
+
+export const ErrorState: Story = {
+ args: {
+ src: 'https://invalid-url-that-will-fail.jpg',
+ alt: 'This image will fail to load',
+ width: '400px',
+ },
+};
+
+export const ErrorStateWithCustomMessage: Story = {
+ args: {
+ src: 'https://invalid-url-that-will-fail.jpg',
+ alt: 'This image will fail to load',
+ width: '400px',
+ errorMessage: 'Custom error: Failed to load image',
+ showErrorDetails: false,
+ },
+};
+
+export const LoadingState: Story = {
+ args: {
+ src: 'https://picsum.photos/800/600', // Larger image to show loading
+ alt: 'Large image that will show loading state',
+ width: '400px',
+ },
+ render: (args) => (
+
+
This image should show a loading skeleton briefly while the image loads.
+
+
+ ),
+};
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedImage/LoadedImage.tsx b/datahub-web-react/src/alchemy-components/components/LoadedImage/LoadedImage.tsx
new file mode 100644
index 00000000000000..795c703924b9cd
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedImage/LoadedImage.tsx
@@ -0,0 +1,94 @@
+import { Skeleton } from 'antd';
+import React, { useState } from 'react';
+import styled from 'styled-components';
+
+import { colors } from '@src/alchemy-components';
+import type { LoadedImageProps } from '@src/alchemy-components/components/LoadedImage/types';
+
+const ImageContainer = styled.div<{ width?: string }>`
+ width: ${(props) => props.width || 'auto'};
+ margin: auto;
+ padding-bottom: 2em;
+`;
+
+const StyledImage = styled.img<{ width?: string }>`
+ width: ${(props) => props.width || 'auto'};
+ height: auto;
+ border-radius: 8px;
+ margin: auto;
+ padding-bottom: 2em;
+`;
+
+const ImageSkeleton = styled(Skeleton.Image)<{ width?: string }>`
+ &&& {
+ width: ${(props) => props.width || 'auto'};
+ height: auto;
+
+ .ant-skeleton-image {
+ width: ${(props) => props.width || 'auto'};
+ height: auto;
+ }
+ }
+`;
+
+const ErrorPlaceholder = styled.div<{ width?: string }>`
+ width: ${(props) => props.width || 'auto'};
+ height: auto;
+ background: ${colors.gray[1500]};
+ border-radius: 8px;
+ color: ${colors.gray[1800]};
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ border: 2px dashed ${colors.gray[1400]};
+ font-size: 14px;
+ min-height: 100px;
+`;
+
+export const LoadedImage = ({
+ src,
+ alt,
+ width,
+ errorMessage = 'Image not found',
+ showErrorDetails = true,
+ onLoad,
+ onError,
+ ...props
+}: LoadedImageProps) => {
+ const [loading, setLoading] = useState(true);
+ const [error, setError] = useState(false);
+
+ const handleImageLoad = () => {
+ setLoading(false);
+ onLoad?.();
+ };
+
+ const handleImageError = () => {
+ setLoading(false);
+ setError(true);
+ onError?.();
+ };
+
+ return (
+
+ {loading && }
+ {!error && (
+
+ )}
+ {error && (
+
+ {errorMessage}
+ {showErrorDetails && `: ${alt}`}
+
+ )}
+
+ );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedImage/index.ts b/datahub-web-react/src/alchemy-components/components/LoadedImage/index.ts
new file mode 100644
index 00000000000000..59c28884863568
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedImage/index.ts
@@ -0,0 +1,2 @@
+export { LoadedImage } from './LoadedImage';
+export type { LoadedImageProps } from './types';
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedImage/types.ts b/datahub-web-react/src/alchemy-components/components/LoadedImage/types.ts
new file mode 100644
index 00000000000000..dde61659e305e7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedImage/types.ts
@@ -0,0 +1,11 @@
+import { ImgHTMLAttributes } from 'react';
+
+export interface LoadedImageProps extends Omit, 'onLoad' | 'onError'> {
+ src: string;
+ alt: string;
+ width?: string;
+ errorMessage?: string;
+ showErrorDetails?: boolean;
+ onLoad?: () => void;
+ onError?: () => void;
+}
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedVideo/LoadedVideo.components.tsx b/datahub-web-react/src/alchemy-components/components/LoadedVideo/LoadedVideo.components.tsx
new file mode 100644
index 00000000000000..5e83132ee46f2c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedVideo/LoadedVideo.components.tsx
@@ -0,0 +1,129 @@
+import styled from 'styled-components';
+
+import colors from '@components/theme/foundations/colors';
+
+const STORY_WIDTH = '600px';
+const STORY_PADDING = '20px';
+
+export const StoryContainer = styled.div`
+ width: ${STORY_WIDTH};
+ padding: ${STORY_PADDING};
+`;
+
+export const StoryTitle = styled.h3`
+ margin-bottom: 20px;
+ text-align: center;
+ color: ${colors.gray[600]};
+ font-size: 18px;
+ font-weight: 600;
+`;
+
+export const StoryDescription = styled.p`
+ margin-bottom: 20px;
+ text-align: center;
+ color: ${colors.gray[1700]};
+ font-size: 14px;
+ line-height: 1.5;
+`;
+
+export const CarouselContainer = styled.div`
+ border: 1px solid ${colors.gray[100]};
+ border-radius: 8px;
+ padding: 20px;
+`;
+
+export const SlideContainer = styled.div`
+ text-align: center;
+ padding: 20px 0;
+`;
+
+export const SlideTitle = styled.h4`
+ margin-bottom: 16px;
+ font-size: 16px;
+ font-weight: 600;
+ color: ${colors.gray[600]};
+`;
+
+export const VideoContainer = styled.div`
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ margin: 0 auto;
+`;
+
+export const ErrorContainer = styled.div`
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ width: 400px;
+ height: 225px;
+ background-color: ${colors.red[0]};
+ border: 2px dashed ${colors.red[500]};
+ border-radius: 8px;
+ margin: 0 auto;
+ font-size: 16px;
+ color: ${colors.red[500]};
+ font-weight: 500;
+`;
+
+export const LoadingContainer = styled.div`
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ width: 400px;
+ height: 225px;
+ background-color: ${colors.gray[1500]};
+ border: 2px dashed ${colors.gray[100]};
+ border-radius: 8px;
+ margin: 0 auto;
+ font-size: 16px;
+ color: ${colors.gray[1700]};
+ font-weight: 500;
+`;
+
+export const LoadingOverlay = styled.div`
+ position: absolute;
+ top: 0;
+ left: 0;
+ right: 0;
+ bottom: 0;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ background-color: ${colors.gray[1500]};
+ z-index: 1;
+ border-radius: 4px;
+`;
+
+export const VideoPlayerContainer = styled.div`
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ margin: 0 auto;
+ border-radius: 8px;
+ overflow: hidden;
+`;
+
+export const IndicatorContainer = styled.div`
+ margin-top: 20px;
+ text-align: center;
+`;
+
+export const IndicatorRow = styled.div`
+ display: flex;
+ justify-content: center;
+ gap: 10px;
+`;
+
+export const IndicatorDot = styled.div<{ $isActive: boolean }>`
+ width: 12px;
+ height: 12px;
+ border-radius: 50%;
+ background-color: ${({ $isActive }) => ($isActive ? colors.primary[500] : colors.gray[100])};
+`;
+
+export const IndicatorText = styled.p`
+ margin-top: 10px;
+ font-size: 12px;
+ color: ${colors.gray[1800]};
+`;
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedVideo/LoadedVideo.stories.tsx b/datahub-web-react/src/alchemy-components/components/LoadedVideo/LoadedVideo.stories.tsx
new file mode 100644
index 00000000000000..75397e67fa909c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedVideo/LoadedVideo.stories.tsx
@@ -0,0 +1,261 @@
+import type { Meta, StoryObj } from '@storybook/react';
+import React from 'react';
+
+import { LoadedVideo } from '@src/alchemy-components/components/LoadedVideo/LoadedVideo';
+import {
+ CarouselContainer,
+ ErrorContainer,
+ IndicatorContainer,
+ IndicatorText,
+ LoadingContainer,
+ SlideContainer,
+ SlideTitle,
+ StoryContainer,
+ StoryDescription,
+ StoryTitle,
+ VideoContainer,
+ VideoPlayerContainer,
+} from '@src/alchemy-components/components/LoadedVideo/LoadedVideo.components';
+import { VIDEO_WIDTH } from '@src/alchemy-components/components/LoadedVideo/constants';
+
+const SAMPLE_VIDEO_URL = 'https://www.sample-videos.com/video321/mp4/720/big_buck_bunny_720p_1mb.mp4';
+
+const meta = {
+ title: 'Example/LoadedVideo',
+ component: LoadedVideo,
+ parameters: {
+ layout: 'centered',
+ },
+ tags: ['autodocs'],
+ argTypes: {
+ mp4Src: {
+ control: 'text',
+ description: 'URL to the MP4 video file',
+ },
+ webmSrc: {
+ control: 'text',
+ description: 'URL to the WebM video file (optional)',
+ },
+ posterSrc: {
+ control: 'text',
+ description: 'URL to the poster image displayed before video loads',
+ },
+ width: {
+ control: 'text',
+ description: 'Width of the video (CSS value)',
+ },
+ height: {
+ control: 'text',
+ description: 'Height of the video (CSS value)',
+ },
+ autoplay: {
+ control: 'boolean',
+ description: 'Whether the video should autoplay',
+ },
+ loop: {
+ control: 'boolean',
+ description: 'Whether the video should loop',
+ },
+ muted: {
+ control: 'boolean',
+ description: 'Whether the video should be muted',
+ },
+ controls: {
+ control: 'boolean',
+ description: 'Whether to show video controls',
+ },
+ playsInline: {
+ control: 'boolean',
+ description: 'Whether the video should play inline on mobile',
+ },
+ },
+} satisfies Meta;
+
+export default meta;
+type Story = StoryObj;
+
+export const Default: Story = {
+ args: {
+ mp4Src: SAMPLE_VIDEO_URL,
+ width: '400px',
+ autoplay: true,
+ loop: true,
+ muted: true,
+ controls: false,
+ playsInline: true,
+ },
+ render: (args) => (
+
+
+
+ ),
+};
+
+export const WithControls: Story = {
+ args: {
+ mp4Src: SAMPLE_VIDEO_URL,
+ width: '400px',
+ autoplay: false,
+ loop: true,
+ muted: false,
+ controls: true,
+ playsInline: true,
+ },
+ render: (args) => (
+
+
+
+ ),
+};
+
+export const CustomSize: Story = {
+ args: {
+ mp4Src: SAMPLE_VIDEO_URL,
+ width: '300px',
+ height: '200px',
+ autoplay: true,
+ loop: true,
+ muted: true,
+ controls: false,
+ playsInline: true,
+ },
+ render: (args) => (
+
+
+
+ ),
+};
+
+export const NoAutoplay: Story = {
+ args: {
+ mp4Src: SAMPLE_VIDEO_URL,
+ width: '400px',
+ autoplay: false,
+ loop: true,
+ muted: true,
+ controls: true,
+ playsInline: true,
+ },
+ render: (args) => (
+
+
+
+ ),
+};
+
+export const WithPoster: Story = {
+ args: {
+ mp4Src: SAMPLE_VIDEO_URL,
+ posterSrc: 'https://via.placeholder.com/400x300/cccccc/666666?text=Video+Poster',
+ width: '400px',
+ autoplay: false,
+ loop: true,
+ muted: true,
+ controls: true,
+ playsInline: true,
+ },
+ render: (args) => (
+
+
+
+ ),
+};
+
+export const LoadingState: Story = {
+ args: {
+ mp4Src: 'https://httpstat.us/200?sleep=3000', // Simulates slow loading
+ width: '400px',
+ autoplay: true,
+ loop: true,
+ muted: true,
+ controls: false,
+ playsInline: true,
+ },
+ render: (_args) => (
+
+ Loading State Demo
+
+ This demonstrates the loading state while the video loads.
+
+ The video has a 3-second delay to simulate slow loading.
+
+ Loading video... Please wait
+
+ Simulated with 3-second delay via httpstat.us
+
+
+ ),
+};
+
+export const ErrorState: Story = {
+ args: {
+ mp4Src: 'https://invalid-url.com/nonexistent-video.mp4',
+ width: '400px',
+ autoplay: true,
+ loop: true,
+ muted: true,
+ controls: false,
+ playsInline: true,
+ },
+ render: (_args) => (
+
+ Error State Demo
+
+ This demonstrates the error state when a video fails to load.
+
+ Using an invalid URL to trigger the error state.
+
+ ⚠️ Failed to load video
+
+ Simulated with invalid video URL
+
+
+ ),
+};
+
+export const SingleVideoDemo: Story = {
+ args: {
+ mp4Src: SAMPLE_VIDEO_URL,
+ width: '520px',
+ autoplay: true,
+ loop: true,
+ muted: true,
+ controls: false,
+ playsInline: true,
+ },
+ render: (args) => {
+ return (
+
+ Single Video Demo
+
+ Demonstrating the LoadedVideo component with a sample video from sample-videos.com.
+
+ This video will automatically loop and play.
+
+
+
+ Big Buck Bunny Sample Video
+
+
+
+
+
+
+
+
+
+ Sample video courtesy of sample-videos.com - Big Buck Bunny (1MB, 720p)
+
+
+
+ );
+ },
+};
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedVideo/LoadedVideo.tsx b/datahub-web-react/src/alchemy-components/components/LoadedVideo/LoadedVideo.tsx
new file mode 100644
index 00000000000000..2e09c64a6b16da
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedVideo/LoadedVideo.tsx
@@ -0,0 +1,146 @@
+import { Spin } from 'antd';
+import React, { useEffect, useRef, useState } from 'react';
+
+import {
+ ErrorContainer,
+ LoadingOverlay,
+ VideoPlayerContainer,
+} from '@src/alchemy-components/components/LoadedVideo/LoadedVideo.components';
+
+interface LoadedVideoProps {
+ mp4Src: string;
+ webmSrc?: string;
+ posterSrc?: string;
+ width?: string;
+ height?: string;
+ autoplay?: boolean;
+ loop?: boolean;
+ muted?: boolean;
+ controls?: boolean;
+ playsInline?: boolean;
+ className?: string;
+ onVideoLoad?: () => void;
+ onVideoError?: () => void;
+ onVideoEnded?: () => void;
+ onVideoTimeUpdate?: (currentTime: number, duration: number) => void;
+ onVideoCanPlay?: () => void;
+ videoRef?: React.RefObject;
+}
+
+export const LoadedVideo: React.FC = ({
+ mp4Src,
+ webmSrc,
+ posterSrc,
+ width,
+ height,
+ autoplay = true,
+ loop = true,
+ muted = true,
+ controls = false,
+ playsInline = true,
+ className,
+ onVideoLoad,
+ onVideoError,
+ onVideoEnded,
+ onVideoTimeUpdate,
+ onVideoCanPlay,
+ videoRef,
+}) => {
+ const [isLoading, setIsLoading] = useState(true);
+ const [hasError, setHasError] = useState(false);
+ const internalVideoRef = useRef(null);
+ const currentVideoRef = videoRef || internalVideoRef;
+
+ useEffect(() => {
+ const video = currentVideoRef.current;
+ if (!video) return undefined;
+
+ const handleLoadedData = () => {
+ setIsLoading(false);
+ onVideoLoad?.();
+ };
+
+ const handleError = () => {
+ setIsLoading(false);
+ setHasError(true);
+ onVideoError?.();
+ };
+
+ const handleCanPlay = () => {
+ onVideoCanPlay?.();
+ };
+
+ const handleEnded = () => {
+ onVideoEnded?.();
+ };
+
+ const handleTimeUpdate = () => {
+ if (onVideoTimeUpdate) {
+ onVideoTimeUpdate(video.currentTime, video.duration);
+ }
+ };
+
+ video.addEventListener('loadeddata', handleLoadedData);
+ video.addEventListener('error', handleError);
+ video.addEventListener('canplay', handleCanPlay);
+ video.addEventListener('ended', handleEnded);
+ video.addEventListener('timeupdate', handleTimeUpdate);
+
+ return () => {
+ video.removeEventListener('loadeddata', handleLoadedData);
+ video.removeEventListener('error', handleError);
+ video.removeEventListener('canplay', handleCanPlay);
+ video.removeEventListener('ended', handleEnded);
+ video.removeEventListener('timeupdate', handleTimeUpdate);
+ };
+ }, [onVideoLoad, onVideoError, onVideoCanPlay, onVideoEnded, onVideoTimeUpdate, currentVideoRef]);
+
+ if (hasError) {
+ return Failed to load video ;
+ }
+
+ return (
+
+ {isLoading && (
+
+
+
+ )}
+
+
+
+ {webmSrc && }
+
+
+ Your browser doesn't support HTML5 video. Here's a{' '}
+ link to the video instead.
+
+
+
+
+ );
+};
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedVideo/__tests__/constants.test.ts b/datahub-web-react/src/alchemy-components/components/LoadedVideo/__tests__/constants.test.ts
new file mode 100644
index 00000000000000..b5de8c151573b7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedVideo/__tests__/constants.test.ts
@@ -0,0 +1,7 @@
+import { VIDEO_WIDTH } from '@components/components/LoadedVideo/constants';
+
+describe('LoadedVideo constants', () => {
+ it('should be a string value', () => {
+ expect(typeof VIDEO_WIDTH).toBe('string');
+ });
+});
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedVideo/constants.ts b/datahub-web-react/src/alchemy-components/components/LoadedVideo/constants.ts
new file mode 100644
index 00000000000000..769ef928a6908c
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedVideo/constants.ts
@@ -0,0 +1 @@
+export const VIDEO_WIDTH = '520px';
diff --git a/datahub-web-react/src/alchemy-components/components/LoadedVideo/index.ts b/datahub-web-react/src/alchemy-components/components/LoadedVideo/index.ts
new file mode 100644
index 00000000000000..cb83d15b9bc5b7
--- /dev/null
+++ b/datahub-web-react/src/alchemy-components/components/LoadedVideo/index.ts
@@ -0,0 +1 @@
+export * from './LoadedVideo';
diff --git a/datahub-web-react/src/alchemy-components/index.ts b/datahub-web-react/src/alchemy-components/index.ts
index 84d0c6bdd5f7a2..1262f50b894a3f 100644
--- a/datahub-web-react/src/alchemy-components/index.ts
+++ b/datahub-web-react/src/alchemy-components/index.ts
@@ -10,6 +10,7 @@ export * from './components/BarChart';
export * from './components/Button';
export * from './components/CalendarChart';
export * from './components/Card';
+export * from './components/Carousel';
export * from './components/Checkbox';
export * from './components/ColorPicker';
export * from './components/DatePicker';
@@ -20,6 +21,8 @@ export * from './components/Heading';
export * from './components/Icon';
export * from './components/Input';
export * from './components/LineChart';
+export * from './components/LoadedImage';
+export * from './components/LoadedVideo';
export * from './components/Loader';
export * from './components/MatchText';
export * from './components/Modal';
diff --git a/datahub-web-react/src/app/AppProviders.tsx b/datahub-web-react/src/app/AppProviders.tsx
index c5237e75d0f18e..f1597b7a16076c 100644
--- a/datahub-web-react/src/app/AppProviders.tsx
+++ b/datahub-web-react/src/app/AppProviders.tsx
@@ -5,6 +5,7 @@ import GlobalSettingsProvider from '@app/context/GlobalSettingsProvider';
import UserContextProvider from '@app/context/UserContextProvider';
import { NavBarProvider } from '@app/homeV2/layout/navBarRedesign/NavBarContext';
import HomePageProvider from '@app/homeV3/context/HomePageProvider';
+import OnboardingTourProvider from '@app/onboarding/OnboardingTourContextProvider';
import SearchContextProvider from '@app/search/context/SearchContextProvider';
import { BrowserTitleProvider } from '@app/shared/BrowserTabTitleContext';
import { EducationStepsProvider } from '@providers/EducationStepsProvider';
@@ -23,13 +24,15 @@ export default function AppProviders({ children }: Props) {
-
-
-
- {children}
-
-
-
+
+
+
+
+ {children}
+
+
+
+
diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts
index 60499954a57545..f9ee264b21ffbb 100644
--- a/datahub-web-react/src/app/analytics/event.ts
+++ b/datahub-web-react/src/app/analytics/event.ts
@@ -137,6 +137,11 @@ export enum EventType {
HomePageTemplateModuleExpandClick,
HomePageTemplateModuleLinkClick,
HomePageTemplateModuleAnnouncementDismiss,
+ WelcomeToDataHubModalViewEvent,
+ WelcomeToDataHubModalInteractEvent,
+ WelcomeToDataHubModalExitEvent,
+ WelcomeToDataHubModalClickViewDocumentationEvent,
+ ProductTourButtonClickEvent,
}
/**
@@ -900,6 +905,33 @@ export interface SearchBarFilterEvent extends BaseEvent {
values: string[]; // the values being filtered for
}
+export interface WelcomeToDataHubModalViewEvent extends BaseEvent {
+ type: EventType.WelcomeToDataHubModalViewEvent;
+}
+
+export interface WelcomeToDataHubModalInteractEvent extends BaseEvent {
+ type: EventType.WelcomeToDataHubModalInteractEvent;
+ currentSlide: number;
+ totalSlides: number;
+}
+
+export interface WelcomeToDataHubModalExitEvent extends BaseEvent {
+ type: EventType.WelcomeToDataHubModalExitEvent;
+ currentSlide: number;
+ totalSlides: number;
+ exitMethod: 'close_button' | 'get_started_button' | 'outside_click' | 'escape_key';
+}
+
+export interface WelcomeToDataHubModalClickViewDocumentationEvent extends BaseEvent {
+ type: EventType.WelcomeToDataHubModalClickViewDocumentationEvent;
+ url: string;
+}
+
+export interface ProductTourButtonClickEvent extends BaseEvent {
+ type: EventType.ProductTourButtonClickEvent;
+ originPage: string; // Page where the button was clicked
+}
+
export interface ClickProductUpdateEvent extends BaseEvent {
type: EventType.ClickProductUpdate;
id: string;
@@ -1113,4 +1145,9 @@ export type Event =
| HomePageTemplateModuleExpandClickEvent
| HomePageTemplateModuleViewAllClickEvent
| HomePageTemplateModuleLinkClickEvent
- | HomePageTemplateModuleAnnouncementDismissEvent;
+ | HomePageTemplateModuleAnnouncementDismissEvent
+ | WelcomeToDataHubModalViewEvent
+ | WelcomeToDataHubModalInteractEvent
+ | WelcomeToDataHubModalExitEvent
+ | WelcomeToDataHubModalClickViewDocumentationEvent
+ | ProductTourButtonClickEvent;
diff --git a/datahub-web-react/src/app/entityV2/Access/RoleEntity.tsx b/datahub-web-react/src/app/entityV2/Access/RoleEntity.tsx
index 25fcad36fd94c5..258af7aea8a9b6 100644
--- a/datahub-web-react/src/app/entityV2/Access/RoleEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/Access/RoleEntity.tsx
@@ -32,10 +32,7 @@ export class RoleEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/application/ApplicationEntity.tsx b/datahub-web-react/src/app/entityV2/application/ApplicationEntity.tsx
index 68d8209851cc99..38a89d1efb691f 100644
--- a/datahub-web-react/src/app/entityV2/application/ApplicationEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/application/ApplicationEntity.tsx
@@ -57,10 +57,7 @@ export class ApplicationEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/businessAttribute/BusinessAttributeEntity.tsx b/datahub-web-react/src/app/entityV2/businessAttribute/BusinessAttributeEntity.tsx
index 115dc95174edbe..8d575ca7c45dfe 100644
--- a/datahub-web-react/src/app/entityV2/businessAttribute/BusinessAttributeEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/businessAttribute/BusinessAttributeEntity.tsx
@@ -41,14 +41,7 @@ export class BusinessAttributeEntity implements Entity {
);
}
- return (
-
- );
+ return ;
};
displayName = (data: BusinessAttribute) => {
diff --git a/datahub-web-react/src/app/entityV2/chart/ChartEntity.tsx b/datahub-web-react/src/app/entityV2/chart/ChartEntity.tsx
index d96b58a6748194..46f51736b5392e 100644
--- a/datahub-web-react/src/app/entityV2/chart/ChartEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/chart/ChartEntity.tsx
@@ -94,10 +94,7 @@ export class ChartEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/container/ContainerEntity.tsx b/datahub-web-react/src/app/entityV2/container/ContainerEntity.tsx
index 12a99f86ee5303..069da5fb21f8d4 100644
--- a/datahub-web-react/src/app/entityV2/container/ContainerEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/container/ContainerEntity.tsx
@@ -59,10 +59,7 @@ export class ContainerEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entityV2/dashboard/DashboardEntity.tsx
index 7fd35970cde99f..9c08d5ed48f291 100644
--- a/datahub-web-react/src/app/entityV2/dashboard/DashboardEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/dashboard/DashboardEntity.tsx
@@ -93,14 +93,7 @@ export class DashboardEntity implements Entity {
);
}
- return (
-
- );
+ return ;
};
isSearchEnabled = () => true;
diff --git a/datahub-web-react/src/app/entityV2/dataContract/DataContractEntity.tsx b/datahub-web-react/src/app/entityV2/dataContract/DataContractEntity.tsx
index 5970ed5c67439a..8d78a7bff0ac06 100644
--- a/datahub-web-react/src/app/entityV2/dataContract/DataContractEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/dataContract/DataContractEntity.tsx
@@ -24,10 +24,7 @@ export class DataContractEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/dataFlow/DataFlowEntity.tsx b/datahub-web-react/src/app/entityV2/dataFlow/DataFlowEntity.tsx
index d9c78b610e6668..0e109f291c3f85 100644
--- a/datahub-web-react/src/app/entityV2/dataFlow/DataFlowEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/dataFlow/DataFlowEntity.tsx
@@ -58,10 +58,7 @@ export class DataFlowEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/dataJob/DataJobEntity.tsx b/datahub-web-react/src/app/entityV2/dataJob/DataJobEntity.tsx
index 797c888b7e6553..f9dc36805e6da7 100644
--- a/datahub-web-react/src/app/entityV2/dataJob/DataJobEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/dataJob/DataJobEntity.tsx
@@ -37,12 +37,13 @@ import { capitalizeFirstLetterOnly } from '@app/shared/textUtil';
import { GetDataJobQuery, useGetDataJobQuery, useUpdateDataJobMutation } from '@graphql/dataJob.generated';
import { DataJob, DataProcessInstanceResult, EntityType, SearchResult } from '@types';
-const getDataJobPlatformName = (data?: DataJob): string => {
- return (
- data?.dataFlow?.platform?.properties?.displayName ||
- capitalizeFirstLetterOnly(data?.dataFlow?.platform?.name) ||
- ''
- );
+const getPlatformForDataJob = (data?: DataJob | null) => {
+ return data?.platform || data?.dataFlow?.platform;
+};
+
+const getDataJobPlatformName = (data?: DataJob | null): string => {
+ const platform = getPlatformForDataJob(data);
+ return platform?.properties?.displayName || capitalizeFirstLetterOnly(platform?.name) || '';
};
const headerDropdownItems = new Set([
@@ -72,10 +73,7 @@ export class DataJobEntity implements Entity {
return (
);
};
@@ -195,7 +193,7 @@ export class DataJobEntity implements Entity {
return {
name,
externalUrl,
- platform: dataJob?.dataFlow?.platform,
+ platform: getPlatformForDataJob(dataJob),
lastRun: ((dataJob as any).lastRun as DataProcessInstanceResult)?.runs?.[0],
lastRunEvent: ((dataJob as any).lastRun as DataProcessInstanceResult)?.runs?.[0]?.state?.[0],
};
@@ -211,7 +209,7 @@ export class DataJobEntity implements Entity {
subtype={data.subTypes?.typeNames?.[0]}
description={data.editableProperties?.description || data.properties?.description}
platformName={getDataJobPlatformName(data)}
- platformLogo={data?.dataFlow?.platform?.properties?.logoUrl || ''}
+ platformLogo={getPlatformForDataJob(data)?.properties?.logoUrl || ''}
owners={data.ownership?.owners}
globalTags={data.globalTags || null}
domain={data.domain?.domain}
@@ -235,7 +233,7 @@ export class DataJobEntity implements Entity {
subtype={data.subTypes?.typeNames?.[0]}
description={data.editableProperties?.description || data.properties?.description}
platformName={getDataJobPlatformName(data)}
- platformLogo={data?.dataFlow?.platform?.properties?.logoUrl || ''}
+ platformLogo={getPlatformForDataJob(data)?.properties?.logoUrl || ''}
platformInstanceId={data.dataPlatformInstance?.instanceId}
owners={data.ownership?.owners}
globalTags={data.globalTags}
@@ -281,7 +279,7 @@ export class DataJobEntity implements Entity {
name: this.displayName(entity),
expandedName: this.getExpandedNameForDataJob(entity),
type: EntityType.DataJob,
- icon: entity?.dataFlow?.platform?.properties?.logoUrl || undefined, // eslint-disable-next-line @typescript-eslint/dot-notation
+ icon: getPlatformForDataJob(entity)?.properties?.logoUrl || undefined, // eslint-disable-next-line @typescript-eslint/dot-notation
downstreamChildren: entity?.['downstream']?.relationships?.map(
(relationship) =>
({
@@ -296,7 +294,7 @@ export class DataJobEntity implements Entity {
type: relationship.entity.type,
}) as EntityAndType,
),
- platform: entity?.dataFlow?.platform,
+ platform: getPlatformForDataJob(entity),
};
};
diff --git a/datahub-web-react/src/app/entityV2/dataPlatform/DataPlatformEntity.tsx b/datahub-web-react/src/app/entityV2/dataPlatform/DataPlatformEntity.tsx
index 213210fc26f764..74a958e2422da7 100644
--- a/datahub-web-react/src/app/entityV2/dataPlatform/DataPlatformEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/dataPlatform/DataPlatformEntity.tsx
@@ -21,10 +21,7 @@ export class DataPlatformEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx b/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx
index c20151bbbaa6f2..05c4a03844974e 100644
--- a/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx
@@ -46,14 +46,7 @@ export class DataProcessInstanceEntity implements Entity {
return ;
}
- return (
-
- );
+ return ;
};
isSearchEnabled = () => false;
diff --git a/datahub-web-react/src/app/entityV2/dataProduct/DataProductEntity.tsx b/datahub-web-react/src/app/entityV2/dataProduct/DataProductEntity.tsx
index 8c77e6960eb1bb..c2eca66fd940cd 100644
--- a/datahub-web-react/src/app/entityV2/dataProduct/DataProductEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/dataProduct/DataProductEntity.tsx
@@ -69,10 +69,7 @@ export class DataProductEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entityV2/dataset/DatasetEntity.tsx
index 7130d8565ae6a0..51c0088135309c 100644
--- a/datahub-web-react/src/app/entityV2/dataset/DatasetEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/dataset/DatasetEntity.tsx
@@ -112,10 +112,7 @@ export class DatasetEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/domain/DomainEntity.tsx b/datahub-web-react/src/app/entityV2/domain/DomainEntity.tsx
index 4d9b0260b8da51..5d65bc76d090f6 100644
--- a/datahub-web-react/src/app/entityV2/domain/DomainEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/domain/DomainEntity.tsx
@@ -60,14 +60,7 @@ export class DomainEntity implements Entity {
);
}
- return (
-
- );
+ return ;
};
isSearchEnabled = () => true;
diff --git a/datahub-web-react/src/app/entityV2/glossaryNode/GlossaryNodeEntity.tsx b/datahub-web-react/src/app/entityV2/glossaryNode/GlossaryNodeEntity.tsx
index 4bb72c6c8612dc..4ece818fb60dc3 100644
--- a/datahub-web-react/src/app/entityV2/glossaryNode/GlossaryNodeEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/glossaryNode/GlossaryNodeEntity.tsx
@@ -56,10 +56,7 @@ class GlossaryNodeEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/glossaryTerm/GlossaryTermEntity.tsx b/datahub-web-react/src/app/entityV2/glossaryTerm/GlossaryTermEntity.tsx
index 23a02ab7a27923..e329f4101df91f 100644
--- a/datahub-web-react/src/app/entityV2/glossaryTerm/GlossaryTermEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/glossaryTerm/GlossaryTermEntity.tsx
@@ -68,10 +68,7 @@ export class GlossaryTermEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/group/Group.tsx b/datahub-web-react/src/app/entityV2/group/Group.tsx
index b5d8309ccecb4c..e210abf415a073 100644
--- a/datahub-web-react/src/app/entityV2/group/Group.tsx
+++ b/datahub-web-react/src/app/entityV2/group/Group.tsx
@@ -28,10 +28,7 @@ export class GroupEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/mlFeature/MLFeatureEntity.tsx b/datahub-web-react/src/app/entityV2/mlFeature/MLFeatureEntity.tsx
index 14004942c15b6a..102c5400bf1ded 100644
--- a/datahub-web-react/src/app/entityV2/mlFeature/MLFeatureEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/mlFeature/MLFeatureEntity.tsx
@@ -59,10 +59,7 @@ export class MLFeatureEntity implements Entity {
return (
);
diff --git a/datahub-web-react/src/app/entityV2/mlFeatureTable/MLFeatureTableEntity.tsx b/datahub-web-react/src/app/entityV2/mlFeatureTable/MLFeatureTableEntity.tsx
index 89de4f2a069673..08f46db6e96fba 100644
--- a/datahub-web-react/src/app/entityV2/mlFeatureTable/MLFeatureTableEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/mlFeatureTable/MLFeatureTableEntity.tsx
@@ -55,10 +55,7 @@ export class MLFeatureTableEntity implements Entity {
return (
);
diff --git a/datahub-web-react/src/app/entityV2/mlModel/MLModelEntity.tsx b/datahub-web-react/src/app/entityV2/mlModel/MLModelEntity.tsx
index e0848e31bfabc9..e87d858107d2c5 100644
--- a/datahub-web-react/src/app/entityV2/mlModel/MLModelEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/mlModel/MLModelEntity.tsx
@@ -61,10 +61,7 @@ export class MLModelEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/mlModelGroup/MLModelGroupEntity.tsx b/datahub-web-react/src/app/entityV2/mlModelGroup/MLModelGroupEntity.tsx
index edf1241a409eed..b253ee47334b3c 100644
--- a/datahub-web-react/src/app/entityV2/mlModelGroup/MLModelGroupEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/mlModelGroup/MLModelGroupEntity.tsx
@@ -56,10 +56,7 @@ export class MLModelGroupEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/mlPrimaryKey/MLPrimaryKeyEntity.tsx b/datahub-web-react/src/app/entityV2/mlPrimaryKey/MLPrimaryKeyEntity.tsx
index 1689758945f142..d2821b9589d50f 100644
--- a/datahub-web-react/src/app/entityV2/mlPrimaryKey/MLPrimaryKeyEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/mlPrimaryKey/MLPrimaryKeyEntity.tsx
@@ -45,10 +45,7 @@ export class MLPrimaryKeyEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/query/QueryEntity.tsx b/datahub-web-react/src/app/entityV2/query/QueryEntity.tsx
index ac2af08c167821..20937aa1826339 100644
--- a/datahub-web-react/src/app/entityV2/query/QueryEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/query/QueryEntity.tsx
@@ -27,10 +27,7 @@ export class QueryEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/schemaField/SchemaFieldEntity.tsx b/datahub-web-react/src/app/entityV2/schemaField/SchemaFieldEntity.tsx
index 03106aad8ca56f..f696db3fee4855 100644
--- a/datahub-web-react/src/app/entityV2/schemaField/SchemaFieldEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/schemaField/SchemaFieldEntity.tsx
@@ -28,7 +28,7 @@ export class SchemaFieldEntity implements Entity {
type: EntityType = EntityType.SchemaField;
icon = (fontSize?: number, styleType?: IconStyleType, color = 'inherit') => (
-
+
);
isSearchEnabled = () => true;
diff --git a/datahub-web-react/src/app/entityV2/shared/components/subtypes.tsx b/datahub-web-react/src/app/entityV2/shared/components/subtypes.tsx
index 35c131965057d9..461bb3e5cc1282 100644
--- a/datahub-web-react/src/app/entityV2/shared/components/subtypes.tsx
+++ b/datahub-web-react/src/app/entityV2/shared/components/subtypes.tsx
@@ -5,7 +5,6 @@ import Icon, {
FilterOutlined,
LineChartOutlined,
} from '@ant-design/icons';
-import ViewComfyOutlinedIcon from '@mui/icons-material/ViewComfyOutlined';
import React from 'react';
import TableauEmbeddedDataSourceLogo from '@images/tableau-embedded-data-source.svg?react';
@@ -47,9 +46,6 @@ export function getSubTypeIcon(subType?: string): JSX.Element | undefined {
if (lowerSubType === SubType.Project.toLowerCase()) {
return ;
}
- if (lowerSubType === SubType.Table.toLowerCase()) {
- return ;
- }
if (lowerSubType === SubType.View.toLowerCase()) {
return ;
}
diff --git a/datahub-web-react/src/app/entityV2/shared/constants.ts b/datahub-web-react/src/app/entityV2/shared/constants.ts
index f7e79bb62cb7d8..6594afb11a8914 100644
--- a/datahub-web-react/src/app/entityV2/shared/constants.ts
+++ b/datahub-web-react/src/app/entityV2/shared/constants.ts
@@ -1,6 +1,7 @@
import { EntityType } from '@types';
// TODO(Gabe): integrate this w/ the theme
+// These colors are deprecated, use the colors in @components/theme/foundations/colors
export const REDESIGN_COLORS = {
BACKGROUND: '#F4F5F7',
GREY: '#e5e5e5',
diff --git a/datahub-web-react/src/app/entityV2/structuredProperty/StructuredPropertyEntity.tsx b/datahub-web-react/src/app/entityV2/structuredProperty/StructuredPropertyEntity.tsx
index adccf052d1c862..2562def98b4a57 100644
--- a/datahub-web-react/src/app/entityV2/structuredProperty/StructuredPropertyEntity.tsx
+++ b/datahub-web-react/src/app/entityV2/structuredProperty/StructuredPropertyEntity.tsx
@@ -32,10 +32,7 @@ export class StructuredPropertyEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/tag/Tag.tsx b/datahub-web-react/src/app/entityV2/tag/Tag.tsx
index e57138e2413cd5..b1e12ba4d2460c 100644
--- a/datahub-web-react/src/app/entityV2/tag/Tag.tsx
+++ b/datahub-web-react/src/app/entityV2/tag/Tag.tsx
@@ -33,10 +33,7 @@ export class TagEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/entityV2/user/User.tsx b/datahub-web-react/src/app/entityV2/user/User.tsx
index dbafb6314eeb7d..fd9f35bd6f81ec 100644
--- a/datahub-web-react/src/app/entityV2/user/User.tsx
+++ b/datahub-web-react/src/app/entityV2/user/User.tsx
@@ -27,10 +27,7 @@ export class UserEntity implements Entity {
return (
);
};
diff --git a/datahub-web-react/src/app/homeV2/HomePage.tsx b/datahub-web-react/src/app/homeV2/HomePage.tsx
index 185e7a03ab22c7..81374cbe67d421 100644
--- a/datahub-web-react/src/app/homeV2/HomePage.tsx
+++ b/datahub-web-react/src/app/homeV2/HomePage.tsx
@@ -9,6 +9,7 @@ import { RightSidebar } from '@app/homeV2/layout/RightSidebar';
import { NavBarStateType, useNavBarContext } from '@app/homeV2/layout/navBarRedesign/NavBarContext';
import PersonalizationLoadingModal from '@app/homeV2/persona/PersonalizationLoadingModal';
import { OnboardingTour } from '@app/onboarding/OnboardingTour';
+import { WelcomeToDataHubModal } from '@app/onboarding/WelcomeToDataHubModal';
import { HOME_PAGE_DOMAINS_ID, HOME_PAGE_PLATFORMS_ID } from '@app/onboarding/config/HomePageOnboardingConfig';
import {
GLOBAL_WELCOME_TO_ACRYL_ID,
@@ -69,6 +70,7 @@ export const HomePage = () => {
+
>
);
};
diff --git a/datahub-web-react/src/app/homeV2/layout/navBarRedesign/NavSidebar.tsx b/datahub-web-react/src/app/homeV2/layout/navBarRedesign/NavSidebar.tsx
index e7ecdffe5fd467..3a6006205475f1 100644
--- a/datahub-web-react/src/app/homeV2/layout/navBarRedesign/NavSidebar.tsx
+++ b/datahub-web-react/src/app/homeV2/layout/navBarRedesign/NavSidebar.tsx
@@ -14,8 +14,10 @@ import {
UserCircle,
} from '@phosphor-icons/react';
import React, { useContext, useEffect } from 'react';
+import { useLocation } from 'react-router-dom';
import styled, { useTheme } from 'styled-components';
+import analytics, { EventType } from '@app/analytics';
import { useUserContext } from '@app/context/useUserContext';
import { useNavBarContext } from '@app/homeV2/layout/navBarRedesign/NavBarContext';
import NavBarHeader from '@app/homeV2/layout/navBarRedesign/NavBarHeader';
@@ -28,6 +30,7 @@ import {
} from '@app/homeV2/layout/navBarRedesign/types';
import useSelectedKey from '@app/homeV2/layout/navBarRedesign/useSelectedKey';
import OnboardingContext from '@app/onboarding/OnboardingContext';
+import { useOnboardingTour } from '@app/onboarding/OnboardingTourContext.hooks';
import { useAppConfig, useBusinessAttributesFlag } from '@app/useAppConfig';
import { colors } from '@src/alchemy-components';
import { getColor } from '@src/alchemy-components/theme/utils';
@@ -35,6 +38,7 @@ import useGetLogoutHandler from '@src/app/auth/useGetLogoutHandler';
import { HOME_PAGE_INGESTION_ID } from '@src/app/onboarding/config/HomePageOnboardingConfig';
import { useHandleOnboardingTour } from '@src/app/onboarding/useHandleOnboardingTour';
import { useUpdateEducationStepsAllowList } from '@src/app/onboarding/useUpdateEducationStepsAllowList';
+import { useIsHomePage } from '@src/app/shared/useIsHomePage';
import { useEntityRegistry } from '@src/app/useEntityRegistry';
import { HelpLinkRoutes, PageRoutes } from '@src/conf/Global';
import { EntityType } from '@src/types.generated';
@@ -86,8 +90,11 @@ export const NavSidebar = () => {
const appConfig = useAppConfig();
const userContext = useUserContext();
const me = useUserContext();
+ const isHomePage = useIsHomePage();
+ const location = useLocation();
const { isUserInitializing } = useContext(OnboardingContext);
+ const { triggerModalTour } = useOnboardingTour();
const { showOnboardingTour } = useHandleOnboardingTour();
const { config } = useAppConfig();
const logout = useGetLogoutHandler();
@@ -255,7 +262,18 @@ export const NavSidebar = () => {
title: 'Product Tour',
description: 'Take a quick tour of this page',
key: 'helpProductTour',
- onClick: showOnboardingTour,
+ onClick: () => {
+ if (isHomePage) {
+ triggerModalTour();
+ } else {
+ // Track Product Tour button click for non-home pages
+ analytics.event({
+ type: EventType.ProductTourButtonClickEvent,
+ originPage: location.pathname,
+ });
+ showOnboardingTour();
+ }
+ },
},
{
type: NavBarMenuItemTypes.DropdownElement,
diff --git a/datahub-web-react/src/app/lineage/utils/__tests__/extendAsyncEntities.test.ts b/datahub-web-react/src/app/lineage/utils/__tests__/extendAsyncEntities.test.ts
index bf54b215138b3a..a540b6ed561ac6 100644
--- a/datahub-web-react/src/app/lineage/utils/__tests__/extendAsyncEntities.test.ts
+++ b/datahub-web-react/src/app/lineage/utils/__tests__/extendAsyncEntities.test.ts
@@ -9,6 +9,7 @@ describe('extendColumnLineage', () => {
const dataJobWithCLL = {
...dataJob1,
name: '',
+ platform: dataJob1.dataFlow?.platform || undefined,
fineGrainedLineages: [
{
upstreams: [{ urn: dataset1.urn, path: 'test1' }],
diff --git a/datahub-web-react/src/app/lineageV2/controls/DownloadLineageScreenshotButton.tsx b/datahub-web-react/src/app/lineageV2/controls/DownloadLineageScreenshotButton.tsx
index 12a2a1e9a21130..0d5d094f37005c 100644
--- a/datahub-web-react/src/app/lineageV2/controls/DownloadLineageScreenshotButton.tsx
+++ b/datahub-web-react/src/app/lineageV2/controls/DownloadLineageScreenshotButton.tsx
@@ -1,8 +1,9 @@
import { CameraOutlined } from '@ant-design/icons';
import { toPng } from 'html-to-image';
-import React from 'react';
+import React, { useContext } from 'react';
import { getRectOfNodes, getTransformForBounds, useReactFlow } from 'reactflow';
+import { LineageNodesContext } from '@app/lineageV2/common';
import { StyledPanelButton } from '@app/lineageV2/controls/StyledPanelButton';
type Props = {
@@ -30,6 +31,7 @@ function downloadImage(dataUrl: string, name?: string) {
export default function DownloadLineageScreenshotButton({ showExpandedText }: Props) {
const { getNodes } = useReactFlow();
+ const { rootUrn, nodes } = useContext(LineageNodesContext);
const getPreviewImage = () => {
const nodesBounds = getRectOfNodes(getNodes());
@@ -37,6 +39,12 @@ export default function DownloadLineageScreenshotButton({ showExpandedText }: Pr
const imageHeight = nodesBounds.height + 200;
const transform = getTransformForBounds(nodesBounds, imageWidth, imageHeight, 0.5, 2);
+ // Get the entity name for the screenshot filename
+ const rootEntity = nodes.get(rootUrn);
+ const entityName = rootEntity?.entity?.name || 'lineage';
+ // Clean the entity name to be safe for filename use
+ const cleanEntityName = entityName.replace(/[^a-zA-Z0-9_-]/g, '_');
+
toPng(document.querySelector('.react-flow__viewport') as HTMLElement, {
backgroundColor: '#f8f8f8',
width: imageWidth,
@@ -47,7 +55,7 @@ export default function DownloadLineageScreenshotButton({ showExpandedText }: Pr
transform: `translate(${transform[0]}px, ${transform[1]}px) scale(${transform[2]})`,
},
}).then((dataUrl) => {
- downloadImage(dataUrl);
+ downloadImage(dataUrl, cleanEntityName);
});
};
diff --git a/datahub-web-react/src/app/lineageV2/controls/__tests__/DownloadLineageScreenshotButton.test.tsx b/datahub-web-react/src/app/lineageV2/controls/__tests__/DownloadLineageScreenshotButton.test.tsx
new file mode 100644
index 00000000000000..1eb933ba59d26e
--- /dev/null
+++ b/datahub-web-react/src/app/lineageV2/controls/__tests__/DownloadLineageScreenshotButton.test.tsx
@@ -0,0 +1,10 @@
+describe('Entity name cleaning', () => {
+ it('should clean special characters', () => {
+ const cleanName = (name: string) => name.replace(/[^a-zA-Z0-9_-]/g, '_');
+
+ expect(cleanName('dataset-with/special@chars#and$symbols')).toBe('dataset-with_special_chars_and_symbols');
+ expect(cleanName('user.transactions')).toBe('user_transactions');
+ expect(cleanName('normal_name')).toBe('normal_name');
+ expect(cleanName('123-valid_name')).toBe('123-valid_name');
+ });
+});
diff --git a/datahub-web-react/src/app/lineageV3/controls/DownloadLineageScreenshotButton.tsx b/datahub-web-react/src/app/lineageV3/controls/DownloadLineageScreenshotButton.tsx
index 678877a0d71c23..9648937f416ea0 100644
--- a/datahub-web-react/src/app/lineageV3/controls/DownloadLineageScreenshotButton.tsx
+++ b/datahub-web-react/src/app/lineageV3/controls/DownloadLineageScreenshotButton.tsx
@@ -1,35 +1,19 @@
import { CameraOutlined } from '@ant-design/icons';
import { toPng } from 'html-to-image';
-import React from 'react';
+import React, { useContext } from 'react';
import { getRectOfNodes, getTransformForBounds, useReactFlow } from 'reactflow';
+import { LineageNodesContext } from '@app/lineageV3/common';
import { StyledPanelButton } from '@app/lineageV3/controls/StyledPanelButton';
+import { downloadImage } from '@app/lineageV3/utils/lineageUtils';
type Props = {
showExpandedText: boolean;
};
-function downloadImage(dataUrl: string, name?: string) {
- const now = new Date();
- const dateStr = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(
- now.getDate(),
- ).padStart(2, '0')}`;
-
- const timeStr = `${String(now.getHours()).padStart(2, '0')}${String(now.getMinutes()).padStart(2, '0')}${String(
- now.getSeconds(),
- ).padStart(2, '0')}`;
-
- const fileNamePrefix = name ? `${name}_` : 'reactflow_';
- const fileName = `${fileNamePrefix}${dateStr}_${timeStr}.png`;
-
- const a = document.createElement('a');
- a.setAttribute('download', fileName);
- a.setAttribute('href', dataUrl);
- a.click();
-}
-
export default function DownloadLineageScreenshotButton({ showExpandedText }: Props) {
const { getNodes } = useReactFlow();
+ const { rootUrn, nodes } = useContext(LineageNodesContext);
const getPreviewImage = () => {
const nodesBounds = getRectOfNodes(getNodes());
@@ -37,6 +21,12 @@ export default function DownloadLineageScreenshotButton({ showExpandedText }: Pr
const imageHeight = nodesBounds.height + 200;
const transform = getTransformForBounds(nodesBounds, imageWidth, imageHeight, 0.5, 2);
+ // Get the entity name for the screenshot filename
+ const rootEntity = nodes.get(rootUrn);
+ const entityName = rootEntity?.entity?.name || 'lineage';
+ // Clean the entity name to be safe for filename use
+ const cleanEntityName = entityName.replace(/[^a-zA-Z0-9_-]/g, '_');
+
toPng(document.querySelector('.react-flow__viewport') as HTMLElement, {
backgroundColor: '#f8f8f8',
width: imageWidth,
@@ -47,7 +37,7 @@ export default function DownloadLineageScreenshotButton({ showExpandedText }: Pr
transform: `translate(${transform[0]}px, ${transform[1]}px) scale(${transform[2]})`,
},
}).then((dataUrl) => {
- downloadImage(dataUrl);
+ downloadImage(dataUrl, cleanEntityName);
});
};
diff --git a/datahub-web-react/src/app/lineageV3/controls/__tests__/DownloadLineageScreenshotButton.test.tsx b/datahub-web-react/src/app/lineageV3/controls/__tests__/DownloadLineageScreenshotButton.test.tsx
new file mode 100644
index 00000000000000..f629b0eebd237b
--- /dev/null
+++ b/datahub-web-react/src/app/lineageV3/controls/__tests__/DownloadLineageScreenshotButton.test.tsx
@@ -0,0 +1,89 @@
+import { downloadImage } from '@app/lineageV3/utils/lineageUtils';
+
+describe('DownloadLineageScreenshotButton', () => {
+ describe('Entity name cleaning', () => {
+ it('should clean special characters', () => {
+ const cleanName = (name: string) => name.replace(/[^a-zA-Z0-9_-]/g, '_');
+
+ expect(cleanName('dataset-with/special@chars#and$symbols')).toBe('dataset-with_special_chars_and_symbols');
+ expect(cleanName('user.transactions')).toBe('user_transactions');
+ expect(cleanName('normal_name')).toBe('normal_name');
+ expect(cleanName('123-valid_name')).toBe('123-valid_name');
+ });
+ });
+
+ describe('downloadImage', () => {
+ let mockAnchorElement: any;
+ let originalCreateElement: typeof document.createElement;
+
+ beforeEach(() => {
+ // Mock anchor element
+ mockAnchorElement = {
+ setAttribute: vi.fn(),
+ click: vi.fn(),
+ };
+
+ // Mock document.createElement
+ originalCreateElement = document.createElement;
+ document.createElement = vi.fn().mockReturnValue(mockAnchorElement);
+ });
+
+ afterEach(() => {
+ document.createElement = originalCreateElement;
+ });
+
+ it('should create anchor element and set download attribute with default filename', () => {
+ const dataUrl = 'data:image/png;base64,mockdata';
+
+ downloadImage(dataUrl);
+
+ expect(document.createElement).toHaveBeenCalledWith('a');
+ expect(mockAnchorElement.setAttribute).toHaveBeenCalledWith('href', dataUrl);
+ expect(mockAnchorElement.setAttribute).toHaveBeenCalledWith(
+ 'download',
+ expect.stringMatching(/^reactflow_\d{4}-\d{2}-\d{2}_\d{6}\.png$/),
+ );
+ expect(mockAnchorElement.click).toHaveBeenCalled();
+ });
+
+ it('should handle empty string name parameter and use default prefix', () => {
+ const dataUrl = 'data:image/png;base64,mockdata';
+
+ downloadImage(dataUrl, '');
+
+ expect(mockAnchorElement.setAttribute).toHaveBeenCalledWith(
+ 'download',
+ expect.stringMatching(/^reactflow_\d{4}-\d{2}-\d{2}_\d{6}\.png$/),
+ );
+ });
+
+ it('should generate filename with correct format and timestamp', () => {
+ const dataUrl = 'data:image/png;base64,mockdata';
+ const name = 'test_entity';
+
+ downloadImage(dataUrl, name);
+
+ // Verify that the anchor element is created and the href attribute is set
+ expect(mockAnchorElement.setAttribute).toHaveBeenCalledWith('href', dataUrl);
+ expect(mockAnchorElement.click).toHaveBeenCalledTimes(1);
+
+ // Get the download filename from the setAttribute calls
+ const setAttributeCalls = mockAnchorElement.setAttribute.mock.calls;
+ const downloadCall = setAttributeCalls.find((call: any[]) => call[0] === 'download');
+ const filename = downloadCall[1];
+
+ // Verify filename format: test_entity_YYYY-MM-DD_HHMMSS.png
+ expect(filename).toMatch(/^test_entity_\d{4}-\d{2}-\d{2}_\d{6}\.png$/);
+
+ // Extract and verify date part (YYYY-MM-DD)
+ const parts = filename.split('_');
+ const datePart = parts[2];
+ expect(datePart).toMatch(/^\d{4}-\d{2}-\d{2}$/);
+
+ // Extract and verify time part (HHMMSS)
+ const timePart = parts[3].replace('.png', '');
+ expect(timePart).toMatch(/^\d{6}$/);
+ expect(timePart.length).toBe(6);
+ });
+ });
+});
diff --git a/datahub-web-react/src/app/lineageV3/utils/lineageUtils.ts b/datahub-web-react/src/app/lineageV3/utils/lineageUtils.ts
index 679ed597c8e3a1..725b1058dca959 100644
--- a/datahub-web-react/src/app/lineageV3/utils/lineageUtils.ts
+++ b/datahub-web-react/src/app/lineageV3/utils/lineageUtils.ts
@@ -66,3 +66,22 @@ export function useGetLineageUrl(urn?: string, type?: EntityType) {
return getLineageUrl(urn, type, location, entityRegistry);
}
+
+export function downloadImage(dataUrl: string, name?: string) {
+ const now = new Date();
+ const dateStr = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(
+ now.getDate(),
+ ).padStart(2, '0')}`;
+
+ const timeStr = `${String(now.getHours()).padStart(2, '0')}${String(now.getMinutes()).padStart(2, '0')}${String(
+ now.getSeconds(),
+ ).padStart(2, '0')}`;
+
+ const fileNamePrefix = name ? `${name}_` : 'reactflow_';
+ const fileName = `${fileNamePrefix}${dateStr}_${timeStr}.png`;
+
+ const a = document.createElement('a');
+ a.setAttribute('download', fileName);
+ a.setAttribute('href', dataUrl);
+ a.click();
+}
diff --git a/datahub-web-react/src/app/onboarding/OnboardingTour.tsx b/datahub-web-react/src/app/onboarding/OnboardingTour.tsx
index a599e564543290..6726daeb0e9984 100644
--- a/datahub-web-react/src/app/onboarding/OnboardingTour.tsx
+++ b/datahub-web-react/src/app/onboarding/OnboardingTour.tsx
@@ -1,9 +1,11 @@
import { Button } from 'antd';
-import React, { useContext, useEffect, useState } from 'react';
+import React, { useContext, useEffect } from 'react';
+import { useLocation } from 'react-router-dom';
import Tour from 'reactour';
import { useUserContext } from '@app/context/useUserContext';
import { REDESIGN_COLORS } from '@app/entityV2/shared/constants';
+import OnboardingContext from '@app/onboarding/OnboardingContext';
import useShouldSkipOnboardingTour from '@app/onboarding/useShouldSkipOnboardingTour';
import { convertStepId, getConditionalStepIdsToAdd, getStepsToRender } from '@app/onboarding/utils';
import { useIsThemeV2 } from '@app/useIsThemeV2';
@@ -19,35 +21,52 @@ type Props = {
export const OnboardingTour = ({ stepIds }: Props) => {
const { educationSteps, setEducationSteps, educationStepIdsAllowlist } = useContext(EducationStepsContext);
const userUrn = useUserContext()?.user?.urn;
- const [isOpen, setIsOpen] = useState(true);
- const [reshow, setReshow] = useState(false);
const isThemeV2 = useIsThemeV2();
+ const { isTourOpen, tourReshow, setTourReshow, setIsTourOpen } = useContext(OnboardingContext);
+ const location = useLocation();
const accentColor = isThemeV2 ? REDESIGN_COLORS.BACKGROUND_PURPLE : '#5cb7b7';
useEffect(() => {
function handleKeyDown(e) {
// Allow reshow if Cmnd + Ctrl + T is pressed
if (e.metaKey && e.ctrlKey && e.key === 't') {
- setReshow(true);
- setIsOpen(true);
+ setTourReshow(true);
+ setIsTourOpen(true);
}
if (e.metaKey && e.ctrlKey && e.key === 'h') {
- setReshow(false);
- setIsOpen(false);
+ setTourReshow(false);
+ setIsTourOpen(false);
}
}
document.addEventListener('keydown', handleKeyDown);
- }, []);
+ }, [setTourReshow, setIsTourOpen]);
- const steps = getStepsToRender(educationSteps, stepIds, userUrn || '', reshow);
+ // Don't show OnboardingTour on homepage - WelcomeToDataHubModal is used there instead
+ const isHomepage = location.pathname === '/';
+
+ const steps = getStepsToRender(educationSteps, stepIds, userUrn || '', tourReshow);
const filteredSteps = steps.filter((step) => step.id && educationStepIdsAllowlist.has(step.id));
const filteredStepIds: string[] = filteredSteps.map((step) => step?.id).filter((stepId) => !!stepId) as string[];
const [batchUpdateStepStates] = useBatchUpdateStepStatesMutation();
+ const shouldSkipOnboardingTour = useShouldSkipOnboardingTour();
+
+ // Automatically open tour for first-time visits when there are unseen steps
+ useEffect(() => {
+ if (
+ !tourReshow && // Only for automatic tours, not manual reshows
+ !shouldSkipOnboardingTour && // Don't show if globally disabled
+ !isHomepage && // Don't show on homepage - WelcomeToDataHubModal is used there
+ filteredSteps.length > 0 && // Only if there are steps to show
+ !isTourOpen // Don't open if already open
+ ) {
+ setIsTourOpen(true);
+ }
+ }, [filteredSteps.length, tourReshow, shouldSkipOnboardingTour, isHomepage, isTourOpen, setIsTourOpen]);
function closeTour() {
- setIsOpen(false);
- setReshow(false);
+ setIsTourOpen(false);
+ setTourReshow(false);
// add conditional steps where its pre-requisite step ID is in our list of IDs we mark as completed
const conditionalStepIds = getConditionalStepIdsToAdd(stepIds, filteredStepIds);
const finalStepIds = [...filteredStepIds, ...conditionalStepIds];
@@ -59,15 +78,15 @@ export const OnboardingTour = ({ stepIds }: Props) => {
});
}
- const shouldSkipOnboardingTour = useShouldSkipOnboardingTour();
-
- if (!filteredSteps.length || shouldSkipOnboardingTour) return null;
+ // For automatic tours (tourReshow=false), only check if we have steps to show and not on homepage
+ // For manual tours (tourReshow=true), also check the global skip flag
+ if (!filteredSteps.length || isHomepage || (tourReshow && shouldSkipOnboardingTour)) return null;
return (
{
+ const context = useContext(OnboardingTourContext);
+ if (context === undefined) {
+ console.error('useOnboardingTour must be used within an OnboardingTourProvider. Returning fallback context.');
+
+ // Return a fallback context to prevent app crash
+ return {
+ isModalTourOpen: false,
+ triggerModalTour: () => console.warn('triggerModalTour called outside of OnboardingTourProvider'),
+ closeModalTour: () => console.warn('closeModalTour called outside of OnboardingTourProvider'),
+ triggerOriginalTour: () => console.warn('triggerOriginalTour called outside of OnboardingTourProvider'),
+ closeOriginalTour: () => console.warn('closeOriginalTour called outside of OnboardingTourProvider'),
+ originalTourStepIds: null,
+ };
+ }
+ return context;
+};
diff --git a/datahub-web-react/src/app/onboarding/OnboardingTourContext.tsx b/datahub-web-react/src/app/onboarding/OnboardingTourContext.tsx
new file mode 100644
index 00000000000000..bacffe2a2af76b
--- /dev/null
+++ b/datahub-web-react/src/app/onboarding/OnboardingTourContext.tsx
@@ -0,0 +1,12 @@
+import { createContext } from 'react';
+
+export type OnboardingTourContextType = {
+ isModalTourOpen: boolean;
+ triggerModalTour: () => void;
+ closeModalTour: () => void;
+ triggerOriginalTour: (stepIds: string[]) => void;
+ closeOriginalTour: () => void;
+ originalTourStepIds: string[] | null;
+};
+
+export const OnboardingTourContext = createContext(undefined);
diff --git a/datahub-web-react/src/app/onboarding/OnboardingTourContextProvider.tsx b/datahub-web-react/src/app/onboarding/OnboardingTourContextProvider.tsx
new file mode 100644
index 00000000000000..529b880f1a3a50
--- /dev/null
+++ b/datahub-web-react/src/app/onboarding/OnboardingTourContextProvider.tsx
@@ -0,0 +1,39 @@
+import React, { ReactNode, useState } from 'react';
+
+import { OnboardingTourContext } from '@app/onboarding/OnboardingTourContext';
+
+type Props = {
+ children: ReactNode;
+};
+
+export default function OnboardingTourProvider({ children }: Props) {
+ const [isModalTourOpen, setIsModalTourOpen] = useState(false);
+ const [originalTourStepIds, setOriginalTourStepIds] = useState(null);
+
+ const triggerModalTour = () => {
+ setIsModalTourOpen(true);
+ };
+
+ const closeModalTour = () => {
+ setIsModalTourOpen(false);
+ };
+
+ const triggerOriginalTour = (stepIds: string[]) => {
+ setOriginalTourStepIds(stepIds);
+ };
+
+ const closeOriginalTour = () => {
+ setOriginalTourStepIds(null);
+ };
+
+ const value = {
+ isModalTourOpen,
+ triggerModalTour,
+ closeModalTour,
+ triggerOriginalTour,
+ closeOriginalTour,
+ originalTourStepIds,
+ };
+
+ return {children} ;
+}
diff --git a/datahub-web-react/src/app/onboarding/WelcomeToDataHubModal.components.tsx b/datahub-web-react/src/app/onboarding/WelcomeToDataHubModal.components.tsx
new file mode 100644
index 00000000000000..8114ae02cdfd33
--- /dev/null
+++ b/datahub-web-react/src/app/onboarding/WelcomeToDataHubModal.components.tsx
@@ -0,0 +1,127 @@
+import { Spin } from 'antd';
+import React from 'react';
+import styled from 'styled-components';
+
+import colors from '@components/theme/foundations/colors';
+
+/**
+ * Container for individual carousel slides with centered content
+ */
+export const SlideContainer = styled.div`
+ position: relative; /* Provide positioning context for absolutely positioned children */
+ text-align: left;
+ margin-bottom: 32px;
+ min-height: 470px;
+`;
+
+/**
+ * Container for video elements with centered alignment
+ */
+export const VideoContainer = styled.div`
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ padding-top: 16px;
+`;
+
+/**
+ * Styled loading container base
+ */
+const LoadingContainerBase = styled.div<{ width: string }>`
+ width: ${(props) => props.width};
+ height: 350px; /* Match video aspect ratio for 620px width */
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ justify-content: center;
+ background-color: ${colors.gray[1500]};
+ border: 2px dashed ${colors.gray[100]};
+ border-radius: 8px;
+ font-size: 16px;
+ color: ${colors.gray[1700]};
+ font-weight: 500;
+ margin: 0 auto;
+ gap: 16px;
+`;
+
+/**
+ * Loading state container with Spin component
+ * @param width - CSS width value for the container
+ * @param children - Optional loading text
+ */
+export const LoadingContainer: React.FC<{ width: string; children?: React.ReactNode }> = ({
+ width,
+ children = 'Loading video...',
+}) => (
+
+
+ {children}
+
+);
+
+/**
+ * Styled anchor for DataHub Docs link
+ */
+export const StyledDocsLink = styled.a`
+ color: ${colors.primary[500]};
+ text-align: center;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 650;
+ line-height: normal;
+ letter-spacing: -0.07px;
+ text-decoration: none;
+ cursor: pointer;
+ border-radius: 4px;
+ padding: 10px 12px;
+
+ &:hover {
+ background-color: ${colors.gray[1500]};
+ }
+`;
+
+/**
+ * Styled video element for visible videos
+ */
+const StyledVideo = styled.video<{ width: string }>`
+ width: ${(props) => props.width};
+`;
+
+/**
+ * Styled video element for hidden preload videos
+ */
+const HiddenPreloadVideo = styled.video<{ width: string }>`
+ width: ${(props) => props.width};
+ opacity: 0;
+ position: absolute;
+ pointer-events: none;
+ top: 0;
+ left: 0;
+`;
+
+/**
+ * Reusable video slide component that handles loading states
+ */
+interface VideoSlideProps {
+ videoSrc?: string;
+ isReady: boolean;
+ onVideoLoad: () => void;
+ width: string;
+}
+
+export const VideoSlide: React.FC = ({ videoSrc, isReady, onVideoLoad, width }) => (
+ <>
+ {isReady ? (
+
+
+
+ ) : (
+ Loading video...
+ )}
+ {videoSrc && !isReady && (
+
+
+
+ )}
+ >
+);
diff --git a/datahub-web-react/src/app/onboarding/WelcomeToDataHubModal.tsx b/datahub-web-react/src/app/onboarding/WelcomeToDataHubModal.tsx
new file mode 100644
index 00000000000000..6dd17471bed941
--- /dev/null
+++ b/datahub-web-react/src/app/onboarding/WelcomeToDataHubModal.tsx
@@ -0,0 +1,310 @@
+import { Button, Carousel, Heading, LoadedImage, Modal } from '@components';
+import React, { useEffect, useRef, useState } from 'react';
+
+import analytics, { EventType } from '@app/analytics';
+import { useOnboardingTour } from '@app/onboarding/OnboardingTourContext.hooks';
+import {
+ LoadingContainer,
+ SlideContainer,
+ StyledDocsLink,
+ VideoContainer,
+ VideoSlide,
+} from '@src/app/onboarding/WelcomeToDataHubModal.components';
+
+import welcomeModalHomeScreenshot from '@images/welcome-modal-home-screenshot.png';
+
+const SLIDE_DURATION_MS = 10000;
+const DATAHUB_DOCS_URL = 'https://docs.datahub.com/docs/category/features';
+const WELCOME_TO_DATAHUB_MODAL_TITLE = 'Welcome to DataHub';
+const SKIP_WELCOME_MODAL_KEY = 'skipWelcomeModal';
+
+interface VideoSources {
+ search: string;
+ lineage: string;
+ impact: string;
+ aiDocs?: string;
+}
+
+function checkShouldSkipWelcomeModal() {
+ return localStorage.getItem(SKIP_WELCOME_MODAL_KEY) === 'true';
+}
+
+export const WelcomeToDataHubModal = () => {
+ const [shouldShow, setShouldShow] = useState(false);
+ const [currentSlide, setCurrentSlide] = useState(0);
+ const [videoSources, setVideoSources] = useState(null);
+ const [videoLoading, setVideoLoading] = useState(false);
+ const [videosReady, setVideosReady] = useState<{ [key in keyof VideoSources]?: boolean }>({});
+ const hasTrackedView = useRef(false);
+ const carouselRef = useRef(null);
+ const { isModalTourOpen, closeModalTour } = useOnboardingTour();
+ const shouldSkipWelcomeModal = checkShouldSkipWelcomeModal();
+ const isDocumentationSlideEnabled = false;
+ const TOTAL_CAROUSEL_SLIDES = isDocumentationSlideEnabled ? 5 : 4;
+ const MODAL_IMAGE_WIDTH_RAW = 620;
+ const MODAL_IMAGE_WIDTH = `${MODAL_IMAGE_WIDTH_RAW}px`;
+ const MODAL_WIDTH_NUM = MODAL_IMAGE_WIDTH_RAW + 45; // Add padding
+ const MODAL_WIDTH = `${MODAL_WIDTH_NUM}px`;
+
+ // Automatic tour for first-time home page visitors
+ useEffect(() => {
+ if (!shouldSkipWelcomeModal) {
+ setShouldShow(true);
+ setCurrentSlide(0);
+ }
+ }, [shouldSkipWelcomeModal]);
+
+ // Manual tour trigger from Product Tour buttons
+ useEffect(() => {
+ if (isModalTourOpen) {
+ setShouldShow(true);
+ setCurrentSlide(0);
+ }
+ }, [isModalTourOpen]);
+
+ // Show modal immediately, load videos individually as they complete
+ useEffect(() => {
+ if (shouldShow && !videoSources) {
+ // Show modal immediately with empty video sources
+ const emptyVideoSources: VideoSources = {
+ search: '',
+ lineage: '',
+ impact: '',
+ aiDocs: undefined,
+ };
+ setVideoSources(emptyVideoSources);
+ setVideoLoading(false);
+
+ // Load all videos in parallel, update each as it completes
+ const loadVideo = async (videoKey: keyof VideoSources, importPromise: Promise<{ default: string }>) => {
+ try {
+ const module = await importPromise;
+ setVideoSources((prev) => (prev ? { ...prev, [videoKey]: module.default } : prev));
+ } catch (error) {
+ console.error(`Failed to load ${videoKey} video:`, error);
+ }
+ };
+
+ // Start loading all videos simultaneously
+ loadVideo('search', import('@images/FTE-search.mp4'));
+ loadVideo('lineage', import('@images/FTE-lineage.mp4'));
+ loadVideo('impact', import('@images/FTE-impact.mp4'));
+
+ if (isDocumentationSlideEnabled) {
+ loadVideo('aiDocs', import('@images/FTE-ai-documentation.mp4'));
+ }
+ }
+ }, [isDocumentationSlideEnabled, shouldShow, videoSources]);
+
+ // Handle when video elements are fully loaded
+ const handleVideoLoad = (videoKey: keyof VideoSources) => {
+ setVideosReady((prev) => ({ ...prev, [videoKey]: true }));
+ };
+
+ // Track page view when modal opens
+ useEffect(() => {
+ if (shouldShow && !hasTrackedView.current) {
+ analytics.page({
+ originPath: '/onboarding-tour',
+ });
+
+ analytics.event({
+ type: EventType.WelcomeToDataHubModalViewEvent,
+ });
+
+ hasTrackedView.current = true;
+ }
+ }, [shouldShow]);
+
+ const handleSlideChange = (current: number) => {
+ // Called after carousel animation completes
+ if (current >= 0 && current < TOTAL_CAROUSEL_SLIDES) {
+ analytics.event({
+ type: EventType.WelcomeToDataHubModalInteractEvent,
+ currentSlide: current + 1,
+ totalSlides: TOTAL_CAROUSEL_SLIDES,
+ });
+
+ setCurrentSlide(current);
+ }
+ };
+
+ function closeTour(
+ exitMethod: 'close_button' | 'get_started_button' | 'outside_click' | 'escape_key' = 'close_button',
+ ) {
+ analytics.event({
+ type: EventType.WelcomeToDataHubModalExitEvent,
+ currentSlide: currentSlide + 1,
+ totalSlides: TOTAL_CAROUSEL_SLIDES,
+ exitMethod,
+ });
+
+ setShouldShow(false);
+ setCurrentSlide(0); // Reset to first slide for next opening
+
+ if (isModalTourOpen) {
+ closeModalTour();
+ } else {
+ // Only set localStorage for automatic first-time tours, not manual triggers
+ localStorage.setItem(SKIP_WELCOME_MODAL_KEY, 'true');
+ }
+ }
+
+ if (!shouldShow) return null;
+
+ // Show loading state while videos are being loaded
+ if (videoLoading || !videoSources) {
+ return (
+ closeTour('close_button')}
+ buttons={[
+ {
+ text: 'Get Started',
+ variant: 'filled',
+ onClick: () => closeTour('get_started_button'),
+ },
+ ]}
+ >
+
+
+
+ Loading...
+
+
+
+ );
+ }
+
+ function trackExternalLinkClick(url: string): void {
+ analytics.event({
+ type: EventType.WelcomeToDataHubModalClickViewDocumentationEvent,
+ url,
+ });
+ }
+
+ return (
+ closeTour('close_button')}
+ buttons={[]}
+ >
+ {
+ trackExternalLinkClick(DATAHUB_DOCS_URL);
+ }}
+ >
+ DataHub Docs
+
+ ) : undefined
+ }
+ rightComponent={
+ currentSlide === TOTAL_CAROUSEL_SLIDES - 1 ? (
+ closeTour('get_started_button')}
+ >
+ Get started
+
+ ) : undefined
+ }
+ infinite={false}
+ >
+
+
+ Find Any Asset, Anywhere
+
+
+ Search datasets, models, dashboards, and more across your entire stack
+
+
+ handleVideoLoad('search')}
+ width={MODAL_IMAGE_WIDTH}
+ />
+
+
+
+
+ Understand Your Data's Origin
+
+
+ See the full story of how your data was created and transformed
+
+
+ handleVideoLoad('lineage')}
+ width={MODAL_IMAGE_WIDTH}
+ />
+
+
+
+
+ Manage Breaking Changes Confidently
+
+
+ Preview the full impact of schema and column changes
+
+
+ handleVideoLoad('impact')}
+ width={MODAL_IMAGE_WIDTH}
+ />
+
+
+ {videoSources.aiDocs && (
+
+
+ Documentation Without the Work
+
+
+ Save hours of manual work while improving discoverability
+
+
+ handleVideoLoad('aiDocs')}
+ width={MODAL_IMAGE_WIDTH}
+ />
+
+
+ )}
+
+
+ Ready to Get Started?
+
+
+ Explore our comprehensive documentation or jump right in and start discovering your data
+
+
+
+
+
+ );
+};
diff --git a/datahub-web-react/src/app/onboarding/__tests__/OnboardingTourContext.hooks.test.tsx b/datahub-web-react/src/app/onboarding/__tests__/OnboardingTourContext.hooks.test.tsx
new file mode 100644
index 00000000000000..c9b481b66dec58
--- /dev/null
+++ b/datahub-web-react/src/app/onboarding/__tests__/OnboardingTourContext.hooks.test.tsx
@@ -0,0 +1,128 @@
+import { renderHook } from '@testing-library/react-hooks';
+import React from 'react';
+
+import { OnboardingTourContext } from '@app/onboarding/OnboardingTourContext';
+import { useOnboardingTour } from '@app/onboarding/OnboardingTourContext.hooks';
+
+const mockContextValue = {
+ isModalTourOpen: false,
+ triggerModalTour: vi.fn(),
+ closeModalTour: vi.fn(),
+ triggerOriginalTour: vi.fn(),
+ closeOriginalTour: vi.fn(),
+ originalTourStepIds: null,
+};
+
+// Mock console methods
+const mockConsoleError = vi.fn();
+const mockConsoleWarn = vi.fn();
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ vi.spyOn(console, 'error').mockImplementation(mockConsoleError);
+ vi.spyOn(console, 'warn').mockImplementation(mockConsoleWarn);
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+describe('useOnboardingTour', () => {
+ describe('when context is available', () => {
+ const wrapper = ({ children }: { children: React.ReactNode }) => (
+ {children}
+ );
+
+ it('should return the context value', () => {
+ const { result } = renderHook(() => useOnboardingTour(), { wrapper });
+
+ expect(result.current).toBe(mockContextValue);
+ });
+
+ it('should not log any errors', () => {
+ renderHook(() => useOnboardingTour(), { wrapper });
+
+ expect(mockConsoleError).not.toHaveBeenCalled();
+ });
+ });
+
+ describe('when context is undefined', () => {
+ const wrapper = ({ children }: { children: React.ReactNode }) => (
+ {children}
+ );
+
+ it('should log an error', () => {
+ renderHook(() => useOnboardingTour(), { wrapper });
+
+ expect(mockConsoleError).toHaveBeenCalledWith(
+ 'useOnboardingTour must be used within an OnboardingTourProvider. Returning fallback context.',
+ );
+ });
+
+ it('should return fallback context with correct structure', () => {
+ const { result } = renderHook(() => useOnboardingTour(), { wrapper });
+
+ expect(result.current).toEqual({
+ isModalTourOpen: false,
+ triggerModalTour: expect.any(Function),
+ closeModalTour: expect.any(Function),
+ triggerOriginalTour: expect.any(Function),
+ closeOriginalTour: expect.any(Function),
+ originalTourStepIds: null,
+ });
+ });
+
+ it('should warn when triggerModalTour is called', () => {
+ const { result } = renderHook(() => useOnboardingTour(), { wrapper });
+
+ result.current.triggerModalTour();
+
+ expect(mockConsoleWarn).toHaveBeenCalledWith('triggerModalTour called outside of OnboardingTourProvider');
+ });
+
+ it('should warn when closeModalTour is called', () => {
+ const { result } = renderHook(() => useOnboardingTour(), { wrapper });
+
+ result.current.closeModalTour();
+
+ expect(mockConsoleWarn).toHaveBeenCalledWith('closeModalTour called outside of OnboardingTourProvider');
+ });
+
+ it('should warn when triggerOriginalTour is called', () => {
+ const { result } = renderHook(() => useOnboardingTour(), { wrapper });
+
+ result.current.triggerOriginalTour(['step1', 'step2']);
+
+ expect(mockConsoleWarn).toHaveBeenCalledWith(
+ 'triggerOriginalTour called outside of OnboardingTourProvider',
+ );
+ });
+
+ it('should warn when closeOriginalTour is called', () => {
+ const { result } = renderHook(() => useOnboardingTour(), { wrapper });
+
+ result.current.closeOriginalTour();
+
+ expect(mockConsoleWarn).toHaveBeenCalledWith('closeOriginalTour called outside of OnboardingTourProvider');
+ });
+ });
+
+ describe('when used outside of provider', () => {
+ it('should log an error and return fallback context', () => {
+ const { result } = renderHook(() => useOnboardingTour());
+
+ expect(mockConsoleError).toHaveBeenCalledWith(
+ 'useOnboardingTour must be used within an OnboardingTourProvider. Returning fallback context.',
+ );
+
+ expect(result.current).toEqual({
+ isModalTourOpen: false,
+ triggerModalTour: expect.any(Function),
+ closeModalTour: expect.any(Function),
+ triggerOriginalTour: expect.any(Function),
+ closeOriginalTour: expect.any(Function),
+ originalTourStepIds: null,
+ });
+ });
+ });
+});
diff --git a/datahub-web-react/src/app/onboarding/__tests__/useHandleOnboardingTour.test.tsx b/datahub-web-react/src/app/onboarding/__tests__/useHandleOnboardingTour.test.tsx
new file mode 100644
index 00000000000000..4a7dfe37e0f839
--- /dev/null
+++ b/datahub-web-react/src/app/onboarding/__tests__/useHandleOnboardingTour.test.tsx
@@ -0,0 +1,170 @@
+import { renderHook } from '@testing-library/react-hooks';
+import React from 'react';
+
+import OnboardingContext from '@app/onboarding/OnboardingContext';
+import { useHandleOnboardingTour } from '@app/onboarding/useHandleOnboardingTour';
+
+const mockSetTourReshow = vi.fn();
+const mockSetIsTourOpen = vi.fn();
+
+const mockContextValue = {
+ tourReshow: false,
+ setTourReshow: mockSetTourReshow,
+ isTourOpen: false,
+ setIsTourOpen: mockSetIsTourOpen,
+ isUserInitializing: false,
+ setIsUserInitializing: vi.fn(),
+};
+
+const wrapper = ({ children }: { children: React.ReactNode }) => (
+ {children}
+);
+
+describe('useHandleOnboardingTour', () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it('should return showOnboardingTour function', () => {
+ const { result } = renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ expect(result.current).toEqual({
+ showOnboardingTour: expect.any(Function),
+ });
+ });
+
+ it('should call setTourReshow and setIsTourOpen when showOnboardingTour is called', () => {
+ const { result } = renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ result.current.showOnboardingTour();
+
+ expect(mockSetTourReshow).toHaveBeenCalledWith(true);
+ expect(mockSetIsTourOpen).toHaveBeenCalledWith(true);
+ });
+
+ it('should add event listener for keydown on mount', () => {
+ const addEventListenerSpy = vi.spyOn(document, 'addEventListener');
+
+ renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ expect(addEventListenerSpy).toHaveBeenCalledWith('keydown', expect.any(Function));
+ });
+
+ it('should remove event listener on unmount', () => {
+ const removeEventListenerSpy = vi.spyOn(document, 'removeEventListener');
+
+ const { unmount } = renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ unmount();
+
+ expect(removeEventListenerSpy).toHaveBeenCalledWith('keydown', expect.any(Function));
+ });
+
+ it('should show onboarding tour when Cmd+Ctrl+T is pressed', () => {
+ renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ const keydownEvent = new KeyboardEvent('keydown', {
+ key: 't',
+ metaKey: true,
+ ctrlKey: true,
+ });
+
+ document.dispatchEvent(keydownEvent);
+
+ expect(mockSetTourReshow).toHaveBeenCalledWith(true);
+ expect(mockSetIsTourOpen).toHaveBeenCalledWith(true);
+ });
+
+ it('should hide onboarding tour when Cmd+Ctrl+H is pressed', () => {
+ renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ const keydownEvent = new KeyboardEvent('keydown', {
+ key: 'h',
+ metaKey: true,
+ ctrlKey: true,
+ });
+
+ document.dispatchEvent(keydownEvent);
+
+ expect(mockSetTourReshow).toHaveBeenCalledWith(false);
+ expect(mockSetIsTourOpen).toHaveBeenCalledWith(false);
+ });
+
+ it('should not trigger tour actions when only metaKey is pressed', () => {
+ renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ const keydownEvent = new KeyboardEvent('keydown', {
+ key: 't',
+ metaKey: true,
+ ctrlKey: false,
+ });
+
+ document.dispatchEvent(keydownEvent);
+
+ expect(mockSetTourReshow).not.toHaveBeenCalled();
+ expect(mockSetIsTourOpen).not.toHaveBeenCalled();
+ });
+
+ it('should not trigger tour actions when only ctrlKey is pressed', () => {
+ renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ const keydownEvent = new KeyboardEvent('keydown', {
+ key: 't',
+ metaKey: false,
+ ctrlKey: true,
+ });
+
+ document.dispatchEvent(keydownEvent);
+
+ expect(mockSetTourReshow).not.toHaveBeenCalled();
+ expect(mockSetIsTourOpen).not.toHaveBeenCalled();
+ });
+
+ it('should not trigger tour actions when wrong key is pressed', () => {
+ renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ const keydownEvent = new KeyboardEvent('keydown', {
+ key: 'x',
+ metaKey: true,
+ ctrlKey: true,
+ });
+
+ document.dispatchEvent(keydownEvent);
+
+ expect(mockSetTourReshow).not.toHaveBeenCalled();
+ expect(mockSetIsTourOpen).not.toHaveBeenCalled();
+ });
+
+ it('should handle multiple show/hide cycles correctly', () => {
+ renderHook(() => useHandleOnboardingTour(), { wrapper });
+
+ // Show tour
+ const showEvent = new KeyboardEvent('keydown', {
+ key: 't',
+ metaKey: true,
+ ctrlKey: true,
+ });
+ document.dispatchEvent(showEvent);
+
+ expect(mockSetTourReshow).toHaveBeenCalledWith(true);
+ expect(mockSetIsTourOpen).toHaveBeenCalledWith(true);
+
+ // Hide tour
+ const hideEvent = new KeyboardEvent('keydown', {
+ key: 'h',
+ metaKey: true,
+ ctrlKey: true,
+ });
+ document.dispatchEvent(hideEvent);
+
+ expect(mockSetTourReshow).toHaveBeenCalledWith(false);
+ expect(mockSetIsTourOpen).toHaveBeenCalledWith(false);
+
+ expect(mockSetTourReshow).toHaveBeenCalledTimes(2);
+ expect(mockSetIsTourOpen).toHaveBeenCalledTimes(2);
+ });
+});
diff --git a/datahub-web-react/src/app/onboarding/useShouldSkipOnboardingTour.ts b/datahub-web-react/src/app/onboarding/useShouldSkipOnboardingTour.ts
index 112c82bf65e993..78e0baa0598ff9 100644
--- a/datahub-web-react/src/app/onboarding/useShouldSkipOnboardingTour.ts
+++ b/datahub-web-react/src/app/onboarding/useShouldSkipOnboardingTour.ts
@@ -1,5 +1,5 @@
// this key is used in commands.js to turn off onboarding tours in cypress tests
-const SKIP_ONBOARDING_TOUR_KEY = 'skipOnboardingTour';
+export const SKIP_ONBOARDING_TOUR_KEY = 'skipOnboardingTour';
export default function useShouldSkipOnboardingTour() {
const shouldSkipOnboardingTour = localStorage.getItem(SKIP_ONBOARDING_TOUR_KEY);
diff --git a/datahub-web-react/src/app/previewV2/ColoredBackgroundPlatformIconGroup.tsx b/datahub-web-react/src/app/previewV2/ColoredBackgroundPlatformIconGroup.tsx
index a1ddc9a99995f7..53f598836470d8 100644
--- a/datahub-web-react/src/app/previewV2/ColoredBackgroundPlatformIconGroup.tsx
+++ b/datahub-web-react/src/app/previewV2/ColoredBackgroundPlatformIconGroup.tsx
@@ -32,6 +32,7 @@ interface Props {
icon?: React.ReactNode;
backgroundSize?: number;
imgSize?: number;
+ className?: string;
}
export default function ColoredBackgroundPlatformIconGroup(props: Props) {
@@ -45,6 +46,7 @@ export default function ColoredBackgroundPlatformIconGroup(props: Props) {
icon,
imgSize = 18,
backgroundSize = 32,
+ className,
} = props;
const shouldShowSeparateSiblings = useIsShowSeparateSiblingsEnabled();
@@ -97,5 +99,5 @@ export default function ColoredBackgroundPlatformIconGroup(props: Props) {
);
};
- return {renderLogoIcon()} ;
+ return {renderLogoIcon()} ;
}
diff --git a/datahub-web-react/src/app/previewV2/CompactView.tsx b/datahub-web-react/src/app/previewV2/CompactView.tsx
index 64176a59a88bbe..3c3a2312a08477 100644
--- a/datahub-web-react/src/app/previewV2/CompactView.tsx
+++ b/datahub-web-react/src/app/previewV2/CompactView.tsx
@@ -7,7 +7,6 @@ import { GenericEntityProperties } from '@app/entity/shared/types';
import { EntityMenuActions } from '@app/entityV2/Entity';
import { EntityMenuItems } from '@app/entityV2/shared/EntityDropdown/EntityMenuActions';
import MoreOptionsMenuAction from '@app/entityV2/shared/EntityDropdown/MoreOptionsMenuAction';
-import { REDESIGN_COLORS } from '@app/entityV2/shared/constants';
import ViewInPlatform from '@app/entityV2/shared/externalUrl/ViewInPlatform';
import ColoredBackgroundPlatformIconGroup, {
PlatformContentWrapper,
@@ -32,18 +31,20 @@ const RowContainer = styled.div`
}
`;
+const StyledPlatformIconGroup = styled(ColoredBackgroundPlatformIconGroup)`
+ margin: 0;
+`;
+
+const ContextPathRowContainer = styled(RowContainer)`
+ align-items: center;
+ justify-content: start;
+`;
+
const CompactActionsAndStatusSection = styled(ActionsAndStatusSection)`
justify-content: end;
margin-right: -0.3rem;
`;
-const PlatformDivider = styled.div`
- font-size: 16px;
- margin-right: 0.5rem;
- margin-top: -3px;
- color: ${REDESIGN_COLORS.TEXT_GREY};
-`;
-
interface Props {
name: string;
urn: string;
@@ -132,10 +133,10 @@ export const CompactView = ({
)}
-
+
{isIconPresent ? (
) : (
@@ -158,7 +158,7 @@ export const CompactView = ({
entityTitleWidth={previewType === PreviewType.HOVER_CARD ? 150 : 200}
isCompactView
/>
-
+
>
);
};
diff --git a/datahub-web-react/src/app/previewV2/ContextPath.tsx b/datahub-web-react/src/app/previewV2/ContextPath.tsx
index 97ef2a4b21d98d..5134d469c394ab 100644
--- a/datahub-web-react/src/app/previewV2/ContextPath.tsx
+++ b/datahub-web-react/src/app/previewV2/ContextPath.tsx
@@ -92,7 +92,7 @@ export default function ContextPath(props: Props) {
const entityRegistry = useEntityRegistryV2();
const entityTypeIcon =
- getSubTypeIcon(displayedEntityType) || entityRegistry.getIcon(entityType, 16, IconStyleType.ACCENT, '#8d95b1');
+ getSubTypeIcon(displayedEntityType) || entityRegistry.getIcon(entityType, undefined, IconStyleType.ACCENT);
const hasBrowsePath = !!browsePaths?.path?.length && !isDefaultBrowsePath(browsePaths);
const hasParentEntities = !!parentEntities?.length;
diff --git a/datahub-web-react/src/app/previewV2/ContextPathEntry.tsx b/datahub-web-react/src/app/previewV2/ContextPathEntry.tsx
index fb67036fb65758..be2d1b3e2c5fcc 100644
--- a/datahub-web-react/src/app/previewV2/ContextPathEntry.tsx
+++ b/datahub-web-react/src/app/previewV2/ContextPathEntry.tsx
@@ -49,6 +49,10 @@ const Contents = styled.div<{ $disabled?: boolean }>`
}
`;
+const StyledLink = styled(Link)`
+ overflow: hidden;
+`;
+
interface Props {
name?: string;
linkUrl?: string;
@@ -93,9 +97,9 @@ function ContextPathEntry(props: Props) {
return (
{showLink ? (
-
+
{contents}
-
+
) : (
contents
)}
diff --git a/datahub-web-react/src/app/shared/__tests__/useIsHomePage.test.ts b/datahub-web-react/src/app/shared/__tests__/useIsHomePage.test.ts
new file mode 100644
index 00000000000000..954d128ddd1127
--- /dev/null
+++ b/datahub-web-react/src/app/shared/__tests__/useIsHomePage.test.ts
@@ -0,0 +1,150 @@
+import { renderHook } from '@testing-library/react-hooks';
+
+import { useIsHomePage } from '@app/shared/useIsHomePage';
+
+// Mock react-router
+const mockUseLocation = vi.fn();
+vi.mock('react-router', () => ({
+ useLocation: () => mockUseLocation(),
+}));
+
+describe('useIsHomePage', () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it('should return true when pathname is "/"', () => {
+ mockUseLocation.mockReturnValue({ pathname: '/' });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(true);
+ });
+
+ it('should return false when pathname is not "/"', () => {
+ mockUseLocation.mockReturnValue({ pathname: '/search' });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(false);
+ });
+
+ it('should return false for nested paths', () => {
+ mockUseLocation.mockReturnValue({ pathname: '/dataset/urn:li:dataset:123' });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(false);
+ });
+
+ it('should return false for paths with query parameters', () => {
+ mockUseLocation.mockReturnValue({ pathname: '/search?q=test' });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(false);
+ });
+
+ it('should return false for paths with hash fragments', () => {
+ mockUseLocation.mockReturnValue({ pathname: '/dashboard#metrics' });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(false);
+ });
+
+ it('should return false for empty pathname', () => {
+ mockUseLocation.mockReturnValue({ pathname: '' });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(false);
+ });
+
+ it('should return false for undefined pathname', () => {
+ mockUseLocation.mockReturnValue({ pathname: undefined });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(false);
+ });
+
+ it('should handle pathname with trailing slash correctly', () => {
+ mockUseLocation.mockReturnValue({ pathname: '//' });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(false);
+ });
+
+ it('should memoize the result and not recompute when pathname stays the same', () => {
+ mockUseLocation.mockReturnValue({ pathname: '/' });
+
+ const { result, rerender } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(true);
+
+ // Force a rerender without changing the pathname
+ rerender();
+
+ expect(result.current).toBe(true);
+ expect(mockUseLocation).toHaveBeenCalledTimes(2); // Called on mount and rerender
+ });
+
+ it('should recompute when pathname changes', () => {
+ // Start with home page
+ mockUseLocation.mockReturnValue({ pathname: '/' });
+
+ const { result, rerender } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(true);
+
+ // Change to different page
+ mockUseLocation.mockReturnValue({ pathname: '/search' });
+ rerender();
+
+ expect(result.current).toBe(false);
+
+ // Change back to home page
+ mockUseLocation.mockReturnValue({ pathname: '/' });
+ rerender();
+
+ expect(result.current).toBe(true);
+ });
+
+ it('should handle case sensitivity correctly', () => {
+ mockUseLocation.mockReturnValue({ pathname: '/HOME' });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(false);
+ });
+
+ it('should handle common route variations', () => {
+ const testCases = [
+ { pathname: '/browse', expected: false },
+ { pathname: '/search', expected: false },
+ { pathname: '/datasets', expected: false },
+ { pathname: '/', expected: true },
+ { pathname: '/settings', expected: false },
+ { pathname: '/profile', expected: false },
+ { pathname: '/lineage', expected: false },
+ { pathname: '/glossary', expected: false },
+ { pathname: '/domains', expected: false },
+ { pathname: '/tags', expected: false },
+ { pathname: '/onboarding', expected: false },
+ ];
+
+ testCases.forEach(({ pathname, expected }) => {
+ mockUseLocation.mockReturnValue({ pathname });
+
+ const { result } = renderHook(() => useIsHomePage());
+
+ expect(result.current).toBe(expected);
+ });
+ });
+});
diff --git a/datahub-web-react/src/app/shared/useIsHomePage.ts b/datahub-web-react/src/app/shared/useIsHomePage.ts
new file mode 100644
index 00000000000000..ba1dfb308a9057
--- /dev/null
+++ b/datahub-web-react/src/app/shared/useIsHomePage.ts
@@ -0,0 +1,14 @@
+import { useMemo } from 'react';
+import { useLocation } from 'react-router';
+
+/**
+ * Hook to detect if the current page is the home page
+ * Abstracts location.pathname for better testability and reusability
+ */
+export const useIsHomePage = (): boolean => {
+ const { pathname } = useLocation();
+
+ return useMemo(() => {
+ return pathname === '/';
+ }, [pathname]);
+};
diff --git a/datahub-web-react/src/app/sharedV2/carousel/Carousel.tsx b/datahub-web-react/src/app/sharedV2/carousel/Carousel.tsx
index ab727aa9b3fba9..977fa10a9eab5e 100644
--- a/datahub-web-react/src/app/sharedV2/carousel/Carousel.tsx
+++ b/datahub-web-react/src/app/sharedV2/carousel/Carousel.tsx
@@ -40,7 +40,6 @@ const ButtonContainer = styled.div<{ left?: boolean; right?: boolean }>`
border-radius: 100px;
box-shadow: 0px 2px 4px rgba(0, 0, 0, 0.1);
- color: ${REDESIGN_COLORS.BLACK};
background-color: ${REDESIGN_COLORS.WHITE};
:hover {
diff --git a/datahub-web-react/src/graphql/dataJob.graphql b/datahub-web-react/src/graphql/dataJob.graphql
index 27aed505de0142..196a7000cbcad1 100644
--- a/datahub-web-react/src/graphql/dataJob.graphql
+++ b/datahub-web-react/src/graphql/dataJob.graphql
@@ -26,6 +26,9 @@ query getDataJob($urn: String!) {
...browsePathV2Fields
}
...notes
+ platform {
+ ...platformFields
+ }
}
}
diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql
index 95afe0e4ea18be..0b340ddec31c1d 100644
--- a/datahub-web-react/src/graphql/fragments.graphql
+++ b/datahub-web-react/src/graphql/fragments.graphql
@@ -532,6 +532,9 @@ fragment nonRecursiveDataJobFields on DataJob {
subTypes {
typeNames
}
+ platform {
+ ...platformFields
+ }
}
fragment dataJobFields on DataJob {
@@ -604,6 +607,9 @@ fragment dataJobFields on DataJob {
dataTransformLogic {
...dataTransformLogicFields
}
+ platform {
+ ...platformFields
+ }
}
fragment dashboardFields on Dashboard {
diff --git a/datahub-web-react/src/images/FTE-ai-documentation.mp4 b/datahub-web-react/src/images/FTE-ai-documentation.mp4
new file mode 100644
index 00000000000000..2d06546d0e83d2
Binary files /dev/null and b/datahub-web-react/src/images/FTE-ai-documentation.mp4 differ
diff --git a/datahub-web-react/src/images/FTE-impact.mp4 b/datahub-web-react/src/images/FTE-impact.mp4
new file mode 100644
index 00000000000000..671d878bafe51d
Binary files /dev/null and b/datahub-web-react/src/images/FTE-impact.mp4 differ
diff --git a/datahub-web-react/src/images/FTE-lineage.mp4 b/datahub-web-react/src/images/FTE-lineage.mp4
new file mode 100644
index 00000000000000..7c4fff73f741c6
Binary files /dev/null and b/datahub-web-react/src/images/FTE-lineage.mp4 differ
diff --git a/datahub-web-react/src/images/FTE-search.mp4 b/datahub-web-react/src/images/FTE-search.mp4
new file mode 100644
index 00000000000000..faad77e1ae2923
Binary files /dev/null and b/datahub-web-react/src/images/FTE-search.mp4 differ
diff --git a/datahub-web-react/src/images/welcome-modal-home-screenshot.png b/datahub-web-react/src/images/welcome-modal-home-screenshot.png
new file mode 100644
index 00000000000000..b3bb5712b9f1de
Binary files /dev/null and b/datahub-web-react/src/images/welcome-modal-home-screenshot.png differ
diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock
index 1ed18770875d63..721b9ea85bdf11 100644
--- a/datahub-web-react/yarn.lock
+++ b/datahub-web-react/yarn.lock
@@ -6455,6 +6455,14 @@ call-bind-apply-helpers@^1.0.0:
es-errors "^1.3.0"
function-bind "^1.1.2"
+call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz#4b5428c222be985d79c3d82657479dbe0b59b2d6"
+ integrity sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==
+ dependencies:
+ es-errors "^1.3.0"
+ function-bind "^1.1.2"
+
call-bind@^1.0.0, call-bind@^1.0.2:
version "1.0.2"
resolved "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz"
@@ -7512,6 +7520,15 @@ dset@^3.1.2:
resolved "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz"
integrity sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==
+dunder-proto@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a"
+ integrity sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==
+ dependencies:
+ call-bind-apply-helpers "^1.0.1"
+ es-errors "^1.3.0"
+ gopd "^1.2.0"
+
duplexer@^0.1.2:
version "0.1.2"
resolved "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz"
@@ -7635,9 +7652,9 @@ es-abstract@^1.19.0, es-abstract@^1.20.4:
string.prototype.trimstart "^1.0.5"
unbox-primitive "^1.0.2"
-es-define-property@^1.0.0:
+es-define-property@^1.0.0, es-define-property@^1.0.1:
version "1.0.1"
- resolved "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz"
+ resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa"
integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==
es-errors@^1.3.0:
@@ -7665,6 +7682,23 @@ es-module-lexer@^1.7.0:
resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-1.7.0.tgz#9159601561880a85f2734560a9099b2c31e5372a"
integrity sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==
+es-object-atoms@^1.0.0, es-object-atoms@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz#1c4f2c4837327597ce69d2ca190a7fdd172338c1"
+ integrity sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==
+ dependencies:
+ es-errors "^1.3.0"
+
+es-set-tostringtag@^2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz#f31dbbe0c183b00a6d26eb6325c810c0fd18bd4d"
+ integrity sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==
+ dependencies:
+ es-errors "^1.3.0"
+ get-intrinsic "^1.2.6"
+ has-tostringtag "^1.0.2"
+ hasown "^2.0.2"
+
es-shim-unscopables@^1.0.0:
version "1.0.0"
resolved "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz"
@@ -8291,12 +8325,14 @@ foreground-child@^3.1.0:
signal-exit "^4.0.1"
form-data@^4.0.0:
- version "4.0.0"
- resolved "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz"
- integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
+ version "4.0.4"
+ resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.4.tgz#784cdcce0669a9d68e94d11ac4eea98088edd2c4"
+ integrity sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==
dependencies:
asynckit "^0.4.0"
combined-stream "^1.0.8"
+ es-set-tostringtag "^2.1.0"
+ hasown "^2.0.2"
mime-types "^2.1.12"
format@^0.2.0:
@@ -8398,6 +8434,30 @@ get-intrinsic@^1.2.4:
has-symbols "^1.0.3"
hasown "^2.0.0"
+get-intrinsic@^1.2.6:
+ version "1.3.0"
+ resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01"
+ integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==
+ dependencies:
+ call-bind-apply-helpers "^1.0.2"
+ es-define-property "^1.0.1"
+ es-errors "^1.3.0"
+ es-object-atoms "^1.1.1"
+ function-bind "^1.1.2"
+ get-proto "^1.0.1"
+ gopd "^1.2.0"
+ has-symbols "^1.1.0"
+ hasown "^2.0.2"
+ math-intrinsics "^1.1.0"
+
+get-proto@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/get-proto/-/get-proto-1.0.1.tgz#150b3f2743869ef3e851ec0c49d15b1d14d00ee1"
+ integrity sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==
+ dependencies:
+ dunder-proto "^1.0.1"
+ es-object-atoms "^1.0.0"
+
get-symbol-description@^1.0.0:
version "1.0.0"
resolved "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz"
@@ -8475,6 +8535,11 @@ gopd@^1.0.1:
dependencies:
get-intrinsic "^1.1.3"
+gopd@^1.2.0:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1"
+ integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==
+
graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0:
version "4.2.11"
resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz"
@@ -8593,6 +8658,11 @@ has-symbols@^1.0.2, has-symbols@^1.0.3:
resolved "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz"
integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==
+has-symbols@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338"
+ integrity sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==
+
has-tostringtag@^1.0.0:
version "1.0.0"
resolved "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz"
@@ -9838,6 +9908,11 @@ math-expression-evaluator@^1.2.14:
resolved "https://registry.npmjs.org/math-expression-evaluator/-/math-expression-evaluator-1.3.7.tgz"
integrity sha512-nrbaifCl42w37hYd6oRLvoymFK42tWB+WQTMFtksDGQMi5GvlJwnz/CsS30FFAISFLtX+A0csJ0xLiuuyyec7w==
+math-intrinsics@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9"
+ integrity sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==
+
mdast-util-definitions@^4.0.0:
version "4.0.0"
resolved "https://registry.npmjs.org/mdast-util-definitions/-/mdast-util-definitions-4.0.0.tgz"
diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile
index 0db5e817683dbb..bf4ccb254d9634 100644
--- a/docker/datahub-frontend/Dockerfile
+++ b/docker/datahub-frontend/Dockerfile
@@ -1,7 +1,7 @@
# Defining environment
ARG APP_ENV=prod
-FROM alpine:3.21 AS base
+FROM alpine:3.22 AS base
# Configurable repositories
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile
index ea6e758d4c499e..e2ec51e315e691 100644
--- a/docker/datahub-gms/Dockerfile
+++ b/docker/datahub-gms/Dockerfile
@@ -6,7 +6,7 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
ARG GITHUB_REPO_URL=https://github.com
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
-FROM golang:1-alpine3.21 AS binary
+FROM golang:1-alpine3.22 AS binary
# Re-declaring arg from above to make it available in this stage (will inherit default value)
ARG ALPINE_REPO_URL
@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
-FROM alpine:3.21 AS base
+FROM alpine:3.22 AS base
ENV JMX_VERSION=0.20.0
diff --git a/docker/datahub-mae-consumer/Dockerfile b/docker/datahub-mae-consumer/Dockerfile
index fbedef5ae392cb..210c112029765d 100644
--- a/docker/datahub-mae-consumer/Dockerfile
+++ b/docker/datahub-mae-consumer/Dockerfile
@@ -6,7 +6,7 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
ARG GITHUB_REPO_URL=https://github.com
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
-FROM golang:1-alpine3.21 AS binary
+FROM golang:1-alpine3.22 AS binary
# Re-declaring arg from above to make it available in this stage (will inherit default value)
ARG ALPINE_REPO_URL
@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
-FROM alpine:3.21 AS base
+FROM alpine:3.22 AS base
# Re-declaring args from above to make them available in this stage (will inherit default values)
ARG ALPINE_REPO_URL
diff --git a/docker/datahub-mce-consumer/Dockerfile b/docker/datahub-mce-consumer/Dockerfile
index 3179e647f51b34..2ca991fa12e276 100644
--- a/docker/datahub-mce-consumer/Dockerfile
+++ b/docker/datahub-mce-consumer/Dockerfile
@@ -6,7 +6,7 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
ARG GITHUB_REPO_URL=https://github.com
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
-FROM golang:1-alpine3.21 AS binary
+FROM golang:1-alpine3.22 AS binary
# Re-declaring arg from above to make it available in this stage (will inherit default value)
ARG ALPINE_REPO_URL
@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
-FROM alpine:3.21 AS base
+FROM alpine:3.22 AS base
# Re-declaring args from above to make them available in this stage (will inherit default values)
ARG ALPINE_REPO_URL
diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile
index c3bd4b3f8d52a5..c44ca0df8b2f0f 100644
--- a/docker/datahub-upgrade/Dockerfile
+++ b/docker/datahub-upgrade/Dockerfile
@@ -6,7 +6,7 @@ ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
ARG GITHUB_REPO_URL=https://github.com
ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
-FROM golang:1-alpine3.21 AS binary
+FROM golang:1-alpine3.22 AS binary
# Re-declaring arg from above to make it available in this stage (will inherit default value)
ARG ALPINE_REPO_URL
@@ -23,7 +23,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
-FROM alpine:3.21 AS base
+FROM alpine:3.22 AS base
# Re-declaring args from above to make them available in this stage (will inherit default values)
ARG ALPINE_REPO_URL
diff --git a/docker/elasticsearch-setup/Dockerfile b/docker/elasticsearch-setup/Dockerfile
index 274cb4130d404d..db1bb116a38a97 100644
--- a/docker/elasticsearch-setup/Dockerfile
+++ b/docker/elasticsearch-setup/Dockerfile
@@ -6,7 +6,7 @@ ARG APP_ENV=prod
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
-FROM golang:1-alpine3.21 AS binary
+FROM golang:1-alpine3.22 AS binary
ARG ALPINE_REPO_URL
@@ -25,7 +25,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
-FROM alpine:3.21 AS base
+FROM alpine:3.22 AS base
ARG ALPINE_REPO_URL
diff --git a/docker/mysql-setup/Dockerfile b/docker/mysql-setup/Dockerfile
index 9da4c7754c2650..b60be6a45dea47 100644
--- a/docker/mysql-setup/Dockerfile
+++ b/docker/mysql-setup/Dockerfile
@@ -1,7 +1,7 @@
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
-FROM golang:1-alpine3.21 AS binary
+FROM golang:1-alpine3.22 AS binary
ARG ALPINE_REPO_URL
@@ -19,7 +19,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
-FROM alpine:3.21
+FROM alpine:3.22
COPY --from=binary /go/bin/dockerize /usr/local/bin
ARG ALPINE_REPO_URL
diff --git a/docker/postgres-setup/Dockerfile b/docker/postgres-setup/Dockerfile
index 36fb95e129f1ee..a66e16ec058c02 100644
--- a/docker/postgres-setup/Dockerfile
+++ b/docker/postgres-setup/Dockerfile
@@ -1,7 +1,7 @@
# Defining custom repo urls for use in enterprise environments. Re-used between stages below.
ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
-FROM golang:1-alpine3.21 AS binary
+FROM golang:1-alpine3.22 AS binary
ARG ALPINE_REPO_URL
@@ -19,7 +19,7 @@ WORKDIR /go/src/github.com/jwilder/dockerize
RUN go install github.com/jwilder/dockerize@$DOCKERIZE_VERSION
-FROM alpine:3.21
+FROM alpine:3.22
COPY --from=binary /go/bin/dockerize /usr/local/bin
ARG ALPINE_REPO_URL
diff --git a/docker/snippets/oracle_instantclient.sh b/docker/snippets/oracle_instantclient.sh
index ff5400c539d5e1..086d85696dbc4f 100755
--- a/docker/snippets/oracle_instantclient.sh
+++ b/docker/snippets/oracle_instantclient.sh
@@ -7,6 +7,7 @@ if [ $(arch) = "x86_64" ]; then
wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/2115000/instantclient-basic-linux.x64-21.15.0.0.0dbru.zip
unzip instantclient-basic-linux.x64-21.15.0.0.0dbru.zip
rm instantclient-basic-linux.x64-21.15.0.0.0dbru.zip
+ (cd /usr/lib/*-linux-gnu/; ln -s ./libaio.so.1t64 ./libaio.so.1)
sh -c "echo /opt/oracle/instantclient_21_15 > /etc/ld.so.conf.d/oracle-instantclient.conf"
ldconfig
else
@@ -15,6 +16,7 @@ else
wget --no-verbose -c https://download.oracle.com/otn_software/linux/instantclient/1923000/instantclient-basic-linux.arm64-19.23.0.0.0dbru.zip
unzip instantclient-basic-linux.arm64-19.23.0.0.0dbru.zip
rm instantclient-basic-linux.arm64-19.23.0.0.0dbru.zip
+ (cd /usr/lib/*-linux-gnu/; ln -s ./libaio.so.1t64 ./libaio.so.1)
sh -c "echo /opt/oracle/instantclient_19_23 > /etc/ld.so.conf.d/oracle-instantclient.conf"
ldconfig
fi
diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
index 689396ae3a35f0..f95a1ac58c0689 100644
--- a/docs-website/sidebars.js
+++ b/docs-website/sidebars.js
@@ -445,6 +445,7 @@ module.exports = {
},
{
"DataHub Cloud Release History": [
+ "docs/managed-datahub/release-notes/v_0_3_13",
"docs/managed-datahub/release-notes/v_0_3_12",
"docs/managed-datahub/release-notes/v_0_3_11",
"docs/managed-datahub/release-notes/v_0_3_10",
diff --git a/docs/lineage/openlineage.md b/docs/lineage/openlineage.md
index 043e5bfc6cd92d..507a28f5e1be8b 100644
--- a/docs/lineage/openlineage.md
+++ b/docs/lineage/openlineage.md
@@ -21,7 +21,7 @@ With Spark and Airflow we recommend using the Spark Lineage or DataHub's Airflow
To send OpenLineage messages to DataHub using the REST endpoint, simply make a POST request to the following endpoint:
```
-POST GMS_SERVER_HOST:GMS_PORT/api/v2/lineage
+POST GMS_SERVER_HOST:GMS_PORT/openapi/openlineage/api/v1/lineage
```
Include the OpenLineage message in the request body in JSON format.
@@ -74,6 +74,30 @@ The transport should look like this:
}
```
+#### How to modify configurations
+
+To modify the configurations for the OpenLineage REST endpoint, you can change it using environment variables. The following configurations are available:
+
+##### DataHub OpenLineage Configuration
+
+This document describes all available configuration options for the DataHub OpenLineage integration, including environment variables, application properties, and their usage.
+
+##### Configuration Overview
+
+The DataHub OpenLineage integration can be configured using environment variables, application properties files (`application.yml` or `application.properties`), or JVM system properties. All configuration options are prefixed with `datahub.openlineage`.
+
+##### Environment Variables
+
+| Environment Variable | Property | Type | Default | Description |
+| ------------------------------------------------------ | ------------------------------------------------------ | ------- | ------- | --------------------------------------------------------------- |
+| `DATAHUB_OPENLINEAGE_PLATFORM_INSTANCE` | `datahub.openlineage.platform-instance` | String | `null` | Specific platform instance identifier |
+| `DATAHUB_OPENLINEAGE_COMMON_DATASET_PLATFORM_INSTANCE` | `datahub.openlineage.common-dataset-platform-instance` | String | `null` | Common platform instance for datasets |
+| `DATAHUB_OPENLINEAGE_MATERIALIZE_DATASET` | `datahub.openlineage.materialize-dataset` | Boolean | `true` | Whether to materialize dataset entities |
+| `DATAHUB_OPENLINEAGE_INCLUDE_SCHEMA_METADATA` | `datahub.openlineage.include-schema-metadata` | Boolean | `true` | Whether to include schema metadata in lineage |
+| `DATAHUB_OPENLINEAGE_CAPTURE_COLUMN_LEVEL_LINEAGE` | `datahub.openlineage.capture-column-level-lineage` | Boolean | `true` | Whether to capture column-level lineage information |
+| `DATAHUB_OPENLINEAGE_FILE_PARTITION_REGEXP_PATTERN` | `datahub.openlineage.file-partition-regexp-pattern` | String | `null` | Regular expression pattern for file partition detection |
+| `DATAHUB_OPENLINEAGE_USE_PATCH` | `datahub.openlineage.use-patch` | Boolean | `false` | Whether to use patch operations for lineage/incremental lineage |
+
#### Known Limitations
With Spark and Airflow we recommend using the Spark Lineage or DataHub's Airflow plugin for tighter integration with DataHub.
diff --git a/docs/managed-datahub/configuring-identity-provisioning-with-okta.md b/docs/managed-datahub/configuring-identity-provisioning-with-okta.md
index 64398279759af7..a4ca601f8118a9 100644
--- a/docs/managed-datahub/configuring-identity-provisioning-with-okta.md
+++ b/docs/managed-datahub/configuring-identity-provisioning-with-okta.md
@@ -16,7 +16,14 @@ This document covers the steps required to enable SCIM provisioning from Okta to
This document assumes you are using OIDC for SSO with DataHub.
Since Okta doesn't currently support SCIM with OIDC, you would need to create an additional SWA-app-integration to enable SCIM provisioning.
-On completing the steps in this guide, Okta will start automatically pushing changes to users/groups of this SWA-app-integration to DataHub, thereby simplifying provisioning of users/groups in DataHub.
+After completing this guide, Okta will automatically sync user and group changes from your SWA app integration to DataHub. This streamlines user and group provisioning in DataHub.
+
+Important notes about roles and permissions:
+
+- User and group roles from Okta are not transferred to DataHub
+- When a group is first synced to DataHub, you can assign roles to that group directly in DataHub
+- All users in the group will inherit the group's assigned roles
+- You can also apply DataHub policies to groups, which will affect users based on their group membership
### Why SCIM provisioning?
@@ -26,19 +33,18 @@ Consider the following configuration in Okta
- A group `governance-team`
- And it has two members `john` and `sid`
-- And the group has role `Reader`
Through SCIM provisioning, the following are enabled:
-- If the `governance-team` group is assigned to the DataHub app in Okta with the role `Reader`, Okta will create the users `john` and `sid` in DataHub with the `Reader` role.
+- Okta can create a group `governance-team` in DataHub when you "Push Groups" in Okta.
+- If the `governance-team` group is assigned to the DataHub app in Okta, Okta will create the users `john` and `sid` in DataHub.
- If you remove `john` from group `governance-team` then `john` would automatically get deactivated in DataHub.
- If you remove `sid` from the DataHub app in Okta, then `sid` would automatically get deactivated in DataHub.
-Generally, any user assignment/unassignment to the app in Okta - directly or through groups - are automatically reflected in the DataHub application.
+By default, groups and users synced from Okta to DataHub through this integration have no roles assigned. You can assign
+roles to these groups directly within DataHub.
-This guide also covers other variations such as how to assign a role to a user directly, and how group-information can be pushed to DataHub.
-
-> Only Admin, Editor and Reader roles are supported in DataHub. These roles are preconfigured/created on DataHub.
+Generally, any user assignment/unassignment to the app in Okta - directly or through groups - are automatically reflected in the DataHub application.
## Configuring SCIM provisioning
@@ -76,29 +82,7 @@ c). Configure the `To App` section as shown below:
**Note**: We are not pushing passwords to DataHub over SCIM, since we are assuming SSO with OIDC as mentioned earlier.
-### 3. Add a custom attribute to represent roles
-
-a). Navigate to `Directory` -> `Profile Editor`, and select the user-profile of this new application.
-
-
-
-
-
-b). Click `Add Attribute` and define a new attribute that will be used to specify the role of a DataHub user.
-
-
-
-
-
-- Set value of `External name` to `roles.^[primary==true].value`
-- Set value of `External namespace` to `urn:ietf:params:scim:schemas:core:2.0:User`
-- Define an enumerated list of values as shown in the above image
-- Mark this attribute as required
-- Select `Attribute type` as `Personal`
-
-c). Add a similar attribute for groups i.e. repeat step (b) above, but select `Attribute Type` as `Group`. (Specify the variable name as, say, `dataHubGroupRoles`.)
-
-### 4. Assign users & groups to the app
+### 3. Assign users & groups to the app
Assign users and groups to the app from the `Assignments` tab:
@@ -106,9 +90,6 @@ Assign users and groups to the app from the `Assignments` tab:
-While assigning a user/group, choose an appropriate value for the dataHubRoles/dataHubGroupRoles attribute.
-Note that when a role is selected for a group, the corresponding role is pushed for all users of that group in DataHub.
-
### The provisioning setup is now complete
Once the above steps are completed, user assignments/unassignments to the DataHub-SCIM-SWA app in Okta will get reflected in DataHub automatically.
@@ -119,7 +100,7 @@ Once the above steps are completed, user assignments/unassignments to the DataHu
> But when a user is _deleted_ in Okta, the corresponding user in DataHub does _not_ get deleted.
> Refer the Okta documentation on [Delete (Deprovision)](https://developer.okta.com/docs/concepts/scim/#delete-deprovision) for more details.
-### 5. (Optional): Configure push groups
+### 5. (Recommended): Configure push groups
When groups are assigned to the app, Okta pushes the group-members as users to DataHub, but the group itself isn't pushed.
To push group information to DataHub, configure the `Push Groups` tab accordingly as shown below:
@@ -128,4 +109,5 @@ To push group information to DataHub, configure the `Push Groups` tab accordingl
+When you assign roles or policies to groups in DataHub, all users in those groups automatically inherit the same roles or policies
Refer to the Okta [Group Push](https://help.okta.com/en-us/content/topics/users-groups-profiles/app-assignments-group-push.htm) documentation for more details.
diff --git a/docs/managed-datahub/release-notes/v0_3_13.md b/docs/managed-datahub/release-notes/v_0_3_13.md
similarity index 99%
rename from docs/managed-datahub/release-notes/v0_3_13.md
rename to docs/managed-datahub/release-notes/v_0_3_13.md
index 8f575bc5256d4e..b2264cd36f30b4 100644
--- a/docs/managed-datahub/release-notes/v0_3_13.md
+++ b/docs/managed-datahub/release-notes/v_0_3_13.md
@@ -15,7 +15,7 @@ This contains detailed release notes, but there's also an [announcement blog pos
- **CLI/SDK**: 1.2.0.1
- **Remote Executor**: v0.3.13-acryl (recommended), v0.3.12.4-acryl, v0.3.11.1-acryl
- **On-Prem Versions**:
- - **Helm**: 1.5.66
+ - **Helm**: 1.5.68
- **API Gateway**: 0.5.3
- **Actions**: 0.0.13
diff --git a/docs/managed-datahub/workflows/access-workflows.md b/docs/managed-datahub/workflows/access-workflows.md
index adea61b5599294..49c8f570010404 100644
--- a/docs/managed-datahub/workflows/access-workflows.md
+++ b/docs/managed-datahub/workflows/access-workflows.md
@@ -249,7 +249,7 @@ To create an approval workflow request, users must simply provide responses for
Once completed, it can be submitted by clicking "Submit".
-
+
Once a request is submitted, your open requests will be visible from within **Tasks** > **Requests** > **My Requests**.
@@ -271,7 +271,7 @@ Users assigned as reviewers can manage requests through the **Task Center**:
- Submitted form responses
-
+
2. **Make a Decision**: For each request, you can:
@@ -282,7 +282,7 @@ Users assigned as reviewers can manage requests through the **Task Center**:
3. **Add Comments**: Provide context for your decision to help requestors understand the outcome
-
+
## Getting Notified About Access Workflows
diff --git a/metadata-ingestion/docs/sources/dbt/dbt.md b/metadata-ingestion/docs/sources/dbt/dbt.md
index 80ed11df53c570..978f930f1b92c9 100644
--- a/metadata-ingestion/docs/sources/dbt/dbt.md
+++ b/metadata-ingestion/docs/sources/dbt/dbt.md
@@ -78,6 +78,7 @@ Note:
1. The dbt `meta_mapping` config works at the model level, while the `column_meta_mapping` config works at the column level. The `add_owner` operation is not supported at the column level.
2. For string meta properties we support regex matching.
+3. **List support**: YAML lists are now supported in meta properties. Each item in the list that matches the regex pattern will be processed.
With regex matching, you can also use the matched value to customize how you populate the tag, term or owner fields. Here are a few advanced examples:
@@ -175,6 +176,29 @@ meta_mapping:
In the examples above, we show two ways of writing the matching regexes. In the first one, `^@(.*)` the first matching group (a.k.a. match.group(1)) is automatically inferred. In the second example, `^@(?P(.*))`, we use a named matching group (called owner, since we are matching an owner) to capture the string we want to provide to the ownership urn.
+#### Working with Lists
+
+YAML lists are fully supported in dbt meta properties. Each item in the list is evaluated against the match pattern, and only matching items are processed.
+
+```yaml
+meta:
+ owners:
+ - alice@company.com
+ - bob@company.com
+ - contractor@external.com
+```
+
+```yaml
+meta_mapping:
+ owners:
+ match: ".*@company.com"
+ operation: "add_owner"
+ config:
+ owner_type: user
+```
+
+This will add `alice@company.com` and `bob@company.com` as owners (matching `.*@company.com`) but skip `contractor@external.com` (doesn't match the pattern).
+
### dbt query_tag automated mappings
This works similarly as the dbt meta mapping but for the query tags
diff --git a/metadata-ingestion/scripts/capability_summary.py b/metadata-ingestion/scripts/capability_summary.py
index c243996653b17d..ca02186a6cf408 100644
--- a/metadata-ingestion/scripts/capability_summary.py
+++ b/metadata-ingestion/scripts/capability_summary.py
@@ -19,6 +19,7 @@
"snowflake-summary",
"snowflake-queries",
"bigquery-queries",
+ "datahub-mock-data",
}
diff --git a/metadata-ingestion/scripts/docgen.py b/metadata-ingestion/scripts/docgen.py
index e55f73737945a9..097422b84c5ed9 100644
--- a/metadata-ingestion/scripts/docgen.py
+++ b/metadata-ingestion/scripts/docgen.py
@@ -24,6 +24,12 @@
logger = logging.getLogger(__name__)
+DENY_LIST = {
+ "snowflake-summary",
+ "snowflake-queries",
+ "bigquery-queries",
+ "datahub-mock-data",
+}
def get_snippet(long_string: str, max_length: int = 100) -> str:
snippet = ""
@@ -302,11 +308,7 @@ def generate(
if source and source != plugin_name:
continue
- if plugin_name in {
- "snowflake-summary",
- "snowflake-queries",
- "bigquery-queries",
- }:
+ if plugin_name in DENY_LIST:
logger.info(f"Skipping {plugin_name} as it is on the deny list")
continue
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
index 27aaf39260a9d8..39465ed9844bed 100644
--- a/metadata-ingestion/setup.py
+++ b/metadata-ingestion/setup.py
@@ -751,6 +751,7 @@
"mongodb",
"slack",
"mssql",
+ "mssql-odbc",
"mysql",
"mariadb",
"redash",
diff --git a/metadata-ingestion/src/datahub/emitter/rest_emitter.py b/metadata-ingestion/src/datahub/emitter/rest_emitter.py
index c35386aadd91d5..46914330c7fc95 100644
--- a/metadata-ingestion/src/datahub/emitter/rest_emitter.py
+++ b/metadata-ingestion/src/datahub/emitter/rest_emitter.py
@@ -95,7 +95,7 @@
TRACE_MAX_BACKOFF = 300.0 # Cap at 5 minutes
TRACE_BACKOFF_FACTOR = 2.0 # Double the wait time each attempt
-# The limit is 16mb. We will use a max of 15mb to have some space
+# The limit is 16,000,000 bytes. We will use a max of 15mb to have some space
# for overhead like request headers.
# This applies to pretty much all calls to GMS.
INGEST_MAX_PAYLOAD_BYTES = int(
@@ -586,6 +586,11 @@ def emit_mce(self, mce: MetadataChangeEvent) -> None:
"systemMetadata": system_metadata_obj,
}
payload = json.dumps(snapshot)
+ if len(payload) > INGEST_MAX_PAYLOAD_BYTES:
+ logger.warning(
+ f"MCE object has size {len(payload)} that exceeds the max payload size of {INGEST_MAX_PAYLOAD_BYTES}, "
+ "so this metadata will likely fail to be emitted."
+ )
self._emit_generic(url, payload)
@@ -764,16 +769,24 @@ def _emit_restli_mcps(
url = f"{self._gms_server}/aspects?action=ingestProposalBatch"
mcp_objs = [pre_json_transform(mcp.to_obj()) for mcp in mcps]
+ if len(mcp_objs) == 0:
+ return 0
# As a safety mechanism, we need to make sure we don't exceed the max payload size for GMS.
# If we will exceed the limit, we need to break it up into chunks.
- mcp_obj_chunks: List[List[str]] = []
- current_chunk_size = INGEST_MAX_PAYLOAD_BYTES
+ mcp_obj_chunks: List[List[str]] = [[]]
+ current_chunk_size = 0
for mcp_obj in mcp_objs:
+ mcp_identifier = f"{mcp_obj.get('entityUrn')}-{mcp_obj.get('aspectName')}"
mcp_obj_size = len(json.dumps(mcp_obj))
if _DATAHUB_EMITTER_TRACE:
logger.debug(
- f"Iterating through object with size {mcp_obj_size} (type: {mcp_obj.get('aspectName')}"
+ f"Iterating through object ({mcp_identifier}) with size {mcp_obj_size}"
+ )
+ if mcp_obj_size > INGEST_MAX_PAYLOAD_BYTES:
+ logger.warning(
+ f"MCP object {mcp_identifier} has size {mcp_obj_size} that exceeds the max payload size of {INGEST_MAX_PAYLOAD_BYTES}, "
+ "so this metadata will likely fail to be emitted."
)
if (
@@ -786,7 +799,7 @@ def _emit_restli_mcps(
current_chunk_size = 0
mcp_obj_chunks[-1].append(mcp_obj)
current_chunk_size += mcp_obj_size
- if len(mcp_obj_chunks) > 0:
+ if len(mcp_obj_chunks) > 1 or _DATAHUB_EMITTER_TRACE:
logger.debug(
f"Decided to send {len(mcps)} MCP batch in {len(mcp_obj_chunks)} chunks"
)
diff --git a/metadata-ingestion/src/datahub/ingestion/api/source.py b/metadata-ingestion/src/datahub/ingestion/api/source.py
index 9ddda6d58c627c..bf1d29f7a5945a 100644
--- a/metadata-ingestion/src/datahub/ingestion/api/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/api/source.py
@@ -81,11 +81,24 @@ class StructuredLogLevel(Enum):
ERROR = logging.ERROR
+class StructuredLogCategory(Enum):
+ """
+ This is used to categorise the errors mainly based on the biggest impact area
+ This is to be used to help in self-serve understand the impact of any log entry
+ More enums to be added as logs are updated to be self-serve
+ """
+
+ LINEAGE = "LINEAGE"
+ USAGE = "USAGE"
+ PROFILING = "PROFILING"
+
+
@dataclass
class StructuredLogEntry(Report):
title: Optional[str]
message: str
context: LossyList[str]
+ log_category: Optional[StructuredLogCategory] = None
@dataclass
@@ -108,9 +121,10 @@ def report_log(
exc: Optional[BaseException] = None,
log: bool = False,
stacklevel: int = 1,
+ log_category: Optional[StructuredLogCategory] = None,
) -> None:
"""
- Report a user-facing warning for the ingestion run.
+ Report a user-facing log for the ingestion run.
Args:
level: The level of the log entry.
@@ -118,6 +132,9 @@ def report_log(
title: The category / heading to present on for this message in the UI.
context: Additional context (e.g. where, how) for the log entry.
exc: The exception associated with the event. We'll show the stack trace when in debug mode.
+ log_category: The type of the log entry. This is used to categorise the log entry.
+ log: Whether to log the entry to the console.
+ stacklevel: The stack level to use for the log entry.
"""
# One for this method, and one for the containing report_* call.
@@ -160,6 +177,7 @@ def report_log(
title=title,
message=message,
context=context_list,
+ log_category=log_category,
)
else:
if context is not None:
@@ -219,9 +237,19 @@ def report_warning(
context: Optional[str] = None,
title: Optional[LiteralString] = None,
exc: Optional[BaseException] = None,
+ log_category: Optional[StructuredLogCategory] = None,
) -> None:
+ """
+ See docs of StructuredLogs.report_log for details of args
+ """
self._structured_logs.report_log(
- StructuredLogLevel.WARN, message, title, context, exc, log=False
+ StructuredLogLevel.WARN,
+ message,
+ title,
+ context,
+ exc,
+ log=False,
+ log_category=log_category,
)
def warning(
@@ -231,9 +259,19 @@ def warning(
title: Optional[LiteralString] = None,
exc: Optional[BaseException] = None,
log: bool = True,
+ log_category: Optional[StructuredLogCategory] = None,
) -> None:
+ """
+ See docs of StructuredLogs.report_log for details of args
+ """
self._structured_logs.report_log(
- StructuredLogLevel.WARN, message, title, context, exc, log=log
+ StructuredLogLevel.WARN,
+ message,
+ title,
+ context,
+ exc,
+ log=log,
+ log_category=log_category,
)
def report_failure(
@@ -243,9 +281,19 @@ def report_failure(
title: Optional[LiteralString] = None,
exc: Optional[BaseException] = None,
log: bool = True,
+ log_category: Optional[StructuredLogCategory] = None,
) -> None:
+ """
+ See docs of StructuredLogs.report_log for details of args
+ """
self._structured_logs.report_log(
- StructuredLogLevel.ERROR, message, title, context, exc, log=log
+ StructuredLogLevel.ERROR,
+ message,
+ title,
+ context,
+ exc,
+ log=log,
+ log_category=log_category,
)
def failure(
@@ -255,9 +303,19 @@ def failure(
title: Optional[LiteralString] = None,
exc: Optional[BaseException] = None,
log: bool = True,
+ log_category: Optional[StructuredLogCategory] = None,
) -> None:
+ """
+ See docs of StructuredLogs.report_log for details of args
+ """
self._structured_logs.report_log(
- StructuredLogLevel.ERROR, message, title, context, exc, log=log
+ StructuredLogLevel.ERROR,
+ message,
+ title,
+ context,
+ exc,
+ log=log,
+ log_category=log_category,
)
def info(
@@ -267,9 +325,19 @@ def info(
title: Optional[LiteralString] = None,
exc: Optional[BaseException] = None,
log: bool = True,
+ log_category: Optional[StructuredLogCategory] = None,
) -> None:
+ """
+ See docs of StructuredLogs.report_log for details of args
+ """
self._structured_logs.report_log(
- StructuredLogLevel.INFO, message, title, context, exc, log=log
+ StructuredLogLevel.INFO,
+ message,
+ title,
+ context,
+ exc,
+ log=log,
+ log_category=log_category,
)
@contextlib.contextmanager
@@ -279,6 +347,7 @@ def report_exc(
title: Optional[LiteralString] = None,
context: Optional[str] = None,
level: StructuredLogLevel = StructuredLogLevel.ERROR,
+ log_category: Optional[StructuredLogCategory] = None,
) -> Iterator[None]:
# Convenience method that helps avoid boilerplate try/except blocks.
# TODO: I'm not super happy with the naming here - it's not obvious that this
@@ -287,7 +356,12 @@ def report_exc(
yield
except Exception as exc:
self._structured_logs.report_log(
- level, message=message, title=title, context=context, exc=exc
+ level,
+ message=message,
+ title=title,
+ context=context,
+ exc=exc,
+ log_category=log_category,
)
def __post_init__(self) -> None:
diff --git a/metadata-ingestion/src/datahub/ingestion/autogenerated/capability_summary.json b/metadata-ingestion/src/datahub/ingestion/autogenerated/capability_summary.json
index c38a3fdce7cf4c..82596cfbb921e2 100644
--- a/metadata-ingestion/src/datahub/ingestion/autogenerated/capability_summary.json
+++ b/metadata-ingestion/src/datahub/ingestion/autogenerated/capability_summary.json
@@ -1,9 +1,18 @@
{
- "generated_at": "2025-07-24T13:24:05.751563+00:00",
+ "generated_at": "2025-07-31T12:54:30.557618+00:00",
"generated_by": "metadata-ingestion/scripts/capability_summary.py",
"plugin_details": {
"abs": {
"capabilities": [
+ {
+ "capability": "CONTAINERS",
+ "description": "Extract ABS containers and folders",
+ "subtype_modifier": [
+ "Folder",
+ "ABS container"
+ ],
+ "supported": true
+ },
{
"capability": "DATA_PROFILING",
"description": "Optionally enabled via configuration",
@@ -468,7 +477,9 @@
{
"capability": "CONTAINERS",
"description": "Enabled by default",
- "subtype_modifier": null,
+ "subtype_modifier": [
+ "Database"
+ ],
"supported": true
},
{
@@ -531,13 +542,6 @@
"platform_name": "File Based Lineage",
"support_status": "CERTIFIED"
},
- "datahub-mock-data": {
- "capabilities": [],
- "classname": "datahub.ingestion.source.mock_data.datahub_mock_data.DataHubMockDataSource",
- "platform_id": "datahubmockdata",
- "platform_name": "DataHubMockData",
- "support_status": "TESTING"
- },
"dbt": {
"capabilities": [
{
@@ -607,7 +611,9 @@
{
"capability": "CONTAINERS",
"description": "Enabled by default",
- "subtype_modifier": null,
+ "subtype_modifier": [
+ "Folder"
+ ],
"supported": true
},
{
@@ -643,6 +649,14 @@
"subtype_modifier": null,
"supported": true
},
+ {
+ "capability": "LINEAGE_FINE",
+ "description": "Extract column-level lineage",
+ "subtype_modifier": [
+ "Table"
+ ],
+ "supported": true
+ },
{
"capability": "DATA_PROFILING",
"description": "Optionally enabled via configuration",
@@ -688,7 +702,9 @@
{
"capability": "LINEAGE_COARSE",
"description": "Enabled by default",
- "subtype_modifier": null,
+ "subtype_modifier": [
+ "Table"
+ ],
"supported": true
}
],
@@ -1229,8 +1245,7 @@
"capability": "CONTAINERS",
"description": "Enabled by default",
"subtype_modifier": [
- "Database",
- "Schema"
+ "Catalog"
],
"supported": true
},
@@ -2387,8 +2402,9 @@
},
{
"capability": "LINEAGE_COARSE",
- "description": "Enabled by default to get lineage for views via `include_view_lineage`",
+ "description": "Extract table-level lineage",
"subtype_modifier": [
+ "Table",
"View"
],
"supported": true
@@ -2411,8 +2427,7 @@
"capability": "CONTAINERS",
"description": "Enabled by default",
"subtype_modifier": [
- "Database",
- "Schema"
+ "Catalog"
],
"supported": true
},
@@ -2598,7 +2613,8 @@
"capability": "CONTAINERS",
"description": "Enabled by default",
"subtype_modifier": [
- "Database"
+ "Database",
+ "Schema"
],
"supported": true
},
@@ -2812,6 +2828,15 @@
"description": "Enabled by default",
"subtype_modifier": null,
"supported": true
+ },
+ {
+ "capability": "LINEAGE_COARSE",
+ "description": "Extract table-level lineage for Salesforce objects",
+ "subtype_modifier": [
+ "Custom Object",
+ "Object"
+ ],
+ "supported": true
}
],
"classname": "datahub.ingestion.source.salesforce.SalesforceSource",
@@ -3207,7 +3232,9 @@
{
"capability": "CONTAINERS",
"description": "Enabled by default",
- "subtype_modifier": null,
+ "subtype_modifier": [
+ "Database"
+ ],
"supported": true
},
{
@@ -3339,8 +3366,9 @@
},
{
"capability": "LINEAGE_COARSE",
- "description": "Enabled by default to get lineage for views via `include_view_lineage`",
+ "description": "Extract table-level lineage",
"subtype_modifier": [
+ "Table",
"View"
],
"supported": true
diff --git a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
index 1d619f678e02e3..b78ae5282bb313 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
@@ -44,6 +44,7 @@
get_key_prefix,
strip_abs_prefix,
)
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
from datahub.ingestion.source.data_lake_common.data_lake_utils import (
ContainerWUCreator,
add_partition_columns_to_schema,
@@ -128,6 +129,14 @@ class TableData:
@support_status(SupportStatus.INCUBATING)
@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
@capability(SourceCapability.TAGS, "Can extract ABS object/container tags if enabled")
+@capability(
+ SourceCapability.CONTAINERS,
+ "Extract ABS containers and folders",
+ subtype_modifier=[
+ SourceCapabilityModifier.FOLDER,
+ SourceCapabilityModifier.ABS_CONTAINER,
+ ],
+)
class ABSSource(StatefulIngestionSourceBase):
source_config: DataLakeSourceConfig
report: DataLakeSourceReport
diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
index ce24bee62641e4..1325341d67c0d3 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py
@@ -19,6 +19,7 @@
auto_workunit_reporter,
)
from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
from datahub.ingestion.source.datahub.config import DataHubSourceConfig
from datahub.ingestion.source.datahub.datahub_api_reader import DataHubApiReader
from datahub.ingestion.source.datahub.datahub_database_reader import (
@@ -39,7 +40,13 @@
@platform_name("DataHub")
@config_class(DataHubSourceConfig)
@support_status(SupportStatus.TESTING)
-@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(
+ SourceCapability.CONTAINERS,
+ "Enabled by default",
+ subtype_modifier=[
+ SourceCapabilityModifier.DATABASE,
+ ],
+)
class DataHubSource(StatefulIngestionSourceBase):
platform: str = "datahub"
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
index e77b601981211b..c41b630ff9f454 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -120,6 +120,7 @@
DBT_PLATFORM = "dbt"
_DEFAULT_ACTOR = mce_builder.make_user_urn("unknown")
+_DBT_MAX_COMPILED_CODE_LENGTH = 1 * 1024 * 1024 # 1MB
@dataclass
@@ -1684,6 +1685,12 @@ def _create_dataset_properties_aspect(
def get_external_url(self, node: DBTNode) -> Optional[str]:
pass
+ @staticmethod
+ def _truncate_code(code: str, max_length: int) -> str:
+ if len(code) > max_length:
+ return code[:max_length] + "..."
+ return code
+
def _create_view_properties_aspect(
self, node: DBTNode
) -> Optional[ViewPropertiesClass]:
@@ -1695,6 +1702,9 @@ def _create_view_properties_aspect(
compiled_code = try_format_query(
node.compiled_code, platform=self.config.target_platform
)
+ compiled_code = self._truncate_code(
+ compiled_code, _DBT_MAX_COMPILED_CODE_LENGTH
+ )
materialized = node.materialization in {"table", "incremental", "snapshot"}
view_properties = ViewPropertiesClass(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py
index 297691d27d3394..1139663a429268 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py
@@ -29,6 +29,7 @@
get_key_prefix,
strip_s3_prefix,
)
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
from datahub.ingestion.source.data_lake_common.data_lake_utils import ContainerWUCreator
from datahub.ingestion.source.delta_lake.config import DeltaLakeSourceConfig
from datahub.ingestion.source.delta_lake.delta_lake_utils import (
@@ -85,7 +86,13 @@
@config_class(DeltaLakeSourceConfig)
@support_status(SupportStatus.INCUBATING)
@capability(SourceCapability.TAGS, "Can extract S3 object/bucket tags if enabled")
-@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(
+ SourceCapability.CONTAINERS,
+ "Enabled by default",
+ subtype_modifier=[
+ SourceCapabilityModifier.FOLDER,
+ ],
+)
class DeltaLakeSource(StatefulIngestionSourceBase):
"""
This plugin extracts:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py
index 6e1cfd7cb15891..82a8f959ee0112 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py
@@ -22,6 +22,7 @@
SourceReport,
)
from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
from datahub.ingestion.source.dremio.dremio_api import (
DremioAPIOperations,
DremioEdition,
@@ -86,11 +87,27 @@ class DremioSourceMapEntry:
@platform_name("Dremio")
@config_class(DremioSourceConfig)
@support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(
+ SourceCapability.CONTAINERS,
+ "Enabled by default",
+)
@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
@capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
-@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
+@capability(
+ SourceCapability.LINEAGE_COARSE,
+ "Enabled by default",
+ subtype_modifier=[
+ SourceCapabilityModifier.TABLE,
+ ],
+)
+@capability(
+ SourceCapability.LINEAGE_FINE,
+ "Extract column-level lineage",
+ subtype_modifier=[
+ SourceCapabilityModifier.TABLE,
+ ],
+)
@capability(SourceCapability.OWNERSHIP, "Enabled by default")
@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
@capability(SourceCapability.USAGE_STATS, "Enabled by default to get usage stats")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
index fe98c26e335fd7..eb4dee3201efc6 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py
@@ -16,7 +16,11 @@
platform_name,
support_status,
)
-from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport
+from datahub.ingestion.api.source import (
+ MetadataWorkUnitProcessor,
+ SourceReport,
+ StructuredLogCategory,
+)
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.fivetran.config import (
KNOWN_DATA_PLATFORM_MAPPING,
@@ -96,8 +100,10 @@ def _extend_lineage(self, connector: Connector, datajob: DataJob) -> Dict[str, s
self.report.info(
title="Guessing source platform for lineage",
message="We encountered a connector type that we don't fully support yet. "
- "We will attempt to guess the platform based on the connector type.",
- context=f"{connector.connector_name} (connector_id: {connector.connector_id}, connector_type: {connector.connector_type})",
+ "We will attempt to guess the platform based on the connector type. "
+ "Note that we use connector_id as the key not connector_name which you may see in the UI of Fivetran. ",
+ context=f"connector_name: {connector.connector_name} (connector_id: {connector.connector_id}, connector_type: {connector.connector_type})",
+ log_category=StructuredLogCategory.LINEAGE,
)
source_details.platform = connector.connector_type
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py
index f48ba6758564bf..57ff9415726d2f 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py
@@ -1,5 +1,5 @@
from functools import lru_cache
-from typing import ClassVar, Optional, TextIO
+from typing import ClassVar, Optional, TextIO, Type
from liquid import Environment
from liquid.ast import Node
@@ -20,16 +20,27 @@ def __init__(self, message):
class ConditionNode(Node):
def __init__(self, tok: Token, sql_or_lookml_reference: str, filter_name: str):
self.tok = tok
-
self.sql_or_lookml_reference = sql_or_lookml_reference
-
self.filter_name = filter_name
def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]:
# This implementation will make sure that sql parse work correctly if looker condition tag
# is used in lookml sql field
buffer.write(f"{self.sql_or_lookml_reference}='dummy_value'")
+ return True
+
+class IncrementConditionNode(Node):
+ def __init__(self, tok: Token, sql_or_lookml_reference: str):
+ self.tok = tok
+ self.sql_or_lookml_reference = sql_or_lookml_reference
+
+ def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]:
+ # For incrementcondition, we need to generate a condition that would be used
+ # in incremental PDT updates. This typically involves date/time comparisons.
+ # We'll render it as a date comparison with a placeholder value
+ # See details in Looker documentation for incrementcondition tag -> cloud.google.com/looker/docs/reference/param-view-increment-key
+ buffer.write(f"{self.sql_or_lookml_reference} > '2023-01-01'")
return True
@@ -44,7 +55,6 @@ class ConditionTag(Tag):
This class render the below tag as order.region='ap-south-1' if order_region is provided in config.liquid_variables
as order_region: 'ap-south-1'
{% condition order_region %} order.region {% endcondition %}
-
"""
TAG_START: ClassVar[str] = "condition"
@@ -79,7 +89,48 @@ def parse(self, stream: TokenStream) -> Node:
)
-custom_tags = [ConditionTag]
+class IncrementConditionTag(Tag):
+ """
+ IncrementConditionTag is the equivalent implementation of looker's custom liquid tag "incrementcondition".
+ Refer doc: https://cloud.google.com/looker/docs/incremental-pdts#using_the_incrementcondition_tag
+
+ This tag is used for incremental PDTs to determine which records should be updated.
+ It typically works with date/time fields to filter data that has changed since the last update.
+
+ Example usage in Looker:
+ {% incrementcondition created_at %} order.created_at {% endincrementcondition %}
+
+ This would generate SQL like: order.created_at > '2023-01-01 00:00:00'
+ """
+
+ TAG_START: ClassVar[str] = "incrementcondition"
+ TAG_END: ClassVar[str] = "endincrementcondition"
+ name: str = "incrementcondition"
+
+ def __init__(self, env: Environment):
+ super().__init__(env)
+ self.parser = get_parser(self.env)
+
+ def parse(self, stream: TokenStream) -> Node:
+ expect(stream, TOKEN_TAG, value=IncrementConditionTag.TAG_START)
+
+ start_token = stream.current
+
+ stream.next_token()
+ expect(stream, TOKEN_LITERAL)
+ sql_or_lookml_reference: str = stream.current.value.strip()
+
+ stream.next_token()
+ expect(stream, TOKEN_TAG, value=IncrementConditionTag.TAG_END)
+
+ return IncrementConditionNode(
+ tok=start_token,
+ sql_or_lookml_reference=sql_or_lookml_reference,
+ )
+
+
+# Updated custom_tags list to include both tags
+custom_tags: list[Type[Tag]] = [ConditionTag, IncrementConditionTag]
@string_filter
diff --git a/metadata-ingestion/src/datahub/ingestion/source/mock_data/datahub_mock_data.py b/metadata-ingestion/src/datahub/ingestion/source/mock_data/datahub_mock_data.py
index 6522f8222acdfe..9077536a7172f4 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/mock_data/datahub_mock_data.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/mock_data/datahub_mock_data.py
@@ -13,7 +13,7 @@
platform_name,
support_status,
)
-from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.source import Source, SourceReport, StructuredLogCategory
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.common.subtypes import DatasetSubTypes
from datahub.ingestion.source.mock_data.datahub_mock_data_report import (
@@ -35,6 +35,8 @@
logger = logging.getLogger(__name__)
+PLATFORM_NAME = "fake"
+
class SubTypePattern(StrEnum):
ALTERNATING = "alternating"
@@ -144,7 +146,7 @@ class DataHubMockDataConfig(ConfigModel):
)
-@platform_name("DataHubMockData")
+@platform_name(PLATFORM_NAME)
@config_class(DataHubMockDataConfig)
@support_status(SupportStatus.TESTING)
class DataHubMockDataSource(Source):
@@ -176,6 +178,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
message="This is test warning",
title="Test Warning",
context=f"This is test warning {i}",
+ log_category=StructuredLogCategory.LINEAGE,
)
# We don't want any implicit aspects to be produced
@@ -309,7 +312,7 @@ def _get_subtypes_aspect(
table_level, table_index, subtype_pattern, subtype_types, level_subtypes
)
- urn = make_dataset_urn(platform="fake", name=table_name)
+ urn = make_dataset_urn(platform=PLATFORM_NAME, name=table_name)
mcp = MetadataChangeProposalWrapper(
entityUrn=urn,
entityType="dataset",
@@ -433,7 +436,7 @@ def _generate_downstream_lineage(
def _get_status_aspect(self, table: str) -> MetadataWorkUnit:
urn = make_dataset_urn(
- platform="fake",
+ platform=PLATFORM_NAME,
name=table,
)
mcp = MetadataChangeProposalWrapper(
@@ -448,7 +451,7 @@ def _get_upstream_aspect(
) -> MetadataWorkUnit:
mcp = MetadataChangeProposalWrapper(
entityUrn=make_dataset_urn(
- platform="fake",
+ platform=PLATFORM_NAME,
name=downstream_table,
),
entityType="dataset",
@@ -456,7 +459,7 @@ def _get_upstream_aspect(
upstreams=[
UpstreamClass(
dataset=make_dataset_urn(
- platform="fake",
+ platform=PLATFORM_NAME,
name=upstream_table,
),
type=DatasetLineageTypeClass.TRANSFORMED,
@@ -468,7 +471,7 @@ def _get_upstream_aspect(
def _get_profile_aspect(self, table: str) -> MetadataWorkUnit:
urn = make_dataset_urn(
- platform="fake",
+ platform=PLATFORM_NAME,
name=table,
)
mcp = MetadataChangeProposalWrapper(
@@ -485,7 +488,7 @@ def _get_profile_aspect(self, table: str) -> MetadataWorkUnit:
def _get_usage_aspect(self, table: str) -> MetadataWorkUnit:
urn = make_dataset_urn(
- platform="fake",
+ platform=PLATFORM_NAME,
name=table,
)
mcp = MetadataChangeProposalWrapper(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
index 9b8dbe98932013..3c80f329da7174 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
@@ -1226,7 +1226,10 @@ def report_to_datahub_work_units(
@platform_name("PowerBI")
@config_class(PowerBiDashboardSourceConfig)
@support_status(SupportStatus.CERTIFIED)
-@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(
+ SourceCapability.CONTAINERS,
+ "Enabled by default",
+)
@capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
@capability(SourceCapability.OWNERSHIP, "Enabled by default")
@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
index d238dd549885eb..72a8b546488359 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
@@ -132,6 +132,7 @@
"Enabled by default",
subtype_modifier=[
SourceCapabilityModifier.DATABASE,
+ SourceCapabilityModifier.SCHEMA,
],
)
@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
diff --git a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py
index 4d3853f8c25792..03b70c72ad092e 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py
@@ -549,6 +549,14 @@ def get_approximate_record_count(self, sObjectName: str) -> SObjectRecordCount:
capability_name=SourceCapability.TAGS,
description="Enabled by default",
)
+@capability(
+ capability_name=SourceCapability.LINEAGE_COARSE,
+ description="Extract table-level lineage for Salesforce objects",
+ subtype_modifier=[
+ SourceCapabilityModifier.SALESFORCE_CUSTOM_OBJECT,
+ SourceCapabilityModifier.SALESFORCE_STANDARD_OBJECT,
+ ],
+)
class SalesforceSource(StatefulIngestionSourceBase):
def __init__(self, config: SalesforceConfig, ctx: PipelineContext) -> None:
super().__init__(config, ctx)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py
index b2b80d7eb0b5e1..858c67019038d9 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py
@@ -27,6 +27,7 @@
from datahub.ingestion.source.common.subtypes import (
DatasetContainerSubTypes,
DatasetSubTypes,
+ SourceCapabilityModifier,
)
from datahub.ingestion.source.sql.sql_common import (
SQLAlchemySource,
@@ -168,6 +169,13 @@ def get_sql_alchemy_url(
@capability(
SourceCapability.LINEAGE_COARSE, "View lineage is not supported", supported=False
)
+@capability(
+ SourceCapability.CONTAINERS,
+ "Enabled by default",
+ subtype_modifier=[
+ SourceCapabilityModifier.CATALOG,
+ ],
+)
class HiveMetastoreSource(SQLAlchemySource):
"""
This plugin extracts the following:
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
index d0c9d724056a69..dacd6a54d8dda5 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py
@@ -160,7 +160,12 @@ def get_sql_alchemy_url(
uri_opts=uri_opts,
)
if self.use_odbc:
- uri = f"{uri}?{urllib.parse.urlencode(self.uri_args)}"
+ uri_args = (
+ {k: v for k, v in self.uri_args.items() if k.lower() != "database"}
+ if current_db
+ else self.uri_args
+ )
+ uri = f"{uri}?{urllib.parse.urlencode(uri_args)}"
return uri
@property
@@ -939,6 +944,24 @@ def construct_flow_workunits(
aspect=data_flow.as_container_aspect,
).as_workunit()
+ def _database_names_from_engine(self, engine):
+ """
+ Helper method to get database names from the engine.
+ This is used to fetch the list of databases in the SQL Server instance.
+ """
+ with engine.begin() as conn:
+ databases = conn.execute(
+ "SELECT name FROM master.sys.databases WHERE name NOT IN \
+ ('master', 'model', 'msdb', 'tempdb', 'Resource', \
+ 'distribution' , 'reportserver', 'reportservertempdb'); "
+ ).fetchall()
+ return [db["name"] for db in databases]
+
+ def _inspector_for_database(self, db_name: str) -> Inspector:
+ url = self.config.get_sql_alchemy_url(current_db=db_name)
+ engine = create_engine(url, **self.config.options)
+ return inspect(engine)
+
def get_inspectors(self) -> Iterable[Inspector]:
# This method can be overridden in the case that you want to dynamically
# run on multiple databases.
@@ -950,19 +973,10 @@ def get_inspectors(self) -> Iterable[Inspector]:
inspector = inspect(engine)
yield inspector
else:
- with engine.begin() as conn:
- databases = conn.execute(
- "SELECT name FROM master.sys.databases WHERE name NOT IN \
- ('master', 'model', 'msdb', 'tempdb', 'Resource', \
- 'distribution' , 'reportserver', 'reportservertempdb'); "
- ).fetchall()
-
- for db in databases:
- if self.config.database_pattern.allowed(db["name"]):
- url = self.config.get_sql_alchemy_url(current_db=db["name"])
- engine = create_engine(url, **self.config.options)
- inspector = inspect(engine)
- self.current_database = db["name"]
+ for db_name in self._database_names_from_engine(engine):
+ if self.config.database_pattern.allowed(db_name):
+ inspector = self._inspector_for_database(db_name)
+ self.current_database = db_name
yield inspector
def get_identifier(
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
index ea6eb14232761d..9157eebd6a39ac 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
@@ -42,6 +42,7 @@
)
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.graph.client import DataHubGraph
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
from datahub.ingestion.source.sql.sql_common import register_custom_type
from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
from datahub.ingestion.source.sql.sql_report import SQLSourceReport
@@ -539,7 +540,13 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
@config_class(TeradataConfig)
@support_status(SupportStatus.TESTING)
@capability(SourceCapability.DOMAINS, "Enabled by default")
-@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(
+ SourceCapability.CONTAINERS,
+ "Enabled by default",
+ subtype_modifier=[
+ SourceCapabilityModifier.DATABASE,
+ ],
+)
@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
@capability(
SourceCapability.DELETION_DETECTION,
diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py
index d7ea886a86b388..60382302fd9e56 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py
@@ -36,6 +36,7 @@
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.extractor import schema_util
from datahub.ingestion.source.common.data_reader import DataReader
+from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
from datahub.ingestion.source.sql.sql_common import (
SQLAlchemySource,
SqlWorkUnit,
@@ -249,6 +250,14 @@ def get_identifier(self: BasicSQLAlchemyConfig, schema: str, table: str) -> str:
@support_status(SupportStatus.CERTIFIED)
@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
+@capability(
+ SourceCapability.LINEAGE_COARSE,
+ "Extract table-level lineage",
+ subtype_modifier=[
+ SourceCapabilityModifier.TABLE,
+ SourceCapabilityModifier.VIEW,
+ ],
+)
class TrinoSource(SQLAlchemySource):
"""
diff --git a/metadata-ingestion/src/datahub/utilities/mapping.py b/metadata-ingestion/src/datahub/utilities/mapping.py
index 4772730c90bf1b..7245c2e7ca9e44 100644
--- a/metadata-ingestion/src/datahub/utilities/mapping.py
+++ b/metadata-ingestion/src/datahub/utilities/mapping.py
@@ -83,7 +83,7 @@ class Constants:
MATCH = "match"
USER_OWNER = "user"
GROUP_OWNER = "group"
- OPERAND_DATATYPE_SUPPORTED = [int, bool, str, float]
+ OPERAND_DATATYPE_SUPPORTED = [int, bool, str, float, list]
TAG_PARTITION_KEY = "PARTITION_KEY"
TAG_DIST_KEY = "DIST_KEY"
TAG_SORT_KEY = "SORT_KEY"
@@ -455,7 +455,34 @@ def get_match(self, match_clause: Any, raw_props_value: Any) -> Optional[Match]:
# function to check if a match clause is satisfied to a value.
if not any(
isinstance(raw_props_value, t) for t in Constants.OPERAND_DATATYPE_SUPPORTED
- ) or not isinstance(raw_props_value, type(match_clause)):
+ ):
+ return None
+
+ # Handle list values by checking if any item in the list matches
+ if isinstance(raw_props_value, list):
+ # For lists, we need to find at least one matching item
+ # Return a match with the concatenated values of all matching items
+ matching_items = []
+ for item in raw_props_value:
+ if isinstance(item, str):
+ match = re.match(match_clause, item)
+ if match:
+ matching_items.append(item)
+ elif isinstance(match_clause, type(item)):
+ match = re.match(str(match_clause), str(item))
+ if match:
+ matching_items.append(str(item))
+
+ if matching_items:
+ # Create a synthetic match object with all matching items joined
+ combined_value = ",".join(matching_items)
+ return re.match(
+ ".*", combined_value
+ ) # Always matches, returns combined value
+ return None
+
+ # Handle scalar values (existing logic)
+ elif not isinstance(raw_props_value, type(match_clause)):
return None
elif isinstance(raw_props_value, str):
return re.match(match_clause, raw_props_value)
diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py
index 13458810096a05..66c579dd561f8d 100644
--- a/metadata-ingestion/tests/integration/lookml/test_lookml.py
+++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py
@@ -1009,6 +1009,46 @@ def test_special_liquid_variables():
assert actual_text == expected_text
+@freeze_time(FROZEN_TIME)
+def test_incremental_liquid_expression():
+ text: str = """SELECT
+ user_id,
+ DATE(event_timestamp) as event_date,
+ COUNT(*) as daily_events,
+ SUM(revenue) as daily_revenue,
+ MAX(event_timestamp) as last_event_time
+ FROM warehouse.events.user_events
+ WHERE {% incrementcondition %} event_timestamp {% endincrementcondition %}
+ AND event_type IN ('purchase', 'signup', 'login')
+ AND user_id IS NOT NULL
+ GROUP BY 1, 2
+ """
+ input_liquid_variable: dict = {}
+
+ # Match template after resolution of liquid variables
+ actual_text = resolve_liquid_variable(
+ text=text,
+ liquid_variable=input_liquid_variable,
+ report=LookMLSourceReport(),
+ view_name="test",
+ )
+
+ expected_text: str = """SELECT
+ user_id,
+ DATE(event_timestamp) as event_date,
+ COUNT(*) as daily_events,
+ SUM(revenue) as daily_revenue,
+ MAX(event_timestamp) as last_event_time
+ FROM warehouse.events.user_events
+ WHERE event_timestamp > '2023-01-01'
+ AND event_type IN ('purchase', 'signup', 'login')
+ AND user_id IS NOT NULL
+ GROUP BY 1, 2
+ """
+
+ assert actual_text == expected_text
+
+
@pytest.mark.parametrize(
"view, expected_result, warning_expected",
[
diff --git a/metadata-ingestion/tests/unit/test_mapping.py b/metadata-ingestion/tests/unit/test_mapping.py
index fff9ba81fb42e5..0a934bed0a6330 100644
--- a/metadata-ingestion/tests/unit/test_mapping.py
+++ b/metadata-ingestion/tests/unit/test_mapping.py
@@ -452,3 +452,65 @@ def test_validate_ownership_type_non_urn_invalid():
# Non-urn input that is not valid should raise ValueError.
with pytest.raises(ValueError):
validate_ownership_type("invalid_type")
+
+
+def test_operation_processor_list_values():
+ """Test that list values are properly handled in operation definitions."""
+ raw_props = {
+ "owners_list": [
+ "owner1@company.com",
+ "owner2@company.com",
+ "owner3@company.com",
+ ],
+ "tags_list": ["tag1", "tag2", "tag3"],
+ "mixed_list": ["match1", "nomatch", "match2"],
+ }
+
+ processor = OperationProcessor(
+ operation_defs={
+ "owners_list": {
+ "match": ".*@company.com",
+ "operation": "add_owner",
+ "config": {"owner_type": "user"},
+ },
+ "tags_list": {
+ "match": "tag.*",
+ "operation": "add_tag",
+ "config": {"tag": "list_{{ $match }}"},
+ },
+ "mixed_list": {
+ "match": "match.*",
+ "operation": "add_term",
+ "config": {"term": "{{ $match }}"},
+ },
+ },
+ strip_owner_email_id=True,
+ )
+
+ aspect_map = processor.process(raw_props)
+
+ # Test owners from list
+ assert "add_owner" in aspect_map
+ ownership_aspect: OwnershipClass = aspect_map["add_owner"]
+ assert len(ownership_aspect.owners) == 3
+ owner_urns = {owner.owner for owner in ownership_aspect.owners}
+ expected_owners = {
+ "urn:li:corpuser:owner1",
+ "urn:li:corpuser:owner2",
+ "urn:li:corpuser:owner3",
+ }
+ assert owner_urns == expected_owners
+
+ # Test tags from list - note: tags use the match replacement but join with comma
+ assert "add_tag" in aspect_map
+ tag_aspect: GlobalTagsClass = aspect_map["add_tag"]
+ assert len(tag_aspect.tags) == 1
+ # The matched values get joined with comma, and commas get URL-encoded in URNs
+ assert tag_aspect.tags[0].tag == "urn:li:tag:list_tag1%2Ctag2%2Ctag3"
+
+ # Test terms from list - only matching items
+ assert "add_term" in aspect_map
+ term_aspect: GlossaryTermsClass = aspect_map["add_term"]
+ assert len(term_aspect.terms) == 1
+ # The matched values get joined with comma - terms don't get URL-encoded
+ assert term_aspect.terms[0].urn == "urn:li:glossaryTerm:match1,match2"
diff --git a/metadata-ingestion/tests/unit/test_mssql.py b/metadata-ingestion/tests/unit/test_mssql.py
index 1256672c1ad2d8..f129164a803db6 100644
--- a/metadata-ingestion/tests/unit/test_mssql.py
+++ b/metadata-ingestion/tests/unit/test_mssql.py
@@ -299,3 +299,44 @@ def test_stored_procedure_vs_direct_query_compatibility(mssql_source):
# Verify database_name is properly set
assert sp_step["database_name"] == "test_db"
assert direct_step["database_name"] == "test_db"
+
+
+def test_get_sql_alchemy_url_ignores_config_db_when_override_is_provided():
+ """Test that the database name is ignored when an override is provided"""
+ override_db = "override_db"
+ config_defined_db = "config_defined_connection_db"
+
+ config = SQLServerConfig(
+ host_port="localhost:1433",
+ username="test",
+ password="test",
+ use_odbc=True,
+ uri_args={"database": config_defined_db, "driver": "some_driver_value"},
+ )
+
+ # Mock to avoid DB connections
+ with (
+ patch("datahub.ingestion.source.sql.sql_common.SQLAlchemySource.__init__"),
+ patch(
+ "datahub.ingestion.source.sql.mssql.source.SQLServerSource._database_names_from_engine"
+ ) as mock_db_names,
+ patch(
+ "datahub.ingestion.source.sql.mssql.source.SQLServerSource._inspector_for_database"
+ ),
+ ):
+ mock_db_names.return_value = [override_db]
+ source = SQLServerSource(config, MagicMock())
+
+ # Call the method with the override
+ default_connection = source.config.get_sql_alchemy_url()
+
+ # assert that when not overridden, the default connection uses the config defined database
+ assert config_defined_db in default_connection
+ assert override_db not in default_connection
+
+ # assert that when overridden, the override database is used
+ override_db_connection = source.config.get_sql_alchemy_url(
+ current_db=override_db
+ )
+ assert override_db in override_db_connection
+ assert config_defined_db not in override_db_connection
diff --git a/metadata-integration/java/acryl-spark-lineage/README.md b/metadata-integration/java/acryl-spark-lineage/README.md
index 27da37ca799a5c..ac535fa798d629 100644
--- a/metadata-integration/java/acryl-spark-lineage/README.md
+++ b/metadata-integration/java/acryl-spark-lineage/README.md
@@ -2,7 +2,7 @@
To integrate Spark with DataHub, we provide a lightweight Java agent that listens for Spark application and job events
and pushes metadata out to DataHub in real-time. The agent listens to events such as application start/end, and
-SQLExecution start/end to create pipelines (i.e. DataJob) and tasks (i.e. DataFlow) in Datahub along with lineage to
+SQLExecution start/end to create pipelines (i.e. DataFlow) and tasks (i.e. DataJob) in DataHub along with lineage to
datasets that are being read from and written to. Read on to learn how to configure this for different Spark scenarios.
## Configuring Spark agent
@@ -18,13 +18,27 @@ available [here](https://github.com/datahub-project/datahub/releases).
Always check [the Maven central repository](https://search.maven.org/search?q=a:acryl-spark-lineage) for the latest
released version.
+**Note**: Starting from version 0.2.18, we provide separate jars for different Scala versions:
+
+- For Scala 2.12: `io.acryl:acryl-spark-lineage_2.12:0.2.18`
+- For Scala 2.13: `io.acryl:acryl-spark-lineage_2.13:0.2.18`
+
### Configuration Instructions: spark-submit
When running jobs using spark-submit, the agent needs to be configured in the config file.
```text
#Configuring DataHub spark agent jar
-spark.jars.packages io.acryl:acryl-spark-lineage:0.2.17
+spark.jars.packages io.acryl:acryl-spark-lineage_2.12:0.2.18
+spark.extraListeners datahub.spark.DatahubSparkListener
+spark.datahub.rest.server http://localhost:8080
+```
+
+For Scala 2.13:
+
+```text
+#Configuring DataHub spark agent jar for Scala 2.13
+spark.jars.packages io.acryl:acryl-spark-lineage_2.13:0.2.18
spark.extraListeners datahub.spark.DatahubSparkListener
spark.datahub.rest.server http://localhost:8080
```
@@ -32,7 +46,13 @@ spark.datahub.rest.server http://localhost:8080
## spark-submit command line
```sh
-spark-submit --packages io.acryl:acryl-spark-lineage:0.2.17 --conf "spark.extraListeners=datahub.spark.DatahubSparkListener" my_spark_job_to_run.py
+spark-submit --packages io.acryl:acryl-spark-lineage_2.12:0.2.18 --conf "spark.extraListeners=datahub.spark.DatahubSparkListener" my_spark_job_to_run.py
+```
+
+For Scala 2.13:
+
+```sh
+spark-submit --packages io.acryl:acryl-spark-lineage_2.13:0.2.18 --conf "spark.extraListeners=datahub.spark.DatahubSparkListener" my_spark_job_to_run.py
```
### Configuration Instructions: Amazon EMR
@@ -41,7 +61,7 @@ Set the following spark-defaults configuration properties as it
stated [here](https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark-configure.html)
```text
-spark.jars.packages io.acryl:acryl-spark-lineage:0.2.17
+spark.jars.packages io.acryl:acryl-spark-lineage_2.12:0.2.18
spark.extraListeners datahub.spark.DatahubSparkListener
spark.datahub.rest.server https://your_datahub_host/gms
#If you have authentication set up then you also need to specify the Datahub access token
@@ -56,7 +76,7 @@ When running interactive jobs from a notebook, the listener can be configured wh
spark = SparkSession.builder
.master("spark://spark-master:7077")
.appName("test-application")
-.config("spark.jars.packages", "io.acryl:acryl-spark-lineage:0.2.17")
+.config("spark.jars.packages", "io.acryl:acryl-spark-lineage_2.12:0.2.18")
.config("spark.extraListeners", "datahub.spark.DatahubSparkListener")
.config("spark.datahub.rest.server", "http://localhost:8080")
.enableHiveSupport()
@@ -79,7 +99,7 @@ appName("test-application")
config("spark.master","spark://spark-master:7077")
.
-config("spark.jars.packages","io.acryl:acryl-spark-lineage:0.2.17")
+config("spark.jars.packages","io.acryl:acryl-spark-lineage_2.12:0.2.18")
.
config("spark.extraListeners","datahub.spark.DatahubSparkListener")
@@ -159,47 +179,48 @@ information like tokens.
## Configuration Options
-| Field | Required | Default | Description |
-| ------------------------------------------------------ | -------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --- |
-| spark.jars.packages | ✅ | | Set with latest/required version io.acryl:acryl-spark-lineage:0.2.15 |
-| spark.extraListeners | ✅ | | datahub.spark.DatahubSparkListener |
-| spark.datahub.emitter | | rest | Specify the ways to emit metadata. By default it sends to DataHub using REST emitter. Valid options are rest, kafka or file |
-| spark.datahub.rest.server | | http://localhost:8080 | Datahub server url eg: |
-| spark.datahub.rest.token | | | Authentication token. |
-| spark.datahub.rest.disable_ssl_verification | | false | Disable SSL certificate validation. Caution: Only use this if you know what you are doing! |
-| spark.datahub.rest.disable_chunked_encoding | | false | Disable Chunked Transfer Encoding. In some environment chunked encoding causes issues. With this config option it can be disabled. | |
-| spark.datahub.rest.max_retries | | 0 | Number of times a request retried if failed |
-| spark.datahub.rest.retry_interval | | 10 | Number of seconds to wait between retries |
-| spark.datahub.file.filename | | | The file where metadata will be written if file emitter is set |
-| spark.datahub.kafka.bootstrap | | | The Kafka bootstrap server url to use if the Kafka emitter is set |
-| spark.datahub.kafka.schema_registry_url | | | The Schema registry url to use if the Kafka emitter is set |
-| spark.datahub.kafka.schema_registry_config. | | | Additional config to pass in to the Schema Registry Client |
-| spark.datahub.kafka.producer_config. | | | Additional config to pass in to the Kafka producer. For example: `--conf "spark.datahub.kafka.producer_config.client.id=my_client_id"` |
-| spark.datahub.metadata.pipeline.platformInstance | | | Pipeline level platform instance |
-| spark.datahub.metadata.dataset.platformInstance | | | dataset level platform instance (it is usefult to set if you have it in your glue ingestion) |
-| spark.datahub.metadata.dataset.env | | PROD | [Supported values](https://docs.datahub.com/docs/graphql/enums#fabrictype). In all other cases, will fallback to PROD |
-| spark.datahub.metadata.dataset.hivePlatformAlias | | hive | By default, datahub assigns Hive-like tables to the Hive platform. If you are using Glue as your Hive metastore, set this config flag to `glue` |
-| spark.datahub.metadata.include_scheme | | true | Include scheme from the path URI (e.g. hdfs://, s3://) in the dataset URN. We recommend setting this value to false, it is set to true for backwards compatibility with previous versions |
-| spark.datahub.metadata.remove_partition_pattern | | | Remove partition pattern. (e.g. /partition=\d+) It change database/table/partition=123 to database/table |
-| spark.datahub.coalesce_jobs | | true | Only one datajob(task) will be emitted containing all input and output datasets for the spark application |
-| spark.datahub.parent.datajob_urn | | | Specified dataset will be set as upstream dataset for datajob created. Effective only when spark.datahub.coalesce_jobs is set to true |
-| spark.datahub.metadata.dataset.materialize | | false | Materialize Datasets in DataHub |
-| spark.datahub.platform.s3.path_spec_list | | | List of pathspec per platform |
-| spark.datahub.metadata.dataset.include_schema_metadata | false | | Emit dataset schema metadata based on the spark execution. It is recommended to get schema information from platform specific DataHub sources as this is less reliable |
-| spark.datahub.flow_name | | | If it is set it will be used as the DataFlow name otherwise it uses spark app name as flow_name |
-| spark.datahub.file_partition_regexp | | | Strip partition part from the path if path end matches with the specified regexp. Example `year=.*/month=.*/day=.*` |
-| spark.datahub.tags | | | Comma separated list of tags to attach to the DataFlow |
-| spark.datahub.domains | | | Comma separated list of domain urns to attach to the DataFlow |
-| spark.datahub.stage_metadata_coalescing | | | Normally it coalesces and sends metadata at the onApplicationEnd event which is never called on Databricks or on Glue. You should enable this on Databricks if you want coalesced run. |
-| spark.datahub.patch.enabled | | false | Set this to true to send lineage as a patch, which appends rather than overwrites existing Dataset lineage edges. By default, it is disabled. |
-| spark.datahub.metadata.dataset.lowerCaseUrns | | false | Set this to true to lowercase dataset urns. By default, it is disabled. |
-| spark.datahub.disableSymlinkResolution | | false | Set this to true if you prefer using the s3 location instead of the Hive table. By default, it is disabled. |
-| spark.datahub.s3.bucket | | | The name of the bucket where metadata will be written if s3 emitter is set |
-| spark.datahub.s3.prefix | | | The prefix for the file where metadata will be written on s3 if s3 emitter is set |
-| spark.datahub.s3.filename | | | The name of the file where metadata will be written if it is not set random filename will be used on s3 if s3 emitter is set |
-| spark.datahub.s3.filename | | | The name of the file where metadata will be written if it is not set random filename will be used on s3 if s3 emitter is set |
-| spark.datahub.log.mcps | | true | Set this to true to log MCPS to the log. By default, it is enabled. |
-| spark.datahub.legacyLineageCleanup.enabled | | false | Set this to true to remove legacy lineages from older Spark Plugin runs. This will remove those lineages from the Datasets which it adds to DataJob. By default, it is disabled. |
+| Field | Required | Default | Description |
+| ---------------------------------------------------------------- | -------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| spark.jars.packages | ✅ | | Set with latest/required version io.acryl:acryl-spark-lineage_2.12:0.2.18 (or io.acryl:acryl-spark-lineage_2.13:0.2.18 for Scala 2.13) |
+| spark.extraListeners | ✅ | | datahub.spark.DatahubSparkListener |
+| spark.datahub.emitter | | rest | Specify the ways to emit metadata. By default it sends to DataHub using REST emitter. Valid options are rest, kafka or file |
+| spark.datahub.rest.server | | | Datahub server url eg: |
+| spark.datahub.rest.token | | | Authentication token. |
+| spark.datahub.rest.disable_ssl_verification | | false | Disable SSL certificate validation. Caution: Only use this if you know what you are doing! |
+| spark.datahub.rest.disable_chunked_encoding | | false | Disable Chunked Transfer Encoding. In some environment chunked encoding causes issues. With this config option it can be disabled. |
+| spark.datahub.rest.max_retries | | 0 | Number of times a request retried if failed |
+| spark.datahub.rest.retry_interval | | 10 | Number of seconds to wait between retries |
+| spark.datahub.file.filename | | | The file where metadata will be written if file emitter is set |
+| spark.datahub.kafka.bootstrap | | | The Kafka bootstrap server url to use if the Kafka emitter is set |
+| spark.datahub.kafka.schema_registry_url | | | The Schema registry url to use if the Kafka emitter is set |
+| spark.datahub.kafka.schema_registry_config. | | | Additional config to pass in to the Schema Registry Client |
+| spark.datahub.kafka.producer_config. | | | Additional config to pass in to the Kafka producer. For example: `--conf "spark.datahub.kafka.producer_config.client.id=my_client_id"` |
+| spark.datahub.metadata.pipeline.platformInstance | | | Pipeline level platform instance |
+| spark.datahub.metadata.dataset.platformInstance | | | Dataset level platform instance (useful when you need to match dataset URNs with those created by other ingestion sources) |
+| spark.datahub.metadata.dataset.env | | PROD | [Supported values](https://docs.datahub.com/docs/graphql/enums#fabrictype). In all other cases, will fall back to PROD |
+| spark.datahub.metadata.dataset.hivePlatformAlias | | hive | By default, DataHub assigns Hive-like tables to the Hive platform. If you are using Glue as your Hive metastore, set this config flag to `glue` |
+| spark.datahub.metadata.include_scheme | | true | Include scheme from the path URI (e.g. hdfs://, s3://) in the dataset URN. We recommend setting this value to false, but it is set to true for backwards compatibility with previous versions |
+| spark.datahub.metadata.remove_partition_pattern | | | Remove partition pattern (e.g. /partition=\d+). It changes database/table/partition=123 to database/table |
+| spark.datahub.coalesce_jobs | | true | Only one DataJob (task) will be emitted containing all input and output datasets for the Spark application |
+| spark.datahub.parent.datajob_urn | | | Specified dataset will be set as upstream dataset for DataJob created. Effective only when spark.datahub.coalesce_jobs is set to true |
+| spark.datahub.metadata.dataset.materialize | | false | Materialize Datasets in DataHub |
+| spark.datahub.platform.s3.path_spec_list | | | List of path specs per platform |
+| spark.datahub.metadata.dataset.include_schema_metadata | | false | Emit dataset schema metadata based on the Spark execution. It is recommended to get schema information from platform-specific DataHub sources as this is less reliable |
+| spark.datahub.flow_name | | | If set, it will be used as the DataFlow name; otherwise, it uses the Spark app name as flow_name |
+| spark.datahub.file_partition_regexp | | | Strip partition part from the path if the path end matches the specified regexp. Example: `year=.*/month=.*/day=.*` |
+| spark.datahub.tags | | | Comma-separated list of tags to attach to the DataFlow |
+| spark.datahub.domains | | | Comma-separated list of domain URNs to attach to the DataFlow |
+| spark.datahub.stage_metadata_coalescing | | false | Normally metadata is coalesced and sent at the onApplicationEnd event, which is never called on Databricks or on Glue. Enable this on Databricks if you want coalesced runs. |
+| spark.datahub.patch.enabled | | false | Set this to true to send lineage as a patch, which appends rather than overwrites existing Dataset lineage edges. By default, it is disabled. |
+| spark.datahub.metadata.dataset.lowerCaseUrns | | false | Set this to true to lowercase dataset URNs. By default, it is disabled. |
+| spark.datahub.disableSymlinkResolution | | false | Set this to true if you prefer using the S3 location instead of the Hive table. By default, it is disabled. |
+| spark.datahub.s3.bucket | | | The name of the bucket where metadata will be written if s3 emitter is set |
+| spark.datahub.s3.prefix | | | The prefix for the file where metadata will be written on s3 if s3 emitter is set |
+| spark.datahub.s3.filename | | | The name of the file where metadata will be written if it is not set random filename will be used on s3 if s3 emitter is set |
+| spark.datahub.log.mcps | | true | Set this to true to log MCPS to the log. By default, it is enabled. |
+| spark.datahub.legacyLineageCleanup.enabled | | false | Set this to true to remove legacy lineages from older Spark Plugin runs. This will remove those lineages from the Datasets which it adds to DataJob. By default, it is disabled. |
+| spark.datahub.capture_spark_plan | | false | Set this to true to capture the Spark plan. By default, it is disabled. |
+| spark.datahub.metadata.dataset.enableEnhancedMergeIntoExtraction | | false | Set this to true to enable enhanced table name extraction for Delta Lake MERGE INTO commands. This improves lineage tracking by including the target table name in the job name. By default, it is disabled. |
## What to Expect: The Metadata Model
@@ -224,6 +245,10 @@ The following custom properties in pipelines and tasks relate to the Spark UI:
For Spark on Databricks, pipeline start time is the cluster start time.
+### Column-level Lineage and Transformation Types
+
+The Spark agent captures fine-grained lineage information, including column-level lineage with transformation types. When available, OpenLineage's [transformation types](https://openlineage.io/docs/spec/facets/dataset-facets/column_lineage_facet/#transformation-type) are captured and mapped to DataHub's FinegrainedLineage `TransformOption`, providing detailed insights into how data transformations occur at the column level.
+
### Spark versions supported
Supports Spark 3.x series.
@@ -253,7 +278,10 @@ Hdfs-based platforms supported explicitly:
- AWS S3 (s3)
- Google Cloud Storage (gcs)
-- local ( local file system) (local)
+- Azure Storage:
+ - Azure Blob Storage (abs) - supports wasb/wasbs protocols
+ - Azure Data Lake Storage Gen2 (abs) - supports abfs/abfss protocols
+- Local file system (file)
All other platforms will have "hdfs" as a platform.
**Name**:
@@ -312,6 +340,25 @@ spark.datahub.platform.s3.path2.path_spec_list: s3://bucket2/*/{table}
- For HDFS sources, the folder (name) is regarded as the dataset (name) to align with typical storage of parquet/csv
formats.
+### Delta Lake MERGE INTO Commands
+
+When working with Delta Lake MERGE INTO commands, the default behavior creates generic job names based on the internal Spark task names.
+To improve lineage tracking, you can enable the enhanced table name extraction feature:
+
+```
+spark.datahub.metadata.dataset.enableEnhancedMergeIntoExtraction=true
+```
+
+When enabled, the agent will:
+
+1. Detect Delta Lake MERGE INTO commands
+2. Extract the target table name from the SQL query, dataset names, or symlinks
+3. Include the table name in the job name, making it easier to trace operations against specific tables
+4. Generate more meaningful lineage in DataHub
+
+For example, a job named `execute_merge_into_command_edge` will be enhanced to `execute_merge_into_command_edge.database_table_name`,
+making it clear which table was being modified.
+
### Debugging
- Following info logs are generated
@@ -364,6 +411,23 @@ Use Java 8 to build the project. The project uses Gradle as the build tool. To b
## Changelog
+### Version 0.2.18
+
+- _Changes_:
+ - OpenLineage 1.33.0 upgrade
+ - Add `spark.datahub.capture_spark_plan` option to capture the Spark plan. By default, it is disabled.
+ - Add proper support for Spark Streaming
+ - Fix issue when Delta table was not within Warehouse location and plugin only captured the path and not the table.
+ - Option for Enhanced Merge Into Extraction
+ - Fix rdd map detection to correctly handle map transformations in the lineage.
+ - **JAR Naming**: Starting from this version, separate jars are built for different Scala versions:
+ - Scala 2.12: `io.acryl:acryl-spark-lineage_2.12:0.2.18`
+ - Scala 2.13: `io.acryl:acryl-spark-lineage_2.13:0.2.18`
+ - **Column-level Lineage Enhancement**: OpenLineage's transformation types are now captured and mapped to DataHub's FinegrainedLineage `TransformOption` as per the [OpenLineage column lineage specification](https://openlineage.io/docs/spec/facets/dataset-facets/column_lineage_facet/#transformation-type)
+ - **Dependency Cleanup**: Removed logback dependency to reduce potential conflicts with user applications
+ - FileStreamMicroBatchStream and foreachBatch for Spark streaming
+ - MERGE INTO operations now capture both dataset-level AND column-level lineage
+
### Version 0.2.17
- _Major changes_:
diff --git a/metadata-integration/java/acryl-spark-lineage/build.gradle b/metadata-integration/java/acryl-spark-lineage/build.gradle
index 9598ab3f1dd729..fa7194501fafd6 100644
--- a/metadata-integration/java/acryl-spark-lineage/build.gradle
+++ b/metadata-integration/java/acryl-spark-lineage/build.gradle
@@ -11,8 +11,8 @@ apply from: '../versioning.gradle'
jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
-//to rename artifacts for publish
-project.archivesBaseName = 'acryl-spark-lineage'
+// Define supported Scala versions - always build for both
+def scalaVersions = ['2.12', '2.13']
//mark implementaion dependencies which needs to excluded along with transitive dependencies from shadowjar
//functionality is exactly same as "implementation"
@@ -38,23 +38,19 @@ dependencies {
provided(externalDependency.sparkHive)
implementation 'org.slf4j:slf4j-log4j12:2.0.7'
implementation externalDependency.httpClient
- implementation externalDependency.logbackClassicJava8
implementation externalDependency.typesafeConfig
implementation externalDependency.commonsLang
-
implementation externalDependency.slf4jApi
compileOnly externalDependency.lombok
annotationProcessor externalDependency.lombok
-
implementation externalDependency.typesafeConfig
implementation externalDependency.json
-
implementation project(':metadata-integration:java:openlineage-converter')
-
implementation project(path: ':metadata-integration:java:datahub-client')
implementation project(path: ':metadata-integration:java:openlineage-converter')
+ compileOnly("io.delta:delta-core_2.12:1.0.0")
- //implementation "io.acryl:datahub-client:0.10.2"
+ // Default to Scala 2.12 for main compilation
implementation "io.openlineage:openlineage-spark_2.12:$openLineageVersion"
compileOnly "org.apache.iceberg:iceberg-spark3-runtime:0.12.1"
compileOnly("org.apache.spark:spark-sql_2.12:3.1.3") {
@@ -73,6 +69,11 @@ dependencies {
testImplementation 'org.apache.logging.log4j:log4j-api:2.17.1'
testImplementation 'org.slf4j:slf4j-log4j12:2.0.7'
+ // JUnit 5 dependencies
+ testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2'
+ testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.8.2'
+ testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.2'
+
testImplementation(externalDependency.postgresql) {
exclude group: "com.fasterxml.jackson.core"
}
@@ -93,86 +94,163 @@ tasks.register('checkShadowJar', Exec) {
commandLine 'sh', '-c', 'scripts/check_jar.sh'
}
-shadowJar {
- zip64 = true
- archiveClassifier = ''
- mergeServiceFiles()
- project.configurations.implementation.canBeResolved = true
- configurations = [project.configurations.implementation]
-
- def exclude_modules = project
- .configurations
- .provided
- .resolvedConfiguration
- .getLenientConfiguration()
- .getAllModuleDependencies()
- .collect {
- it.name
- }
- dependencies {
+// Create separate source and javadoc JARs for each Scala version
+scalaVersions.each { sv ->
+ def scalaVersionUnderscore = sv.replace('.', '_')
+
+ tasks.register("sourcesJar_${scalaVersionUnderscore}", Jar) {
+ archiveClassifier = 'sources'
+ archiveBaseName = "acryl-spark-lineage_${sv}"
+ from sourceSets.main.allJava
+ }
+
+ tasks.register("javadocJar_${scalaVersionUnderscore}", Jar) {
+ dependsOn javadoc
+ archiveClassifier = 'javadoc'
+ archiveBaseName = "acryl-spark-lineage_${sv}"
+ from javadoc.destinationDir
+ }
+}
- exclude(dependency {
- exclude_modules.contains(it.name)
- })
- exclude(dependency("org.slf4j::"))
- exclude("org/apache/commons/logging/**")
+// Create shadow JAR tasks for each Scala version
+scalaVersions.each { sv ->
+ tasks.register("shadowJar_${sv.replace('.', '_')}", com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+ group = 'build'
+ description = "Build shadow jar for Scala ${sv}"
+
+ zip64 = true
+ archiveClassifier = ''
+ archiveBaseName = "acryl-spark-lineage_${sv}"
+ mergeServiceFiles()
+
+ from(sourceSets.main.output)
+
+ // Add manifest with version information
+ manifest {
+ attributes(
+ 'Implementation-Version': project.version,
+ 'Scala-Version': sv
+ )
+ }
+
+ // Create a completely separate configuration for each Scala version
+ def scalaConfig = project.configurations.detachedConfiguration()
+
+ // Manually add all base dependencies except OpenLineage
+ scalaConfig.dependencies.add(project.dependencies.create('org.slf4j:slf4j-log4j12:2.0.7'))
+ scalaConfig.dependencies.add(project.dependencies.create(externalDependency.httpClient))
+ scalaConfig.dependencies.add(project.dependencies.create(externalDependency.typesafeConfig))
+ scalaConfig.dependencies.add(project.dependencies.create(externalDependency.commonsLang))
+ scalaConfig.dependencies.add(project.dependencies.create(externalDependency.slf4jApi))
+ scalaConfig.dependencies.add(project.dependencies.create(externalDependency.json))
+
+ // Add project dependencies
+ scalaConfig.dependencies.add(project.dependencies.create(project(':metadata-integration:java:openlineage-converter')))
+ scalaConfig.dependencies.add(project.dependencies.create(project(':metadata-integration:java:datahub-client')))
+
+ // Add the Scala-specific OpenLineage dependency - THIS IS THE KEY PART
+ scalaConfig.dependencies.add(project.dependencies.create("io.openlineage:openlineage-spark_${sv}:${openLineageVersion}"))
+
+ scalaConfig.canBeResolved = true
+ configurations = [scalaConfig]
+
+ def exclude_modules = project
+ .configurations
+ .provided
+ .resolvedConfiguration
+ .getLenientConfiguration()
+ .getAllModuleDependencies()
+ .collect {
+ it.name
+ }
+
+ dependencies {
+ exclude(dependency {
+ exclude_modules.contains(it.name)
+ })
+ exclude(dependency("org.slf4j::"))
+ exclude(dependency("ch.qos.logback:"))
+ exclude("org/apache/commons/logging/**")
+ }
+
+ exclude('module-info.class', 'META-INF/versions/**', 'LICENSE', 'NOTICE')
+ exclude '**/libzstd-jni.*'
+ exclude '**/com_github_luben_zstd_*'
+
+ // Apply all the relocations
+ relocate 'avro.com', 'io.acryl.shaded.avro.com'
+ relocate 'org.json', 'io.acryl.shaded.org.json'
+ relocate 'com.github', 'io.acryl.shaded.com.github'
+ relocate 'avroutil1', 'io.acryl.shaded.avroutil1'
+ relocate 'com.sun.activation', 'io.acryl.shaded.com.sun.activation'
+ relocate 'com.sun.codemodel', 'io.acryl.shaded.com.sun.codemodel'
+ relocate 'com.sun.mail', 'io.acryl.shaded.com.sun.mail'
+ relocate 'org.apache.hc', 'io.acryl.shaded.http'
+ relocate 'org.apache.commons.codec', 'io.acryl.shaded.org.apache.commons.codec'
+ relocate 'org.apache.commons.compress', 'io.acryl.shaded.org.apache.commons.compress'
+ relocate 'org.apache.commons.lang3', 'io.acryl.shaded.org.apache.commons.lang3'
+ relocate 'mozilla', 'datahub.spark2.shaded.mozilla'
+ relocate 'com.typesafe', 'io.acryl.shaded.com.typesafe'
+ relocate 'io.opentracing', 'io.acryl.shaded.io.opentracing'
+ relocate 'io.netty', 'io.acryl.shaded.io.netty'
+ relocate 'ch.randelshofer', 'io.acryl.shaded.ch.randelshofer'
+ relocate 'ch.qos', 'io.acryl.shaded.ch.qos'
+ relocate 'org.springframework', 'io.acryl.shaded.org.springframework'
+ relocate 'com.fasterxml.jackson', 'io.acryl.shaded.jackson'
+ relocate 'org.yaml', 'io.acryl.shaded.org.yaml'
+ relocate 'net.jcip.annotations', 'io.acryl.shaded.annotations'
+ relocate 'javassist', 'io.acryl.shaded.javassist'
+ relocate 'edu.umd.cs.findbugs', 'io.acryl.shaded.findbugs'
+ relocate 'com.google.common', 'io.acryl.shaded.com.google.common'
+ relocate 'org.reflections', 'io.acryl.shaded.org.reflections'
+ relocate 'st4hidden', 'io.acryl.shaded.st4hidden'
+ relocate 'org.stringtemplate', 'io.acryl.shaded.org.stringtemplate'
+ relocate 'org.abego.treelayout', 'io.acryl.shaded.treelayout'
+ relocate 'javax.annotation', 'io.acryl.shaded.javax.annotation'
+ relocate 'com.github.benmanes.caffeine', 'io.acryl.shaded.com.github.benmanes.caffeine'
+ relocate 'org.checkerframework', 'io.acryl.shaded.org.checkerframework'
+ relocate 'com.google.errorprone', 'io.acryl.shaded.com.google.errorprone'
+ relocate 'com.sun.jna', 'io.acryl.shaded.com.sun.jna'
+
+ // Debug output to verify we're using the right dependency
+ doFirst {
+ println "Building JAR for Scala ${sv}"
+ println "OpenLineage dependency: io.openlineage:openlineage-spark_${sv}:${openLineageVersion}"
+ println "Configuration dependencies:"
+ scalaConfig.allDependencies.each { dep ->
+ println " - ${dep.group}:${dep.name}:${dep.version}"
+ }
+ }
}
+}
- // preventing java multi-release JAR leakage
- // https://github.com/johnrengelman/shadow/issues/729
- exclude('module-info.class', 'META-INF/versions/**', 'LICENSE', 'NOTICE')
-
- // prevent jni conflict with spark
- exclude '**/libzstd-jni.*'
- exclude '**/com_github_luben_zstd_*'
-
- relocate 'avro.com', 'io.acryl.shaded.avro.com'
- relocate 'org.json', 'io.acryl.shaded.org.json'
- relocate 'com.github', 'io.acryl.shaded.com.github'
- relocate 'avroutil1', 'io.acryl.shaded.avroutil1'
- relocate 'com.sun.activation', 'io.acryl.shaded.com.sun.activation'
- relocate 'com.sun.codemodel', 'io.acryl.shaded.com.sun.codemodel'
- relocate 'com.sun.mail', 'io.acryl.shaded.com.sun.mail'
- //
- relocate 'org.apache.hc', 'io.acryl.shaded.http'
- relocate 'org.apache.commons.codec', 'io.acryl.shaded.org.apache.commons.codec'
- relocate 'org.apache.commons.compress', 'io.acryl.shaded.org.apache.commons.compress'
- relocate 'org.apache.commons.lang3', 'io.acryl.shaded.org.apache.commons.lang3'
- relocate 'mozilla', 'datahub.spark2.shaded.mozilla'
- relocate 'com.typesafe', 'io.acryl.shaded.com.typesafe'
- relocate 'io.opentracing', 'io.acryl.shaded.io.opentracing'
- relocate 'io.netty', 'io.acryl.shaded.io.netty'
- relocate 'ch.randelshofer', 'io.acryl.shaded.ch.randelshofer'
- relocate 'ch.qos', 'io.acryl.shaded.ch.qos'
- relocate 'org.springframework', 'io.acryl.shaded.org.springframework'
- relocate 'com.fasterxml.jackson', 'io.acryl.shaded.jackson'
- relocate 'org.yaml', 'io.acryl.shaded.org.yaml' // Required for shading snakeyaml
- relocate 'net.jcip.annotations', 'io.acryl.shaded.annotations'
- relocate 'javassist', 'io.acryl.shaded.javassist'
- relocate 'edu.umd.cs.findbugs', 'io.acryl.shaded.findbugs'
- //relocate 'org.antlr', 'io.acryl.shaded.org.antlr'
- //relocate 'antlr', 'io.acryl.shaded.antlr'
- relocate 'com.google.common', 'io.acryl.shaded.com.google.common'
- relocate 'org.reflections', 'io.acryl.shaded.org.reflections'
- relocate 'st4hidden', 'io.acryl.shaded.st4hidden'
- relocate 'org.stringtemplate', 'io.acryl.shaded.org.stringtemplate'
- relocate 'org.abego.treelayout', 'io.acryl.shaded.treelayout'
- relocate 'javax.annotation', 'io.acryl.shaded.javax.annotation'
- relocate 'com.github.benmanes.caffeine', 'io.acryl.shaded.com.github.benmanes.caffeine'
- relocate 'org.checkerframework', 'io.acryl.shaded.org.checkerframework'
- relocate 'com.google.errorprone', 'io.acryl.shaded.com.google.errorprone'
- relocate 'com.sun.jna', 'io.acryl.shaded.com.sun.jna'
+// Keep the original shadowJar task and make it build all versions
+shadowJar {
+ // Make shadowJar depend on all Scala version builds
+ dependsOn scalaVersions.collect { "shadowJar_${it.replace('.', '_')}" }
+
+ // Disable actual JAR creation for this task since we create versioned ones
+ enabled = false
+ doLast {
+ println "Built shadow JARs for all Scala versions: ${scalaVersions.join(', ')}"
+ }
}
checkShadowJar {
dependsOn shadowJar
}
+// Task to build all Scala versions (always runs)
+tasks.register('buildAllScalaVersions') {
+ group = 'build'
+ description = 'Build shadow jars for all Scala versions'
+ dependsOn scalaVersions.collect { "shadowJar_${it.replace('.', '_')}" }
+}
test {
forkEvery = 1
- useJUnit()
+ useJUnitPlatform()
}
assemble {
@@ -184,46 +262,110 @@ task integrationTest(type: Exec, dependsOn: [shadowJar, ':docker:quickstart']) {
commandLine "spark-smoke-test/smoke.sh"
}
-task sourcesJar(type: Jar) {
- archiveClassifier = 'sources'
- from sourceSets.main.allJava
-}
+// Remove the old shared tasks since we now create version-specific ones
+// task sourcesJar(type: Jar) {
+// archiveClassifier = 'sources'
+// from sourceSets.main.allJava
+// }
+
+// task javadocJar(type: Jar, dependsOn: javadoc) {
+// archiveClassifier = 'javadoc'
+// from javadoc.destinationDir
+// }
+
+// Task to debug dependency resolution for each Scala version
+tasks.register('debugDependencies') {
+ group = 'help'
+ description = 'Show what dependencies are resolved for each Scala version'
+
+ doLast {
+ def supportedScalaVersions = ['2.12', '2.13']
+
+ println "=== Base Implementation Dependencies ==="
+ project.configurations.implementation.allDependencies.each { dep ->
+ println " ${dep.group}:${dep.name}:${dep.version}"
+ }
+
+ supportedScalaVersions.each { sv ->
+ println "\n=== Dependencies for Scala ${sv} ==="
+
+ // Create the same configuration as the shadow task
+ def scalaConfig = project.configurations.detachedConfiguration()
+
+ // Add the same dependencies as in the shadow task
+ scalaConfig.dependencies.add(project.dependencies.create('org.slf4j:slf4j-log4j12:2.0.7'))
+ scalaConfig.dependencies.add(project.dependencies.create(externalDependency.typesafeConfig))
+ scalaConfig.dependencies.add(project.dependencies.create(externalDependency.json))
+ scalaConfig.dependencies.add(project.dependencies.create("io.openlineage:openlineage-spark_${sv}:${openLineageVersion}"))
+
+ println "Configured dependencies for Scala ${sv}:"
+ scalaConfig.allDependencies.each { dep ->
+ println " ADDED: ${dep.group}:${dep.name}:${dep.version}"
+ }
-task javadocJar(type: Jar, dependsOn: javadoc) {
- archiveClassifier = 'javadoc'
- from javadoc.destinationDir
+ try {
+ scalaConfig.canBeResolved = true
+ println "\nResolved dependencies for Scala ${sv}:"
+ scalaConfig.resolvedConfiguration.resolvedArtifacts.each { artifact ->
+ def id = artifact.moduleVersion.id
+ if (id.name.contains('openlineage')) {
+ println " ✅ OPENLINEAGE: ${id.group}:${id.name}:${id.version}"
+ } else {
+ println " ${id.group}:${id.name}:${id.version}"
+ }
+ }
+ } catch (Exception e) {
+ println " ERROR resolving dependencies: ${e.message}"
+ }
+ }
+
+ println "\n=== Summary ==="
+ println "The key difference should be in the OpenLineage Spark dependency:"
+ println " - Scala 2.12 should have: openlineage-spark_2.12"
+ println " - Scala 2.13 should have: openlineage-spark_2.13"
+ println "Note: Scala itself won't be in the JARs (it's provided/compileOnly)"
+ }
}
publishing {
publications {
- shadow(MavenPublication) { publication ->
- project.shadow.component(publication)
- pom {
- name = 'Acryl Spark Lineage'
- group = 'io.acryl'
- artifactId = 'acryl-spark-lineage'
- description = 'Library to push data lineage from spark to datahub'
- url = 'https://docs.datahub.com'
- artifacts = [shadowJar, javadocJar, sourcesJar]
-
- scm {
- connection = 'scm:git:git://github.com/datahub-project/datahub.git'
- developerConnection = 'scm:git:ssh://github.com:datahub-project/datahub.git'
- url = 'https://github.com/datahub-project/datahub.git'
- }
+ // Create publications for each Scala version - always build both
+ scalaVersions.each { sv ->
+ def scalaVersionUnderscore = sv.replace('.', '_')
+
+ "shadow_${scalaVersionUnderscore}"(MavenPublication) { publication ->
+ artifactId = "acryl-spark-lineage_${sv}"
+
+ artifact tasks["shadowJar_${scalaVersionUnderscore}"]
+ artifact tasks["javadocJar_${scalaVersionUnderscore}"]
+ artifact tasks["sourcesJar_${scalaVersionUnderscore}"]
+
+ pom {
+ name = "Acryl Spark Lineage (Scala ${sv})"
+ group = 'io.acryl'
+ artifactId = "acryl-spark-lineage_${sv}"
+ description = "Library to push data lineage from spark to datahub (Scala ${sv})"
+ url = 'https://docs.datahub.com'
+
+ scm {
+ connection = 'scm:git:git://github.com/datahub-project/datahub.git'
+ developerConnection = 'scm:git:ssh://github.com:datahub-project/datahub.git'
+ url = 'https://github.com/datahub-project/datahub.git'
+ }
- licenses {
- license {
- name = 'The Apache License, Version 2.0'
- url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+ licenses {
+ license {
+ name = 'The Apache License, Version 2.0'
+ url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+ }
}
- }
- developers {
- developer {
- id = 'datahub'
- name = 'Datahub'
- email = 'datahub@acryl.io'
+ developers {
+ developer {
+ id = 'datahub'
+ name = 'Datahub'
+ email = 'datahub@acryl.io'
+ }
}
}
}
@@ -232,8 +374,8 @@ publishing {
repositories {
maven {
- def releasesRepoUrl = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/"
- def snapshotsRepoUrl = "https://s01.oss.sonatype.org/content/repositories/snapshots/"
+ def releasesRepoUrl = "https://ossrh-staging-api.central.sonatype.com/service/local/staging/deploy/maven2/"
+ def snapshotsRepoUrl = "https://ossrh-staging-api.central.sonatype.com/content/repositories/snapshots/"
def ossrhUsername = System.getenv('RELEASE_USERNAME')
def ossrhPassword = System.getenv('RELEASE_PASSWORD')
credentials {
@@ -249,12 +391,16 @@ signing {
def signingKey = findProperty("signingKey")
def signingPassword = System.getenv("SIGNING_PASSWORD")
useInMemoryPgpKeys(signingKey, signingPassword)
- sign publishing.publications.shadow
+
+ // Sign all publications
+ publishing.publications.each { publication ->
+ sign publication
+ }
}
nexusStaging {
- serverUrl = "https://s01.oss.sonatype.org/service/local/"
+ serverUrl = "https://ossrh-staging-api.central.sonatype.com/service/local/"
//required only for projects registered in Sonatype after 2021-02-24
- username = System.getenv("NEXUS_USERNAME")
- password = System.getenv("NEXUS_PASSWORD")
-}
+ username = System.getenv("RELEASE_USERNAME")
+ password = System.getenv("RELEASE_PASSWORD")
+}
\ No newline at end of file
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java
index 84f397226ce912..965bc9c18bdf1b 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubEventEmitter.java
@@ -58,7 +58,6 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
-import org.apache.spark.sql.streaming.StreamingQueryProgress;
@Slf4j
public class DatahubEventEmitter extends EventEmitter {
@@ -404,19 +403,6 @@ private void mergeCustomProperties(DatahubJob datahubJob, DatahubJob storedDatah
}
}
- public void emit(StreamingQueryProgress event) throws URISyntaxException {
- List mcps = new ArrayList<>();
- for (MetadataChangeProposalWrapper mcpw :
- generateMcpFromStreamingProgressEvent(event, datahubConf, schemaMap)) {
- try {
- mcps.add(eventFormatter.convert(mcpw));
- } catch (IOException e) {
- log.error("Failed to convert mcpw to mcp", e);
- }
- }
- emitMcps(mcps);
- }
-
protected void emitMcps(List mcps) {
Optional emitter = getEmitter();
if (emitter.isPresent()) {
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java
index b594f6bae954fa..35c39b12e2a562 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java
@@ -36,6 +36,7 @@
import java.time.Instant;
import java.util.HashMap;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import org.apache.spark.SparkConf;
@@ -51,7 +52,6 @@
import org.apache.spark.scheduler.SparkListenerJobEnd;
import org.apache.spark.scheduler.SparkListenerJobStart;
import org.apache.spark.scheduler.SparkListenerTaskEnd;
-import org.apache.spark.sql.streaming.StreamingQueryListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Function0;
@@ -67,6 +67,7 @@ public class DatahubSparkListener extends SparkListener {
private static ContextFactory contextFactory;
private static CircuitBreaker circuitBreaker = new NoOpCircuitBreaker();
private static final String sparkVersion = package$.MODULE$.SPARK_VERSION();
+ private final SparkConf conf;
private final Function0> activeSparkContext =
ScalaConversionUtils.toScalaFn(SparkContext$.MODULE$::getActive);
@@ -74,8 +75,14 @@ public class DatahubSparkListener extends SparkListener {
private static MeterRegistry meterRegistry;
private boolean isDisabled;
- public DatahubSparkListener() throws URISyntaxException {
- listener = new OpenLineageSparkListener();
+ public DatahubSparkListener(SparkConf conf) throws URISyntaxException {
+ this.conf = ((SparkConf) Objects.requireNonNull(conf)).clone();
+
+ listener = new OpenLineageSparkListener(conf);
+ log.info(
+ "Initializing DatahubSparkListener. Version: {} with Spark version: {}",
+ VersionUtil.getVersion(),
+ sparkVersion);
}
private static SparkAppContext getSparkAppContext(
@@ -255,7 +262,10 @@ private synchronized SparkLineageConf loadDatahubConfig(
SparkEnv sparkEnv = SparkEnv$.MODULE$.get();
if (sparkEnv != null) {
log.info("sparkEnv: {}", sparkEnv.conf().toDebugString());
- sparkEnv.conf().set("spark.openlineage.facets.disabled", "[spark_unknown;spark.logicalPlan]");
+ if (datahubConf.hasPath("capture_spark_plan")
+ && datahubConf.getBoolean("capture_spark_plan")) {
+ sparkEnv.conf().set("spark.openlineage.facets.spark.logicalPlan.disabled", "false");
+ }
}
if (properties != null) {
@@ -322,53 +332,8 @@ public void onJobStart(SparkListenerJobStart jobStart) {
public void onOtherEvent(SparkListenerEvent event) {
long startTime = System.currentTimeMillis();
-
log.debug("Other event called {}", event.getClass().getName());
- // Switch to streaming mode if streaming mode is not set, but we get a progress event
- if ((event instanceof StreamingQueryListener.QueryProgressEvent)
- || (event instanceof StreamingQueryListener.QueryStartedEvent)) {
- if (!emitter.isStreaming()) {
- if (!datahubConf.hasPath(STREAMING_JOB)) {
- log.info("Streaming mode not set explicitly, switching to streaming mode");
- emitter.setStreaming(true);
- } else {
- emitter.setStreaming(datahubConf.getBoolean(STREAMING_JOB));
- log.info("Streaming mode set to {}", datahubConf.getBoolean(STREAMING_JOB));
- }
- }
- }
-
- if (datahubConf.hasPath(STREAMING_JOB) && !datahubConf.getBoolean(STREAMING_JOB)) {
- log.info("Not in streaming mode");
- return;
- }
-
listener.onOtherEvent(event);
-
- if (event instanceof StreamingQueryListener.QueryProgressEvent) {
- int streamingHeartbeatIntervalSec = SparkConfigParser.getStreamingHeartbeatSec(datahubConf);
- StreamingQueryListener.QueryProgressEvent queryProgressEvent =
- (StreamingQueryListener.QueryProgressEvent) event;
- ((StreamingQueryListener.QueryProgressEvent) event).progress().id();
- if ((batchLastUpdated.containsKey(queryProgressEvent.progress().id().toString()))
- && (batchLastUpdated
- .get(queryProgressEvent.progress().id().toString())
- .isAfter(Instant.now().minusSeconds(streamingHeartbeatIntervalSec)))) {
- log.debug(
- "Skipping lineage emit as it was emitted in the last {} seconds",
- streamingHeartbeatIntervalSec);
- return;
- }
- try {
- batchLastUpdated.put(queryProgressEvent.progress().id().toString(), Instant.now());
- emitter.emit(queryProgressEvent.progress());
- } catch (URISyntaxException e) {
- throw new RuntimeException(e);
- }
- log.debug("Query progress event: {}", queryProgressEvent.progress());
- long elapsedTime = System.currentTimeMillis() - startTime;
- log.debug("onOtherEvent completed successfully in {} ms", elapsedTime);
- }
}
private static void initializeMetrics(OpenLineageConfig openLineageConfig) {
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/VersionUtil.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/VersionUtil.java
new file mode 100644
index 00000000000000..53adf39d380dc7
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/VersionUtil.java
@@ -0,0 +1,111 @@
+package datahub.spark;
+
+import java.io.InputStream;
+import java.util.jar.Attributes;
+import java.util.jar.Manifest;
+
+public class VersionUtil {
+
+ private static final String UNKNOWN_VERSION = "unknown";
+ private static String cachedVersion = null;
+
+ /** Get the version of the library from the JAR manifest */
+ public static String getVersion() {
+ if (cachedVersion != null) {
+ return cachedVersion;
+ }
+
+ cachedVersion = getVersionFromManifest();
+ return cachedVersion;
+ }
+
+ /** Get version from JAR manifest */
+ private static String getVersionFromManifest() {
+ try {
+ // Get the class's package
+ Package pkg = VersionUtil.class.getPackage();
+ if (pkg != null && pkg.getImplementationVersion() != null) {
+ return pkg.getImplementationVersion();
+ }
+
+ // Fallback: read manifest directly
+ Class> clazz = VersionUtil.class;
+ String className = clazz.getSimpleName() + ".class";
+ String classPath = clazz.getResource(className).toString();
+
+ if (classPath.startsWith("jar")) {
+ String manifestPath =
+ classPath.substring(0, classPath.lastIndexOf("!") + 1) + "/META-INF/MANIFEST.MF";
+ try (InputStream manifestStream = new java.net.URL(manifestPath).openStream()) {
+ Manifest manifest = new Manifest(manifestStream);
+ Attributes attributes = manifest.getMainAttributes();
+ String version = attributes.getValue("Implementation-Version");
+ if (version != null) {
+ return version;
+ }
+ }
+ }
+ } catch (Exception e) {
+ // Ignore and fall through to unknown
+ }
+
+ return UNKNOWN_VERSION;
+ }
+
+ /** Get detailed version information including Scala version */
+ public static String getDetailedVersion() {
+ try {
+ Class> clazz = VersionUtil.class;
+ String className = clazz.getSimpleName() + ".class";
+ String classPath = clazz.getResource(className).toString();
+
+ if (classPath.startsWith("jar")) {
+ String manifestPath =
+ classPath.substring(0, classPath.lastIndexOf("!") + 1) + "/META-INF/MANIFEST.MF";
+ try (InputStream manifestStream = new java.net.URL(manifestPath).openStream()) {
+ Manifest manifest = new Manifest(manifestStream);
+ Attributes attributes = manifest.getMainAttributes();
+
+ String version = attributes.getValue("Implementation-Version");
+ String scalaVersion = attributes.getValue("Scala-Version");
+ String builtBy = attributes.getValue("Built-By");
+ String builtDate = attributes.getValue("Built-Date");
+
+ StringBuilder info = new StringBuilder();
+ info.append("Version: ").append(version != null ? version : UNKNOWN_VERSION);
+ if (scalaVersion != null) {
+ info.append(", Scala: ").append(scalaVersion);
+ }
+ if (builtDate != null) {
+ info.append(", Built: ").append(builtDate);
+ }
+ if (builtBy != null) {
+ info.append(", By: ").append(builtBy);
+ }
+
+ return info.toString();
+ }
+ }
+ } catch (Exception e) {
+ // Ignore and fall through
+ }
+
+ return "Version: " + UNKNOWN_VERSION;
+ }
+
+ /** Print version information to console */
+ public static void printVersion() {
+ System.out.println("Acryl Spark Lineage - " + getDetailedVersion());
+ }
+
+ /** Main method for testing or standalone version checking */
+ public static void main(String[] args) {
+ if (args.length > 0 && "--version".equals(args[0])) {
+ printVersion();
+ return;
+ }
+
+ System.out.println("Simple version: " + getVersion());
+ System.out.println("Detailed version: " + getDetailedVersion());
+ }
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java
index 824cd1a687b264..30f997830c9193 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java
@@ -81,6 +81,8 @@ public class SparkConfigParser {
public static final String DATABRICKS_CLUSTER_KEY = "databricks.cluster";
public static final String PIPELINE_KEY = "metadata.pipeline";
public static final String PIPELINE_PLATFORM_INSTANCE_KEY = PIPELINE_KEY + ".platformInstance";
+ public static final String ENABLE_ENHANCED_MERGE_INTO_EXTRACTION =
+ "metadata.dataset.enableEnhancedMergeIntoExtraction";
public static final String TAGS_KEY = "tags";
@@ -178,6 +180,8 @@ public static DatahubOpenlineageConfig sparkConfigToDatahubOpenlineageConf(
builder.removeLegacyLineage(SparkConfigParser.isLegacyLineageCleanupEnabled(sparkConfig));
builder.disableSymlinkResolution(SparkConfigParser.isDisableSymlinkResolution(sparkConfig));
builder.lowerCaseDatasetUrns(SparkConfigParser.isLowerCaseDatasetUrns(sparkConfig));
+ builder.enhancedMergeIntoExtraction(
+ SparkConfigParser.isEnhancedMergeIntoExtractionEnabled(sparkConfig));
try {
String parentJob = SparkConfigParser.getParentJobKey(sparkConfig);
if (parentJob != null) {
@@ -395,4 +399,9 @@ public static boolean isLowerCaseDatasetUrns(Config datahubConfig) {
return datahubConfig.hasPath(DATASET_LOWERCASE_URNS)
&& datahubConfig.getBoolean(DATASET_LOWERCASE_URNS);
}
+
+ public static boolean isEnhancedMergeIntoExtractionEnabled(Config datahubConfig) {
+ return datahubConfig.hasPath(ENABLE_ENHANCED_MERGE_INTO_EXTRACTION)
+ && datahubConfig.getBoolean(ENABLE_ENHANCED_MERGE_INTO_EXTRACTION);
+ }
}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/converter/SparkStreamingEventToDatahub.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/converter/SparkStreamingEventToDatahub.java
index f0bfc021bbea92..179d6d5e963631 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/converter/SparkStreamingEventToDatahub.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/converter/SparkStreamingEventToDatahub.java
@@ -43,8 +43,66 @@ public static List generateMcpFromStreamingProgre
Map schemaMap) {
List mcps = new ArrayList<>();
+ String pipelineName = conf.getOpenLineageConf().getPipelineName();
+ if (pipelineName == null || pipelineName.trim().isEmpty()) {
+ // For streaming queries, we need a consistent identifier across runs
+ String streamingQueryName;
+ if (event.name() != null) {
+ // Use the query name if set - this should be consistent across runs
+ streamingQueryName = event.name();
+ } else {
+ // If no name is set, try to create a consistent identifier from the query details
+ JsonElement root = new JsonParser().parse(event.json());
+ String sinkDescription =
+ root.getAsJsonObject().get("sink").getAsJsonObject().get("description").getAsString();
+ String sinkType = sinkDescription.split("\\[")[0];
+ String readableSinkType = getDatahubPlatform(sinkType);
+ // Extract content between brackets and sanitize the entire identifier
+ String sinkPath = StringUtils.substringBetween(sinkDescription, "[", "]");
+ if (sinkPath == null) {
+ sinkPath = sinkDescription; // Fallback if no brackets found
+ }
+ // First replace slashes with dots
+ String sanitizedPath = sinkPath.replace('/', '.');
+ // Then replace any remaining special characters with underscores
+ sanitizedPath = sanitizedPath.replaceAll("[^a-zA-Z0-9_.]", "_");
+ // Remove any leading/trailing dots that might have come from leading/trailing slashes
+ sanitizedPath = sanitizedPath.replaceAll("^\\.|\\.$", "");
+ // Ensure we have a valid path that won't cause URN creation issues
+ if (StringUtils.isBlank(sanitizedPath)) {
+ // Create a meaningful identifier using sink type and batch ID
+ sanitizedPath = String.format("unnamed_%s_batch_%d", readableSinkType, event.batchId());
+ log.warn(
+ "Could not extract path from sink description, using generated identifier: {}",
+ sanitizedPath);
+ }
+ streamingQueryName = readableSinkType + "_sink_" + sanitizedPath;
+ log.info(
+ "No query name set, using sink description to create stable identifier: {}",
+ streamingQueryName);
+ }
+
+ String appId =
+ conf.getSparkAppContext() != null ? conf.getSparkAppContext().getAppId() : null;
+
+ // Ensure we have valid values for URN creation
+ if (StringUtils.isBlank(appId)) {
+ log.warn("No app ID available, using streaming query name as pipeline name");
+ pipelineName = streamingQueryName;
+ } else {
+ pipelineName = String.format("%s.%s", appId, streamingQueryName);
+ }
+
+ // Final validation to ensure we have a valid pipeline name for URN creation
+ if (StringUtils.isBlank(pipelineName)) {
+ log.error("Unable to generate valid pipeline name from available information");
+ return new ArrayList<>(); // Return empty list rather than cause NPE
+ }
+ log.debug("No pipeline name configured, using streaming query details: {}", pipelineName);
+ }
+
DataFlowInfo dataFlowInfo = new DataFlowInfo();
- dataFlowInfo.setName(conf.getOpenLineageConf().getPipelineName());
+ dataFlowInfo.setName(pipelineName);
StringMap flowCustomProperties = new StringMap();
Long appStartTime;
@@ -60,17 +118,20 @@ public static List generateMcpFromStreamingProgre
flowCustomProperties.put("plan", event.json());
dataFlowInfo.setCustomProperties(flowCustomProperties);
- DataFlowUrn flowUrn =
- flowUrn(
- conf.getOpenLineageConf().getPlatformInstance(),
- conf.getOpenLineageConf().getPipelineName());
+ DataFlowUrn flowUrn = flowUrn(conf.getOpenLineageConf().getPlatformInstance(), pipelineName);
+
+ log.debug(
+ "Creating streaming flow URN with namespace: {}, name: {}",
+ conf.getOpenLineageConf().getPlatformInstance(),
+ pipelineName);
+
MetadataChangeProposalWrapper dataflowMcp =
MetadataChangeProposalWrapper.create(
b -> b.entityType("dataFlow").entityUrn(flowUrn).upsert().aspect(dataFlowInfo));
mcps.add(dataflowMcp);
DataJobInfo dataJobInfo = new DataJobInfo();
- dataJobInfo.setName(conf.getOpenLineageConf().getPipelineName());
+ dataJobInfo.setName(pipelineName);
dataJobInfo.setType(DataJobInfo.Type.create("SPARK"));
StringMap jobCustomProperties = new StringMap();
@@ -81,7 +142,7 @@ public static List generateMcpFromStreamingProgre
jobCustomProperties.put("numInputRows", Long.toString(event.numInputRows()));
dataJobInfo.setCustomProperties(jobCustomProperties);
- DataJobUrn jobUrn = jobUrn(flowUrn, conf.getOpenLineageConf().getPipelineName());
+ DataJobUrn jobUrn = jobUrn(flowUrn, pipelineName);
MetadataChangeProposalWrapper dataJobMcp =
MetadataChangeProposalWrapper.create(
b -> b.entityType("dataJob").entityUrn(jobUrn).upsert().aspect(dataJobInfo));
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/SparkOpenLineageExtensionVisitorWrapper.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/SparkOpenLineageExtensionVisitorWrapper.java
new file mode 100644
index 00000000000000..166205ec91b82e
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/SparkOpenLineageExtensionVisitorWrapper.java
@@ -0,0 +1,378 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+package io.openlineage.spark.agent.lifecycle;
+
+import io.openlineage.client.OpenLineage;
+import io.openlineage.client.OpenLineage.InputDataset;
+import io.openlineage.client.OpenLineageClientUtils;
+import io.openlineage.client.utils.DatasetIdentifier;
+import io.openlineage.client.utils.DatasetIdentifier.Symlink;
+import io.openlineage.spark.agent.util.ExtensionClassloader;
+import io.openlineage.spark.api.SparkOpenLineageConfig;
+import io.openlineage.spark.extension.OpenLineageExtensionProvider;
+import io.openlineage.spark.shaded.com.fasterxml.jackson.annotation.JsonCreator;
+import io.openlineage.spark.shaded.com.fasterxml.jackson.annotation.JsonProperty;
+import io.openlineage.spark.shaded.com.fasterxml.jackson.core.type.TypeReference;
+import io.openlineage.spark.shaded.com.fasterxml.jackson.databind.ObjectMapper;
+import io.openlineage.spark.shaded.org.apache.commons.lang3.tuple.ImmutablePair;
+import io.openlineage.spark.shaded.org.apache.commons.lang3.tuple.Pair;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.ServiceLoader;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+
+// We only shadow this jar to silence warnings about illegal reflective access
+
+/**
+ * A helper class that uses reflection to call all methods of SparkOpenLineageExtensionVisitor,
+ * which are exposed by the extensions implementing interfaces from `spark-extension-interfaces`
+ * package.
+ */
+@Slf4j
+public final class SparkOpenLineageExtensionVisitorWrapper {
+
+ private static final String providerCanonicalName =
+ "io.openlineage.spark.extension.OpenLineageExtensionProvider";
+
+ private static ByteBuffer providerClassBytes;
+ private static boolean providerFailWarned;
+
+ static {
+ try {
+ providerClassBytes = getProviderClassBytes(Thread.currentThread().getContextClassLoader());
+ } catch (IOException e) {
+ providerFailWarned = true;
+ log.warn("Failed to load provider class bytes.", e);
+ }
+ }
+
+ private static final ClassLoader currentThreadClassloader =
+ Thread.currentThread().getContextClassLoader();
+
+ /**
+ * Stores instances of SparkOpenLineageExtensionVisitor provided by connectors that implement
+ * interfaces from spark-extension-interfaces.
+ */
+ private final List extensionObjects;
+
+ private final boolean hasLoadedObjects;
+ private final ObjectMapper objectMapper =
+ OpenLineageClientUtils.newObjectMapper()
+ .addMixIn(DatasetIdentifier.class, DatasetIdentifierMixin.class)
+ .addMixIn(Symlink.class, SymlinkMixin.class);
+
+ public SparkOpenLineageExtensionVisitorWrapper(SparkOpenLineageConfig config) {
+ try {
+ extensionObjects = init(config.getTestExtensionProvider());
+ this.hasLoadedObjects = !extensionObjects.isEmpty();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public boolean isDefinedAt(Object object) {
+ return hasLoadedObjects
+ && extensionObjects.stream()
+ .map(o -> getMethod(o, "isDefinedAt", Object.class))
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .anyMatch(
+ objectAndMethod -> {
+ try {
+ return (boolean)
+ objectAndMethod.getRight().invoke(objectAndMethod.getLeft(), object);
+ } catch (Exception e) {
+ log.error(
+ "Can't invoke 'isDefinedAt' method on {} class instance",
+ objectAndMethod.getLeft().getClass().getCanonicalName());
+ }
+ return false;
+ });
+ }
+
+ public DatasetIdentifier getLineageDatasetIdentifier(
+ Object lineageNode, String sparkListenerEventName, Object sqlContext, Object parameters) {
+ if (!hasLoadedObjects) {
+ return null;
+ } else {
+ final List> methodsToCall =
+ extensionObjects.stream()
+ .map(
+ o ->
+ getMethod(o, "apply", Object.class, String.class, Object.class, Object.class))
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .collect(Collectors.toList());
+
+ for (ImmutablePair objectAndMethod : methodsToCall) {
+ try {
+ Map result =
+ (Map)
+ objectAndMethod
+ .getRight()
+ .invoke(
+ objectAndMethod.getLeft(),
+ lineageNode,
+ sparkListenerEventName,
+ sqlContext,
+ parameters);
+ if (result != null && !result.isEmpty()) {
+ return objectMapper.convertValue(result, DatasetIdentifier.class);
+ }
+ } catch (Exception e) {
+ log.warn(
+ "Can't invoke apply method on {} class instance",
+ objectAndMethod.getLeft().getClass().getCanonicalName());
+ }
+ }
+ }
+ return null;
+ }
+
+ public DatasetIdentifier getLineageDatasetIdentifier(
+ Object lineageNode, String sparkListenerEventName) {
+ Map datasetIdentifier = callApply(lineageNode, sparkListenerEventName);
+ return objectMapper.convertValue(datasetIdentifier, DatasetIdentifier.class);
+ }
+
+ @SuppressWarnings("unchecked")
+ public Pair, List> getInputs(
+ Object lineageNode, String sparkListenerEventName) {
+ Map inputs = callApply(lineageNode, sparkListenerEventName);
+
+ List> datasets = (List>) inputs.get("datasets");
+ List delagateNodes = (List) inputs.get("delegateNodes");
+ return ImmutablePair.of(
+ objectMapper.convertValue(datasets, new TypeReference>() {}),
+ delagateNodes);
+ }
+
+ @SuppressWarnings("unchecked")
+ public Pair, List> getOutputs(
+ Object lineageNode, String sparkListenerEventName) {
+ Map outputs = callApply(lineageNode, sparkListenerEventName);
+ List> datasets = (List>) outputs.get("datasets");
+ List delagateNodes = (List) outputs.get("delegateNodes");
+ return ImmutablePair.of(
+ objectMapper.convertValue(
+ datasets, new TypeReference>() {}),
+ delagateNodes);
+ }
+
+ private Map callApply(Object lineageNode, String sparkListenerEventName) {
+ if (!hasLoadedObjects) {
+ return Collections.emptyMap();
+ } else {
+ final List> methodsToCall =
+ extensionObjects.stream()
+ .map(o -> getMethod(o, "apply", Object.class, String.class))
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .collect(Collectors.toList());
+
+ for (ImmutablePair objectAndMethod : methodsToCall) {
+ try {
+ Map result =
+ (Map)
+ objectAndMethod
+ .getRight()
+ .invoke(objectAndMethod.getLeft(), lineageNode, sparkListenerEventName);
+ if (result != null && !result.isEmpty()) {
+ return result;
+ }
+ } catch (Exception e) {
+ log.error(
+ "Can't invoke apply method on {} class instance",
+ objectAndMethod.getLeft().getClass().getCanonicalName());
+ }
+ }
+ }
+ return Collections.emptyMap();
+ }
+
+ @SuppressWarnings("PMD") // always point locally
+ private Optional> getMethod(
+ Object classInstance, String methodName, Class>... parameterTypes) {
+ try {
+ Method method = classInstance.getClass().getMethod(methodName, parameterTypes);
+ method.setAccessible(true);
+ return Optional.of(ImmutablePair.of(classInstance, method));
+ } catch (NoSuchMethodException e) {
+ log.warn(
+ "No '{}' method found on {} class instance",
+ methodName,
+ classInstance.getClass().getCanonicalName());
+ }
+ return Optional.empty();
+ }
+
+ private static List init(String testExtensionProvider)
+ throws ClassNotFoundException, IOException, InstantiationException, IllegalAccessException {
+ List objects = new ArrayList<>();
+ // The following sequence of operations must be preserved as is.
+ // We cannot use ResourceFinder or ServiceLoader to determine if there are any
+ // OpenLineageExtensionProvider implementations because doing so involves classloading
+ // machinery.
+ // As a result, there is no reliable way to bypass the entire mechanism, even if
+ // no OpenLineageExtensionProvider implementations are present.
+
+ List availableClassloaders =
+ Thread.getAllStackTraces().keySet().stream()
+ .map(Thread::getContextClassLoader)
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+
+ // Mutates the state of available classloader(s)
+ loadProviderToAvailableClassloaders(availableClassloaders);
+ ExtensionClassloader classLoader = new ExtensionClassloader(availableClassloaders);
+
+ ServiceLoader serviceLoader =
+ ServiceLoader.load(OpenLineageExtensionProvider.class, classLoader);
+
+ for (OpenLineageExtensionProvider service : serviceLoader) {
+ String className = service.getVisitorClassName();
+ if (testExtensionProvider == null) {
+ final Object classInstance = getClassInstance(className);
+ objects.add(classInstance);
+ } else if (testExtensionProvider.equals(service.getClass().getCanonicalName())) {
+ Object classInstance = getClassInstance(className);
+ objects.add(classInstance);
+ break;
+ }
+ }
+ return objects;
+ }
+
+ // FIXED: This method now uses safer classloader handling to avoid illegal reflective access
+ private static void loadProviderToAvailableClassloaders(List classloaders)
+ throws IOException {
+ List filteredClassloaders =
+ classloaders.stream()
+ // Skip the class loader associated with SparkOpenLineageExtensionVisitorWrapper
+ .filter(cl -> !(currentThreadClassloader.equals(cl)))
+ // Skip class loaders that have already loaded OpenLineageExtensionProvider
+ .filter(cl -> !hasLoadedProvider(cl))
+ .collect(Collectors.toList());
+
+ if (!filteredClassloaders.isEmpty()) {
+ log.warn(
+ "Different classloaders detected for openlineage-spark integration and Spark connector. "
+ + "This may cause extension loading issues. "
+ + "For optimal compatibility, ensure both libraries are loaded using the same classloader by: \n"
+ + "1. Placing both libraries in the /usr/lib/spark/jars directory, or \n"
+ + "2. Loading both libraries through the --jars parameter.");
+ }
+
+ // FIXED: Instead of using dangerous reflective defineClass, try to load extensions
+ // using available classloaders without illegal reflection
+ filteredClassloaders.forEach(
+ cl -> {
+ try {
+ // SAFE APPROACH: Try to load the class normally first
+ try {
+ cl.loadClass(providerCanonicalName);
+ log.debug("Provider class already available in classloader: {}", cl);
+ return;
+ } catch (ClassNotFoundException e) {
+ // Class not found, but we won't force-define it using illegal reflection
+ log.debug("Provider class not found in classloader {}, skipping unsafe loading", cl);
+ }
+
+ // ALTERNATIVE SAFE APPROACH: Try using parent classloader delegation
+ ClassLoader parent = cl.getParent();
+ if (parent != null && parent != currentThreadClassloader) {
+ try {
+ parent.loadClass(providerCanonicalName);
+ log.debug("Provider class found via parent delegation in classloader: {}", cl);
+ } catch (ClassNotFoundException e) {
+ log.trace("Provider class not found via parent delegation either");
+ }
+ }
+
+ } catch (Exception e) {
+ log.debug("Safe provider loading failed for classloader {}: {}", cl, e.getMessage());
+ }
+ });
+ }
+
+ private static boolean hasLoadedProvider(ClassLoader classLoader) {
+ try {
+ classLoader.loadClass(providerCanonicalName);
+ return true;
+ } catch (Exception | Error e) {
+ log.trace("{} classloader failed to load OpenLineageExtensionProvider class", classLoader, e);
+ return false;
+ }
+ }
+
+ private static ByteBuffer getProviderClassBytes(ClassLoader classLoader) throws IOException {
+ String classPath =
+ SparkOpenLineageExtensionVisitorWrapper.providerCanonicalName.replace('.', '/') + ".class";
+
+ try (InputStream is = classLoader.getResourceAsStream(classPath)) {
+ if (is == null) {
+ throw new IOException(
+ "Class not found: "
+ + SparkOpenLineageExtensionVisitorWrapper.providerCanonicalName
+ + " using classloader: "
+ + classLoader);
+ }
+
+ byte[] bytes = IOUtils.toByteArray(is);
+ return ByteBuffer.wrap(bytes);
+ }
+ }
+
+ private static Object getClassInstance(String className)
+ throws ClassNotFoundException, InstantiationException, IllegalAccessException {
+ Class> loadedClass = Class.forName(className);
+ Object classInstance = loadedClass.newInstance();
+ return classInstance;
+ }
+
+ @SuppressWarnings("PMD") // always point locally
+ private abstract static class DatasetIdentifierMixin {
+ private final String name;
+ private final String namespace;
+ private final List symlinks;
+
+ @JsonCreator
+ public DatasetIdentifierMixin(
+ @JsonProperty("name") String name,
+ @JsonProperty("namespace") String namespace,
+ @JsonProperty("symlinks") List symlinks) {
+ this.name = name;
+ this.namespace = namespace;
+ this.symlinks = symlinks;
+ }
+ }
+
+ @SuppressWarnings("PMD") // always point locally
+ private abstract static class SymlinkMixin {
+ private final String name;
+ private final String namespace;
+ private final DatasetIdentifier.SymlinkType type;
+
+ @JsonCreator
+ private SymlinkMixin(
+ @JsonProperty("name") String name,
+ @JsonProperty("namespace") String namespace,
+ @JsonProperty("type") DatasetIdentifier.SymlinkType type) {
+ this.name = name;
+ this.namespace = namespace;
+ this.type = type;
+ }
+ }
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/FileStreamMicroBatchStreamStrategy.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/FileStreamMicroBatchStreamStrategy.java
new file mode 100644
index 00000000000000..f85b9144fa8fa5
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/FileStreamMicroBatchStreamStrategy.java
@@ -0,0 +1,128 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+
+package io.openlineage.spark.agent.lifecycle.plan;
+
+import io.openlineage.client.OpenLineage;
+import io.openlineage.client.utils.DatasetIdentifier;
+import io.openlineage.spark.agent.util.PathUtils;
+import io.openlineage.spark.api.DatasetFactory;
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation;
+
+/**
+ * Strategy for handling file-based streaming sources (CSV, Parquet, JSON, etc.) in micro-batch
+ * streaming operations.
+ */
+@Slf4j
+public class FileStreamMicroBatchStreamStrategy extends StreamStrategy {
+
+ private final StreamingDataSourceV2Relation relation;
+
+ public FileStreamMicroBatchStreamStrategy(
+ DatasetFactory datasetFactory,
+ StreamingDataSourceV2Relation relation) {
+ super(datasetFactory, relation.schema(), relation.stream(), Optional.empty());
+ this.relation = relation;
+ }
+
+ @Override
+ public List getInputDatasets() {
+ log.info("Extracting input datasets from file-based streaming source");
+
+ try {
+ // Get the streaming source path
+ Optional pathOpt = getStreamingSourcePath();
+ if (!pathOpt.isPresent()) {
+ log.warn("Could not extract path from file-based streaming source");
+ return Collections.emptyList();
+ }
+
+ String path = pathOpt.get();
+ log.info("Found streaming source path: {}", path);
+
+ // Create dataset from path
+ URI uri = URI.create(path);
+ DatasetIdentifier identifier = PathUtils.fromURI(uri);
+ String namespace = identifier.getNamespace();
+ String name = identifier.getName();
+
+ log.info("Creating input dataset with namespace: {}, name: {}", namespace, name);
+
+ // Use the inherited datasetFactory to create the dataset
+ OpenLineage.InputDataset dataset = datasetFactory.getDataset(name, namespace, schema);
+
+ return Collections.singletonList(dataset);
+
+ } catch (Exception e) {
+ log.error("Error extracting input datasets from file streaming source", e);
+ return Collections.emptyList();
+ }
+ }
+
+ /**
+ * Extract the path from file-based streaming source. This handles various file formats (CSV,
+ * Parquet, JSON, etc.)
+ */
+ private Optional getStreamingSourcePath() {
+ try {
+ // Try to get path from the streaming source options
+ Object streamObj = relation.stream();
+ if (streamObj == null) {
+ return Optional.empty();
+ }
+
+ // Use reflection to get the path from various file-based stream types
+ String streamClassName = streamObj.getClass().getCanonicalName();
+ log.info("Processing stream class: {}", streamClassName);
+
+ // Handle different file-based streaming sources
+ if (streamClassName != null) {
+ if (streamClassName.contains("FileStreamSource")
+ || streamClassName.contains("TextFileStreamSource")
+ || streamClassName.contains("org.apache.spark.sql.execution.streaming.sources")) {
+
+ // Try to extract path using reflection
+ try {
+ java.lang.reflect.Method getPathMethod = streamObj.getClass().getMethod("path");
+ if (getPathMethod != null) {
+ Object pathObj = getPathMethod.invoke(streamObj);
+ if (pathObj != null) {
+ return Optional.of(pathObj.toString());
+ }
+ }
+ } catch (Exception e) {
+ log.debug("Could not extract path via reflection: {}", e.getMessage());
+ }
+
+ // Try alternative methods for getting path
+ try {
+ java.lang.reflect.Field pathField = streamObj.getClass().getDeclaredField("path");
+ pathField.setAccessible(true);
+ Object pathObj = pathField.get(streamObj);
+ if (pathObj != null) {
+ return Optional.of(pathObj.toString());
+ }
+ } catch (Exception e) {
+ log.debug("Could not extract path via field access: {}", e.getMessage());
+ }
+ }
+ }
+
+ // Fallback: return a generic path if we can't extract the real one
+ log.debug("Could not extract specific path, using generic file path");
+ return Optional.of("file:///streaming/input");
+
+ } catch (Exception e) {
+ log.error("Error extracting path from streaming source", e);
+ }
+
+ return Optional.empty();
+ }
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/SaveIntoDataSourceCommandVisitor.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/SaveIntoDataSourceCommandVisitor.java
new file mode 100644
index 00000000000000..1ac5863b2368bf
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/SaveIntoDataSourceCommandVisitor.java
@@ -0,0 +1,353 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+
+package io.openlineage.spark.agent.lifecycle.plan;
+
+import static io.openlineage.client.OpenLineage.LifecycleStateChangeDatasetFacet.LifecycleStateChange.CREATE;
+import static io.openlineage.client.OpenLineage.LifecycleStateChangeDatasetFacet.LifecycleStateChange.OVERWRITE;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
+import io.openlineage.client.OpenLineage;
+import io.openlineage.client.OpenLineage.LifecycleStateChangeDatasetFacet.LifecycleStateChange;
+import io.openlineage.client.OpenLineage.OutputDataset;
+import io.openlineage.client.utils.DatasetIdentifier;
+import io.openlineage.client.utils.jdbc.JdbcDatasetUtils;
+import io.openlineage.spark.agent.util.DatasetFacetsUtils;
+import io.openlineage.spark.agent.util.PathUtils;
+import io.openlineage.spark.agent.util.PlanUtils;
+import io.openlineage.spark.agent.util.ScalaConversionUtils;
+import io.openlineage.spark.api.AbstractQueryPlanDatasetBuilder;
+import io.openlineage.spark.api.JobNameSuffixProvider;
+import io.openlineage.spark.api.OpenLineageContext;
+import java.net.URI;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.spark.scheduler.SparkListenerEvent;
+import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SaveMode;
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
+import org.apache.spark.sql.execution.QueryExecution;
+import org.apache.spark.sql.execution.datasources.LogicalRelation;
+import org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand;
+import org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider;
+import org.apache.spark.sql.sources.BaseRelation;
+import org.apache.spark.sql.sources.RelationProvider;
+import org.apache.spark.sql.sources.SchemaRelationProvider;
+import org.apache.spark.sql.types.StructType;
+import scala.Option;
+
+/**
+ * {@link LogicalPlan} visitor that matches an {@link SaveIntoDataSourceCommand} and extracts the
+ * output {@link OpenLineage.Dataset} being written. Since the output datasource is a {@link
+ * BaseRelation}, we wrap it with an artificial {@link LogicalRelation} so we can delegate to other
+ * plan visitors.
+ */
+@Slf4j
+public class SaveIntoDataSourceCommandVisitor
+ extends AbstractQueryPlanDatasetBuilder<
+ SparkListenerEvent, SaveIntoDataSourceCommand, OutputDataset>
+ implements JobNameSuffixProvider {
+
+ public SaveIntoDataSourceCommandVisitor(OpenLineageContext context) {
+ super(context, false);
+ }
+
+ @Override
+ public boolean isDefinedAtLogicalPlan(LogicalPlan x) {
+ if (context.getSparkSession().isPresent() && x instanceof SaveIntoDataSourceCommand) {
+ SaveIntoDataSourceCommand command = (SaveIntoDataSourceCommand) x;
+ if (PlanUtils.safeIsInstanceOf(
+ command.dataSource(), "com.google.cloud.spark.bigquery.BigQueryRelationProvider")) {
+ return false;
+ }
+ return command.dataSource() instanceof SchemaRelationProvider
+ || context.getSparkExtensionVisitorWrapper().isDefinedAt(command.dataSource())
+ || command.dataSource() instanceof RelationProvider;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean isDefinedAt(SparkListenerEvent x) {
+ return super.isDefinedAt(x)
+ && context
+ .getQueryExecution()
+ .filter(qe -> isDefinedAtLogicalPlan(qe.optimizedPlan()))
+ .isPresent();
+ }
+
+ @Override
+ public List apply(SaveIntoDataSourceCommand cmd) {
+ // intentionally unimplemented
+ throw new UnsupportedOperationException("apply(LogicalPlay) is not implemented");
+ }
+
+ @Override
+ @SuppressWarnings("PMD.AvoidDuplicateLiterals")
+ public List apply(SparkListenerEvent event, SaveIntoDataSourceCommand command) {
+ BaseRelation relation;
+
+ if (context.getSparkExtensionVisitorWrapper().isDefinedAt(command.dataSource())) {
+ DatasetIdentifier datasetIdentifier =
+ context
+ .getSparkExtensionVisitorWrapper()
+ .getLineageDatasetIdentifier(
+ command.dataSource(),
+ event.getClass().getName(),
+ context.getSparkSession().get().sqlContext(),
+ command.options());
+
+ return datasetIdentifier != null
+ ? Collections.singletonList(
+ outputDataset().getDataset(datasetIdentifier, getSchema(command)))
+ : Collections.emptyList();
+ }
+
+ // Kafka has some special handling because the Source and Sink relations require different
+ // options. A KafkaRelation for writes uses the "topic" option, while the same relation for
+ // reads requires the "subscribe" option. The KafkaSourceProvider never returns a KafkaRelation
+ // for write operations (it executes the real writer, then returns a dummy relation), so we have
+ // to use it to construct a reader, meaning we need to change the "topic" option to "subscribe".
+ // Since it requires special handling anyway, we just go ahead and extract the Dataset(s)
+ // directly.
+ // TODO- it may be the case that we need to extend this pattern to support arbitrary relations,
+ // as other impls of CreatableRelationProvider may not be able to be handled in the generic way.
+ if (KafkaRelationVisitor.isKafkaSource(command.dataSource())) {
+ return KafkaRelationVisitor.createKafkaDatasets(
+ outputDataset(),
+ command.dataSource(),
+ command.options(),
+ command.mode(),
+ command.schema());
+ }
+
+ // Similar to Kafka, Azure Kusto also has some special handling. So we use the method
+ // below for extracting the dataset from Kusto write operations.
+ if (KustoRelationVisitor.isKustoSource(command.dataSource())) {
+ return KustoRelationVisitor.createKustoDatasets(
+ outputDataset(), command.options(), command.schema());
+ }
+
+ StructType schema = getSchema(command);
+ LifecycleStateChange lifecycleStateChange =
+ (SaveMode.Overwrite == command.mode()) ? OVERWRITE : CREATE;
+
+ if (command.dataSource().getClass().getName().contains("DeltaDataSource")) {
+ // Handle path-based Delta tables
+ if (command.options().contains("path")) {
+ URI uri = URI.create(command.options().get("path").get());
+ return Collections.singletonList(
+ outputDataset().getDataset(PathUtils.fromURI(uri), schema, lifecycleStateChange));
+ }
+
+ // Handle catalog-based Delta tables (saveAsTable scenarios)
+ if (command.options().contains("table")) {
+ String tableName = command.options().get("table").get();
+ // For catalog tables, use the default namespace or catalog
+ String namespace = "spark_catalog"; // Default Spark catalog namespace
+ DatasetIdentifier identifier = new DatasetIdentifier(tableName, namespace);
+ return Collections.singletonList(
+ outputDataset().getDataset(identifier, schema, lifecycleStateChange));
+ }
+
+ // Handle saveAsTable without explicit table option - check for table info in query execution
+ if (context.getQueryExecution().isPresent()) {
+ QueryExecution qe = context.getQueryExecution().get();
+ // Try to extract table name from query execution context
+ String extractedTableName = extractTableNameFromContext(qe);
+ if (extractedTableName != null) {
+ String namespace = "spark_catalog";
+ DatasetIdentifier identifier = new DatasetIdentifier(extractedTableName, namespace);
+ return Collections.singletonList(
+ outputDataset().getDataset(identifier, schema, lifecycleStateChange));
+ }
+ }
+
+ log.debug(
+ "Delta table detected but could not determine path or table name from options: {}",
+ command.options());
+ }
+
+ if (command
+ .dataSource()
+ .getClass()
+ .getCanonicalName()
+ .equals(JdbcRelationProvider.class.getCanonicalName())) {
+ String tableName = command.options().get("dbtable").get();
+ String url = command.options().get("url").get();
+ DatasetIdentifier identifier =
+ JdbcDatasetUtils.getDatasetIdentifier(url, tableName, new Properties());
+ return Collections.singletonList(
+ outputDataset().getDataset(identifier, schema, lifecycleStateChange));
+ }
+
+ SQLContext sqlContext = context.getSparkSession().get().sqlContext();
+ try {
+ if (command.dataSource() instanceof RelationProvider) {
+ RelationProvider p = (RelationProvider) command.dataSource();
+ relation = p.createRelation(sqlContext, command.options());
+ } else {
+ SchemaRelationProvider p = (SchemaRelationProvider) command.dataSource();
+ relation = p.createRelation(sqlContext, command.options(), schema);
+ }
+ } catch (Exception ex) {
+ // Bad detection of errors in scala
+ if (ex instanceof SQLException) {
+ // This can happen on SparkListenerSQLExecutionStart for example for sqlite, when database
+ // does not exist yet - it will be created as command execution
+ // Still, we can just ignore it on start, because it will work on end
+ // see SparkReadWriteIntegTest.testReadFromFileWriteToJdbc
+ log.warn("Can't create relation: ", ex);
+ return Collections.emptyList();
+ }
+ throw ex;
+ }
+ LogicalRelation logicalRelation =
+ new LogicalRelation(
+ relation,
+ ScalaConversionUtils.asScalaSeqEmpty(),
+ Option.empty(),
+ command.isStreaming());
+ return delegate(
+ context.getOutputDatasetQueryPlanVisitors(), context.getOutputDatasetBuilders(), event)
+ .applyOrElse(
+ logicalRelation,
+ ScalaConversionUtils.toScalaFn((lp) -> Collections.emptyList()))
+ .stream()
+ // constructed datasets don't include the output stats, so add that facet here
+ .map(
+ ds -> {
+ Builder facetsMap =
+ ImmutableMap.builder();
+ if (ds.getFacets().getAdditionalProperties() != null) {
+ facetsMap.putAll(ds.getFacets().getAdditionalProperties());
+ }
+ ds.getFacets().getAdditionalProperties().putAll(facetsMap.build());
+
+ // rebuild whole dataset with a LifecycleStateChange facet added
+ OpenLineage.DatasetFacets facets =
+ DatasetFacetsUtils.copyToBuilder(context, ds.getFacets())
+ .lifecycleStateChange(
+ context
+ .getOpenLineage()
+ .newLifecycleStateChangeDatasetFacet(
+ OpenLineage.LifecycleStateChangeDatasetFacet.LifecycleStateChange
+ .OVERWRITE,
+ null))
+ .build();
+
+ OpenLineage.OutputDataset newDs =
+ context
+ .getOpenLineage()
+ .newOutputDataset(
+ ds.getNamespace(), ds.getName(), facets, ds.getOutputFacets());
+ return newDs;
+ })
+ .collect(Collectors.toList());
+ }
+
+ private StructType getSchema(SaveIntoDataSourceCommand command) {
+ StructType schema = command.schema();
+ if ((schema == null || schema.fields() == null || schema.fields().length == 0)
+ && command.query() != null
+ && command.query().output() != null) {
+ // get schema from logical plan's output
+ schema = PlanUtils.toStructType(ScalaConversionUtils.fromSeq(command.query().output()));
+ }
+ return schema;
+ }
+
+ /**
+ * Attempts to extract table name from QueryExecution context for saveAsTable operations. This
+ * handles cases where the table name isn't explicitly in the command options.
+ */
+ private String extractTableNameFromContext(QueryExecution qe) {
+ try {
+ // Try to get table name from SQL text if available
+ // Note: sqlText() is not available in all Spark versions, use reflection
+ try {
+ java.lang.reflect.Method sqlTextMethod = qe.getClass().getMethod("sqlText");
+ Object sqlOption = sqlTextMethod.invoke(qe);
+ if (sqlOption != null && ((Option>) sqlOption).isDefined()) {
+ String sql = (String) ((Option>) sqlOption).get();
+ log.debug("Attempting to extract table name from SQL: {}", sql);
+
+ // Look for saveAsTable pattern which typically generates CREATE TABLE statements
+ if (sql.toLowerCase().contains("create table")) {
+ // Extract table name using regex pattern matching
+ String[] tokens = sql.split("\\s+");
+ for (int i = 0; i < tokens.length - 1; i++) {
+ if (tokens[i].toLowerCase().equals("table")) {
+ String candidateTableName = tokens[i + 1];
+ // Clean up table name (remove backticks, quotes, database prefix)
+ candidateTableName = candidateTableName.replaceAll("[`'\"]", "");
+ // Handle database.table format by taking just the table name
+ if (candidateTableName.contains(".")) {
+ String[] parts = candidateTableName.split("\\.");
+ candidateTableName = parts[parts.length - 1]; // Take the last part (table name)
+ }
+ if (!candidateTableName.isEmpty()
+ && !candidateTableName.toLowerCase().equals("if")) {
+ log.debug("Extracted table name from SQL: {}", candidateTableName);
+ return candidateTableName;
+ }
+ }
+ }
+ }
+ }
+ } catch (Exception reflectionEx) {
+ log.debug(
+ "sqlText() method not available in this Spark version: {}", reflectionEx.getMessage());
+ }
+
+ log.debug("Could not extract table name from QueryExecution SQL text");
+ } catch (Exception e) {
+ log.debug("Error extracting table name from QueryExecution: {}", e.getMessage());
+ }
+
+ return null;
+ }
+
+ @Override
+ public Optional jobNameSuffix(OpenLineageContext context) {
+ return context
+ .getQueryExecution()
+ .map(QueryExecution::optimizedPlan)
+ .filter(p -> p instanceof SaveIntoDataSourceCommand)
+ .map(p -> (SaveIntoDataSourceCommand) p)
+ .map(p -> jobNameSuffix(p))
+ .filter(Optional::isPresent)
+ .map(Optional::get);
+ }
+
+ @SuppressWarnings("PMD.AvoidDuplicateLiterals")
+ public Optional jobNameSuffix(SaveIntoDataSourceCommand command) {
+ if (command.dataSource().getClass().getName().contains("DeltaDataSource")
+ && command.options().contains("path")) {
+ return Optional.of(trimPath(command.options().get("path").get()));
+ } else if (KustoRelationVisitor.isKustoSource(command.dataSource())) {
+ return Optional.ofNullable(command.options().get("kustotable"))
+ .filter(Option::isDefined)
+ .map(Option::get);
+ } else if (command.options().get("table").isDefined()) {
+ return Optional.of(command.options().get("table").get());
+ } else if (command.dataSource() instanceof RelationProvider
+ || command.dataSource() instanceof SchemaRelationProvider) {
+ return ScalaConversionUtils.fromMap(command.options()).keySet().stream()
+ .filter(key -> key.toLowerCase(Locale.ROOT).contains("table"))
+ .findAny()
+ .map(key -> command.options().get(key).get());
+ }
+
+ return Optional.empty();
+ }
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/StreamingDataSourceV2RelationVisitor.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/StreamingDataSourceV2RelationVisitor.java
new file mode 100644
index 00000000000000..911c2d2cb543ee
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/StreamingDataSourceV2RelationVisitor.java
@@ -0,0 +1,113 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+
+package io.openlineage.spark.agent.lifecycle.plan;
+
+import io.openlineage.client.OpenLineage.InputDataset;
+import io.openlineage.spark.agent.util.ScalaConversionUtils;
+import io.openlineage.spark.api.OpenLineageContext;
+import io.openlineage.spark.api.QueryPlanVisitor;
+import java.util.List;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
+import org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2Relation;
+
+@Slf4j
+public class StreamingDataSourceV2RelationVisitor
+ extends QueryPlanVisitor {
+ private static final String KAFKA_MICRO_BATCH_STREAM_CLASS_NAME =
+ "org.apache.spark.sql.kafka010.KafkaMicroBatchStream";
+ private static final String KINESIS_MICRO_BATCH_STREAM_CLASS_NAME =
+ "org.apache.spark.sql.connector.kinesis.KinesisV2MicrobatchStream";
+ private static final String MONGO_MICRO_BATCH_STREAM_CLASS_NAME =
+ "com.mongodb.spark.sql.connector.read.MongoMicroBatchStream";
+ private static final String FILE_STREAM_MICRO_BATCH_STREAM_CLASS_NAME =
+ "org.apache.spark.sql.execution.streaming.sources.FileStreamSourceV2";
+
+ public StreamingDataSourceV2RelationVisitor(@NonNull OpenLineageContext context) {
+ super(context);
+ }
+
+ @Override
+ public List apply(LogicalPlan x) {
+ log.info(
+ "Applying {} to a logical plan with type {}",
+ this.getClass().getSimpleName(),
+ x.getClass().getCanonicalName());
+ final StreamingDataSourceV2Relation relation = (StreamingDataSourceV2Relation) x;
+ final StreamStrategy streamStrategy = selectStrategy(relation);
+ return streamStrategy.getInputDatasets();
+ }
+
+ @Override
+ public boolean isDefinedAt(LogicalPlan x) {
+ boolean result = x instanceof StreamingDataSourceV2Relation;
+ if (log.isDebugEnabled()) {
+ log.debug(
+ "The result of checking whether {} is an instance of {} is {}",
+ x.getClass().getCanonicalName(),
+ StreamingDataSourceV2Relation.class.getCanonicalName(),
+ result);
+ }
+ return result;
+ }
+
+ public StreamStrategy selectStrategy(StreamingDataSourceV2Relation relation) {
+ StreamStrategy streamStrategy;
+ Class> streamClass = relation.stream().getClass();
+ String streamClassName = streamClass.getCanonicalName();
+ if (KAFKA_MICRO_BATCH_STREAM_CLASS_NAME.equals(streamClassName)) {
+ streamStrategy =
+ new KafkaMicroBatchStreamStrategy(
+ inputDataset(),
+ relation.schema(),
+ relation.stream(),
+ ScalaConversionUtils.asJavaOptional(relation.startOffset()));
+ } else if (KINESIS_MICRO_BATCH_STREAM_CLASS_NAME.equals(streamClassName)) {
+ streamStrategy = new KinesisMicroBatchStreamStrategy(inputDataset(), relation);
+ } else if (MONGO_MICRO_BATCH_STREAM_CLASS_NAME.equals(streamClassName)) {
+ streamStrategy = new MongoMicroBatchStreamStrategy(inputDataset(), relation);
+ } else if (FILE_STREAM_MICRO_BATCH_STREAM_CLASS_NAME.equals(streamClassName)
+ || isFileBasedStreamingSource(streamClassName)) {
+ streamStrategy = new FileStreamMicroBatchStreamStrategy(inputDataset(), relation);
+ } else {
+ log.warn(
+ "The {} has been selected because no rules have matched for the stream class of {}",
+ NoOpStreamStrategy.class,
+ streamClassName);
+ streamStrategy =
+ new NoOpStreamStrategy(
+ inputDataset(),
+ relation.schema(),
+ relation.stream(),
+ ScalaConversionUtils.asJavaOptional(relation.startOffset()));
+ }
+
+ log.info(
+ "Selected this strategy: {} for stream class: {}",
+ streamStrategy.getClass().getSimpleName(),
+ streamClassName);
+ return streamStrategy;
+ }
+
+ /** Check if the stream class name indicates a file-based streaming source. */
+ private boolean isFileBasedStreamingSource(String streamClassName) {
+ if (streamClassName == null) {
+ return false;
+ }
+
+ return streamClassName.contains("FileStreamSource")
+ || streamClassName.contains("TextFileStreamSource")
+ || streamClassName.contains("FileSource")
+ || streamClassName.contains("ParquetFileSource")
+ || streamClassName.contains("JsonFileSource")
+ || streamClassName.contains("CsvFileSource")
+ || streamClassName.contains("org.apache.spark.sql.execution.streaming.sources")
+ || streamClassName.contains("org.apache.spark.sql.execution.datasources.v2.csv")
+ || streamClassName.contains("org.apache.spark.sql.execution.datasources.v2.json")
+ || streamClassName.contains("org.apache.spark.sql.execution.datasources.v2.parquet");
+ }
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/WriteToDataSourceV2Visitor.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/WriteToDataSourceV2Visitor.java
new file mode 100644
index 00000000000000..f3ba56802cafe6
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/lifecycle/plan/WriteToDataSourceV2Visitor.java
@@ -0,0 +1,286 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+/*
+This class is shadowed from Openlineage to support foreachBatch in streaming
+*/
+package io.openlineage.spark.agent.lifecycle.plan;
+
+import io.openlineage.client.OpenLineage.OutputDataset;
+import io.openlineage.client.utils.DatasetIdentifier;
+import io.openlineage.spark.agent.util.PathUtils;
+import io.openlineage.spark.agent.util.ScalaConversionUtils;
+import io.openlineage.spark.api.OpenLineageContext;
+import io.openlineage.spark.api.QueryPlanVisitor;
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
+import org.apache.spark.sql.connector.write.BatchWrite;
+import org.apache.spark.sql.connector.write.streaming.StreamingWrite;
+import org.apache.spark.sql.execution.datasources.v2.WriteToDataSourceV2;
+import org.apache.spark.sql.execution.streaming.sources.MicroBatchWrite;
+import org.apache.spark.sql.types.StructType;
+import org.jetbrains.annotations.NotNull;
+import scala.Option;
+
+@Slf4j
+public final class WriteToDataSourceV2Visitor
+ extends QueryPlanVisitor {
+ private static final String KAFKA_STREAMING_WRITE_CLASS_NAME =
+ "org.apache.spark.sql.kafka010.KafkaStreamingWrite";
+ private static final String FOREACH_BATCH_SINK_CLASS_NAME =
+ "org.apache.spark.sql.execution.streaming.sources.ForeachBatchSink";
+
+ public WriteToDataSourceV2Visitor(@NonNull OpenLineageContext context) {
+ super(context);
+ }
+
+ @Override
+ public boolean isDefinedAt(LogicalPlan plan) {
+ boolean result = plan instanceof WriteToDataSourceV2;
+ if (log.isDebugEnabled()) {
+ log.debug(
+ "The supplied logical plan {} {} an instance of {}",
+ plan.getClass().getCanonicalName(),
+ result ? "IS" : "IS NOT",
+ WriteToDataSourceV2.class.getCanonicalName());
+ }
+ return result;
+ }
+
+ @Override
+ public List apply(LogicalPlan plan) {
+ List result = Collections.emptyList();
+ WriteToDataSourceV2 write = (WriteToDataSourceV2) plan;
+ BatchWrite batchWrite = write.batchWrite();
+ if (batchWrite instanceof MicroBatchWrite) {
+ MicroBatchWrite microBatchWrite = (MicroBatchWrite) batchWrite;
+ StreamingWrite streamingWrite = microBatchWrite.writeSupport();
+ Class extends StreamingWrite> streamingWriteClass = streamingWrite.getClass();
+ String streamingWriteClassName = streamingWriteClass.getCanonicalName();
+ if (KAFKA_STREAMING_WRITE_CLASS_NAME.equals(streamingWriteClassName)) {
+ result = handleKafkaStreamingWrite(streamingWrite);
+ } else if (streamingWriteClassName != null
+ && (streamingWriteClassName.contains("FileStreamSink")
+ || streamingWriteClassName.contains("ForeachBatchSink")
+ || streamingWriteClassName.contains("ConsoleSink")
+ || streamingWriteClassName.contains("DeltaSink")
+ || streamingWriteClassName.contains("ParquetSink"))) {
+ result = handleFileBasedStreamingWrite(streamingWrite, write);
+ } else {
+ log.warn(
+ "The streaming write class '{}' for '{}' is not supported",
+ streamingWriteClass,
+ MicroBatchWrite.class.getCanonicalName());
+ }
+ } else {
+ log.warn("Unsupported batch write class: {}", batchWrite.getClass().getCanonicalName());
+ }
+
+ return result;
+ }
+
+ private @NotNull List handleFileBasedStreamingWrite(
+ StreamingWrite streamingWrite, WriteToDataSourceV2 write) {
+ log.debug(
+ "Handling file-based streaming write: {}", streamingWrite.getClass().getCanonicalName());
+
+ try {
+ // Try to extract path from streaming write
+ Optional pathOpt = extractPathFromStreamingWrite(streamingWrite);
+ if (!pathOpt.isPresent()) {
+ log.warn("Could not extract path from file-based streaming write");
+ return Collections.emptyList();
+ }
+
+ String path = pathOpt.get();
+ log.debug("Found streaming write path: {}", path);
+
+ // Create dataset from path
+ URI uri = URI.create(path);
+ DatasetIdentifier identifier = PathUtils.fromURI(uri);
+ String namespace = identifier.getNamespace();
+ String name = identifier.getName();
+
+ log.debug("Creating output dataset with namespace: {}, name: {}", namespace, name);
+
+ // Get schema from the write operation
+ StructType schema = null;
+ if (write.query() != null) {
+ schema = write.query().schema();
+ }
+
+ // Use the inherited outputDataset() method to create the dataset
+ OutputDataset dataset = outputDataset().getDataset(name, namespace, schema);
+ return Collections.singletonList(dataset);
+
+ } catch (Exception e) {
+ log.error("Error extracting output dataset from file-based streaming write", e);
+ return Collections.emptyList();
+ }
+ }
+
+ private Optional extractPathFromStreamingWrite(StreamingWrite streamingWrite) {
+ try {
+ // Try to get path using reflection from various sink types
+ String className = streamingWrite.getClass().getCanonicalName();
+
+ // For ForeachBatchSink, try to get the underlying sink's path
+ if (className != null && className.contains("ForeachBatchSink")) {
+ // ForeachBatchSink typically wraps another sink or has batch function
+ // We need to extract path from the context of how it's used
+ return tryExtractPathFromForeachBatch(streamingWrite);
+ }
+
+ // For file-based sinks, try standard path extraction
+ if (className != null
+ && (className.contains("FileStreamSink")
+ || className.contains("ParquetSink")
+ || className.contains("DeltaSink"))) {
+ return tryExtractPathFromFileSink(streamingWrite);
+ }
+
+ // For console sink, return console identifier
+ if (className != null && className.contains("ConsoleSink")) {
+ return Optional.of("console://output");
+ }
+
+ } catch (Exception e) {
+ log.debug("Error extracting path from streaming write: {}", e.getMessage());
+ }
+
+ return Optional.empty();
+ }
+
+ private Optional tryExtractPathFromForeachBatch(StreamingWrite streamingWrite) {
+ try {
+ // ForeachBatchSink doesn't have a direct path since outputs are determined
+ // dynamically by the user's foreachBatch function. The actual lineage
+ // will be captured when the user's function executes batch operations.
+ //
+ // For now, we return empty to indicate that this sink doesn't have
+ // a predetermined output path, and rely on the batch operations
+ // within the foreachBatch function to generate proper lineage events.
+ log.debug("ForeachBatchSink detected - outputs will be tracked from batch operations");
+ return Optional.empty();
+ } catch (Exception e) {
+ log.debug("Could not extract path from ForeachBatchSink: {}", e.getMessage());
+ return Optional.empty();
+ }
+ }
+
+ private Optional tryExtractPathFromFileSink(StreamingWrite streamingWrite) {
+ try {
+ // Try to extract path using reflection
+ Optional pathOpt = tryReadField(streamingWrite, "path");
+ if (pathOpt.isPresent()) {
+ return pathOpt;
+ }
+
+ // Try alternative field names
+ pathOpt = tryReadField(streamingWrite, "outputPath");
+ if (pathOpt.isPresent()) {
+ return pathOpt;
+ }
+
+ pathOpt = tryReadField(streamingWrite, "location");
+ if (pathOpt.isPresent()) {
+ return pathOpt;
+ }
+
+ } catch (Exception e) {
+ log.debug("Error extracting path from file sink: {}", e.getMessage());
+ }
+
+ return Optional.empty();
+ }
+
+ private Optional tryReadField(Object target, String fieldName) {
+ try {
+ T result = (T) FieldUtils.readDeclaredField(target, fieldName, true);
+ return result == null ? Optional.empty() : Optional.of(result);
+ } catch (IllegalAccessException e) {
+ log.debug("Could not read field {}: {}", fieldName, e.getMessage());
+ return Optional.empty();
+ }
+ }
+
+ private @NotNull List handleKafkaStreamingWrite(StreamingWrite streamingWrite) {
+ KafkaStreamWriteProxy proxy = new KafkaStreamWriteProxy(streamingWrite);
+ Optional topicOpt = proxy.getTopic();
+ StructType schemaOpt = proxy.getSchema();
+
+ Optional bootstrapServersOpt = proxy.getBootstrapServers();
+ String namespace = KafkaBootstrapServerResolver.resolve(bootstrapServersOpt);
+
+ if (topicOpt.isPresent() && bootstrapServersOpt.isPresent()) {
+ String topic = topicOpt.get();
+
+ OutputDataset dataset = outputDataset().getDataset(topic, namespace, schemaOpt);
+ return Collections.singletonList(dataset);
+ } else {
+ String topicPresent =
+ topicOpt.isPresent() ? "Topic **IS** present" : "Topic **IS NOT** present";
+ String bootstrapServersPresent =
+ bootstrapServersOpt.isPresent()
+ ? "Bootstrap servers **IS** present"
+ : "Bootstrap servers **IS NOT** present";
+ log.warn(
+ "Both topic and bootstrapServers need to be present in order to construct an output dataset. {}. {}",
+ bootstrapServersPresent,
+ topicPresent);
+ return Collections.emptyList();
+ }
+ }
+
+ @Slf4j
+ private static final class KafkaStreamWriteProxy {
+ private final StreamingWrite streamingWrite;
+
+ public KafkaStreamWriteProxy(StreamingWrite streamingWrite) {
+ String incomingClassName = streamingWrite.getClass().getCanonicalName();
+ if (!KAFKA_STREAMING_WRITE_CLASS_NAME.equals(incomingClassName)) {
+ throw new IllegalArgumentException(
+ "Expected the supplied argument to be of type '"
+ + KAFKA_STREAMING_WRITE_CLASS_NAME
+ + "' but received '"
+ + incomingClassName
+ + "' instead");
+ }
+
+ this.streamingWrite = streamingWrite;
+ }
+
+ public Optional getTopic() {
+ return this.>tryReadField(streamingWrite, "topic")
+ .flatMap(ScalaConversionUtils::asJavaOptional);
+ }
+
+ public StructType getSchema() {
+ Optional schema = this.tryReadField(streamingWrite, "schema");
+ return schema.orElseGet(StructType::new);
+ }
+
+ public Optional getBootstrapServers() {
+ Optional> producerParams = tryReadField(streamingWrite, "producerParams");
+ return producerParams.flatMap(
+ props -> Optional.ofNullable((String) props.get("bootstrap.servers")));
+ }
+
+ private Optional tryReadField(Object target, String fieldName) {
+ try {
+ T result = (T) FieldUtils.readDeclaredField(target, fieldName, true);
+ return result == null ? Optional.empty() : Optional.of(result);
+ } catch (IllegalAccessException e) {
+ return Optional.empty();
+ }
+ }
+ }
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PathUtils.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PathUtils.java
new file mode 100644
index 00000000000000..bdc06093de7974
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PathUtils.java
@@ -0,0 +1,191 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+
+package io.openlineage.spark.agent.util;
+
+import io.openlineage.client.utils.DatasetIdentifier;
+import io.openlineage.client.utils.filesystem.FilesystemDatasetUtils;
+import java.net.URI;
+import java.util.Optional;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.TableIdentifier;
+import org.apache.spark.sql.catalyst.catalog.CatalogTable;
+import org.apache.spark.sql.internal.StaticSQLConf;
+
+@Slf4j
+@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
+public class PathUtils {
+ private static final String DEFAULT_DB = "default";
+ public static final String GLUE_TABLE_PREFIX = "table/";
+
+ public static DatasetIdentifier fromPath(Path path) {
+ return fromURI(path.toUri());
+ }
+
+ public static DatasetIdentifier fromURI(URI location) {
+ return FilesystemDatasetUtils.fromLocation(location);
+ }
+
+ /**
+ * Create DatasetIdentifier from CatalogTable, using storage's locationURI if it exists. In other
+ * way, use defaultTablePath.
+ */
+ public static DatasetIdentifier fromCatalogTable(
+ CatalogTable catalogTable, SparkSession sparkSession) {
+ URI locationUri;
+ if (catalogTable.storage() != null && catalogTable.storage().locationUri().isDefined()) {
+ locationUri = catalogTable.storage().locationUri().get();
+ } else {
+ locationUri = getDefaultLocationUri(sparkSession, catalogTable.identifier());
+ }
+ return fromCatalogTable(catalogTable, sparkSession, locationUri);
+ }
+
+ /** Create DatasetIdentifier from CatalogTable, using provided location. */
+ @SneakyThrows
+ public static DatasetIdentifier fromCatalogTable(
+ CatalogTable catalogTable, SparkSession sparkSession, Path location) {
+ return fromCatalogTable(catalogTable, sparkSession, location.toUri());
+ }
+
+ /** Create DatasetIdentifier from CatalogTable, using provided location. */
+ @SneakyThrows
+ public static DatasetIdentifier fromCatalogTable(
+ CatalogTable catalogTable, SparkSession sparkSession, URI location) {
+ // perform URL normalization
+ DatasetIdentifier locationDataset = fromURI(location);
+ URI locationUri = FilesystemDatasetUtils.toLocation(locationDataset);
+
+ Optional symlinkDataset = Optional.empty();
+
+ SparkContext sparkContext = sparkSession.sparkContext();
+ SparkConf sparkConf = sparkContext.getConf();
+ Configuration hadoopConf = sparkContext.hadoopConfiguration();
+
+ Optional metastoreUri = getMetastoreUri(sparkContext);
+ Optional glueArn = AwsUtils.getGlueArn(sparkConf, hadoopConf);
+
+ if (glueArn.isPresent()) {
+ // Even if glue catalog is used, it will have a hive metastore URI
+ // Use ARN format 'arn:aws:glue:{region}:{account_id}:table/{database}/{table}'
+ String tableName = nameFromTableIdentifier(catalogTable.identifier(), "/");
+ symlinkDataset =
+ Optional.of(new DatasetIdentifier(GLUE_TABLE_PREFIX + tableName, glueArn.get()));
+ } else if (metastoreUri.isPresent()) {
+ // dealing with Hive tables
+ URI hiveUri = prepareHiveUri(metastoreUri.get());
+ String tableName = nameFromTableIdentifier(catalogTable.identifier());
+ symlinkDataset = Optional.of(FilesystemDatasetUtils.fromLocationAndName(hiveUri, tableName));
+ } else {
+ Optional warehouseLocation =
+ getWarehouseLocation(sparkConf, hadoopConf)
+ // perform normalization
+ .map(FilesystemDatasetUtils::fromLocation)
+ .map(FilesystemDatasetUtils::toLocation);
+
+ if (warehouseLocation.isPresent()) {
+ URI relativePath = warehouseLocation.get().relativize(locationUri);
+ if (!relativePath.equals(locationUri)) {
+ // if there is no metastore, and table has custom location,
+ // it cannot be accessed via default warehouse location
+ String tableName = nameFromTableIdentifier(catalogTable.identifier());
+ symlinkDataset =
+ Optional.of(
+ FilesystemDatasetUtils.fromLocationAndName(warehouseLocation.get(), tableName));
+ } else {
+ // Table is outside warehouse, but we create symlink to actual location + tableName
+ String tableName = nameFromTableIdentifier(catalogTable.identifier());
+ symlinkDataset =
+ Optional.of(FilesystemDatasetUtils.fromLocationAndName(locationUri, tableName));
+ }
+ }
+ }
+
+ if (symlinkDataset.isPresent()) {
+ locationDataset.withSymlink(
+ symlinkDataset.get().getName(),
+ symlinkDataset.get().getNamespace(),
+ DatasetIdentifier.SymlinkType.TABLE);
+ }
+
+ return locationDataset;
+ }
+
+ public static URI getDefaultLocationUri(SparkSession sparkSession, TableIdentifier identifier) {
+ return sparkSession.sessionState().catalog().defaultTablePath(identifier);
+ }
+
+ public static Path reconstructDefaultLocation(String warehouse, String[] namespace, String name) {
+ String database = null;
+ if (namespace.length == 1) {
+ // {"database"}
+ database = namespace[0];
+ } else if (namespace.length > 1) {
+ // {"spark_catalog", "database"}
+ database = namespace[1];
+ }
+
+ // /warehouse/mytable
+ if (database == null || database.equals(DEFAULT_DB)) {
+ return new Path(warehouse, name);
+ }
+
+ // /warehouse/mydb.db/mytable
+ return new Path(warehouse, database + ".db", name);
+ }
+
+ @SneakyThrows
+ public static URI prepareHiveUri(URI uri) {
+ return new URI("hive", uri.getAuthority(), null, null, null);
+ }
+
+ @SneakyThrows
+ private static Optional getWarehouseLocation(SparkConf sparkConf, Configuration hadoopConf) {
+ Optional warehouseLocation =
+ SparkConfUtils.findSparkConfigKey(sparkConf, StaticSQLConf.WAREHOUSE_PATH().key());
+ if (!warehouseLocation.isPresent()) {
+ warehouseLocation =
+ SparkConfUtils.findHadoopConfigKey(hadoopConf, "hive.metastore.warehouse.dir");
+ }
+ return warehouseLocation.map(URI::create);
+ }
+
+ private static Optional getMetastoreUri(SparkContext context) {
+ // make sure enableHiveSupport is called
+ Optional setting =
+ SparkConfUtils.findSparkConfigKey(
+ context.getConf(), StaticSQLConf.CATALOG_IMPLEMENTATION().key());
+ if (!setting.isPresent() || !"hive".equals(setting.get())) {
+ return Optional.empty();
+ }
+ return SparkConfUtils.getMetastoreUri(context);
+ }
+
+ /** Get DatasetIdentifier name in format database.table or table */
+ private static String nameFromTableIdentifier(TableIdentifier identifier) {
+ return nameFromTableIdentifier(identifier, ".");
+ }
+
+ private static String nameFromTableIdentifier(TableIdentifier identifier, String delimiter) {
+ // calling `unquotedString` method includes `spark_catalog`, so instead get proper identifier
+ // manually
+ String name;
+ if (identifier.database().isDefined()) {
+ // include database in name
+ name = identifier.database().get() + delimiter + identifier.table();
+ } else {
+ // just table name
+ name = identifier.table();
+ }
+
+ return name;
+ }
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java
index 5f87df2a65d6c2..4ac3fa002ad177 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java
@@ -1,5 +1,5 @@
/*
-/* Copyright 2018-2024 contributors to the OpenLineage project
+/* Copyright 2018-2025 contributors to the OpenLineage project
/* SPDX-License-Identifier: Apache-2.0
*/
@@ -173,20 +173,34 @@ public static OpenLineage.DatasourceDatasetFacet datasourceFacet(
* and namespace.
*
* @param parentRunId
- * @param parentJob
+ * @param parentJobName
* @param parentJobNamespace
* @return
*/
public static OpenLineage.ParentRunFacet parentRunFacet(
- UUID parentRunId, String parentJob, String parentJobNamespace) {
+ UUID parentRunId,
+ String parentJobName,
+ String parentJobNamespace,
+ UUID rootParentRunId,
+ String rootParentJobName,
+ String rootParentJobNamespace) {
return new OpenLineage(Versions.OPEN_LINEAGE_PRODUCER_URI)
.newParentRunFacetBuilder()
.run(new OpenLineage.ParentRunFacetRunBuilder().runId(parentRunId).build())
.job(
new OpenLineage.ParentRunFacetJobBuilder()
- .name(NameNormalizer.normalize(parentJob))
+ .name(NameNormalizer.normalize(parentJobName))
.namespace(parentJobNamespace)
.build())
+ .root(
+ new OpenLineage.ParentRunFacetRootBuilder()
+ .run(new OpenLineage.RootRunBuilder().runId(rootParentRunId).build())
+ .job(
+ new OpenLineage.RootJobBuilder()
+ .namespace(rootParentJobNamespace)
+ .name(rootParentJobName)
+ .build())
+ .build())
.build();
}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java
index 6ef7403362a909..987da0f6bd06e3 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java
@@ -1,5 +1,5 @@
/*
-/* Copyright 2018-2024 contributors to the OpenLineage project
+/* Copyright 2018-2025 contributors to the OpenLineage project
/* SPDX-License-Identifier: Apache-2.0
*/
@@ -23,6 +23,7 @@
import org.apache.spark.sql.execution.datasources.FileScanRDD;
import scala.Tuple2;
import scala.collection.immutable.Seq;
+import scala.collection.mutable.ArrayBuffer;
/** Utility class to extract paths from RDD nodes. */
@Slf4j
@@ -65,6 +66,11 @@ public boolean isDefinedAt(Object rdd) {
public Stream extract(HadoopRDD rdd) {
org.apache.hadoop.fs.Path[] inputPaths = FileInputFormat.getInputPaths(rdd.getJobConf());
Configuration hadoopConf = rdd.getConf();
+ if (log.isDebugEnabled()) {
+ log.debug("Hadoop RDD class {}", rdd.getClass());
+ log.debug("Hadoop RDD input paths {}", Arrays.toString(inputPaths));
+ log.debug("Hadoop RDD job conf {}", rdd.getJobConf());
+ }
return Arrays.stream(inputPaths).map(p -> PlanUtils.getDirectoryPath(p, hadoopConf));
}
}
@@ -78,6 +84,9 @@ public boolean isDefinedAt(Object rdd) {
@Override
public Stream extract(MapPartitionsRDD rdd) {
+ if (log.isDebugEnabled()) {
+ log.debug("Parent RDD: {}", rdd.prev());
+ }
return findRDDPaths(rdd.prev());
}
}
@@ -122,7 +131,9 @@ public Stream extract(ParallelCollectionRDD rdd) {
try {
Object data = FieldUtils.readField(rdd, "data", true);
log.debug("ParallelCollectionRDD data: {}", data);
- if ((data instanceof Seq) && ((Seq) data).head() instanceof Tuple2) {
+ if ((data instanceof Seq)
+ && (!((Seq>) data).isEmpty())
+ && ((Seq) data).head() instanceof Tuple2) {
// exit if the first element is invalid
Seq data_slice = (Seq) ((Seq) data).slice(0, SEQ_LIMIT);
return ScalaConversionUtils.fromSeq(data_slice).stream()
@@ -140,6 +151,11 @@ public Stream extract(ParallelCollectionRDD rdd) {
return path;
})
.filter(Objects::nonNull);
+ } else if ((data instanceof ArrayBuffer) && !((ArrayBuffer>) data).isEmpty()) {
+ ArrayBuffer> dataBuffer = (ArrayBuffer>) data;
+ return ScalaConversionUtils.fromSeq(dataBuffer.toSeq()).stream()
+ .map(o -> parentOf(o.toString()))
+ .filter(Objects::nonNull);
} else {
// Changed to debug to silence error
log.debug("Cannot extract path from ParallelCollectionRDD {}", data);
@@ -156,6 +172,9 @@ private static Path parentOf(String path) {
try {
return new Path(path).getParent();
} catch (Exception e) {
+ if (log.isDebugEnabled()) {
+ log.debug("Cannot get parent of path {}", path, e);
+ }
return null;
}
}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java
index 841298ab0e037f..31dbe3813e83b7 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RemovePathPatternUtils.java
@@ -1,5 +1,5 @@
/*
-/* Copyright 2018-2024 contributors to the OpenLineage project
+/* Copyright 2018-2025 contributors to the OpenLineage project
/* SPDX-License-Identifier: Apache-2.0
*/
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/Vendors.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/Vendors.java
index 967935cb40468e..94312e59ea051e 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/Vendors.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/Vendors.java
@@ -1,5 +1,5 @@
/*
-/* Copyright 2018-2024 contributors to the OpenLineage project
+/* Copyright 2018-2025 contributors to the OpenLineage project
/* SPDX-License-Identifier: Apache-2.0
*/
@@ -21,6 +21,8 @@ public interface Vendors {
Arrays.asList(
// Add vendor classes here
"io.openlineage.spark.agent.vendor.snowflake.SnowflakeVendor",
+ "io.openlineage.spark.agent.vendor.iceberg.IcebergVendor",
+ "io.openlineage.spark.agent.vendor.gcp.GcpVendor",
// This is the only chance we have to add the RedshiftVendor to the list of vendors
"io.openlineage.spark.agent.vendor.redshift.RedshiftVendor");
@@ -56,7 +58,7 @@ static Vendors getVendors(List additionalVendors) {
// and the app
// https://github.com/OpenLineage/OpenLineage/issues/1860
// ServiceLoader serviceLoader = ServiceLoader.load(Vendor.class);
- return new VendorsImpl(vendors);
+ return new VendorsImpl(vendors, new VendorsContext());
}
static Vendors empty() {
@@ -71,10 +73,17 @@ public Collection getVisitorFactories() {
public Collection getEventHandlerFactories() {
return Collections.emptyList();
}
+
+ @Override
+ public VendorsContext getVendorsContext() {
+ return new VendorsContext();
+ }
};
}
Collection getVisitorFactories();
Collection getEventHandlerFactories();
+
+ VendorsContext getVendorsContext();
}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/VendorsContext.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/VendorsContext.java
new file mode 100644
index 00000000000000..3707241d732f13
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/VendorsContext.java
@@ -0,0 +1,28 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+
+package io.openlineage.spark.api;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/** Class to store all the vendors related context information. */
+public class VendorsContext {
+ private final Map contextMap = new HashMap<>();
+
+ public void register(String key, Object value) {
+ contextMap.put(key, value);
+ }
+
+ public Optional fromVendorsContext(String key) {
+ return Optional.ofNullable(contextMap.get(key));
+ }
+
+ public boolean contains(String key) {
+ return contextMap.containsKey(key);
+ }
+ ;
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/VendorsImpl.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/VendorsImpl.java
index 66db4cf4f4e43e..6878c7f058f7d5 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/VendorsImpl.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/api/VendorsImpl.java
@@ -1,29 +1,27 @@
/*
-/* Copyright 2018-2024 contributors to the OpenLineage project
+/* Copyright 2018-2025 contributors to the OpenLineage project
/* SPDX-License-Identifier: Apache-2.0
*/
package io.openlineage.spark.api;
import io.openlineage.spark.agent.lifecycle.VisitorFactory;
-import io.openlineage.spark.agent.vendor.redshift.RedshiftVendor;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
-import lombok.extern.slf4j.Slf4j;
-@Slf4j
public class VendorsImpl implements Vendors {
private final List vendors;
+ private final VendorsContext vendorsContext;
- public VendorsImpl(List vendors) {
+ public VendorsImpl(List vendors, VendorsContext vendorsContext) {
this.vendors = vendors;
+ this.vendorsContext = vendorsContext;
}
@Override
public Collection getVisitorFactories() {
- vendors.add(new RedshiftVendor());
return vendors.stream()
.map(Vendor::getVisitorFactory)
.filter(Optional::isPresent)
@@ -39,4 +37,9 @@ public Collection getEventHandlerFactories() {
.map(Optional::get)
.collect(Collectors.toList());
}
+
+ @Override
+ public VendorsContext getVendorsContext() {
+ return vendorsContext;
+ }
}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark3/agent/lifecycle/plan/MergeIntoCommandEdgeInputDatasetBuilder.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark3/agent/lifecycle/plan/MergeIntoCommandEdgeInputDatasetBuilder.java
new file mode 100644
index 00000000000000..ccb38947547f28
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark3/agent/lifecycle/plan/MergeIntoCommandEdgeInputDatasetBuilder.java
@@ -0,0 +1,95 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+
+package io.openlineage.spark3.agent.lifecycle.plan;
+
+import io.openlineage.client.OpenLineage.InputDataset;
+import io.openlineage.spark.api.AbstractQueryPlanInputDatasetBuilder;
+import io.openlineage.spark.api.OpenLineageContext;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.reflect.MethodUtils;
+import org.apache.spark.scheduler.SparkListenerEvent;
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
+
+@Slf4j
+public class MergeIntoCommandEdgeInputDatasetBuilder
+ extends AbstractQueryPlanInputDatasetBuilder {
+
+ public MergeIntoCommandEdgeInputDatasetBuilder(OpenLineageContext context) {
+ super(context, false);
+ }
+
+ @Override
+ public boolean isDefinedAtLogicalPlan(LogicalPlan x) {
+ return x.getClass()
+ .getCanonicalName()
+ .endsWith("sql.transaction.tahoe.commands.MergeIntoCommandEdge");
+ }
+
+ @Override
+ protected List apply(SparkListenerEvent event, LogicalPlan x) {
+ Object o1 = null;
+ Object o2 = null;
+ List inputs = new ArrayList<>();
+
+ try {
+ o1 = MethodUtils.invokeExactMethod(x, "target", new Object[] {});
+ o2 = MethodUtils.invokeExactMethod(x, "source", new Object[] {});
+ } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
+ log.error("Cannot extract target from Databricks classes", e);
+ }
+
+ if (o1 != null && o1 instanceof LogicalPlan) {
+ inputs.addAll(delegate((LogicalPlan) o1, event));
+ }
+ if (o2 != null && o2 instanceof LogicalPlan) {
+ List sourceDatasets = delegate((LogicalPlan) o2, event);
+ inputs.addAll(sourceDatasets);
+
+ // Handle complex subqueries that aren't captured by standard delegation
+ if (sourceDatasets.isEmpty()) {
+ inputs.addAll(extractInputDatasetsFromComplexSource((LogicalPlan) o2, event));
+ }
+ }
+
+ return inputs;
+ }
+
+ /**
+ * Extracts input datasets from complex source plans like subqueries with DISTINCT, PROJECT, etc.
+ * This handles cases where the standard delegation doesn't work due to missing builders for
+ * intermediate logical plan nodes.
+ */
+ private List extractInputDatasetsFromComplexSource(
+ LogicalPlan source, SparkListenerEvent event) {
+ List datasets = new ArrayList<>();
+
+ // Use a queue to traverse the logical plan tree depth-first
+ java.util.Queue queue = new java.util.LinkedList<>();
+ queue.offer(source);
+
+ while (!queue.isEmpty()) {
+ LogicalPlan current = queue.poll();
+
+ // Try to delegate this node directly
+ List currentDatasets = delegate(current, event);
+ datasets.addAll(currentDatasets);
+
+ // If this node didn't produce any datasets, traverse its children
+ if (currentDatasets.isEmpty()) {
+ // Add all children to the queue for traversal
+ scala.collection.Iterator children = current.children().iterator();
+ while (children.hasNext()) {
+ queue.offer(children.next());
+ }
+ }
+ }
+
+ return datasets;
+ }
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark3/agent/lifecycle/plan/MergeIntoCommandInputDatasetBuilder.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark3/agent/lifecycle/plan/MergeIntoCommandInputDatasetBuilder.java
new file mode 100644
index 00000000000000..c8cacbcf2ba807
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark3/agent/lifecycle/plan/MergeIntoCommandInputDatasetBuilder.java
@@ -0,0 +1,85 @@
+/*
+/* Copyright 2018-2025 contributors to the OpenLineage project
+/* SPDX-License-Identifier: Apache-2.0
+*/
+
+package io.openlineage.spark3.agent.lifecycle.plan;
+
+import io.openlineage.client.OpenLineage;
+import io.openlineage.spark.api.AbstractQueryPlanInputDatasetBuilder;
+import io.openlineage.spark.api.OpenLineageContext;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.spark.scheduler.SparkListenerEvent;
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
+import org.apache.spark.sql.delta.commands.MergeIntoCommand;
+
+@Slf4j
+public class MergeIntoCommandInputDatasetBuilder
+ extends AbstractQueryPlanInputDatasetBuilder {
+
+ public MergeIntoCommandInputDatasetBuilder(OpenLineageContext context) {
+ super(context, true); // FIXED: This enables recursive traversal of subqueries
+ }
+
+ @Override
+ public boolean isDefinedAtLogicalPlan(LogicalPlan x) {
+ return x instanceof MergeIntoCommand;
+ }
+
+ @Override
+ protected List apply(SparkListenerEvent event, MergeIntoCommand x) {
+ List datasets = new ArrayList<>();
+
+ // Process target table
+ List targetDatasets = delegate(x.target(), event);
+ datasets.addAll(targetDatasets);
+
+ // Process source - this will recursively process all datasets in the source plan,
+ // including those in subqueries
+ List sourceDatasets = delegate(x.source(), event);
+ datasets.addAll(sourceDatasets);
+
+ // Handle complex subqueries that aren't captured by standard delegation
+ if (sourceDatasets.isEmpty()) {
+ sourceDatasets.addAll(extractInputDatasetsFromComplexSource(x.source(), event));
+ datasets.addAll(sourceDatasets);
+ }
+
+ return datasets;
+ }
+
+ /**
+ * Extracts input datasets from complex source plans like subqueries with DISTINCT, PROJECT, etc.
+ * This handles cases where the standard delegation doesn't work due to missing builders for
+ * intermediate logical plan nodes.
+ */
+ private List extractInputDatasetsFromComplexSource(
+ LogicalPlan source, SparkListenerEvent event) {
+ List datasets = new ArrayList<>();
+
+ // Use a queue to traverse the logical plan tree depth-first
+ java.util.Queue queue = new java.util.LinkedList<>();
+ queue.offer(source);
+
+ while (!queue.isEmpty()) {
+ LogicalPlan current = queue.poll();
+
+ // Try to delegate this node directly
+ List currentDatasets = delegate(current, event);
+ datasets.addAll(currentDatasets);
+
+ // If this node didn't produce any datasets, traverse its children
+ if (currentDatasets.isEmpty()) {
+ // Add all children to the queue for traversal
+ scala.collection.Iterator children = current.children().iterator();
+ while (children.hasNext()) {
+ queue.offer(children.next());
+ }
+ }
+ }
+
+ return datasets;
+ }
+}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/test/java/datahub/spark/HdfsPathDatasetTest.java b/metadata-integration/java/acryl-spark-lineage/src/test/java/datahub/spark/HdfsPathDatasetTest.java
index 9d7637c6742b87..84b1df821c2f99 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/test/java/datahub/spark/HdfsPathDatasetTest.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/test/java/datahub/spark/HdfsPathDatasetTest.java
@@ -1,5 +1,7 @@
package datahub.spark;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import datahub.spark.conf.SparkAppContext;
@@ -12,8 +14,7 @@
import java.net.URISyntaxException;
import java.util.HashMap;
import lombok.extern.slf4j.Slf4j;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
@Slf4j
public class HdfsPathDatasetTest {
@@ -39,7 +40,7 @@ public void testNoPathSpecList()
HdfsPathDataset.create(
new URI("s3://my-bucket/foo/tests/bar.avro"),
sparkLineageConfBuilder.build().getOpenLineageConf());
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket/foo/tests/bar.avro,PROD)",
dataset.urn().toString());
}
@@ -71,7 +72,7 @@ public void testPathSpecList()
HdfsPathDataset.create(
new URI("s3a://my-bucket/foo/tests/bar.avro"),
sparkLineageConfBuilder.build().getOpenLineageConf());
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket/foo/tests,PROD)",
dataset.urn().toString());
}
@@ -98,7 +99,7 @@ public void testUrisWithPartitionRegexp()
HdfsPathDataset.create(
new URI("s3://bucket-a/kafka_backup/my-table/year=2022/month=10/day=11/my-file.tx"),
datahubConfig);
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:s3,bucket-a/kafka_backup/my-table,PROD)",
dataset.urn().toString());
@@ -106,7 +107,7 @@ public void testUrisWithPartitionRegexp()
HdfsPathDataset.create(
new URI("s3://bucket-b/kafka_backup/my-table/year=2023/month=11/day=23/my-file.tx"),
datahubConfig);
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:s3,bucket-b/kafka_backup/my-table,PROD)",
dataset.urn().toString());
@@ -115,14 +116,14 @@ public void testUrisWithPartitionRegexp()
new URI(
"s3://bucket-c/my-backup/my-other-folder/my-table/year=2023/month=11/day=23/my-file.tx"),
datahubConfig);
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:s3,bucket-c/my-backup/my-other-folder/my-table,PROD)",
dataset.urn().toString());
dataset =
HdfsPathDataset.create(
new URI("s3://bucket-d/kafka_backup/my-table/non-partitioned/"), datahubConfig);
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:s3,bucket-d/kafka_backup/my-table/non-partitioned,PROD)",
dataset.urn().toString());
}
@@ -149,7 +150,7 @@ public void testNoMatchPathSpecListWithFolder()
DatahubOpenlineageConfig datahubConfig = sparkLineageConfBuilder.build().getOpenLineageConf();
SparkDataset dataset = HdfsPathDataset.create(new URI(gcsPath), datahubConfig);
- Assert.assertEquals(expectedUrn, dataset.urn().toString());
+ assertEquals(expectedUrn, dataset.urn().toString());
}
@Test
@@ -174,7 +175,7 @@ public void testNoMatchPathSpecList()
SparkDataset dataset =
HdfsPathDataset.create(
new URI("s3a://my-bucket/foo/tests/bar.avro"), sparkLineageConf.getOpenLineageConf());
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket/foo/tests/bar.avro,PROD)",
dataset.urn().toString());
}
@@ -204,7 +205,7 @@ public void testPathSpecListPlatformInstance()
SparkDataset dataset =
HdfsPathDataset.create(
new URI("s3a://my-bucket/foo/tests/bar.avro"), sparkLineageConf.getOpenLineageConf());
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket/foo/tests,PROD)",
dataset.urn().toString());
}
@@ -232,7 +233,7 @@ public void testPathAliasList()
SparkDataset dataset =
HdfsPathDataset.create(
new URI("s3a://my-bucket/foo/tests/bar.avro"), sparkLineageConf.getOpenLineageConf());
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket/foo,PROD)", dataset.urn().toString());
}
@@ -255,7 +256,7 @@ public void testGcsNoPathSpecList()
SparkDataset dataset =
HdfsPathDataset.create(
new URI("gs://my-bucket/foo/tests/bar.avro"), sparkLineageConf.getOpenLineageConf());
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:gcs,my-bucket/foo/tests/bar.avro,PROD)",
dataset.urn().toString());
}
@@ -284,7 +285,7 @@ public void testGcsPathSpecList()
SparkDataset dataset =
HdfsPathDataset.create(
new URI("gs://my-bucket/foo/tests/bar.avro"), sparkLineageConf.getOpenLineageConf());
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:gcs,my-bucket/foo/tests,PROD)",
dataset.urn().toString());
}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/test/java/datahub/spark/OpenLineageEventToDatahubTest.java b/metadata-integration/java/acryl-spark-lineage/src/test/java/datahub/spark/OpenLineageEventToDatahubTest.java
index b9a142364d4e89..830b416c2f7b93 100644
--- a/metadata-integration/java/acryl-spark-lineage/src/test/java/datahub/spark/OpenLineageEventToDatahubTest.java
+++ b/metadata-integration/java/acryl-spark-lineage/src/test/java/datahub/spark/OpenLineageEventToDatahubTest.java
@@ -1,5 +1,9 @@
package datahub.spark;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
import com.linkedin.common.FabricType;
import com.linkedin.common.urn.DatasetUrn;
import com.linkedin.dataprocess.RunResultType;
@@ -21,14 +25,15 @@
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
+import java.util.Objects;
import java.util.Optional;
import java.util.stream.Stream;
-import junit.framework.TestCase;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.tuple.Triple;
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
-public class OpenLineageEventToDatahubTest extends TestCase {
+public class OpenLineageEventToDatahubTest {
+ @Test
public void testGenerateUrnFromStreamingDescriptionFile() throws URISyntaxException {
Config datahubConfig =
ConfigFactory.parseMap(
@@ -52,6 +57,7 @@ public void testGenerateUrnFromStreamingDescriptionFile() throws URISyntaxExcept
assertEquals("/tmp/streaming_output", urn.get().getDatasetNameEntity());
}
+ @Test
public void testGenerateUrnFromStreamingDescriptionS3File() throws URISyntaxException {
Config datahubConfig =
ConfigFactory.parseMap(
@@ -74,6 +80,7 @@ public void testGenerateUrnFromStreamingDescriptionS3File() throws URISyntaxExce
assertEquals("bucket/streaming_output", urn.get().getDatasetNameEntity());
}
+ @Test
public void testGenerateUrnFromStreamingDescriptionS3AFile() throws URISyntaxException {
Config datahubConfig =
ConfigFactory.parseMap(
@@ -97,6 +104,7 @@ public void testGenerateUrnFromStreamingDescriptionS3AFile() throws URISyntaxExc
assertEquals("bucket/streaming_output", urn.get().getDatasetNameEntity());
}
+ @Test
public void testGenerateUrnFromStreamingDescriptionGCSFile() throws URISyntaxException {
Config datahubConfig =
ConfigFactory.parseMap(
@@ -120,6 +128,7 @@ public void testGenerateUrnFromStreamingDescriptionGCSFile() throws URISyntaxExc
assertEquals("bucket/streaming_output", urn.get().getDatasetNameEntity());
}
+ @Test
public void testGenerateUrnFromStreamingDescriptionDeltaFile() throws URISyntaxException {
Config datahubConfig =
ConfigFactory.parseMap(
@@ -143,6 +152,7 @@ public void testGenerateUrnFromStreamingDescriptionDeltaFile() throws URISyntaxE
assertEquals("/tmp/streaming_output", urn.get().getDatasetNameEntity());
}
+ @Test
public void testGenerateUrnFromStreamingDescriptionGCSWithPathSpec()
throws InstantiationException, IllegalArgumentException, URISyntaxException {
Config datahubConfig =
@@ -171,10 +181,11 @@ public void testGenerateUrnFromStreamingDescriptionGCSWithPathSpec()
sparkLineageConfBuilder.build());
assert (urn.isPresent());
- Assert.assertEquals(
+ assertEquals(
"urn:li:dataset:(urn:li:dataPlatform:gcs,my-bucket/foo/tests,PROD)", urn.get().toString());
}
+ @Test
public void testGcsDataset() throws URISyntaxException {
OpenLineage.OutputDataset outputDataset =
new OpenLineage.OutputDatasetBuilder()
@@ -198,6 +209,7 @@ public void testGcsDataset() throws URISyntaxException {
urn.get().getDatasetNameEntity());
}
+ @Test
public void testGcsDatasetWithoutSlashInName() throws URISyntaxException {
OpenLineage.OutputDataset outputDataset =
new OpenLineage.OutputDatasetBuilder()
@@ -221,6 +233,7 @@ public void testGcsDatasetWithoutSlashInName() throws URISyntaxException {
urn.get().getDatasetNameEntity());
}
+ @Test
public void testRemoveFilePrefixFromPath() throws URISyntaxException {
OpenLineage.OutputDataset outputDataset =
new OpenLineage.OutputDatasetBuilder()
@@ -241,6 +254,7 @@ public void testRemoveFilePrefixFromPath() throws URISyntaxException {
assertEquals("/tmp/streaming_output/file.txt", urn.get().getDatasetNameEntity());
}
+ @Test
public void testRemoveFilePrefixFromPathWithPlatformInstance() throws URISyntaxException {
Config datahubConfig =
ConfigFactory.parseMap(
@@ -270,6 +284,7 @@ public void testRemoveFilePrefixFromPathWithPlatformInstance() throws URISyntaxE
"my-platfrom-instance./tmp/streaming_output/file.txt", urn.get().getDatasetNameEntity());
}
+ @Test
public void testOpenlineageDatasetWithPathSpec() throws URISyntaxException {
Config datahubConfig =
ConfigFactory.parseMap(
@@ -309,6 +324,7 @@ public void testOpenlineageDatasetWithPathSpec() throws URISyntaxException {
urn.get().getDatasetNameEntity());
}
+ @Test
public void testOpenlineageTableDataset() throws URISyntaxException {
// https://openlineage.io/docs/spec/naming#dataset-naming
Stream> testCases =
@@ -384,6 +400,7 @@ public void testOpenlineageTableDataset() throws URISyntaxException {
});
}
+ @Test
public void testProcessOlEvent() throws URISyntaxException, IOException {
OpenLineage.OutputDataset outputDataset =
new OpenLineage.OutputDatasetBuilder()
@@ -408,6 +425,7 @@ public void testProcessOlEvent() throws URISyntaxException, IOException {
assertNotNull(datahubJob);
}
+ @Test
public void testProcessOlFailedEvent() throws URISyntaxException, IOException {
Config datahubConfig = ConfigFactory.empty();
@@ -430,6 +448,7 @@ public void testProcessOlFailedEvent() throws URISyntaxException, IOException {
RunResultType.FAILURE, datahubJob.getDataProcessInstanceRunEvent().getResult().getType());
}
+ @Test
public void testProcessOlEventWithSetFlowname() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -459,6 +478,7 @@ public void testProcessOlEventWithSetFlowname() throws URISyntaxException, IOExc
RunResultType.FAILURE, datahubJob.getDataProcessInstanceRunEvent().getResult().getType());
}
+ @Test
public void testProcessOlEventWithSetDatasetFabricType() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -486,6 +506,7 @@ public void testProcessOlEventWithSetDatasetFabricType() throws URISyntaxExcepti
}
}
+ @Test
public void testProcessGlueOlEvent() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -513,6 +534,7 @@ public void testProcessGlueOlEvent() throws URISyntaxException, IOException {
}
}
+ @Test
public void testProcess_OL17_GlueOlEvent() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -540,6 +562,7 @@ public void testProcess_OL17_GlueOlEvent() throws URISyntaxException, IOExceptio
}
}
+ @Test
public void testProcessGlueOlEventSymlinkDisabled() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -568,6 +591,7 @@ public void testProcessGlueOlEventSymlinkDisabled() throws URISyntaxException, I
}
}
+ @Test
public void testProcessGlueOlEventWithHiveAlias() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -596,6 +620,7 @@ public void testProcessGlueOlEventWithHiveAlias() throws URISyntaxException, IOE
}
}
+ @Test
public void testProcessRedshiftOutput() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -628,6 +653,7 @@ public void testProcessRedshiftOutput() throws URISyntaxException, IOException {
}
}
+ @Test
public void testProcessRedshiftOutputWithPlatformInstance()
throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
@@ -662,6 +688,7 @@ public void testProcessRedshiftOutputWithPlatformInstance()
}
}
+ @Test
public void testProcessRedshiftOutputWithPlatformSpecificPlatformInstance()
throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
@@ -707,6 +734,7 @@ public void testProcessRedshiftOutputWithPlatformSpecificPlatformInstance()
}
}
+ @Test
public void testProcessRedshiftOutputWithPlatformSpecificEnv()
throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
@@ -748,6 +776,7 @@ public void testProcessRedshiftOutputWithPlatformSpecificEnv()
}
}
+ @Test
public void testProcessRedshiftOutputLowercasedUrns() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -782,6 +811,7 @@ public void testProcessRedshiftOutputLowercasedUrns() throws URISyntaxException,
}
}
+ @Test
public void testProcessGCSInputsOutputs() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -815,6 +845,7 @@ public void testProcessGCSInputsOutputs() throws URISyntaxException, IOException
}
}
+ @Test
public void testProcessMappartitionJob() throws URISyntaxException, IOException {
DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
DatahubOpenlineageConfig.builder();
@@ -842,4 +873,144 @@ public void testProcessMappartitionJob() throws URISyntaxException, IOException
}
assertEquals(0, datahubJob.getOutSet().size());
}
+
+ @Test
+ public void testCaptureTransformOption() throws URISyntaxException, IOException {
+ DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
+ DatahubOpenlineageConfig.builder();
+ builder.fabricType(FabricType.DEV);
+ builder.lowerCaseDatasetUrns(true);
+ builder.materializeDataset(true);
+ builder.includeSchemaMetadata(true);
+ builder.isSpark(true);
+
+ String olEvent =
+ IOUtils.toString(
+ this.getClass().getResourceAsStream("/ol_events/sample_spark_with_transformation.json"),
+ StandardCharsets.UTF_8);
+
+ OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent);
+ DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build());
+
+ assertNotNull(datahubJob);
+
+ assertEquals(1, datahubJob.getInSet().size());
+ for (DatahubDataset dataset : datahubJob.getInSet()) {
+ assertEquals(
+ "urn:li:dataset:(urn:li:dataPlatform:file,/spark-test/people.parquet,DEV)",
+ dataset.getUrn().toString());
+ }
+ for (DatahubDataset dataset : datahubJob.getOutSet()) {
+ assertEquals(
+ "urn:li:dataset:(urn:li:dataPlatform:file,/spark-test/result_test,DEV)",
+ dataset.getUrn().toString());
+ assertEquals(
+ "DIRECT:IDENTITY,INDIRECT:FILTER",
+ Objects.requireNonNull(dataset.getLineage().getFineGrainedLineages())
+ .get(0)
+ .getTransformOperation());
+ }
+ }
+
+ @Test
+ public void testFlinkJobEvent() throws URISyntaxException, IOException {
+ DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
+ DatahubOpenlineageConfig.builder();
+ builder.fabricType(FabricType.DEV);
+ builder.lowerCaseDatasetUrns(true);
+ builder.materializeDataset(true);
+ builder.includeSchemaMetadata(true);
+ builder.isSpark(false);
+
+ String olEvent =
+ IOUtils.toString(
+ this.getClass().getResourceAsStream("/ol_events/flink_job_test.json"),
+ StandardCharsets.UTF_8);
+
+ OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent);
+ DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build());
+
+ assertNotNull(datahubJob);
+
+ assertEquals(1, datahubJob.getInSet().size());
+ for (DatahubDataset dataset : datahubJob.getInSet()) {
+ assertEquals(
+ "urn:li:dataset:(urn:li:dataPlatform:kafka,lineage-test-topic-json,DEV)",
+ dataset.getUrn().toString());
+ }
+ for (DatahubDataset dataset : datahubJob.getOutSet()) {
+ assertEquals(
+ "urn:li:dataset:(urn:li:dataPlatform:kafka,lineage-test-topic-json-flinkoutput,DEV)",
+ dataset.getUrn().toString());
+ }
+ }
+
+ @Test
+ public void testDebeziumJobEvent() throws URISyntaxException, IOException {
+ DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
+ DatahubOpenlineageConfig.builder();
+ builder.fabricType(FabricType.DEV);
+ builder.lowerCaseDatasetUrns(true);
+ builder.materializeDataset(true);
+ builder.includeSchemaMetadata(true);
+ builder.isSpark(false);
+ builder.usePatch(true);
+
+ String olEvent =
+ IOUtils.toString(
+ this.getClass().getResourceAsStream("/ol_events/debezium_event.json"),
+ StandardCharsets.UTF_8);
+
+ OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent);
+ DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build());
+
+ assertNotNull(datahubJob);
+
+ assertEquals(0, datahubJob.getInSet().size());
+ for (DatahubDataset dataset : datahubJob.getOutSet()) {
+ assertEquals(
+ "urn:li:dataset:(urn:li:dataPlatform:kafka,debezium.public.product,DEV)",
+ dataset.getUrn().toString());
+ }
+ }
+
+ @Test
+ public void testDatabricksMergeIntoStartEvent() throws URISyntaxException, IOException {
+ DatahubOpenlineageConfig.DatahubOpenlineageConfigBuilder builder =
+ DatahubOpenlineageConfig.builder();
+ builder.fabricType(FabricType.PROD);
+ builder.materializeDataset(true);
+ builder.includeSchemaMetadata(true);
+ builder.isSpark(true);
+
+ String olEvent =
+ IOUtils.toString(
+ this.getClass().getResourceAsStream("/ol_events/databricks_mergeinto_start_event.json"),
+ StandardCharsets.UTF_8);
+
+ OpenLineage.RunEvent runEvent = OpenLineageClientUtils.runEventFromJson(olEvent);
+ DatahubJob datahubJob = OpenLineageToDataHub.convertRunEventToJob(runEvent, builder.build());
+
+ assertNotNull(datahubJob);
+ assertEquals("my-docuemnt-merge-job", datahubJob.getDataFlowInfo().getName());
+ assertEquals("my-docuemnt-merge-job", datahubJob.getJobInfo().getName());
+
+ assertEquals(1, datahubJob.getInSet().size());
+ for (DatahubDataset dataset : datahubJob.getInSet()) {
+ assertEquals(
+ "urn:li:dataset:(urn:li:dataPlatform:hive,documentraw.document,PROD)",
+ dataset.getUrn().toString());
+ }
+
+ // This test verifies the bug: outputs should be present but the converter returns empty outSet
+ // "Expected at least one output dataset but found none. This indicates the bug where outputs
+ // are not being processed correctly for MERGE INTO START events."
+ assertTrue(datahubJob.getOutSet().size() > 0);
+
+ for (DatahubDataset dataset : datahubJob.getOutSet()) {
+ assertEquals(
+ "urn:li:dataset:(urn:li:dataPlatform:hive,documentraw.document,PROD)",
+ dataset.getUrn().toString());
+ }
+ }
}
diff --git a/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/databricks_mergeinto_start_event.json b/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/databricks_mergeinto_start_event.json
new file mode 100644
index 00000000000000..d9951d45d42e6b
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/databricks_mergeinto_start_event.json
@@ -0,0 +1,446 @@
+{
+ "eventTime": "2025-07-10T12:24:34.186Z",
+ "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent",
+ "eventType": "START",
+ "run": {
+ "runId": "0197f44b-7231-7dcb-8d0c-349ce8b0f2c2",
+ "facets": {
+ "parent": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-0/ParentRunFacet.json#/$defs/ParentRunFacet",
+ "run": {
+ "runId": "0197f44a-aaaa-7d9f-a2c1-96ebf920205d"
+ },
+ "job": {
+ "namespace": "default",
+ "name": "databricks_shell"
+ },
+ "root": {
+ "run": {
+ "runId": "0197f44a-aaaa-bbbb-a2c1-96ebf920205d"
+ },
+ "job": {
+ "namespace": "default",
+ "name": "Databricks Shell"
+ }
+ }
+ },
+ "processing_engine": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json#/$defs/ProcessingEngineRunFacet",
+ "version": "3.5.0",
+ "name": "spark",
+ "openlineageAdapterVersion": "0.2.18-rc7"
+ },
+ "spark_properties": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet",
+ "properties": {
+ "spark.master": "spark://127.0.0.1:7077",
+ "spark.app.name": "Databricks Shell"
+ }
+ }
+ }
+ },
+ "job": {
+ "namespace": "default",
+ "name": "my-docuemnt-merge-job",
+ "facets": {
+ "jobType": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/2-0-3/JobTypeJobFacet.json#/$defs/JobTypeJobFacet",
+ "processingType": "BATCH",
+ "integration": "SPARK",
+ "jobType": "SQL_JOB"
+ }
+ }
+ },
+ "inputs": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/document",
+ "facets": {
+ "dataSource": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet",
+ "name": "dbfs",
+ "uri": "dbfs"
+ },
+ "schema": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet",
+ "fields": [
+ {
+ "name": "DocYear",
+ "type": "integer"
+ },
+ {
+ "name": "DocSystem",
+ "type": "string"
+ },
+ {
+ "name": "DocNumber",
+ "type": "long"
+ },
+ {
+ "name": "DocName",
+ "type": "string"
+ },
+ {
+ "name": "DocLongName",
+ "type": "string"
+ },
+ {
+ "name": "DocType",
+ "type": "string"
+ },
+ {
+ "name": "FilePath",
+ "type": "string"
+ },
+ {
+ "name": "FileName",
+ "type": "string"
+ },
+ {
+ "name": "FileSize",
+ "type": "long"
+ },
+ {
+ "name": "FileModificationTime",
+ "type": "timestamp"
+ },
+ {
+ "name": "RecordInsertDateTime",
+ "type": "timestamp"
+ },
+ {
+ "name": "RecordUpdateDateTime",
+ "type": "timestamp"
+ },
+ {
+ "name": "BatchID",
+ "type": "long"
+ }
+ ]
+ },
+ "symlinks": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet",
+ "identifiers": [
+ {
+ "namespace": "dbfs:/demo-warehouse/document",
+ "name": "documentraw.document",
+ "type": "TABLE"
+ }
+ ]
+ }
+ },
+ "inputFacets": {}
+ }
+ ],
+ "outputs": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/document",
+ "facets": {
+ "dataSource": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet",
+ "name": "dbfs",
+ "uri": "dbfs"
+ },
+ "columnLineage": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-2-0/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet",
+ "fields": {
+ "DocYear": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/document",
+ "field": "DocYear",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ },
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/documentstage",
+ "field": "DocYear",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "DocSystem": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/document",
+ "field": "DocSystem",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "DocNumber": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/documentstage",
+ "field": "DocNumber",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ },
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/document",
+ "field": "DocNumber",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "DocName": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/documentstage",
+ "field": "DocName",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "DocLongName": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/documentstage",
+ "field": "DocLongName",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "DocType": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/documentstage",
+ "field": "DocType",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "FilePath": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/documentstage",
+ "field": "FilePath",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "FileName": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/documentstage",
+ "field": "FileName",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "FileSize": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/documentstage",
+ "field": "FileSize",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "FileModificationTime": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/documentstage",
+ "field": "FileModificationTime",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "RecordInsertDateTime": {
+ "inputFields": [
+ {
+ "namespace": "dbfs",
+ "name": "/demo-warehouse/document",
+ "field": "RecordInsertDateTime",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "schema": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet",
+ "fields": [
+ {
+ "name": "DocYear",
+ "type": "integer"
+ },
+ {
+ "name": "DocSystem",
+ "type": "string"
+ },
+ {
+ "name": "DocNumber",
+ "type": "long"
+ },
+ {
+ "name": "DocName",
+ "type": "string"
+ },
+ {
+ "name": "DocLongName",
+ "type": "string"
+ },
+ {
+ "name": "DocType",
+ "type": "string"
+ },
+ {
+ "name": "FilePath",
+ "type": "string"
+ },
+ {
+ "name": "FileName",
+ "type": "string"
+ },
+ {
+ "name": "FileSize",
+ "type": "long"
+ },
+ {
+ "name": "FileModificationTime",
+ "type": "timestamp"
+ },
+ {
+ "name": "RecordInsertDateTime",
+ "type": "timestamp"
+ },
+ {
+ "name": "RecordUpdateDateTime",
+ "type": "timestamp"
+ },
+ {
+ "name": "BatchID",
+ "type": "long"
+ }
+ ]
+ },
+ "symlinks": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json#/$defs/SymlinksDatasetFacet",
+ "identifiers": [
+ {
+ "namespace": "dbfs:/demo-warehouse/document",
+ "name": "documentraw.document",
+ "type": "TABLE"
+ }
+ ]
+ }
+ },
+ "outputFacets": {}
+ }
+ ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/debezium_event.json b/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/debezium_event.json
new file mode 100644
index 00000000000000..721c28ca04cde1
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/debezium_event.json
@@ -0,0 +1,249 @@
+{
+ "eventTime": "2025-06-26T18:33:50.391222794Z",
+ "producer": "https://github.com/debezium/debezium/v3.2.0.CR1",
+ "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent",
+ "eventType": "RUNNING",
+ "run": {
+ "runId": "0197ad84-7a0d-7527-9f8a-a08602319b4d",
+ "facets": {
+ "processing_engine": {
+ "_producer": "https://github.com/debezium/debezium/v3.2.0.CR1",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json#/$defs/ProcessingEngineRunFacet",
+ "version": "3.2.0.CR1",
+ "name": "Debezium",
+ "openlineageAdapterVersion": "1.31.0"
+ },
+ "debezium_config": {
+ "configs": [
+ "connector.class=io.debezium.connector.postgresql.PostgresConnector",
+ "database.user=postgres",
+ "database.dbname=debezium_poc_db",
+ "transforms.openlineage.type=io.debezium.transforms.openlineage.OpenLineage",
+ "slot.name=debezium",
+ "openlineage.integration.job.namespace=dbz-jobs",
+ "openlineage.integration.job.owners=Data Team=Jan Doe",
+ "openlineage.integration.config.file.path=/debezium/openlineage.yml",
+ "transforms=openlineage",
+ "database.server.name=dbserver1",
+ "schema.history.internal.kafka.bootstrap.servers=kafka:9094",
+ "database.port=5432",
+ "plugin.name=pgoutput",
+ "topic.prefix=debezium",
+ "task.class=io.debezium.connector.postgresql.PostgresConnectorTask",
+ "database.hostname=postgres",
+ "database.password=postgres",
+ "name=postgres-connector-2",
+ "table.include.list=public.product",
+ "value.converter=org.apache.kafka.connect.json.JsonConverter",
+ "openlineage.integration.job.description=Debezium Postgres Connector Job",
+ "openlineage.integration.enabled=true",
+ "openlineage.integration.job.tags=env=test"
+ ],
+ "additionalProperties": {},
+ "_producer": "https://github.com/debezium/debezium/v3.2.0.CR1",
+ "_schemaURL": "https://github.com/debezium/debezium/tree/main/debezium-core/src/main/java/io/debezium/openlineage/facets/spec/DebeziumRunFacet.json"
+ }
+ }
+ },
+ "job": {
+ "namespace": "dbz-jobs",
+ "name": "debezium.0",
+ "facets": {
+ "jobType": {
+ "_producer": "https://github.com/debezium/debezium/v3.2.0.CR1",
+ "_schemaURL": "https://openlineage.io/spec/facets/2-0-3/JobTypeJobFacet.json#/$defs/JobTypeJobFacet",
+ "processingType": "STREAMING",
+ "integration": "DEBEZIUM",
+ "jobType": "TASK"
+ },
+ "ownership": {
+ "_producer": "https://github.com/debezium/debezium/v3.2.0.CR1",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/OwnershipJobFacet.json#/$defs/OwnershipJobFacet",
+ "owners": [
+ {
+ "name": "Data Team",
+ "type": "Jan Doe"
+ }
+ ]
+ },
+ "tags": {
+ "_producer": "https://github.com/debezium/debezium/v3.2.0.CR1",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/TagsJobFacet.json#/$defs/TagsJobFacet",
+ "tags": [
+ {
+ "key": "env",
+ "value": "test",
+ "source": "CONFIG"
+ }
+ ]
+ },
+ "documentation": {
+ "_producer": "https://github.com/debezium/debezium/v3.2.0.CR1",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DocumentationJobFacet.json#/$defs/DocumentationJobFacet",
+ "description": "Debezium Postgres Connector Job"
+ }
+ }
+ },
+ "inputs": [],
+ "outputs": [
+ {
+ "namespace": "kafka://kafka:9094",
+ "name": "debezium.public.product",
+ "facets": {
+ "datasetType": {
+ "_producer": "https://github.com/debezium/debezium/v3.2.0.CR1",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetTypeDatasetFacet.json#/$defs/DatasetTypeDatasetFacet",
+ "datasetType": "TABLE",
+ "subType": ""
+ },
+ "schema": {
+ "_producer": "https://github.com/debezium/debezium/v3.2.0.CR1",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet",
+ "fields": [
+ {
+ "name": "before",
+ "type": "STRUCT",
+ "fields": [
+ {
+ "name": "id",
+ "type": "INT32"
+ },
+ {
+ "name": "name",
+ "type": "STRING"
+ },
+ {
+ "name": "price",
+ "type": "INT32"
+ },
+ {
+ "name": "creation_date",
+ "type": "INT64"
+ }
+ ]
+ },
+ {
+ "name": "after",
+ "type": "STRUCT",
+ "fields": [
+ {
+ "name": "id",
+ "type": "INT32"
+ },
+ {
+ "name": "name",
+ "type": "STRING"
+ },
+ {
+ "name": "price",
+ "type": "INT32"
+ },
+ {
+ "name": "creation_date",
+ "type": "INT64"
+ }
+ ]
+ },
+ {
+ "name": "source",
+ "type": "STRUCT",
+ "fields": [
+ {
+ "name": "version",
+ "type": "STRING"
+ },
+ {
+ "name": "connector",
+ "type": "STRING"
+ },
+ {
+ "name": "name",
+ "type": "STRING"
+ },
+ {
+ "name": "ts_ms",
+ "type": "INT64"
+ },
+ {
+ "name": "snapshot",
+ "type": "STRING"
+ },
+ {
+ "name": "db",
+ "type": "STRING"
+ },
+ {
+ "name": "sequence",
+ "type": "STRING"
+ },
+ {
+ "name": "ts_us",
+ "type": "INT64"
+ },
+ {
+ "name": "ts_ns",
+ "type": "INT64"
+ },
+ {
+ "name": "schema",
+ "type": "STRING"
+ },
+ {
+ "name": "table",
+ "type": "STRING"
+ },
+ {
+ "name": "txId",
+ "type": "INT64"
+ },
+ {
+ "name": "lsn",
+ "type": "INT64"
+ },
+ {
+ "name": "xmin",
+ "type": "INT64"
+ }
+ ]
+ },
+ {
+ "name": "transaction",
+ "type": "STRUCT",
+ "fields": [
+ {
+ "name": "id",
+ "type": "STRING"
+ },
+ {
+ "name": "total_order",
+ "type": "INT64"
+ },
+ {
+ "name": "data_collection_order",
+ "type": "INT64"
+ }
+ ]
+ },
+ {
+ "name": "op",
+ "type": "STRING"
+ },
+ {
+ "name": "ts_ms",
+ "type": "INT64"
+ },
+ {
+ "name": "ts_us",
+ "type": "INT64"
+ },
+ {
+ "name": "ts_ns",
+ "type": "INT64"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+
diff --git a/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/flink_job_test.json b/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/flink_job_test.json
new file mode 100644
index 00000000000000..2d6c395cbfde64
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/flink_job_test.json
@@ -0,0 +1,85 @@
+{
+ "eventTime": "2025-06-24T10:40:40.472698302Z",
+ "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.35.0-SNAPSHOT/integration/flink",
+ "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent",
+ "eventType": "START",
+ "run": {
+ "runId": "0197a186-8ffc-73ac-b5c7-5b168e062215",
+ "facets": {
+ "processing_engine": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.35.0-SNAPSHOT/integration/flink",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json#/$defs/ProcessingEngineRunFacet",
+ "version": "2.0.0",
+ "name": "flink",
+ "openlineageAdapterVersion": "1.35.0-SNAPSHOT"
+ },
+ "flink_job": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.35.0-SNAPSHOT/integration/flink",
+ "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet",
+ "jobId": "21789fe91d5cf53f916a59bf8ea6c765"
+ }
+ }
+ },
+ "job": {
+ "namespace": "flink-jobs",
+ "name": "flink-sql-job-json",
+ "facets": {
+ "jobType": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.35.0-SNAPSHOT/integration/flink",
+ "_schemaURL": "https://openlineage.io/spec/facets/2-0-3/JobTypeJobFacet.json#/$defs/JobTypeJobFacet",
+ "processingType": "STREAMING",
+ "integration": "FLINK",
+ "jobType": "JOB"
+ }
+ }
+ },
+ "inputs": [
+ {
+ "namespace": "kafka://kafka-prod",
+ "name": "lineage-test-topic-json",
+ "facets": {
+ "documentation": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.35.0-SNAPSHOT/integration/flink",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-0/DocumentationDatasetFacet.json#/$defs/DocumentationDatasetFacet",
+ "description": ""
+ },
+ "schema": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.35.0-SNAPSHOT/integration/flink",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet",
+ "fields": [
+ {
+ "name": "text",
+ "type": "STRING",
+ "description": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "outputs": [
+ {
+ "namespace": "kafka://kafka-prod",
+ "name": "lineage-test-topic-json-flinkoutput",
+ "facets": {
+ "documentation": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.35.0-SNAPSHOT/integration/flink",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-0/DocumentationDatasetFacet.json#/$defs/DocumentationDatasetFacet",
+ "description": ""
+ },
+ "schema": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.35.0-SNAPSHOT/integration/flink",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet",
+ "fields": [
+ {
+ "name": "text",
+ "type": "STRING",
+ "description": ""
+ }
+ ]
+ }
+ }
+ }
+ ]
+}
+}
\ No newline at end of file
diff --git a/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/sample_spark_with_transformation.json b/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/sample_spark_with_transformation.json
new file mode 100644
index 00000000000000..a5e3335c144d8b
--- /dev/null
+++ b/metadata-integration/java/acryl-spark-lineage/src/test/resources/ol_events/sample_spark_with_transformation.json
@@ -0,0 +1,200 @@
+{
+ "eventTime": "2025-06-26T09:10:58.948Z",
+ "producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent",
+ "eventType": "COMPLETE",
+ "run": {
+ "runId": "0197ab81-29e9-7df5-a5f0-0ecdcfcbab5d",
+ "facets": {
+ "parent": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-0/ParentRunFacet.json#/$defs/ParentRunFacet",
+ "run": {
+ "runId": "0197ab81-2733-7bbc-b360-b605458dfc8c"
+ },
+ "job": {
+ "namespace": "default",
+ "name": "simple_app_parquet_demo"
+ },
+ "root": {
+ "run": {
+ "runId": "0197ab81-2733-7bbc-b360-b605458dfc8c"
+ },
+ "job": {
+ "namespace": "default",
+ "name": "SimpleAppParquetDemo"
+ }
+ }
+ },
+ "processing_engine": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json#/$defs/ProcessingEngineRunFacet",
+ "version": "3.5.5",
+ "name": "spark"
+ },
+ "environment-properties": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet",
+ "environment-properties": {}
+ },
+ "spark_properties": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet",
+ "properties": {
+ "spark.master": "local[*]",
+ "spark.app.name": "SimpleAppParquetDemo"
+ }
+ }
+ }
+ },
+ "job": {
+ "namespace": "default",
+ "name": "simple_app_parquet_demo.adaptive_spark_plan.spark-test_result_test",
+ "facets": {
+ "jobType": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/2-0-3/JobTypeJobFacet.json#/$defs/JobTypeJobFacet",
+ "processingType": "BATCH",
+ "integration": "SPARK",
+ "jobType": "SQL_JOB"
+ }
+ }
+ },
+ "inputs": [
+ {
+ "namespace": "file",
+ "name": "/spark-test/people.parquet",
+ "facets": {
+ "dataSource": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet",
+ "name": "file",
+ "uri": "file"
+ },
+ "schema": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet",
+ "fields": [
+ {
+ "name": "age",
+ "type": "long"
+ },
+ {
+ "name": "name",
+ "type": "string"
+ }
+ ]
+ }
+ },
+ "inputFacets": {
+ "inputStatistics": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/InputStatisticsInputDatasetFacet.json#/$defs/InputStatisticsInputDatasetFacet",
+ "size": 738,
+ "fileCount": 1
+ }
+ }
+ }
+ ],
+ "outputs": [
+ {
+ "namespace": "file",
+ "name": "/spark-test/result_test",
+ "facets": {
+ "dataSource": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json#/$defs/DatasourceDatasetFacet",
+ "name": "file",
+ "uri": "file"
+ },
+ "columnLineage": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-2-0/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet",
+ "fields": {
+ "age": {
+ "inputFields": [
+ {
+ "namespace": "file",
+ "name": "/spark-test/people.parquet",
+ "field": "age",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ },
+ {
+ "type": "INDIRECT",
+ "subtype": "FILTER",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ },
+ "name": {
+ "inputFields": [
+ {
+ "namespace": "file",
+ "name": "/spark-test/people.parquet",
+ "field": "age",
+ "transformations": [
+ {
+ "type": "INDIRECT",
+ "subtype": "FILTER",
+ "description": "",
+ "masking": false
+ }
+ ]
+ },
+ {
+ "namespace": "file",
+ "name": "/spark-test/people.parquet",
+ "field": "name",
+ "transformations": [
+ {
+ "type": "DIRECT",
+ "subtype": "IDENTITY",
+ "description": "",
+ "masking": false
+ }
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "lifecycleStateChange": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/LifecycleStateChangeDatasetFacet.json#/$defs/LifecycleStateChangeDatasetFacet",
+ "lifecycleStateChange": "OVERWRITE"
+ },
+ "schema": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json#/$defs/SchemaDatasetFacet",
+ "fields": [
+ {
+ "name": "age",
+ "type": "long"
+ },
+ {
+ "name": "name",
+ "type": "string"
+ }
+ ]
+ }
+ },
+ "outputFacets": {
+ "outputStatistics": {
+ "_producer": "https://github.com/OpenLineage/OpenLineage/tree/1.33.0/integration/spark",
+ "_schemaURL": "https://openlineage.io/spec/facets/1-0-2/OutputStatisticsOutputDatasetFacet.json#/$defs/OutputStatisticsOutputDatasetFacet",
+ "rowCount": 1,
+ "size": 729,
+ "fileCount": 1
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/config/DatahubOpenlineageConfig.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/config/DatahubOpenlineageConfig.java
index c725673eae47b5..fd54fc4fd86033 100644
--- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/config/DatahubOpenlineageConfig.java
+++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/config/DatahubOpenlineageConfig.java
@@ -36,6 +36,7 @@ public class DatahubOpenlineageConfig {
@Builder.Default private final boolean disableSymlinkResolution = false;
@Builder.Default private final boolean lowerCaseDatasetUrns = false;
@Builder.Default private final boolean removeLegacyLineage = false;
+ @Builder.Default private final boolean enhancedMergeIntoExtraction = false;
public List getPathSpecsForPlatform(String platform) {
if ((pathSpecs == null) || (pathSpecs.isEmpty())) {
diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java
index 9fcfc68bd03f55..6255c57570c720 100644
--- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java
+++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/converter/OpenLineageToDataHub.java
@@ -67,7 +67,9 @@
import java.net.URISyntaxException;
import java.time.ZonedDateTime;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Comparator;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -80,16 +82,37 @@
import lombok.extern.slf4j.Slf4j;
import org.json.JSONArray;
import org.json.JSONException;
+import org.json.JSONObject;
@Slf4j
public class OpenLineageToDataHub {
+ // Constants
public static final String FILE_NAMESPACE = "file";
public static final String SCHEME_SEPARATOR = "://";
public static final String URN_LI_CORPUSER = "urn:li:corpuser:";
public static final String URN_LI_CORPUSER_DATAHUB = URN_LI_CORPUSER + "datahub";
public static final String URN_LI_DATA_PROCESS_INSTANCE = "urn:li:dataProcessInstance:";
+ // Custom property keys
+ public static final String PROCESSING_ENGINE_KEY = "processingEngine";
+ public static final String PROCESSING_ENGINE_VERSION_KEY = "processingEngineVersion";
+ public static final String OPENLINEAGE_ADAPTER_VERSION_KEY = "openlineageAdapterVersion";
+ public static final String JOB_ID_KEY = "jobId";
+ public static final String JOB_DESCRIPTION_KEY = "jobDescription";
+ public static final String JOB_GROUP_KEY = "jobGroup";
+ public static final String JOB_CALL_SITE_KEY = "jobCallSite";
+ public static final String SPARK_VERSION_KEY = "spark-version";
+ public static final String OPENLINEAGE_SPARK_VERSION_KEY = "openlineage-spark-version";
+ public static final String SPARK_LOGICAL_PLAN_KEY = "spark.logicalPlan";
+
+ // SQL patterns
+ public static final String MERGE_INTO_COMMAND_PATTERN = "execute_merge_into_command_edge";
+ public static final String MERGE_INTO_SQL_PATTERN = "MERGE INTO";
+ public static final String TABLE_PREFIX = "table/";
+ public static final String WAREHOUSE_PATH_PATTERN = "/warehouse/";
+ public static final String DB_SUFFIX = ".db/";
+
public static final Map PLATFORM_MAP =
Stream.of(
new String[][] {
@@ -105,6 +128,7 @@ public static Optional convertOpenlineageDatasetToDatasetUrn(
String namespace = dataset.getNamespace();
String datasetName = dataset.getName();
Optional datahubUrn;
+
if (dataset.getFacets() != null
&& dataset.getFacets().getSymlinks() != null
&& !mappingConfig.isDisableSymlinkResolution()) {
@@ -112,7 +136,7 @@ public static Optional convertOpenlineageDatasetToDatasetUrn(
getDatasetUrnFromOlDataset(namespace, datasetName, mappingConfig);
for (OpenLineage.SymlinksDatasetFacetIdentifiers symlink :
dataset.getFacets().getSymlinks().getIdentifiers()) {
- if (symlink.getType().equals("TABLE")) {
+ if ("TABLE".equals(symlink.getType())) {
// Before OpenLineage 0.17.1 the namespace started with "aws:glue:" and after that it was
// changed to :arn:aws:glue:"
if (symlink.getNamespace().startsWith("aws:glue:")
@@ -121,8 +145,8 @@ public static Optional convertOpenlineageDatasetToDatasetUrn(
} else {
namespace = mappingConfig.getHivePlatformAlias();
}
- if (symlink.getName().startsWith("table/")) {
- datasetName = symlink.getName().replaceFirst("table/", "").replace("/", ".");
+ if (symlink.getName().startsWith(TABLE_PREFIX)) {
+ datasetName = symlink.getName().replaceFirst(TABLE_PREFIX, "").replace("/", ".");
} else {
datasetName = symlink.getName();
}
@@ -151,6 +175,7 @@ public static Optional convertOpenlineageDatasetToDatasetUrn(
mappingConfig.getUrnAliases().get(datahubUrn.get().toString())));
return datahubUrn;
} catch (URISyntaxException e) {
+ log.warn("Failed to create URN from alias: {}", e.getMessage());
return Optional.empty();
}
}
@@ -183,7 +208,8 @@ private static Optional getDatasetUrnFromOlDataset(
datasetName = datasetUri.getPath();
try {
HdfsPathDataset hdfsPathDataset = HdfsPathDataset.create(datasetUri, mappingConfig);
- return Optional.of(hdfsPathDataset.urn());
+ DatasetUrn urn = hdfsPathDataset.urn();
+ return Optional.of(urn);
} catch (InstantiationException e) {
log.warn(
"Unable to create urn from namespace: {} and dataset {}.", namespace, datasetName);
@@ -200,15 +226,16 @@ private static Optional getDatasetUrnFromOlDataset(
String platformInstance = getPlatformInstance(mappingConfig, platform);
FabricType env = getEnv(mappingConfig, platform);
- return Optional.of(DatahubUtils.createDatasetUrn(platform, platformInstance, datasetName, env));
+ DatasetUrn urn = DatahubUtils.createDatasetUrn(platform, platformInstance, datasetName, env);
+ return Optional.of(urn);
}
private static FabricType getEnv(DatahubOpenlineageConfig mappingConfig, String platform) {
FabricType fabricType = mappingConfig.getFabricType();
if (mappingConfig.getPathSpecs() != null
&& mappingConfig.getPathSpecs().containsKey(platform)) {
- List path_specs = mappingConfig.getPathSpecs().get(platform);
- for (PathSpec pathSpec : path_specs) {
+ List pathSpecs = mappingConfig.getPathSpecs().get(platform);
+ for (PathSpec pathSpec : pathSpecs) {
if (pathSpec.getEnv().isPresent()) {
try {
fabricType = FabricType.valueOf(pathSpec.getEnv().get());
@@ -229,8 +256,8 @@ private static String getPlatformInstance(
String platformInstance = mappingConfig.getCommonDatasetPlatformInstance();
if (mappingConfig.getPathSpecs() != null
&& mappingConfig.getPathSpecs().containsKey(platform)) {
- List path_specs = mappingConfig.getPathSpecs().get(platform);
- for (PathSpec pathSpec : path_specs) {
+ List pathSpecs = mappingConfig.getPathSpecs().get(platform);
+ for (PathSpec pathSpec : pathSpecs) {
if (pathSpec.getPlatformInstance().isPresent()) {
return pathSpec.getPlatformInstance().get();
}
@@ -267,8 +294,7 @@ public static Domains generateDomains(List domains) {
return datahubDomains;
}
- public static Urn dataPlatformInstanceUrn(String platform, String instance)
- throws URISyntaxException {
+ public static Urn dataPlatformInstanceUrn(String platform, String instance) {
return new Urn(
"dataPlatformInstance",
new TupleKey(Arrays.asList(new DataPlatformUrn(platform).toString(), instance)));
@@ -286,11 +312,18 @@ public static DatahubJob convertRunEventToJob(
log.info("Emitting lineage: {}", OpenLineageClientUtils.toJson(event));
DataFlowInfo dfi = convertRunEventToDataFlowInfo(event, datahubConf.getPipelineName());
+ String processingEngine = null;
+
+ if (event.getRun().getFacets() != null
+ && event.getRun().getFacets().getProcessing_engine() != null) {
+ processingEngine = event.getRun().getFacets().getProcessing_engine().getName();
+ }
+
DataFlowUrn dataFlowUrn =
getFlowUrn(
event.getJob().getNamespace(),
event.getJob().getName(),
- null,
+ processingEngine,
event.getProducer(),
datahubConf);
jobBuilder.flowUrn(dataFlowUrn);
@@ -328,7 +361,9 @@ public static DatahubJob convertRunEventToJob(
static void forEachValue(Map source, StringMap customProperties) {
for (final Map.Entry entry : source.entrySet()) {
if (entry.getValue() instanceof Map) {
- forEachValue((Map) entry.getValue(), customProperties);
+ @SuppressWarnings("unchecked")
+ Map nestedMap = (Map) entry.getValue();
+ forEachValue(nestedMap, customProperties);
} else {
customProperties.put(entry.getKey(), entry.getValue().toString());
}
@@ -344,7 +379,7 @@ private static Ownership generateOwnership(OpenLineage.RunEvent event) {
event.getJob().getFacets().getOwnership().getOwners()) {
Owner owner = new Owner();
try {
- owner.setOwner(Urn.createFromString(URN_LI_CORPUSER + ":" + ownerFacet.getName()));
+ owner.setOwner(Urn.createFromString(URN_LI_CORPUSER + ownerFacet.getName()));
owner.setType(OwnershipType.DEVELOPER);
OwnershipSource source = new OwnershipSource();
source.setType(OwnershipSourceType.SERVICE);
@@ -384,9 +419,9 @@ private static UpstreamLineage getFineGrainedLineage(
return null;
}
- OpenLineage.ColumnLineageDatasetFacet columLineage = dataset.getFacets().getColumnLineage();
+ OpenLineage.ColumnLineageDatasetFacet columnLineage = dataset.getFacets().getColumnLineage();
Set> fields =
- columLineage.getFields().getAdditionalProperties().entrySet();
+ columnLineage.getFields().getAdditionalProperties().entrySet();
for (Map.Entry field : fields) {
FineGrainedLineage fgl = new FineGrainedLineage();
@@ -398,6 +433,8 @@ private static UpstreamLineage getFineGrainedLineage(
urn ->
downstreamsFields.add(
UrnUtils.getUrn("urn:li:schemaField:" + "(" + urn + "," + field.getKey() + ")")));
+
+ LinkedHashSet transformationTexts = new LinkedHashSet<>();
OpenLineage.StaticDatasetBuilder staticDatasetBuilder =
new OpenLineage.StaticDatasetBuilder();
field
@@ -410,6 +447,15 @@ private static UpstreamLineage getFineGrainedLineage(
.name(inputField.getName())
.namespace(inputField.getNamespace())
.build();
+
+ if (inputField.getTransformations() != null) {
+ for (OpenLineage.InputFieldTransformations transformation :
+ inputField.getTransformations()) {
+ transformationTexts.add(
+ String.format(
+ "%s:%s", transformation.getType(), transformation.getSubtype()));
+ }
+ }
Optional urn =
convertOpenlineageDatasetToDatasetUrn(staticDataset, mappingConfig);
if (urn.isPresent()) {
@@ -434,7 +480,6 @@ private static UpstreamLineage getFineGrainedLineage(
}
});
- // fgl.set(upstreamFields);
upstreamFields.sort(Comparator.comparing(Urn::toString));
fgl.setUpstreams(upstreamFields);
fgl.setConfidenceScore(0.5f);
@@ -443,6 +488,16 @@ private static UpstreamLineage getFineGrainedLineage(
downstreamsFields.sort(Comparator.comparing(Urn::toString));
fgl.setDownstreams(downstreamsFields);
fgl.setDownstreamType(FineGrainedLineageDownstreamType.FIELD_SET);
+
+ // Capture transformation information from OpenLineage
+ if (!transformationTexts.isEmpty()) {
+ List sortedList =
+ transformationTexts.stream()
+ .sorted(String::compareToIgnoreCase)
+ .collect(Collectors.toList());
+ fgl.setTransformOperation(String.join(",", sortedList));
+ }
+
fgla.add(fgl);
}
@@ -473,6 +528,7 @@ private static GlobalTags generateTags(OpenLineage.RunEvent event) {
.getAdditionalProperties()
.get("airflow")
.getAdditionalProperties();
+ @SuppressWarnings("unchecked")
Map dagProperties = (Map) airflowProperties.get("dag");
if (dagProperties.get("tags") != null) {
try {
@@ -498,20 +554,18 @@ private static StringMap generateCustomProperties(
&& (event.getRun().getFacets().getProcessing_engine() != null)) {
if (event.getRun().getFacets().getProcessing_engine().getName() != null) {
customProperties.put(
- "processingEngine", event.getRun().getFacets().getProcessing_engine().getName());
+ PROCESSING_ENGINE_KEY, event.getRun().getFacets().getProcessing_engine().getName());
}
- customProperties.put(
- "processingEngine", event.getRun().getFacets().getProcessing_engine().getName());
if (event.getRun().getFacets().getProcessing_engine().getVersion() != null) {
customProperties.put(
- "processingEngineVersion",
+ PROCESSING_ENGINE_VERSION_KEY,
event.getRun().getFacets().getProcessing_engine().getVersion());
}
if (event.getRun().getFacets().getProcessing_engine().getOpenlineageAdapterVersion()
!= null) {
customProperties.put(
- "openlineageAdapterVersion",
+ OPENLINEAGE_ADAPTER_VERSION_KEY,
event.getRun().getFacets().getProcessing_engine().getOpenlineageAdapterVersion());
}
}
@@ -523,122 +577,180 @@ private static StringMap generateCustomProperties(
for (Map.Entry entry :
event.getRun().getFacets().getAdditionalProperties().entrySet()) {
- switch (entry.getKey()) {
- case "spark_jobDetails":
- if (entry.getValue().getAdditionalProperties().get("jobId") != null) {
- customProperties.put(
- "jobId",
- (String) entry.getValue().getAdditionalProperties().get("jobId").toString());
- }
- if (entry.getValue().getAdditionalProperties().get("jobDescription") != null) {
- customProperties.put(
- "jobDescription",
- (String) entry.getValue().getAdditionalProperties().get("jobDescription"));
- }
- if (entry.getValue().getAdditionalProperties().get("jobGroup") != null) {
- customProperties.put(
- "jobGroup", (String) entry.getValue().getAdditionalProperties().get("jobGroup"));
- }
- if (entry.getValue().getAdditionalProperties().get("jobCallSite") != null) {
- customProperties.put(
- "jobCallSite",
- (String) entry.getValue().getAdditionalProperties().get("jobCallSite"));
- }
- case "processing_engine":
- if (entry.getValue().getAdditionalProperties().get("processing-engine") != null) {
- customProperties.put(
- "processing-engine",
- (String) entry.getValue().getAdditionalProperties().get("name"));
- }
- if (entry.getValue().getAdditionalProperties().get("processing-engine-version") != null) {
- customProperties.put(
- "processing-engine-version",
- (String) entry.getValue().getAdditionalProperties().get("version"));
- }
- if (entry.getValue().getAdditionalProperties().get("openlineage-adapter-version")
- != null) {
- customProperties.put(
- "openlineage-adapter-version",
- (String)
- entry.getValue().getAdditionalProperties().get("openlineageAdapterVersion"));
- }
+ processRunFacetEntry(entry, customProperties, flowProperties);
+ }
+ return customProperties;
+ }
- case "spark_version":
- {
- if (entry.getValue().getAdditionalProperties().get("spark-version") != null) {
- customProperties.put(
- "spark-version",
- (String) entry.getValue().getAdditionalProperties().get("spark-version"));
- }
- if (entry.getValue().getAdditionalProperties().get("openlineage-spark-version")
- != null) {
- customProperties.put(
- "openlineage-spark-version",
- (String)
- entry.getValue().getAdditionalProperties().get("openlineage-spark-version"));
- }
- }
- break;
- case "spark_properties":
- {
- if (entry.getValue() != null) {
- Map sparkProperties =
- (Map)
- entry.getValue().getAdditionalProperties().get("properties");
- log.info("Spark properties: {}, Properties: {}", entry.getValue(), sparkProperties);
- if (sparkProperties != null) {
- forEachValue(sparkProperties, customProperties);
- }
- }
- }
- break;
- case "airflow":
- {
- Map airflowProperties;
- if (flowProperties) {
- airflowProperties =
- (Map) entry.getValue().getAdditionalProperties().get("dag");
- } else {
- airflowProperties =
- (Map) entry.getValue().getAdditionalProperties().get("task");
- }
- forEachValue(airflowProperties, customProperties);
- }
- break;
- case "unknownSourceAttribute":
- {
- if (!flowProperties) {
- List> unknownItems =
- (List>)
- entry.getValue().getAdditionalProperties().get("unknownItems");
- for (Map item : unknownItems) {
- forEachValue(item, customProperties);
- }
- }
- }
- break;
- default:
- break;
+ private static void processRunFacetEntry(
+ Map.Entry entry,
+ StringMap customProperties,
+ boolean flowProperties) {
+ switch (entry.getKey()) {
+ case "spark_jobDetails":
+ processSparkJobDetails(entry.getValue(), customProperties);
+ break;
+ case "processing_engine":
+ processProcessingEngine(entry.getValue(), customProperties);
+ break;
+ case "spark_version":
+ processSparkVersion(entry.getValue(), customProperties);
+ break;
+ case "spark_properties":
+ processSparkProperties(entry.getValue(), customProperties);
+ break;
+ case "airflow":
+ processAirflowProperties(entry.getValue(), customProperties, flowProperties);
+ break;
+ case "spark.logicalPlan":
+ processSparkLogicalPlan(entry.getValue(), customProperties, flowProperties);
+ break;
+ case "unknownSourceAttribute":
+ processUnknownSourceAttributes(entry.getValue(), customProperties, flowProperties);
+ break;
+ default:
+ break;
+ }
+ }
+
+ private static void processSparkJobDetails(
+ OpenLineage.RunFacet facet, StringMap customProperties) {
+ Map properties = facet.getAdditionalProperties();
+ if (properties.get("jobId") != null) {
+ customProperties.put(JOB_ID_KEY, properties.get("jobId").toString());
+ }
+ if (properties.get("jobDescription") != null) {
+ customProperties.put(JOB_DESCRIPTION_KEY, (String) properties.get("jobDescription"));
+ }
+ if (properties.get("jobGroup") != null) {
+ customProperties.put(JOB_GROUP_KEY, (String) properties.get("jobGroup"));
+ }
+ if (properties.get("jobCallSite") != null) {
+ customProperties.put(JOB_CALL_SITE_KEY, (String) properties.get("jobCallSite"));
+ }
+ }
+
+ private static void processProcessingEngine(
+ OpenLineage.RunFacet facet, StringMap customProperties) {
+ Map properties = facet.getAdditionalProperties();
+ if (properties.get("name") != null) {
+ customProperties.put(PROCESSING_ENGINE_KEY, (String) properties.get("name"));
+ }
+ if (properties.get("version") != null) {
+ customProperties.put(PROCESSING_ENGINE_VERSION_KEY, (String) properties.get("version"));
+ }
+ if (properties.get("openlineageAdapterVersion") != null) {
+ customProperties.put(
+ OPENLINEAGE_ADAPTER_VERSION_KEY, (String) properties.get("openlineageAdapterVersion"));
+ }
+ }
+
+ private static void processSparkVersion(OpenLineage.RunFacet facet, StringMap customProperties) {
+ Map properties = facet.getAdditionalProperties();
+ if (properties.get("spark-version") != null) {
+ customProperties.put(SPARK_VERSION_KEY, (String) properties.get("spark-version"));
+ }
+ if (properties.get("openlineage-spark-version") != null) {
+ customProperties.put(
+ OPENLINEAGE_SPARK_VERSION_KEY, (String) properties.get("openlineage-spark-version"));
+ }
+ }
+
+ private static void processSparkProperties(
+ OpenLineage.RunFacet facet, StringMap customProperties) {
+ if (facet != null) {
+ @SuppressWarnings("unchecked")
+ Map sparkProperties =
+ (Map) facet.getAdditionalProperties().get("properties");
+ log.info("Spark properties: {}, Properties: {}", facet, sparkProperties);
+ if (sparkProperties != null) {
+ forEachValue(sparkProperties, customProperties);
}
}
- return customProperties;
+ }
+
+ private static void processAirflowProperties(
+ OpenLineage.RunFacet facet, StringMap customProperties, boolean flowProperties) {
+ @SuppressWarnings("unchecked")
+ Map airflowProperties =
+ flowProperties
+ ? (Map) facet.getAdditionalProperties().get("dag")
+ : (Map) facet.getAdditionalProperties().get("task");
+ if (airflowProperties != null) {
+ forEachValue(airflowProperties, customProperties);
+ }
+ }
+
+ private static void processSparkLogicalPlan(
+ OpenLineage.RunFacet facet, StringMap customProperties, boolean flowProperties) {
+ if (flowProperties) {
+ JSONObject jsonObject = new JSONObject(facet.getAdditionalProperties());
+ customProperties.put(SPARK_LOGICAL_PLAN_KEY, jsonObject.toString());
+ }
+ }
+
+ private static void processUnknownSourceAttributes(
+ OpenLineage.RunFacet facet, StringMap customProperties, boolean flowProperties) {
+ if (!flowProperties) {
+ @SuppressWarnings("unchecked")
+ List> unknownItems =
+ (List>)
+ facet.getAdditionalProperties().getOrDefault("unknownItems", Collections.emptyList());
+ for (Map item : unknownItems) {
+ forEachValue(item, customProperties);
+ }
+ }
+ }
+
+ // Helper method to check for RDD transformations that don't create new datasets
+ private static boolean isNonMaterializingRddTransformation(String jobName) {
+ // These transformations work on the same logical dataset without materializing new ones
+ String[] nonMaterializingTransformations = {
+ // Element-wise transformations (1-to-1 mapping)
+ "map_parallel_collection",
+ "map_text_file",
+ "map_hadoopfile",
+ "map_partitions_parallel_collection",
+ "map_partitions_text_file",
+ "map_partitions_hadoopfile",
+ "flatmap_parallel_collection",
+ "flatmap_text_file",
+ "flatmap_hadoopfile",
+
+ // Filtering operations (subset of same dataset)
+ "filter_parallel_collection",
+ "filter_text_file",
+ "filter_hadoopfile",
+
+ // Deduplication (subset of same dataset)
+ "distinct_parallel_collection",
+ "distinct_text_file",
+ "distinct_hadoopfile"
+ };
+
+ for (String transformation : nonMaterializingTransformations) {
+ if (jobName.endsWith(transformation)) {
+ return true;
+ }
+ }
+
+ return false;
}
private static void convertJobToDataJob(
DatahubJob datahubJob, OpenLineage.RunEvent event, DatahubOpenlineageConfig datahubConf)
- throws URISyntaxException, IOException {
+ throws URISyntaxException {
OpenLineage.Job job = event.getJob();
DataJobInfo dji = new DataJobInfo();
log.debug("Datahub Config: {}", datahubConf);
- if (job.getName().contains(".")) {
- String jobName = job.getName().substring(job.getName().indexOf(".") + 1);
- dji.setName(jobName);
- } else {
- dji.setName(job.getName());
- }
+ // Extract job names using helper method
+ JobNameResult jobNames = extractJobNames(job, event, datahubConf);
+
+ // Set the display name
+ dji.setName(jobNames.displayName);
String jobProcessingEngine = null;
if ((event.getRun().getFacets() != null)
@@ -649,7 +761,7 @@ private static void convertJobToDataJob(
DataFlowUrn flowUrn =
getFlowUrn(
event.getJob().getNamespace(),
- event.getJob().getName(),
+ job.getName(), // Use original job name for flow URN
jobProcessingEngine,
event.getProducer(),
datahubConf);
@@ -657,8 +769,10 @@ private static void convertJobToDataJob(
dji.setFlowUrn(flowUrn);
dji.setType(DataJobInfo.Type.create(flowUrn.getOrchestratorEntity()));
- DataJobUrn dataJobUrn = new DataJobUrn(flowUrn, job.getName());
+ // Use the jobNameForUrn (which includes table name for MERGE commands)
+ DataJobUrn dataJobUrn = new DataJobUrn(flowUrn, jobNames.urnName);
datahubJob.setJobUrn(dataJobUrn);
+
StringMap customProperties = generateCustomProperties(event, false);
dji.setCustomProperties(customProperties);
@@ -673,26 +787,9 @@ private static void convertJobToDataJob(
dji.setDescription(description);
}
datahubJob.setJobInfo(dji);
- DataJobInputOutput inputOutput = new DataJobInputOutput();
- boolean inputsEqualOutputs = false;
- if ((datahubConf.isSpark())
- && ((event.getInputs() != null && event.getOutputs() != null)
- && (event.getInputs().size() == event.getOutputs().size()))) {
- inputsEqualOutputs =
- event.getInputs().stream()
- .map(OpenLineage.Dataset::getName)
- .collect(Collectors.toSet())
- .equals(
- event.getOutputs().stream()
- .map(OpenLineage.Dataset::getName)
- .collect(Collectors.toSet()));
- if (inputsEqualOutputs) {
- log.info(
- "Inputs equals Outputs: {}. This is most probably because of an rdd map operation and we only process Inputs",
- inputsEqualOutputs);
- }
- }
+ // Process inputs and outputs
+ boolean inputsEqualOutputs = checkInputsEqualOutputs(event, job, datahubConf);
processJobInputs(datahubJob, event, datahubConf);
@@ -700,13 +797,16 @@ private static void convertJobToDataJob(
processJobOutputs(datahubJob, event, datahubConf);
}
+ // Set run event and instance properties
DataProcessInstanceRunEvent dpire = processDataProcessInstanceResult(event);
datahubJob.setDataProcessInstanceRunEvent(dpire);
DataProcessInstanceProperties dpiProperties = getJobDataProcessInstanceProperties(event);
datahubJob.setDataProcessInstanceProperties(dpiProperties);
- processParentJob(event, job, inputOutput, datahubConf);
+ // Create input/output edges and relationships
+ DataJobInputOutput inputOutput = new DataJobInputOutput();
+ processParentJob(event, job, jobNames.urnName, inputOutput, datahubConf);
DataProcessInstanceRelationships dataProcessInstanceRelationships =
new DataProcessInstanceRelationships();
@@ -723,6 +823,222 @@ private static void convertJobToDataJob(
}
}
+ private static class JobNameResult {
+ final String displayName;
+ final String urnName;
+
+ JobNameResult(String displayName, String urnName) {
+ this.displayName = displayName;
+ this.urnName = urnName;
+ }
+ }
+
+ private static JobNameResult extractJobNames(
+ OpenLineage.Job job, OpenLineage.RunEvent event, DatahubOpenlineageConfig datahubConf) {
+
+ // Check if we have a MERGE INTO command
+ boolean isMergeIntoCommand = job.getName().contains(MERGE_INTO_COMMAND_PATTERN);
+ String tableName = null;
+
+ // If this is a MERGE INTO command and enhanced extraction is enabled, try to extract the target
+ // table name
+ if (isMergeIntoCommand && datahubConf.isEnhancedMergeIntoExtraction()) {
+ log.info("Detected MERGE INTO command in job: {} - using enhanced extraction", job.getName());
+ tableName = extractTableNameFromMergeCommand(job, event);
+ }
+
+ // Prepare job names - one for display and one for the URN
+ String jobNameForDisplay = job.getName();
+ String jobNameForUrn = job.getName();
+
+ // If this is a merge command with an identified table, include the table name
+ if (isMergeIntoCommand && tableName != null && datahubConf.isEnhancedMergeIntoExtraction()) {
+ // Create modified job names that include the table name
+ String tablePart = tableName.replace(".", "_").replace(" ", "_").toLowerCase();
+ String enhancedJobName = job.getName() + "." + tablePart;
+
+ log.info("Modified job name for MERGE INTO: {} -> {}", job.getName(), enhancedJobName);
+
+ // Use the enhanced name for URN
+ jobNameForUrn = enhancedJobName;
+
+ // For display name, first add the table part, then remove everything before first dot
+ jobNameForDisplay = enhancedJobName;
+ if (jobNameForDisplay.contains(".")) {
+ jobNameForDisplay = jobNameForDisplay.substring(jobNameForDisplay.indexOf(".") + 1);
+ }
+ } else if (job.getName().contains(".")) {
+ // Normal case - use part after the dot for display only
+ jobNameForDisplay = job.getName().substring(job.getName().indexOf(".") + 1);
+ }
+
+ return new JobNameResult(jobNameForDisplay, jobNameForUrn);
+ }
+
+ private static String extractTableNameFromMergeCommand(
+ OpenLineage.Job job, OpenLineage.RunEvent event) {
+ String tableName;
+
+ // Method 1: Check for table name in the SQL facet (most reliable)
+ tableName = extractTableNameFromSql(job);
+ if (tableName != null) {
+ return tableName;
+ }
+
+ // Method 2: Look for direct table names in the outputs
+ tableName = extractTableNameFromOutputs(event);
+ if (tableName != null) {
+ return tableName;
+ }
+
+ // Method 3: Check for table identifiers in symlinks
+ tableName = extractTableNameFromSymlinks(event);
+ if (tableName != null) {
+ return tableName;
+ }
+
+ // Method 4: Extract table name from warehouse paths (as a last resort)
+ tableName = extractTableNameFromWarehousePaths(event);
+ return tableName;
+ }
+
+ private static String extractTableNameFromSql(OpenLineage.Job job) {
+ if (job.getFacets() != null && job.getFacets().getSql() != null) {
+ String sqlQuery = job.getFacets().getSql().getQuery();
+ if (sqlQuery != null && sqlQuery.toUpperCase().contains(MERGE_INTO_SQL_PATTERN)) {
+ // Extract table name from the MERGE INTO SQL statement
+ String[] lines = sqlQuery.split("\n");
+ for (String line : lines) {
+ line = line.trim();
+ if (line.toUpperCase().startsWith(MERGE_INTO_SQL_PATTERN)) {
+ // Format: MERGE INTO schema.table target
+ String[] parts = line.split("\\s+");
+ if (parts.length >= 3) {
+ String tableName = parts[2].replace("`", "").trim();
+ // If there's an alias (target/t/etc.), remove it
+ int spaceIndex = tableName.indexOf(' ');
+ if (spaceIndex > 0) {
+ tableName = tableName.substring(0, spaceIndex);
+ }
+ log.info("Extracted table name from SQL: {}", tableName);
+ return tableName;
+ }
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ private static String extractTableNameFromOutputs(OpenLineage.RunEvent event) {
+ if (event.getOutputs() != null) {
+ for (OpenLineage.OutputDataset output : event.getOutputs()) {
+ // First check if the name itself is a table name (e.g., "delta_demo.customers")
+ String name = output.getName();
+ if (name != null && name.contains(".") && !name.startsWith("/")) {
+ log.info("Using table name directly from output dataset name: {}", name);
+ return name;
+ }
+ }
+ }
+ return null;
+ }
+
+ private static String extractTableNameFromSymlinks(OpenLineage.RunEvent event) {
+ if (event.getOutputs() != null) {
+ for (OpenLineage.OutputDataset output : event.getOutputs()) {
+ if (output.getFacets() != null && output.getFacets().getSymlinks() != null) {
+ for (OpenLineage.SymlinksDatasetFacetIdentifiers symlink :
+ output.getFacets().getSymlinks().getIdentifiers()) {
+ if ("TABLE".equals(symlink.getType())) {
+ String name = symlink.getName();
+ if (name != null) {
+ // Handle table/name format
+ if (name.startsWith(TABLE_PREFIX)) {
+ name = name.replaceFirst(TABLE_PREFIX, "").replace("/", ".");
+ }
+ log.info("Extracted table name from symlink: {}", name);
+ return name;
+ }
+ }
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ private static String extractTableNameFromWarehousePaths(OpenLineage.RunEvent event) {
+ if (event.getOutputs() != null) {
+ for (OpenLineage.OutputDataset output : event.getOutputs()) {
+ String path = output.getName();
+ if (path != null && path.contains(WAREHOUSE_PATH_PATTERN)) {
+ // Extract table name from warehouse path pattern /warehouse/db.name/ or similar
+ if (path.contains(DB_SUFFIX)) {
+ int dbIndex = path.lastIndexOf(DB_SUFFIX);
+ String tablePart = path.substring(dbIndex + 4);
+ // Remove trailing slashes
+ tablePart = tablePart.replaceAll("/+$", "");
+ // Construct the full table name including db
+ int warehouseIndex = path.lastIndexOf(WAREHOUSE_PATH_PATTERN);
+ if (warehouseIndex >= 0) {
+ String dbPart = path.substring(warehouseIndex + 11, dbIndex);
+ String tableName = dbPart + "." + tablePart;
+ log.info("Extracted table name from warehouse path: {}", tableName);
+ return tableName;
+ }
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ // Some rdd map operation generate inputs that are equal to outputs even though it doesn't come
+ // from
+ // the spark job. We try to handle it by checking if the job is a non-materializing RDD
+ // transformation.
+ // This logic might not be perfect, but it should cover most cases.
+ private static boolean checkInputsEqualOutputs(
+ OpenLineage.RunEvent event, OpenLineage.Job job, DatahubOpenlineageConfig datahubConf) {
+ if (!datahubConf.isSpark()) {
+ return false;
+ }
+
+ if (job.getFacets() == null
+ || job.getFacets().getJobType() == null
+ || !"RDD_JOB".equals(job.getFacets().getJobType().getJobType())) {
+ return false;
+ }
+
+ if (!isNonMaterializingRddTransformation(job.getName())) {
+ return false;
+ }
+
+ if (event.getInputs() == null
+ || event.getOutputs() == null
+ || event.getInputs().size() != event.getOutputs().size()) {
+ return false;
+ }
+
+ boolean inputsEqualOutputs =
+ event.getInputs().stream()
+ .map(OpenLineage.Dataset::getName)
+ .collect(Collectors.toSet())
+ .equals(
+ event.getOutputs().stream()
+ .map(OpenLineage.Dataset::getName)
+ .collect(Collectors.toSet()));
+
+ if (inputsEqualOutputs) {
+ log.info(
+ "Inputs equals Outputs: {}. This is most probably because of an rdd map operation and we only process Inputs",
+ inputsEqualOutputs);
+ }
+
+ return inputsEqualOutputs;
+ }
+
private static DataProcessInstanceProperties getJobDataProcessInstanceProperties(
OpenLineage.RunEvent event) throws URISyntaxException {
DataProcessInstanceProperties dpiProperties = new DataProcessInstanceProperties();
@@ -761,18 +1077,21 @@ public static AuditStamp createAuditStamp(ZonedDateTime eventTime) {
private static void processParentJob(
OpenLineage.RunEvent event,
OpenLineage.Job job,
+ String jobNameForUrn,
DataJobInputOutput inputOutput,
DatahubOpenlineageConfig datahubConf) {
if ((event.getRun().getFacets() != null) && (event.getRun().getFacets().getParent() != null)) {
+ OpenLineage.ParentRunFacetJob parentRunFacetJob =
+ event.getRun().getFacets().getParent().getJob();
DataJobUrn parentDataJobUrn =
new DataJobUrn(
getFlowUrn(
- event.getRun().getFacets().getParent().getJob().getNamespace(),
- event.getRun().getFacets().getParent().getJob().getName(),
+ parentRunFacetJob.getNamespace(),
+ parentRunFacetJob.getName(),
null,
event.getRun().getFacets().getParent().get_producer(),
datahubConf),
- job.getName());
+ jobNameForUrn);
Edge edge = createEdge(parentDataJobUrn, event.getEventTime());
EdgeArray array = new EdgeArray();
@@ -788,8 +1107,10 @@ private static void processJobInputs(
return;
}
- for (OpenLineage.InputDataset input :
- event.getInputs().stream().distinct().collect(Collectors.toList())) {
+ // Use LinkedHashSet to maintain order and remove duplicates more efficiently
+ Set uniqueInputs = new LinkedHashSet<>(event.getInputs());
+
+ for (OpenLineage.InputDataset input : uniqueInputs) {
Optional datasetUrn = convertOpenlineageDatasetToDatasetUrn(input, datahubConf);
if (datasetUrn.isPresent()) {
DatahubDataset.DatahubDatasetBuilder builder = DatahubDataset.builder();
@@ -815,8 +1136,10 @@ private static void processJobOutputs(
return;
}
- for (OpenLineage.OutputDataset output :
- event.getOutputs().stream().distinct().collect(Collectors.toList())) {
+ // Use LinkedHashSet to maintain order and remove duplicates more efficiently
+ Set uniqueOutputs = new LinkedHashSet<>(event.getOutputs());
+
+ for (OpenLineage.OutputDataset output : uniqueOutputs) {
Optional datasetUrn = convertOpenlineageDatasetToDatasetUrn(output, datahubConf);
if (datasetUrn.isPresent()) {
DatahubDataset.DatahubDatasetBuilder builder = DatahubDataset.builder();
@@ -910,7 +1233,7 @@ public static DataFlowUrn getFlowUrn(
}
public static DataFlowInfo convertRunEventToDataFlowInfo(
- OpenLineage.RunEvent event, String flowName) throws IOException {
+ OpenLineage.RunEvent event, String flowName) {
DataFlowInfo dataFlowInfo = new DataFlowInfo();
dataFlowInfo.setName(getFlowName(event.getJob().getName(), flowName));
return dataFlowInfo;
diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPathDataset.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPathDataset.java
index b938db24fc626d..467493e1c17702 100644
--- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPathDataset.java
+++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPathDataset.java
@@ -179,7 +179,8 @@ static String getMatchedUri(String pathUri, String pathSpec) {
public enum HdfsPlatform {
S3(Arrays.asList("s3", "s3a", "s3n"), "s3"),
GCS(Arrays.asList("gs", "gcs"), "gcs"),
- ABFS(Arrays.asList("abfs", "abfss"), "abfs"),
+ ABFS(Arrays.asList("abfs", "abfss"), "abs"),
+ WASB(Arrays.asList("wasb", "wasbs"), "abs"),
DBFS(Collections.singletonList("dbfs"), "dbfs"),
FILE(Collections.singletonList("file"), "file"),
// default platform
diff --git a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPlatform.java b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPlatform.java
index dcaf34f9d7b0fd..76099626bbe414 100644
--- a/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPlatform.java
+++ b/metadata-integration/java/openlineage-converter/src/main/java/io/datahubproject/openlineage/dataset/HdfsPlatform.java
@@ -7,7 +7,8 @@
public enum HdfsPlatform {
S3(Arrays.asList("s3", "s3a", "s3n"), "s3"),
GCS(Arrays.asList("gs", "gcs"), "gcs"),
- ABFS(Arrays.asList("abfs", "abfss"), "abfs"),
+ ABFS(Arrays.asList("abfs", "abfss"), "abs"),
+ WASB(Arrays.asList("wasb", "wasbs"), "abs"),
DBFS(Collections.singletonList("dbfs"), "dbfs"),
FILE(Collections.singletonList("file"), "file"),
// default platform
diff --git a/metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java b/metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java
index 95d73bbc02f2ec..47832ed4e754a4 100644
--- a/metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java
+++ b/metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java
@@ -782,7 +782,8 @@ public PropertiesCollector propertiesCollector(Environment environment) {
// Gradle and test-specific properties
"org.gradle.internal.worker.tmpdir",
- "org.springframework.boot.test.context.SpringBootTestContextBootstrapper"
+ "org.springframework.boot.test.context.SpringBootTestContextBootstrapper",
+ "datahub.policies.systemPolicyUrnList"
// TODO: Add more properties as they are discovered during testing
// When this test fails due to unclassified properties, add them to
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/config/DatahubOpenlineageProperties.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/config/DatahubOpenlineageProperties.java
new file mode 100644
index 00000000000000..20a32ba63a9e53
--- /dev/null
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/config/DatahubOpenlineageProperties.java
@@ -0,0 +1,21 @@
+package io.datahubproject.openapi.openlineage.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Data
+@Component
+@ConfigurationProperties(prefix = "datahub.openlineage")
+public class DatahubOpenlineageProperties {
+
+ private String pipelineName;
+ private String platformInstance;
+ private String commonDatasetPlatformInstance;
+ private String platform;
+ private String filePartitionRegexpPattern;
+ private boolean materializeDataset = true;
+ private boolean includeSchemaMetadata = true;
+ private boolean captureColumnLevelLineage = true;
+ private boolean usePatch = false;
+}
diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/config/OpenLineageServletConfig.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/config/OpenLineageServletConfig.java
index fa1569fa8cf621..ffcf0d4f85f62e 100644
--- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/config/OpenLineageServletConfig.java
+++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/openlineage/config/OpenLineageServletConfig.java
@@ -2,28 +2,35 @@
import io.datahubproject.openapi.openlineage.mapping.RunEventMapper;
import io.datahubproject.openlineage.config.DatahubOpenlineageConfig;
+import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
+@Slf4j
public class OpenLineageServletConfig {
+ private final DatahubOpenlineageProperties properties;
+
+ public OpenLineageServletConfig(DatahubOpenlineageProperties properties) {
+ this.properties = properties;
+ }
+
@Bean
public RunEventMapper.MappingConfig mappingConfig() {
DatahubOpenlineageConfig datahubOpenlineageConfig =
DatahubOpenlineageConfig.builder()
- .isStreaming(false)
- .pipelineName(null)
- .platformInstance(null)
- .commonDatasetPlatformInstance(null)
- .platform(null)
- .filePartitionRegexpPattern(null)
- .materializeDataset(true)
- .includeSchemaMetadata(true)
- .captureColumnLevelLineage(true)
- .usePatch(false)
+ .platformInstance(properties.getPlatformInstance())
+ .commonDatasetPlatformInstance(properties.getCommonDatasetPlatformInstance())
+ .platform(properties.getPlatform())
+ .filePartitionRegexpPattern(properties.getFilePartitionRegexpPattern())
+ .materializeDataset(properties.isMaterializeDataset())
+ .includeSchemaMetadata(properties.isIncludeSchemaMetadata())
+ .captureColumnLevelLineage(properties.isCaptureColumnLevelLineage())
+ .usePatch(properties.isUsePatch())
.parentJobUrn(null)
.build();
+ log.info("Starting OpenLineage Endpoint with config: {}", datahubOpenlineageConfig);
return RunEventMapper.MappingConfig.builder().datahubConfig(datahubOpenlineageConfig).build();
}
}
diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java
index 39e6ef787c764b..3624491612906e 100644
--- a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java
+++ b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java
@@ -131,7 +131,13 @@ public enum DataHubUsageEventType {
ENTITY_EVENT("EntityEvent"),
FAILED_LOGIN_EVENT("FailedLogInEvent"),
DELETE_POLICY_EVENT("DeletePolicyEvent"),
- CLICK_PRODUCT_UPDATE_EVENT("ClickProductUpdateEvent");
+ CLICK_PRODUCT_UPDATE_EVENT("ClickProductUpdateEvent"),
+ WELCOME_TO_DATAHUB_MODAL_VIEW_EVENT("WelcomeToDataHubModalViewEvent"),
+ WELCOME_TO_DATAHUB_MODAL_INTERACT_EVENT("WelcomeToDataHubModalInteractEvent"),
+ WELCOME_TO_DATAHUB_MODAL_EXIT_EVENT("WelcomeToDataHubModalExitEvent"),
+ WELCOME_TO_DATAHUB_MODAL_CLICK_VIEW_DOCUMENTATION_EVENT(
+ "WelcomeToDataHubModalClickViewDocumentationEvent"),
+ PRODUCT_TOUR_BUTTON_CLICK_EVENT("ProductTourButtonClickEvent");
private final String type;
diff --git a/metadata-service/services/src/test/java/com/linkedin/metadata/datahubusage/DataHubUsageEventTypeTest.java b/metadata-service/services/src/test/java/com/linkedin/metadata/datahubusage/DataHubUsageEventTypeTest.java
new file mode 100644
index 00000000000000..665b5e86809832
--- /dev/null
+++ b/metadata-service/services/src/test/java/com/linkedin/metadata/datahubusage/DataHubUsageEventTypeTest.java
@@ -0,0 +1,64 @@
+package com.linkedin.metadata.datahubusage;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+
+import org.testng.annotations.Test;
+
+public class DataHubUsageEventTypeTest {
+
+ @Test
+ public void testGetTypeMethod() {
+ // Test that getType method works for various event types
+ assertEquals(
+ DataHubUsageEventType.getType("PageViewEvent"), DataHubUsageEventType.PAGE_VIEW_EVENT);
+ assertEquals(DataHubUsageEventType.getType("LogInEvent"), DataHubUsageEventType.LOG_IN_EVENT);
+ assertEquals(
+ DataHubUsageEventType.getType("WelcomeToDataHubModalViewEvent"),
+ DataHubUsageEventType.WELCOME_TO_DATAHUB_MODAL_VIEW_EVENT);
+
+ // Test the new enum value
+ assertEquals(
+ DataHubUsageEventType.getType("ProductTourButtonClickEvent"),
+ DataHubUsageEventType.PRODUCT_TOUR_BUTTON_CLICK_EVENT);
+
+ // Test case insensitive matching
+ assertEquals(
+ DataHubUsageEventType.getType("producttourButtonClickEvent"),
+ DataHubUsageEventType.PRODUCT_TOUR_BUTTON_CLICK_EVENT);
+
+ // Test non-existent event type
+ assertNull(DataHubUsageEventType.getType("NonExistentEvent"));
+ }
+
+ @Test
+ public void testEnumValues() {
+ // Test that enum values have correct type strings
+ assertEquals(
+ DataHubUsageEventType.PRODUCT_TOUR_BUTTON_CLICK_EVENT.getType(),
+ "ProductTourButtonClickEvent");
+ assertEquals(
+ DataHubUsageEventType.WELCOME_TO_DATAHUB_MODAL_VIEW_EVENT.getType(),
+ "WelcomeToDataHubModalViewEvent");
+ assertEquals(
+ DataHubUsageEventType.WELCOME_TO_DATAHUB_MODAL_INTERACT_EVENT.getType(),
+ "WelcomeToDataHubModalInteractEvent");
+ assertEquals(
+ DataHubUsageEventType.WELCOME_TO_DATAHUB_MODAL_EXIT_EVENT.getType(),
+ "WelcomeToDataHubModalExitEvent");
+ assertEquals(
+ DataHubUsageEventType.WELCOME_TO_DATAHUB_MODAL_CLICK_VIEW_DOCUMENTATION_EVENT.getType(),
+ "WelcomeToDataHubModalClickViewDocumentationEvent");
+ }
+
+ @Test
+ public void testAllEnumValuesHaveTypes() {
+ // Ensure all enum values have non-null type strings
+ for (DataHubUsageEventType eventType : DataHubUsageEventType.values()) {
+ assertNotNull(eventType.getType(), "Event type should not be null for " + eventType.name());
+ // Verify that getType method can find each enum value
+ assertEquals(DataHubUsageEventType.getType(eventType.getType()), eventType);
+ }
+ }
+}
diff --git a/smoke-test/tests/cypress/cypress.config.js b/smoke-test/tests/cypress/cypress.config.js
index e6913a4791d812..dcab42dfa8a7d7 100644
--- a/smoke-test/tests/cypress/cypress.config.js
+++ b/smoke-test/tests/cypress/cypress.config.js
@@ -8,7 +8,7 @@ module.exports = defineConfig({
projectId: "hkrxk5",
defaultCommandTimeout: 10000,
retries: {
- runMode: 2,
+ runMode: 5,
openMode: 0,
},
video: false,
diff --git a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js
index ba1263df53ad89..695d69913cda0e 100644
--- a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js
+++ b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js
@@ -51,9 +51,7 @@ const getDomainList = (domainName) => {
cy.contains("span.ant-typography-ellipsis", domainName)
.parent('[data-testid="domain-list-item"]')
.find(
- '[data-testid="open-domain-action-item-urn:li:domain:' +
- domainName.toLowerCase() +
- '"]',
+ `[data-testid="open-domain-action-item-urn:li:domain:${domainName.toLowerCase()}"]`,
)
.click();
};
diff --git a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js b/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
deleted file mode 100644
index 553c0fb1626bcb..00000000000000
--- a/smoke-test/tests/cypress/cypress/e2e/glossary/glossary_navigation.js
+++ /dev/null
@@ -1,116 +0,0 @@
-const glossaryTerm = "CypressGlosssaryNavigationTerm";
-const glossarySecondTerm = "CypressGlossarySecondTerm";
-const glossaryTermGroup = "CypressGlosssaryNavigationGroup";
-const glossaryParentGroup = "CypressNode";
-
-const createTerm = (glossaryTerm) => {
- cy.waitTextVisible("Create Glossary Term");
- cy.enterTextInTestId("create-glossary-entity-modal-name", glossaryTerm);
- cy.clickOptionWithTestId("glossary-entity-modal-create-button");
-};
-
-const navigateToParentAndCheckTermGroup = (parentGroup, termGroup) => {
- cy.get('[data-testid="glossary-browser-sidebar"]')
- .contains(parentGroup)
- .click();
- cy.get('*[class^="GlossaryEntitiesList"]')
- .contains(termGroup)
- .should("be.visible");
-};
-
-const moveGlossaryEntityToGroup = (
- sourceEntity,
- targetEntity,
- confirmationMsg,
-) => {
- cy.clickOptionWithText(sourceEntity);
- cy.get('[data-testid="entity-header-dropdown"]').should("be.visible");
- cy.openThreeDotDropdown();
- cy.clickOptionWithText("Move");
- cy.get('[data-testid="move-glossary-entity-modal"]')
- .contains(targetEntity)
- .click({ force: true });
- cy.get('[data-testid="move-glossary-entity-modal"]')
- .contains(targetEntity)
- .should("be.visible");
- cy.clickOptionWithTestId("glossary-entity-modal-move-button");
- cy.waitTextVisible(confirmationMsg);
-};
-
-const deleteGlossaryTerm = (parentGroup, termGroup, term) => {
- cy.goToGlossaryList();
- cy.clickOptionWithText(parentGroup);
- cy.clickOptionWithText(termGroup);
- cy.clickOptionWithText(term);
- cy.deleteFromDropdown();
- cy.waitTextVisible("Deleted Glossary Term!");
-};
-
-describe("glossary sidebar navigation test", () => {
- it("create term and term parent group, move and delete term group", () => {
- cy.loginWithCredentials();
-
- // Create term group and term
- cy.createGlossaryTermGroup(glossaryTermGroup);
- cy.clickOptionWithTestId("add-term-button");
- createTerm(glossaryTerm);
- moveGlossaryEntityToGroup(
- glossaryTerm,
- glossaryTermGroup,
- `Moved Glossary Term!`,
- );
- navigateToParentAndCheckTermGroup(glossaryTermGroup, glossaryTerm);
-
- // Create another term and move it to the same term group
- cy.clickOptionWithText(glossaryTermGroup);
- cy.openThreeDotDropdown();
- cy.clickOptionWithTestId("entity-menu-add-term-button");
- createTerm(glossarySecondTerm);
- moveGlossaryEntityToGroup(
- glossarySecondTerm,
- glossaryTermGroup,
- `Moved Glossary Term!`,
- );
- navigateToParentAndCheckTermGroup(glossaryTermGroup, glossarySecondTerm);
-
- // Switch between terms and ensure the "Properties" tab is active
- cy.clickOptionWithText(glossaryTerm);
- cy.get('[data-testid="entity-tab-headers-test-id"]')
- .contains("Properties")
- .click({ force: true });
- cy.get('[data-node-key="Properties"]')
- .contains("Properties")
- .should("have.attr", "aria-selected", "true");
- cy.clickOptionWithText(glossarySecondTerm);
- cy.get('[data-node-key="Properties"]')
- .contains("Properties")
- .should("have.attr", "aria-selected", "true");
-
- // Move a term group from the root level to be under a parent term group
- cy.goToGlossaryList();
- moveGlossaryEntityToGroup(
- glossaryTermGroup,
- glossaryParentGroup,
- "Moved Term Group!",
- );
- navigateToParentAndCheckTermGroup(glossaryParentGroup, glossaryTermGroup);
-
- // Delete glossary terms and term group
- deleteGlossaryTerm(glossaryParentGroup, glossaryTermGroup, glossaryTerm);
- deleteGlossaryTerm(
- glossaryParentGroup,
- glossaryTermGroup,
- glossarySecondTerm,
- );
-
- cy.goToGlossaryList();
- cy.clickOptionWithText(glossaryParentGroup);
- cy.clickOptionWithText(glossaryTermGroup);
- cy.deleteFromDropdown();
- cy.waitTextVisible("Deleted Term Group!");
-
- // Ensure it is no longer in the sidebar navigator
- cy.ensureTextNotPresent(glossaryTerm);
- cy.ensureTextNotPresent(glossaryTermGroup);
- });
-});
diff --git a/smoke-test/tests/cypress/cypress/e2e/incidentsV2/v2_incidents.js b/smoke-test/tests/cypress/cypress/e2e/incidentsV2/v2_incidents.js
index d0e54b13167a47..9ae80e38373f45 100644
--- a/smoke-test/tests/cypress/cypress/e2e/incidentsV2/v2_incidents.js
+++ b/smoke-test/tests/cypress/cypress/e2e/incidentsV2/v2_incidents.js
@@ -174,8 +174,8 @@ describe("incidents", () => {
cy.visit(
"/dataset/urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.customers,PROD)/Incidents?is_lineage_mode=false&separate_siblings=false",
);
- cy.get('[data-testid="create-incident-btn-main"]').trigger("mouseover");
- cy.get(".ant-dropdown-menu-item").first().click();
+ cy.findByTestId("create-incident-btn-main").as("btn");
+ cy.get("@btn").click();
cy.get('[data-testid="drawer-header-title"]').should(
"contain.text",
"Create New Incident",
diff --git a/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js b/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js
index 623d5078e77bca..f772620fb93212 100644
--- a/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js
+++ b/smoke-test/tests/cypress/cypress/e2e/mutations/edit_documentation.js
@@ -4,6 +4,12 @@ const wrong_url = "https://www.linkedincom";
const correct_url = "https://www.linkedin.com";
describe("edit documentation and link to dataset", () => {
+ beforeEach(() => {
+ cy.window().then((win) => {
+ win.localStorage.setItem("isThemeV2Enabled", "false");
+ });
+ });
+
it("open test dataset page, edit documentation", () => {
// edit documentation and verify changes saved
cy.loginWithCredentials();
diff --git a/smoke-test/tests/cypress/cypress/e2e/onboarding/welcome-to-datahub-modal.cy.js b/smoke-test/tests/cypress/cypress/e2e/onboarding/welcome-to-datahub-modal.cy.js
new file mode 100644
index 00000000000000..e0670e8acd2aa8
--- /dev/null
+++ b/smoke-test/tests/cypress/cypress/e2e/onboarding/welcome-to-datahub-modal.cy.js
@@ -0,0 +1,95 @@
+import { aliasQuery, hasOperationName } from "../utils";
+
+describe("WelcomeToDataHubModal", () => {
+ const SKIP_WELCOME_MODAL_KEY = "skipWelcomeModal";
+ const THEME_V2_STATUS_KEY = "isThemeV2Enabled";
+
+ beforeEach(() => {
+ cy.window().then((win) => {
+ win.localStorage.removeItem(SKIP_WELCOME_MODAL_KEY);
+ cy.skipIntroducePage();
+ win.localStorage.setItem(THEME_V2_STATUS_KEY, "true");
+ });
+
+ // Intercept GraphQL requests if needed
+ cy.intercept("POST", "/api/v2/graphql", (req) => {
+ aliasQuery(req, "appConfig");
+
+ // Mock app config to enable theme V2
+ if (hasOperationName(req, "appConfig")) {
+ req.alias = "gqlappConfigQuery";
+ req.continue((res) => {
+ res.body.data.appConfig.featureFlags.themeV2Enabled = true;
+ res.body.data.appConfig.featureFlags.themeV2Default = true;
+ });
+ }
+ });
+ });
+
+ afterEach(() => {
+ cy.window().then((win) => {
+ win.localStorage.removeItem(SKIP_WELCOME_MODAL_KEY);
+ win.localStorage.removeItem(THEME_V2_STATUS_KEY);
+ });
+ });
+
+ it("should display the modal for first-time users", () => {
+ cy.intercept("POST", "**/track**", { statusCode: 200 }).as("trackEvents");
+
+ cy.loginForOnboarding();
+ cy.visit("/");
+
+ cy.findByRole("dialog").should("be.visible");
+
+ // Click the last carousel dot
+ cy.findByRole("dialog").find("li").last().click();
+
+ // Click Get Started button
+ cy.findByRole("button", { name: /get started/i }).click();
+ cy.findByRole("dialog").should("not.exist");
+
+ cy.window().then((win) => {
+ expect(win.localStorage.getItem(SKIP_WELCOME_MODAL_KEY)).to.equal("true");
+ });
+ });
+
+ it("should not display the modal if user has already seen it", () => {
+ cy.window().then((win) => {
+ win.localStorage.setItem(SKIP_WELCOME_MODAL_KEY, "true");
+ });
+
+ cy.loginForOnboarding();
+ cy.visit("/");
+
+ cy.findByRole("dialog").should("not.exist");
+ });
+
+ it("should handle user interactions and track events", () => {
+ // Set up intercept that captures all track events
+ cy.intercept("POST", "**/track**", { statusCode: 200 }).as("trackEvents");
+
+ cy.loginForOnboarding();
+ cy.visit("/");
+
+ cy.findByRole("dialog").should("be.visible");
+ cy.findByLabelText(/close/i).click();
+ cy.findByRole("dialog").should("not.exist");
+
+ // Wait specifically for the Exit event to appear and verify all properties
+ cy.get("@trackEvents.all").should((interceptions) => {
+ const modalExitEvent = interceptions.find(
+ (interception) =>
+ interception.request.body.type === "WelcomeToDataHubModalExitEvent",
+ );
+
+ // Verify the event exists and has all expected properties
+ expect(modalExitEvent.request.body.exitMethod).to.equal("close_button");
+ expect(modalExitEvent.request.body.currentSlide).to.be.a("number");
+ expect(modalExitEvent.request.body.totalSlides).to.be.a("number");
+ });
+
+ cy.window().then((win) => {
+ expect(win.localStorage.getItem(SKIP_WELCOME_MODAL_KEY)).to.equal("true");
+ });
+ });
+});
diff --git a/smoke-test/tests/cypress/cypress/e2e/ownership/manage_ownership.js b/smoke-test/tests/cypress/cypress/e2e/ownership/manage_ownership.js
index 17825881fb0b56..2441363073610c 100644
--- a/smoke-test/tests/cypress/cypress/e2e/ownership/manage_ownership.js
+++ b/smoke-test/tests/cypress/cypress/e2e/ownership/manage_ownership.js
@@ -20,7 +20,9 @@ describe("manage ownership", () => {
cy.get(
'[data-row-key="Test Ownership Type"] > :nth-child(3) > .anticon > svg',
- ).click();
+ )
+ .first()
+ .click();
cy.clickOptionWithText("Edit");
cy.get('[data-testid="ownership-type-description-input"]').clear(
"This is an test ownership type description.",
@@ -34,7 +36,9 @@ describe("manage ownership", () => {
cy.get(
'[data-row-key="Test Ownership Type"] > :nth-child(3) > .anticon > svg',
- ).click();
+ )
+ .first()
+ .click();
cy.clickOptionWithText("Delete");
cy.get(".ant-popover-buttons > .ant-btn-primary").click();
cy.wait(3000);
diff --git a/smoke-test/tests/cypress/cypress/support/commands.js b/smoke-test/tests/cypress/cypress/support/commands.js
index 46e45a4061a6a2..bc83988da94105 100644
--- a/smoke-test/tests/cypress/cypress/support/commands.js
+++ b/smoke-test/tests/cypress/cypress/support/commands.js
@@ -23,6 +23,14 @@ export function getTimestampMillisNumDaysAgo(numDays) {
}
const SKIP_ONBOARDING_TOUR_KEY = "skipOnboardingTour";
+const SKIP_WELCOME_MODAL_KEY = "skipWelcomeModal";
+
+function notFirstTimeVisit() {
+ cy.window().then((win) => {
+ win.localStorage.setItem(SKIP_ONBOARDING_TOUR_KEY, "true");
+ win.localStorage.setItem(SKIP_WELCOME_MODAL_KEY, "true");
+ });
+}
Cypress.Commands.add("login", () => {
cy.request({
@@ -33,31 +41,45 @@ Cypress.Commands.add("login", () => {
password: Cypress.env("ADMIN_PASSWORD"),
},
retryOnStatusCodeFailure: true,
- }).then(() => localStorage.setItem(SKIP_ONBOARDING_TOUR_KEY, "true"));
+ }).then(() => notFirstTimeVisit());
});
Cypress.Commands.add("loginWithCredentials", (username, password) => {
cy.visit("/login");
- if ((username, password)) {
- cy.get("input[data-testid=username]").type(username);
- cy.get("input[data-testid=password]").type(password);
- } else {
- cy.get("input[data-testid=username]").type(Cypress.env("ADMIN_USERNAME"));
- cy.get("input[data-testid=password]").type(Cypress.env("ADMIN_PASSWORD"));
- }
+ cy.get("input[data-testid=username]").type(
+ username || Cypress.env("ADMIN_USERNAME"),
+ { delay: 0 },
+ );
+ cy.get("input[data-testid=password]").type(
+ password || Cypress.env("ADMIN_PASSWORD"),
+ { delay: 0 },
+ );
cy.contains("Sign In").click();
cy.get(".ant-avatar-circle").should("be.visible");
- localStorage.setItem(SKIP_ONBOARDING_TOUR_KEY, "true");
+ notFirstTimeVisit();
});
Cypress.Commands.add("visitWithLogin", (url) => {
cy.visit(url);
cy.get("input[data-testid=username]").type(Cypress.env("ADMIN_USERNAME"));
cy.get("input[data-testid=password]").type(Cypress.env("ADMIN_PASSWORD"));
- localStorage.setItem(SKIP_ONBOARDING_TOUR_KEY, "true");
+ notFirstTimeVisit();
cy.contains("Sign In").click();
});
+// Login commands for onboarding tour testing (without setting skipOnboardingTour)
+Cypress.Commands.add("loginForOnboarding", () => {
+ cy.request({
+ method: "POST",
+ url: "/logIn",
+ body: {
+ username: Cypress.env("ADMIN_USERNAME"),
+ password: Cypress.env("ADMIN_PASSWORD"),
+ },
+ retryOnStatusCodeFailure: true,
+ });
+});
+
Cypress.Commands.add("deleteUrn", (urn) => {
cy.request({
method: "POST",
diff --git a/smoke-test/tests/cypress/cypress/support/e2e.js b/smoke-test/tests/cypress/cypress/support/e2e.js
index a0acff87361440..db66eb9f08ef6e 100644
--- a/smoke-test/tests/cypress/cypress/support/e2e.js
+++ b/smoke-test/tests/cypress/cypress/support/e2e.js
@@ -16,6 +16,9 @@
// Import commands.js using ES2015 syntax:
import "./commands";
+// Import Testing Library commands
+import "@testing-library/cypress/add-commands";
+
// Alternatively you can use CommonJS syntax:
// require('./commands')
diff --git a/smoke-test/tests/cypress/package.json b/smoke-test/tests/cypress/package.json
index 66301bf20e83cd..674e2710515f87 100644
--- a/smoke-test/tests/cypress/package.json
+++ b/smoke-test/tests/cypress/package.json
@@ -12,7 +12,8 @@
"dependencies": {
"cypress": "^14.5.1",
"cypress-timestamps": "^1.2.0",
- "dayjs": "^1.11.7"
+ "dayjs": "^1.11.7",
+ "@testing-library/cypress": "^10.0.1"
},
"devDependencies": {
"cypress-junit-reporter": "^1.3.1",
diff --git a/smoke-test/tests/cypress/yarn.lock b/smoke-test/tests/cypress/yarn.lock
index c16eb2589dee31..53fc0eabeddc4a 100644
--- a/smoke-test/tests/cypress/yarn.lock
+++ b/smoke-test/tests/cypress/yarn.lock
@@ -7,6 +7,25 @@
resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf"
integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==
+"@babel/code-frame@^7.10.4":
+ version "7.27.1"
+ resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.27.1.tgz#200f715e66d52a23b221a9435534a91cc13ad5be"
+ integrity sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==
+ dependencies:
+ "@babel/helper-validator-identifier" "^7.27.1"
+ js-tokens "^4.0.0"
+ picocolors "^1.1.1"
+
+"@babel/helper-validator-identifier@^7.27.1":
+ version "7.27.1"
+ resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz#a7054dcc145a967dd4dc8fee845a57c1316c9df8"
+ integrity sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==
+
+"@babel/runtime@^7.12.5", "@babel/runtime@^7.14.6":
+ version "7.27.6"
+ resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.27.6.tgz#ec4070a04d76bae8ddbb10770ba55714a417b7c6"
+ integrity sha512-vbavdySgbTTrmFE+EsiqUTzlOr5bzlnJtUv9PynGCAKvfQqjIXbvFdumPM/GxMDfyuGMJaJAU6TO4zc1Jf1i8Q==
+
"@cypress/request@^3.0.8":
version "3.0.8"
resolved "https://registry.yarnpkg.com/@cypress/request/-/request-3.0.8.tgz#992f1f42ba03ebb14fa5d97290abe9d015ed0815"
@@ -40,16 +59,16 @@
lodash.once "^4.1.1"
"@eslint-community/eslint-utils@^4.2.0":
- version "4.4.0"
- resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59"
- integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==
+ version "4.7.0"
+ resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.7.0.tgz#607084630c6c033992a082de6e6fbc1a8b52175a"
+ integrity sha512-dyybb3AcajC7uha6CvhdVRJqaKyn7w2YKqKyAN37NKYgZT36w+iRb0Dymmc5qEJ549c/S31cMMSFd75bteCpCw==
dependencies:
- eslint-visitor-keys "^3.3.0"
+ eslint-visitor-keys "^3.4.3"
"@eslint-community/regexpp@^4.6.1":
- version "4.10.0"
- resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.10.0.tgz#548f6de556857c8bb73bbee70c35dc82a2e74d63"
- integrity sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==
+ version "4.12.1"
+ resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.12.1.tgz#cfc6cffe39df390a3841cde2abccf92eaa7ae0e0"
+ integrity sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==
"@eslint/eslintrc@^2.1.4":
version "2.1.4"
@@ -66,17 +85,17 @@
minimatch "^3.1.2"
strip-json-comments "^3.1.1"
-"@eslint/js@8.57.0":
- version "8.57.0"
- resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.0.tgz#a5417ae8427873f1dd08b70b3574b453e67b5f7f"
- integrity sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==
+"@eslint/js@8.57.1":
+ version "8.57.1"
+ resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.1.tgz#de633db3ec2ef6a3c89e2f19038063e8a122e2c2"
+ integrity sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==
-"@humanwhocodes/config-array@^0.11.14":
- version "0.11.14"
- resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b"
- integrity sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==
+"@humanwhocodes/config-array@^0.13.0":
+ version "0.13.0"
+ resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.13.0.tgz#fb907624df3256d04b9aa2df50d7aa97ec648748"
+ integrity sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==
dependencies:
- "@humanwhocodes/object-schema" "^2.0.2"
+ "@humanwhocodes/object-schema" "^2.0.3"
debug "^4.3.1"
minimatch "^3.0.5"
@@ -85,10 +104,10 @@
resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c"
integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==
-"@humanwhocodes/object-schema@^2.0.2":
- version "2.0.2"
- resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz#d9fae00a2d5cb40f92cfe64b47ad749fbc38f917"
- integrity sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==
+"@humanwhocodes/object-schema@^2.0.3":
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz#4a2868d75d6d6963e423bcf90b7fd1be343409d3"
+ integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==
"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
@@ -111,15 +130,49 @@
"@nodelib/fs.scandir" "2.1.5"
fastq "^1.6.0"
+"@rtsao/scc@^1.1.0":
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8"
+ integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==
+
+"@testing-library/cypress@^10.0.1":
+ version "10.0.3"
+ resolved "https://registry.yarnpkg.com/@testing-library/cypress/-/cypress-10.0.3.tgz#4514b1bcace4f28acab7ba7db6b8f5c017f0bfd1"
+ integrity sha512-TeZJMCNtiS59cPWalra7LgADuufO5FtbqQBYxuAgdX6ZFAR2D9CtQwAG8VbgvFcchW3K414va/+7P4OkQ80UVg==
+ dependencies:
+ "@babel/runtime" "^7.14.6"
+ "@testing-library/dom" "^10.1.0"
+
+"@testing-library/dom@^10.1.0":
+ version "10.4.0"
+ resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-10.4.0.tgz#82a9d9462f11d240ecadbf406607c6ceeeff43a8"
+ integrity sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==
+ dependencies:
+ "@babel/code-frame" "^7.10.4"
+ "@babel/runtime" "^7.12.5"
+ "@types/aria-query" "^5.0.1"
+ aria-query "5.3.0"
+ chalk "^4.1.0"
+ dom-accessibility-api "^0.5.9"
+ lz-string "^1.5.0"
+ pretty-format "^27.0.2"
+
+"@types/aria-query@^5.0.1":
+ version "5.0.4"
+ resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.4.tgz#1a31c3d378850d2778dabb6374d036dcba4ba708"
+ integrity sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==
+
"@types/json5@^0.0.29":
version "0.0.29"
resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==
"@types/node@*":
- version "16.11.11"
- resolved "https://registry.npmjs.org/@types/node/-/node-16.11.11.tgz"
- integrity sha512-KB0sixD67CeecHC33MYn+eYARkqTheIRNuu97y2XMjR7Wu3XibO1vaY6VBV6O/a89SPI81cEUIYT87UqUWlZNw==
+ version "24.0.14"
+ resolved "https://registry.yarnpkg.com/@types/node/-/node-24.0.14.tgz#6e3d4fb6d858c48c69707394e1a0e08ce1ecc1bc"
+ integrity sha512-4zXMWD91vBLGRtHK3YbIoFMia+1nqEz72coM42C5ETjnNCa/heoj7NT1G67iAfOqMmcfhuCZ4uNpyz8EjlAejw==
+ dependencies:
+ undici-types "~7.8.0"
"@types/sinonjs__fake-timers@8.1.1":
version "8.1.1"
@@ -127,9 +180,9 @@
integrity sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g==
"@types/sizzle@^2.3.2":
- version "2.3.3"
- resolved "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.3.tgz"
- integrity sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ==
+ version "2.3.9"
+ resolved "https://registry.yarnpkg.com/@types/sizzle/-/sizzle-2.3.9.tgz#d4597dbd4618264c414d7429363e3f50acb66ea2"
+ integrity sha512-xzLEyKB50yqCUPUJkIsrVvoWNfFUbIZI+RspLWt8u+tIW/BetMBZtgV2LY/2o+tYH8dRvQ+eoPf3NdhQCcLE2w==
"@types/yauzl@^2.9.1":
version "2.9.2"
@@ -139,9 +192,9 @@
"@types/node" "*"
"@ungap/structured-clone@^1.2.0":
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406"
- integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==
+ version "1.3.0"
+ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.3.0.tgz#d06bbb384ebcf6c505fde1c3d0ed4ddffe0aaff8"
+ integrity sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==
acorn-jsx@^5.3.2:
version "5.3.2"
@@ -149,9 +202,9 @@ acorn-jsx@^5.3.2:
integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==
acorn@^8.9.0:
- version "8.11.3"
- resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a"
- integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==
+ version "8.15.0"
+ resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.15.0.tgz#a360898bc415edaac46c8241f6383975b930b816"
+ integrity sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==
aggregate-error@^3.0.0:
version "3.1.0"
@@ -200,6 +253,11 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0:
dependencies:
color-convert "^2.0.1"
+ansi-styles@^5.0.0:
+ version "5.2.0"
+ resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b"
+ integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==
+
arch@^2.2.0:
version "2.2.0"
resolved "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz"
@@ -210,24 +268,34 @@ argparse@^2.0.1:
resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38"
integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==
-array-buffer-byte-length@^1.0.1:
- version "1.0.1"
- resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz#1e5583ec16763540a27ae52eed99ff899223568f"
- integrity sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg==
+aria-query@5.3.0:
+ version "5.3.0"
+ resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-5.3.0.tgz#650c569e41ad90b51b3d7df5e5eed1c7549c103e"
+ integrity sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==
dependencies:
- call-bind "^1.0.5"
- is-array-buffer "^3.0.4"
+ dequal "^2.0.3"
-array-includes@^3.1.7:
- version "3.1.7"
- resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.7.tgz#8cd2e01b26f7a3086cbc87271593fe921c62abda"
- integrity sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ==
+array-buffer-byte-length@^1.0.1, array-buffer-byte-length@^1.0.2:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz#384d12a37295aec3769ab022ad323a18a51ccf8b"
+ integrity sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==
dependencies:
- call-bind "^1.0.2"
- define-properties "^1.2.0"
- es-abstract "^1.22.1"
- get-intrinsic "^1.2.1"
- is-string "^1.0.7"
+ call-bound "^1.0.3"
+ is-array-buffer "^3.0.5"
+
+array-includes@^3.1.9:
+ version "3.1.9"
+ resolved "https://registry.yarnpkg.com/array-includes/-/array-includes-3.1.9.tgz#1f0ccaa08e90cdbc3eb433210f903ad0f17c3f3a"
+ integrity sha512-FmeCCAenzH0KH381SPT5FZmiA/TmpndpcaShhfgEN9eCVjnFBqq3l1xrI42y8+PPLI6hypzou4GXw00WHmPBLQ==
+ dependencies:
+ call-bind "^1.0.8"
+ call-bound "^1.0.4"
+ define-properties "^1.2.1"
+ es-abstract "^1.24.0"
+ es-object-atoms "^1.1.1"
+ get-intrinsic "^1.3.0"
+ is-string "^1.1.1"
+ math-intrinsics "^1.1.0"
array.prototype.filter@^1.0.3:
version "1.0.3"
@@ -240,50 +308,51 @@ array.prototype.filter@^1.0.3:
es-array-method-boxes-properly "^1.0.0"
is-string "^1.0.7"
-array.prototype.findlastindex@^1.2.3:
- version "1.2.4"
- resolved "https://registry.yarnpkg.com/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.4.tgz#d1c50f0b3a9da191981ff8942a0aedd82794404f"
- integrity sha512-hzvSHUshSpCflDR1QMUBLHGHP1VIEBegT4pix9H/Z92Xw3ySoy6c2qh7lJWTJnRJ8JCZ9bJNCgTyYaJGcJu6xQ==
+array.prototype.findlastindex@^1.2.6:
+ version "1.2.6"
+ resolved "https://registry.yarnpkg.com/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.6.tgz#cfa1065c81dcb64e34557c9b81d012f6a421c564"
+ integrity sha512-F/TKATkzseUExPlfvmwQKGITM3DGTK+vkAsCZoDc5daVygbJBnjEUCbgkAvVFsgfXfX4YIqZ/27G3k3tdXrTxQ==
dependencies:
- call-bind "^1.0.5"
+ call-bind "^1.0.8"
+ call-bound "^1.0.4"
define-properties "^1.2.1"
- es-abstract "^1.22.3"
+ es-abstract "^1.23.9"
es-errors "^1.3.0"
- es-shim-unscopables "^1.0.2"
+ es-object-atoms "^1.1.1"
+ es-shim-unscopables "^1.1.0"
-array.prototype.flat@^1.3.2:
- version "1.3.2"
- resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz#1476217df8cff17d72ee8f3ba06738db5b387d18"
- integrity sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==
+array.prototype.flat@^1.3.3:
+ version "1.3.3"
+ resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.3.3.tgz#534aaf9e6e8dd79fb6b9a9917f839ef1ec63afe5"
+ integrity sha512-rwG/ja1neyLqCuGZ5YYrznA62D4mZXg0i1cIskIUKSiqF3Cje9/wXAls9B9s1Wa2fomMsIv8czB8jZcPmxCXFg==
dependencies:
- call-bind "^1.0.2"
- define-properties "^1.2.0"
- es-abstract "^1.22.1"
- es-shim-unscopables "^1.0.0"
+ call-bind "^1.0.8"
+ define-properties "^1.2.1"
+ es-abstract "^1.23.5"
+ es-shim-unscopables "^1.0.2"
-array.prototype.flatmap@^1.3.2:
- version "1.3.2"
- resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz#c9a7c6831db8e719d6ce639190146c24bbd3e527"
- integrity sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ==
+array.prototype.flatmap@^1.3.3:
+ version "1.3.3"
+ resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.3.3.tgz#712cc792ae70370ae40586264629e33aab5dd38b"
+ integrity sha512-Y7Wt51eKJSyi80hFrJCePGGNo5ktJCslFuboqJsbf57CCPcm5zztluPlc4/aD8sWsKvlwatezpV4U1efk8kpjg==
dependencies:
- call-bind "^1.0.2"
- define-properties "^1.2.0"
- es-abstract "^1.22.1"
- es-shim-unscopables "^1.0.0"
+ call-bind "^1.0.8"
+ define-properties "^1.2.1"
+ es-abstract "^1.23.5"
+ es-shim-unscopables "^1.0.2"
-arraybuffer.prototype.slice@^1.0.3:
- version "1.0.3"
- resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.3.tgz#097972f4255e41bc3425e37dc3f6421cf9aefde6"
- integrity sha512-bMxMKAjg13EBSVscxTaYA4mRc5t1UAXa2kXiGTNfZ079HIWXEkKmkgFrh/nJqamaLSrXO5H4WFFkPEaLJWbs3A==
+arraybuffer.prototype.slice@^1.0.3, arraybuffer.prototype.slice@^1.0.4:
+ version "1.0.4"
+ resolved "https://registry.yarnpkg.com/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.4.tgz#9d760d84dbdd06d0cbf92c8849615a1a7ab3183c"
+ integrity sha512-BNoCY6SXXPQ7gF2opIP4GBE+Xw7U+pHMYKuzjgCN3GwiaIR09UUeKfheyIry77QtrCBlC0KK0q5/TER/tYh3PQ==
dependencies:
array-buffer-byte-length "^1.0.1"
- call-bind "^1.0.5"
+ call-bind "^1.0.8"
define-properties "^1.2.1"
- es-abstract "^1.22.3"
- es-errors "^1.2.1"
- get-intrinsic "^1.2.3"
+ es-abstract "^1.23.5"
+ es-errors "^1.3.0"
+ get-intrinsic "^1.2.6"
is-array-buffer "^3.0.4"
- is-shared-array-buffer "^1.0.2"
asn1@~0.2.3:
version "0.2.6"
@@ -302,10 +371,15 @@ astral-regex@^2.0.0:
resolved "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz"
integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==
+async-function@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/async-function/-/async-function-1.0.0.tgz#509c9fca60eaf85034c6829838188e4e4c8ffb2b"
+ integrity sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==
+
async@^3.2.0:
- version "3.2.2"
- resolved "https://registry.npmjs.org/async/-/async-3.2.2.tgz"
- integrity sha512-H0E+qZaDEfx/FY4t7iLRv1W2fFI6+pyCeTw1uN20AQPiwqwM6ojPxHxdLv4z8hi2DtnW9BOckSspLucW7pIE5g==
+ version "3.2.6"
+ resolved "https://registry.yarnpkg.com/async/-/async-3.2.6.tgz#1b0728e14929d51b85b449b7f06e27c1145e38ce"
+ integrity sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==
asynckit@^0.4.0:
version "0.4.0"
@@ -330,9 +404,9 @@ aws-sign2@~0.7.0:
integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=
aws4@^1.8.0:
- version "1.11.0"
- resolved "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz"
- integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==
+ version "1.13.2"
+ resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.13.2.tgz#0aa167216965ac9474ccfa83892cfb6b3e1e52ef"
+ integrity sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==
balanced-match@^1.0.0:
version "1.0.2"
@@ -387,7 +461,7 @@ cachedir@^2.3.0:
resolved "https://registry.npmjs.org/cachedir/-/cachedir-2.3.0.tgz"
integrity sha512-A+Fezp4zxnit6FanDmv9EqXNAi3vt9DWp51/71UEhXukb7QUuvtv9344h91dyAxuTLoSYJFU299qzR3tzwPAhw==
-call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2:
+call-bind-apply-helpers@^1.0.0, call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz#4b5428c222be985d79c3d82657479dbe0b59b2d6"
integrity sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==
@@ -395,18 +469,17 @@ call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2:
es-errors "^1.3.0"
function-bind "^1.1.2"
-call-bind@^1.0.2, call-bind@^1.0.5, call-bind@^1.0.6, call-bind@^1.0.7:
- version "1.0.7"
- resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.7.tgz#06016599c40c56498c18769d2730be242b6fa3b9"
- integrity sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==
+call-bind@^1.0.2, call-bind@^1.0.5, call-bind@^1.0.7, call-bind@^1.0.8:
+ version "1.0.8"
+ resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.8.tgz#0736a9660f537e3388826f440d5ec45f744eaa4c"
+ integrity sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==
dependencies:
+ call-bind-apply-helpers "^1.0.0"
es-define-property "^1.0.0"
- es-errors "^1.3.0"
- function-bind "^1.1.2"
get-intrinsic "^1.2.4"
- set-function-length "^1.2.1"
+ set-function-length "^1.2.2"
-call-bound@^1.0.2:
+call-bound@^1.0.2, call-bound@^1.0.3, call-bound@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/call-bound/-/call-bound-1.0.4.tgz#238de935d2a2a692928c538c7ccfa91067fd062a"
integrity sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==
@@ -563,9 +636,9 @@ cypress-timestamps@^1.2.0:
format-duration "^2.0.0"
cypress@^14.5.1:
- version "14.5.1"
- resolved "https://registry.yarnpkg.com/cypress/-/cypress-14.5.1.tgz#0af3f2ce7beb82f8d88a8a3cb7d8e40326114ce2"
- integrity sha512-vYBeZKW3UAtxwv5mFuSlOBCYhyO0H86TeDKRJ7TgARyHiREIaiDjeHtqjzrXRFrdz9KnNavqlm+z+hklC7v8XQ==
+ version "14.5.2"
+ resolved "https://registry.yarnpkg.com/cypress/-/cypress-14.5.2.tgz#b45563bf9a96b815ab6e5d028b49ce0b0fe80cb2"
+ integrity sha512-O4E4CEBqDHLDrJD/dfStHPcM+8qFgVVZ89Li7xDU0yL/JxO/V0PEcfF2I8aGa7uA2MGNLkNUAnghPM83UcHOJw==
dependencies:
"@cypress/request" "^3.0.8"
"@cypress/xvfb" "^1.2.4"
@@ -619,15 +692,37 @@ dashdash@^1.12.0:
dependencies:
assert-plus "^1.0.0"
-dayjs@^1.10.4:
- version "1.10.7"
- resolved "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz"
- integrity sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig==
+data-view-buffer@^1.0.2:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/data-view-buffer/-/data-view-buffer-1.0.2.tgz#211a03ba95ecaf7798a8c7198d79536211f88570"
+ integrity sha512-EmKO5V3OLXh1rtK2wgXRansaK1/mtVdTUEiEI0W8RkvgT05kfxaH29PliLnpLP73yYO6142Q72QNa8Wx/A5CqQ==
+ dependencies:
+ call-bound "^1.0.3"
+ es-errors "^1.3.0"
+ is-data-view "^1.0.2"
+
+data-view-byte-length@^1.0.2:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/data-view-byte-length/-/data-view-byte-length-1.0.2.tgz#9e80f7ca52453ce3e93d25a35318767ea7704735"
+ integrity sha512-tuhGbE6CfTM9+5ANGf+oQb72Ky/0+s3xKUpHvShfiz2RxMFgFPjsXuRLBVMtvMs15awe45SRb83D6wH4ew6wlQ==
+ dependencies:
+ call-bound "^1.0.3"
+ es-errors "^1.3.0"
+ is-data-view "^1.0.2"
+
+data-view-byte-offset@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/data-view-byte-offset/-/data-view-byte-offset-1.0.1.tgz#068307f9b71ab76dbbe10291389e020856606191"
+ integrity sha512-BS8PfmtDGnrgYdOonGZQdLZslWIeCGFP9tpan0hi1Co2Zr2NKADsvGYA8XxuG/4UWgJ6Cjtv+YJnB6MM69QGlQ==
+ dependencies:
+ call-bound "^1.0.2"
+ es-errors "^1.3.0"
+ is-data-view "^1.0.1"
-dayjs@^1.11.7:
- version "1.11.7"
- resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2"
- integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==
+dayjs@^1.10.4, dayjs@^1.11.7:
+ version "1.11.13"
+ resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.13.tgz#92430b0139055c3ebb60150aa13e860a4b5a366c"
+ integrity sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==
debug@^2.2.0:
version "2.6.9"
@@ -643,21 +738,7 @@ debug@^3.1.0, debug@^3.2.7:
dependencies:
ms "^2.1.1"
-debug@^4.1.1, debug@^4.3.2:
- version "4.3.3"
- resolved "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz"
- integrity sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==
- dependencies:
- ms "2.1.2"
-
-debug@^4.3.1:
- version "4.3.4"
- resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
- integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
- dependencies:
- ms "2.1.2"
-
-debug@^4.3.4:
+debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4:
version "4.4.1"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.1.tgz#e5a8bc6cbc4c6cd3e64308b0693a3d4fa550189b"
integrity sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==
@@ -692,6 +773,11 @@ delayed-stream@~1.0.0:
resolved "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz"
integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk=
+dequal@^2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/dequal/-/dequal-2.0.3.tgz#2644214f1997d39ed0ee0ece72335490a7ac67be"
+ integrity sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==
+
doctrine@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d"
@@ -706,7 +792,12 @@ doctrine@^3.0.0:
dependencies:
esutils "^2.0.2"
-dunder-proto@^1.0.1:
+dom-accessibility-api@^0.5.9:
+ version "0.5.16"
+ resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz#5a7429e6066eb3664d911e33fb0e45de8eb08453"
+ integrity sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==
+
+dunder-proto@^1.0.0, dunder-proto@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a"
integrity sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==
@@ -729,9 +820,9 @@ emoji-regex@^8.0.0:
integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
end-of-stream@^1.1.0:
- version "1.4.4"
- resolved "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz"
- integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
+ version "1.4.5"
+ resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.5.tgz#7344d711dea40e0b74abc2ed49778743ccedb08c"
+ integrity sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==
dependencies:
once "^1.4.0"
@@ -789,36 +880,82 @@ es-abstract@^1.22.1, es-abstract@^1.22.3:
unbox-primitive "^1.0.2"
which-typed-array "^1.1.14"
+es-abstract@^1.23.5, es-abstract@^1.23.9, es-abstract@^1.24.0:
+ version "1.24.0"
+ resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.24.0.tgz#c44732d2beb0acc1ed60df840869e3106e7af328"
+ integrity sha512-WSzPgsdLtTcQwm4CROfS5ju2Wa1QQcVeT37jFjYzdFz1r9ahadC8B8/a4qxJxM+09F18iumCdRmlr96ZYkQvEg==
+ dependencies:
+ array-buffer-byte-length "^1.0.2"
+ arraybuffer.prototype.slice "^1.0.4"
+ available-typed-arrays "^1.0.7"
+ call-bind "^1.0.8"
+ call-bound "^1.0.4"
+ data-view-buffer "^1.0.2"
+ data-view-byte-length "^1.0.2"
+ data-view-byte-offset "^1.0.1"
+ es-define-property "^1.0.1"
+ es-errors "^1.3.0"
+ es-object-atoms "^1.1.1"
+ es-set-tostringtag "^2.1.0"
+ es-to-primitive "^1.3.0"
+ function.prototype.name "^1.1.8"
+ get-intrinsic "^1.3.0"
+ get-proto "^1.0.1"
+ get-symbol-description "^1.1.0"
+ globalthis "^1.0.4"
+ gopd "^1.2.0"
+ has-property-descriptors "^1.0.2"
+ has-proto "^1.2.0"
+ has-symbols "^1.1.0"
+ hasown "^2.0.2"
+ internal-slot "^1.1.0"
+ is-array-buffer "^3.0.5"
+ is-callable "^1.2.7"
+ is-data-view "^1.0.2"
+ is-negative-zero "^2.0.3"
+ is-regex "^1.2.1"
+ is-set "^2.0.3"
+ is-shared-array-buffer "^1.0.4"
+ is-string "^1.1.1"
+ is-typed-array "^1.1.15"
+ is-weakref "^1.1.1"
+ math-intrinsics "^1.1.0"
+ object-inspect "^1.13.4"
+ object-keys "^1.1.1"
+ object.assign "^4.1.7"
+ own-keys "^1.0.1"
+ regexp.prototype.flags "^1.5.4"
+ safe-array-concat "^1.1.3"
+ safe-push-apply "^1.0.0"
+ safe-regex-test "^1.1.0"
+ set-proto "^1.0.0"
+ stop-iteration-iterator "^1.1.0"
+ string.prototype.trim "^1.2.10"
+ string.prototype.trimend "^1.0.9"
+ string.prototype.trimstart "^1.0.8"
+ typed-array-buffer "^1.0.3"
+ typed-array-byte-length "^1.0.3"
+ typed-array-byte-offset "^1.0.4"
+ typed-array-length "^1.0.7"
+ unbox-primitive "^1.1.0"
+ which-typed-array "^1.1.19"
+
es-array-method-boxes-properly@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/es-array-method-boxes-properly/-/es-array-method-boxes-properly-1.0.0.tgz#873f3e84418de4ee19c5be752990b2e44718d09e"
integrity sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==
-es-define-property@^1.0.0:
- version "1.0.0"
- resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.0.tgz#c7faefbdff8b2696cf5f46921edfb77cc4ba3845"
- integrity sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==
- dependencies:
- get-intrinsic "^1.2.4"
-
-es-define-property@^1.0.1:
+es-define-property@^1.0.0, es-define-property@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa"
integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==
-es-errors@^1.0.0, es-errors@^1.2.1, es-errors@^1.3.0:
+es-errors@^1.0.0, es-errors@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f"
integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==
-es-object-atoms@^1.0.0:
- version "1.0.0"
- resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.0.0.tgz#ddb55cd47ac2e240701260bc2a8e31ecb643d941"
- integrity sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==
- dependencies:
- es-errors "^1.3.0"
-
-es-object-atoms@^1.1.1:
+es-object-atoms@^1.0.0, es-object-atoms@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz#1c4f2c4837327597ce69d2ca190a7fdd172338c1"
integrity sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==
@@ -844,21 +981,21 @@ es-set-tostringtag@^2.1.0:
has-tostringtag "^1.0.2"
hasown "^2.0.2"
-es-shim-unscopables@^1.0.0, es-shim-unscopables@^1.0.2:
- version "1.0.2"
- resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz#1f6942e71ecc7835ed1c8a83006d8771a63a3763"
- integrity sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==
+es-shim-unscopables@^1.0.2, es-shim-unscopables@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.1.0.tgz#438df35520dac5d105f3943d927549ea3b00f4b5"
+ integrity sha512-d9T8ucsEhh8Bi1woXCf+TIKDIROLG5WCkxg8geBCbvk22kzwC5G2OnXVMO6FUsvQlgUUXQ2itephWDLqDzbeCw==
dependencies:
- hasown "^2.0.0"
+ hasown "^2.0.2"
-es-to-primitive@^1.2.1:
- version "1.2.1"
- resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a"
- integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==
+es-to-primitive@^1.2.1, es-to-primitive@^1.3.0:
+ version "1.3.0"
+ resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.3.0.tgz#96c89c82cc49fd8794a24835ba3e1ff87f214e18"
+ integrity sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==
dependencies:
- is-callable "^1.1.4"
- is-date-object "^1.0.1"
- is-symbol "^1.0.2"
+ is-callable "^1.2.7"
+ is-date-object "^1.0.5"
+ is-symbol "^1.0.4"
escape-string-regexp@^1.0.5:
version "1.0.5"
@@ -894,10 +1031,10 @@ eslint-import-resolver-node@^0.3.9:
is-core-module "^2.13.0"
resolve "^1.22.4"
-eslint-module-utils@^2.8.0:
- version "2.8.0"
- resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.8.0.tgz#e439fee65fc33f6bba630ff621efc38ec0375c49"
- integrity sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw==
+eslint-module-utils@^2.12.1:
+ version "2.12.1"
+ resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.12.1.tgz#f76d3220bfb83c057651359295ab5854eaad75ff"
+ integrity sha512-L8jSWTze7K2mTg0vos/RuLRS5soomksDPoJLXIslC7c8Wmut3bx7CPpJijDcBZtxQ5lrbUdM+s0OlNbz0DCDNw==
dependencies:
debug "^3.2.7"
@@ -909,26 +1046,28 @@ eslint-plugin-cypress@^2.15.1:
globals "^13.20.0"
eslint-plugin-import@^2.25.2:
- version "2.29.1"
- resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz#d45b37b5ef5901d639c15270d74d46d161150643"
- integrity sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==
- dependencies:
- array-includes "^3.1.7"
- array.prototype.findlastindex "^1.2.3"
- array.prototype.flat "^1.3.2"
- array.prototype.flatmap "^1.3.2"
+ version "2.32.0"
+ resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.32.0.tgz#602b55faa6e4caeaa5e970c198b5c00a37708980"
+ integrity sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==
+ dependencies:
+ "@rtsao/scc" "^1.1.0"
+ array-includes "^3.1.9"
+ array.prototype.findlastindex "^1.2.6"
+ array.prototype.flat "^1.3.3"
+ array.prototype.flatmap "^1.3.3"
debug "^3.2.7"
doctrine "^2.1.0"
eslint-import-resolver-node "^0.3.9"
- eslint-module-utils "^2.8.0"
- hasown "^2.0.0"
- is-core-module "^2.13.1"
+ eslint-module-utils "^2.12.1"
+ hasown "^2.0.2"
+ is-core-module "^2.16.1"
is-glob "^4.0.3"
minimatch "^3.1.2"
- object.fromentries "^2.0.7"
- object.groupby "^1.0.1"
- object.values "^1.1.7"
+ object.fromentries "^2.0.8"
+ object.groupby "^1.0.3"
+ object.values "^1.2.1"
semver "^6.3.1"
+ string.prototype.trimend "^1.0.9"
tsconfig-paths "^3.15.0"
eslint-scope@^7.2.2:
@@ -939,21 +1078,21 @@ eslint-scope@^7.2.2:
esrecurse "^4.3.0"
estraverse "^5.2.0"
-eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4.3:
+eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4.3:
version "3.4.3"
resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800"
integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==
"eslint@^7.32.0 || ^8.2.0":
- version "8.57.0"
- resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.57.0.tgz#c786a6fd0e0b68941aaf624596fb987089195668"
- integrity sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==
+ version "8.57.1"
+ resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.57.1.tgz#7df109654aba7e3bbe5c8eae533c5e461d3c6ca9"
+ integrity sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==
dependencies:
"@eslint-community/eslint-utils" "^4.2.0"
"@eslint-community/regexpp" "^4.6.1"
"@eslint/eslintrc" "^2.1.4"
- "@eslint/js" "8.57.0"
- "@humanwhocodes/config-array" "^0.11.14"
+ "@eslint/js" "8.57.1"
+ "@humanwhocodes/config-array" "^0.13.0"
"@humanwhocodes/module-importer" "^1.0.1"
"@nodelib/fs.walk" "^1.2.8"
"@ungap/structured-clone" "^1.2.0"
@@ -1090,9 +1229,9 @@ fast-levenshtein@^2.0.6:
integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==
fastq@^1.6.0:
- version "1.17.1"
- resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.17.1.tgz#2a523f07a4e7b1e81a42b91b8bf2254107753b47"
- integrity sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==
+ version "1.19.1"
+ resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.19.1.tgz#d50eaba803c8846a883c16492821ebcd2cda55f5"
+ integrity sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==
dependencies:
reusify "^1.0.4"
@@ -1135,16 +1274,16 @@ flat-cache@^3.0.4:
rimraf "^3.0.2"
flatted@^3.2.9:
- version "3.3.1"
- resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a"
- integrity sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==
+ version "3.3.3"
+ resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.3.tgz#67c8fad95454a7c7abebf74bb78ee74a44023358"
+ integrity sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==
-for-each@^0.3.3:
- version "0.3.3"
- resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e"
- integrity sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==
+for-each@^0.3.3, for-each@^0.3.5:
+ version "0.3.5"
+ resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.5.tgz#d650688027826920feeb0af747ee7b9421a41d47"
+ integrity sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==
dependencies:
- is-callable "^1.1.3"
+ is-callable "^1.2.7"
forever-agent@~0.6.1:
version "0.6.1"
@@ -1187,22 +1326,24 @@ function-bind@^1.1.2:
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c"
integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==
-function.prototype.name@^1.1.6:
- version "1.1.6"
- resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.6.tgz#cdf315b7d90ee77a4c6ee216c3c3362da07533fd"
- integrity sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==
+function.prototype.name@^1.1.6, function.prototype.name@^1.1.8:
+ version "1.1.8"
+ resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.8.tgz#e68e1df7b259a5c949eeef95cdbde53edffabb78"
+ integrity sha512-e5iwyodOHhbMr/yNrc7fDYG4qlbIvI5gajyzPnb5TCwyhjApznQh1BMFou9b30SevY43gCJKXycoCBjMbsuW0Q==
dependencies:
- call-bind "^1.0.2"
- define-properties "^1.2.0"
- es-abstract "^1.22.1"
+ call-bind "^1.0.8"
+ call-bound "^1.0.3"
+ define-properties "^1.2.1"
functions-have-names "^1.2.3"
+ hasown "^2.0.2"
+ is-callable "^1.2.7"
functions-have-names@^1.2.3:
version "1.2.3"
resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834"
integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==
-get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2, get-intrinsic@^1.2.3, get-intrinsic@^1.2.4:
+get-intrinsic@^1.2.2, get-intrinsic@^1.2.3:
version "1.2.4"
resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd"
integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==
@@ -1213,7 +1354,7 @@ get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2, get-intrinsic@
has-symbols "^1.0.3"
hasown "^2.0.0"
-get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.3.0:
+get-intrinsic@^1.2.4, get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.2.7, get-intrinsic@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01"
integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==
@@ -1229,7 +1370,7 @@ get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.3.0:
hasown "^2.0.2"
math-intrinsics "^1.1.0"
-get-proto@^1.0.1:
+get-proto@^1.0.0, get-proto@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/get-proto/-/get-proto-1.0.1.tgz#150b3f2743869ef3e851ec0c49d15b1d14d00ee1"
integrity sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==
@@ -1244,14 +1385,14 @@ get-stream@^5.0.0, get-stream@^5.1.0:
dependencies:
pump "^3.0.0"
-get-symbol-description@^1.0.2:
- version "1.0.2"
- resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.2.tgz#533744d5aa20aca4e079c8e5daf7fd44202821f5"
- integrity sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg==
+get-symbol-description@^1.0.2, get-symbol-description@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.1.0.tgz#7bdd54e0befe8ffc9f3b4e203220d9f1e881b6ee"
+ integrity sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==
dependencies:
- call-bind "^1.0.5"
+ call-bound "^1.0.3"
es-errors "^1.3.0"
- get-intrinsic "^1.2.4"
+ get-intrinsic "^1.2.6"
getos@^3.2.1:
version "3.2.1"
@@ -1307,14 +1448,15 @@ globalthis@^1.0.3:
dependencies:
define-properties "^1.1.3"
-gopd@^1.0.1:
- version "1.0.1"
- resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c"
- integrity sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==
+globalthis@^1.0.4:
+ version "1.0.4"
+ resolved "https://registry.yarnpkg.com/globalthis/-/globalthis-1.0.4.tgz#7430ed3a975d97bfb59bcce41f5cabbafa651236"
+ integrity sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==
dependencies:
- get-intrinsic "^1.1.3"
+ define-properties "^1.2.1"
+ gopd "^1.0.1"
-gopd@^1.2.0:
+gopd@^1.0.1, gopd@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1"
integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==
@@ -1329,10 +1471,10 @@ graphemer@^1.4.0:
resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6"
integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==
-has-bigints@^1.0.1, has-bigints@^1.0.2:
- version "1.0.2"
- resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa"
- integrity sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==
+has-bigints@^1.0.2:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.1.0.tgz#28607e965ac967e03cd2a2c70a2636a1edad49fe"
+ integrity sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==
has-flag@^4.0.0:
version "4.0.0"
@@ -1346,22 +1488,19 @@ has-property-descriptors@^1.0.0, has-property-descriptors@^1.0.1, has-property-d
dependencies:
es-define-property "^1.0.0"
-has-proto@^1.0.1, has-proto@^1.0.3:
- version "1.0.3"
- resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.3.tgz#b31ddfe9b0e6e9914536a6ab286426d0214f77fd"
- integrity sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==
-
-has-symbols@^1.0.2, has-symbols@^1.0.3:
- version "1.0.3"
- resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8"
- integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==
+has-proto@^1.0.1, has-proto@^1.2.0:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.2.0.tgz#5de5a6eabd95fdffd9818b43055e8065e39fe9d5"
+ integrity sha512-KIL7eQPfHQRC8+XluaIw7BHUwwqL19bQn4hzNgdr+1wXoU0KKj6rufu47lhY7KbJR2C6T6+PfyN0Ea7wkSS+qQ==
+ dependencies:
+ dunder-proto "^1.0.0"
-has-symbols@^1.1.0:
+has-symbols@^1.0.3, has-symbols@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338"
integrity sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==
-has-tostringtag@^1.0.0, has-tostringtag@^1.0.1, has-tostringtag@^1.0.2:
+has-tostringtag@^1.0.1, has-tostringtag@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc"
integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==
@@ -1408,9 +1547,9 @@ ignore@^5.2.0:
integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==
import-fresh@^3.2.1:
- version "3.3.0"
- resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b"
- integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==
+ version "3.3.1"
+ resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.1.tgz#9cecb56503c0ada1f2741dbbd6546e4b13b57ccf"
+ integrity sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==
dependencies:
parent-module "^1.0.0"
resolve-from "^4.0.0"
@@ -1443,72 +1582,111 @@ ini@2.0.0:
resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz"
integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==
-internal-slot@^1.0.7:
- version "1.0.7"
- resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.7.tgz#c06dcca3ed874249881007b0a5523b172a190802"
- integrity sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g==
+internal-slot@^1.0.7, internal-slot@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.1.0.tgz#1eac91762947d2f7056bc838d93e13b2e9604961"
+ integrity sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==
dependencies:
es-errors "^1.3.0"
- hasown "^2.0.0"
- side-channel "^1.0.4"
+ hasown "^2.0.2"
+ side-channel "^1.1.0"
-is-array-buffer@^3.0.4:
- version "3.0.4"
- resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.4.tgz#7a1f92b3d61edd2bc65d24f130530ea93d7fae98"
- integrity sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw==
+is-array-buffer@^3.0.4, is-array-buffer@^3.0.5:
+ version "3.0.5"
+ resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.5.tgz#65742e1e687bd2cc666253068fd8707fe4d44280"
+ integrity sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==
dependencies:
- call-bind "^1.0.2"
- get-intrinsic "^1.2.1"
+ call-bind "^1.0.8"
+ call-bound "^1.0.3"
+ get-intrinsic "^1.2.6"
-is-bigint@^1.0.1:
- version "1.0.4"
- resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.4.tgz#08147a1875bc2b32005d41ccd8291dffc6691df3"
- integrity sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==
+is-async-function@^2.0.0:
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/is-async-function/-/is-async-function-2.1.1.tgz#3e69018c8e04e73b738793d020bfe884b9fd3523"
+ integrity sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ==
dependencies:
- has-bigints "^1.0.1"
+ async-function "^1.0.0"
+ call-bound "^1.0.3"
+ get-proto "^1.0.1"
+ has-tostringtag "^1.0.2"
+ safe-regex-test "^1.1.0"
-is-boolean-object@^1.1.0:
- version "1.1.2"
- resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.1.2.tgz#5c6dc200246dd9321ae4b885a114bb1f75f63719"
- integrity sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==
+is-bigint@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.1.0.tgz#dda7a3445df57a42583db4228682eba7c4170672"
+ integrity sha512-n4ZT37wG78iz03xPRKJrHTdZbe3IicyucEtdRsV5yglwc3GyUfbAfpSeD0FJ41NbUNSt5wbhqfp1fS+BgnvDFQ==
dependencies:
- call-bind "^1.0.2"
- has-tostringtag "^1.0.0"
+ has-bigints "^1.0.2"
+
+is-boolean-object@^1.2.1:
+ version "1.2.2"
+ resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.2.2.tgz#7067f47709809a393c71ff5bb3e135d8a9215d9e"
+ integrity sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A==
+ dependencies:
+ call-bound "^1.0.3"
+ has-tostringtag "^1.0.2"
is-buffer@~1.1.6:
version "1.1.6"
resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
-is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7:
+is-callable@^1.2.7:
version "1.2.7"
resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055"
integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==
-is-core-module@^2.13.0, is-core-module@^2.13.1:
- version "2.13.1"
- resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384"
- integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==
+is-core-module@^2.13.0, is-core-module@^2.16.0, is-core-module@^2.16.1:
+ version "2.16.1"
+ resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.16.1.tgz#2a98801a849f43e2add644fbb6bc6229b19a4ef4"
+ integrity sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==
dependencies:
- hasown "^2.0.0"
+ hasown "^2.0.2"
-is-date-object@^1.0.1:
- version "1.0.5"
- resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.5.tgz#0841d5536e724c25597bf6ea62e1bd38298df31f"
- integrity sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==
+is-data-view@^1.0.1, is-data-view@^1.0.2:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/is-data-view/-/is-data-view-1.0.2.tgz#bae0a41b9688986c2188dda6657e56b8f9e63b8e"
+ integrity sha512-RKtWF8pGmS87i2D6gqQu/l7EYRlVdfzemCJN/P3UOs//x1QE7mfhvzHIApBTRf7axvT6DMGwSwBXYCT0nfB9xw==
dependencies:
- has-tostringtag "^1.0.0"
+ call-bound "^1.0.2"
+ get-intrinsic "^1.2.6"
+ is-typed-array "^1.1.13"
+
+is-date-object@^1.0.5, is-date-object@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.1.0.tgz#ad85541996fc7aa8b2729701d27b7319f95d82f7"
+ integrity sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg==
+ dependencies:
+ call-bound "^1.0.2"
+ has-tostringtag "^1.0.2"
is-extglob@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==
+is-finalizationregistry@^1.1.0:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/is-finalizationregistry/-/is-finalizationregistry-1.1.1.tgz#eefdcdc6c94ddd0674d9c85887bf93f944a97c90"
+ integrity sha512-1pC6N8qWJbWoPtEjgcL2xyhQOP491EQjeUo3qTKcmV8YSDDJrOepfG8pcC7h/QgnQHYSv0mJ3Z/ZWxmatVrysg==
+ dependencies:
+ call-bound "^1.0.3"
+
is-fullwidth-code-point@^3.0.0:
version "3.0.0"
resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz"
integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==
+is-generator-function@^1.0.10:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/is-generator-function/-/is-generator-function-1.1.0.tgz#bf3eeda931201394f57b5dba2800f91a238309ca"
+ integrity sha512-nPUB5km40q9e8UfN/Zc24eLlzdSf9OfKByBw9CIdw4H1giPMeA0OIJvbchsCu4npfI2QcMVBsGEBHKZ7wLTWmQ==
+ dependencies:
+ call-bound "^1.0.3"
+ get-proto "^1.0.0"
+ has-tostringtag "^1.0.2"
+ safe-regex-test "^1.1.0"
+
is-glob@^4.0.0, is-glob@^4.0.3:
version "4.0.3"
resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084"
@@ -1524,30 +1702,43 @@ is-installed-globally@~0.4.0:
global-dirs "^3.0.0"
is-path-inside "^3.0.2"
-is-negative-zero@^2.0.2:
+is-map@^2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.3.tgz#ede96b7fe1e270b3c4465e3a465658764926d62e"
+ integrity sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw==
+
+is-negative-zero@^2.0.2, is-negative-zero@^2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.3.tgz#ced903a027aca6381b777a5743069d7376a49747"
integrity sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw==
-is-number-object@^1.0.4:
- version "1.0.7"
- resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.7.tgz#59d50ada4c45251784e9904f5246c742f07a42fc"
- integrity sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==
+is-number-object@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.1.1.tgz#144b21e95a1bc148205dcc2814a9134ec41b2541"
+ integrity sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw==
dependencies:
- has-tostringtag "^1.0.0"
+ call-bound "^1.0.3"
+ has-tostringtag "^1.0.2"
is-path-inside@^3.0.2, is-path-inside@^3.0.3:
version "3.0.3"
resolved "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz"
integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==
-is-regex@^1.1.4:
- version "1.1.4"
- resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.4.tgz#eef5663cd59fa4c0ae339505323df6854bb15958"
- integrity sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==
+is-regex@^1.1.4, is-regex@^1.2.1:
+ version "1.2.1"
+ resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.2.1.tgz#76d70a3ed10ef9be48eb577887d74205bf0cad22"
+ integrity sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==
dependencies:
- call-bind "^1.0.2"
- has-tostringtag "^1.0.0"
+ call-bound "^1.0.2"
+ gopd "^1.2.0"
+ has-tostringtag "^1.0.2"
+ hasown "^2.0.2"
+
+is-set@^2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.3.tgz#8ab209ea424608141372ded6e0cb200ef1d9d01d"
+ integrity sha512-iPAjerrse27/ygGLxw+EBR9agv9Y6uLeYVJMu+QNCoouJ1/1ri0mGrcWpfCqFZuzzx3WjtwxG098X+n4OuRkPg==
is-shared-array-buffer@^1.0.2:
version "1.0.3"
@@ -1556,31 +1747,41 @@ is-shared-array-buffer@^1.0.2:
dependencies:
call-bind "^1.0.7"
+is-shared-array-buffer@^1.0.4:
+ version "1.0.4"
+ resolved "https://registry.yarnpkg.com/is-shared-array-buffer/-/is-shared-array-buffer-1.0.4.tgz#9b67844bd9b7f246ba0708c3a93e34269c774f6f"
+ integrity sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==
+ dependencies:
+ call-bound "^1.0.3"
+
is-stream@^2.0.0:
version "2.0.1"
resolved "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz"
integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==
-is-string@^1.0.5, is-string@^1.0.7:
- version "1.0.7"
- resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.7.tgz#0dd12bf2006f255bb58f695110eff7491eebc0fd"
- integrity sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==
+is-string@^1.0.7, is-string@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.1.1.tgz#92ea3f3d5c5b6e039ca8677e5ac8d07ea773cbb9"
+ integrity sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==
dependencies:
- has-tostringtag "^1.0.0"
+ call-bound "^1.0.3"
+ has-tostringtag "^1.0.2"
-is-symbol@^1.0.2, is-symbol@^1.0.3:
- version "1.0.4"
- resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.4.tgz#a6dac93b635b063ca6872236de88910a57af139c"
- integrity sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==
+is-symbol@^1.0.4, is-symbol@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.1.1.tgz#f47761279f532e2b05a7024a7506dbbedacd0634"
+ integrity sha512-9gGx6GTtCQM73BgmHQXfDmLtfjjTUDSyoxTCbp5WtoixAhfgsDirWIcVQ/IHpvI5Vgd5i/J5F7B9cN/WlVbC/w==
dependencies:
- has-symbols "^1.0.2"
+ call-bound "^1.0.2"
+ has-symbols "^1.1.0"
+ safe-regex-test "^1.1.0"
-is-typed-array@^1.1.13:
- version "1.1.13"
- resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.13.tgz#d6c5ca56df62334959322d7d7dd1cca50debe229"
- integrity sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==
+is-typed-array@^1.1.13, is-typed-array@^1.1.14, is-typed-array@^1.1.15:
+ version "1.1.15"
+ resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.15.tgz#4bfb4a45b61cee83a5a46fba778e4e8d59c0ce0b"
+ integrity sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==
dependencies:
- which-typed-array "^1.1.14"
+ which-typed-array "^1.1.16"
is-typedarray@~1.0.0:
version "1.0.0"
@@ -1592,12 +1793,25 @@ is-unicode-supported@^0.1.0:
resolved "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz"
integrity sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==
-is-weakref@^1.0.2:
- version "1.0.2"
- resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.0.2.tgz#9529f383a9338205e89765e0392efc2f100f06f2"
- integrity sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==
+is-weakmap@^2.0.2:
+ version "2.0.2"
+ resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.2.tgz#bf72615d649dfe5f699079c54b83e47d1ae19cfd"
+ integrity sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w==
+
+is-weakref@^1.0.2, is-weakref@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/is-weakref/-/is-weakref-1.1.1.tgz#eea430182be8d64174bd96bffbc46f21bf3f9293"
+ integrity sha512-6i9mGWSlqzNMEqpCp93KwRS1uUOodk2OJ6b+sq7ZPDSy2WuI5NFIxp/254TytR8ftefexkWn5xNiHUNpPOfSew==
dependencies:
- call-bind "^1.0.2"
+ call-bound "^1.0.3"
+
+is-weakset@^2.0.3:
+ version "2.0.4"
+ resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.4.tgz#c9f5deb0bc1906c6d6f1027f284ddf459249daca"
+ integrity sha512-mfcwb6IzQyOKTs84CQMrOwW4gQcaTOAWJ0zzJCl2WSPDrWk/OzDaImWFH3djXhb24g4eudZfLRozAvPGw4d9hQ==
+ dependencies:
+ call-bound "^1.0.3"
+ get-intrinsic "^1.2.6"
isarray@^2.0.5:
version "2.0.5"
@@ -1614,6 +1828,11 @@ isstream@~0.1.2:
resolved "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=
+js-tokens@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
+ integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==
+
js-yaml@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602"
@@ -1751,6 +1970,11 @@ log-update@^4.0.0:
slice-ansi "^4.0.0"
wrap-ansi "^6.2.0"
+lz-string@^1.5.0:
+ version "1.5.0"
+ resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.5.0.tgz#c1ab50f77887b712621201ba9fd4e3a6ed099941"
+ integrity sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==
+
math-intrinsics@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9"
@@ -1816,11 +2040,6 @@ ms@2.0.0:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==
-ms@2.1.2:
- version "2.1.2"
- resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz"
- integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
-
ms@^2.1.1, ms@^2.1.3:
version "2.1.3"
resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz"
@@ -1843,7 +2062,7 @@ object-inspect@^1.13.1:
resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.1.tgz#b96c6109324ccfef6b12216a956ca4dc2ff94bc2"
integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==
-object-inspect@^1.13.3:
+object-inspect@^1.13.3, object-inspect@^1.13.4:
version "1.13.4"
resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.4.tgz#8375265e21bc20d0fa582c22e1b13485d6e00213"
integrity sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==
@@ -1853,53 +2072,56 @@ object-keys@^1.1.1:
resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e"
integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==
-object.assign@^4.1.2, object.assign@^4.1.5:
- version "4.1.5"
- resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.5.tgz#3a833f9ab7fdb80fc9e8d2300c803d216d8fdbb0"
- integrity sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==
+object.assign@^4.1.2, object.assign@^4.1.5, object.assign@^4.1.7:
+ version "4.1.7"
+ resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.7.tgz#8c14ca1a424c6a561b0bb2a22f66f5049a945d3d"
+ integrity sha512-nK28WOo+QIjBkDduTINE4JkF/UJJKyf2EJxvJKfblDpyg0Q+pkOHNTL0Qwy6NP6FhE/EnzV73BxxqcJaXY9anw==
dependencies:
- call-bind "^1.0.5"
+ call-bind "^1.0.8"
+ call-bound "^1.0.3"
define-properties "^1.2.1"
- has-symbols "^1.0.3"
+ es-object-atoms "^1.0.0"
+ has-symbols "^1.1.0"
object-keys "^1.1.1"
object.entries@^1.1.5:
- version "1.1.7"
- resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.7.tgz#2b47760e2a2e3a752f39dd874655c61a7f03c131"
- integrity sha512-jCBs/0plmPsOnrKAfFQXRG2NFjlhZgjjcBLSmTnEhU8U6vVTsVe8ANeQJCHTl3gSsI4J+0emOoCgoKlmQPMgmA==
+ version "1.1.9"
+ resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.9.tgz#e4770a6a1444afb61bd39f984018b5bede25f8b3"
+ integrity sha512-8u/hfXFRBD1O0hPUjioLhoWFHRmt6tKA4/vZPyckBr18l1KE9uHrFaFaUi8MDRTpi4uak2goyPTSNJLXX2k2Hw==
dependencies:
- call-bind "^1.0.2"
- define-properties "^1.2.0"
- es-abstract "^1.22.1"
+ call-bind "^1.0.8"
+ call-bound "^1.0.4"
+ define-properties "^1.2.1"
+ es-object-atoms "^1.1.1"
-object.fromentries@^2.0.7:
- version "2.0.7"
- resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.7.tgz#71e95f441e9a0ea6baf682ecaaf37fa2a8d7e616"
- integrity sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA==
+object.fromentries@^2.0.8:
+ version "2.0.8"
+ resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.8.tgz#f7195d8a9b97bd95cbc1999ea939ecd1a2b00c65"
+ integrity sha512-k6E21FzySsSK5a21KRADBd/NGneRegFO5pLHfdQLpRDETUNJueLXs3WCzyQ3tFRDYgbq3KHGXfTbi2bs8WQ6rQ==
dependencies:
- call-bind "^1.0.2"
- define-properties "^1.2.0"
- es-abstract "^1.22.1"
+ call-bind "^1.0.7"
+ define-properties "^1.2.1"
+ es-abstract "^1.23.2"
+ es-object-atoms "^1.0.0"
-object.groupby@^1.0.1:
- version "1.0.2"
- resolved "https://registry.yarnpkg.com/object.groupby/-/object.groupby-1.0.2.tgz#494800ff5bab78fd0eff2835ec859066e00192ec"
- integrity sha512-bzBq58S+x+uo0VjurFT0UktpKHOZmv4/xePiOA1nbB9pMqpGK7rUPNgf+1YC+7mE+0HzhTMqNUuCqvKhj6FnBw==
+object.groupby@^1.0.3:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/object.groupby/-/object.groupby-1.0.3.tgz#9b125c36238129f6f7b61954a1e7176148d5002e"
+ integrity sha512-+Lhy3TQTuzXI5hevh8sBGqbmurHbbIjAi0Z4S63nthVLmLxfbj4T54a4CfZrXIrt9iP4mVAPYMo/v99taj3wjQ==
dependencies:
- array.prototype.filter "^1.0.3"
- call-bind "^1.0.5"
+ call-bind "^1.0.7"
define-properties "^1.2.1"
- es-abstract "^1.22.3"
- es-errors "^1.0.0"
+ es-abstract "^1.23.2"
-object.values@^1.1.7:
- version "1.1.7"
- resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.1.7.tgz#617ed13272e7e1071b43973aa1655d9291b8442a"
- integrity sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng==
+object.values@^1.2.1:
+ version "1.2.1"
+ resolved "https://registry.yarnpkg.com/object.values/-/object.values-1.2.1.tgz#deed520a50809ff7f75a7cfd4bc64c7a038c6216"
+ integrity sha512-gXah6aZrcUxjWg2zR2MwouP2eHlCBzdV4pygudehaKXSGW4v2AsRQUK+lwwXhii6KFZcunEnmSUoYp5CXibxtA==
dependencies:
- call-bind "^1.0.2"
- define-properties "^1.2.0"
- es-abstract "^1.22.1"
+ call-bind "^1.0.8"
+ call-bound "^1.0.3"
+ define-properties "^1.2.1"
+ es-object-atoms "^1.0.0"
once@^1.3.0, once@^1.3.1, once@^1.4.0:
version "1.4.0"
@@ -1932,6 +2154,15 @@ ospath@^1.2.2:
resolved "https://registry.npmjs.org/ospath/-/ospath-1.2.2.tgz"
integrity sha1-EnZjl3Sj+O8lcvf+QoDg6kVQwHs=
+own-keys@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/own-keys/-/own-keys-1.0.1.tgz#e4006910a2bf913585289676eebd6f390cf51358"
+ integrity sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg==
+ dependencies:
+ get-intrinsic "^1.2.6"
+ object-keys "^1.1.1"
+ safe-push-apply "^1.0.0"
+
p-limit@^3.0.2:
version "3.1.0"
resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b"
@@ -1990,15 +2221,20 @@ performance-now@^2.1.0:
resolved "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz"
integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
+picocolors@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b"
+ integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==
+
pify@^2.2.0:
version "2.3.0"
resolved "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz"
integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw=
possible-typed-array-names@^1.0.0:
- version "1.0.0"
- resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f"
- integrity sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz#93e3582bc0e5426586d9d07b79ee40fc841de4ae"
+ integrity sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==
prelude-ls@^1.2.1:
version "1.2.1"
@@ -2006,15 +2242,24 @@ prelude-ls@^1.2.1:
integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==
prettier@^3.2.5:
- version "3.2.5"
- resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.5.tgz#e52bc3090586e824964a8813b09aba6233b28368"
- integrity sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==
+ version "3.6.2"
+ resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.6.2.tgz#ccda02a1003ebbb2bfda6f83a074978f608b9393"
+ integrity sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==
pretty-bytes@^5.6.0:
version "5.6.0"
resolved "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz"
integrity sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==
+pretty-format@^27.0.2:
+ version "27.5.1"
+ resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e"
+ integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==
+ dependencies:
+ ansi-regex "^5.0.1"
+ ansi-styles "^5.0.0"
+ react-is "^17.0.1"
+
process@^0.11.10:
version "0.11.10"
resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
@@ -2022,13 +2267,13 @@ process@^0.11.10:
proxy-from-env@1.0.0:
version "1.0.0"
- resolved "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz"
- integrity sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=
+ resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee"
+ integrity sha512-F2JHgJQ1iqwnHDcQjVBsq3n/uoaFL+iPW/eAeL7kVxy/2RrWaN4WroKjjvbsoRtv0ftelNyC01bjRhn/bhcf4A==
pump@^3.0.0:
- version "3.0.0"
- resolved "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz"
- integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.3.tgz#151d979f1a29668dc0025ec589a455b53282268d"
+ integrity sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==
dependencies:
end-of-stream "^1.1.0"
once "^1.3.1"
@@ -2050,15 +2295,36 @@ queue-microtask@^1.2.2:
resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243"
integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==
-regexp.prototype.flags@^1.5.2:
- version "1.5.2"
- resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz#138f644a3350f981a858c44f6bb1a61ff59be334"
- integrity sha512-NcDiDkTLuPR+++OCKB0nWafEmhg/Da8aUPLPMQbK+bxKKCm1/S5he+AqYa4PlMCVBalb4/yxIRub6qkEx5yJbw==
+react-is@^17.0.1:
+ version "17.0.2"
+ resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0"
+ integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==
+
+reflect.getprototypeof@^1.0.6, reflect.getprototypeof@^1.0.9:
+ version "1.0.10"
+ resolved "https://registry.yarnpkg.com/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz#c629219e78a3316d8b604c765ef68996964e7bf9"
+ integrity sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==
dependencies:
- call-bind "^1.0.6"
+ call-bind "^1.0.8"
define-properties "^1.2.1"
+ es-abstract "^1.23.9"
es-errors "^1.3.0"
- set-function-name "^2.0.1"
+ es-object-atoms "^1.0.0"
+ get-intrinsic "^1.2.7"
+ get-proto "^1.0.1"
+ which-builtin-type "^1.2.1"
+
+regexp.prototype.flags@^1.5.2, regexp.prototype.flags@^1.5.4:
+ version "1.5.4"
+ resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz#1ad6c62d44a259007e55b3970e00f746efbcaa19"
+ integrity sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==
+ dependencies:
+ call-bind "^1.0.8"
+ define-properties "^1.2.1"
+ es-errors "^1.3.0"
+ get-proto "^1.0.1"
+ gopd "^1.2.0"
+ set-function-name "^2.0.2"
request-progress@^3.0.0:
version "3.0.0"
@@ -2073,11 +2339,11 @@ resolve-from@^4.0.0:
integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
resolve@^1.22.4:
- version "1.22.8"
- resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d"
- integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==
+ version "1.22.10"
+ resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.10.tgz#b663e83ffb09bbf2386944736baae803029b8b39"
+ integrity sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==
dependencies:
- is-core-module "^2.13.0"
+ is-core-module "^2.16.0"
path-parse "^1.0.7"
supports-preserve-symlinks-flag "^1.0.0"
@@ -2090,9 +2356,9 @@ restore-cursor@^3.1.0:
signal-exit "^3.0.2"
reusify@^1.0.4:
- version "1.0.4"
- resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
- integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.1.0.tgz#0fe13b9522e1473f51b558ee796e08f11f9b489f"
+ integrity sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==
rfdc@^1.3.0:
version "1.3.0"
@@ -2130,19 +2396,38 @@ safe-array-concat@^1.1.0:
has-symbols "^1.0.3"
isarray "^2.0.5"
+safe-array-concat@^1.1.3:
+ version "1.1.3"
+ resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.1.3.tgz#c9e54ec4f603b0bbb8e7e5007a5ee7aecd1538c3"
+ integrity sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==
+ dependencies:
+ call-bind "^1.0.8"
+ call-bound "^1.0.2"
+ get-intrinsic "^1.2.6"
+ has-symbols "^1.1.0"
+ isarray "^2.0.5"
+
safe-buffer@^5.0.1, safe-buffer@^5.1.2:
version "5.2.1"
resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz"
integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
-safe-regex-test@^1.0.3:
- version "1.0.3"
- resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.3.tgz#a5b4c0f06e0ab50ea2c395c14d8371232924c377"
- integrity sha512-CdASjNJPvRa7roO6Ra/gLYBTzYzzPyyBXxIMdGW3USQLyjWEls2RgW5UBTXaQVp+OrpeCK3bLem8smtmheoRuw==
+safe-push-apply@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/safe-push-apply/-/safe-push-apply-1.0.0.tgz#01850e981c1602d398c85081f360e4e6d03d27f5"
+ integrity sha512-iKE9w/Z7xCzUMIZqdBsp6pEQvwuEebH4vdpjcDWnyzaI6yl6O9FHvVpmGelvEHNsoY6wGblkxR6Zty/h00WiSA==
dependencies:
- call-bind "^1.0.6"
es-errors "^1.3.0"
- is-regex "^1.1.4"
+ isarray "^2.0.5"
+
+safe-regex-test@^1.0.3, safe-regex-test@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.1.0.tgz#7f87dfb67a3150782eaaf18583ff5d1711ac10c1"
+ integrity sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==
+ dependencies:
+ call-bound "^1.0.2"
+ es-errors "^1.3.0"
+ is-regex "^1.2.1"
safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
version "2.1.2"
@@ -2159,19 +2444,19 @@ semver@^7.7.1:
resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.2.tgz#67d99fdcd35cec21e6f8b87a7fd515a33f982b58"
integrity sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==
-set-function-length@^1.2.1:
- version "1.2.1"
- resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.1.tgz#47cc5945f2c771e2cf261c6737cf9684a2a5e425"
- integrity sha512-j4t6ccc+VsKwYHso+kElc5neZpjtq9EnRICFZtWyBsLojhmeF/ZBd/elqm22WJh/BziDe/SBiOeAt0m2mfLD0g==
+set-function-length@^1.2.2:
+ version "1.2.2"
+ resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449"
+ integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==
dependencies:
- define-data-property "^1.1.2"
+ define-data-property "^1.1.4"
es-errors "^1.3.0"
function-bind "^1.1.2"
- get-intrinsic "^1.2.3"
+ get-intrinsic "^1.2.4"
gopd "^1.0.1"
- has-property-descriptors "^1.0.1"
+ has-property-descriptors "^1.0.2"
-set-function-name@^2.0.1:
+set-function-name@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/set-function-name/-/set-function-name-2.0.2.tgz#16a705c5a0dc2f5e638ca96d8a8cd4e1c2b90985"
integrity sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==
@@ -2181,6 +2466,15 @@ set-function-name@^2.0.1:
functions-have-names "^1.2.3"
has-property-descriptors "^1.0.2"
+set-proto@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/set-proto/-/set-proto-1.0.0.tgz#0760dbcff30b2d7e801fd6e19983e56da337565e"
+ integrity sha512-RJRdvCo6IAnPdsvP/7m6bsQqNnn1FCBX5ZNtFL98MmFF/4xAIJTIg1YbHW5DC2W5SKZanrC6i4HsJqlajw/dZw==
+ dependencies:
+ dunder-proto "^1.0.1"
+ es-errors "^1.3.0"
+ es-object-atoms "^1.0.0"
+
shebang-command@^2.0.0:
version "2.0.0"
resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz"
@@ -2222,16 +2516,6 @@ side-channel-weakmap@^1.0.2:
object-inspect "^1.13.3"
side-channel-map "^1.0.1"
-side-channel@^1.0.4:
- version "1.0.5"
- resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.5.tgz#9a84546599b48909fb6af1211708d23b1946221b"
- integrity sha512-QcgiIWV4WV7qWExbN5llt6frQB/lBven9pqliLXfGPB+K9ZYXxDozp0wLkHS24kWCm+6YXH/f0HhnObZnZOBnQ==
- dependencies:
- call-bind "^1.0.6"
- es-errors "^1.3.0"
- get-intrinsic "^1.2.4"
- object-inspect "^1.13.1"
-
side-channel@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.1.0.tgz#c3fcff9c4da932784873335ec9765fa94ff66bc9"
@@ -2281,6 +2565,14 @@ sshpk@^1.18.0:
safer-buffer "^2.0.2"
tweetnacl "~0.14.0"
+stop-iteration-iterator@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz#f481ff70a548f6124d0312c3aa14cbfa7aa542ad"
+ integrity sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==
+ dependencies:
+ es-errors "^1.3.0"
+ internal-slot "^1.1.0"
+
string-width@^4.1.0, string-width@^4.2.0:
version "4.2.3"
resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
@@ -2290,6 +2582,19 @@ string-width@^4.1.0, string-width@^4.2.0:
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.1"
+string.prototype.trim@^1.2.10:
+ version "1.2.10"
+ resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.10.tgz#40b2dd5ee94c959b4dcfb1d65ce72e90da480c81"
+ integrity sha512-Rs66F0P/1kedk5lyYyH9uBzuiI/kNRmwJAR9quK6VOtIpZ2G+hMZd+HQbbv25MgCA6gEffoMZYxlTod4WcdrKA==
+ dependencies:
+ call-bind "^1.0.8"
+ call-bound "^1.0.2"
+ define-data-property "^1.1.4"
+ define-properties "^1.2.1"
+ es-abstract "^1.23.5"
+ es-object-atoms "^1.0.0"
+ has-property-descriptors "^1.0.2"
+
string.prototype.trim@^1.2.8:
version "1.2.8"
resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz#f9ac6f8af4bd55ddfa8895e6aea92a96395393bd"
@@ -2308,6 +2613,16 @@ string.prototype.trimend@^1.0.7:
define-properties "^1.2.0"
es-abstract "^1.22.1"
+string.prototype.trimend@^1.0.9:
+ version "1.0.9"
+ resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.9.tgz#62e2731272cd285041b36596054e9f66569b6942"
+ integrity sha512-G7Ok5C6E/j4SGfyLCloXTrngQIQU3PWtXGst3yM7Bea9FRURf1S42ZHlZZtsNque2FN2PoUhfZXYLNWwEr4dLQ==
+ dependencies:
+ call-bind "^1.0.8"
+ call-bound "^1.0.2"
+ define-properties "^1.2.1"
+ es-object-atoms "^1.0.0"
+
string.prototype.trimstart@^1.0.7:
version "1.0.7"
resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.7.tgz#d4cdb44b83a4737ffbac2d406e405d43d0184298"
@@ -2317,6 +2632,15 @@ string.prototype.trimstart@^1.0.7:
define-properties "^1.2.0"
es-abstract "^1.22.1"
+string.prototype.trimstart@^1.0.8:
+ version "1.0.8"
+ resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.8.tgz#7ee834dda8c7c17eff3118472bb35bfedaa34dde"
+ integrity sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg==
+ dependencies:
+ call-bind "^1.0.7"
+ define-properties "^1.2.1"
+ es-object-atoms "^1.0.0"
+
strip-ansi@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f"
@@ -2467,6 +2791,15 @@ typed-array-buffer@^1.0.1:
es-errors "^1.3.0"
is-typed-array "^1.1.13"
+typed-array-buffer@^1.0.3:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz#a72395450a4869ec033fd549371b47af3a2ee536"
+ integrity sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==
+ dependencies:
+ call-bound "^1.0.3"
+ es-errors "^1.3.0"
+ is-typed-array "^1.1.14"
+
typed-array-byte-length@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/typed-array-byte-length/-/typed-array-byte-length-1.0.1.tgz#d92972d3cff99a3fa2e765a28fcdc0f1d89dec67"
@@ -2478,6 +2811,17 @@ typed-array-byte-length@^1.0.0:
has-proto "^1.0.3"
is-typed-array "^1.1.13"
+typed-array-byte-length@^1.0.3:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/typed-array-byte-length/-/typed-array-byte-length-1.0.3.tgz#8407a04f7d78684f3d252aa1a143d2b77b4160ce"
+ integrity sha512-BaXgOuIxz8n8pIq3e7Atg/7s+DpiYrxn4vdot3w9KbnBhcRQq6o3xemQdIfynqSeXeDrF32x+WvfzmOjPiY9lg==
+ dependencies:
+ call-bind "^1.0.8"
+ for-each "^0.3.3"
+ gopd "^1.2.0"
+ has-proto "^1.2.0"
+ is-typed-array "^1.1.14"
+
typed-array-byte-offset@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/typed-array-byte-offset/-/typed-array-byte-offset-1.0.2.tgz#f9ec1acb9259f395093e4567eb3c28a580d02063"
@@ -2490,6 +2834,19 @@ typed-array-byte-offset@^1.0.0:
has-proto "^1.0.3"
is-typed-array "^1.1.13"
+typed-array-byte-offset@^1.0.4:
+ version "1.0.4"
+ resolved "https://registry.yarnpkg.com/typed-array-byte-offset/-/typed-array-byte-offset-1.0.4.tgz#ae3698b8ec91a8ab945016108aef00d5bff12355"
+ integrity sha512-bTlAFB/FBYMcuX81gbL4OcpH5PmlFHqlCCpAl8AlEzMz5k53oNDvN8p1PNOWLEmI2x4orp3raOFB51tv9X+MFQ==
+ dependencies:
+ available-typed-arrays "^1.0.7"
+ call-bind "^1.0.8"
+ for-each "^0.3.3"
+ gopd "^1.2.0"
+ has-proto "^1.2.0"
+ is-typed-array "^1.1.15"
+ reflect.getprototypeof "^1.0.9"
+
typed-array-length@^1.0.4:
version "1.0.5"
resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.5.tgz#57d44da160296d8663fd63180a1802ebf25905d5"
@@ -2502,15 +2859,32 @@ typed-array-length@^1.0.4:
is-typed-array "^1.1.13"
possible-typed-array-names "^1.0.0"
-unbox-primitive@^1.0.2:
- version "1.0.2"
- resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e"
- integrity sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==
+typed-array-length@^1.0.7:
+ version "1.0.7"
+ resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.7.tgz#ee4deff984b64be1e118b0de8c9c877d5ce73d3d"
+ integrity sha512-3KS2b+kL7fsuk/eJZ7EQdnEmQoaho/r6KUef7hxvltNA5DR8NAUM+8wJMbJyZ4G9/7i3v5zPBIMN5aybAh2/Jg==
dependencies:
- call-bind "^1.0.2"
+ call-bind "^1.0.7"
+ for-each "^0.3.3"
+ gopd "^1.0.1"
+ is-typed-array "^1.1.13"
+ possible-typed-array-names "^1.0.0"
+ reflect.getprototypeof "^1.0.6"
+
+unbox-primitive@^1.0.2, unbox-primitive@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.1.0.tgz#8d9d2c9edeea8460c7f35033a88867944934d1e2"
+ integrity sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==
+ dependencies:
+ call-bound "^1.0.3"
has-bigints "^1.0.2"
- has-symbols "^1.0.3"
- which-boxed-primitive "^1.0.2"
+ has-symbols "^1.1.0"
+ which-boxed-primitive "^1.1.1"
+
+undici-types@~7.8.0:
+ version "7.8.0"
+ resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.8.0.tgz#de00b85b710c54122e44fbfd911f8d70174cd294"
+ integrity sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==
universalify@^2.0.0:
version "2.0.0"
@@ -2543,16 +2917,45 @@ verror@1.10.0:
core-util-is "1.0.2"
extsprintf "^1.2.0"
-which-boxed-primitive@^1.0.2:
+which-boxed-primitive@^1.1.0, which-boxed-primitive@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.1.1.tgz#d76ec27df7fa165f18d5808374a5fe23c29b176e"
+ integrity sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA==
+ dependencies:
+ is-bigint "^1.1.0"
+ is-boolean-object "^1.2.1"
+ is-number-object "^1.1.1"
+ is-string "^1.1.1"
+ is-symbol "^1.1.1"
+
+which-builtin-type@^1.2.1:
+ version "1.2.1"
+ resolved "https://registry.yarnpkg.com/which-builtin-type/-/which-builtin-type-1.2.1.tgz#89183da1b4907ab089a6b02029cc5d8d6574270e"
+ integrity sha512-6iBczoX+kDQ7a3+YJBnh3T+KZRxM/iYNPXicqk66/Qfm1b93iu+yOImkg0zHbj5LNOcNv1TEADiZ0xa34B4q6Q==
+ dependencies:
+ call-bound "^1.0.2"
+ function.prototype.name "^1.1.6"
+ has-tostringtag "^1.0.2"
+ is-async-function "^2.0.0"
+ is-date-object "^1.1.0"
+ is-finalizationregistry "^1.1.0"
+ is-generator-function "^1.0.10"
+ is-regex "^1.2.1"
+ is-weakref "^1.0.2"
+ isarray "^2.0.5"
+ which-boxed-primitive "^1.1.0"
+ which-collection "^1.0.2"
+ which-typed-array "^1.1.16"
+
+which-collection@^1.0.2:
version "1.0.2"
- resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"
- integrity sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==
+ resolved "https://registry.yarnpkg.com/which-collection/-/which-collection-1.0.2.tgz#627ef76243920a107e7ce8e96191debe4b16c2a0"
+ integrity sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw==
dependencies:
- is-bigint "^1.0.1"
- is-boolean-object "^1.1.0"
- is-number-object "^1.0.4"
- is-string "^1.0.5"
- is-symbol "^1.0.3"
+ is-map "^2.0.3"
+ is-set "^2.0.3"
+ is-weakmap "^2.0.2"
+ is-weakset "^2.0.3"
which-typed-array@^1.1.14:
version "1.1.14"
@@ -2565,6 +2968,19 @@ which-typed-array@^1.1.14:
gopd "^1.0.1"
has-tostringtag "^1.0.1"
+which-typed-array@^1.1.16, which-typed-array@^1.1.19:
+ version "1.1.19"
+ resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.19.tgz#df03842e870b6b88e117524a4b364b6fc689f956"
+ integrity sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==
+ dependencies:
+ available-typed-arrays "^1.0.7"
+ call-bind "^1.0.8"
+ call-bound "^1.0.4"
+ for-each "^0.3.5"
+ get-proto "^1.0.1"
+ gopd "^1.2.0"
+ has-tostringtag "^1.0.2"
+
which@^2.0.1:
version "2.0.2"
resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz"