Skip to content

Commit 8f4bf29

Browse files
authored
feat: Support the --asset-type argument for individual imports (#337)
* feat: Support the --asset-type argument for individual imports * Removing duplicated comma * fixing logic
1 parent acc41f8 commit 8f4bf29

File tree

2 files changed

+363
-59
lines changed

2 files changed

+363
-59
lines changed

src/preset_cli/cli/superset/sync/native/command.py

Lines changed: 30 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from io import BytesIO
1515
from pathlib import Path
1616
from types import ModuleType
17-
from typing import Any, Dict, Iterator, Optional, Set, Tuple
17+
from typing import Any, Dict, Iterator, Set, Tuple
1818
from zipfile import ZipFile
1919

2020
import backoff
@@ -58,7 +58,7 @@ class ResourceType(Enum):
5858
def __new__(
5959
cls,
6060
resource_name: str,
61-
metadata_type: Optional[str] = None,
61+
metadata_type: str | None = None,
6262
) -> "ResourceType":
6363
"""
6464
ResourceType Constructor.
@@ -83,6 +83,7 @@ def metadata_type(self) -> str:
8383
"""
8484
return self._metadata_type # type: ignore
8585

86+
ASSET = ("assets", "assets")
8687
CHART = ("chart", "Slice")
8788
DASHBOARD = ("dashboard", "Dashboard")
8889
DATABASE = ("database", "Database")
@@ -92,14 +93,14 @@ def metadata_type(self) -> str:
9293
def normalize_to_enum( # pylint: disable=unused-argument
9394
ctx: click.core.Context,
9495
param: str,
95-
value: Optional[str],
96+
value: str | None,
9697
):
9798
"""
9899
Normalize the ``--asset-type`` option value and return the
99100
corresponding ResourceType Enum.
100101
"""
101102
if value is None:
102-
return None
103+
return ResourceType.ASSET
103104
return ResourceType(value.lower())
104105

105106

@@ -230,14 +231,14 @@ def native( # pylint: disable=too-many-locals, too-many-arguments, too-many-bra
230231
ctx: click.core.Context,
231232
directory: str,
232233
option: Tuple[str, ...],
234+
asset_type: ResourceType,
233235
overwrite: bool = False,
234236
disable_jinja_templating: bool = False,
235237
disallow_edits: bool = True, # pylint: disable=unused-argument
236238
external_url_prefix: str = "",
237239
load_env: bool = False,
238240
split: bool = False,
239241
continue_on_error: bool = False,
240-
asset_type: Optional[ResourceType] = None,
241242
) -> None:
242243
"""
243244
Sync exported DBs/datasets/charts/dashboards to Superset.
@@ -316,16 +317,23 @@ def native( # pylint: disable=too-many-locals, too-many-arguments, too-many-bra
316317
configs["bundle" / relative_path] = config
317318

318319
if split:
319-
import_resources_individually(configs, client, overwrite, continue_on_error)
320+
import_resources_individually(
321+
configs,
322+
client,
323+
overwrite,
324+
asset_type,
325+
continue_on_error,
326+
)
320327
else:
321328
contents = {str(k): yaml.dump(v) for k, v in configs.items()}
322-
import_resources(contents, client, overwrite, asset_type=asset_type)
329+
import_resources(contents, client, overwrite, asset_type)
323330

324331

325332
def import_resources_individually( # pylint: disable=too-many-locals
326333
configs: Dict[Path, AssetConfig],
327334
client: SupersetClient,
328335
overwrite: bool,
336+
asset_type: ResourceType,
329337
continue_on_error: bool = False,
330338
) -> None:
331339
"""
@@ -354,10 +362,20 @@ def import_resources_individually( # pylint: disable=too-many-locals
354362
with open(log_file_path, "w", encoding="utf-8") as log_file:
355363
for resource_name, get_related_uuids in imports:
356364
for path, config in configs.items():
357-
if path.parts[1] != resource_name or path in assets_to_skip:
365+
if path.parts[1] != resource_name:
358366
continue
359367

360368
asset_configs = {path: config}
369+
for uuid in get_related_uuids(config):
370+
asset_configs.update(related_configs[uuid])
371+
related_configs[config["uuid"]] = asset_configs
372+
373+
if path in assets_to_skip or (
374+
asset_type != ResourceType.ASSET
375+
and asset_type.resource_name not in resource_name
376+
):
377+
continue
378+
361379
_logger.info("Importing %s", path.relative_to("bundle"))
362380
asset_log = {
363381
"uuid": config["uuid"],
@@ -366,16 +384,13 @@ def import_resources_individually( # pylint: disable=too-many-locals
366384
}
367385

368386
try:
369-
for uuid in get_related_uuids(config):
370-
asset_configs.update(related_configs[uuid])
371387
contents = {str(k): yaml.dump(v) for k, v in asset_configs.items()}
372-
import_resources(contents, client, overwrite)
388+
import_resources(contents, client, overwrite, asset_type)
373389
except Exception: # pylint: disable=broad-except
374390
if not continue_on_error:
375391
raise
376392
asset_log["status"] = "FAILED"
377393

378-
related_configs[config["uuid"]] = asset_configs
379394
logs[LogType.ASSETS].append(asset_log)
380395
assets_to_skip.add(path)
381396
write_logs_to_file(log_file, logs)
@@ -464,17 +479,15 @@ def import_resources(
464479
contents: Dict[str, str],
465480
client: SupersetClient,
466481
overwrite: bool,
467-
asset_type: Optional[ResourceType] = None,
482+
asset_type: ResourceType,
468483
) -> None:
469484
"""
470485
Import a bundle of assets.
471486
"""
472-
metadata_type = asset_type.metadata_type if asset_type else "assets"
473-
resource_name = asset_type.resource_name if asset_type else "assets"
474487
contents["bundle/metadata.yaml"] = yaml.dump(
475488
dict(
476489
version="1.0.0",
477-
type=metadata_type,
490+
type=asset_type.metadata_type,
478491
timestamp=datetime.now(tz=timezone.utc).isoformat(),
479492
),
480493
)
@@ -486,7 +499,7 @@ def import_resources(
486499
output.write(file_content.encode())
487500
buf.seek(0)
488501
try:
489-
client.import_zip(resource_name, buf, overwrite=overwrite)
502+
client.import_zip(asset_type.resource_name, buf, overwrite=overwrite)
490503
except SupersetError as ex:
491504
click.echo(
492505
click.style(

0 commit comments

Comments
 (0)