From 4d3d037b392eda0001361e82980e348c0a8f87fc Mon Sep 17 00:00:00 2001 From: Fabian Zills Date: Thu, 5 Dec 2024 14:20:11 +0100 Subject: [PATCH 1/9] performance benchmarking --- zntrack/state.py | 10 ++++++++++ zntrack/utils/misc.py | 2 ++ zntrack/utils/node_wd.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/zntrack/state.py b/zntrack/state.py index b644c4d2..a136f66f 100644 --- a/zntrack/state.py +++ b/zntrack/state.py @@ -26,6 +26,8 @@ PLUGIN_LIST = list[t.Type[ZnTrackPlugin]] PLUGIN_DICT = dict[str, ZnTrackPlugin] +COUNT = 0 + @dataclasses.dataclass(frozen=True) class NodeStatus: @@ -53,11 +55,19 @@ def name(self) -> str: def nwd(self): if self.tmp_path is not None: return self.tmp_path + if "nwd" not in self.node.__dict__: + global COUNT + COUNT += 1 + print(f"nwd: {COUNT}") + self.node.__dict__["nwd"] = get_nwd(self.node) + return get_nwd(self.node) @property def fs(self) -> AbstractFileSystem: """Get the file system of the Node.""" + # print("ACCESS FILE SYSTEM") + print("This should not be called during build.") if self.remote is None and self.rev is None: return LocalFileSystem() return dvc.api.DVCFileSystem( diff --git a/zntrack/utils/misc.py b/zntrack/utils/misc.py index 0bcf39d1..d1b427d3 100644 --- a/zntrack/utils/misc.py +++ b/zntrack/utils/misc.py @@ -77,6 +77,8 @@ def default(self, value, **kwargs): nwd_handler = NWDReplaceHandler() original_nwd = get_nwd(instance) + if "nwd" not in instance.__dict__: + print("tmp-path-loader") tmp_nwd = instance.nwd original_path = pathlib.Path(nwd_handler(value, nwd=original_nwd)) diff --git a/zntrack/utils/node_wd.py b/zntrack/utils/node_wd.py index f4b78c8d..398ad2ec 100644 --- a/zntrack/utils/node_wd.py +++ b/zntrack/utils/node_wd.py @@ -58,7 +58,7 @@ def get_nwd(node: "Node") -> pathlib.Path: if ( node.state.remote is None and node.state.rev is None - and node.state.state == NodeStatusEnum.FINISHED + and node.state.state != NodeStatusEnum.FINISHED ): nwd = pathlib.Path(NWD_PATH, node.name) else: From 2ce928429b7f86afed960021c8abd6493f89ca08 Mon Sep 17 00:00:00 2001 From: Fabian Zills Date: Thu, 5 Dec 2024 20:31:39 +0100 Subject: [PATCH 2/9] move stuff around --- zntrack/state.py | 8 +++----- zntrack/utils/node_wd.py | 11 ++++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/zntrack/state.py b/zntrack/state.py index a136f66f..fdddd6f2 100644 --- a/zntrack/state.py +++ b/zntrack/state.py @@ -26,7 +26,7 @@ PLUGIN_LIST = list[t.Type[ZnTrackPlugin]] PLUGIN_DICT = dict[str, ZnTrackPlugin] -COUNT = 0 +COUNT = datetime.datetime.now() @dataclasses.dataclass(frozen=True) @@ -57,8 +57,8 @@ def nwd(self): return self.tmp_path if "nwd" not in self.node.__dict__: global COUNT - COUNT += 1 - print(f"nwd: {COUNT}") + print(f"nwd: {(datetime.datetime.now() - COUNT).total_seconds()} for {self.node.name}") + COUNT = datetime.datetime.now() self.node.__dict__["nwd"] = get_nwd(self.node) return get_nwd(self.node) @@ -66,8 +66,6 @@ def nwd(self): @property def fs(self) -> AbstractFileSystem: """Get the file system of the Node.""" - # print("ACCESS FILE SYSTEM") - print("This should not be called during build.") if self.remote is None and self.rev is None: return LocalFileSystem() return dvc.api.DVCFileSystem( diff --git a/zntrack/utils/node_wd.py b/zntrack/utils/node_wd.py index 398ad2ec..b508d814 100644 --- a/zntrack/utils/node_wd.py +++ b/zntrack/utils/node_wd.py @@ -6,6 +6,7 @@ import pathlib import shutil import typing as t +import datetime import znflow.utils import znjson @@ -70,11 +71,11 @@ def get_nwd(node: "Node") -> pathlib.Path: except (FileNotFoundError, KeyError): nwd = pathlib.Path(NWD_PATH, node.name) - if node.state.group is not None: - # strip the groups from node_name - to_replace = "_".join(node.state.group.name) + "_" - replacement = "/".join(node.state.group.name) + "/" - nwd = pathlib.Path(str(nwd).replace(to_replace, replacement)) + if node.state.group is not None: + # strip the groups from node_name + to_replace = "_".join(node.state.group.name) + "_" + replacement = "/".join(node.state.group.name) + "/" + nwd = pathlib.Path(str(nwd).replace(to_replace, replacement)) return nwd From c7dbebc8055413e465ca374c819e90dd3b3d8810 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 19:31:56 +0000 Subject: [PATCH 3/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- zntrack/state.py | 4 +++- zntrack/utils/node_wd.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/zntrack/state.py b/zntrack/state.py index fdddd6f2..432ecaa7 100644 --- a/zntrack/state.py +++ b/zntrack/state.py @@ -57,7 +57,9 @@ def nwd(self): return self.tmp_path if "nwd" not in self.node.__dict__: global COUNT - print(f"nwd: {(datetime.datetime.now() - COUNT).total_seconds()} for {self.node.name}") + print( + f"nwd: {(datetime.datetime.now() - COUNT).total_seconds()} for {self.node.name}" + ) COUNT = datetime.datetime.now() self.node.__dict__["nwd"] = get_nwd(self.node) diff --git a/zntrack/utils/node_wd.py b/zntrack/utils/node_wd.py index b508d814..a6de3e0e 100644 --- a/zntrack/utils/node_wd.py +++ b/zntrack/utils/node_wd.py @@ -6,7 +6,6 @@ import pathlib import shutil import typing as t -import datetime import znflow.utils import znjson From fcddc051c56af28cad6c293195ebcabdafe9ba36 Mon Sep 17 00:00:00 2001 From: Fabian Zills Date: Thu, 5 Dec 2024 20:49:01 +0100 Subject: [PATCH 4/9] remove print --- zntrack/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zntrack/state.py b/zntrack/state.py index fdddd6f2..a58c83d5 100644 --- a/zntrack/state.py +++ b/zntrack/state.py @@ -57,7 +57,7 @@ def nwd(self): return self.tmp_path if "nwd" not in self.node.__dict__: global COUNT - print(f"nwd: {(datetime.datetime.now() - COUNT).total_seconds()} for {self.node.name}") + # print(f"nwd: {(datetime.datetime.now() - COUNT).total_seconds()} for {self.node.name}") COUNT = datetime.datetime.now() self.node.__dict__["nwd"] = get_nwd(self.node) From 1a84f930295ef34ede83769747e9825d727b8c0c Mon Sep 17 00:00:00 2001 From: Fabian Zills Date: Fri, 6 Dec 2024 15:53:30 +0100 Subject: [PATCH 5/9] remove debug statements --- zntrack/state.py | 4 ---- zntrack/utils/misc.py | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/zntrack/state.py b/zntrack/state.py index a58c83d5..c3e37b33 100644 --- a/zntrack/state.py +++ b/zntrack/state.py @@ -26,7 +26,6 @@ PLUGIN_LIST = list[t.Type[ZnTrackPlugin]] PLUGIN_DICT = dict[str, ZnTrackPlugin] -COUNT = datetime.datetime.now() @dataclasses.dataclass(frozen=True) @@ -56,9 +55,6 @@ def nwd(self): if self.tmp_path is not None: return self.tmp_path if "nwd" not in self.node.__dict__: - global COUNT - # print(f"nwd: {(datetime.datetime.now() - COUNT).total_seconds()} for {self.node.name}") - COUNT = datetime.datetime.now() self.node.__dict__["nwd"] = get_nwd(self.node) return get_nwd(self.node) diff --git a/zntrack/utils/misc.py b/zntrack/utils/misc.py index d1b427d3..0bd4d459 100644 --- a/zntrack/utils/misc.py +++ b/zntrack/utils/misc.py @@ -76,9 +76,7 @@ def default(self, value, **kwargs): nwd_handler = NWDReplaceHandler() - original_nwd = get_nwd(instance) - if "nwd" not in instance.__dict__: - print("tmp-path-loader") + original_nwd = get_nwd(instance) tmp_nwd = instance.nwd original_path = pathlib.Path(nwd_handler(value, nwd=original_nwd)) From 58afdbb70fbb13d830e67ded402e50d6b95a2ff3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 14:54:03 +0000 Subject: [PATCH 6/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- zntrack/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zntrack/utils/misc.py b/zntrack/utils/misc.py index 0bd4d459..0bcf39d1 100644 --- a/zntrack/utils/misc.py +++ b/zntrack/utils/misc.py @@ -76,7 +76,7 @@ def default(self, value, **kwargs): nwd_handler = NWDReplaceHandler() - original_nwd = get_nwd(instance) + original_nwd = get_nwd(instance) tmp_nwd = instance.nwd original_path = pathlib.Path(nwd_handler(value, nwd=original_nwd)) From 51f7773ae0191108c96b2546b9fa6022b14c6cb4 Mon Sep 17 00:00:00 2001 From: Fabian Zills Date: Fri, 6 Dec 2024 15:59:08 +0100 Subject: [PATCH 7/9] fix indent --- zntrack/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zntrack/state.py b/zntrack/state.py index c3e37b33..6fe3f2a1 100644 --- a/zntrack/state.py +++ b/zntrack/state.py @@ -55,7 +55,7 @@ def nwd(self): if self.tmp_path is not None: return self.tmp_path if "nwd" not in self.node.__dict__: - self.node.__dict__["nwd"] = get_nwd(self.node) + self.node.__dict__["nwd"] = get_nwd(self.node) return get_nwd(self.node) From 9f4087b95fc4af2e5b1ee6a79e1ee69b75eded91 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:00:05 +0000 Subject: [PATCH 8/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- zntrack/state.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zntrack/state.py b/zntrack/state.py index 6fe3f2a1..d64691a3 100644 --- a/zntrack/state.py +++ b/zntrack/state.py @@ -27,7 +27,6 @@ PLUGIN_DICT = dict[str, ZnTrackPlugin] - @dataclasses.dataclass(frozen=True) class NodeStatus: remote: str | None = None From 393c4730dd93785da530d9f90617ebe62617bb9b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Mar 2025 14:01:29 +0000 Subject: [PATCH 9/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- zntrack/utils/node_wd.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zntrack/utils/node_wd.py b/zntrack/utils/node_wd.py index 169755b3..239157f1 100644 --- a/zntrack/utils/node_wd.py +++ b/zntrack/utils/node_wd.py @@ -5,7 +5,6 @@ import pathlib import shutil import typing as t -import warnings import znflow.utils