Add test for copying across filesystems.

Author: bekadavis9

tiledb/tests/common.py

@@ -70,18 +70,31 @@ def create_vfs_dir(path):
     vfs.create_dir(path)
 
 
-def vfs_path(scheme: str, prefix: str = None) -> str:
-    """Create a VFS-compatible path"""
-    if not prefix:
-        if scheme == "s3":
-            prefix: str = "tiledb-" + str(random.randint(0, 10e10))
-        else:
-            prefix: str = "tiledb-" + str(uuid.uuid4())
+def vfs_path(scheme: str) -> str:
+    prefix = "tiledb-"
+    if scheme == "s3":
+        prefix += str(random.randint(0, 10000000000))
+    else:
+        prefix += str(uuid.uuid4())
+
     if scheme == "file":
         return f"{scheme}://{tempfile.mktemp(prefix=prefix)}"
-    if "/" not in prefix:
-        prefix: str = f"{uuid.uuid4()}/{prefix}"
-    return f"{scheme}://{prefix}"
+    else:
+        return f"{scheme}://{prefix}"
+
+
+# def vfs_path(scheme: str, prefix: str = None) -> str:
+#     """Create a VFS-compatible path"""
+#     if not prefix:
+#         if scheme == "s3":
+#             prefix: str = "tiledb-" + str(random.randint(0, 10e10))
+#         else :
+#             prefix: str = "tiledb-" + str(uuid.uuid4())
+#     if scheme == "file":
+#         return f"{scheme}://{tempfile.mktemp(prefix=prefix)}"
+#     if "/" not in prefix:
+#         prefix: str = f"{uuid.uuid4()}/{prefix}"
+#     return f"{scheme}://{prefix}"
 
 
 class DiskTestCase:

tiledb/tests/conftest.py

@@ -72,3 +72,36 @@ def original_os_fork():
     """Provides the original unpatched os.fork."""
     if sys.platform != "win32":
         return os.fork
+
+
+@pytest.fixture
+def vfs_config() -> dict[str, str]:
+    config: dict[str, str] = {}
+    # Configure S3
+    if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"):
+        config["vfs.s3.aws_access_key_id"] = os.getenv("AWS_ACCESS_KEY_ID")
+        config["vfs.s3.aws_secret_access_key"] = os.getenv("AWS_SECRET_ACCESS_KEY")
+        if os.getenv("VFS_S3_USE_MINIO"):
+            config["vfs.s3.endpoint_override"] = "localhost:9999"
+            config["vfs.s3.scheme"] = "https"
+            config["vfs.s3.use_virtual_addressing"] = "false"
+            config["vfs.s3.verify_ssl"] = "false"
+
+    #   Configure Azure
+    if os.getenv("AZURE_BLOB_ENDPOINT"):
+        config["vfs.azure.blob_endpoint"] = os.getenv("AZURE_BLOB_ENDPOINT")
+    if os.getenv("AZURE_STORAGE_ACCOUNT_TOKEN"):
+        config["vfs.azure.storage_sas_token"] = os.getenv("AZURE_STORAGE_ACCOUNT_TOKEN")
+    elif os.getenv("AZURE_STORAGE_ACCOUNT_NAME") and os.getenv(
+        "AZURE_STORAGE_ACCOUNT_KEY"
+    ):
+        config["vfs.azure.storage_account_name"] = os.getenv(
+            "AZURE_STORAGE_ACCOUNT_NAME"
+        )
+        config["vfs.azure.storage_account_key"] = os.getenv("AZURE_STORAGE_ACCOUNT_KEY")
+
+    #   Configure Google Cloud
+    if os.getenv("TILEDB_TEST_GCS_ENDPOINT"):
+        config["vfs.gcs.endpoint"] = os.getenv("TILEDB_TEST_GCS_ENDPOINT")
+
+    return config

tiledb/tests/test_vfs.py

@@ -118,32 +118,46 @@ def test_copy(self):
             with self.assertRaises(tiledb.TileDBError):
                 vfs.copy_dir(self.path("foo/baz"), self.path("do_not_exist/baz"))
 
+    # Note: Azure tests are intermittently failing.
+    # Seemingly mostly azure->azure, but also test teardown.
+    # I think it's an Azurite bucket storage thing, actually.
     @pytest.mark.skipif(
         sys.platform == "win32",
         reason="Windows paths are difficult; Posix is sufficient for testing.",
     )
-    @pytest.mark.parametrize("src", ["file"])
-    @pytest.mark.parametrize("dst", ["file"])
-    # @pytest.mark.parametrize("src", ["file", "s3", "azure", "gcs"])
-    # @pytest.mark.parametrize("dst", ["file", "s3", "azure", "gcs"])
-    def test_copy_across(self, src: str, dst: str):
+    @pytest.mark.parametrize("src", ["file", "s3", "azure", "gcs"])
+    @pytest.mark.parametrize("dst", ["file", "s3", "azure", "gcs"])
+    def test_copy_across(self, src: str, dst: str, vfs_config):
         # Currently, skip if both FSes are the same. This is covered above.
         # However, this test might ought to replace the above.
-        # if src == dst:
-        #     return
+        if src == dst:
+            return
 
-        vfs = tiledb.VFS()
+        # Set configuration options
+        if (src == "s3" or dst == "s3") and not vfs_config.get(
+            "vfs.s3.aws_access_key_id"
+        ):
+            return
+        if (src == "azure" or dst == "azure") and not any(
+            x.startswith("vfs.azure") for x in vfs_config.keys()
+        ):
+            return
+        if (src == "gcs" or dst == "gcs") and not vfs_config.get("vfs.gcs.endpoint"):
+            return
+        vfs = tiledb.VFS(vfs_config)
 
         # Return if neither filesystem is supported.
         if not vfs.supports(src) or not vfs.supports(dst):
             return
 
         # Create source file, and write to it.
-        srcdir: str = vfs_path(src, prefix="tiledb-copy-src")
+        srcdir: str = vfs_path(src)
         if src == "file":
             vfs.create_dir(srcdir)
+            self.assertTrue(vfs.is_dir(srcdir))
         else:
             vfs.create_bucket(srcdir)
+            self.assertTrue(vfs.is_bucket(srcdir))
         srcfile: str = f"{srcdir}/testfile"
         vfs.touch(srcfile)
         self.assertTrue(vfs.isfile(srcfile))
@@ -154,23 +168,18 @@ def test_copy_across(self, src: str, dst: str):
             self.assertEqual(handle.read(), contents)
 
         # Copy src -> dst and assert the file contents are unchanged.
-        dstdir: str = vfs_path(dst, prefix="tiledb-copy-dst")
+        dstdir: str = vfs_path(dst)
         if dst == "file":
             vfs.create_dir(dstdir)
+            self.assertTrue(vfs.is_dir(dstdir))
         else:
             vfs.create_bucket(dstdir)
+            self.assertTrue(vfs.is_bucket(dstdir))
         dstfile: str = f"{dstdir}/testfile"
-        vfs.copy_file(srcfile, dstfile)
+        vfs.copy_dir(srcdir, dstdir)
         with vfs.open(dstfile) as handle:
             self.assertEqual(handle.read(), contents)
 
-        # Copy back dst -> src and assert the file contents are unchanged.
-        vfs.remove_file(srcfile)
-        self.assertFalse(vfs.isfile(srcfile))
-        vfs.copy_file(dstfile, srcfile)
-        with vfs.open(srcfile) as handle:
-            self.assertEqual(handle.read(), contents)
-
         # Clean up
         if src == "file":
             vfs.remove_dir(srcdir)