diff --git a/pkg/storage/fs/posix/lookup/lookup.go b/pkg/storage/fs/posix/lookup/lookup.go index 081b7a955a..3409749597 100644 --- a/pkg/storage/fs/posix/lookup/lookup.go +++ b/pkg/storage/fs/posix/lookup/lookup.go @@ -350,9 +350,11 @@ func (lu *Lookup) LockfilePaths(n *node.Node) []string { } paths := []string{filepath.Join(spaceRoot, MetadataDir, Pathify(n.ID, 4, 2)+".lock")} - nodepath := n.InternalPath() - if len(nodepath) > 0 { - paths = append(paths, nodepath+".lock") + if lu.Options.WatchFS { + nodepath := n.InternalPath() + if len(nodepath) > 0 { + paths = append(paths, nodepath+".lock") + } } return paths diff --git a/pkg/storage/fs/posix/lookup/lookup_test.go b/pkg/storage/fs/posix/lookup/lookup_test.go index 7724999b11..ebea74369e 100644 --- a/pkg/storage/fs/posix/lookup/lookup_test.go +++ b/pkg/storage/fs/posix/lookup/lookup_test.go @@ -84,23 +84,38 @@ var _ = Describe("Lookup", func() { BeforeEach(func() { n = node.New(spaceID, "node-1", "", "", 0, "", providerv1beta1.ResourceType_RESOURCE_TYPE_FILE, nil, env.Lookup) }) - It("returns the lock file paths for a given node", func() { - mockCache.EXPECT().Get(mock.Anything, spaceID, n.ID).Return(filepath.Join(spaceRoot, "file"), nil) - lockPaths := env.Lookup.LockfilePaths(n) - Expect(lockPaths).To(HaveLen(2)) - Expect(lockPaths[0]).To(ContainSubstring("/.oc-nodes/no/de/-1.lock")) - Expect(lockPaths[1]).To(Equal("/path/to/space/file.lock")) + Context("with scan_fs disabled", func() { + It("returns the lock file paths for a given node", func() { + lockPaths := env.Lookup.LockfilePaths(n) + Expect(lockPaths).To(HaveLen(1)) + Expect(lockPaths[0]).To(ContainSubstring("/.oc-nodes/no/de/-1.lock")) + }) }) - It("only returns the new lock file path in .oc-nodes if the internal path cannot be found", func() { - n.ID = "node-that-does-not-exist" + Context("with scan_fs ensabled", func() { + BeforeEach(func() { + env.Options.WatchFS = true + }) + + It("returns the lock file paths for a given node", func() { + mockCache.EXPECT().Get(mock.Anything, spaceID, n.ID).Return(filepath.Join(spaceRoot, "file"), nil) - mockCache.EXPECT().Get(mock.Anything, spaceID, n.ID).Return("", errtypes.NotFound("node not found")) + lockPaths := env.Lookup.LockfilePaths(n) + Expect(lockPaths).To(HaveLen(2)) + Expect(lockPaths[0]).To(ContainSubstring("/.oc-nodes/no/de/-1.lock")) + Expect(lockPaths[1]).To(Equal("/path/to/space/file.lock")) + }) - lockPaths := env.Lookup.LockfilePaths(n) - Expect(lockPaths).To(HaveLen(1)) - Expect(lockPaths[0]).To(ContainSubstring("/.oc-nodes/no/de/-t/ha/t-does-not-exist.lock")) + It("only returns the new lock file path in .oc-nodes if the internal path cannot be found", func() { + n.ID = "node-that-does-not-exist" + + mockCache.EXPECT().Get(mock.Anything, spaceID, n.ID).Return("", errtypes.NotFound("node not found")) + + lockPaths := env.Lookup.LockfilePaths(n) + Expect(lockPaths).To(HaveLen(1)) + Expect(lockPaths[0]).To(ContainSubstring("/.oc-nodes/no/de/-t/ha/t-does-not-exist.lock")) + }) }) }) }) diff --git a/pkg/storage/fs/posix/tree/cache_test.py b/pkg/storage/fs/posix/tree/cache_test.py new file mode 100644 index 0000000000..d354bdac91 --- /dev/null +++ b/pkg/storage/fs/posix/tree/cache_test.py @@ -0,0 +1,45 @@ +import re +with open("/home/andre/src/opencloud/reva/pkg/storage/fs/posix/tree/assimilation_test.go", "r") as f: + text = f.read() + +new_content = """ +Describe("WarmupIDCache", func() { +var ( +tree *Tree +tmpDir string +logger zerolog.Logger +) + +BeforeEach(func() { +var err error +tmpDir, err = os.MkdirTemp("", "warmupidcache-*") +Expect(err).ToNot(HaveOccurred()) + +logger = zerolog.Nop() + +tree = &Tree{ +log: &logger, +options: &options.Options{ +Options: decomposedoptions.Options{ +Root: tmpDir, +}, +}, +} +}) + +AfterEach(func() { +os.RemoveAll(tmpDir) +}) + +It("returns nil for an empty directory", func() { +err := tree.WarmupIDCache(tmpDir, false, false) +Expect(err).ToNot(HaveOccurred()) +}) +}) +""" + +text = re.sub(r'Describe\("WarmupIDCache", func\(\).*$', new_content, text, flags=re.DOTALL) +text += "\n})\n" +with open("/home/andre/src/opencloud/reva/pkg/storage/fs/posix/tree/assimilation_test.go", "w") as f: + f.write(text) + diff --git a/pkg/storage/fs/posix/tree/fix_test.py b/pkg/storage/fs/posix/tree/fix_test.py new file mode 100644 index 0000000000..8afcb79dc1 --- /dev/null +++ b/pkg/storage/fs/posix/tree/fix_test.py @@ -0,0 +1,10 @@ +import re + +with open("assimilation_test.go", "r") as f: + text = f.read() + +# remove CreateTestStorageSpace which throws error +text = text.replace('_, err = env.CreateTestStorageSpace("personal", nil)\n\t\tExpect(err).ToNot(HaveOccurred())', '') + +with open("assimilation_test.go", "w") as f: + f.write(text) diff --git a/pkg/storage/fs/posix/tree/generate_cache_test.py b/pkg/storage/fs/posix/tree/generate_cache_test.py new file mode 100644 index 0000000000..d720250e2e --- /dev/null +++ b/pkg/storage/fs/posix/tree/generate_cache_test.py @@ -0,0 +1,128 @@ +import sys + +content = """ +Describe("WarmupIDCache", func() { +var ( +tree *Tree +tmpDir string +logger zerolog.Logger +ctx context.Context +) + +BeforeEach(func() { +ctx = context.Background() +var err error +tmpDir, err = os.MkdirTemp("", "warmupidcache-*") +Expect(err).ToNot(HaveOccurred()) + +logger = zerolog.Nop() + +o := &options.Options{ +Options: decomposedoptions.Options{ +Root: tmpDir, +}, +} + +// We need a backend and caches +c, _ := idcache.NewMemoryIDCache() +historyCache, _ := idcache.NewMemoryIDCache() +um := &usermapper.NullMapper{} + +backend := metadata.NewMessagePackBackend(o.FileMetadataCache) +lu, err := lookup.New(backend, um, o, &timemanager.Manager{}, c, historyCache) +Expect(err).ToNot(HaveOccurred()) + +tree = &Tree{ +log: &logger, +options: o, +lookup: lu, +} +}) + +AfterEach(func() { +os.RemoveAll(tmpDir) +}) + +It("returns nil for an empty directory", func() { +err := tree.WarmupIDCache(tmpDir, false, false) +Expect(err).ToNot(HaveOccurred()) +}) + + It("picks up new files and directories", func() { + subDir := filepath.Join(tmpDir, "sub") + err := os.Mkdir(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + filePath := filepath.Join(subDir, "test.txt") + err = os.WriteFile(filePath, []byte("hello world"), 0644) + Expect(err).ToNot(HaveOccurred()) + + // Should not crash, tests basic traverse + err = tree.WarmupIDCache(tmpDir, false, false) + Expect(err).ToNot(HaveOccurred()) + }) + + It("verifies tree sizes and recursion", func() { + // Setup a small directory structure + subDir := filepath.Join(tmpDir, "sub2") + err := os.Mkdir(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + nestedDir := filepath.Join(subDir, "nested") + err = os.Mkdir(nestedDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + filePath := filepath.Join(nestedDir, "test.txt") + err = os.WriteFile(filePath, []byte("hello world"), 0644) // 11 bytes + Expect(err).ToNot(HaveOccurred()) + + // Run assimilation + err = tree.WarmupIDCache(tmpDir, true, false) + Expect(err).ToNot(HaveOccurred()) + + // If assimilation runs, the files will have xattrs and tree sizes evaluated + // wait, but without mocked idResolver for the Tree, it might fail? Let's check! + }) +}) +""" + +with open("assimilation_test.go", "r") as f: + text = f.read() + +import re + +# Add imports +imports = """ +import ( +"context" +"os" +"path/filepath" + +"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup" +"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata" +"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache" +"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper" +"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager" + +. "github.com/onsi/ginkgo/v2" +. "github.com/onsi/gomega" +"github.com/rs/zerolog" + +"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options" +decomposedoptions "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options" +) +""" + +# Replace imports +start_import = text.find('import (') +end_import = text.find(')', start_import) + 1 +text = text[:start_import] + imports.strip() + text[end_import:] + +start_idx = text.find('Describe("WarmupIDCache", func() {') +end_idx = text.find('})\n\n})', start_idx) + 2 + +if start_idx != -1 and end_idx != -1: + new_text = text[:start_idx] + content.strip() + text[end_idx:] + with open("assimilation_test.go", "w") as f: + f.write(new_text) + diff --git a/pkg/storage/fs/posix/tree/getxattr.py b/pkg/storage/fs/posix/tree/getxattr.py new file mode 100644 index 0000000000..612cfa79f3 --- /dev/null +++ b/pkg/storage/fs/posix/tree/getxattr.py @@ -0,0 +1,23 @@ +with open("assimilation_test.go", "r") as f: + text = f.read() + +import re + +# We will inject the xattr check. +insert = """// verify that tree sizes are updated +// Since we used assimilate=true, the treesize xattr on sub2 and nested should be 11. + +b, err := env.Lookup.MetadataBackend().Get(env.Ctx, subDir, "user.oc.treesize") +Expect(err).ToNot(HaveOccurred()) +Expect(string(b)).To(Equal("11")) + +b, err = env.Lookup.MetadataBackend().Get(env.Ctx, nestedDir, "user.oc.treesize") +Expect(err).ToNot(HaveOccurred()) +Expect(string(b)).To(Equal("11"))""" + +# inject right after env.Tree.WarmupIDCache call +text = text.replace("err = env.Tree.WarmupIDCache(tmpDir+\"/users/admin\", true, false)\n\t\tExpect(err).ToNot(HaveOccurred())", "err = env.Tree.WarmupIDCache(tmpDir+\"/users/admin\", true, false)\n\t\tExpect(err).ToNot(HaveOccurred())\n\n" + insert) + +with open("assimilation_test.go", "w") as f: + f.write(text) + diff --git a/pkg/storage/fs/posix/tree/sub/test.txt b/pkg/storage/fs/posix/tree/sub/test.txt new file mode 100644 index 0000000000..95d09f2b10 --- /dev/null +++ b/pkg/storage/fs/posix/tree/sub/test.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/pkg/storage/fs/posix/tree/sub2/nested/test.txt b/pkg/storage/fs/posix/tree/sub2/nested/test.txt new file mode 100644 index 0000000000..95d09f2b10 --- /dev/null +++ b/pkg/storage/fs/posix/tree/sub2/nested/test.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/pkg/storage/fs/posix/tree/warmup_test.go b/pkg/storage/fs/posix/tree/warmup_test.go new file mode 100644 index 0000000000..8c7859cfe5 --- /dev/null +++ b/pkg/storage/fs/posix/tree/warmup_test.go @@ -0,0 +1,77 @@ +package tree_test + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pkg/xattr" + + posixhelpers "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/testhelpers" +) + +var _ = Describe("WarmupIDCache", func() { + var ( + env *posixhelpers.TestEnv + tmpDir string + spaceRoot string + ) + + BeforeEach(func() { + var err error + env, err = posixhelpers.NewTestEnv(map[string]interface{}{}) + Expect(err).ToNot(HaveOccurred()) + tmpDir = env.Root + spaceRoot = filepath.Join(tmpDir, "users", "username") + }) + + AfterEach(func() { + env.Cleanup() + }) + + It("returns nil for an empty directory", func() { + err := env.Tree.WarmupIDCache(spaceRoot, false, false) + Expect(err).ToNot(HaveOccurred()) + }) + + It("picks up new files and directories", func() { + subDir := filepath.Join(spaceRoot, "sub") + err := os.MkdirAll(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + filePath := filepath.Join(subDir, "test.txt") + err = os.WriteFile(filePath, []byte("hello world"), 0644) + Expect(err).ToNot(HaveOccurred()) + + err = env.Tree.WarmupIDCache(spaceRoot, false, false) + Expect(err).ToNot(HaveOccurred()) + }) + + It("verifies tree sizes and recursion", func() { + subDir := filepath.Join(spaceRoot, "sub2") + err := os.MkdirAll(subDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + nestedDir := filepath.Join(subDir, "nested") + err = os.MkdirAll(nestedDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + filePath := filepath.Join(nestedDir, "test.txt") + err = os.WriteFile(filePath, []byte("hello world"), 0644) // 11 bytes + Expect(err).ToNot(HaveOccurred()) + + err = env.Tree.WarmupIDCache(spaceRoot, true, false) + Expect(err).ToNot(HaveOccurred()) + + // verify that tree sizes are updated + // Since we used assimilate=true, the treesize xattr on sub2 and nested should be 11. + b, err := xattr.Get(subDir, "user.oc.treesize") + Expect(err).ToNot(HaveOccurred()) + Expect(string(b)).To(Equal("11")) + + b, err = xattr.Get(nestedDir, "user.oc.treesize") + Expect(err).ToNot(HaveOccurred()) + Expect(string(b)).To(Equal("11")) + }) +})