Merge pull request #6 from zweihander/plainfiles

feat(dl): add plainfiles cache

Author: ernado

cmd/getdoc/main.go

@@ -28,11 +28,6 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	defer func() {
-		if err := client.Close(); err != nil {
-			panic(err)
-		}
-	}()
 
 	ctx := context.Background()
 	fmt.Println("Extracting")

dl/dl.go

@@ -20,44 +20,48 @@ import (
 type Client struct {
 	rate     ratelimit.Limiter
 	http     HTTPClient
-	cache    *cache.Cacher
+	cache    cache.Cacher
 	readonly bool
 }
 
 type Options struct {
 	Client   HTTPClient
 	Path     string
 	Readonly bool
+	FromZip  bool
 }
 
 type HTTPClient interface {
 	Do(req *http.Request) (*http.Response, error)
 }
 
 func NewClient(opt Options) (*Client, error) {
-	c, err := cache.NewCacher(opt.Path, opt.Readonly)
-	if err != nil {
-		return nil, err
-	}
-
 	if opt.Client == nil {
 		opt.Client = http.DefaultClient
 	}
 
-	return &Client{
+	c := &Client{
 		http:     opt.Client,
 		readonly: opt.Readonly,
 		rate:     ratelimit.New(10),
-		cache:    c,
-	}, nil
-}
+	}
 
-var ErrReadOnly = errors.New("write operation in read only mode")
+	if opt.FromZip {
+		zipCache, err := cache.NewFromZip(opt.Path)
+		if err != nil {
+			return nil, err
+		}
+
+		c.cache = zipCache
+	} else {
+		c.cache = cache.NewFromDirectory(opt.Path)
+	}
 
-func (c *Client) Close() error {
-	return c.cache.Close()
+	return c, nil
 }
 
+var ErrReadOnly = errors.New("write operation in read only mode")
+
 // NoLayer can be passed as "layer" argument.
 const NoLayer = 0
 
@@ -117,9 +121,9 @@ func (c *Client) download(ctx context.Context, layer int, key string) ([]byte, e
 //
 // Blank key is invalid.
 func (c *Client) Get(ctx context.Context, layer int, key string) ([]byte, error) {
-	cacheKey := fmt.Sprintf("%d/%s", layer, key)
-	if layer == NoLayer {
-		cacheKey = "default-layer/" + key
+	cacheKey := key
+	if layer != NoLayer {
+		cacheKey = fmt.Sprintf("%d/%s", layer, key)
 	}
 
 	// Trying to get from cache.

dl/dl_test.go

@@ -18,15 +18,11 @@ func TestClient_Download(t *testing.T) {
 		Client:   unusableHTTPClient{},
 		Path:     filepath.Join("_testdata", "121.zip"),
 		Readonly: true,
+		FromZip:  true,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer func() {
-		if err := c.Close(); err != nil {
-			t.Fatal(err)
-		}
-	}()
 
 	data, err := c.Get(context.Background(), 121, "schema")
 	if err != nil {

dl/internal/cache/cache.go

@@ -0,0 +1,10 @@
+package cache
+
+import "errors"
+
+type Cacher interface {
+	Get(key string) ([]byte, error)
+	Set(key string, value []byte) error
+}
+
+var ErrNotFound = errors.New("not found")

dl/internal/cache/cacher.go

@@ -0,0 +1,35 @@
+package cache
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+type PlainFiles struct {
+	path string
+}
+
+func NewFromDirectory(path string) PlainFiles {
+	return PlainFiles{path}
+}
+
+func (p PlainFiles) Get(key string) ([]byte, error) {
+	path := filepath.Join(p.path, key)
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return nil, ErrNotFound
+		}
+	}
+
+	return ioutil.ReadFile(path)
+}
+
+func (p PlainFiles) Set(key string, value []byte) error {
+	path := filepath.Join(p.path, key)
+	if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(path, value, 0600)
+}

dl/internal/cache/plain.go

@@ -4,8 +4,6 @@ import (
 	"archive/zip"
 	"bytes"
 	"io/ioutil"
-	"os"
-	"path/filepath"
 )
 
 func filesFromZip(p string) (map[string][]byte, error) {
@@ -42,32 +40,3 @@ func filesFromZip(p string) (map[string][]byte, error) {
 
 	return files, nil
 }
-
-func dumpZip(files map[string][]byte, path string) error {
-	if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
-		return err
-	}
-
-	f, err := os.Create(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	w := zip.NewWriter(f)
-	defer w.Close()
-
-	for path, data := range files {
-		path = filepath.ToSlash(path)
-		fw, err := w.Create(path)
-		if err != nil {
-			return err
-		}
-
-		if _, err := fw.Write(data); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

dl/internal/cache/utils.go

@@ -0,0 +1,33 @@
+package cache
+
+import (
+	"fmt"
+)
+
+type ZipReadonly struct {
+	files map[string][]byte
+}
+
+func NewFromZip(path string) (*ZipReadonly, error) {
+	files, err := filesFromZip(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ZipReadonly{
+		files: files,
+	}, nil
+}
+
+func (c *ZipReadonly) Get(key string) ([]byte, error) {
+	data, found := c.files[key]
+	if !found {
+		return nil, ErrNotFound
+	}
+
+	return data, nil
+}
+
+func (c *ZipReadonly) Set(key string, value []byte) error {
+	return fmt.Errorf("unsupported operation: set")
+}

dl/internal/cache/zip.go

@@ -22,15 +22,11 @@ func TestExtract(t *testing.T) {
 		Client:   unusableHTTPClient{},
 		Path:     filepath.Join("dl", "_testdata", "121.zip"),
 		Readonly: true,
+		FromZip:  true,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer func() {
-		if err := c.Close(); err != nil {
-			t.Fatal(err)
-		}
-	}()
 
 	doc, err := ExtractLayer(context.Background(), 121, c)
 	if err != nil {