Merge pull request #482 from qiniu/etag_v2

bachue · web-flow · commit 46ea0ec07129 · 2020-07-24T14:41:19.000+08:00
Etag v2
diff --git a/src/main/java/com/qiniu/storage/FixBlockUploader.java b/src/main/java/com/qiniu/storage/FixBlockUploader.java
@@ -29,14 +29,12 @@ public class FixBlockUploader {
     private String host = null;
 
     /**
-     * @param blockSize     must be multiples of 4M.
+     * @param blockSize     block size, eg: 1024 * 1024 * 8.
      * @param configuration Nullable, if null, then create a new one.
      * @param client        Nullable, if null, then create a new one with configuration.
      * @param recorder      Nullable.
      */
     public FixBlockUploader(int blockSize, Configuration configuration, Client client, Recorder recorder) {
-        assert blockSize > 0 && blockSize % (4 * 1024 * 1024) == 0 : "blockSize must be multiples of 4M ";
-
         if (configuration == null) {
             configuration = new Configuration();
         }
diff --git a/src/main/java/com/qiniu/util/EtagV2.java b/src/main/java/com/qiniu/util/EtagV2.java
@@ -0,0 +1,134 @@
+package com.qiniu.util;
+
+import java.io.*;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+
+public class EtagV2 {
+    private EtagV2() {
+    }
+
+    public static String data(byte[] data, int offset, int length, long blockSize) {
+        try {
+            return stream(new ByteArrayInputStream(data, offset, length), length, blockSize);
+        } catch (IOException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public static String data(byte[] data, long blockSize) {
+        return data(data, 0, data.length, blockSize);
+    }
+
+    public static String data(byte[] data, int offset, int length, long[] parts) {
+        try {
+            return stream(new ByteArrayInputStream(data, offset, length), length, parts);
+        } catch (IOException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public static String data(byte[] data, long[] parts) {
+        return data(data, 0, data.length, parts);
+    }
+
+    public static String file(String filePath, long blockSize) throws IOException {
+        return file(new File(filePath), blockSize);
+    }
+
+    public static String file(String filePath, long[] parts) throws IOException {
+        return file(new File(filePath), parts);
+    }
+
+    public static String file(File file, long blockSize) throws IOException {
+        FileInputStream fi = null;
+        try {
+            fi = new FileInputStream(file);
+            return stream(fi, file.length(), blockSize);
+        } finally {
+            if (fi != null) {
+                try {
+                    fi.close();
+                } catch (Throwable t) {
+                }
+            }
+        }
+    }
+
+    public static String file(File file, long[] parts) throws IOException {
+        FileInputStream fi = null;
+        try {
+            fi = new FileInputStream(file);
+            return stream(fi, file.length(), parts);
+        } finally {
+            if (fi != null) {
+                try {
+                    fi.close();
+                } catch (Throwable t) {
+                }
+            }
+        }
+    }
+
+
+    public static String stream(InputStream in, long len, long blockSize) throws IOException {
+        // // 和 is4MBParts(new long[]{len}) 实质一样，只有一个 part 的简化版 // //
+        if (blockSize == 1024 * 1024 * 4 || (len <= blockSize && len <= 1024 * 1024 * 4)) {
+            return Etag.stream(in, len);
+        }
+        int size = (int) ((len + blockSize - 1) / blockSize);
+        long[] parts = new long[size];
+        Arrays.fill(parts, 0, size - 1, blockSize);
+        parts[size - 1] = len % blockSize;
+
+        return etagV2(in, len, parts);
+    }
+
+    public static String stream(InputStream in, long len, long[] parts) throws IOException {
+        if (is4MBParts(parts)) {
+            return Etag.stream(in, len);
+        }
+
+        return etagV2(in, len, parts);
+    }
+
+    private static boolean is4MBParts(long[] parts) {
+        int idx = 0;
+        int last = parts.length - 1;
+        for (long part : parts) {
+            if (idx != last && part != 1024 * 1024 * 4 || part > 1024 * 1024 * 4) {
+                return false;
+            }
+            idx += 1;
+        }
+        return true;
+    }
+
+    private static String etagV2(InputStream in, long len, long[] parts) throws IOException {
+        long partSize = 0;
+        for (long part : parts) {
+            partSize += part;
+        }
+        if (len != partSize) {
+            throw new IOException("etag calc failed: size not equal with part size");
+        }
+
+        MessageDigest sha1;
+        try {
+            sha1 = MessageDigest.getInstance("sha-1");
+        } catch (NoSuchAlgorithmException e) {
+            throw new AssertionError(e);
+        }
+        for (long part : parts) {
+            String partEtag = Etag.stream(in, part);
+            byte[] bytes = UrlSafeBase64.decode(partEtag);
+            sha1.update(bytes, 1, bytes.length - 1);
+        }
+        byte[] digest = sha1.digest();
+        byte[] ret = new byte[digest.length + 1];
+        ret[0] = (byte) 0x9e;
+        System.arraycopy(digest, 0, ret, 1, digest.length);
+        return UrlSafeBase64.encodeToString(ret);
+    }
+}
diff --git a/src/test/java/com/qiniu/storage/FixBlockUploaderWithRecorderTest.java b/src/test/java/com/qiniu/storage/FixBlockUploaderWithRecorderTest.java
@@ -5,7 +5,7 @@
 import com.qiniu.http.Client;
 import com.qiniu.http.Response;
 import com.qiniu.storage.persistent.FileRecorder;
-import com.qiniu.util.Etag;
+import com.qiniu.util.EtagV2;
 import com.qiniu.util.StringMap;
 import com.qiniu.util.UrlSafeBase64;
 import org.junit.Before;
@@ -96,7 +96,7 @@ public void breakThenUpload(final ExecutorService pool1, final ExecutorService p
         final File f = TempFile.createFileOld(size);
         final FixBlockUploader.FileBlockData fbd = new FixBlockUploader.FileBlockData(blockSize, f);
         System.out.println(f.getAbsolutePath());
-        final String etag = Etag.file(f);
+        final String etag = EtagV2.file(f, blockSize);
         final String returnBody = "{\"key\":\"$(key)\",\"hash\":\"$(etag)\",\"fsize\":\"$(fsize)\""
                 + ",\"fname\":\"$(fname)\",\"mimeType\":\"$(mimeType)\"}";
 
diff --git a/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java b/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java
@@ -7,7 +7,7 @@
 import com.qiniu.storage.BucketManager;
 import com.qiniu.storage.Configuration;
 import com.qiniu.storage.FixBlockUploader;
-import com.qiniu.util.Etag;
+import com.qiniu.util.EtagV2;
 import com.qiniu.util.StringMap;
 import org.junit.Assert;
 import org.junit.Before;
@@ -24,7 +24,7 @@
 import static org.junit.Assert.assertEquals;
 
 public class FixBlockUploaderTest {
-    int blockSize = 1024 * 1024 * 8;
+    int blockSize = 1024 * 1021 * 7;
     Configuration config;
     Client client;
     FixBlockUploader up;
@@ -147,7 +147,7 @@ private void template(boolean isStream, int size, boolean https, boolean fixFile
             f = TempFile.createFileOld(size);
         }
         System.out.println(f.getAbsolutePath());
-        final String etag = Etag.file(f);
+        final String etag = EtagV2.file(f, blockSize);
         System.out.println(etag);
         final String returnBody = "{\"key\":\"$(key)\",\"hash\":\"$(etag)\",\"fsize\":\"$(fsize)\""
                 + ",\"fname\":\"$(fname)\",\"mimeType\":\"$(mimeType)\"}";
@@ -200,7 +200,7 @@ private void template(boolean isStream, int size, boolean https, boolean fixFile
     @Test
     public void testEmptyKey() throws IOException {
         File f = TempFile.createFileOld(1);
-        String etag = Etag.file(f);
+        String etag = EtagV2.file(f, blockSize);
         String token = TestConfig.testAuth.uploadToken(bucket, null);
         Response res = up.upload(f, token, "");
         System.out.println(res.getInfo());
@@ -212,7 +212,7 @@ public void testEmptyKey() throws IOException {
     @Test
     public void testNullKey() throws IOException {
         File f = TempFile.createFile(2);
-        String etag = Etag.file(f);
+        String etag = EtagV2.file(f, blockSize);
         String token = TestConfig.testAuth.uploadToken(bucket, null);
         Response res = up.upload(f, token, null);
         System.out.println(res.getInfo());
@@ -224,7 +224,7 @@ public void testNullKey() throws IOException {
     @Test
     public void testKey2() throws IOException {
         File f = TempFile.createFile(2);
-        String etag = Etag.file(f);
+        String etag = EtagV2.file(f, blockSize);
         String token = TestConfig.testAuth.uploadToken(bucket, "err");
         try {
             Response res = up.upload(f, token, null);
@@ -238,7 +238,7 @@ public void testKey2() throws IOException {
     @Test
     public void testMeat() throws IOException {
         File f = TempFile.createFile(1);
-        String etag = Etag.file(f);
+        String etag = EtagV2.file(f, blockSize);
         String returnBody = "{\"key\":\"$(key)\",\"hash\":\"$(etag)\",\"fsize\":\"$(fsize)\""
                 + ",\"fname\":\"$(x:biubiu)_$(fname)\",\"mimeType\":\"$(mimeType)\",\"biu2biu\":\"$(x:biu2biu)\"}";
 
