implement xxh64

src/build/gen_sources.mvel

@@ -56,22 +56,26 @@ def generate_hc_compressors() {
 }
 
 def generate_xxhash() {
-  compiledTemplate = get_template("xxhash.template");
-  for (type : ["Safe", "Unsafe"]) {
-    dest = dest_file("xxhash/XXHash32Java" + type + ".java");
-    args = new HashMap();
-    args.put("type", type);
-    execute_template(compiledTemplate, dest, args);
+  for (bitness : ["32", "64"]) {
+    compiledTemplate = get_template("xxhash" + bitness + ".template");
+    for (type : ["Safe", "Unsafe"]) {
+      dest = dest_file("xxhash/XXHash" + bitness + "Java" + type + ".java");
+      args = new HashMap();
+      args.put("type", type);
+      execute_template(compiledTemplate, dest, args);
+    }
   }
 }
 
 def generate_streaming_xxhash() {
-  compiledTemplate = get_template("xxhash_streaming.template");
-  for (type : ["Safe", "Unsafe"]) {
-    dest = dest_file("xxhash/StreamingXXHash32Java" + type + ".java");
-    args = new HashMap();
-    args.put("type", type);
-    execute_template(compiledTemplate, dest, args);
+  for (bitness : ["32", "64"]) {
+    compiledTemplate = get_template("xxhash" + bitness + "_streaming.template");
+    for (type : ["Safe", "Unsafe"]) {
+      dest = dest_file("xxhash/StreamingXXHash" + bitness + "Java" + type + ".java");
+      args = new HashMap();
+      args.put("type", type);
+      execute_template(compiledTemplate, dest, args);
+    }
   }
 }
 

src/build/source_templates/xxhash64.template

@@ -0,0 +1,104 @@
+// Auto-generated: DO NOT EDIT
+
+package net.jpountz.xxhash;
+
+import static net.jpountz.xxhash.XXHashConstants.*;
+@if{ type == "Unsafe" }
+import static net.jpountz.util.UnsafeUtils.*;
+import static net.jpountz.util.Utils.checkRange;
+import static net.jpountz.util.Utils.rotateLeft;
+@else{}
+import static net.jpountz.util.Utils.*;
+@end{}
+
+/**
+ * {@link XXHash64} implementation.
+ */
+final class XXHash64Java${type} extends XXHash64 {
+
+  public static final XXHash64 INSTANCE = new XXHash64Java${type}();
+
+  @Override
+  public long hash(byte[] buf, int off, int len, long seed) {
+    checkRange(buf, off, len);
+
+    final int end = off + len;
+    long h64;
+
+    if (len >= 32) {
+      final int limit = end - 32;
+      long v1 = seed + PRIME64_1 + PRIME64_2;
+      long v2 = seed + PRIME64_2;
+      long v3 = seed + 0;
+      long v4 = seed - PRIME64_1;
+      do {
+        v1 += readLongLE(buf, off) * PRIME64_2;
+        v1 = rotateLeft(v1, 31);
+        v1 *= PRIME64_1;
+        off += 8;
+
+        v2 += readLongLE(buf, off) * PRIME64_2;
+        v2 = rotateLeft(v2, 31);
+        v2 *= PRIME64_1;
+        off += 8;
+
+        v3 += readLongLE(buf, off) * PRIME64_2;
+        v3 = rotateLeft(v3, 31);
+        v3 *= PRIME64_1;
+        off += 8;
+
+        v4 += readLongLE(buf, off) * PRIME64_2;
+        v4 = rotateLeft(v4, 31);
+        v4 *= PRIME64_1;
+        off += 8;
+      } while (off <= limit);
+
+      h64 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18);
+
+      v1 *= PRIME64_2; v1 = rotateLeft(v1, 31); v1 *= PRIME64_1; h64 ^= v1;
+      h64 = h64 * PRIME64_1 + PRIME64_4;
+
+      v2 *= PRIME64_2; v2 = rotateLeft(v2, 31); v2 *= PRIME64_1; h64 ^= v2;
+      h64 = h64 * PRIME64_1 + PRIME64_4;
+
+      v3 *= PRIME64_2; v3 = rotateLeft(v3, 31); v3 *= PRIME64_1; h64 ^= v3;
+      h64 = h64 * PRIME64_1 + PRIME64_4;
+
+      v4 *= PRIME64_2; v4 = rotateLeft(v4, 31); v4 *= PRIME64_1; h64 ^= v4;
+      h64 = h64 * PRIME64_1 + PRIME64_4;
+    } else {
+      h64 = seed + PRIME64_5;
+    }
+
+    h64 += len;
+
+    while (off <= end - 8) {
+      long k1 = readLongLE(buf, off);
+	  k1 *= PRIME64_2; k1 = rotateLeft(k1, 31); k1 *= PRIME64_1; h64 ^= k1;
+      h64 = rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4;
+      off += 8;
+    }
+
+    if (off <= end - 4) {
+      h64 ^= (readIntLE(buf, off) & 0xFFFFFFFFL) * PRIME64_1;
+      h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3;
+      off += 4;
+    }
+
+    while (off < end) {
+      h64 ^= (buf[off] & 0xFF) * PRIME64_5;
+      h64 = rotateLeft(h64, 11) * PRIME64_1;
+      ++off;
+    }
+
+    h64 ^= h64 >>> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >>> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >>> 32;
+
+    return h64;
+  }
+
+}
+

src/build/source_templates/xxhash64_streaming.template

@@ -0,0 +1,170 @@
+// Auto-generated: DO NOT EDIT
+
+package net.jpountz.xxhash;
+
+import static net.jpountz.xxhash.XXHashConstants.*;
+@if{ type == "Unsafe" }
+import static net.jpountz.util.UnsafeUtils.*;
+import static net.jpountz.util.Utils.checkRange;
+import static net.jpountz.util.Utils.rotateLeft;
+@else{}
+import static net.jpountz.util.Utils.*;
+@end{}
+
+/**
+ * Streaming xxhash.
+ */
+final class StreamingXXHash64Java${type} extends AbstractStreamingXXHash64Java {
+
+  static class Factory implements StreamingXXHash64.Factory {
+
+    public static final StreamingXXHash64.Factory INSTANCE = new Factory();
+
+    @Override
+    public StreamingXXHash64 newStreamingHash(long seed) {
+      return new StreamingXXHash64Java${type}(seed);
+    }
+
+  }
+
+  StreamingXXHash64Java${type}(long seed) {
+    super(seed);
+  }
+
+  @Override
+  public long getValue() {
+    long h64;
+    if (totalLen >= 32) {
+      long v1 = this.v1;
+      long v2 = this.v2;
+      long v3 = this.v3;
+      long v4 = this.v4;
+
+      h64 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18);
+
+      v1 *= PRIME64_2; v1 = rotateLeft(v1, 31); v1 *= PRIME64_1; h64 ^= v1;
+      h64 = h64*PRIME64_1 + PRIME64_4;
+
+      v2 *= PRIME64_2; v2 = rotateLeft(v2, 31); v2 *= PRIME64_1; h64 ^= v2;
+      h64 = h64*PRIME64_1 + PRIME64_4;
+
+      v3 *= PRIME64_2; v3 = rotateLeft(v3, 31); v3 *= PRIME64_1; h64 ^= v3;
+      h64 = h64*PRIME64_1 + PRIME64_4;
+
+      v4 *= PRIME64_2; v4 = rotateLeft(v4, 31); v4 *= PRIME64_1; h64 ^= v4;
+      h64 = h64*PRIME64_1 + PRIME64_4;
+    } else {
+      h64 = seed + PRIME64_5;
+    }
+
+    h64 += totalLen;
+
+    int off = 0;
+    while (off <= memSize - 8) {
+      long k1 = readLongLE(memory, off);
+	  k1 *= PRIME64_2; k1 = rotateLeft(k1, 31); k1 *= PRIME64_1; h64 ^= k1;
+      h64 = rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4;
+      off += 8;
+    }
+
+    if (off <= memSize - 4) {
+      h64 ^= (readIntLE(memory, off) & 0xFFFFFFFFL) * PRIME64_1;
+      h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3;
+      off += 4;
+    }
+
+    while (off < memSize) {
+      h64 ^= (memory[off] & 0xFF) * PRIME64_5;
+      h64 = rotateLeft(h64, 11) * PRIME64_1;
+      ++off;
+    }
+
+    h64 ^= h64 >>> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >>> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >>> 32;
+
+    return h64;
+  }
+
+  @Override
+  public void update(byte[] buf, int off, int len) {
+    checkRange(buf, off, len);
+
+    totalLen += len;
+
+    if (memSize + len < 32) { // fill in tmp buffer
+      System.arraycopy(buf, off, memory, memSize, len);
+      memSize += len;
+      return;
+    }
+
+    final int end = off + len;
+
+    if (memSize > 0) { // data left from previous update
+      System.arraycopy(buf, off, memory, memSize, 32 - memSize);
+
+      v1 += readLongLE(memory, 0) * PRIME64_2;
+      v1 = rotateLeft(v1, 31);
+      v1 *= PRIME64_1;
+
+      v2 += readLongLE(memory, 8) * PRIME64_2;
+      v2 = rotateLeft(v2, 31);
+      v2 *= PRIME64_1;
+
+      v3 += readLongLE(memory, 16) * PRIME64_2;
+      v3 = rotateLeft(v3, 31);
+      v3 *= PRIME64_1;
+
+      v4 += readLongLE(memory, 24) * PRIME64_2;
+      v4 = rotateLeft(v4, 31);
+      v4 *= PRIME64_1;
+
+      off += 32 - memSize;
+      memSize = 0;
+    }
+
+    {
+      final int limit = end - 32;
+      long v1 = this.v1;
+      long v2 = this.v2;
+      long v3 = this.v3;
+      long v4 = this.v4;
+
+      while (off <= limit) {
+        v1 += readLongLE(buf, off) * PRIME64_2;
+        v1 = rotateLeft(v1, 31);
+        v1 *= PRIME64_1;
+        off += 8;
+
+        v2 += readLongLE(buf, off) * PRIME64_2;
+        v2 = rotateLeft(v2, 31);
+        v2 *= PRIME64_1;
+        off += 8;
+
+        v3 += readLongLE(buf, off) * PRIME64_2;
+        v3 = rotateLeft(v3, 31);
+        v3 *= PRIME64_1;
+        off += 8;
+
+        v4 += readLongLE(buf, off) * PRIME64_2;
+        v4 = rotateLeft(v4, 31);
+        v4 *= PRIME64_1;
+        off += 8;
+      }
+
+      this.v1 = v1;
+      this.v2 = v2;
+      this.v3 = v3;
+      this.v4 = v4;
+    }
+
+    if (off < end) {
+      System.arraycopy(buf, off, memory, 0, end - off);
+      memSize = end - off;
+    }
+  }
+
+}
+

src/java-unsafe/net/jpountz/util/UnsafeUtils.java

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import static java.lang.Integer.reverseBytes;
 import static net.jpountz.util.Utils.NATIVE_BYTE_ORDER;
 
 import java.lang.reflect.Field;
@@ -69,6 +68,14 @@ public static long readLong(byte[] src, int srcOff) {
     return UNSAFE.getLong(src, BYTE_ARRAY_OFFSET + srcOff);
   }
 
+  public static long readLongLE(byte[] src, int srcOff) {
+    long i = readLong(src, srcOff);
+    if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
+      i = Long.reverseBytes(i);
+    }
+    return i;
+  }
+
   public static void writeLong(byte[] dest, int destOff, long value) {
     UNSAFE.putLong(dest, BYTE_ARRAY_OFFSET + destOff, value);
   }
@@ -80,7 +87,7 @@ public static int readInt(byte[] src, int srcOff) {
   public static int readIntLE(byte[] src, int srcOff) {
     int i = readInt(src, srcOff);
     if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
-      i = reverseBytes(i);
+      i = Integer.reverseBytes(i);
     }
     return i;
   }

src/java/net/jpountz/util/Utils.java

@@ -72,6 +72,11 @@ public static int readInt(byte[] buf, int i) {
     }
   }
 
+  public static long readLongLE(byte[] buf, int i) {
+    return (buf[i] & 0xFFL) | ((buf[i+1] & 0xFFL) << 8) | ((buf[i+2] & 0xFFL) << 16) | ((buf[i+3] & 0xFFL) << 24)
+         | ((buf[i+4] & 0xFFL) << 32) | ((buf[i+5] & 0xFFL) << 40) | ((buf[i+6] & 0xFFL) << 48) | ((buf[i+7] & 0xFFL) << 56);
+  }
+
   public static void writeShortLittleEndian(byte[] buf, int off, int v) {
     buf[off++] = (byte) v;
     buf[off++] = (byte) (v >>> 8);
@@ -97,4 +102,7 @@ public static int readShort(short[] buf, int off) {
     return buf[off] & 0xFFFF;
   }
 
+  public static long rotateLeft(long val, int x) {
+    return (val << x) | (val >>> (64-x));
+  }
 }