From ca0829b664b35e0639e6ee668710de40b4066d64 Mon Sep 17 00:00:00 2001 From: Linnaea Von Lavia Date: Sun, 3 Aug 2014 12:07:02 +0800 Subject: [PATCH] implement xxh64 --- src/build/gen_sources.mvel | 28 +- .../{xxhash.template => xxhash32.template} | 0 ...g.template => xxhash32_streaming.template} | 0 src/build/source_templates/xxhash64.template | 104 ++ .../xxhash64_streaming.template | 170 +++ .../net/jpountz/util/UnsafeUtils.java | 11 +- src/java/net/jpountz/util/Utils.java | 8 + .../xxhash/AbstractStreamingXXHash64Java.java | 43 + .../net/jpountz/xxhash/StreamingXXHash64.java | 111 ++ .../jpountz/xxhash/StreamingXXHash64JNI.java | 71 + src/java/net/jpountz/xxhash/XXHash64.java | 35 + src/java/net/jpountz/xxhash/XXHash64JNI.java | 29 + .../net/jpountz/xxhash/XXHashConstants.java | 5 + .../net/jpountz/xxhash/XXHashFactory.java | 20 + src/java/net/jpountz/xxhash/XXHashJNI.java | 6 + src/jni/net_jpountz_xxhash_XXHashJNI.c | 96 +- .../{XXHashTest.java => XXHash32Test.java} | 2 +- src/test/net/jpountz/xxhash/XXHash64Test.java | 151 ++ .../net/jpountz/xxhash/XXHashFactoryTest.java | 6 + src/xxhash/Makefile | 32 +- src/xxhash/bench.c | 142 +- src/xxhash/xxhash.c | 1307 +++++++++++------ src/xxhash/xxhash.h | 331 ++--- 23 files changed, 2021 insertions(+), 687 deletions(-) rename src/build/source_templates/{xxhash.template => xxhash32.template} (100%) rename src/build/source_templates/{xxhash_streaming.template => xxhash32_streaming.template} (100%) create mode 100644 src/build/source_templates/xxhash64.template create mode 100644 src/build/source_templates/xxhash64_streaming.template create mode 100644 src/java/net/jpountz/xxhash/AbstractStreamingXXHash64Java.java create mode 100644 src/java/net/jpountz/xxhash/StreamingXXHash64.java create mode 100644 src/java/net/jpountz/xxhash/StreamingXXHash64JNI.java create mode 100644 src/java/net/jpountz/xxhash/XXHash64.java create mode 100644 src/java/net/jpountz/xxhash/XXHash64JNI.java rename src/test/net/jpountz/xxhash/{XXHashTest.java => XXHash32Test.java} (98%) create mode 100644 src/test/net/jpountz/xxhash/XXHash64Test.java diff --git a/src/build/gen_sources.mvel b/src/build/gen_sources.mvel index 40bd7806..0f326711 100644 --- a/src/build/gen_sources.mvel +++ b/src/build/gen_sources.mvel @@ -56,22 +56,26 @@ def generate_hc_compressors() { } def generate_xxhash() { - compiledTemplate = get_template("xxhash.template"); - for (type : ["Safe", "Unsafe"]) { - dest = dest_file("xxhash/XXHash32Java" + type + ".java"); - args = new HashMap(); - args.put("type", type); - execute_template(compiledTemplate, dest, args); + for (bitness : ["32", "64"]) { + compiledTemplate = get_template("xxhash" + bitness + ".template"); + for (type : ["Safe", "Unsafe"]) { + dest = dest_file("xxhash/XXHash" + bitness + "Java" + type + ".java"); + args = new HashMap(); + args.put("type", type); + execute_template(compiledTemplate, dest, args); + } } } def generate_streaming_xxhash() { - compiledTemplate = get_template("xxhash_streaming.template"); - for (type : ["Safe", "Unsafe"]) { - dest = dest_file("xxhash/StreamingXXHash32Java" + type + ".java"); - args = new HashMap(); - args.put("type", type); - execute_template(compiledTemplate, dest, args); + for (bitness : ["32", "64"]) { + compiledTemplate = get_template("xxhash" + bitness + "_streaming.template"); + for (type : ["Safe", "Unsafe"]) { + dest = dest_file("xxhash/StreamingXXHash" + bitness + "Java" + type + ".java"); + args = new HashMap(); + args.put("type", type); + execute_template(compiledTemplate, dest, args); + } } } diff --git a/src/build/source_templates/xxhash.template b/src/build/source_templates/xxhash32.template similarity index 100% rename from src/build/source_templates/xxhash.template rename to src/build/source_templates/xxhash32.template diff --git a/src/build/source_templates/xxhash_streaming.template b/src/build/source_templates/xxhash32_streaming.template similarity index 100% rename from src/build/source_templates/xxhash_streaming.template rename to src/build/source_templates/xxhash32_streaming.template diff --git a/src/build/source_templates/xxhash64.template b/src/build/source_templates/xxhash64.template new file mode 100644 index 00000000..493e84f3 --- /dev/null +++ b/src/build/source_templates/xxhash64.template @@ -0,0 +1,104 @@ +// Auto-generated: DO NOT EDIT + +package net.jpountz.xxhash; + +import static net.jpountz.xxhash.XXHashConstants.*; +@if{ type == "Unsafe" } +import static net.jpountz.util.UnsafeUtils.*; +import static net.jpountz.util.Utils.checkRange; +import static net.jpountz.util.Utils.rotateLeft; +@else{} +import static net.jpountz.util.Utils.*; +@end{} + +/** + * {@link XXHash64} implementation. + */ +final class XXHash64Java${type} extends XXHash64 { + + public static final XXHash64 INSTANCE = new XXHash64Java${type}(); + + @Override + public long hash(byte[] buf, int off, int len, long seed) { + checkRange(buf, off, len); + + final int end = off + len; + long h64; + + if (len >= 32) { + final int limit = end - 32; + long v1 = seed + PRIME64_1 + PRIME64_2; + long v2 = seed + PRIME64_2; + long v3 = seed + 0; + long v4 = seed - PRIME64_1; + do { + v1 += readLongLE(buf, off) * PRIME64_2; + v1 = rotateLeft(v1, 31); + v1 *= PRIME64_1; + off += 8; + + v2 += readLongLE(buf, off) * PRIME64_2; + v2 = rotateLeft(v2, 31); + v2 *= PRIME64_1; + off += 8; + + v3 += readLongLE(buf, off) * PRIME64_2; + v3 = rotateLeft(v3, 31); + v3 *= PRIME64_1; + off += 8; + + v4 += readLongLE(buf, off) * PRIME64_2; + v4 = rotateLeft(v4, 31); + v4 *= PRIME64_1; + off += 8; + } while (off <= limit); + + h64 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); + + v1 *= PRIME64_2; v1 = rotateLeft(v1, 31); v1 *= PRIME64_1; h64 ^= v1; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; v2 = rotateLeft(v2, 31); v2 *= PRIME64_1; h64 ^= v2; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; v3 = rotateLeft(v3, 31); v3 *= PRIME64_1; h64 ^= v3; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; v4 = rotateLeft(v4, 31); v4 *= PRIME64_1; h64 ^= v4; + h64 = h64 * PRIME64_1 + PRIME64_4; + } else { + h64 = seed + PRIME64_5; + } + + h64 += len; + + while (off <= end - 8) { + long k1 = readLongLE(buf, off); + k1 *= PRIME64_2; k1 = rotateLeft(k1, 31); k1 *= PRIME64_1; h64 ^= k1; + h64 = rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + off += 8; + } + + if (off <= end - 4) { + h64 ^= (readIntLE(buf, off) & 0xFFFFFFFFL) * PRIME64_1; + h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + off += 4; + } + + while (off < end) { + h64 ^= (buf[off] & 0xFF) * PRIME64_5; + h64 = rotateLeft(h64, 11) * PRIME64_1; + ++off; + } + + h64 ^= h64 >>> 33; + h64 *= PRIME64_2; + h64 ^= h64 >>> 29; + h64 *= PRIME64_3; + h64 ^= h64 >>> 32; + + return h64; + } + +} + diff --git a/src/build/source_templates/xxhash64_streaming.template b/src/build/source_templates/xxhash64_streaming.template new file mode 100644 index 00000000..078efe8d --- /dev/null +++ b/src/build/source_templates/xxhash64_streaming.template @@ -0,0 +1,170 @@ +// Auto-generated: DO NOT EDIT + +package net.jpountz.xxhash; + +import static net.jpountz.xxhash.XXHashConstants.*; +@if{ type == "Unsafe" } +import static net.jpountz.util.UnsafeUtils.*; +import static net.jpountz.util.Utils.checkRange; +import static net.jpountz.util.Utils.rotateLeft; +@else{} +import static net.jpountz.util.Utils.*; +@end{} + +/** + * Streaming xxhash. + */ +final class StreamingXXHash64Java${type} extends AbstractStreamingXXHash64Java { + + static class Factory implements StreamingXXHash64.Factory { + + public static final StreamingXXHash64.Factory INSTANCE = new Factory(); + + @Override + public StreamingXXHash64 newStreamingHash(long seed) { + return new StreamingXXHash64Java${type}(seed); + } + + } + + StreamingXXHash64Java${type}(long seed) { + super(seed); + } + + @Override + public long getValue() { + long h64; + if (totalLen >= 32) { + long v1 = this.v1; + long v2 = this.v2; + long v3 = this.v3; + long v4 = this.v4; + + h64 = rotateLeft(v1, 1) + rotateLeft(v2, 7) + rotateLeft(v3, 12) + rotateLeft(v4, 18); + + v1 *= PRIME64_2; v1 = rotateLeft(v1, 31); v1 *= PRIME64_1; h64 ^= v1; + h64 = h64*PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; v2 = rotateLeft(v2, 31); v2 *= PRIME64_1; h64 ^= v2; + h64 = h64*PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; v3 = rotateLeft(v3, 31); v3 *= PRIME64_1; h64 ^= v3; + h64 = h64*PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; v4 = rotateLeft(v4, 31); v4 *= PRIME64_1; h64 ^= v4; + h64 = h64*PRIME64_1 + PRIME64_4; + } else { + h64 = seed + PRIME64_5; + } + + h64 += totalLen; + + int off = 0; + while (off <= memSize - 8) { + long k1 = readLongLE(memory, off); + k1 *= PRIME64_2; k1 = rotateLeft(k1, 31); k1 *= PRIME64_1; h64 ^= k1; + h64 = rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4; + off += 8; + } + + if (off <= memSize - 4) { + h64 ^= (readIntLE(memory, off) & 0xFFFFFFFFL) * PRIME64_1; + h64 = rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3; + off += 4; + } + + while (off < memSize) { + h64 ^= (memory[off] & 0xFF) * PRIME64_5; + h64 = rotateLeft(h64, 11) * PRIME64_1; + ++off; + } + + h64 ^= h64 >>> 33; + h64 *= PRIME64_2; + h64 ^= h64 >>> 29; + h64 *= PRIME64_3; + h64 ^= h64 >>> 32; + + return h64; + } + + @Override + public void update(byte[] buf, int off, int len) { + checkRange(buf, off, len); + + totalLen += len; + + if (memSize + len < 32) { // fill in tmp buffer + System.arraycopy(buf, off, memory, memSize, len); + memSize += len; + return; + } + + final int end = off + len; + + if (memSize > 0) { // data left from previous update + System.arraycopy(buf, off, memory, memSize, 32 - memSize); + + v1 += readLongLE(memory, 0) * PRIME64_2; + v1 = rotateLeft(v1, 31); + v1 *= PRIME64_1; + + v2 += readLongLE(memory, 8) * PRIME64_2; + v2 = rotateLeft(v2, 31); + v2 *= PRIME64_1; + + v3 += readLongLE(memory, 16) * PRIME64_2; + v3 = rotateLeft(v3, 31); + v3 *= PRIME64_1; + + v4 += readLongLE(memory, 24) * PRIME64_2; + v4 = rotateLeft(v4, 31); + v4 *= PRIME64_1; + + off += 32 - memSize; + memSize = 0; + } + + { + final int limit = end - 32; + long v1 = this.v1; + long v2 = this.v2; + long v3 = this.v3; + long v4 = this.v4; + + while (off <= limit) { + v1 += readLongLE(buf, off) * PRIME64_2; + v1 = rotateLeft(v1, 31); + v1 *= PRIME64_1; + off += 8; + + v2 += readLongLE(buf, off) * PRIME64_2; + v2 = rotateLeft(v2, 31); + v2 *= PRIME64_1; + off += 8; + + v3 += readLongLE(buf, off) * PRIME64_2; + v3 = rotateLeft(v3, 31); + v3 *= PRIME64_1; + off += 8; + + v4 += readLongLE(buf, off) * PRIME64_2; + v4 = rotateLeft(v4, 31); + v4 *= PRIME64_1; + off += 8; + } + + this.v1 = v1; + this.v2 = v2; + this.v3 = v3; + this.v4 = v4; + } + + if (off < end) { + System.arraycopy(buf, off, memory, 0, end - off); + memSize = end - off; + } + } + +} + diff --git a/src/java-unsafe/net/jpountz/util/UnsafeUtils.java b/src/java-unsafe/net/jpountz/util/UnsafeUtils.java index 5de8f049..f6c5812d 100644 --- a/src/java-unsafe/net/jpountz/util/UnsafeUtils.java +++ b/src/java-unsafe/net/jpountz/util/UnsafeUtils.java @@ -14,7 +14,6 @@ * limitations under the License. */ -import static java.lang.Integer.reverseBytes; import static net.jpountz.util.Utils.NATIVE_BYTE_ORDER; import java.lang.reflect.Field; @@ -69,6 +68,14 @@ public static long readLong(byte[] src, int srcOff) { return UNSAFE.getLong(src, BYTE_ARRAY_OFFSET + srcOff); } + public static long readLongLE(byte[] src, int srcOff) { + long i = readLong(src, srcOff); + if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { + i = Long.reverseBytes(i); + } + return i; + } + public static void writeLong(byte[] dest, int destOff, long value) { UNSAFE.putLong(dest, BYTE_ARRAY_OFFSET + destOff, value); } @@ -80,7 +87,7 @@ public static int readInt(byte[] src, int srcOff) { public static int readIntLE(byte[] src, int srcOff) { int i = readInt(src, srcOff); if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) { - i = reverseBytes(i); + i = Integer.reverseBytes(i); } return i; } diff --git a/src/java/net/jpountz/util/Utils.java b/src/java/net/jpountz/util/Utils.java index 828812ef..15ed4cb8 100644 --- a/src/java/net/jpountz/util/Utils.java +++ b/src/java/net/jpountz/util/Utils.java @@ -72,6 +72,11 @@ public static int readInt(byte[] buf, int i) { } } + public static long readLongLE(byte[] buf, int i) { + return (buf[i] & 0xFFL) | ((buf[i+1] & 0xFFL) << 8) | ((buf[i+2] & 0xFFL) << 16) | ((buf[i+3] & 0xFFL) << 24) + | ((buf[i+4] & 0xFFL) << 32) | ((buf[i+5] & 0xFFL) << 40) | ((buf[i+6] & 0xFFL) << 48) | ((buf[i+7] & 0xFFL) << 56); + } + public static void writeShortLittleEndian(byte[] buf, int off, int v) { buf[off++] = (byte) v; buf[off++] = (byte) (v >>> 8); @@ -97,4 +102,7 @@ public static int readShort(short[] buf, int off) { return buf[off] & 0xFFFF; } + public static long rotateLeft(long val, int x) { + return (val << x) | (val >>> (64-x)); + } } diff --git a/src/java/net/jpountz/xxhash/AbstractStreamingXXHash64Java.java b/src/java/net/jpountz/xxhash/AbstractStreamingXXHash64Java.java new file mode 100644 index 00000000..97e294dd --- /dev/null +++ b/src/java/net/jpountz/xxhash/AbstractStreamingXXHash64Java.java @@ -0,0 +1,43 @@ +package net.jpountz.xxhash; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static net.jpountz.xxhash.XXHashConstants.PRIME64_1; +import static net.jpountz.xxhash.XXHashConstants.PRIME64_2; + +abstract class AbstractStreamingXXHash64Java extends StreamingXXHash64 { + + int memSize; + long v1, v2, v3, v4; + long totalLen; + final byte[] memory; + + AbstractStreamingXXHash64Java(long seed) { + super(seed); + memory = new byte[32]; + reset(); + } + + @Override + public void reset() { + v1 = seed + PRIME64_1 + PRIME64_2; + v2 = seed + PRIME64_2; + v3 = seed + 0; + v4 = seed - PRIME64_1; + totalLen = 0; + memSize = 0; + } + +} diff --git a/src/java/net/jpountz/xxhash/StreamingXXHash64.java b/src/java/net/jpountz/xxhash/StreamingXXHash64.java new file mode 100644 index 00000000..c68fb4d1 --- /dev/null +++ b/src/java/net/jpountz/xxhash/StreamingXXHash64.java @@ -0,0 +1,111 @@ +package net.jpountz.xxhash; + +import java.util.zip.Checksum; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +/** + * Streaming interface for {@link XXHash64}. + *

+ * This API is compatible with the {@link XXHash64 block API} and the following + * code samples are equivalent: + * 

+ *   long hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, long seed) {
+ *     return xxhashFactory.hash64().hash(buf, off, len, seed);
+ *   }
+ *
+ * 
+ *   long hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, long seed) {
+ *     StreamingXXHash64 sh64 = xxhashFactory.newStreamingHash64(seed);
+ *     sh64.update(buf, off, len);
+ *     return sh64.getValue();
+ *   }
+ *
+ *

+ * Instances of this class are not thread-safe. + */ +public abstract class StreamingXXHash64 { + + interface Factory { + + StreamingXXHash64 newStreamingHash(long seed); + + } + + final long seed; + + StreamingXXHash64(long seed) { + this.seed = seed; + } + + /** + * Get the value of the checksum. + */ + public abstract long getValue(); + + /** + * Update the value of the hash with buf[off:off+len]. + */ + public abstract void update(byte[] buf, int off, int len); + + /** + * Reset this instance to the state it had right after instantiation. The + * seed remains unchanged. + */ + public abstract void reset(); + + @Override + public String toString() { + return getClass().getSimpleName() + "(seed=" + seed + ")"; + } + + /** + * Return a {@link Checksum} view of this instance. Modifications to the view + * will modify this instance too and vice-versa. + */ + public final Checksum asChecksum() { + return new Checksum() { + + @Override + public long getValue() { + return StreamingXXHash64.this.getValue(); + } + + @Override + public void reset() { + StreamingXXHash64.this.reset(); + } + + @Override + public void update(int b) { + StreamingXXHash64.this.update(new byte[] {(byte) b}, 0, 1); + } + + @Override + public void update(byte[] b, int off, int len) { + StreamingXXHash64.this.update(b, off, len); + } + + @Override + public String toString() { + return StreamingXXHash64.this.toString(); + } + + }; + } + +} \ No newline at end of file diff --git a/src/java/net/jpountz/xxhash/StreamingXXHash64JNI.java b/src/java/net/jpountz/xxhash/StreamingXXHash64JNI.java new file mode 100644 index 00000000..a8ae202d --- /dev/null +++ b/src/java/net/jpountz/xxhash/StreamingXXHash64JNI.java @@ -0,0 +1,71 @@ +package net.jpountz.xxhash; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +final class StreamingXXHash64JNI extends StreamingXXHash64 { + + static class Factory implements StreamingXXHash64.Factory { + + public static final StreamingXXHash64.Factory INSTANCE = new Factory(); + + @Override + public StreamingXXHash64 newStreamingHash(long seed) { + return new StreamingXXHash64JNI(seed); + } + + } + + private long state; + + StreamingXXHash64JNI(long seed) { + super(seed); + state = XXHashJNI.XXH64_init(seed); + } + + private void checkState() { + if (state == 0) { + throw new AssertionError("Already finalized"); + } + } + + @Override + public void reset() { + checkState(); + XXHashJNI.XXH64_free(state); + state = XXHashJNI.XXH64_init(seed); + } + + @Override + public long getValue() { + checkState(); + return XXHashJNI.XXH64_intermediateDigest(state); + } + + @Override + public void update(byte[] bytes, int off, int len) { + checkState(); + XXHashJNI.XXH64_update(state, bytes, off, len); + } + + @Override + protected void finalize() throws Throwable { + super.finalize(); + // free memory + XXHashJNI.XXH64_free(state); + state = 0; + } + +} diff --git a/src/java/net/jpountz/xxhash/XXHash64.java b/src/java/net/jpountz/xxhash/XXHash64.java new file mode 100644 index 00000000..d414db3d --- /dev/null +++ b/src/java/net/jpountz/xxhash/XXHash64.java @@ -0,0 +1,35 @@ +package net.jpountz.xxhash; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A 64-bits hash. + *

+ * Instances of this class are thread-safe. + */ +public abstract class XXHash64 { + + /** + * Compute the 64-bits hash of buf[off:off+len] using seed + * seed. + */ + public abstract long hash(byte[] buf, int off, int len, long seed); + + @Override + public String toString() { + return getClass().getSimpleName(); + } + +} diff --git a/src/java/net/jpountz/xxhash/XXHash64JNI.java b/src/java/net/jpountz/xxhash/XXHash64JNI.java new file mode 100644 index 00000000..87cc38f4 --- /dev/null +++ b/src/java/net/jpountz/xxhash/XXHash64JNI.java @@ -0,0 +1,29 @@ +package net.jpountz.xxhash; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static net.jpountz.util.Utils.checkRange; + +final class XXHash64JNI extends XXHash64 { + + public static final XXHash64 INSTANCE = new XXHash64JNI(); + + @Override + public long hash(byte[] buf, int off, int len, long seed) { + checkRange(buf, off, len); + return XXHashJNI.XXH64(buf, off, len, seed); + } + +} diff --git a/src/java/net/jpountz/xxhash/XXHashConstants.java b/src/java/net/jpountz/xxhash/XXHashConstants.java index 771a0d2c..2e887dfd 100644 --- a/src/java/net/jpountz/xxhash/XXHashConstants.java +++ b/src/java/net/jpountz/xxhash/XXHashConstants.java @@ -23,4 +23,9 @@ enum XXHashConstants { static final int PRIME4 = 668265263; static final int PRIME5 = 374761393; + static final long PRIME64_1 = -7046029288634856825L; //11400714785074694791 + static final long PRIME64_2 = -4417276706812531889L; //14029467366897019727 + static final long PRIME64_3 = 1609587929392839161L; + static final long PRIME64_4 = -8796714831421723037L; //9650029242287828579 + static final long PRIME64_5 = 2870177450012600261L; } diff --git a/src/java/net/jpountz/xxhash/XXHashFactory.java b/src/java/net/jpountz/xxhash/XXHashFactory.java index 656b2312..b67f1dfd 100644 --- a/src/java/net/jpountz/xxhash/XXHashFactory.java +++ b/src/java/net/jpountz/xxhash/XXHashFactory.java @@ -147,12 +147,16 @@ private static T classInstance(String cls) throws NoSuchFieldException, Secu private final String impl; private final XXHash32 hash32; + private final XXHash64 hash64; private final StreamingXXHash32.Factory streamingHash32Factory; + private final StreamingXXHash64.Factory streamingHash64Factory; private XXHashFactory(String impl) throws ClassNotFoundException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { this.impl = impl; hash32 = classInstance("net.jpountz.xxhash.XXHash32" + impl); streamingHash32Factory = classInstance("net.jpountz.xxhash.StreamingXXHash32" + impl + "$Factory"); + hash64 = classInstance("net.jpountz.xxhash.XXHash64" + impl); + streamingHash64Factory = classInstance("net.jpountz.xxhash.StreamingXXHash64" + impl + "$Factory"); // make sure it can run final byte[] bytes = new byte[100]; @@ -164,6 +168,10 @@ private XXHashFactory(String impl) throws ClassNotFoundException, NoSuchFieldExc final StreamingXXHash32 streamingHash32 = newStreamingHash32(seed); streamingHash32.update(bytes, 0, bytes.length); final int h2 = streamingHash32.getValue(); + final long h3 = hash64.hash(bytes, 0, bytes.length, seed); + final StreamingXXHash64 streamingHash64 = newStreamingHash64(seed); + streamingHash64.update(bytes, 0, bytes.length); + final long h4 = streamingHash64.getValue(); if (h1 != h2) { throw new AssertionError(); } @@ -174,6 +182,11 @@ public XXHash32 hash32() { return hash32; } + /** Return a {@link XXHash64} instance. */ + public XXHash64 hash64() { + return hash64; + } + /** * Return a new {@link StreamingXXHash32} instance. */ @@ -181,6 +194,13 @@ public StreamingXXHash32 newStreamingHash32(int seed) { return streamingHash32Factory.newStreamingHash(seed); } + /** + * Return a new {@link StreamingXXHash64} instance. + */ + public StreamingXXHash64 newStreamingHash64(long seed) { + return streamingHash64Factory.newStreamingHash(seed); + } + /** Prints the fastest instance. */ public static void main(String[] args) { System.out.println("Fastest instance is " + fastestInstance()); diff --git a/src/java/net/jpountz/xxhash/XXHashJNI.java b/src/java/net/jpountz/xxhash/XXHashJNI.java index 74c40211..17bc08a0 100644 --- a/src/java/net/jpountz/xxhash/XXHashJNI.java +++ b/src/java/net/jpountz/xxhash/XXHashJNI.java @@ -32,4 +32,10 @@ enum XXHashJNI { static native int XXH32_digest(long state); static native void XXH32_free(long state); + static native long XXH64(byte[] input, int offset, int len, long seed); + static native long XXH64_init(long seed); + static native void XXH64_update(long state, byte[] input, int offset, int len); + static native long XXH64_intermediateDigest(long state); + static native long XXH64_digest(long state); + static native void XXH64_free(long state); } diff --git a/src/jni/net_jpountz_xxhash_XXHashJNI.c b/src/jni/net_jpountz_xxhash_XXHashJNI.c index e8e1eb43..425457bb 100644 --- a/src/jni/net_jpountz_xxhash_XXHashJNI.c +++ b/src/jni/net_jpountz_xxhash_XXHashJNI.c @@ -70,7 +70,7 @@ JNIEXPORT jlong JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1init /* * Class: net_jpountz_xxhash_XXHashJNI - * Method: XXH32_feed + * Method: XXH32_update * Signature: (J[BII)V */ JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1update @@ -90,7 +90,7 @@ JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1update /* * Class: net_jpountz_xxhash_XXHashJNI - * Method: XXH32_getIntermediateResult + * Method: XXH32_intermediateDigest * Signature: (J)I */ JNIEXPORT jint JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1intermediateDigest @@ -102,7 +102,7 @@ JNIEXPORT jint JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1intermediateDige /* * Class: net_jpountz_xxhash_XXHashJNI - * Method: XXH32_result + * Method: XXH32_digest * Signature: (J)I */ JNIEXPORT jint JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1digest @@ -124,3 +124,93 @@ JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1free } +/* + * Class: net_jpountz_xxhash_XXHashJNI + * Method: XXH64 + * Signature: ([BIIJ)J + */ +JNIEXPORT jlong JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64 + (JNIEnv *env, jclass cls, jbyteArray buf, jint off, jint len, jlong seed) { + + char* in; + jlong h64; + + in = (char*) (*env)->GetPrimitiveArrayCritical(env, buf, 0); + if (in == NULL) { + throw_OOM(env); + return 0; + } + + h64 = XXH64(in + off, len, seed); + + (*env)->ReleasePrimitiveArrayCritical(env, buf, in, 0); + + return h64; +} + +/* + * Class: net_jpountz_xxhash_XXHashJNI + * Method: XXH64_init + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64_1init + (JNIEnv *env, jclass cls, jlong seed) { + + return (jlong) XXH64_init(seed); +} + +/* + * Class: net_jpountz_xxhash_XXHashJNI + * Method: XXH64_update + * Signature: (J[BII)V + */ +JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64_1update + (JNIEnv *env, jclass cls, jlong state, jbyteArray src, jint off, jint len) { + + char* in = (char*) (*env)->GetPrimitiveArrayCritical(env, src, 0); + if (in == NULL) { + throw_OOM(env); + return; + } + + XXH64_update((void*) state, in + off, len); + + (*env)->ReleasePrimitiveArrayCritical(env, src, in, 0); + +} + +/* + * Class: net_jpountz_xxhash_XXHashJNI + * Method: XXH64_intermediateDigest + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64_1intermediateDigest + (JNIEnv *env, jclass cls, jlong state) { + + return XXH64_intermediateDigest((void*) state); + +} + +/* + * Class: net_jpountz_xxhash_XXHashJNI + * Method: XXH64_digest + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64_1digest + (JNIEnv *env, jclass cls, jlong state) { + + return XXH64_digest((void*) state); + +} + +/* + * Class: net_jpountz_xxhash_XXHashJNI + * Method: XXH64_free + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64_1free + (JNIEnv *env, jclass cls, jlong state) { + + free((void*) state); + +} diff --git a/src/test/net/jpountz/xxhash/XXHashTest.java b/src/test/net/jpountz/xxhash/XXHash32Test.java similarity index 98% rename from src/test/net/jpountz/xxhash/XXHashTest.java rename to src/test/net/jpountz/xxhash/XXHash32Test.java index a8a36834..3318d108 100644 --- a/src/test/net/jpountz/xxhash/XXHashTest.java +++ b/src/test/net/jpountz/xxhash/XXHash32Test.java @@ -24,7 +24,7 @@ import com.carrotsearch.randomizedtesting.annotations.Repeat; @RunWith(RandomizedRunner.class) -public class XXHashTest extends RandomizedTest { +public class XXHash32Test extends RandomizedTest { private static abstract class StreamingXXHash32Adapter extends XXHash32 { diff --git a/src/test/net/jpountz/xxhash/XXHash64Test.java b/src/test/net/jpountz/xxhash/XXHash64Test.java new file mode 100644 index 00000000..98cad69f --- /dev/null +++ b/src/test/net/jpountz/xxhash/XXHash64Test.java @@ -0,0 +1,151 @@ +package net.jpountz.xxhash; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import net.jpountz.util.Utils; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.carrotsearch.randomizedtesting.RandomizedRunner; +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.Repeat; + +@RunWith(RandomizedRunner.class) +public class XXHash64Test extends RandomizedTest { + + private static abstract class StreamingXXHash64Adapter extends XXHash64 { + + protected abstract StreamingXXHash64 streamingHash(long seed); + + @Override + public long hash(byte[] buf, int off, int len, long seed) { + Utils.checkRange(buf, off, len); + int originalOff = off; + int remainingPasses = randomInt(5); + StreamingXXHash64 h = streamingHash(seed); + final int end = off + len; + while (off < end) { + final int l = randomIntBetween(off, end) - off; + h.update(buf, off, l); + off += l; + if (remainingPasses > 0 && randomInt(5) == 0) { + h.reset(); + --remainingPasses; + off = originalOff; + } + if (randomBoolean()) { + h.getValue(); + } + } + return h.getValue(); + } + + public String toString() { + return streamingHash(0).toString(); + } + + } + + private static XXHash64[] INSTANCES = new XXHash64[] { + XXHashFactory.nativeInstance().hash64(), + XXHashFactory.unsafeInstance().hash64(), + XXHashFactory.safeInstance().hash64(), + new StreamingXXHash64Adapter() { + protected StreamingXXHash64 streamingHash(long seed) { + return XXHashFactory.nativeInstance().newStreamingHash64(seed); + } + }, + new StreamingXXHash64Adapter() { + protected StreamingXXHash64 streamingHash(long seed) { + return XXHashFactory.unsafeInstance().newStreamingHash64(seed); + } + }, + new StreamingXXHash64Adapter() { + protected StreamingXXHash64 streamingHash(long seed) { + return XXHashFactory.safeInstance().newStreamingHash64(seed); + } + } + }; + + @Test + public void testEmpty() { + final long seed = randomLong(); + for (XXHash64 xxHash : INSTANCES) { + xxHash.hash(new byte[0], 0, 0, seed); + } + } + + @Test + @Repeat(iterations = 20) + public void testAIOOBE() { + final long seed = randomLong(); + final int max = randomBoolean() ? 64 : 1000; + final int bufLen = randomIntBetween(1, max); + final byte[] buf = new byte[bufLen]; + for (int i = 0; i < buf.length; ++i) { + buf[i] = randomByte(); + } + final int off = randomInt(buf.length - 1); + final int len = randomInt(buf.length - off); + for (XXHash64 xxHash : INSTANCES) { + xxHash.hash(buf, off, len, seed); + } + } + + @Test + @Repeat(iterations=40) + public void testInstances() { + final int maxLenLog = randomInt(20); + final int bufLen = randomInt(1 << maxLenLog); + byte[] buf = new byte[bufLen]; + for (int i = 0; i < bufLen; ++i) { + buf[i] = randomByte(); + } + final long seed = randomLong(); + final int off = randomIntBetween(0, Math.max(0, bufLen - 1)); + final int len = randomIntBetween(0, bufLen - off); + + final long ref = XXHashFactory.nativeInstance().hash64().hash(buf, off, len, seed); + for (XXHash64 hash : INSTANCES) { + final long h = hash.hash(buf, off, len, seed); + assertEquals(hash.toString(), ref, h); + } + } + + @Test + public void test4GB() { + byte[] bytes = new byte[randomIntBetween(1 << 22, 1 << 26)]; + for (int i = 0; i < bytes.length; ++i) { + bytes[i] = randomByte(); + } + final int off = randomInt(5); + final int len = randomIntBetween(bytes.length - off - 1024, bytes.length - off); + long totalLen = 0; + final long seed = randomLong(); + StreamingXXHash64 hash1 = XXHashFactory.nativeInstance().newStreamingHash64(seed); + StreamingXXHash64 hash2 = XXHashFactory.unsafeInstance().newStreamingHash64(seed); + StreamingXXHash64 hash3 = XXHashFactory.safeInstance().newStreamingHash64(seed); + while (totalLen < (1L << 33)) { + hash1.update(bytes, off, len); + hash2.update(bytes, off, len); + hash3.update(bytes, off, len); + assertEquals(hash2.toString() + " " + totalLen, hash1.getValue(), hash2.getValue()); + assertEquals(hash3.toString() + " " + totalLen, hash1.getValue(), hash3.getValue()); + totalLen += len; + } + } + +} diff --git a/src/test/net/jpountz/xxhash/XXHashFactoryTest.java b/src/test/net/jpountz/xxhash/XXHashFactoryTest.java index 7dd6f5a6..c4102201 100644 --- a/src/test/net/jpountz/xxhash/XXHashFactoryTest.java +++ b/src/test/net/jpountz/xxhash/XXHashFactoryTest.java @@ -25,6 +25,12 @@ public void test() { assertTrue(XXHashFactory.unsafeInstance().newStreamingHash32(0) instanceof StreamingXXHash32JavaUnsafe); assertEquals(XXHash32JavaSafe.INSTANCE, XXHashFactory.safeInstance().hash32()); assertTrue(XXHashFactory.safeInstance().newStreamingHash32(0) instanceof StreamingXXHash32JavaSafe); + assertEquals(XXHash64JNI.INSTANCE, XXHashFactory.nativeInstance().hash64()); + assertTrue(XXHashFactory.nativeInstance().newStreamingHash64(0) instanceof StreamingXXHash64JNI); + assertEquals(XXHash64JavaUnsafe.INSTANCE, XXHashFactory.unsafeInstance().hash64()); + assertTrue(XXHashFactory.unsafeInstance().newStreamingHash64(0) instanceof StreamingXXHash64JavaUnsafe); + assertEquals(XXHash64JavaSafe.INSTANCE, XXHashFactory.safeInstance().hash64()); + assertTrue(XXHashFactory.safeInstance().newStreamingHash64(0) instanceof StreamingXXHash64JavaSafe); } } diff --git a/src/xxhash/Makefile b/src/xxhash/Makefile index 598f5e3f..e5ed5c78 100644 --- a/src/xxhash/Makefile +++ b/src/xxhash/Makefile @@ -24,7 +24,8 @@ # ################################################################ CC=gcc -CFLAGS=-I. -std=c99 -Wall -W -Wundef -Wno-implicit-function-declaration +CFLAGS+= -I. -std=c99 -O3 -Wall -Wextra -Wundef -Wshadow -Wstrict-prototypes + OS := $(shell uname) ifeq ($(OS),Linux) @@ -33,12 +34,35 @@ else EXT =.exe endif +# Minimize test target for Travis CI's Build Matrix +ifeq ($(XXH_TRAVIS_CI_ENV),-m32) +TEST_TARGETS=test-32 +else ifeq ($(XXH_TRAVIS_CI_ENV),-m64) +TEST_TARGETS=test-64 +else +TEST_TARGETS=test-64 test-32 +endif + default: xxHash -all: xxHash +all: xxHash xxHash32 xxHash: xxhash.c bench.c - $(CC) -O2 $(CFLAGS) $^ -o $@$(EXT) + $(CC) $(CFLAGS) $^ -o $@$(EXT) + +xxHash32: xxhash.c bench.c + $(CC) -m32 $(CFLAGS) $^ -o $@$(EXT) + +test: $(TEST_TARGETS) + +test-64: xxHash + ./xxHash bench.c + valgrind ./xxHash -i1 bench.c + +test-32: xxHash32 + ./xxHash32 bench.c clean: - rm -f core *.o xxHash$(EXT) + rm -f core *.o xxHash$(EXT) xxHash32$(EXT) + + diff --git a/src/xxhash/bench.c b/src/xxhash/bench.c index 773177d7..c7f94286 100644 --- a/src/xxhash/bench.c +++ b/src/xxhash/bench.c @@ -25,7 +25,7 @@ You can contact the author at : // Compiler Options //************************************** // Visual warning messages (must be first line) -#define _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS // Under Linux at least, pull in the *64 commands #define _LARGEFILE64_SOURCE @@ -48,7 +48,6 @@ You can contact the author at : # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) #endif - //************************************** // Hash Functions to test //************************************** @@ -56,6 +55,14 @@ You can contact the author at : #define DEFAULTHASH XXH32 #define HASH0 XXH32 +// Making a wrapper to fit into the 32 bit api +unsigned int XXH64_32(const void* key, unsigned int len, unsigned int seed) +{ + unsigned long long hash = XXH64(key, len, seed); + return (unsigned int)(hash & 0xFFFFFFFF); +} +#define HASH1 XXH64_32 + //************************************** // Basic Types @@ -79,14 +86,14 @@ You can contact the author at : //************************************** // Constants //************************************** -#define PROGRAM_NAME "Benchmark utility using xxHash algorithm" +#define PROGRAM_NAME "xxHash tester" #define PROGRAM_VERSION "" #define COMPILED __DATE__ #define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", PROGRAM_NAME, PROGRAM_VERSION, AUTHOR, COMPILED +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", PROGRAM_NAME, (int)(sizeof(void*)*8), PROGRAM_VERSION, AUTHOR, COMPILED #define NBLOOPS 3 // Default number of benchmark iterations -#define TIMELOOP 2000 // Minimum timing per iteration +#define TIMELOOP 2500 // Minimum timing per iteration #define KB *(1U<<10) #define MB *(1U<<20) @@ -99,10 +106,9 @@ You can contact the author at : //************************************** // Local structures //************************************** - struct hashFunctionPrototype { - unsigned int (*hashFunction)(const void*, int, unsigned int); + unsigned int (*hashFunction)(const void*, unsigned int, unsigned int); }; @@ -130,7 +136,7 @@ void BMK_SetNbIterations(int nbLoops) // Benchmark Functions //********************************************************* -static int BMK_GetMilliStart() +static int BMK_GetMilliStart(void) { // Supposed to be portable // Rolls over every ~ 12.1 days (0x100000/24/60/60) @@ -193,7 +199,7 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int selection) { int fileIdx=0; struct hashFunctionPrototype hashP; - unsigned int hashResult=0; + U32 hashResult=0; U64 totals = 0; double totalc = 0.; @@ -214,6 +220,8 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int selection) default: hashP.hashFunction = DEFAULTHASH; } + DISPLAY("Selected fn %d", selection); + // Loop for each file while (fileIdx\r", interationNb, inFileName, (int)benchedSize); + DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH32", (int)benchedSize); // Hash loop milliTime = BMK_GetMilliStart(); @@ -293,15 +301,15 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int selection) } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, inFileName, (int)benchedSize, (double)benchedSize / fastestC / 1000.); + DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000.); } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", inFileName, (int)benchedSize, (double)benchedSize / fastestC / 1000., hashResult); + DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000., hashResult); totals += benchedSize; totalc += fastestC; } - // Bench Unaligned + // Bench Unaligned XXH32 { int interationNb; double fastestC = 100000000.; @@ -328,11 +336,48 @@ int BMK_benchFile(char** fileNamesTable, int nbFiles, int selection) } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "(unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); + DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "(unaligned)", (int)benchedSize-1, (double)(benchedSize-1) / fastestC / 1000.); + DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)benchedSize-1, (double)(benchedSize-1) / fastestC / 1000.); } + // Bench XXH64 + { + int interationNb; + double fastestC = 100000000.; + unsigned long long h64 = 0; + + DISPLAY("\r%79s\r", ""); // Clean display line + for (interationNb = 1; interationNb <= nbIterations; interationNb++) + { + int nbHashes = 0; + int milliTime; + + DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH64", (int)benchedSize); + + // Hash loop + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i<100; i++) + { + h64 = XXH64(alignedBuffer, (int)benchedSize, 0); + nbHashes++; + } + } + milliTime = BMK_GetMilliSpan(milliTime); + if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; + DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000.); + } + DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X%08X\n", "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000., (U32)(h64>>32), (U32)(h64)); + + totals += benchedSize; + totalc += fastestC; + } + free(buffer); } @@ -349,7 +394,7 @@ static void BMK_checkResult(U32 r1, U32 r2) static int nbTests = 1; if (r1==r2) DISPLAY("\rTest%3i : %08X == %08X ok ", nbTests, r1, r2); - else + else { DISPLAY("\rERROR : Test%3i : %08X <> %08X !!!!! \n", nbTests, r1, r2); exit(1); @@ -358,6 +403,41 @@ static void BMK_checkResult(U32 r1, U32 r2) } +static void BMK_checkResult64(U64 r1, U64 r2) +{ + static int nbTests = 1; + + if (r1!=r2) + { + DISPLAY("\rERROR : Test%3i : 64-bits values non equals !!!!! \n", nbTests); + DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2<<32), (U32)r2); + exit(1); + } + nbTests++; +} + + +static void BMK_testSequence64(void* sentence, int len, U64 seed, U64 Nresult) +{ + U64 Dresult; + void* state; + int index; + + Dresult = XXH64(sentence, len, seed); + BMK_checkResult64(Dresult, Nresult); + + state = XXH64_init(seed); + XXH64_update(state, sentence, len); + Dresult = XXH64_digest(state); + BMK_checkResult64(Dresult, Nresult); + + state = XXH64_init(seed); + for (index=0; index>24); - random *= random; + sanityBuffer[i] = (BYTE)(prime>>24); + prime *= prime; } BMK_testSequence(sanityBuffer, 1, 0, 0xB85CBEE5); @@ -400,8 +479,15 @@ static void BMK_sanityCheck() BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x1F1AA412); BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0x498EC8E2); - DISPLAY(" -- all tests ok"); + BMK_testSequence64(sanityBuffer, 1, 0, 0x4FCE394CC88952D8ULL); + BMK_testSequence64(sanityBuffer, 1, PRIME, 0x739840CB819FA723ULL); + BMK_testSequence64(sanityBuffer, 14, 0, 0xCFFA8DB881BC3A3DULL); + BMK_testSequence64(sanityBuffer, 14, PRIME, 0x5B9611585EFCC9CBULL); + BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x0EAB543384F878ADULL); + BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0xCAA65939306F1E21ULL); + DISPLAY("\r%79s\r", ""); // Clean display line + DISPLAY("Sanity check -- all tests ok\n"); } @@ -415,6 +501,7 @@ int usage(char* exename) DISPLAY( " %s [arg] filename\n", exename); DISPLAY( "Arguments :\n"); DISPLAY( " -i# : number of iterations \n"); + DISPLAY( " -s# : Function selection [0,1]. Default is 0 \n"); DISPLAY( " -h : help (this text)\n"); return 0; } @@ -433,6 +520,7 @@ int main(int argc, char** argv) int i, filenamesStart=2; char* input_filename=0; + int fn_selection = 0; // Welcome message DISPLAY( WELCOME_MESSAGE ); @@ -459,6 +547,8 @@ int main(int argc, char** argv) // Modify Nb Iterations (benchmark only) if ( argument[0] =='i' ) { int iters = argument[1] - '0'; BMK_SetNbIterations(iters); continue; } + // select function + if ( argument[0] =='s' ) { fn_selection = argument[1] - '0'; continue; } } // first provided filename is input @@ -469,7 +559,7 @@ int main(int argc, char** argv) // No input filename ==> Error if(!input_filename) { badusage(argv[0]); return 1; } - return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, 0); + if(fn_selection < 0 || fn_selection > 1) { badusage(argv[0]); return 1; } + return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, fn_selection); } - diff --git a/src/xxhash/xxhash.c b/src/xxhash/xxhash.c index 6e92d250..6b60db15 100644 --- a/src/xxhash/xxhash.c +++ b/src/xxhash/xxhash.c @@ -1,475 +1,832 @@ -/* -xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -*/ - - -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. -// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). -#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_USE_UNALIGNED_ACCESS 1 -#endif - -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. -// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. -// This option has a very small performance cost (only measurable on small inputs). -// By default, this option is disabled. To enable it, uncomment below define : -//#define XXH_ACCEPT_NULL_INPUT_POINTER 1 - -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. -// Results are therefore identical for little-endian and big-endian CPU. -// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -// Should endian-independance be of no importance for your application, you may set the #define below to 1. -// It will improve speed for Big-endian CPU. -// This option has no impact on Little_Endian CPU. -#define XXH_FORCE_NATIVE_FORMAT 0 - - -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif - -#ifdef _MSC_VER // Visual Studio -# define FORCE_INLINE static __forceinline -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -//************************************** -// Includes & Memory related functions -//************************************** -#include "xxhash.h" -// Modify the local functions below should you wish to use some other memory related routines -// for malloc(), free() -#include -FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } -FORCE_INLINE void XXH_free (void* p) { free(p); } -// for memcpy() -#include -FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } - - -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct _U32_S { U32 v; } _PACKED U32_S; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif - -#define A32(x) (((U32_S *)(x))->v) - - -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -// Note : although _rotl exists for minGW (GCC under windows), performance seems poor -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -#endif - -#if defined(_MSC_VER) // Visual Studio -# define XXH_swap32 _byteswap_ulong -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -#else -static inline U32 XXH_swap32 (U32 x) { - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff );} -#endif - - -//************************************** -// Constants -//************************************** -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - - -//************************************** -// Architecture Macros -//************************************** -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch - static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) -#endif - - -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations - - -//**************************** -// Memory reads -//**************************** -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); - else - return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } - - -//**************************** -// Simple Hash Functions -//**************************** -FORCE_INLINE U32 XXH32_endian_align(const void* input, int len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U32 h32; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { len=0; p=(const BYTE*)(size_t)16; } -#endif - - if (len>=16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do - { - v1 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } - else - { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p<=bEnd-4) - { - h32 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32(const void* input, int len, U32 seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - void* state = XXH32_init(seed); - XXH32_update(state, input, len); - return XXH32_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -# if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } -# endif - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -//**************************** -// Advanced Hash Functions -//**************************** - -struct XXH_state32_t -{ - U64 total_len; - U32 seed; - U32 v1; - U32 v2; - U32 v3; - U32 v4; - int memsize; - char memory[16]; -}; - - -int XXH32_sizeofState() -{ - XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough - return sizeof(struct XXH_state32_t); -} - - -XXH_errorcode XXH32_resetState(void* state_in, U32 seed) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - state->seed = seed; - state->v1 = seed + PRIME32_1 + PRIME32_2; - state->v2 = seed + PRIME32_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME32_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - - -void* XXH32_init (U32 seed) -{ - void* state = XXH_malloc (sizeof(struct XXH_state32_t)); - XXH32_resetState(state, seed); - return state; -} - - -FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 16) // fill in tmp buffer - { - XXH_memcpy(state->memory + state->memsize, input, len); - state->memsize += len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); - { - const U32* p32 = (const U32*)state->memory; - state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++; - state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; - state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++; - state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do - { - v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->memory, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_errorcode XXH32_update (void* state_in, const void* input, int len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - const BYTE * p = (const BYTE*)state->memory; - BYTE* bEnd = (BYTE*)state->memory + state->memsize; - U32 h32; - - if (state->total_len >= 16) - { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } - else - { - h32 = state->seed + PRIME32_5; - } - - h32 += (U32) state->total_len; - - while (p<=bEnd-4) - { - h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32_intermediateDigest (void* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian); - else - return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian); -} - - -U32 XXH32_digest (void* state_in) -{ - U32 h32 = XXH32_intermediateDigest(state_in); - - XXH_free(state_in); - - return h32; -} +/* +xxHash - Fast Hash algorithm +Copyright (C) 2012-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : http://code.google.com/p/xxhash/ +*/ + + +//************************************** +// Tuning parameters +//************************************** +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. +// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_USE_UNALIGNED_ACCESS 1 +#endif + +// XXH_ACCEPT_NULL_INPUT_POINTER : +// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. +// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. +// This option has a very small performance cost (only measurable on small inputs). +// By default, this option is disabled. To enable it, uncomment below define : +// #define XXH_ACCEPT_NULL_INPUT_POINTER 1 + +// XXH_FORCE_NATIVE_FORMAT : +// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. +// Results are therefore identical for little-endian and big-endian CPU. +// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. +// Should endian-independance be of no importance for your application, you may set the #define below to 1. +// It will improve speed for Big-endian CPU. +// This option has no impact on Little_Endian CPU. +#define XXH_FORCE_NATIVE_FORMAT 0 + +//************************************** +// Compiler Specific Options +//************************************** +// Disable some Visual warning messages +#ifdef _MSC_VER // Visual Studio +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#endif + +#ifdef _MSC_VER // Visual Studio +# define FORCE_INLINE static __forceinline +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + +//************************************** +// Includes & Memory related functions +//************************************** +#include "xxhash.h" +// Modify the local functions below should you wish to use some other memory related routines +// for malloc(), free() +#include +FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } +FORCE_INLINE void XXH_free (void* p) { free(p); } +// for memcpy() +#include +FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# ifdef __IBMC__ +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct _U32_S { U32 v; } _PACKED U32_S; +typedef struct _U64_S { U64 v; } _PACKED U64_S; + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(pop) +#endif + +#define A32(x) (((U32_S *)(x))->v) +#define A64(x) (((U64_S *)(x))->v) + + +//*************************************** +// Compiler-specific Functions and Macros +//*************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +// Note : although _rotl exists for minGW (GCC under windows), performance seems poor +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) // Visual Studio +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static inline U32 XXH_swap32 (U32 x) { + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff );} +static inline U64 XXH_swap64 (U64 x) { + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL);} +#endif + + +//************************************** +// Constants +//************************************** +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + +#define PRIME64_1 11400714785074694791ULL +#define PRIME64_2 14029467366897019727ULL +#define PRIME64_3 1609587929392839161ULL +#define PRIME64_4 9650029242287828579ULL +#define PRIME64_5 2870177450012600261ULL + +//************************************** +// Architecture Macros +//************************************** +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch + static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) +#endif + + +//************************************** +// Macros +//************************************** +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations + + +//**************************** +// Memory reads +//**************************** +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + else + return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } + +FORCE_INLINE U64 XXH_readLE64_align(const U64* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); + else + return endian==XXH_littleEndian ? *ptr : XXH_swap64(*ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const U64* ptr, XXH_endianess endian) { return XXH_readLE64_align(ptr, endian, XXH_unaligned); } + + +//**************************** +// Simple Hash Functions +//**************************** +FORCE_INLINE U32 XXH32_endian_align(const void* input, unsigned int len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align((const U32*)p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)16; } +#endif + + if (len>=16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do + { + v1 += XXH_get32bits(p) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; + v2 += XXH_get32bits(p) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; + v3 += XXH_get32bits(p) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; + v4 += XXH_get32bits(p) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } + else + { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p<=bEnd-4) + { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32(const void* input, unsigned int len, U32 seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + void* state = XXH32_init(seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, unsigned int len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align((const U64*)p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)32; } +#endif + + if (len>=32) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do + { + v1 += XXH_get64bits(p) * PRIME64_2; p+=8; v1 = XXH_rotl64(v1, 31); v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; p+=8; v2 = XXH_rotl64(v2, 31); v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; p+=8; v3 = XXH_rotl64(v3, 31); v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; p+=8; v4 = XXH_rotl64(v4, 31); v4 *= PRIME64_1; + } while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; v1 = XXH_rotl64(v1, 31); v1 *= PRIME64_1; h64 ^= v1; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; v2 = XXH_rotl64(v2, 31); v2 *= PRIME64_1; h64 ^= v2; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; v3 = XXH_rotl64(v3, 31); v3 *= PRIME64_1; h64 ^= v3; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; v4 = XXH_rotl64(v4, 31); v4 *= PRIME64_1; h64 ^= v4; + h64 = h64 * PRIME64_1 + PRIME64_4; + } + else + { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p<=bEnd-8) + { + U64 k1 = XXH_get64bits(p); + k1 *= PRIME64_2; k1 = XXH_rotl64(k1,31); k1 *= PRIME64_1; h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + +#if 1 + + if (p<=bEnd-4) + { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64(const void* input, unsigned int len, unsigned long long seed) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +} + +//**************************** +// Advanced Hash Functions +//**************************** + +struct XXH_state32_t +{ + U64 total_len; + U32 seed; + U32 v1; + U32 v2; + U32 v3; + U32 v4; + int memsize; + char memory[16]; +}; + +struct XXH_state64_t +{ + U64 total_len; + U64 seed; + U64 v1; + U64 v2; + U64 v3; + U64 v4; + int memsize; + char memory[32]; +}; + + +int XXH32_sizeofState(void) +{ + XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough + return sizeof(struct XXH_state32_t); +} + +int XXH64_sizeofState(void) +{ + XXH_STATIC_ASSERT(XXH64_SIZEOFSTATE >= sizeof(struct XXH_state64_t)); // A compilation error here means XXH64_SIZEOFSTATE is not large enough + return sizeof(struct XXH_state64_t); +} + + +XXH_errorcode XXH32_resetState(void* state_in, U32 seed) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + state->seed = seed; + state->v1 = seed + PRIME32_1 + PRIME32_2; + state->v2 = seed + PRIME32_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME32_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + +XXH_errorcode XXH64_resetState(void* state_in, unsigned long long seed) +{ + struct XXH_state64_t * state = (struct XXH_state64_t *) state_in; + state->seed = seed; + state->v1 = seed + PRIME64_1 + PRIME64_2; + state->v2 = seed + PRIME64_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME64_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + + +void* XXH32_init (U32 seed) +{ + void* state = XXH_malloc (sizeof(struct XXH_state32_t)); + XXH32_resetState(state, seed); + return state; +} + +void* XXH64_init (unsigned long long seed) +{ + void* state = XXH_malloc (sizeof(struct XXH_state64_t)); + XXH64_resetState(state, seed); + return state; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) // fill in tmp buffer + { + XXH_memcpy(state->memory + state->memsize, input, len); + state->memsize += len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); + { + const U32* p32 = (const U32*)state->memory; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do + { + v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; + v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; + v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; + v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->memory, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH32_update (void* state_in, const void* input, unsigned int len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + const BYTE * p = (const BYTE*)state->memory; + BYTE* bEnd = (BYTE*)state->memory + state->memsize; + U32 h32; + + if (state->total_len >= 16) + { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } + else + { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p<=bEnd-4) + { + h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32_intermediateDigest (void* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian); + else + return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian); +} + + +U32 XXH32_digest (void* state_in) +{ + U32 h32 = XXH32_intermediateDigest(state_in); + + XXH_free(state_in); + + return h32; +} + + +FORCE_INLINE XXH_errorcode XXH64_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) +{ + struct XXH_state64_t * state = (struct XXH_state64_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) // fill in tmp buffer + { + XXH_memcpy(state->memory + state->memsize, input, len); + state->memsize += len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(state->memory + state->memsize, input, 32-state->memsize); + { + const U64* p64 = (const U64*)state->memory; + state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; state->v1 = XXH_rotl64(state->v1, 31); state->v1 *= PRIME64_1; p64++; + state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; state->v2 = XXH_rotl64(state->v2, 31); state->v2 *= PRIME64_1; p64++; + state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; state->v3 = XXH_rotl64(state->v3, 31); state->v3 *= PRIME64_1; p64++; + state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; state->v4 = XXH_rotl64(state->v4, 31); state->v4 *= PRIME64_1; p64++; + } + p += 32-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-32) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do + { + v1 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; v1 = XXH_rotl64(v1, 31); v1 *= PRIME64_1; p+=8; + v2 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; v2 = XXH_rotl64(v2, 31); v2 *= PRIME64_1; p+=8; + v3 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; v3 = XXH_rotl64(v3, 31); v3 *= PRIME64_1; p+=8; + v4 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; v4 = XXH_rotl64(v4, 31); v4 *= PRIME64_1; p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->memory, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH64_update (void* state_in, const void* input, unsigned int len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U64 XXH64_intermediateDigest_endian (void* state_in, XXH_endianess endian) +{ + struct XXH_state64_t * state = (struct XXH_state64_t *) state_in; + const BYTE * p = (const BYTE*)state->memory; + BYTE* bEnd = (BYTE*)state->memory + state->memsize; + U64 h64; + + if (state->total_len >= 32) + { + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; v1 = XXH_rotl64(v1, 31); v1 *= PRIME64_1; h64 ^= v1; + h64 = h64*PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; v2 = XXH_rotl64(v2, 31); v2 *= PRIME64_1; h64 ^= v2; + h64 = h64*PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; v3 = XXH_rotl64(v3, 31); v3 *= PRIME64_1; h64 ^= v3; + h64 = h64*PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; v4 = XXH_rotl64(v4, 31); v4 *= PRIME64_1; h64 ^= v4; + h64 = h64*PRIME64_1 + PRIME64_4; + } + else + { + h64 = state->seed + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p<=bEnd-8) + { + U64 k1 = XXH_readLE64((const U64*)p, endian); + k1 *= PRIME64_2; k1 = XXH_rotl64(k1,31); k1 *= PRIME64_1; h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p<=bEnd-4) + { + h64 ^= (U64)(XXH_readLE32((const U32*)p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64_intermediateDigest (void* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_intermediateDigest_endian(state_in, XXH_littleEndian); + else + return XXH64_intermediateDigest_endian(state_in, XXH_bigEndian); +} + + +unsigned long long XXH64_digest (void* state_in) +{ + U64 h64 = XXH64_intermediateDigest(state_in); + + XXH_free(state_in); + + return h64; +} + diff --git a/src/xxhash/xxhash.h b/src/xxhash/xxhash.h index 4435906b..63a83c9c 100644 --- a/src/xxhash/xxhash.h +++ b/src/xxhash/xxhash.h @@ -1,164 +1,167 @@ -/* - xxHash - Fast Hash algorithm - Header File - Copyright (C) 2012-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. -*/ - -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - - -//**************************** -// Type -//**************************** -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - - -//**************************** -// Simple Hash Functions -//**************************** - -unsigned int XXH32 (const void* input, int len, unsigned int seed); - -/* -XXH32() : - Calculate the 32-bits hash of sequence of length "len" stored at memory address "input". - The memory between input & input+len must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - This function successfully passes all SMHasher tests. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s - Note that "len" is type "int", which means it is limited to 2^31-1. - If your data is larger, use the advanced functions below. -*/ - - - -//**************************** -// Advanced Hash Functions -//**************************** - -void* XXH32_init (unsigned int seed); -XXH_errorcode XXH32_update (void* state, const void* input, int len); -unsigned int XXH32_digest (void* state); - -/* -These functions calculate the xxhash of an input provided in several small packets, -as opposed to an input provided as a single block. - -It must be started with : -void* XXH32_init() -The function returns a pointer which holds the state of calculation. - -This pointer must be provided as "void* state" parameter for XXH32_update(). -XXH32_update() can be called as many times as necessary. -The user must provide a valid (allocated) input. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. -Note that "len" is type "int", which means it is limited to 2^31-1. -If your data is larger, it is recommended to chunk your data into blocks -of size for example 2^30 (1GB) to avoid any "int" overflow issue. - -Finally, you can end the calculation anytime, by using XXH32_digest(). -This function returns the final 32-bits hash. -You must provide the same "void* state" parameter created by XXH32_init(). -Memory will be freed by XXH32_digest(). -*/ - - -int XXH32_sizeofState(void); -XXH_errorcode XXH32_resetState(void* state, unsigned int seed); - -#define XXH32_SIZEOFSTATE 48 -typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t; -/* -These functions allow user application to make its own allocation for state. - -XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state. -Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer. -This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state. - -For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()), -use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields. -*/ - - -unsigned int XXH32_intermediateDigest (void* state); -/* -This function does the same as XXH32_digest(), generating a 32-bit hash, -but preserve memory context. -This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update(). -To free memory context, use XXH32_digest(), or free(). -*/ - - - -//**************************** -// Deprecated function names -//**************************** -// The following translations are provided to ease code transition -// You are encouraged to no longer this function names -#define XXH32_feed XXH32_update -#define XXH32_result XXH32_digest -#define XXH32_getIntermediateResult XXH32_intermediateDigest - - - -#if defined (__cplusplus) -} -#endif +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : http://code.google.com/p/xxhash/ +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +/***************************** + Type +*****************************/ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + + +/***************************** + Simple Hash Functions +*****************************/ + +unsigned int XXH32 (const void* input, unsigned int len, unsigned int seed); +unsigned long long XXH64 (const void* input, unsigned int len, unsigned long long seed); + +/* +XXH32() : + Calculate the 32-bits hash of sequence of length "len" stored at memory address "input". + The memory between input & input+len must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + This function successfully passes all SMHasher tests. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s + Note that "len" is type "int", which means it is limited to 2^31-1. + If your data is larger, use the advanced functions below. +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". +*/ + + + +/***************************** + Advanced Hash Functions +*****************************/ + +void* XXH32_init (unsigned int seed); +XXH_errorcode XXH32_update (void* state, const void* input, unsigned int len); +unsigned int XXH32_digest (void* state); + +void* XXH64_init (unsigned long long seed); +XXH_errorcode XXH64_update (void* state, const void* input, unsigned int len); +unsigned long long XXH64_digest (void* state); + +/* +These functions calculate the xxhash of an input provided in several small packets, +as opposed to an input provided as a single block. + +It must be started with : +void* XXHnn_init() +The function returns a pointer which holds the state of calculation. + +This pointer must be provided as "void* state" parameter for XXHnn_update(). +XXHnn_update() can be called as many times as necessary. +The user must provide a valid (allocated) input. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. +Note that "len" is type "int", which means it is limited to 2^31-1. +If your data is larger, it is recommended to chunk your data into blocks +of size for example 2^30 (1GB) to avoid any "int" overflow issue. + +Finally, you can end the calculation anytime, by using XXHnn_digest(). +This function returns the final nn-bits hash. +You must provide the same "void* state" parameter created by XXHnn_init(). +Memory will be freed by XXHnn_digest(). +*/ + + +int XXH32_sizeofState(void); +XXH_errorcode XXH32_resetState(void* state, unsigned int seed); + +#define XXH32_SIZEOFSTATE 48 +typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t; + +int XXH64_sizeofState(void); +XXH_errorcode XXH64_resetState(void* state, unsigned long long seed); + +#define XXH64_SIZEOFSTATE 88 +typedef struct { long long ll[(XXH64_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH64_stateSpace_t; + +/* +These functions allow user application to make its own allocation for state. + +XXHnn_sizeofState() is used to know how much space must be allocated for the xxHash nn-bits state. +Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer. +This pointer must then be provided as 'state' into XXHnn_resetState(), which initializes the state. + +For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()), +use the structure XXHnn_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields. +*/ + + +unsigned int XXH32_intermediateDigest (void* state); +unsigned long long XXH64_intermediateDigest (void* state); +/* +This function does the same as XXHnn_digest(), generating a nn-bit hash, +but preserve memory context. +This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXHnn_update(). +To free memory context, use XXHnn_digest(), or free(). +*/ + + +#if defined (__cplusplus) +} +#endif