Loading include/common/rpc/distributor.hpp +38 −0 Original line number Diff line number Diff line Loading @@ -36,6 +36,7 @@ #include <unordered_map> #include <fstream> #include <map> #include "rapidhash.h" namespace gkfs::rpc { Loading Loading @@ -228,6 +229,43 @@ public: locate_directory_metadata() const override; }; class RapidHashDistributor : public Distributor { private: host_t localhost_; unsigned int hosts_size_{0}; std::vector<host_t> all_hosts_; std::hash<std::string> str_hash; public: RapidHashDistributor(); RapidHashDistributor(host_t localhost, unsigned int hosts_size); unsigned int hosts_size() const override; void hosts_size(unsigned int size) override; host_t localhost() const override; host_t locate_data(const std::string& path, const chunkid_t& chnk_id, const int num_copy) const override; host_t locate_data(const std::string& path, const chunkid_t& chnk_id, unsigned int host_size, const int num_copy); host_t locate_file_metadata(const std::string& path, const int num_copy) const override; std::vector<host_t> locate_directory_metadata() const override; }; } // namespace gkfs::rpc #endif // GEKKOFS_RPC_LOCATOR_HPP include/common/rpc/rapidhash.h 0 → 100644 +323 −0 Original line number Diff line number Diff line /* * rapidhash - Very fast, high quality, platform-independent hashing algorithm. * Copyright (C) 2024 Nicolas De Carli * * Based on 'wyhash', by Wang Yi <godspeed_china@yeah.net> * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * You can contact the author at: * - rapidhash source repository: https://github.com/Nicoshev/rapidhash */ /* * Includes. */ #include <stdint.h> #include <string.h> #if defined(_MSC_VER) #include <intrin.h> #if defined(_M_X64) && !defined(_M_ARM64EC) #pragma intrinsic(_umul128) #endif #endif /* * C++ macros. * * RAPIDHASH_INLINE can be overridden to be stronger than a hint, i.e. by adding __attribute__((always_inline)). */ #ifdef __cplusplus #define RAPIDHASH_NOEXCEPT noexcept #define RAPIDHASH_CONSTEXPR constexpr #ifndef RAPIDHASH_INLINE #define RAPIDHASH_INLINE inline #endif #else #define RAPIDHASH_NOEXCEPT #define RAPIDHASH_CONSTEXPR static const #ifndef RAPIDHASH_INLINE #define RAPIDHASH_INLINE static inline #endif #endif /* * Protection macro, alters behaviour of rapid_mum multiplication function. * * RAPIDHASH_FAST: Normal behavior, max speed. * RAPIDHASH_PROTECTED: Extra protection against entropy loss. */ #ifndef RAPIDHASH_PROTECTED #define RAPIDHASH_FAST #elif defined(RAPIDHASH_FAST) #error "cannot define RAPIDHASH_PROTECTED and RAPIDHASH_FAST simultaneously." #endif /* * Unrolling macros, changes code definition for main hash function. * * RAPIDHASH_COMPACT: Legacy variant, each loop process 48 bytes. * RAPIDHASH_UNROLLED: Unrolled variant, each loop process 96 bytes. * * Most modern CPUs should benefit from having RAPIDHASH_UNROLLED. * * These macros do not alter the output hash. */ #ifndef RAPIDHASH_COMPACT #define RAPIDHASH_UNROLLED #elif defined(RAPIDHASH_UNROLLED) #error "cannot define RAPIDHASH_COMPACT and RAPIDHASH_UNROLLED simultaneously." #endif /* * Likely and unlikely macros. */ #if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) #define _likely_(x) __builtin_expect(x,1) #define _unlikely_(x) __builtin_expect(x,0) #else #define _likely_(x) (x) #define _unlikely_(x) (x) #endif /* * Endianness macros. */ #ifndef RAPIDHASH_LITTLE_ENDIAN #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) #define RAPIDHASH_LITTLE_ENDIAN #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #define RAPIDHASH_BIG_ENDIAN #else #warning "could not determine endianness! Falling back to little endian." #define RAPIDHASH_LITTLE_ENDIAN #endif #endif /* * Default seed. */ #define RAPID_SEED (0xbdd89aa982704029ull) /* * Default secret parameters. */ RAPIDHASH_CONSTEXPR uint64_t rapid_secret[3] = {0x2d358dccaa6c78a5ull, 0x8bb84b93962eacc9ull, 0x4b33a62ed433d4a3ull}; /* * 64*64 -> 128bit multiply function. * * @param A Address of 64-bit number. * @param B Address of 64-bit number. * * Calculates 128-bit C = *A * *B. * * When RAPIDHASH_FAST is defined: * Overwrites A contents with C's low 64 bits. * Overwrites B contents with C's high 64 bits. * * When RAPIDHASH_PROTECTED is defined: * Xors and overwrites A contents with C's low 64 bits. * Xors and overwrites B contents with C's high 64 bits. */ RAPIDHASH_INLINE void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { #if defined(__SIZEOF_INT128__) __uint128_t r=*A; r*=*B; #ifdef RAPIDHASH_PROTECTED *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); #else *A=(uint64_t)r; *B=(uint64_t)(r>>64); #endif #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) #if defined(_M_X64) #ifdef RAPIDHASH_PROTECTED uint64_t a, b; a=_umul128(*A,*B,&b); *A^=a; *B^=b; #else *A=_umul128(*A,*B,B); #endif #else #ifdef RAPIDHASH_PROTECTED uint64_t a, b; b = __umulh(*A, *B); a = *A * *B; *A^=a; *B^=b; #else uint64_t c = __umulh(*A, *B); *A = *A * *B; *B = c; #endif #endif #else uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B, hi, lo; uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t<rl; lo=t+(rm1<<32); c+=lo<t; hi=rh+(rm0>>32)+(rm1>>32)+c; #ifdef RAPIDHASH_PROTECTED *A^=lo; *B^=hi; #else *A=lo; *B=hi; #endif #endif } /* * Multiply and xor mix function. * * @param A 64-bit number. * @param B 64-bit number. * * Calculates 128-bit C = A * B. * Returns 64-bit xor between high and low 64 bits of C. */ RAPIDHASH_INLINE uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } /* * Read functions. */ #ifdef RAPIDHASH_LITTLE_ENDIAN RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} #elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return __builtin_bswap64(v);} RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return __builtin_bswap32(v);} #elif defined(_MSC_VER) RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return _byteswap_uint64(v);} RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return _byteswap_ulong(v);} #else RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, 8); return (((v >> 56) & 0xff)| ((v >> 40) & 0xff00)| ((v >> 24) & 0xff0000)| ((v >> 8) & 0xff000000)| ((v << 8) & 0xff00000000)| ((v << 24) & 0xff0000000000)| ((v << 40) & 0xff000000000000)| ((v << 56) & 0xff00000000000000)); } RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, 4); return (((v >> 24) & 0xff)| ((v >> 8) & 0xff00)| ((v << 8) & 0xff0000)| ((v << 24) & 0xff000000)); } #endif /* * Reads and combines 3 bytes of input. * * @param p Buffer to read from. * @param k Length of @p, in bytes. * * Always reads and combines 3 bytes from memory. * Guarantees to read each buffer position at least once. * * Returns a 64-bit value containing all three bytes read. */ RAPIDHASH_INLINE uint64_t rapid_readSmall(const uint8_t *p, size_t k) RAPIDHASH_NOEXCEPT { return (((uint64_t)p[0])<<56)|(((uint64_t)p[k>>1])<<32)|p[k-1];} /* * rapidhash main function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * @param secret Triplet of 64-bit secrets used to alter hash result predictably. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { const uint8_t *p=(const uint8_t *)key; seed^=rapid_mix(seed^secret[0],secret[1])^len; uint64_t a, b; if(_likely_(len<=16)){ if(_likely_(len>=4)){ const uint8_t * plast = p + len - 4; a = (rapid_read32(p) << 32) | rapid_read32(plast); const uint64_t delta = ((len&24)>>(len>>3)); b = ((rapid_read32(p + delta) << 32) | rapid_read32(plast - delta)); } else if(_likely_(len>0)){ a=rapid_readSmall(p,len); b=0;} else a=b=0; } else{ size_t i=len; if(_unlikely_(i>48)){ uint64_t see1=seed, see2=seed; #ifdef RAPIDHASH_UNROLLED while(_likely_(i>=96)){ seed=rapid_mix(rapid_read64(p)^secret[0],rapid_read64(p+8)^seed); see1=rapid_mix(rapid_read64(p+16)^secret[1],rapid_read64(p+24)^see1); see2=rapid_mix(rapid_read64(p+32)^secret[2],rapid_read64(p+40)^see2); seed=rapid_mix(rapid_read64(p+48)^secret[0],rapid_read64(p+56)^seed); see1=rapid_mix(rapid_read64(p+64)^secret[1],rapid_read64(p+72)^see1); see2=rapid_mix(rapid_read64(p+80)^secret[2],rapid_read64(p+88)^see2); p+=96; i-=96; } if(_unlikely_(i>=48)){ seed=rapid_mix(rapid_read64(p)^secret[0],rapid_read64(p+8)^seed); see1=rapid_mix(rapid_read64(p+16)^secret[1],rapid_read64(p+24)^see1); see2=rapid_mix(rapid_read64(p+32)^secret[2],rapid_read64(p+40)^see2); p+=48; i-=48; } #else do { seed=rapid_mix(rapid_read64(p)^secret[0],rapid_read64(p+8)^seed); see1=rapid_mix(rapid_read64(p+16)^secret[1],rapid_read64(p+24)^see1); see2=rapid_mix(rapid_read64(p+32)^secret[2],rapid_read64(p+40)^see2); p+=48; i-=48; } while (_likely_(i>=48)); #endif seed^=see1^see2; } if(i>16){ seed=rapid_mix(rapid_read64(p)^secret[2],rapid_read64(p+8)^seed^secret[1]); if(i>32) seed=rapid_mix(rapid_read64(p+16)^secret[2],rapid_read64(p+24)^seed); } a=rapid_read64(p+i-16); b=rapid_read64(p+i-8); } a^=secret[1]; b^=seed; rapid_mum(&a,&b); return rapid_mix(a^secret[0]^len,b^secret[1]); } /* * rapidhash default seeded hash function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * * Calls rapidhash_internal using provided parameters and default secrets. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { return rapidhash_internal(key, len, seed, rapid_secret); } /* * rapidhash default hash function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * * Calls rapidhash_withSeed using provided parameters and the default seed. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE uint64_t rapidhash(const void *key, size_t len) RAPIDHASH_NOEXCEPT { return rapidhash_withSeed(key, len, RAPID_SEED); } src/common/rpc/distributor.cpp +55 −0 Original line number Diff line number Diff line Loading @@ -306,5 +306,60 @@ GuidedDistributor::locate_directory_metadata() const { return all_hosts_; } RapidHashDistributor::RapidHashDistributor(host_t localhost, unsigned int hosts_size) : localhost_(localhost), hosts_size_(hosts_size), all_hosts_(hosts_size) { ::iota(all_hosts_.begin(), all_hosts_.end(), 0); } RapidHashDistributor::RapidHashDistributor() {} host_t RapidHashDistributor::localhost() const { return localhost_; } unsigned int RapidHashDistributor::hosts_size() const { return hosts_size_; } void RapidHashDistributor::hosts_size(unsigned int size) { hosts_size_ = size; } host_t RapidHashDistributor::locate_data(const string& path, const chunkid_t& chnk_id, const int num_copy) const { std::string hash_buf = path + ::to_string(chnk_id); return (rapidhash(hash_buf.data(), hash_buf.length()) + num_copy) % hosts_size_; } host_t RapidHashDistributor::locate_data(const string& path, const chunkid_t& chnk_id, unsigned int hosts_size, const int num_copy) { if(hosts_size_ != hosts_size) { hosts_size_ = hosts_size; all_hosts_ = std::vector<unsigned int>(hosts_size); ::iota(all_hosts_.begin(), all_hosts_.end(), 0); } std::string hash_buf = path + ::to_string(chnk_id); return (rapidhash(hash_buf.data(), hash_buf.length()) + num_copy) % hosts_size_; } host_t RapidHashDistributor::locate_file_metadata(const string& path, const int num_copy) const { return (rapidhash(path.data(), path.length()) + num_copy) % hosts_size_; } ::vector<host_t> RapidHashDistributor::locate_directory_metadata() const { return all_hosts_; } } // namespace rpc } // namespace gkfs Loading
include/common/rpc/distributor.hpp +38 −0 Original line number Diff line number Diff line Loading @@ -36,6 +36,7 @@ #include <unordered_map> #include <fstream> #include <map> #include "rapidhash.h" namespace gkfs::rpc { Loading Loading @@ -228,6 +229,43 @@ public: locate_directory_metadata() const override; }; class RapidHashDistributor : public Distributor { private: host_t localhost_; unsigned int hosts_size_{0}; std::vector<host_t> all_hosts_; std::hash<std::string> str_hash; public: RapidHashDistributor(); RapidHashDistributor(host_t localhost, unsigned int hosts_size); unsigned int hosts_size() const override; void hosts_size(unsigned int size) override; host_t localhost() const override; host_t locate_data(const std::string& path, const chunkid_t& chnk_id, const int num_copy) const override; host_t locate_data(const std::string& path, const chunkid_t& chnk_id, unsigned int host_size, const int num_copy); host_t locate_file_metadata(const std::string& path, const int num_copy) const override; std::vector<host_t> locate_directory_metadata() const override; }; } // namespace gkfs::rpc #endif // GEKKOFS_RPC_LOCATOR_HPP
include/common/rpc/rapidhash.h 0 → 100644 +323 −0 Original line number Diff line number Diff line /* * rapidhash - Very fast, high quality, platform-independent hashing algorithm. * Copyright (C) 2024 Nicolas De Carli * * Based on 'wyhash', by Wang Yi <godspeed_china@yeah.net> * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * You can contact the author at: * - rapidhash source repository: https://github.com/Nicoshev/rapidhash */ /* * Includes. */ #include <stdint.h> #include <string.h> #if defined(_MSC_VER) #include <intrin.h> #if defined(_M_X64) && !defined(_M_ARM64EC) #pragma intrinsic(_umul128) #endif #endif /* * C++ macros. * * RAPIDHASH_INLINE can be overridden to be stronger than a hint, i.e. by adding __attribute__((always_inline)). */ #ifdef __cplusplus #define RAPIDHASH_NOEXCEPT noexcept #define RAPIDHASH_CONSTEXPR constexpr #ifndef RAPIDHASH_INLINE #define RAPIDHASH_INLINE inline #endif #else #define RAPIDHASH_NOEXCEPT #define RAPIDHASH_CONSTEXPR static const #ifndef RAPIDHASH_INLINE #define RAPIDHASH_INLINE static inline #endif #endif /* * Protection macro, alters behaviour of rapid_mum multiplication function. * * RAPIDHASH_FAST: Normal behavior, max speed. * RAPIDHASH_PROTECTED: Extra protection against entropy loss. */ #ifndef RAPIDHASH_PROTECTED #define RAPIDHASH_FAST #elif defined(RAPIDHASH_FAST) #error "cannot define RAPIDHASH_PROTECTED and RAPIDHASH_FAST simultaneously." #endif /* * Unrolling macros, changes code definition for main hash function. * * RAPIDHASH_COMPACT: Legacy variant, each loop process 48 bytes. * RAPIDHASH_UNROLLED: Unrolled variant, each loop process 96 bytes. * * Most modern CPUs should benefit from having RAPIDHASH_UNROLLED. * * These macros do not alter the output hash. */ #ifndef RAPIDHASH_COMPACT #define RAPIDHASH_UNROLLED #elif defined(RAPIDHASH_UNROLLED) #error "cannot define RAPIDHASH_COMPACT and RAPIDHASH_UNROLLED simultaneously." #endif /* * Likely and unlikely macros. */ #if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) #define _likely_(x) __builtin_expect(x,1) #define _unlikely_(x) __builtin_expect(x,0) #else #define _likely_(x) (x) #define _unlikely_(x) (x) #endif /* * Endianness macros. */ #ifndef RAPIDHASH_LITTLE_ENDIAN #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) #define RAPIDHASH_LITTLE_ENDIAN #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #define RAPIDHASH_BIG_ENDIAN #else #warning "could not determine endianness! Falling back to little endian." #define RAPIDHASH_LITTLE_ENDIAN #endif #endif /* * Default seed. */ #define RAPID_SEED (0xbdd89aa982704029ull) /* * Default secret parameters. */ RAPIDHASH_CONSTEXPR uint64_t rapid_secret[3] = {0x2d358dccaa6c78a5ull, 0x8bb84b93962eacc9ull, 0x4b33a62ed433d4a3ull}; /* * 64*64 -> 128bit multiply function. * * @param A Address of 64-bit number. * @param B Address of 64-bit number. * * Calculates 128-bit C = *A * *B. * * When RAPIDHASH_FAST is defined: * Overwrites A contents with C's low 64 bits. * Overwrites B contents with C's high 64 bits. * * When RAPIDHASH_PROTECTED is defined: * Xors and overwrites A contents with C's low 64 bits. * Xors and overwrites B contents with C's high 64 bits. */ RAPIDHASH_INLINE void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { #if defined(__SIZEOF_INT128__) __uint128_t r=*A; r*=*B; #ifdef RAPIDHASH_PROTECTED *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); #else *A=(uint64_t)r; *B=(uint64_t)(r>>64); #endif #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) #if defined(_M_X64) #ifdef RAPIDHASH_PROTECTED uint64_t a, b; a=_umul128(*A,*B,&b); *A^=a; *B^=b; #else *A=_umul128(*A,*B,B); #endif #else #ifdef RAPIDHASH_PROTECTED uint64_t a, b; b = __umulh(*A, *B); a = *A * *B; *A^=a; *B^=b; #else uint64_t c = __umulh(*A, *B); *A = *A * *B; *B = c; #endif #endif #else uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B, hi, lo; uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t<rl; lo=t+(rm1<<32); c+=lo<t; hi=rh+(rm0>>32)+(rm1>>32)+c; #ifdef RAPIDHASH_PROTECTED *A^=lo; *B^=hi; #else *A=lo; *B=hi; #endif #endif } /* * Multiply and xor mix function. * * @param A 64-bit number. * @param B 64-bit number. * * Calculates 128-bit C = A * B. * Returns 64-bit xor between high and low 64 bits of C. */ RAPIDHASH_INLINE uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } /* * Read functions. */ #ifdef RAPIDHASH_LITTLE_ENDIAN RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} #elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return __builtin_bswap64(v);} RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return __builtin_bswap32(v);} #elif defined(_MSC_VER) RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return _byteswap_uint64(v);} RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return _byteswap_ulong(v);} #else RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, 8); return (((v >> 56) & 0xff)| ((v >> 40) & 0xff00)| ((v >> 24) & 0xff0000)| ((v >> 8) & 0xff000000)| ((v << 8) & 0xff00000000)| ((v << 24) & 0xff0000000000)| ((v << 40) & 0xff000000000000)| ((v << 56) & 0xff00000000000000)); } RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, 4); return (((v >> 24) & 0xff)| ((v >> 8) & 0xff00)| ((v << 8) & 0xff0000)| ((v << 24) & 0xff000000)); } #endif /* * Reads and combines 3 bytes of input. * * @param p Buffer to read from. * @param k Length of @p, in bytes. * * Always reads and combines 3 bytes from memory. * Guarantees to read each buffer position at least once. * * Returns a 64-bit value containing all three bytes read. */ RAPIDHASH_INLINE uint64_t rapid_readSmall(const uint8_t *p, size_t k) RAPIDHASH_NOEXCEPT { return (((uint64_t)p[0])<<56)|(((uint64_t)p[k>>1])<<32)|p[k-1];} /* * rapidhash main function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * @param secret Triplet of 64-bit secrets used to alter hash result predictably. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { const uint8_t *p=(const uint8_t *)key; seed^=rapid_mix(seed^secret[0],secret[1])^len; uint64_t a, b; if(_likely_(len<=16)){ if(_likely_(len>=4)){ const uint8_t * plast = p + len - 4; a = (rapid_read32(p) << 32) | rapid_read32(plast); const uint64_t delta = ((len&24)>>(len>>3)); b = ((rapid_read32(p + delta) << 32) | rapid_read32(plast - delta)); } else if(_likely_(len>0)){ a=rapid_readSmall(p,len); b=0;} else a=b=0; } else{ size_t i=len; if(_unlikely_(i>48)){ uint64_t see1=seed, see2=seed; #ifdef RAPIDHASH_UNROLLED while(_likely_(i>=96)){ seed=rapid_mix(rapid_read64(p)^secret[0],rapid_read64(p+8)^seed); see1=rapid_mix(rapid_read64(p+16)^secret[1],rapid_read64(p+24)^see1); see2=rapid_mix(rapid_read64(p+32)^secret[2],rapid_read64(p+40)^see2); seed=rapid_mix(rapid_read64(p+48)^secret[0],rapid_read64(p+56)^seed); see1=rapid_mix(rapid_read64(p+64)^secret[1],rapid_read64(p+72)^see1); see2=rapid_mix(rapid_read64(p+80)^secret[2],rapid_read64(p+88)^see2); p+=96; i-=96; } if(_unlikely_(i>=48)){ seed=rapid_mix(rapid_read64(p)^secret[0],rapid_read64(p+8)^seed); see1=rapid_mix(rapid_read64(p+16)^secret[1],rapid_read64(p+24)^see1); see2=rapid_mix(rapid_read64(p+32)^secret[2],rapid_read64(p+40)^see2); p+=48; i-=48; } #else do { seed=rapid_mix(rapid_read64(p)^secret[0],rapid_read64(p+8)^seed); see1=rapid_mix(rapid_read64(p+16)^secret[1],rapid_read64(p+24)^see1); see2=rapid_mix(rapid_read64(p+32)^secret[2],rapid_read64(p+40)^see2); p+=48; i-=48; } while (_likely_(i>=48)); #endif seed^=see1^see2; } if(i>16){ seed=rapid_mix(rapid_read64(p)^secret[2],rapid_read64(p+8)^seed^secret[1]); if(i>32) seed=rapid_mix(rapid_read64(p+16)^secret[2],rapid_read64(p+24)^seed); } a=rapid_read64(p+i-16); b=rapid_read64(p+i-8); } a^=secret[1]; b^=seed; rapid_mum(&a,&b); return rapid_mix(a^secret[0]^len,b^secret[1]); } /* * rapidhash default seeded hash function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * @param seed 64-bit seed used to alter the hash result predictably. * * Calls rapidhash_internal using provided parameters and default secrets. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { return rapidhash_internal(key, len, seed, rapid_secret); } /* * rapidhash default hash function. * * @param key Buffer to be hashed. * @param len @key length, in bytes. * * Calls rapidhash_withSeed using provided parameters and the default seed. * * Returns a 64-bit hash. */ RAPIDHASH_INLINE uint64_t rapidhash(const void *key, size_t len) RAPIDHASH_NOEXCEPT { return rapidhash_withSeed(key, len, RAPID_SEED); }
src/common/rpc/distributor.cpp +55 −0 Original line number Diff line number Diff line Loading @@ -306,5 +306,60 @@ GuidedDistributor::locate_directory_metadata() const { return all_hosts_; } RapidHashDistributor::RapidHashDistributor(host_t localhost, unsigned int hosts_size) : localhost_(localhost), hosts_size_(hosts_size), all_hosts_(hosts_size) { ::iota(all_hosts_.begin(), all_hosts_.end(), 0); } RapidHashDistributor::RapidHashDistributor() {} host_t RapidHashDistributor::localhost() const { return localhost_; } unsigned int RapidHashDistributor::hosts_size() const { return hosts_size_; } void RapidHashDistributor::hosts_size(unsigned int size) { hosts_size_ = size; } host_t RapidHashDistributor::locate_data(const string& path, const chunkid_t& chnk_id, const int num_copy) const { std::string hash_buf = path + ::to_string(chnk_id); return (rapidhash(hash_buf.data(), hash_buf.length()) + num_copy) % hosts_size_; } host_t RapidHashDistributor::locate_data(const string& path, const chunkid_t& chnk_id, unsigned int hosts_size, const int num_copy) { if(hosts_size_ != hosts_size) { hosts_size_ = hosts_size; all_hosts_ = std::vector<unsigned int>(hosts_size); ::iota(all_hosts_.begin(), all_hosts_.end(), 0); } std::string hash_buf = path + ::to_string(chnk_id); return (rapidhash(hash_buf.data(), hash_buf.length()) + num_copy) % hosts_size_; } host_t RapidHashDistributor::locate_file_metadata(const string& path, const int num_copy) const { return (rapidhash(path.data(), path.length()) + num_copy) % hosts_size_; } ::vector<host_t> RapidHashDistributor::locate_directory_metadata() const { return all_hosts_; } } // namespace rpc } // namespace gkfs
