From dcc0fa791500db5e67cd5ea979c78e04370b2851 Mon Sep 17 00:00:00 2001 From: Anush Date: Sun, 7 Jul 2024 00:45:38 +0530 Subject: [PATCH 1/2] chore: mmh3.hash(token, signed=False) instead of abs() --- fastembed/sparse/bm42.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastembed/sparse/bm42.py b/fastembed/sparse/bm42.py index f9b7d6c8f..7d8a0c228 100644 --- a/fastembed/sparse/bm42.py +++ b/fastembed/sparse/bm42.py @@ -169,7 +169,7 @@ def _rescore_vector(self, vector: Dict[str, float]) -> Dict[int, float]: new_vector = {} for token, value in vector.items(): - token_id = abs(mmh3.hash(token)) + token_id = mmh3.hash(token, signed=False) # Examples: # Num 0: Log(1/1 + 1) = 0.6931471805599453 # Num 1: Log(1/2 + 1) = 0.4054651081081644 @@ -260,7 +260,7 @@ def embed( def _query_rehash(cls, tokens: Iterable[str]) -> Dict[int, float]: result = {} for token in tokens: - token_id = abs(mmh3.hash(token)) + token_id = mmh3.hash(token, signed=False) result[token_id] = 1.0 return result From 062fb2c39590c5aafcfc57a67d87f05f0caad33e Mon Sep 17 00:00:00 2001 From: Anush Date: Sun, 7 Jul 2024 01:00:08 +0530 Subject: [PATCH 2/2] chore: Updated bm25.py --- fastembed/sparse/bm25.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastembed/sparse/bm25.py b/fastembed/sparse/bm25.py index fd248b708..2a7f222f2 100644 --- a/fastembed/sparse/bm25.py +++ b/fastembed/sparse/bm25.py @@ -237,7 +237,7 @@ def _term_frequency(self, tokens: List[str]) -> Dict[int, float]: @classmethod def compute_token_id(cls, token: str) -> int: - return abs(mmh3.hash(token)) + return mmh3.hash(token, signed=False) def query_embed(self, query: Union[str, Iterable[str]], **kwargs) -> Iterable[SparseEmbedding]: """To emulate BM25 behaviour, we don't need to use weights in the query, and