Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/rapidfuzz/process_cpp_impl.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -1671,8 +1671,8 @@ cdef inline vector[PyObjectWrapper] preprocess_py(queries, processor) except *:
cdef int64_t queries_len = <int64_t>len(queries)
proc_queries.reserve(queries_len)

# processor None/False
if not processor:
# Preserve the legacy False sentinel without truth-testing callables.
if processor is None or processor is False:
for query in queries:
proc_queries.emplace_back(<PyObject*>query)
# processor has to be called through python
Expand All @@ -1691,8 +1691,8 @@ cdef inline vector[RF_StringWrapper] preprocess(const RF_ScorerFlags* scorer_fla
flags = scorer_flags.flags
proc_queries.reserve(queries_len)

# No processor
if not processor:
# Preserve the legacy False sentinel without truth-testing callables.
if processor is None or processor is False:
for query in queries:
conv_sequence_with_none(query, &proc_str)
if proc_str.data == NULL:
Expand Down
10 changes: 7 additions & 3 deletions src/rapidfuzz/process_py.py
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,8 @@ def cdist(

_ = workers, score_hint
scorer_kwargs = scorer_kwargs or {}
if processor is False:
processor = None
dtype = _dtype_to_type_num(dtype, scorer, scorer_kwargs)
results = np.zeros((len(queries), len(choices)), dtype=dtype)

Expand Down Expand Up @@ -551,7 +553,7 @@ def cdist(
results[i, j] = results[j, i] = score
else:
for i, query in enumerate(queries):
proc_query = processor(query) if (processor and not is_none(query)) else query
proc_query = processor(query) if (processor is not None and not is_none(query)) else query
for j, choice in enumerate(proc_choices):
score = (
scorer(
Expand Down Expand Up @@ -651,14 +653,16 @@ def cpdist(

_ = workers, score_hint
scorer_kwargs = scorer_kwargs or {}
if processor is False:
processor = None
dtype = _dtype_to_type_num(dtype, scorer, scorer_kwargs)
results = np.zeros((len_queries,), dtype=dtype)

setupPandas()

for i, (query, choice) in enumerate(zip(queries, choices)):
proc_query = processor(query) if (processor and not is_none(query)) else query
proc_choice = processor(choice) if (processor and not is_none(choice)) else choice
proc_query = processor(query) if (processor is not None and not is_none(query)) else query
proc_choice = processor(choice) if (processor is not None and not is_none(choice)) else choice
score = scorer(
proc_query,
proc_choice,
Expand Down
65 changes: 65 additions & 0 deletions tests/test_process.py
Original file line number Diff line number Diff line change
Expand Up @@ -772,3 +772,68 @@ def test_cpdist_integral_dtype_rounding():
assert float_result % 1 >= 0.5
# Check if the rounded float result is equal to the integer result
assert round(float_result) == int_result


class _FalseBoolProcessor:
def __call__(self, value):
return value.lower()

def __bool__(self):
return False


class _EmptyProcessor:
def __call__(self, value):
return value.lower()

def __len__(self):
return 0


class _NoTruthTestProcessor:
def __call__(self, value):
return value.lower()

def __bool__(self):
message = "processor truth value must not be tested"
raise AssertionError(message)


@pytest.mark.parametrize("module", [process_py, process_cpp])
@pytest.mark.parametrize("method", ["cdist", "cpdist"])
@pytest.mark.parametrize("processor_type", [_FalseBoolProcessor, _EmptyProcessor, _NoTruthTestProcessor])
@pytest.mark.parametrize("custom_scorer", [False, True])
def test_pairwise_processor_truth_value(module, method, processor_type, custom_scorer):
pytest.importorskip("numpy")

def scorer(first, second, **kwargs):
return fuzz.ratio(first, second, **kwargs)

result = getattr(module, method)(
["A"], ["a"], processor=processor_type(), scorer=scorer if custom_scorer else fuzz.ratio
)
assert result.item() == 100


@pytest.mark.parametrize("module", [process_py, process_cpp])
@pytest.mark.parametrize("processor_type", [_FalseBoolProcessor, _EmptyProcessor, _NoTruthTestProcessor])
def test_cdist_symmetric_processor_truth_value(module, processor_type):
np = pytest.importorskip("numpy")

values = ["A", "a", None]
result = module.cdist(values, values, processor=processor_type())
np.testing.assert_array_equal(result, [[100, 100, 0], [100, 100, 0], [0, 0, 0]])


@pytest.mark.parametrize("module", [process_py, process_cpp])
@pytest.mark.parametrize("method", ["cdist", "cpdist"])
@pytest.mark.parametrize("processor", [None, False, str.lower])
@pytest.mark.parametrize("custom_scorer", [False, True])
def test_pairwise_processor_compatibility(module, method, processor, custom_scorer):
pytest.importorskip("numpy")

def scorer(first, second, **kwargs):
return fuzz.ratio(first, second, **kwargs)

result = getattr(module, method)(["A"], ["a"], processor=processor, scorer=scorer if custom_scorer else fuzz.ratio)
assert result.item() == (100 if processor is str.lower else 0)