Update Duplicate Detection

centrallix-lib/Makefile.in

@@ -6,7 +6,7 @@
 ## This library is made available under the GNU Lesser General Public
 ## License, version 2.1.
 ##
-## Copyright (C) 1998-2001 LightSys Technology Services, Inc.
+## Copyright (C) 1998-2026 LightSys Technology Services, Inc.
 ##
 
 @SET_MAKE@
@@ -63,10 +63,10 @@ CFLAGS=@CFLAGS@ @DEFS@ -Iinclude -DCXLIB_INTERNAL -DNM_USE_SYSMALLOC -Wall -Wfor
 MTCFLAGS=@CFLAGS@ @DEFS@ -Iinclude -DCXLIB_INTERNAL -DNM_USE_SYSMALLOC -Wall -Wformat-nonliteral $(PROFILE) $(COVERAGE) -g -O0
 TCFLAGS=$(patsubst -DNDEBUG,,$(CFLAGS)) -DCX_TESTING
 
-XSTATICFILES=mtask.o mtlexer.o memstr.o xarray.o xhash.o xstring.o mtsession.o newmalloc.o xhashqueue.o bdqs_transport.o xhandle.o xringqueue.o cxsec.o smmalloc.o qprintf.o strtcpy.o util.o timer.o warn.o range.o
+XSTATICFILES=mtask.o mtlexer.o memstr.o xarray.o xhash.o xstring.o mtsession.o newmalloc.o xhashqueue.o bdqs_transport.o xhandle.o xringqueue.o cxsec.o smmalloc.o clusters.o qprintf.o strtcpy.o util.o timer.o warn.o range.o
 STATICFILES=$(patsubst %,src/%,$(XSTATICFILES))
 
-XDYNAMICFILES=mtask.lo mtlexer.lo memstr.lo xarray.lo xhash.lo xstring.lo mtsession.lo newmalloc.lo xhashqueue.lo bdqs_transport.lo xhandle.lo xringqueue.lo cxsec.lo smmalloc.lo qprintf.lo strtcpy.lo util.lo timer.lo warn.lo range.lo
+XDYNAMICFILES=mtask.lo mtlexer.lo memstr.lo xarray.lo xhash.lo xstring.lo mtsession.lo newmalloc.lo xhashqueue.lo bdqs_transport.lo xhandle.lo xringqueue.lo cxsec.lo smmalloc.lo clusters.lo qprintf.lo strtcpy.lo util.lo timer.lo warn.lo range.lo
 DYNAMICFILES=$(patsubst %,src/%,$(XDYNAMICFILES))
 
 INCLUDEFILES:=$(wildcard include/*.h)
@@ -76,6 +76,19 @@ INCLUDEFILES:=$(patsubst include/cxlibconfig-all.h,,$(INCLUDEFILES))
 TESTFILES:=$(wildcard tests/test_$(TONLY)*.c)
 TESTPROGS:=$(patsubst %.c,%.bin,$(TESTFILES))
 
+## Valgrind setup for "make valtest".  Override VALGRINDFLAGS to change the
+## checks performed, and set VALGRINDLOG to a directory to get one valgrind
+## log per test instead of the errors being printed with the test results.
+VALGRIND?=valgrind
+VALGRINDFLAGS?=-q
+VALGRINDLOG?=
+
+## How many tests "make test" and "make valtest" run at once (1 for one at a
+## time).  Each test is a single-threaded process, so this scales with the core
+## count; the driver's lockup timeout is wall clock, so going well past the
+## core count risks reporting LOCKUP for a test that was only waiting for one.
+TESTJOBS?=$(shell nproc 2>/dev/null || echo 1)
+
 .SUFFIXES:	.c .o .lo
 
 .PHONY: all
@@ -157,29 +170,106 @@ install: all
 	@echo "***************************************************************"
 	@echo ""
 
-%.bin:	%.c tests/t_driver.c libCentrallix.a
-	$(CC) $(TCFLAGS) tests/t_driver.c $< libCentrallix.a -o $@ ${LIBS}
+%.bin:	%.c tests/t_driver.c tests/t_utils.c tests/t_utils.h libCentrallix.a
+	$(CC) $(TCFLAGS) tests/t_driver.c tests/t_utils.c $< libCentrallix.a -o $@ ${LIBS}
+
+## Runs the selected tests $(TESTJOBS) at a time.  Output is captured per test
+## rather than written straight to the terminal, then printed in listing order
+## as each test finishes, so parallel tests cannot interleave.
+##
+## TESTCMD runs once per test with the binary as "$$1" and its capture file as
+## "$$2".  It writes to "$$2.part" and renames that to "$$2" when done so that
+## the rename is what marks the test as finished, then it exits with the test's
+## status.  The driver exits 0 even when reporting FAIL or CRASH (see
+## tests/t_driver.c), so a nonzero status means the test did not complete
+## normally and must fail the target; only valgrind's --error-exitcode is
+## dropped.  A test whose output never appears is reported and also fails.
+##
+## On SIGINT or SIGTERM, the shell stops the tests, removes the captured
+## output without printing it, and then dies from the same signal.  xargs
+## runs in its own process group so that the whole group can be killed:
+## a background job ignores SIGINT, so Ctrl+C never reaches the tests.
+RUN_TESTS = \
+    RESULTS=$$(mktemp -d) || exit 1; \
+    XARGS=; \
+    stop() { \
+	trap - EXIT INT TERM; \
+	if [ -n "$$XARGS" ]; then \
+	    kill -s TERM -- -"$$XARGS" 2>/dev/null; \
+	    wait "$$XARGS" 2>/dev/null; \
+	fi; \
+	rm -rf "$$RESULTS"; \
+	kill -s "$$1" $$$$; \
+    }; \
+    trap 'rm -rf "$$RESULTS"' EXIT; \
+    trap 'stop INT' INT; \
+    trap 'stop TERM' TERM; \
+    N=0; \
+    for PROG in $$( ls $(TESTPROGS) ); do \
+	case "$$PROG" in tests/test_$$TONLY*) \
+	    N=$$((N+1)); \
+	    printf '%s\n%s/%04d.out\n' "$$PROG" "$$RESULTS" "$$N";; \
+	esac; \
+    done > "$$RESULTS/list"; \
+    if [ "$$N" -eq 0 ]; then exit 0; fi; \
+    setsid xargs -r -n 2 -P $(TESTJOBS) sh -c "$$TESTCMD" sh \
+	< "$$RESULTS/list" 2> "$$RESULTS/stderr" & \
+    XARGS=$$!; \
+    RC=0; \
+    I=0; \
+    while [ "$$I" -lt "$$N" ]; do \
+	I=$$((I+1)); \
+	OUT=$$(printf '%s/%04d.out' "$$RESULTS" "$$I"); \
+	while [ ! -e "$$OUT" ]; do \
+	    kill -0 "$$XARGS" 2>/dev/null || break; \
+	    sleep 0.05; \
+	done; \
+	if [ -e "$$OUT" ]; then \
+	    cat "$$OUT"; \
+	else \
+	    printf '%-62.62s  FAIL (produced no result)\n' \
+		"$$(sed -n "$$((2 * I - 1))p" "$$RESULTS/list")"; \
+	    RC=1; \
+	fi; \
+    done; \
+    wait "$$XARGS" || RC=$$?; \
+    cat "$$RESULTS/stderr" >&2; \
+    exit "$$RC"
 
 .PHONY: test
 test:	$(TESTPROGS)
 	@printf "%-62.62s  %s\n" "Test Name" "Stat Ops/sec"
 	@printf "%-62.62s  %s\n" "----------------------------------------------------------------" "--------------"
-	@for PROG in $$( ls $(TESTPROGS) ); do \
-	    if [ "$$TONLY" = "" -o "$${PROG#tests/test_$$TONLY}" != "$$PROG" ]; then \
-		$$PROG; \
-	    fi \
-	done 
+	@TESTCMD='"$$1" > "$$2.part" 2>&1; \
+		S=$$?; \
+		mv "$$2.part" "$$2"; \
+		exit $$S'; \
+	$(RUN_TESTS)
 
 .PHONY:	valtest
 valtest:	$(TESTPROGS)
 	@printf "Running Regression Tests Under Valgrind...\n"
 	@printf "%-62.62s  %s\n" "Test Name" "Stat Ops/sec"
 	@printf "%-62.62s  %s\n" "----------------------------------------------------------------" "--------------"
-	@for PROG in $$( ls $(TESTPROGS) ); do \
-	    if [ "$$TONLY" = "" -o "$${PROG#tests/test_$$TONLY}" != "$$PROG" ]; then \
-		valgrind -q $$PROG; \
-	    fi \
-	done 
+	@if [ "$(VALGRINDLOG)" != "" ]; then mkdir -p "$(VALGRINDLOG)"; fi
+	@if [ "$(VALGRINDLOG)" = "" ]; then \
+	    TESTCMD='$(VALGRIND) $(VALGRINDFLAGS) "$$1" > "$$2.part" 2>&1; \
+		S=$$?; \
+		mv "$$2.part" "$$2"; \
+		exit $$S'; \
+	else \
+	    TESTCMD='NAME=$${1##*/}; \
+		LOG="$(VALGRINDLOG)/$${NAME%.bin}.log"; \
+		$(VALGRIND) $(VALGRINDFLAGS) --error-exitcode=125 --log-file="$$LOG" "$$1" > "$$2.part" 2>&1; \
+		S=$$?; \
+		if [ "$$S" = "125" ]; then \
+		    printf "  valgrind: errors reported -- %s\n" "$$LOG" >> "$$2.part"; \
+		    S=0; \
+		fi; \
+		mv "$$2.part" "$$2"; \
+		exit $$S'; \
+	fi; \
+	$(RUN_TESTS)
 
 .PHONY: cov-clean
 cov-clean:

centrallix-lib/include/clusters.h

@@ -0,0 +1,166 @@
+#ifndef CLUSTERS_H
+#define	CLUSTERS_H
+
+/************************************************************************/
+/* Centrallix Application Server System					*/
+/* Centrallix Core							*/
+/* 									*/
+/* Copyright (C) 1998-2026 LightSys Technology Services, Inc.		*/
+/* 									*/
+/* This program is free software; you can redistribute it and/or modify	*/
+/* it under the terms of the GNU General Public License as published by	*/
+/* the Free Software Foundation; either version 2 of the License, or	*/
+/* (at your option) any later version.					*/
+/* 									*/
+/* This program is distributed in the hope that it will be useful,	*/
+/* but WITHOUT ANY WARRANTY; without even the implied warranty of	*/
+/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the	*/
+/* GNU General Public License for more details.				*/
+/* 									*/
+/* You should have received a copy of the GNU General Public License	*/
+/* along with this program; if not, write to the Free Software		*/
+/* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA		*/
+/* 02111-1307  USA							*/
+/* 									*/
+/* A copy of the GNU General Public License has been included in this	*/
+/* distribution in the file "COPYING".					*/
+/* 									*/
+/* Module:	clusters.c, clusters.h					*/
+/* Author:	Israel Fuller						*/
+/* Creation:	September 29, 2025					*/
+/* Description:	Clustering library used to cluster and search data with	*/
+/*		cosine or Levenshtein (aka. edit distance) similarity 	*/
+/*		measures. Used by the "clustering driver".		*/
+/*		For more information on how to use this library, see	*/
+/*		string_similarity.md in the centrallix-sysdoc folder.	*/
+/************************************************************************/
+
+#include <stdlib.h>
+#include <stdbool.h>
+
+#ifdef CXLIB_INTERNAL
+#include "xarray.h"
+#else
+#include "cxlib/xarray.h"
+#endif
+
+/** This file has additional documentation in string_similarity.md. **/
+
+
+/*** This value defines the number of dimensions used for a sparse
+ *** vector.  The higher the number, the fewer collisions will be
+ *** encountered when using these vectors for cosine comparisons.
+ *** This is also called the vector table size, if viewing the
+ *** vector as a hash table of character pairs.
+ *** 
+ *** 2147483647 is the signed int max, and is also a prime number.
+ *** Using this value ensures that the longest run of 0s will not
+ *** cause an int underflow with the current encoding scheme.
+ *** 
+ *** Unfortunately, we can't use a number this large yet because
+ *** the kmeans algorithm creates densely allocated centroids with
+ *** `CA_NUM_DIMS` dimensions, so a large number causes it to fail.
+ *** Thus, we use 251 as the largest prime number less than 256,
+ *** giving us a decent balance between collision reduction and
+ *** kmeans centroid performance/memory overhead.
+ ***/
+#define CA_NUM_DIMS 251
+
+/*** The character used to create a pair with the first and last characters
+ *** of a string.  Currently set to 96, the character just before 'a' (97)
+ *** in the ASCII table.
+ ***/
+#define CA_BOUNDARY_CHAR ((unsigned char)('a' - 1))
+
+/** Types. **/
+typedef int* pVector;      /* Sparse vector. */
+typedef double* pCentroid; /* Dense centroid. */
+#define CENTROID_SIZE (CA_NUM_DIMS * sizeof(double))
+
+/*** Information about detected matching pairs.
+ ***
+ *** @param i The index into the provided data for the first element of the pair.
+ *** @param j The index into the provided data for the second element of the pair.
+ *** @param similarity A number from 0 to 1, from a similarity function, showing
+ *** 	how similar the pair is.
+ ***/
+typedef struct
+    {
+    unsigned int i, j;
+    double similarity;
+    }
+    Pair, *pPair;
+
+
+/** Edit distance function. **/
+int caEditDist(const char* str1, const char* str2, const size_t str1_length, const size_t str2_length);
+
+/** Vector functions. **/
+pVector caBuildVector(const char* str);
+bool caEql(pVector v1, pVector v2);
+unsigned int caSparseLen(const pVector vector);
+void caPrintVector(const pVector vector);
+void caFreeVector(pVector sparse_vector);
+
+/** k-means function. **/
+int caKmeans(
+    pVector* vectors,
+    const unsigned int num_vectors,
+    const unsigned int num_clusters,
+    const unsigned int max_iter,
+    const double min_improvement,
+    unsigned int* labels,
+    double* vector_sims,
+    bool auto_seed);
+
+/** Vector helper macros. **/
+#define caIsEmpty(vector) (vector[0] == -CA_NUM_DIMS)
+/*** Note: Given that CA_NUM_DIMS == 251, caBuildVector("") will give the
+ *** vector we check for in the caHasNoPairs() macro, [-172, 11, -78],
+ *** which has a single pair of boundary characters.
+ *** If CA_NUM_DIMS is modified, this macro will need to be updated, hence the
+ *** compiler directive causing it to be undefined in this case, likely leading
+ *** to a lot of compiler or linker issues to remind the developer about this.
+ ***/
+#if CA_NUM_DIMS == 251
+#define caHasNoPairs(vector) \
+    ({ \
+	__typeof__ (vector) _v = (vector); \
+	_v[0] == -172 && _v[1] == 11 && _v[2] == -78; \
+    })
+#endif
+
+/*** A function which takes two data items and returns their similarity, a
+ *** value between `0.0` (completely different) and `1.0` (identical).
+ ***/
+typedef double (*pSimilarityFn)(void*, void*);
+
+/** Comparison functions (matching the pSimilarityFn signature). **/
+double caCosCompare(void* v1, void* v2);
+double caLevCompare(void* str1, void* str2);
+
+/** Similarity search functions. **/
+void* caMostSimilar(
+    void* target,
+    void** data,
+    const unsigned int num_data,
+    pSimilarityFn similarity,
+    const double threshold);
+pXArray caSlidingSearch(
+    void** data,
+    const unsigned int num_data,
+    const unsigned int window_size,
+    pSimilarityFn similarity,
+    const double threshold,
+    pXArray maybe_pairs);
+pXArray caCompleteSearch(
+    void** data,
+    const unsigned int num_data,
+    pSimilarityFn similarity,
+    const double threshold,
+    pXArray maybe_pairs);
+
+/** Module management functions. **/
+void caInit(void);
+
+#endif /* End of .h file. */