Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tools/mtmd/clip-model.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ enum resize_algo {
RESIZE_ALGO_BILINEAR, // stretch to target resolution
RESIZE_ALGO_BICUBIC, // center-crop when aspect ratio doesn't match
RESIZE_ALGO_BICUBIC_PILLOW,
// RESIZE_ALGO_LANCZOS, // TODO
RESIZE_ALGO_LANCZOS,
};

// Padding style for img_tool::resize
Expand Down
60 changes: 50 additions & 10 deletions tools/mtmd/mtmd-image.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ struct img_tool {
case RESIZE_ALGO_BICUBIC_PILLOW:
resize_bicubic_pillow(src, dst, target_resolution.width, target_resolution.height);
break;
case RESIZE_ALGO_LANCZOS:
resize_lanczos_pillow(src, dst, target_resolution.width, target_resolution.height);
break;
default:
throw std::runtime_error("Unsupported resize algorithm");
}
Expand Down Expand Up @@ -97,6 +100,9 @@ struct img_tool {
case RESIZE_ALGO_BICUBIC_PILLOW:
resize_bicubic_pillow(src, resized_image, new_width, new_height);
break;
case RESIZE_ALGO_LANCZOS:
resize_lanczos_pillow(src, resized_image, new_width, new_height);
break;
default:
throw std::runtime_error("Unsupported resize algorithm");
}
Expand Down Expand Up @@ -337,22 +343,50 @@ struct img_tool {
}
}

// Bicubic resize function using Pillow's ImagingResample algorithm
// Pillow-compatible separable resampling (Bicubic and Lanczos)
// Adapted from https://github.com/python-pillow/Pillow/blob/main/src/libImaging/Resample.c
//
// Key Difference with resize_bicubic:
// 1. Uses separable filtering: horizontal pass followed by vertical pass
// Key properties:
// 1. Separable filtering: horizontal pass followed by vertical pass
// 2. Pre-computes normalized filter coefficients for each output pixel
// 3. Applies convolution using fixed-point integer arithmetic for performance
// 3. Fixed-point integer arithmetic (22 fractional bits) for speed and determinism
static bool resize_bicubic_pillow(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) {
return resize_pillow(img, dst, target_width, target_height, /*use_lanczos=*/false);
}

// Lanczos-3 (support radius 3), matches Pillow's Image.LANCZOS
static bool resize_lanczos_pillow(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) {
return resize_pillow(img, dst, target_width, target_height, /*use_lanczos=*/true);
}

static bool resize_pillow(
const clip_image_u8 & img,
clip_image_u8 & dst,
int target_width,
int target_height,
bool use_lanczos) {
// Fixed-point precision: 22 bits = 32 (int32_t) - 8 (uint8_t pixels) - 2 (headroom for accumulation)
// This allows encoding fractional weights as integers: weight * 2^22
const int PRECISION_BITS = 32 - 8 - 2;

// Bicubic filter function with a = -0.5 (Note that GGML/PyTorch takes a = -0.75)
// Resample filter: Lanczos-3 (support [-3, 3]) or bicubic with a = -0.5 (support [-2, 2])
// Note: GGML/PyTorch bicubic uses a = -0.75, Pillow uses a = -0.5
// Returns filter weight for distance x from pixel center
// Support: [-2, 2], meaning the filter influences pixels within 2 units of distance
auto bicubic_filter = [](double x) -> double {
auto resample_filter = [use_lanczos](double x) -> double {
if (use_lanczos) {
if (-3.0 <= x && x < 3.0) {
auto sinc = [](double v) {
if (v == 0.0) {
return 1.0;
}
const double pi_v = v * 3.141592653589793238462643383279502884;
return std::sin(pi_v) / pi_v;
};
return sinc(x) * sinc(x / 3.0);
}
return 0.0;
}

constexpr double a = -0.5;
if (x < 0.0) {
x = -x;
Expand All @@ -366,8 +400,8 @@ struct img_tool {
return 0.0; // Zero outside [-2, 2]
};

// Filter support radius: bicubic extends 2 pixels in each direction
constexpr double filter_support = 2.0;
// Filter support radius: 2 for bicubic, 3 for lanczos
const double filter_support = use_lanczos ? 3.0 : 2.0;

// Clipping function for 8-bit values
auto clip8 = [](int val) -> uint8_t {
Expand Down Expand Up @@ -434,7 +468,7 @@ struct img_tool {
// Compute filter weights for each contributing input pixel
for (x = 0; x < xmax; x++) {
// Distance from input pixel center to output pixel center in input space
double w = bicubic_filter((x + xmin - center + 0.5) * ss);
double w = resample_filter((x + xmin - center + 0.5) * ss);
pre_weights[xx * ksize + x] = w;
ww += w; // Accumulate for normalization
}
Expand Down Expand Up @@ -463,6 +497,12 @@ struct img_tool {
const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS

for (int i = 0; i < outSize * ksize; i++) {
if (use_lanczos) {
// Pillow adds +/- 0.5 then truncates toward zero; std::round would round twice
const double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5);
weights[i] = static_cast<int32_t>(rounded);
continue;
}
double tmp_val = pre_weights[i] * fxp_scale;
if (pre_weights[i] < 0) {
tmp_val -= 0.5;
Expand Down
Loading