Skip to content

fix(cohorts): Handle string 'null' values correctly in IS_SET/IS_NOT_SET operators #36729

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
haacked wants to merge 3 commits into from
Closed

fix(cohorts): Handle string 'null' values correctly in IS_SET/IS_NOT_SET operators #36729

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
24dafee
fix(cohorts): Handle string 'null' values correctly in IS_SET/IS_NOT_…
haacked Aug 15, 2025
b098f6d
Update query snapshots
github-actions[bot] Aug 15, 2025
9a74047
Update query snapshots
github-actions[bot] Aug 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 21 additions & 5 deletions posthog/hogql/property.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,18 +170,34 @@ def _expr_to_compare_op(
expr: ast.Expr, value: ValueT, operator: PropertyOperator, property: Property, is_json_field: bool, team: Team
) -> ast.Expr:
if operator == PropertyOperator.IS_SET:
return ast.CompareOperation(
op=ast.CompareOperationOp.NotEq,
left=expr,
right=ast.Constant(value=None),
# IS_SET should exclude both SQL NULL and string "null" values
return ast.And(
exprs=[
ast.CompareOperation(
op=ast.CompareOperationOp.NotEq,
left=expr,
right=ast.Constant(value=None),
),
ast.CompareOperation(
op=ast.CompareOperationOp.NotEq,
left=expr,
right=ast.Constant(value="null"),
),
]
)
elif operator == PropertyOperator.IS_NOT_SET:
# IS_NOT_SET should include SQL NULL, missing keys, and string "null" values
exprs: list[ast.Expr] = [
ast.CompareOperation(
op=ast.CompareOperationOp.Eq,
left=expr,
right=ast.Constant(value=None),
)
),
ast.CompareOperation(
op=ast.CompareOperationOp.Eq,
left=expr,
right=ast.Constant(value="null"),
),
]

if is_json_field:
Expand Down
57 changes: 54 additions & 3 deletions posthog/hogql/test/test_property.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ def test_property_to_expr_group(self):
self._parse_expr("group_3.properties.a = 'b'"),
)
self.assertEqual(
self._parse_expr("group_0.properties.a = NULL OR (NOT JSONHas(group_0.properties, 'a'))"),
self._parse_expr(
"group_0.properties.a = NULL OR group_0.properties.a = 'null' OR (NOT JSONHas(group_0.properties, 'a'))"
),
self._property_to_expr(
{"type": "group", "group_type_index": 0, "key": "a", "value": "b", "operator": "is_not_set"}
),
Expand Down Expand Up @@ -135,10 +137,10 @@ def test_property_to_expr_event(self):
)
self.assertEqual(
self._property_to_expr({"type": "event", "key": "a", "value": "b", "operator": "is_set"}),
self._parse_expr("properties.a != NULL"),
self._parse_expr("properties.a != NULL AND properties.a != 'null'"),
)
self.assertEqual(
self._parse_expr("properties.a = NULL OR (NOT JSONHas(properties, 'a'))"),
self._parse_expr("properties.a = NULL OR properties.a = 'null' OR (NOT JSONHas(properties, 'a'))"),
self._property_to_expr({"type": "event", "key": "a", "value": "b", "operator": "is_not_set"}),
)
self.assertEqual(
Expand Down Expand Up @@ -909,3 +911,52 @@ def test_map_virtual_properties(self):
chain=["person", "properties", 42]
)
assert map_virtual_properties(ast.Field(chain=["properties", 42])) == ast.Field(chain=["properties", 42])

def test_is_set_with_string_null_values(self):
"""Test that IS_SET properly excludes string 'null' values."""
# IS_SET should exclude both SQL NULL and string "null"
self.assertEqual(
self._property_to_expr({"type": "event", "key": "test_prop", "operator": "is_set"}),
self._parse_expr("properties.test_prop != NULL AND properties.test_prop != 'null'"),
)

# Test with person properties too
self.assertEqual(
self._property_to_expr({"type": "person", "key": "test_prop", "operator": "is_set"}),
self._parse_expr("person.properties.test_prop != NULL AND person.properties.test_prop != 'null'"),
)

def test_is_not_set_with_string_null_values(self):
"""Test that IS_NOT_SET properly includes string 'null' values."""
# IS_NOT_SET should include SQL NULL, missing keys, and string "null"
self.assertEqual(
self._property_to_expr({"type": "event", "key": "test_prop", "operator": "is_not_set"}),
self._parse_expr(
"properties.test_prop = NULL OR properties.test_prop = 'null' OR (NOT JSONHas(properties, 'test_prop'))"
),
)

# Test with person properties too
self.assertEqual(
self._property_to_expr({"type": "person", "key": "test_prop", "operator": "is_not_set"}),
self._parse_expr(
"person.properties.test_prop = NULL OR person.properties.test_prop = 'null' OR (NOT JSONHas(person.properties, 'test_prop'))"
),
)

def test_is_set_is_not_set_with_groups(self):
"""Test IS_SET/IS_NOT_SET with group properties."""
# Group properties should also handle string "null" correctly
self.assertEqual(
self._property_to_expr({"type": "group", "group_type_index": 0, "key": "test_prop", "operator": "is_set"}),
self._parse_expr("group_0.properties.test_prop != NULL AND group_0.properties.test_prop != 'null'"),
)

self.assertEqual(
self._property_to_expr(
{"type": "group", "group_type_index": 0, "key": "test_prop", "operator": "is_not_set"}
),
self._parse_expr(
"group_0.properties.test_prop = NULL OR group_0.properties.test_prop = 'null' OR (NOT JSONHas(group_0.properties, 'test_prop'))"
),
)
2 changes: 1 addition & 1 deletion posthog/hogql_queries/web_analytics/test/__snapshots__/test_web_stats_table.ambr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1643,7 +1643,7 @@
WHERE equals(person_distinct_id_overrides.team_id, 99999)
GROUP BY person_distinct_id_overrides.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id)
WHERE and(equals(events.team_id, 99999), and(or(and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2024-06-26 00:00:00', 'UTC'))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2024-06-27 23:59:59', 'UTC')))), 0), or(equals(events.event, '$pageview'), equals(events.event, '$screen')), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'utm_source'), ''), 'null'), '^"|"$', '')), not(JSONHas(events.properties, 'utm_source'))), 1))
WHERE and(equals(events.team_id, 99999), and(or(and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2024-06-26 00:00:00', 'UTC'))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2024-06-27 23:59:59', 'UTC')))), 0), or(equals(events.event, '$pageview'), equals(events.event, '$screen')), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'utm_source'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'utm_source'), ''), 'null'), '^"|"$', ''), 'null'), 0), not(JSONHas(events.properties, 'utm_source'))), 1))
GROUP BY session_id,
breakdown_value)
GROUP BY `context.columns.breakdown_value`
Expand Down
6 changes: 3 additions & 3 deletions ...queries/test/listing_recordings/__snapshots__/test_session_recording_list_from_query.ambr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1052,7 +1052,7 @@
WHERE and(equals(s.team_id, 99999), greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-31 20:00:00.000000', 6, 'UTC')), greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-18 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('today', 6, 'UTC')), globalIn(s.session_id,
(SELECT events.`$session_id` AS session_id
FROM events
WHERE and(equals(events.team_id, 99999), notEmpty(events.`$session_id`), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-12-31 20:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), now64(6, 'UTC')), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$feature/target-flag'), ''), 'null'), '^"|"$', '')), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2021-01-17 00:00:00.000000', 6, 'UTC')))
WHERE and(equals(events.team_id, 99999), notEmpty(events.`$session_id`), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-12-31 20:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), now64(6, 'UTC')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$feature/target-flag'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$feature/target-flag'), ''), 'null'), '^"|"$', ''), 'null'), 1)), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2021-01-17 00:00:00.000000', 6, 'UTC')))
GROUP BY events.`$session_id`
HAVING 1
ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC)))
Expand Down Expand Up @@ -1095,9 +1095,9 @@
WHERE and(equals(s.team_id, 99999), greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2020-12-31 20:00:00.000000', 6, 'UTC')), greaterOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('2021-01-18 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(s.min_first_timestamp, 'UTC'), toDateTime64('today', 6, 'UTC')), globalIn(s.session_id,
(SELECT events.`$session_id` AS session_id
FROM events
WHERE and(equals(events.team_id, 99999), notEmpty(events.`$session_id`), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-12-31 20:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), now64(6, 'UTC')), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$feature/target-flag'), ''), 'null'), '^"|"$', '')), not(JSONHas(events.properties, '$feature/target-flag'))), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2021-01-17 00:00:00.000000', 6, 'UTC')))
WHERE and(equals(events.team_id, 99999), notEmpty(events.`$session_id`), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-12-31 20:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), now64(6, 'UTC')), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$feature/target-flag'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$feature/target-flag'), ''), 'null'), '^"|"$', ''), 'null'), 0), not(JSONHas(events.properties, '$feature/target-flag'))), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2021-01-17 00:00:00.000000', 6, 'UTC')))
GROUP BY events.`$session_id`
HAVING ifNull(equals(countIf(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$feature/target-flag'), ''), 'null'), '^"|"$', ''))), 0), 0)
HAVING ifNull(equals(countIf(and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$feature/target-flag'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$feature/target-flag'), ''), 'null'), '^"|"$', ''), 'null'), 1))), 0), 0)
ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC)))
GROUP BY s.session_id
HAVING 1
Expand Down