Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions products/signals/backend/personal_ranking/list_sort.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
"""Read one person's stored list for `sort=personal` on the report list. The read never builds.

It reads one `PersonalInboxRanking` row and nothing else: no candidate pools, no queued build, no model call. So a
page load can use it. The order puts the ranked reports first by stored rank, then the folded reports, and the view
orders every other report in the person's `recommended` order. With no usable row there is no order, and the list is
the plain `recommended` order.
"""

import uuid
from collections.abc import Mapping

from posthog.dataclasses import frozen
from posthog.models import Team, User

from products.signals.backend.models import PersonalInboxRanking
from products.signals.backend.personal_ranking.config import read_config


@frozen
class PersonalRowNote:
tier: str
reason: str


@frozen
class PersonalListOrder:
ranking_id: str
# The ranked reports by stored rank, then the folded reports.
report_ids: tuple[uuid.UUID, ...]
# Only the reports the language model ranked, by report id.
notes: Mapping[str, PersonalRowNote]


def _parse_id(value: object) -> uuid.UUID | None:
try:
return uuid.UUID(str(value))
except ValueError:
return None


def personal_list_order(team: Team, user: User, ranking_id: str | None = None) -> PersonalListOrder | None:
"""The order of the row `ranking_id` names, else of the latest personalized row. None when nothing is ranked."""
if not read_config().enabled:
return None
rows = PersonalInboxRanking.objects.for_team(team.id).filter(user_id=user.id, personalized=True)
# A later page names the row of the first page, so a rebuild between pages does not reshuffle them.
row = rows.filter(id=ranking_id).first() if ranking_id else None
if row is None:
row = rows.order_by("-created_at", "-id").first()
if row is None:
return None
result = row.result or {}
ranked = sorted(result.get("ranked") or (), key=lambda entry: entry["rank"])
report_ids: dict[uuid.UUID, None] = {}
notes: dict[str, PersonalRowNote] = {}
for entry in [*ranked, *(result.get("folded") or ())]:
report_id = _parse_id(entry.get("report_id"))
if report_id is None:
continue
report_ids.setdefault(report_id)
if entry.get("ranked_by") == "llm":
notes[str(report_id)] = PersonalRowNote(tier=entry["tier"], reason=entry["reason"])
if not report_ids:
return None
return PersonalListOrder(ranking_id=str(row.id), report_ids=tuple(report_ids), notes=notes)
3 changes: 3 additions & 0 deletions products/signals/backend/scout_report/trial_inbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
"count_only",
"ordering",
"sort",
"ranking_id",
"limit",
"offset",
"format",
Expand Down Expand Up @@ -115,7 +116,7 @@
return document

@classmethod
def overlay(cls, report: TrialReport, live: Mapping[str, object] | None = None) -> dict[str, JsonValue]:

Check warning on line 119 in products/signals/backend/scout_report/trial_inbox.py

View workflow job for this annotation

GitHub Actions / Python code quality (depot-ubuntu-24.04)

lint:complexity

`overlay` has cyclomatic complexity 11 (warn >10)

Check warning on line 119 in products/signals/backend/scout_report/trial_inbox.py

View workflow job for this annotation

GitHub Actions / Python code quality (depot-ubuntu-24.04)

`overlay` has cyclomatic complexity 11 (warn >10)
if report.source_report_id is None:
return dict(report.document)
if live is None:
Expand Down Expand Up @@ -296,6 +297,8 @@
include_source_metadata=include_source_metadata,
)
document["inbox_merge_key"] = None
document["personal_tier"] = None
document["personal_reason"] = None
if document.get("status") not in statuses:
continue
if search and not any(search in str(document.get(field, "")).casefold() for field in ("title", "summary")):
Expand Down
55 changes: 48 additions & 7 deletions products/signals/backend/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@
SignalTeamConfig,
SignalUserAutonomyConfig,
)
from .personal_ranking.list_sort import PersonalListOrder, PersonalRowNote
from .pull_request_label import DEFAULT_PULL_REQUEST_LABEL
from .ranking.inbox_sort import inbox_merge_key
from .ranking.public_outcomes import RankingOutcome
Expand Down Expand Up @@ -337,7 +338,7 @@
self._data_import_statuses_by_team[team_id] = statuses
return statuses

def validate(self, attrs: dict) -> dict:

Check warning on line 341 in products/signals/backend/serializers.py

View workflow job for this annotation

GitHub Actions / Python code quality (depot-ubuntu-24.04)

lint:complexity

`validate` has cyclomatic complexity 29 (warn >10)

Check warning on line 341 in products/signals/backend/serializers.py

View workflow job for this annotation

GitHub Actions / Python code quality (depot-ubuntu-24.04)

`validate` has cyclomatic complexity 29 (warn >10)
source_product = attrs.get("source_product", getattr(self.instance, "source_product", None))
source_type = attrs.get("source_type", getattr(self.instance, "source_type", None))
enabled = attrs.get("enabled", getattr(self.instance, "enabled", False))
Expand Down Expand Up @@ -2647,24 +2648,57 @@
)
inbox_merge_key = serializers.SerializerMethodField(
help_text=(
"Sort key from the ranker named by the list response's ranker field, for sort=recommended only. "
"Lower keys rank first, so rows from several requests merge in the order of one request. "
"For an outcome ranker (open, action or fix), a scored report gets 1 minus its score, from 0 to 1, "
"and a report without a usable score gets 2 plus its priority rank. The priority ranker uses the "
"priority rank only, from 0 (P0) to 5 (no priority). Null for every other sort."
"Sort key from the ranker named by the list response's ranker field, for sort=recommended and "
"sort=personal only. Lower keys rank first, so rows from several requests merge in the order of one "
"request. For an outcome ranker (open, action or fix), a scored report gets 1 minus its score, from "
"0 to 1, and a report without a usable score gets 2 plus its priority rank. The priority ranker uses "
"the priority rank only, from 0 (P0) to 5 (no priority). For the personal ranker, a report in the "
"stored personal ranking gets its position in that ranking from 0, and every other report gets the "
"number of those positions plus its key from the recommended ranker. Null for every other sort."
),
)
personal_tier = serializers.SerializerMethodField(
help_text=(
"The tier your personal ranking gave this report, for sort=personal. Null when the language model "
"did not rank the report, and for every other sort."
),
)
personal_reason = serializers.SerializerMethodField(
help_text=(
"Why your personal ranking put this report where it is, for sort=personal. Null when the language "
"model did not rank the report, and for every other sort."
),
)

class Meta(SignalReportSerializer.Meta):
fields = [*SignalReportSerializer.Meta.fields, "inbox_merge_key"]
fields = [*SignalReportSerializer.Meta.fields, "inbox_merge_key", "personal_tier", "personal_reason"]
read_only_fields = fields

def get_inbox_merge_key(self, obj: SignalReport) -> float | None:
ranker = self.context.get("inbox_ranker")
if ranker is None:
return None
score = None if ranker == "priority" else getattr(obj, f"ranking_{ranker}_score", None)
return inbox_merge_key(ranker, score, getattr(obj, "priority_sort_rank", None))
key = inbox_merge_key(ranker, score, getattr(obj, "priority_sort_rank", None))
personal_order: PersonalListOrder | None = self.context.get("personal_order")
if personal_order is None:
return key
personal_rank = getattr(obj, "personal_rank", None)
if personal_rank is not None:
return float(personal_rank - 1)
return len(personal_order.report_ids) + key

def _personal_note(self, obj: SignalReport) -> PersonalRowNote | None:
personal_order: PersonalListOrder | None = self.context.get("personal_order")
return personal_order.notes.get(str(obj.id)) if personal_order is not None else None

def get_personal_tier(self, obj: SignalReport) -> str | None:
note = self._personal_note(obj)
return note.tier if note is not None else None

def get_personal_reason(self, obj: SignalReport) -> str | None:
note = self._personal_note(obj)
return note.reason if note is not None else None


class SignalReportSectionCountsSerializer(serializers.Serializer):
Expand Down Expand Up @@ -2695,6 +2729,13 @@


class SignalReportListQuerySerializer(SignalReportFilterQuerySerializer):
ranking_id = serializers.UUIDField(
required=False,
help_text=(
"For sort=personal: the ranking_id of the first page. Pass it on later pages, so a new personal "
"ranking does not reorder them. When that ranking no longer exists, the latest one applies."
),
)
count_only = serializers.BooleanField(
required=False,
default=False,
Expand Down
141 changes: 141 additions & 0 deletions products/signals/backend/test/test_personal_ranking.py
Original file line number Diff line number Diff line change
Expand Up @@ -1173,3 +1173,144 @@ def test_unavailable(self, _name: str, payload: dict[str, Any], flag_enabled: bo
else:
assert response.status_code == 404
assert (self._rows(), self.queued) == (0, [])


class TestPersonalListSort(APIBaseTest):
def setUp(self) -> None:
super().setUp()
self.payload: dict[str, Any] = {}
self.flag_enabled = True
self.analytics = MagicMock()
self.analytics.get_feature_flag.return_value = "control"
self.gather = MagicMock()
self.delay = MagicMock()
for target, value in (
(
"products.signals.backend.personal_ranking.config.read_flag_payload",
MagicMock(side_effect=lambda *_: self.payload),
),
(
"products.signals.backend.ranking.access.feature_enabled_or_false",
MagicMock(side_effect=lambda *_args, **_kwargs: self.flag_enabled),
),
("products.signals.backend.ranking.inbox_sort.posthoganalytics", self.analytics),
("products.signals.backend.personal_ranking.serve.gather_candidates", self.gather),
("products.signals.backend.personal_ranking.serve.build_personal_inbox_ranking.delay", self.delay),
):
patcher = patch(target, value)
patcher.start()
self.addCleanup(patcher.stop)
self.reports = [
SignalReport.objects.create(
team=self.team, title=f"R{index}", summary="s", signal_count=1, total_weight=1.0, status="ready"
)
for index in range(5)
]

def _ranking(self, ranked: list[SignalReport], folded: list[SignalReport], **kwargs: Any) -> PersonalInboxRanking:
defaults: dict[str, Any] = {
"personalized": True,
"result": {
"ranked": [
{
"report_id": str(report.id),
"rank": rank,
"tier": "top" if rank == 1 else "later",
"reason": f"reason {report.title}",
"ranked_by": "llm" if rank == 1 else "model",
}
# Stored out of rank order, so the read must sort by rank.
for rank, report in reversed(list(enumerate(ranked, start=1)))
],
"folded": [
{"report_id": str(report.id), "into_report_id": str(ranked[0].id), "reason": "same"}
for report in folded
],
},
}
return PersonalInboxRanking.objects.create(
team=self.team,
user=self.user,
input_hash="h",
model="m",
prompt_version=1,
status="ok",
**{**defaults, **kwargs},
)

def _list(self, **params: Any) -> dict[str, Any]:
query = "&".join(f"{key}={value}" for key, value in {"status": "ready", **params}.items())
response = self.client.get(f"/api/projects/{self.team.id}/signals/reports/?{query}")
assert response.status_code == 200, response.json()
return response.json()

def _ids(self, body: dict[str, Any]) -> list[str]:
return [row["id"] for row in body["results"]]

def test_puts_ranked_then_folded_reports_first_without_a_build_or_an_exposure(self) -> None:
r0, r1, r2, r3, r4 = self.reports
row = self._ranking([r2, r0], [r1])
base = self._ids(self._list(sort="recommended"))
self.analytics.reset_mock()

body = self._list(sort="personal")

rest = [report_id for report_id in base if report_id not in {str(r2.id), str(r0.id), str(r1.id)}]
assert self._ids(body) == [str(r2.id), str(r0.id), str(r1.id), *rest]
assert (body["ranker"], body["ranking_id"], body["ranking_pill"]) == ("personal", str(row.id), False)
assert [row["inbox_merge_key"] for row in body["results"]] == [0, 1, 2, 3 + 5, 3 + 5]
assert [(row["personal_tier"], row["personal_reason"]) for row in body["results"][:3]] == [
("top", "reason R2"),
(None, None),
(None, None),
]
assert self.analytics.get_feature_flag.call_args.kwargs["send_feature_flag_events"] is False
assert (self.gather.call_count, self.delay.call_count) == (0, 0)
assert PersonalInboxRanking.objects.for_team(self.team.id).count() == 1

@parameterized.expand(
[
("no_row", None, True, True),
("empty_row", {"ranked": [], "folded": []}, True, True),
("not_personalized", "not_personalized", True, True),
("flag_off", "ranked", False, True),
("disabled", "ranked", True, False),
]
)
def test_falls_back_to_the_recommended_order(
self, _name: str, row: Any, flag_enabled: bool, config_enabled: bool
) -> None:
r0, r1, *_ = self.reports
if row == "not_personalized":
self._ranking([r0], [r1], personalized=False)
elif row == "ranked":
self._ranking([r0], [r1])
elif row is not None:
self._ranking([], [], result=row)
base = self._list(sort="recommended")
self.flag_enabled = flag_enabled
self.payload = {} if config_enabled else {"enabled": False}

body = self._list(sort="personal")

assert self._ids(body) == self._ids(base)
assert (body["ranker"], body["ranking_id"]) == ("priority", None)
assert [row["inbox_merge_key"] for row in body["results"]] == [
row["inbox_merge_key"] for row in base["results"]
]
assert all(row["personal_reason"] is None for row in body["results"])
assert (self.gather.call_count, self.delay.call_count) == (0, 0)

def test_later_pages_keep_the_order_of_the_first_page_after_a_rebuild(self) -> None:
r0, r1, r2, r3, r4 = self.reports
old = self._ranking([r0, r1, r2, r3], [])
first = self._list(sort="personal", limit=2)
assert (self._ids(first), first["ranking_id"]) == ([str(r0.id), str(r1.id)], str(old.id))
self._ranking([r3, r2, r1, r0], [])

second = self._list(sort="personal", limit=2, offset=2, ranking_id=first["ranking_id"])
assert (self._ids(second), second["ranking_id"]) == ([str(r2.id), str(r3.id)], str(old.id))

old.delete()
latest = self._list(sort="personal", limit=2, offset=2, ranking_id=first["ranking_id"])
assert self._ids(latest) == [str(r1.id), str(r0.id)]
Loading
Loading