Skip to content

Insights: don't count never-answered journal days as "without" #682

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
digitalerdude wants to merge 1 commit into from
Closed

Insights: don't count never-answered journal days as "without" #682

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions Packages/StrandAnalytics/Sources/StrandAnalytics/BehaviorInsights.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,18 +78,25 @@ public enum BehaviorInsights {
// MARK: - Single behavior effect

/// Compute the effect of `behavior` on `outcome`. Days are partitioned into
/// "with" (day ∈ behaviorDays) and "without" (day ∉ behaviorDays), restricted
/// to days that have an outcome value in `outcomeByDay`.
/// "with" (day ∈ behaviorDays) and "without" (day ∈ askedDays, day ∉ behaviorDays),
/// restricted to days that have an outcome value in `outcomeByDay`. A day that was never
/// asked/answered (day ∉ behaviorDays ∪ askedDays) counts in NEITHER group — a day just
/// missing a journal answer is not the same as an explicit "no" (issue #631).
///
/// `behaviorDays` is expected to be a subset of `askedDays`; the check below is an OR so a
/// "with" day is never dropped even if a caller's askedDays build has a gap.
///
/// Returns nil unless BOTH groups are non-empty AND the total is large enough
/// to form a variance estimate (at least 1 value per side and ≥ 3 total).
public static func effect(behaviorDays: Set<String>,
askedDays: Set<String>,
outcomeByDay: [String: Double],
behavior: String,
outcome: String) -> BehaviorEffect? {
var withVals: [Double] = []
var withoutVals: [Double] = []
for (day, value) in outcomeByDay {
guard behaviorDays.contains(day) || askedDays.contains(day) else { continue }
if behaviorDays.contains(day) { withVals.append(value) }
else { withoutVals.append(value) }
}
Expand Down Expand Up @@ -125,11 +132,13 @@ public enum BehaviorInsights {
/// return them sorted by |cohensD| descending, with significant effects first.
/// Behaviors that don't yield a computable effect are dropped.
public static func rank(behaviors: [String: Set<String>],
asked: [String: Set<String>],
outcomeByDay: [String: Double],
outcome: String) -> [BehaviorEffect] {
var effects: [BehaviorEffect] = []
for (name, days) in behaviors {
if let e = effect(behaviorDays: days, outcomeByDay: outcomeByDay,
let askedDays = asked[name] ?? days
if let e = effect(behaviorDays: days, askedDays: askedDays, outcomeByDay: outcomeByDay,
behavior: name, outcome: outcome) {
effects.append(e)
}
Expand Down
8 changes: 7 additions & 1 deletion Packages/StrandAnalytics/Sources/StrandAnalytics/EffectRanker.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,15 @@ public enum EffectRanker {
/// significant-first, |cohensD| desc, then behaviour name asc. Behaviours with no
/// computable lag are dropped.
public static func rank(behaviors: [String: Set<String>],
asked: [String: Set<String>],
outcomeByDay: [String: Double],
outcome: String) -> [RankedEffect] {
var rows: [RankedEffect] = []
// Sort behaviour names so the build order is deterministic regardless of dict order.
for name in behaviors.keys.sorted() {
let days = behaviors[name]!
if let row = bestLag(behaviorDays: days, outcomeByDay: outcomeByDay,
let askedDays = asked[name] ?? days
if let row = bestLag(behaviorDays: days, askedDays: askedDays, outcomeByDay: outcomeByDay,
behavior: name, outcome: outcome) {
rows.append(row)
}
Expand All @@ -121,13 +123,17 @@ public enum EffectRanker {
/// Find the best-lag RankedEffect for ONE behaviour against ONE outcome, or nil when no
/// lag in `lagSet` yields a computable effect that clears the group gate.
public static func bestLag(behaviorDays: Set<String>,
askedDays: Set<String>,
outcomeByDay: [String: Double],
behavior: String,
outcome: String) -> RankedEffect? {
var best: (lag: Int, effect: BehaviorEffect)?
for lag in lagSet {
let shifted = shiftedOutcome(outcomeByDay, byLag: lag)
// askedDays/behaviorDays already live in behaviour-day space — only the outcome
// series gets re-keyed per lag, never the asked/behaviour sets.
guard let e = BehaviorInsights.effect(behaviorDays: behaviorDays,
askedDays: askedDays,
outcomeByDay: shifted,
behavior: behavior,
outcome: outcome) else { continue }
Expand Down
83 changes: 82 additions & 1 deletion Packages/StrandAnalytics/Tests/StrandAnalyticsTests/BehaviorInsightsTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ final class BehaviorInsightsTests: XCTestCase {
"d13": 70, "d14": 68,
]
let behaviorDays: Set<String> = ["d01", "d02", "d03", "d04", "d05", "d06"]
let e = BehaviorInsights.effect(behaviorDays: behaviorDays, outcomeByDay: outcome,
let e = BehaviorInsights.effect(behaviorDays: behaviorDays, askedDays: Set(outcome.keys),
outcomeByDay: outcome,
behavior: "Alcohol", outcome: "Recovery")!
XCTAssertEqual(e.nWith, 6)
XCTAssertEqual(e.nWithout, 8)
Expand All @@ -34,6 +35,7 @@ final class BehaviorInsightsTests: XCTestCase {
"f": 70, "g": 72, "h": 68, "i": 71, "j": 69, // without (mean 70)
]
let e = BehaviorInsights.effect(behaviorDays: ["a", "b", "c", "d", "e"],
askedDays: Set(outcome.keys),
outcomeByDay: outcome,
behavior: "Meditation", outcome: "Recovery")!
XCTAssertEqual(e.delta, 10.0, accuracy: 1e-9)
Expand All @@ -46,10 +48,12 @@ final class BehaviorInsightsTests: XCTestCase {
// Behavior logged every day → no "without" group.
let outcome: [String: Double] = ["a": 60, "b": 61, "c": 62]
XCTAssertNil(BehaviorInsights.effect(behaviorDays: ["a", "b", "c"],
askedDays: Set(outcome.keys),
outcomeByDay: outcome,
behavior: "X", outcome: "Recovery"))
// Behavior never logged → no "with" group.
XCTAssertNil(BehaviorInsights.effect(behaviorDays: [],
askedDays: Set(outcome.keys),
outcomeByDay: outcome,
behavior: "X", outcome: "Recovery"))
}
Expand All @@ -58,12 +62,65 @@ final class BehaviorInsightsTests: XCTestCase {
// "z" is in behaviorDays but has no outcome value → not counted in nWith.
let outcome: [String: Double] = ["a": 60, "b": 62, "c": 70, "d": 72]
let e = BehaviorInsights.effect(behaviorDays: ["a", "b", "z"],
askedDays: Set(outcome.keys).union(["z"]),
outcomeByDay: outcome,
behavior: "X", outcome: "Recovery")!
XCTAssertEqual(e.nWith, 2) // a, b only
XCTAssertEqual(e.nWithout, 2) // c, d
}

// MARK: - never-asked days (#631: a missing journal answer is not an implicit "no")

func testEffectExcludesUnaskedDaysFromBothGroups() {
// "z" has an outcome value but was never asked (not in behaviorDays or askedDays) →
// dropped entirely, not counted as "without".
let outcome: [String: Double] = ["a": 60, "b": 62, "c": 70, "d": 72, "z": 65]
let e = BehaviorInsights.effect(behaviorDays: ["a", "b"],
askedDays: ["a", "b", "c", "d"],
outcomeByDay: outcome,
behavior: "X", outcome: "Recovery")!
XCTAssertEqual(e.nWith, 2) // a, b
XCTAssertEqual(e.nWithout, 2) // c, d — "z" excluded, not counted as without
}

func testEffectAskedNoStillCountsAsWithout() {
// A day that WAS asked and explicitly answered "no" (in askedDays, not in behaviorDays)
// still counts as "without" — the fix only excludes NEVER-asked days.
let outcome: [String: Double] = ["a": 60, "b": 61, "c": 70, "d": 71, "e": 72]
let e = BehaviorInsights.effect(behaviorDays: ["a", "b"],
askedDays: ["a", "b", "c", "d", "e"],
outcomeByDay: outcome,
behavior: "X", outcome: "Recovery")!
XCTAssertEqual(e.nWith, 2)
XCTAssertEqual(e.nWithout, 3)
}

func testEffectBehaviorDayNotInAskedStillCountsAsWith() {
// Defensive OR: a "yes" day missing from askedDays (a caller build gap) is never dropped
// from "with".
let outcome: [String: Double] = ["a": 80, "b": 60, "c": 61]
let e = BehaviorInsights.effect(behaviorDays: ["a"],
askedDays: ["b", "c"], // "a" deliberately missing
outcomeByDay: outcome,
behavior: "X", outcome: "Recovery")!
XCTAssertEqual(e.nWith, 1) // "a" still counted despite the askedDays gap
XCTAssertEqual(e.nWithout, 2)
}

func testEffectRealisticSparseJournalQuestion() {
// #631 shape: a rarely-asked question logged "yes" on 3 days and "no" on 2 days, out of
// 300 total outcome-days. The other 295 days must be excluded, not lumped into "without".
var outcome: [String: Double] = [:]
for i in 0..<300 { outcome["d\(i)"] = Double(60 + i % 10) }
let behaviorDays: Set<String> = ["d0", "d1", "d2"]
let askedDays: Set<String> = behaviorDays.union(["d3", "d4"])
let e = BehaviorInsights.effect(behaviorDays: behaviorDays, askedDays: askedDays,
outcomeByDay: outcome,
behavior: "Travelled in a car or train?", outcome: "Charge")!
XCTAssertEqual(e.nWith, 3)
XCTAssertEqual(e.nWithout, 2) // NOT 297
}

// MARK: - significance flips

func testSignificanceFlipsWithGroupSize() {
Expand All @@ -74,6 +131,7 @@ final class BehaviorInsightsTests: XCTestCase {
"o1": 70, "o2": 71, "o3": 69, "o4": 70,
]
let small = BehaviorInsights.effect(behaviorDays: ["w1", "w2", "w3", "w4"],
askedDays: Set(smallOutcome.keys),
outcomeByDay: smallOutcome,
behavior: "X", outcome: "Recovery")!
XCTAssertLessThan(small.pApprox, 0.05) // strong evidence numerically…
Expand All @@ -86,6 +144,7 @@ final class BehaviorInsightsTests: XCTestCase {
"o1": 70, "o2": 71, "o3": 69, "o4": 70, "o5": 70,
]
let big = BehaviorInsights.effect(behaviorDays: ["w1", "w2", "w3", "w4", "w5"],
askedDays: Set(bigOutcome.keys),
outcomeByDay: bigOutcome,
behavior: "X", outcome: "Recovery")!
XCTAssertEqual(Swift.min(big.nWith, big.nWithout), 5)
Expand All @@ -99,6 +158,7 @@ final class BehaviorInsightsTests: XCTestCase {
"o1": 64, "o2": 72, "o3": 61, "o4": 74, "o5": 67, "o6": 69,
]
let e = BehaviorInsights.effect(behaviorDays: ["w1", "w2", "w3", "w4", "w5", "w6"],
askedDays: Set(outcome.keys),
outcomeByDay: outcome,
behavior: "X", outcome: "Recovery")!
XCTAssertGreaterThan(e.pApprox, 0.05) // no separation → weak evidence
Expand All @@ -119,8 +179,12 @@ final class BehaviorInsightsTests: XCTestCase {
let weak: Set<String> = ["d1", "d7", "d2"]
// Tiny-but-significant-impossible: 2 days only.
let tiny: Set<String> = ["d3", "d9"]
let asked: [String: Set<String>] = [
"Strong": Set(outcome.keys), "Weak": Set(outcome.keys), "Tiny": Set(outcome.keys),
]

let ranked = BehaviorInsights.rank(behaviors: ["Strong": strong, "Weak": weak, "Tiny": tiny],
asked: asked,
outcomeByDay: outcome, outcome: "Recovery")
XCTAssertEqual(ranked.count, 3)
XCTAssertEqual(ranked.first?.behavior, "Strong") // significant + largest |d|
Expand All @@ -138,11 +202,25 @@ final class BehaviorInsightsTests: XCTestCase {
let ranked = BehaviorInsights.rank(behaviors: [
"AllDays": ["a", "b", "c", "d"],
"Half": ["a", "b"],
], asked: [
"AllDays": Set(outcome.keys),
"Half": Set(outcome.keys),
], outcomeByDay: outcome, outcome: "Recovery")
XCTAssertEqual(ranked.count, 1)
XCTAssertEqual(ranked.first?.behavior, "Half")
}

func testRankAskedDaysFallsBackToBehaviorDaysWhenMissing() {
// A behaviour missing from `asked` falls back to its own behaviorDays — i.e. only the
// "with" days count as asked, so there's no "without" group and the behaviour is dropped,
// rather than crashing or silently reproducing the old "everything else is without" bug.
let outcome: [String: Double] = ["a": 60, "b": 62, "c": 70, "d": 72]
let ranked = BehaviorInsights.rank(behaviors: ["Half": ["a", "b"]],
asked: [:],
outcomeByDay: outcome, outcome: "Recovery")
XCTAssertTrue(ranked.isEmpty)
}

// MARK: - sentence

func testSentenceLowerWithPercent() {
Expand All @@ -153,6 +231,7 @@ final class BehaviorInsightsTests: XCTestCase {
"o1": 78, "o2": 82, "o3": 80, "o4": 79, "o5": 81, // mean 80
]
let e = BehaviorInsights.effect(behaviorDays: ["w1", "w2", "w3", "w4", "w5"],
askedDays: Set(outcome.keys),
outcomeByDay: outcome,
behavior: "Alcohol", outcome: "Recovery")!
let s = BehaviorInsights.sentence(e)
Expand All @@ -165,6 +244,7 @@ final class BehaviorInsightsTests: XCTestCase {
"o1": 49, "o2": 51, "o3": 50, // mean 50
]
let e = BehaviorInsights.effect(behaviorDays: ["w1", "w2", "w3"],
askedDays: Set(outcome.keys),
outcomeByDay: outcome,
behavior: "Meditation", outcome: "Recovery")!
let s = BehaviorInsights.sentence(e)
Expand All @@ -179,6 +259,7 @@ final class BehaviorInsightsTests: XCTestCase {
"o1": 0, "o2": 0, "o3": 0,
]
let e = BehaviorInsights.effect(behaviorDays: ["w1", "w2", "w3"],
askedDays: Set(outcome.keys),
outcomeByDay: outcome,
behavior: "X", outcome: "HRV")!
XCTAssertNil(e.pctChange)
Expand Down
Loading
Loading