diff --git a/import_data/activity_parsers.py b/import_data/activity_parsers.py index 15c3474..da22e27 100644 --- a/import_data/activity_parsers.py +++ b/import_data/activity_parsers.py @@ -84,6 +84,31 @@ def fitbit_parser(fitbit_info, event_start, event_end=None): return returned_fitbit_data + +def apple_health_parser(apple_health_info, event_start, event_end=None): + if not event_end: + event_end = event_start + apple_health_data = requests.get(apple_health_info["download_url"]).text + apple_health_data = apple_health_data.split("\n") + start_date = arrow.get(event_start) + end_date = arrow.get(event_end) + period_start = start_date.shift(weeks=WEEKS_BEFORE_SICK * -1) + period_end = end_date.shift(weeks=WEEKS_AFTER_SICK) + + returned_apple_data = [] + + for entry in apple_health_data: + if entry.endswith("R"): + entry = entry.split(",") + sdate = arrow.get(entry[1]) + if sdate >= period_start and sdate <= period_end: + returned_apple_data.append( + {"timestamp": entry[1], "data": {"heart_rate": entry[0]},} + ) + + return returned_apple_data + + def googlefit_to_qf(json_data, min_date, max_date): res = [] data = json_data @@ -96,7 +121,7 @@ def googlefit_to_qf(json_data, min_date, max_date): for col_name in df.columns: # the col_name should be of the format heart_rate.bpm.INT # named this way by the parsing functionality in get_dataframe_with_all - if 'heart_rate' not in col_name: + if "heart_rate" not in col_name: continue data_points = len(df[col_name].dropna()) if data_points > max_data_points: @@ -112,7 +137,7 @@ def googlefit_to_qf(json_data, min_date, max_date): ts = ts.to_pydatetime() if ts < min_date or ts > max_date: continue - rec = {"timestamp": ts.isoformat(), "data":{"heart_rate": value}} + rec = {"timestamp": ts.isoformat(), "data": {"heart_rate": value}} res.append(rec) return res @@ -121,7 +146,9 @@ def googlefit_parser(googlefit_files_info, event_start, event_end=None): print(event_start) if event_end is None: event_end = event_start - event_start = datetime(event_start.year, event_start.month, event_start.day, 0, 0, 0) + event_start = datetime( + event_start.year, event_start.month, event_start.day, 0, 0, 0 + ) event_end = datetime(event_end.year, event_end.month, event_end.day, 23, 59, 59) min_date = event_start - timedelta(days=21) max_date = event_end + timedelta(days=14) @@ -143,7 +170,7 @@ def googlefit_parser(googlefit_files_info, event_start, event_end=None): data_in_qf_format = googlefit_to_qf(googlefit_json, min_date, max_date) if data_in_qf_format: - returned_googlefit_data+=data_in_qf_format + returned_googlefit_data += data_in_qf_format return returned_googlefit_data diff --git a/quantified_flu/helpers.py b/quantified_flu/helpers.py index 271aafd..173b9e3 100644 --- a/quantified_flu/helpers.py +++ b/quantified_flu/helpers.py @@ -9,7 +9,12 @@ def identify_missing_sources(oh_member): - missing_sources = {"oura": True, "fitbit": True, "googlefit": True} + missing_sources = { + "oura": True, + "fitbit": True, + "googlefit": True, + "apple_health": True, + } # Check data already in Open Humans. for i in oh_member.list_files(): @@ -17,8 +22,12 @@ def identify_missing_sources(oh_member): missing_sources.pop("oura", None) if i["basename"] == "fitbit-data.json" and i["source"] == "direct-sharing-102": missing_sources.pop("fitbit", None) - if "googlefit" in i["basename"]: # GoogleFit is stored in monthly files + if "googlefit" in i["basename"]: # GoogleFit is stored in monthly files missing_sources.pop("googlefit", None) + if i["source"] == "direct-sharing-453" and i["basename"].startswith( + "heartrate_samples" + ): + missing_sources.pop("apple_health", None) # Check data imported by this app. if hasattr(oh_member, "fitbit_member"): diff --git a/quantified_flu/templates/quantified_flu/about.html b/quantified_flu/templates/quantified_flu/about.html index f69daa7..a7684f3 100644 --- a/quantified_flu/templates/quantified_flu/about.html +++ b/quantified_flu/templates/quantified_flu/about.html @@ -74,6 +74,7 @@

Which wearables are supported?

Right now we support the following wearable devices: diff --git a/quantified_flu/templates/quantified_flu/home.html b/quantified_flu/templates/quantified_flu/home.html index 20f056d..fd47822 100644 --- a/quantified_flu/templates/quantified_flu/home.html +++ b/quantified_flu/templates/quantified_flu/home.html @@ -12,7 +12,7 @@

Whether the flu or the common cold: Can weara

We have two ways you can contribute…

Report past illness

- Report when you got sick and share your wearable device data. We'll plot your data for you! Currently we support Fitbit & Oura Ring and Google Fit. + Report when you got sick and share your wearable device data. We'll plot your data for you! Currently we support Fitbit & Oura Ring, Apple Watch and Google Fit. Ask us to add support for your wearable, if yours is missing so far!

Track symptoms going forward

@@ -37,7 +37,7 @@

Using your data for good

{% if request.user.is_authenticated %} -{% if missing_sources|length == 3 %} +{% if missing_sources|length == 4 %}

Data Sources

You can connect these data sources to correlate them with reported illness. @@ -53,10 +53,16 @@

Data Sources

Connect Oura account - - Connect GoogleFit account - +
+ + Connect Apple Watch + +
+ + Connect GoogleFit account +
{% endif %} @@ -102,7 +108,7 @@

Schedule check-ins

If at least one data source has been connected, management of data sources is moved to below the "actions" (retrospective analysis & scheduling check-ins). {% endcomment %} -{% if missing_sources|length < 3 %} +{% if missing_sources|length < 4 %}

Data Sources

{% comment %} @@ -129,6 +135,11 @@

Data Sources

  • Connect GoogleFit account
    • {% endif %} + {% if source == 'apple_health' %} + +
  • Connect Apple Watch / Apple Health
    • +     + {% endif %} {% endfor %}

    @@ -169,9 +180,10 @@

    Manage Fitbit

    You can manage this connection at:
    fitbit.openhumans.org

    - {% endif %} + {% endif %} {# if request.user.openhumansmember.fitbit_member #} - {% endif %} + {% endif %} {# if 'fitbit' not in missing_sources #} + {% if 'oura' not in missing_sources %}

    Manage Oura

    @@ -202,10 +214,30 @@

    Manage Oura

    You can manage this connection at:
    oura.openhumans.org

    - {% endif %} - {% endif %} + {% endif %} {# if request.user.openhumansmember.oura_user #} +
    + {% endif %} {# if 'oura' not in missing_sources #} - {% if 'googlefit' not in missing_sources %} + {% if 'apple_health' not in missing_sources %} +
    +

    Manage Apple Watch / Apple Health

    + +

    + Your data files: You've shared Oura data via the + Open Humans importer app. Your data file ("heartrate_samples*.csv") + should be available in your Open Humans account here:
    + + https://www.openhumans.org/activity/oh-data-port-for-apple-health/ + +

    +

    + You can update your data through the + iPhone app. +

    +
    + {% endif %} {# if 'apple_health' not in missing_sources #} + + {% if 'googlefit' not in missing_sources %}

    Manage GoogleFit

    {% if request.user.openhumansmember.googlefit_member %} @@ -235,11 +267,11 @@

    Manage GoogleFit

    You can manage this connection at:
    google-fit.openhumans.org

    - {% endif %} + {% endif %} {# if request.user.openhumansmember.googlefit_member #}
    - {% endif %} + {% endif %} {# if 'googlefit' not in missing_sources #} -{% endif %} {# if missing_sources < 3 #} +{% endif %} {# if missing_sources < 4 #} {% endif %} {# if user.is_authenticated #} {% endblock main %} diff --git a/retrospective/tasks.py b/retrospective/tasks.py index ecadda0..ae7c322 100644 --- a/retrospective/tasks.py +++ b/retrospective/tasks.py @@ -14,7 +14,8 @@ oura_parser, fitbit_parser, fitbit_intraday_parser, - googlefit_parser + googlefit_parser, + apple_health_parser, ) from import_data.celery_fitbit import fetch_fitbit_data from import_data.celery_oura import fetch_oura_data @@ -45,6 +46,7 @@ def get_wearable_data(member_files): fitbit_data = None googlefit_data = [] fitbit_intraday_data = [] + apple_health_data = None # Get info for relevant files. # Take the first match for Fitbit and Oura, where all data is in a single file. @@ -76,11 +78,15 @@ def get_wearable_data(member_files): if source == "direct-sharing-191": fitbit_intraday_data.append(file_info) + if source == "direct-sharing-453" and basename.startswith("heartrate_samples"): + apple_health_data = file_info + return { "oura_data": oura_data, "googlefit_data": googlefit_data, "fitbit_data": fitbit_data, "fitbit_intraday_data": fitbit_intraday_data, + "apple_health_data": apple_health_data, } @@ -90,12 +96,13 @@ def analyze_googlefit_event(googlefit_data, event): googlefit_hr_analysis = RetrospectiveEventAnalysis( event=event, graph_data=json.dumps(googlefit_hr_data), - graph_type="googlefit_heartrate" + graph_type="googlefit_heartrate", ) googlefit_hr_analysis.save() else: print("No GoogleFit data available around event {}".format(event.date)) + @task def analyze_event(event_id): event = RetrospectiveEvent.objects.get(id=event_id) @@ -106,6 +113,7 @@ def analyze_event(event_id): fitbit_data = wearable_data["fitbit_data"] fitbit_intraday_data = wearable_data["fitbit_intraday_data"] googlefit_data = wearable_data["googlefit_data"] + apple_health_data = wearable_data["apple_health_data"] if googlefit_data: analyze_googlefit_event(googlefit_data, event) @@ -130,6 +138,20 @@ def analyze_event(event_id): ) oura_temp_analysis.save() + if apple_health_data: + apple_analyses = event.retrospectiveeventanalysis_set.filter( + graph_type__exact="apple_health_summary" + ) + if not apple_analyses: + apple_hr_data = apple_health_parser(apple_health_data, event.date) + if apple_hr_data: + apple_hr_analysis = RetrospectiveEventAnalysis( + event=event, + graph_data=json.dumps(apple_hr_data), + graph_type="apple_health_summary", + ) + apple_hr_analysis.save() + if fitbit_data: fitbit_analyses = event.retrospectiveeventanalysis_set.filter( graph_type__exact="fitbit_summary" @@ -153,15 +175,20 @@ def analyze_event(event_id): ) if not fitbit_intraday_analyses: - fb_intraday_data = fitbit_intraday_parser( - fitbit_data, fitbit_intraday_data, event.date - ) - new_analysis = RetrospectiveEventAnalysis( - event=event, - graph_data=json.dumps(fb_intraday_data), - graph_type="fitbit_intraday", - ) - new_analysis.save() + try: + fb_intraday_data = fitbit_intraday_parser( + fitbit_data, fitbit_intraday_data, event.date + ) + new_analysis = RetrospectiveEventAnalysis( + event=event, + graph_data=json.dumps(fb_intraday_data), + graph_type="fitbit_intraday", + ) + new_analysis.save() + except: + # The fitbit intraday parsing is very fickle and often breaks + # Blanket except here to allow the pipeline to work regardless. + pass @task @@ -192,7 +219,6 @@ def update_googlefit_data(oh_id, django_user_id): analyze_existing_reports(django_user_id) - def set_symptomwearablereport(oh_member, data_source, start, end, data): values = json.dumps(data) wearable_report, created = SymptomReportPhysiology.objects.get_or_create( @@ -216,6 +242,7 @@ def add_wearable_to_symptom(oh_member_id): fitbit_data = wearable_data["fitbit_data"] fitbit_intraday_data = wearable_data["fitbit_intraday_data"] googlefit_data = wearable_data["googlefit_data"] + apple_health_data = wearable_data["apple_health_data"] symptoms_start = ( SymptomReport.objects.filter(member=oh_member).earliest("created").created @@ -233,7 +260,20 @@ def add_wearable_to_symptom(oh_member_id): data_source="googlefit_heartrate", start=symptoms_start, end=symptoms_end, - data=googlefit_hr_data) + data=googlefit_hr_data, + ) + + if apple_health_data: + apple_hr_data = apple_health_parser( + apple_health_data, symptoms_start, symptoms_end + ) + set_symptomwearablereport( + oh_member=oh_member, + data_source="apple_health_summary", + start=symptoms_start, + end=symptoms_end, + data=apple_hr_data, + ) if oura_data: oura_hr_data, oura_temp_data = oura_parser( diff --git a/retrospective/templates/retrospective/event.html b/retrospective/templates/retrospective/event.html index dfbdef6..3b2e7dc 100644 --- a/retrospective/templates/retrospective/event.html +++ b/retrospective/templates/retrospective/event.html @@ -57,7 +57,7 @@

    Sickness event: {{ object.date }}

    {# Explanations if analyses aren't available. #} {% if not object.retrospectiveeventanalysis_set.all %} - {% if request.user.openhumansmember|missing_sources|length == 3 %} + {% if request.user.openhumansmember|missing_sources|length == 4 %}