blob_db: shrink growable settings files on existing devices [FIRM-1893]

include/pbl/services/blob_db/api.h

@@ -110,6 +110,12 @@ typedef BlobDBDirtyItem *(*BlobDBGetDirtyListImpl)(void);
 //! \returns S_SUCCESS if the item was marked synced, an error code otherwise
 typedef status_t (*BlobDBMarkSyncedImpl)(const uint8_t *key, int key_len);
 
+//! Implements the Compact API. Reclaims unused space in the underlying
+//! settings file. Note that this function should be blocking; only blob DBs
+//! backed by a settings_file need to implement this.
+//! \returns S_SUCCESS on success, an error code otherwise
+typedef status_t (*BlobDBCompactImpl)(void);
+
 //! Emits a Blob DB event.
 //! \param type The type of event to emit
 //! \param db_id the ID of the blob DB
@@ -120,6 +126,12 @@ void blob_db_event_put(BlobDBEventType type, BlobDBId db_id, const uint8_t *key,
 //! Call the BlobDBInitImpl for all the databases
 void blob_db_init_dbs(void);
 
+//! Call the BlobDBCompactImpl for every database that implements one. Used to
+//! reclaim space in growable settings_file-backed databases. Must be called
+//! after blob_db_init_dbs(). Safe to call from a system task callback; do not
+//! call from the kernel main loop as compaction performs disk I/O.
+void blob_db_compact_growable_dbs(void);
+
 //! Call the BlobDBIsDirtyImpl for each database, and fill the 'ids' list
 //! with all the dirty DB ids
 //! \param[out] ids an array of BlobDbIds of size NumBlobDBs or more.

include/pbl/services/blob_db/app_db.h

@@ -59,5 +59,7 @@ status_t app_db_delete(const uint8_t *key, int key_len);
 
 status_t app_db_flush(void);
 
+status_t app_db_compact(void);
+
 /* TEST */
 AppInstallId app_db_check_next_unique_id(void);

include/pbl/services/blob_db/app_glance_db.h

@@ -29,6 +29,11 @@ void app_glance_db_init(void);
 
 status_t app_glance_db_flush(void);
 
+//! Compact and shrink the on-disk settings file. Forces growable files that
+//! grew before the growable change landed (or under heavy load) to drop back
+//! toward the initial allocation.
+status_t app_glance_db_compact(void);
+
 status_t app_glance_db_insert(const uint8_t *key, int key_len, const uint8_t *val, int val_len);
 
 int app_glance_db_get_len(const uint8_t *key, int key_len);

include/pbl/services/blob_db/contacts_db.h

@@ -42,3 +42,5 @@ status_t contacts_db_read(const uint8_t *key, int key_len, uint8_t *val_out, int
 status_t contacts_db_delete(const uint8_t *key, int key_len);
 
 status_t contacts_db_flush(void);
+
+status_t contacts_db_compact(void);

include/pbl/services/blob_db/health_db.h

@@ -45,3 +45,5 @@ status_t health_db_read(const uint8_t *key, int key_len, uint8_t *val_out, int v
 status_t health_db_delete(const uint8_t *key, int key_len);
 
 status_t health_db_flush(void);
+
+status_t health_db_compact(void);

include/pbl/services/blob_db/ios_notif_pref_db.h

@@ -57,6 +57,8 @@ status_t ios_notif_pref_db_delete(const uint8_t *key, int key_len);
 
 status_t ios_notif_pref_db_flush(void);
 
+status_t ios_notif_pref_db_compact(void);
+
 status_t ios_notif_pref_db_is_dirty(bool *is_dirty_out);
 
 BlobDBDirtyItem* ios_notif_pref_db_get_dirty_list(void);

include/pbl/services/blob_db/pin_db.h

@@ -58,6 +58,8 @@ status_t pin_db_delete(const uint8_t *key, int key_len);
 
 status_t pin_db_flush(void);
 
+status_t pin_db_compact(void);
+
 status_t pin_db_is_dirty(bool *is_dirty_out);
 
 BlobDBDirtyItem* pin_db_get_dirty_list(void);

include/pbl/services/blob_db/reminder_db.h

@@ -73,6 +73,8 @@ status_t reminder_db_delete(const uint8_t *key, int key_len);
 
 status_t reminder_db_flush(void);
 
+status_t reminder_db_compact(void);
+
 status_t reminder_db_is_dirty(bool *is_dirty_out);
 
 BlobDBDirtyItem* reminder_db_get_dirty_list(void);

include/pbl/services/blob_db/timeline_item_storage.h

@@ -31,6 +31,9 @@ void timeline_item_storage_init(TimelineItemStorage *storage,
 
 void timeline_item_storage_deinit(TimelineItemStorage *storage);
 
+//! Compact and shrink the underlying settings file.
+status_t timeline_item_storage_compact(TimelineItemStorage *storage);
+
 bool timeline_item_storage_exists_with_parent(TimelineItemStorage *storage, const Uuid *parent_id);
 
 status_t timeline_item_storage_flush(TimelineItemStorage *storage);

include/pbl/services/blob_db/watch_app_prefs_db.h

@@ -43,3 +43,5 @@ status_t watch_app_prefs_db_read(const uint8_t *key, int key_len, uint8_t *val_o
 status_t watch_app_prefs_db_delete(const uint8_t *key, int key_len);
 
 status_t watch_app_prefs_db_flush(void);
+
+status_t watch_app_prefs_db_compact(void);

include/pbl/services/blob_db/weather_db.h

@@ -57,6 +57,8 @@ void weather_db_init(void);
 
 status_t weather_db_flush(void);
 
+status_t weather_db_compact(void);
+
 status_t weather_db_insert(const uint8_t *key, int key_len, const uint8_t *val, int val_len);
 
 int weather_db_get_len(const uint8_t *key, int key_len);

include/pbl/services/settings/settings_file.h

@@ -196,3 +196,8 @@ typedef bool (*SettingsFileRewriteFilterCallback)(void *key, size_t key_len, voi
 //! settings_file_rewrite if all you are doing is excluding specific records from the old file.
 status_t settings_file_rewrite_filtered(SettingsFile *file,
                                         SettingsFileRewriteFilterCallback filter_cb, void *context);
+
+//! Compact the file: rewrite all live records and, for growable files, drop
+//! alloc_used_space toward min_alloc_used_space. Useful for shrinking growable
+//! settings files that grew under load and have since had records removed.
+status_t settings_file_compact(SettingsFile *file);

src/fw/apps/system/settings/system.c

@@ -39,6 +39,8 @@
 #include "system/version.h"
 
 #include "pbl/services/activity/activity.h"
+#include "pbl/services/blob_db/api.h"
+#include "system/logging.h"
 
 #include <stdio.h>
 #include <string.h>
@@ -71,6 +73,7 @@ enum {
 #endif
   DebuggingItemAccelShakeLogInfo,
   DebuggingItemVibeLogInfo,
+  DebuggingItemCompactSettingsDbs,
   DebuggingItem_Count,
 };
 
@@ -549,6 +552,17 @@ static void prv_power_mode_menu_push(SettingsSystemData *data) {
       true /* icons_enabled */, s_power_mode_labels, data);
 }
 
+// Compact growable settings DBs
+////////////////////////////////
+
+static void prv_compact_settings_dbs_task_cb(void *data) {
+  blob_db_compact_growable_dbs();
+}
+
+static void prv_compact_settings_dbs(void) {
+  system_task_add_callback(prv_compact_settings_dbs_task_cb, NULL);
+}
+
 // Debug options window
 ///////////////////////
 
@@ -565,6 +579,7 @@ static const char* s_debugging_titles[DebuggingItem_Count] = {
 #endif
   [DebuggingItemAccelShakeLogInfo] = i18n_noop("Shake Log Info"),
   [DebuggingItemVibeLogInfo] = i18n_noop("Vibe Log Info"),
+  [DebuggingItemCompactSettingsDbs] = i18n_noop("Compact Settings DBs"),
 };
 
 static void prv_debugging_draw_row_callback(GContext* ctx, const Layer *cell_layer,
@@ -666,6 +681,9 @@ static void prv_debugging_select_callback(MenuLayer *menu_layer,
       shell_prefs_set_vibe_log_info_enabled(
           !shell_prefs_get_vibe_log_info_enabled());
       break;
+    case DebuggingItemCompactSettingsDbs:
+      prv_compact_settings_dbs();
+      break;
     default:
       WTF;
   }

src/fw/services/blob_db/api.c

@@ -23,6 +23,8 @@
 #include "kernel/pbl_malloc.h"
 #include "system/logging.h"
 
+#include <inttypes.h>
+
 typedef struct {
   BlobDBInitImpl init;
   BlobDBInsertImpl insert;
@@ -33,6 +35,8 @@ typedef struct {
   BlobDBIsDirtyImpl is_dirty;
   BlobDBGetDirtyListImpl get_dirty_list;
   BlobDBMarkSyncedImpl mark_synced;
+  BlobDBCompactImpl compact;
+  const char *name;
   bool disabled;
 } BlobDB;
 
@@ -47,6 +51,8 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .is_dirty = pin_db_is_dirty,
     .get_dirty_list = pin_db_get_dirty_list,
     .mark_synced = pin_db_mark_synced,
+    .compact = pin_db_compact,
+    .name = "pin_db",
   },
   [BlobDBIdApps] = {
     .init = app_db_init,
@@ -55,6 +61,8 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .read = app_db_read,
     .del = app_db_delete,
     .flush = app_db_flush,
+    .compact = app_db_compact,
+    .name = "app_db",
   },
   [BlobDBIdReminders] = {
     .init = reminder_db_init,
@@ -66,6 +74,8 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .is_dirty = reminder_db_is_dirty,
     .get_dirty_list = reminder_db_get_dirty_list,
     .mark_synced = reminder_db_mark_synced,
+    .compact = reminder_db_compact,
+    .name = "reminder_db",
   },
   [BlobDBIdNotifs] = {
     .init = notif_db_init,
@@ -74,6 +84,7 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .read = notif_db_read,
     .del = notif_db_delete,
     .flush = notif_db_flush,
+    .name = "notif_db",
   },
   [BlobDBIdWeather] = {
     .init = weather_db_init,
@@ -82,6 +93,8 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .read = weather_db_read,
     .del = weather_db_delete,
     .flush = weather_db_flush,
+    .compact = weather_db_compact,
+    .name = "weather_db",
   },
   [BlobDBIdiOSNotifPref] = {
     .init = ios_notif_pref_db_init,
@@ -93,6 +106,8 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .is_dirty = ios_notif_pref_db_is_dirty,
     .get_dirty_list = ios_notif_pref_db_get_dirty_list,
     .mark_synced = ios_notif_pref_db_mark_synced,
+    .compact = ios_notif_pref_db_compact,
+    .name = "ios_notif_pref_db",
   },
   [BlobDBIdPrefs] = {
     .init = prefs_db_init,
@@ -101,6 +116,7 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .read = prefs_db_read,
     .del = prefs_db_delete,
     .flush = prefs_db_flush,
+    .name = "prefs_db",
   },
   [BlobDBIdContacts] = {
     .init = contacts_db_init,
@@ -109,6 +125,8 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .read = contacts_db_read,
     .del = contacts_db_delete,
     .flush = contacts_db_flush,
+    .compact = contacts_db_compact,
+    .name = "contacts_db",
   },
   [BlobDBIdWatchAppPrefs] = {
     .init = watch_app_prefs_db_init,
@@ -117,6 +135,8 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .read = watch_app_prefs_db_read,
     .del = watch_app_prefs_db_delete,
     .flush = watch_app_prefs_db_flush,
+    .compact = watch_app_prefs_db_compact,
+    .name = "watch_app_prefs_db",
   },
   [BlobDBIdHealth] = {
     .init = health_db_init,
@@ -125,6 +145,8 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .read = health_db_read,
     .del = health_db_delete,
     .flush = health_db_flush,
+    .compact = health_db_compact,
+    .name = "health_db",
   },
   [BlobDBIdAppGlance] = {
     .init = app_glance_db_init,
@@ -133,6 +155,8 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .read = app_glance_db_read,
     .del = app_glance_db_delete,
     .flush = app_glance_db_flush,
+    .compact = app_glance_db_compact,
+    .name = "app_glance_db",
   },
   [BlobDBIdSettings] = {
     .init = settings_blob_db_init,
@@ -144,6 +168,7 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = {
     .is_dirty = settings_blob_db_is_dirty,
     .get_dirty_list = settings_blob_db_get_dirty_list,
     .mark_synced = settings_blob_db_mark_synced,
+    .name = "settings_blob_db",
   },
 };
 
@@ -181,6 +206,20 @@ void blob_db_init_dbs(void) {
   }
 }
 
+void blob_db_compact_growable_dbs(void) {
+  PBL_LOG_INFO("blob_db_compact_growable_dbs: start");
+  const BlobDB *db = s_blob_dbs;
+  for (int i = 0; i < NumBlobDBs; ++i, ++db) {
+    if (!db->compact) {
+      continue;
+    }
+    const status_t rv = db->compact();
+    PBL_LOG_INFO("blob_db_compact_growable_dbs: %s -> %"PRIi32,
+                 db->name ? db->name : "?", rv);
+  }
+  PBL_LOG_INFO("blob_db_compact_growable_dbs: done");
+}
+
 void blob_db_get_dirty_dbs(uint8_t *ids, uint8_t *num_ids) {
   const BlobDB *db = s_blob_dbs;
   *num_ids = 0;

src/fw/services/blob_db/app_db.c

@@ -402,6 +402,16 @@ status_t app_db_flush(void) {
   return S_SUCCESS;
 }
 
+status_t app_db_compact(void) {
+  status_t rv = prv_lock_mutex_and_open_file();
+  if (rv != S_SUCCESS) {
+    return rv;
+  }
+  rv = settings_file_compact(&s_app_db.settings_file);
+  prv_close_file_and_unlock_mutex();
+  return rv;
+}
+
 //////////////////////
 // Test functions
 //////////////////////

src/fw/services/blob_db/app_glance_db.c

@@ -632,6 +632,16 @@ status_t app_glance_db_flush(void) {
   return S_SUCCESS;
 }
 
+status_t app_glance_db_compact(void) {
+  status_t rv = prv_lock_mutex_and_open_file();
+  if (rv != S_SUCCESS) {
+    return rv;
+  }
+  rv = settings_file_compact(&s_app_glance_db.settings_file);
+  prv_close_file_and_unlock_mutex();
+  return rv;
+}
+
 static status_t prv_validate_glance(const Uuid *app_uuid,
                              const SerializedAppGlanceHeader *serialized_glance, size_t *len) {
   // Change this block if we support multiple app glance versions in the future

src/fw/services/blob_db/contacts_db.c

@@ -162,3 +162,13 @@ status_t contacts_db_flush(void) {
   mutex_unlock(s_contacts_db.mutex);
   return rv;
 }
+
+status_t contacts_db_compact(void) {
+  status_t rv = prv_lock_mutex_and_open_file();
+  if (rv != S_SUCCESS) {
+    return rv;
+  }
+  rv = settings_file_compact(&s_contacts_db.settings_file);
+  prv_close_file_and_unlock_mutex();
+  return rv;
+}

src/fw/services/blob_db/health_db.c

@@ -423,3 +423,14 @@ status_t health_db_flush(void) {
   return rv;
 }
 
+status_t health_db_compact(void) {
+  SettingsFile file;
+  status_t rv = prv_file_open_and_lock(&file);
+  if (rv != S_SUCCESS) {
+    return rv;
+  }
+  rv = settings_file_compact(&file);
+  prv_file_close_and_unlock(&file);
+  return rv;
+}
+

src/fw/services/blob_db/ios_notif_pref_db.c

@@ -297,6 +297,17 @@ status_t ios_notif_pref_db_flush(void) {
   return rv;
 }
 
+status_t ios_notif_pref_db_compact(void) {
+  SettingsFile file;
+  status_t rv = prv_file_open_and_lock(&file);
+  if (rv != S_SUCCESS) {
+    return rv;
+  }
+  rv = settings_file_compact(&file);
+  prv_file_close_and_unlock(&file);
+  return rv;
+}
+
 status_t ios_notif_pref_db_is_dirty(bool *is_dirty_out) {
   SettingsFile file;
   status_t rv = prv_file_open_and_lock(&file);