diff --git a/Cargo.lock b/Cargo.lock index 7664f492b..6ec093588 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,6 +117,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "aegis" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41585f248ed9406567ae6d61ca2a55ed3dc7348e65203a01229881253acb95e3" +dependencies = [ + "cc", + "softaes", +] + [[package]] name = "aes" version = "0.8.4" @@ -173,7 +183,7 @@ dependencies = [ "futures", "futures-concurrency", "rustc-hash 2.1.3", - "rustix", + "rustix 1.1.4", "schemars 1.2.2", "serde", "serde_json", @@ -205,7 +215,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "strum", + "strum 0.28.0", "tracing", ] @@ -264,6 +274,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "allocator-api2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880" + [[package]] name = "android-activity" version = "0.6.1" @@ -329,7 +345,7 @@ checksum = "f211a51805bc641f3ad5b7664c77d2547af685cc33b4cd8d31964027a46f13f1" dependencies = [ "anstyle", "memchr", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -382,6 +398,22 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "antithesis_sdk" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8ed6751597568a284314fd9e2db420665898600109dedc7d8d9b588f00ae9c4" +dependencies = [ + "libc", + "libloading", + "linkme", + "once_cell", + "rand 0.8.6", + "rustc_version_runtime", + "serde", + "serde_json", +] + [[package]] name = "anyhow" version = "1.0.103" @@ -417,6 +449,26 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "aristo" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcdeefa800110050103e2459a2f8dbdbd560ad046e30f1f87e24ce19c2cb8bde" +dependencies = [ + "aristo-macros", +] + +[[package]] +name = "aristo-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64a66d21a80182b35b1741997a6d2456911f54b7eb1918aa4e2382fc205268d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "arraydeque" version = "0.5.1" @@ -516,6 +568,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "assoc" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdc70193dadb9d7287fa4b633f15f90c876915b31f6af17da307fc59c9859a8" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -590,7 +648,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix", + "rustix 1.1.4", "slab", "windows-sys 0.61.2", ] @@ -632,7 +690,7 @@ dependencies = [ "cfg-if", "event-listener", "futures-lite", - "rustix", + "rustix 1.1.4", ] [[package]] @@ -658,7 +716,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix", + "rustix 1.1.4", "signal-hook-registry", "slab", "windows-sys 0.61.2", @@ -956,6 +1014,19 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -965,6 +1036,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.11.0", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 2.0.118", + "which 4.4.2", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -1045,6 +1139,18 @@ dependencies = [ "no_std_io2", ] +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake3" version = "1.8.7" @@ -1132,6 +1238,15 @@ dependencies = [ "cfg_aliases 0.2.2", ] +[[package]] +name = "branches" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7436b21fd6195415058eb41c9090e156b556bc7e0377bb57c5c026a421521525" +dependencies = [ + "rustc_version", +] + [[package]] name = "bs58" version = "0.5.1" @@ -1165,9 +1280,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" dependencies = [ "bytemuck_derive", ] @@ -1243,7 +1358,7 @@ checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" dependencies = [ "bitflags 2.13.2", "polling", - "rustix", + "rustix 1.1.4", "slab", "tracing", ] @@ -1255,7 +1370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" dependencies = [ "calloop", - "rustix", + "rustix 1.1.4", "wayland-backend", "wayland-client", ] @@ -1295,9 +1410,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.66" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "jobserver", @@ -1348,6 +1463,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "cfg_block" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18758054972164c3264f7c8386f5fc6da6114cb46b619fd365d4e3b2dc3ae487" + [[package]] name = "cgl" version = "0.3.2" @@ -1563,7 +1684,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" dependencies = [ "serde", "termcolor", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -1935,6 +2056,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2035,6 +2165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -2826,6 +2957,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fancy-regex" version = "0.16.2" @@ -2837,6 +2974,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fastbloom" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" +dependencies = [ + "getrandom 0.3.4", + "libm", + "siphasher 1.0.3", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -2900,9 +3048,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "fixedbitset" @@ -3075,6 +3223,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futf" version = "0.1.5" @@ -3306,6 +3460,21 @@ dependencies = [ "x11", ] +[[package]] +name = "genawaiter" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" +dependencies = [ + "genawaiter-macro", +] + +[[package]] +name = "genawaiter-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" + [[package]] name = "generator" version = "0.8.10" @@ -3337,7 +3506,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix", + "rustix 1.1.4", "windows-link 0.2.1", ] @@ -3627,7 +3796,7 @@ dependencies = [ "async-channel", "async-task", "backtrace", - "bindgen", + "bindgen 0.72.1", "bitflags 2.13.2", "chrono", "collections", @@ -3670,7 +3839,7 @@ dependencies = [ "slotmap", "smallvec", "spin 0.10.0", - "strum", + "strum 0.28.0", "sum_tree", "taffy", "thiserror 2.0.19", @@ -3959,7 +4128,7 @@ dependencies = [ "raw-window-handle", "smallvec", "smol", - "strum", + "strum 0.28.0", "url", "uuid", "wayland-backend", @@ -4017,7 +4186,7 @@ dependencies = [ "raw-window-handle", "semver", "smallvec", - "strum", + "strum 0.28.0", "uuid", "zed-font-kit 0.14.1-zed (git+https://github.com/zed-industries/font-kit?rev=94b0f28166665e8fd2f53ff6d268a14955c82269)", ] @@ -4063,7 +4232,7 @@ source = "git+https://github.com/longbridge/gpui-fast#1b381adb68a6e5a4550f73b0cd dependencies = [ "anyhow", "log", - "which", + "which 8.0.6", ] [[package]] @@ -4290,7 +4459,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "allocator-api2", + "allocator-api2 0.2.21", "equivalent", "foldhash 0.2.0", ] @@ -4301,7 +4470,7 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "allocator-api2", + "allocator-api2 0.2.21", "equivalent", "foldhash 0.2.0", ] @@ -4373,6 +4542,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "html5ever" version = "0.27.0" @@ -4571,11 +4749,36 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collator" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08984ed58ac439ebf3e13d2cf26b0c46a60afcd21721c1d14087c0b240344dda" +dependencies = [ + "icu_collator_data", + "icu_collections", + "icu_locale_core", + "icu_locale_fallback", + "icu_normalizer", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_collator_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d7e54efdddeb1208c08dd5d32b53a879ac00d2d3d051b2255fd26821d16368" + [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -4585,45 +4788,91 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_locale" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "785f595c61ef57169a467eeed7b4b6936a66f6cacbb116a96ee86da983251bf4" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_locale_data", + "icu_locale_fallback", + "icu_provider", + "tinystr", + "zerovec", +] + [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d37d91460e362a5cf58907cd7ce871411775ee6294e49a5cc8e6cec16dfa75ea" + +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", "icu_properties", "icu_provider", "smallvec", + "utf16_iter", + "utf8_iter", + "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -4634,18 +4883,20 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -4843,6 +5094,15 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "intrusive-collections" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189d0897e4cbe8c75efedf3502c18c887b05046e59d28404d4d8e46cbc4d1e86" +dependencies = [ + "memoffset", +] + [[package]] name = "inventory" version = "0.3.24" @@ -4864,6 +5124,17 @@ dependencies = [ "leaky-cow", ] +[[package]] +name = "io-uring" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "libc", +] + [[package]] name = "ipconfig" version = "0.3.4" @@ -4926,7 +5197,7 @@ dependencies = [ "rustls-pki-types", "serde", "smallvec", - "strum", + "strum 0.28.0", "time", "tokio", "tokio-stream", @@ -4972,7 +5243,7 @@ dependencies = [ "rand 0.10.2", "rustls", "simple-dns", - "strum", + "strum 0.28.0", "tokio", "tracing", "url", @@ -5055,7 +5326,7 @@ dependencies = [ "serde_json", "sha1", "simdutf8", - "strum", + "strum 0.28.0", "time", "tokio", "tokio-rustls", @@ -5329,6 +5600,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lb-wry" version = "0.53.3" @@ -5455,12 +5732,38 @@ version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "linktime-proc-macro" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -5779,7 +6082,7 @@ version = "0.1.0" source = "git+https://github.com/longbridge/gpui-fast#1b381adb68a6e5a4550f73b0cd29a231ce6c7281" dependencies = [ "anyhow", - "bindgen", + "bindgen 0.72.1", "core-foundation 0.10.1", "core-video", "foreign-types", @@ -5843,6 +6146,28 @@ dependencies = [ "paste", ] +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "mime" version = "0.3.17" @@ -7120,6 +7445,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "pack1" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b7bb0ecf2e447b1f20ee94ee79ef6eed1e9d4b3c36ce1903b9dea3bf205523" +dependencies = [ + "bytemuck", +] + [[package]] name = "page_size" version = "0.6.0" @@ -7615,7 +7955,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -7767,6 +8107,8 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -7939,15 +8281,38 @@ dependencies = [ ] [[package]] -name = "pulp" -version = "0.22.3" +name = "prost" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ - "bytemuck", - "cfg-if", - "libm", - "num-complex", + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", "paste", "pulp-wasm-simd-flag", "raw-cpuid", @@ -8098,6 +8463,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -8225,6 +8596,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_pcg" version = "0.10.2" @@ -8255,6 +8635,15 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + [[package]] name = "rav1e" version = "0.8.1" @@ -8700,6 +9089,16 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + [[package]] name = "ropey" version = "2.0.0-beta.1" @@ -8847,6 +9246,16 @@ dependencies = [ "semver", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -8856,6 +9265,19 @@ dependencies = [ "nom 7.1.3", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -8865,7 +9287,7 @@ dependencies = [ "bitflags 2.13.2", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -9505,6 +9927,26 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "shuttle" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab17edba38d63047f46780cf7360acf7467fec2c048928689a5c1dd1c2b4e31" +dependencies = [ + "assoc", + "bitvec", + "cfg-if", + "generator", + "hex", + "owo-colors", + "rand 0.8.6", + "rand_core 0.6.4", + "rand_pcg 0.3.1", + "scoped-tls", + "smallvec", + "tracing", +] + [[package]] name = "signal-hook" version = "0.4.4" @@ -9593,6 +10035,15 @@ dependencies = [ "log", ] +[[package]] +name = "simsimd" +version = "6.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fb3bc3cdce07a7d7d4caa4c54f8aa967f6be41690482b54b24100a2253fa70" +dependencies = [ + "cc", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -9684,6 +10135,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "softaes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4fdfe13e1c42e45f497416afb8dc4e783bf0e25b7be9077c1c77c5fb14cb998" + [[package]] name = "sorted-index-buffer" version = "0.2.1" @@ -9835,13 +10292,35 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", +] + [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros", + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.118", ] [[package]] @@ -9979,6 +10458,12 @@ dependencies = [ "zeno", ] +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -10153,6 +10638,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.46" @@ -10329,6 +10820,7 @@ dependencies = [ "chrono", "dirs 7.0.0", "flate2", + "futures-lite", "image", "log", "serde", @@ -10337,6 +10829,7 @@ dependencies = [ "tar", "tcode-core", "tcode-protocol", + "turso", "ureq", "uuid", "zip", @@ -10461,7 +10954,7 @@ dependencies = [ "tcode-voice", "tempfile", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", "url", "web-time", "webview2-com", @@ -10517,7 +11010,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -10683,11 +11176,12 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -11027,6 +11521,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.19", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.31" @@ -11119,6 +11626,209 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "turso" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8cbeace6385bf3fcb346d4aace8d9ba0de5cffe5e5a0ed210beebd671c208" +dependencies = [ + "thiserror 2.0.19", + "tracing", + "tracing-subscriber", + "turso_core", + "turso_sdk_kit", + "turso_sync_sdk_kit", +] + +[[package]] +name = "turso_core" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e5f09aaead5dfc87c5a22a2b3e95fb85e2d666e27e5f4a2474c42198af20ec" +dependencies = [ + "aegis", + "aes", + "aes-gcm", + "allocator-api2 0.4.0", + "antithesis_sdk", + "arc-swap", + "aristo", + "bigdecimal", + "bitflags 2.13.2", + "branches", + "bumpalo", + "bytemuck", + "cfg_aliases 0.2.2", + "cfg_block", + "chrono", + "crc32c", + "crossbeam-epoch", + "crossbeam-utils", + "either", + "fallible-iterator", + "fastbloom", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "icu_collator", + "icu_locale", + "intrusive-collections", + "io-uring", + "libc", + "libloading", + "libm", + "loom", + "miette", + "num-bigint", + "num-traits", + "pack1", + "parking_lot", + "pastey 0.2.3", + "polling", + "rand 0.9.4", + "rapidhash", + "regex", + "regex-syntax", + "roaring", + "rustc-hash 2.1.3", + "rustix 1.1.4", + "ryu", + "serde_json", + "shuttle", + "simdutf8", + "simsimd", + "smallvec", + "strum 0.26.3", + "strum_macros 0.26.4", + "tempfile", + "thiserror 2.0.19", + "tracing", + "tracing-subscriber", + "turso_ext", + "turso_macros", + "turso_parser", + "twox-hash", + "uncased", + "uuid", + "windows-sys 0.61.2", +] + +[[package]] +name = "turso_ext" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3902472c63d19191d5982d28542595d1babef787a8482175422ecf7f28100261" +dependencies = [ + "chrono", + "getrandom 0.4.3", + "turso_macros", +] + +[[package]] +name = "turso_macros" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63bd5f0f7186707a8b4cf54bbaddff9056c218823befab15508ef6e7502c2340" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "turso_parser" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b6d3428f16fff69a13bf7f966628d9fc8e0d097fec6cd84ce1af3d5a9be6b1" +dependencies = [ + "bitflags 2.13.2", + "memchr", + "miette", + "strum 0.26.3", + "strum_macros 0.26.4", + "thiserror 2.0.19", + "turso_macros", +] + +[[package]] +name = "turso_sdk_kit" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f96fb1bf669ebd868caf17d865cfa7dc94ad5b2e840402bb28525be09767fc" +dependencies = [ + "bindgen 0.69.5", + "env_logger", + "parking_lot", + "tracing", + "tracing-appender", + "tracing-subscriber", + "turso_core", + "turso_ext", + "turso_sdk_kit_macros", +] + +[[package]] +name = "turso_sdk_kit_macros" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "592f57f212b0ce58c876ab760ef84b219719ee2ade738b53f180db87c0232432" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "turso_sync_engine" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ef57fe72ac7905ad0cfe1cb3f1f24ee1d671176c57a5151709ba9bbf4e1f050" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "genawaiter", + "http", + "libc", + "prost", + "roaring", + "serde", + "serde_json", + "thiserror 2.0.19", + "tracing", + "turso_core", + "turso_parser", + "uuid", +] + +[[package]] +name = "turso_sync_sdk_kit" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b83c90ebfc04dc00d1c98cf030510b0d132ec819c355b9ed64d6b767f13db686" +dependencies = [ + "bindgen 0.69.5", + "env_logger", + "genawaiter", + "parking_lot", + "tracing", + "tracing-appender", + "tracing-subscriber", + "turso_core", + "turso_sdk_kit", + "turso_sdk_kit_macros", + "turso_sync_engine", +] + +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" +dependencies = [ + "rand 0.10.2", +] + [[package]] name = "typeid" version = "1.0.3" @@ -11178,6 +11888,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.9.0" @@ -11244,6 +11963,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1d386ff53b415b7fe27b50bb44679e2cc4660272694b7b6f3326d8480823a94" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.2" @@ -11334,6 +12059,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -11610,7 +12341,7 @@ dependencies = [ "cc", "downcast-rs", "log", - "rustix", + "rustix 1.1.4", "scoped-tls", "smallvec", "wayland-sys", @@ -11623,7 +12354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" dependencies = [ "bitflags 2.13.2", - "rustix", + "rustix 1.1.4", "wayland-backend", "wayland-scanner", ] @@ -11634,7 +12365,7 @@ version = "0.31.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a52d18780be9b1314328a3de5f930b73d2200112e3849ca6cb11822793fb34d" dependencies = [ - "rustix", + "rustix 1.1.4", "wayland-client", "xcursor", ] @@ -12011,6 +12742,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "which" version = "8.0.6" @@ -12805,11 +13548,17 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "ws_stream_wasm" @@ -12830,6 +13579,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x11" version = "2.21.0" @@ -12870,7 +13628,7 @@ dependencies = [ "as-raw-xcb-connection", "gethostname", "libc", - "rustix", + "rustix 1.1.4", "x11rb-protocol", "xcursor", ] @@ -12906,7 +13664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix", + "rustix 1.1.4", ] [[package]] @@ -13071,7 +13829,7 @@ dependencies = [ "hex", "libc", "ordered-stream", - "rustix", + "rustix 1.1.4", "serde", "serde_repr", "tracing", @@ -13348,21 +14106,23 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -13370,13 +14130,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.5", ] [[package]] diff --git a/crates/app/src/main.rs b/crates/app/src/main.rs index 83dcd71b8..f9782394f 100644 --- a/crates/app/src/main.rs +++ b/crates/app/src/main.rs @@ -2,7 +2,7 @@ // Debug builds keep the console so `RUST_LOG` output stays visible. #![cfg_attr(all(windows, not(debug_assertions)), windows_subsystem = "windows")] -use std::{borrow::Cow, rc::Rc, time::Duration}; +use std::{borrow::Cow, cell::RefCell, rc::Rc, time::Duration}; use gpui::{ App, BorrowAppContext as _, Entity, ParentElement as _, Styled as _, TitlebarOptions, @@ -20,6 +20,7 @@ use tcode_ui::{assets, settings}; use tcode_ui::overlay::{DialogActions, OverlayExt as _}; use tcode_ui::widgets::button::{Button, ButtonVariants as _}; +mod migration; #[cfg(not(target_os = "linux"))] mod preview_smoke; @@ -45,6 +46,10 @@ fn main_window_background() -> WindowBackgroundAppearance { } } +/// Background redraws of an inactive window are capped to this interval +/// unless Settings turns the throttle off. +const INACTIVE_FRAME_INTERVAL: Duration = Duration::from_millis(500); + const QUIT_PROMPT_TIMEOUT: Duration = Duration::from_secs(15); fn finish_quit_prompt(window_state: &Entity, epoch: u64, cx: &mut App) -> bool { @@ -183,7 +188,7 @@ fn pair_command(args: &[String], client_host: &NativeClientHost) -> Result (SpawnedHost, HostMux) { +fn start_local(store: SessionStore) -> std::io::Result<(SpawnedHost, HostMux)> { let mut host_services = HostServices { background_startup_probes: true, ai_title_generation: true, @@ -203,9 +208,108 @@ fn start_local(store: SessionStore) -> (SpawnedHost, HostMux) { } Err(error) => log::warn!("MCP host failed to bind: {error}"), } - let host = spawn_host(store, host_services).expect("failed to start tcode host thread"); + let host = spawn_host(store, host_services)?; let mux = HostMux::new(host.to_host.clone(), host.from_host.clone()); - (host, mux) + Ok((host, mux)) +} + +/// The local host did not start — most often because another Tcode already +/// owns the data directory. Say why on stderr and in a dialog, then exit +/// non-zero. +fn exit_with_startup_failure(error: std::io::Error) -> ! { + struct StartupFailure; + impl gpui::Render for StartupFailure { + fn render( + &mut self, + _: &mut gpui::Window, + _: &mut gpui::Context, + ) -> impl gpui::IntoElement { + gpui::div() + } + } + + eprintln!("tcode: {error}"); + let detail = error.to_string(); + gpui_platform::application().run(move |cx| { + // gpui ends a quit with status 0; this observer is the last code that + // runs before it. + cx.on_app_quit(|_| async { std::process::exit(1) }).detach(); + let options = WindowOptions { + window_bounds: Some(WindowBounds::centered(size(px(420.), px(160.)), cx)), + ..Default::default() + }; + let opened = cx.open_window(options, |_, cx| { + gpui::AppContext::new(cx, |_| StartupFailure) + }); + let Ok(window) = opened else { + cx.quit(); + return; + }; + cx.activate(true); + let _ = window.update(cx, |_, window, cx| { + let quit = tcode_ui::tr!("quit.confirm"); + let answer = window.prompt( + gpui::PromptLevel::Critical, + &tcode_ui::tr!("quit.startup_failed"), + Some(&detail), + &[quit.as_ref()], + cx, + ); + cx.spawn(async move |_, cx| { + let _ = answer.await; + cx.update(|cx| cx.quit()); + }) + .detach(); + }); + }); + std::process::exit(1) +} + +/// Every desktop window: the main one and the startup migration's. +fn window_options(cx: &App, inactive_frame_interval: Option) -> WindowOptions { + WindowOptions { + window_bounds: Some(WindowBounds::centered(size(px(1200.), px(800.)), cx)), + // Low enough that the window can actually be dragged into the + // compact layout; the launch geometry above is unchanged. + window_min_size: Some(size(px(360.), px(480.))), + // macOS: seamless titlebar — transparent, with the traffic lights + // nudged down to sit vertically centered in the 52px top strip. + // + // Windows: also client-decorated — a transparent titlebar hides + // the system one — and `tcode_ui`'s window caption cluster draws + // the minimize/maximize/close controls into whichever top strip + // is rightmost (`crates/ui/src/window_caption.rs`). + // + // Linux: we draw no controls of our own there, so a transparent + // titlebar would leave the window with no way to be closed from + // the chrome. Keep the native system titlebar; our top strip + // simply sits below it. + titlebar: Some(TitlebarOptions { + title: None, + appears_transparent: cfg!(any(target_os = "macos", target_os = "windows")), + traffic_light_position: Some(point(px(12.), px(19.))), + }), + // macOS: the app owns titlebar dragging. AppKit's native + // titlebar-region drag sits on top of whatever gpui draws in the + // top strip — dragging the Preview URL bar moved the window + // instead of selecting text. Every draggable strip already goes + // through `window_drag_area` (`start_window_move`), so hand the + // whole content view to the app and let controls own their input. + app_owns_titlebar_drag: true, + // Spell out that Windows is client-decorated. (This field is + // advisory off Wayland; leaving it `None` elsewhere keeps Linux + // on whatever its compositor/backend already chose.) + window_decorations: cfg!(target_os = "windows").then_some(WindowDecorations::Client), + // Persistent windows use the platform's system material: + // macOS sidebar vibrancy, Windows Acrylic, or an opaque fallback. + window_background: main_window_background(), + // Throttle background redraws (spinners, streaming output) to + // ~2 FPS while the window is inactive; gpui lifts the cap the + // moment the window is active or receiving high-rate input. + // This setting is captured at window creation and requires a restart. + inactive_frame_interval, + ..Default::default() + } } struct LocalKernel { @@ -216,18 +320,18 @@ struct LocalKernel { } impl LocalKernel { - fn start(store: SessionStore) -> Self { - let (host, mux) = start_local(store); + fn start(store: SessionStore) -> std::io::Result { + let (host, mux) = start_local(store)?; let connection = mux.attach(); let control_link = HostLink::new(connection.to_host, connection.from_host); let pump_link = control_link.clone(); let control_pump = smol::spawn(async move { pump_link.pump().await }); - Self { + Ok(Self { host, mux, control_link, _control_pump: control_pump, - } + }) } /// A window's link to the local kernel. The mux keeps the kernel alive @@ -354,10 +458,19 @@ fn main() { } None => AttachmentTarget::Local, }; + // An older build's threads move into tcode.db before the kernel starts, + // behind a window of their own. With nothing to migrate, the kernel + // starts before the event loop. + let migration_needed = match store.needs_migration() { + Ok(needed) => needed, + Err(error) => exit_with_startup_failure(error), + }; // Kernel ownership is process composition, not a property of whichever // host the window currently views. - let kernel = Rc::new(LocalKernel::start(store)); - let local_settings = kernel.settings(); + let kernel = (!migration_needed).then(|| match LocalKernel::start(store.clone()) { + Ok(kernel) => Rc::new(kernel), + Err(error) => exit_with_startup_failure(error), + }); gpui_platform::application() .with_assets(assets::Assets) @@ -381,191 +494,182 @@ fn main() { Cow::Owned(tcode_ui::flattened_theme_json()) }; - // Hosting belongs to the process-owned local kernel; it carries no - // current-attachment mode. - cx.set_global(RemoteController::new( - kernel.mux.clone(), - data_dir.clone(), - kernel.control_link.clone(), - local_settings.clone(), - )); - if local_settings.remote_hosting_enabled { - let name = local_settings - .remote_host_name - .clone() - .unwrap_or_else(machine_name); - cx.update_global::(|controller, _| { - if let Err(error) = controller.start_hosting(&local_settings.traverse, name) { - log::error!("remote hosting could not start: {error}"); - } - }); - } + let shell_options = ShellOptions { + title: tcode_ui::tr!("app.name").into(), + fonts: application_fonts, + theme_json, + activate: true, + system_locale: None, + // A desktop window is never suspended by the OS. + lifecycle: None, + ..ShellOptions::default() + }; + tcode_ui::init_client(cx, native_client.as_ref(), &shell_options); #[cfg(target_os = "macos")] cx.set_menus([gpui::Menu::new(tcode_ui::tr!("app.name")).items([ gpui::MenuItem::action(tcode_ui::tr!("quit.menu_item"), Quit), ])]); - // Process ownership, not the window's current attachment, grants - // authority to stop the local kernel on application quit. - let quit_subscription = cx.on_app_quit({ - let link = kernel.control_link.clone(); - let to_host = kernel.host.to_host.clone(); - let stopped = kernel.host.stopped.clone(); - move |_cx| { - let link = link.clone(); - let to_host = to_host.clone(); - let stopped = stopped.clone(); - async move { - let _ = link.shutdown().await; - // `shutdown` only closes this client's mux connection; - // the host loop ends when its own inbox closes. - to_host.close(); - let _ = stopped.recv().await; + let appearance = native_client.load_preferences().appearance; + let shell_slot: Rc>>> = Rc::default(); + cx.on_action::({ + let shell_slot = shell_slot.clone(); + move |action, cx| { + if migration::quit(cx) { + return; + } + let shell = shell_slot.borrow().clone(); + match shell { + Some(shell) => handle_quit(action, &shell, cx), + None => cx.quit(), } } }); - quit_subscription.detach(); - - let window_options = WindowOptions { - window_bounds: Some(WindowBounds::centered(size(px(1200.), px(800.)), cx)), - // Low enough that the window can actually be dragged into the - // compact layout; the launch geometry above is unchanged. - window_min_size: Some(size(px(360.), px(480.))), - // macOS: seamless titlebar — transparent, with the traffic lights - // nudged down to sit vertically centered in the 52px top strip. - // - // Windows: also client-decorated — a transparent titlebar hides - // the system one — and `tcode_ui`'s window caption cluster draws - // the minimize/maximize/close controls into whichever top strip - // is rightmost (`crates/ui/src/window_caption.rs`). - // - // Linux: we draw no controls of our own there, so a transparent - // titlebar would leave the window with no way to be closed from - // the chrome. Keep the native system titlebar; our top strip - // simply sits below it. - titlebar: Some(TitlebarOptions { - title: None, - appears_transparent: cfg!(any(target_os = "macos", target_os = "windows")), - traffic_light_position: Some(point(px(12.), px(19.))), - }), - // macOS: the app owns titlebar dragging. AppKit's native - // titlebar-region drag sits on top of whatever gpui draws in the - // top strip — dragging the Preview URL bar moved the window - // instead of selecting text. Every draggable strip already goes - // through `window_drag_area` (`start_window_move`), so hand the - // whole content view to the app and let controls own their input. - app_owns_titlebar_drag: true, - // Spell out that Windows is client-decorated. (This field is - // advisory off Wayland; leaving it `None` elsewhere keeps Linux - // on whatever its compositor/backend already chose.) - window_decorations: cfg!(target_os = "windows") - .then_some(WindowDecorations::Client), - // Persistent windows use the platform's system material: - // macOS sidebar vibrancy, Windows Acrylic, or an opaque fallback. - window_background: main_window_background(), - // Throttle background redraws (spinners, streaming output) to - // ~2 FPS while the window is inactive; gpui lifts the cap the - // moment the window is active or receiving high-rate input. - // This setting is captured at window creation and requires a restart. - inactive_frame_interval: (!local_settings.inactive_frame_throttle_disabled) - .then(|| Duration::from_millis(500)), - ..Default::default() - }; - - let local_kernel = kernel.clone(); - let (window, shell) = tcode_ui::run_shell( - cx, - native_client.clone(), - ShellOptions { - window: window_options, - title: tcode_ui::tr!("app.name").into(), - fonts: application_fonts, - theme_json, - activate: true, - system_locale: None, - // A desktop window is never suspended by the OS. - lifecycle: None, - setup: ShellSetup { - client_host: Some(native_client.clone()), - local: Some(Rc::new(move || local_kernel.transport())), - initial: Some(initial_target.clone()), - initial_pairing_error: None, - // Only here: bootstrap applies locale and theme from the - // host's own settings before the first frame. - seed_blocking: true, - restore_navigation: false, - }, - }, - ); - cx.on_action::({ - let shell = shell.clone(); - move |action, cx| handle_quit(action, &shell, cx) - }); - // Restart continuity: if this launch follows a permission-grant - // relaunch, reopen the recorded session and Settings page. Only - // meaningful for a host in this process: the marker lives in this - // machine's data dir, and a remote host's marker is its own. - if shell - .read(cx) - .store() - .is_some_and(|store| !store.read(cx).is_remote()) - && let Ok(CommandResponse::PendingRelaunchSection { - section: Some(section), - session_id, - }) = kernel - .control_link - .command_blocking(Command::ApplyPendingRelaunch) - { - if let Some(id) = session_id - && let Some(store) = shell.read(cx).store() - { - store.update(cx, |store, _| store.select_session(id)); + let launch = move |cx: &mut App, kernel: Rc| { + let local_settings = kernel.settings(); + // Hosting belongs to the process-owned local kernel; it carries no + // current-attachment mode. + cx.set_global(RemoteController::new( + kernel.mux.clone(), + data_dir.clone(), + kernel.control_link.clone(), + local_settings.clone(), + )); + if local_settings.remote_hosting_enabled { + let name = local_settings + .remote_host_name + .clone() + .unwrap_or_else(machine_name); + cx.update_global::(|controller, _| { + if let Err(error) = controller.start_hosting(&local_settings.traverse, name) + { + log::error!("remote hosting could not start: {error}"); + } + }); } - let window_state = shell.read(cx).window_state(); - window_state.update(cx, |state, cx| { - state.pending_settings_section = Some(section); - state.open_settings(cx); + // Process ownership, not the window's current attachment, grants + // authority to stop the local kernel on application quit. + let quit_subscription = cx.on_app_quit({ + let link = kernel.control_link.clone(); + let to_host = kernel.host.to_host.clone(); + let stopped = kernel.host.stopped.clone(); + move |_cx| { + let link = link.clone(); + let to_host = to_host.clone(); + let stopped = stopped.clone(); + async move { + let _ = link.shutdown().await; + // `shutdown` only closes this client's mux connection; + // the host loop ends when its own inbox closes. + to_host.close(); + let _ = stopped.recv().await; + } + } }); - } + quit_subscription.detach(); - cx.spawn(async move |cx| { - #[cfg(not(target_os = "linux"))] - if let Some(watchdog) = preview_smoke_watchdog { - preview_smoke::run(watchdog, shell, window, cx).await; - return; + let local_kernel = kernel.clone(); + let options = window_options( + cx, + (!local_settings.inactive_frame_throttle_disabled) + .then_some(INACTIVE_FRAME_INTERVAL), + ); + let (window, shell) = tcode_ui::run_shell( + cx, + native_client.clone(), + ShellOptions { + window: options, + setup: ShellSetup { + client_host: Some(native_client.clone()), + local: Some(Rc::new(move || local_kernel.transport())), + initial: Some(initial_target.clone()), + initial_pairing_error: None, + // Only here: bootstrap applies locale and theme from the + // host's own settings before the first frame. + seed_blocking: true, + restore_navigation: false, + }, + ..shell_options + }, + ); + *shell_slot.borrow_mut() = Some(shell.clone()); + // Restart continuity: if this launch follows a permission-grant + // relaunch, reopen the recorded session and Settings page. Only + // meaningful for a host in this process: the marker lives in this + // machine's data dir, and a remote host's marker is its own. + if shell + .read(cx) + .store() + .is_some_and(|store| !store.read(cx).is_remote()) + && let Ok(CommandResponse::PendingRelaunchSection { + section: Some(section), + session_id, + }) = kernel + .control_link + .command_blocking(Command::ApplyPendingRelaunch) + { + if let Some(id) = session_id + && let Some(store) = shell.read(cx).store() + { + store.update(cx, |store, _| store.select_session(id)); + } + let window_state = shell.read(cx).window_state(); + window_state.update(cx, |state, cx| { + state.pending_settings_section = Some(section); + state.open_settings(cx); + }); } - let _ = window; - if open_latest { - let Some(link) = cx.update(|cx| shell.read(cx).link()) else { + cx.spawn(async move |cx| { + #[cfg(not(target_os = "linux"))] + if let Some(watchdog) = preview_smoke_watchdog { + preview_smoke::run(watchdog, shell, window, cx).await; return; - }; - if let Ok(CommandResponse::SessionId(Some(id))) = - link.command(Command::OpenLatestSession).await - && let Some(store) = cx.update(|cx| shell.read(cx).store()) - { - store.update(cx, |store, _| store.select_session(id)); } - for _ in 0..100 { - if cx.update(|cx| { - shell - .read(cx) - .store() - .is_some_and(|store| store.read(cx).active_session_id().is_some()) - }) { - break; + let _ = window; + + if open_latest { + let Some(link) = cx.update(|cx| shell.read(cx).link()) else { + return; + }; + if let Ok(CommandResponse::SessionId(Some(id))) = + link.command(Command::OpenLatestSession).await + && let Some(store) = cx.update(|cx| shell.read(cx).store()) + { + store.update(cx, |store, _| store.select_session(id)); + } + for _ in 0..100 { + if cx.update(|cx| { + shell.read(cx).store().is_some_and(|store| { + store.read(cx).active_session_id().is_some() + }) + }) { + break; + } + cx.background_executor() + .timer(std::time::Duration::from_millis(10)) + .await; + } + if let Some(store) = cx.update(|cx| shell.read(cx).store()) { + store.update(cx, |store, _cx| { + store.sync_active_conversation_ui(); + }); } - cx.background_executor() - .timer(std::time::Duration::from_millis(10)) - .await; - } - if let Some(store) = cx.update(|cx| shell.read(cx).store()) { - store.update(cx, |store, _cx| { - store.sync_active_conversation_ui(); - }); } + }) + .detach(); + }; + + match kernel { + Some(kernel) => launch(cx, kernel), + None => { + let options = window_options(cx, Some(INACTIVE_FRAME_INTERVAL)); + let kernel_store = store.clone(); + migration::run(cx, store, options, appearance, move |cx| { + launch(cx, Rc::new(LocalKernel::start(kernel_store)?)); + Ok(()) + }); } - }) - .detach(); + } }); } diff --git a/crates/app/src/migration.rs b/crates/app/src/migration.rs new file mode 100644 index 000000000..66a0377b8 --- /dev/null +++ b/crates/app/src/migration.rs @@ -0,0 +1,325 @@ +//! The one-time migration into `tcode.db`, run before the local kernel starts +//! behind a window that offers nothing but its progress and Quit. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use gpui::{ + AnyWindowHandle, App, AppContext as _, Context, Entity, Global, IntoElement, + ParentElement as _, Render, Styled as _, Window, WindowOptions, div, + prelude::FluentBuilder as _, px, +}; +use tcode_services::store::{Migration, MigrationPhase, MigrationProgress, SessionStore}; +use tcode_ui::overlay::{DialogActions, OverlayExt as _}; +use tcode_ui::theme::{self, ActiveTheme as _, ThemeMode}; +use tcode_ui::widgets::{Button, Progress}; + +/// What the dialog shows. +enum Status { + Running(Option), + /// Quit was asked for: the migration stops at its next chunk, and the + /// process exits once it has. + Quitting, + Failed(String), +} + +struct MigrationView { + status: Status, + cancel: Arc, +} + +/// The view a Quit reaches while the migration window is up. +struct Running(Entity); +impl Global for Running {} + +/// Behind the dialog; the window shows nothing else. +struct Backdrop; + +impl Render for Backdrop { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl IntoElement { + div().size_full() + } +} + +enum Update { + Progress(MigrationProgress), + Done(std::io::Result), +} + +/// Migrate `store` behind a modal progress dialog, then hand over to `start` +/// and close the window. Quit, ⌘Q or closing the window cancels the +/// migration and exits once it has stopped; a failure, of the migration or of +/// `start`, stays in the dialog until Quit, which then exits non-zero. +pub(crate) fn run( + cx: &mut App, + store: SessionStore, + window_options: WindowOptions, + appearance: Option, + start: impl FnOnce(&mut App) -> std::io::Result<()> + 'static, +) { + let cancel = Arc::new(AtomicBool::new(false)); + let view = cx.new(|_| MigrationView { + status: Status::Running(None), + cancel: cancel.clone(), + }); + cx.set_global(Running(view.clone())); + + let (updates, received) = async_channel::unbounded(); + let (finished, stopped) = async_channel::bounded::<()>(1); + let spawned = std::thread::Builder::new() + .name("tcode-migration".into()) + .spawn({ + let cancel = cancel.clone(); + move || { + let progress = updates.clone(); + let result = store.migrate( + |value| { + let _ = progress.try_send(Update::Progress(value)); + }, + &cancel, + ); + let _ = updates.try_send(Update::Done(result)); + drop(finished); + } + }); + // A quit the dialog does not see first (the Dock menu, logging out) + // still stops the migration; gpui waits for it only briefly, and a + // migration it outlasts is discarded by the next start. + cx.on_app_quit({ + let cancel = cancel.clone(); + move |_| { + cancel.store(true, Ordering::Relaxed); + let stopped = stopped.clone(); + async move { + let _ = stopped.recv().await; + } + } + }) + .detach(); + + let window = tcode_ui::open_client_window( + cx, + window_options, + tcode_ui::tr!("app.name").into(), + |_, cx| cx.new(|_| Backdrop), + ); + cx.activate(true); + let _ = window.update(cx, |_, window, cx| { + // The host's theme setting is unknown until its kernel starts; this + // client's own choice is not. + match appearance.as_deref() { + Some("light") => theme::change_mode(ThemeMode::Light, Some(window), cx), + Some("dark") => theme::change_mode(ThemeMode::Dark, Some(window), cx), + _ => {} + } + window.on_window_should_close(cx, |_, cx| { + quit(cx); + false + }); + let content = view.clone(); + window.open_dialog(cx, move |dialog, _, _| { + let content = content.clone(); + dialog + .title(tcode_ui::tr!("migration.title").into_owned()) + .close_button(false) + .overlay_closable(false) + .keyboard(false) + .content(move |body, _, _| body.child(content.clone())) + .footer( + DialogActions::new().child( + Button::new("migration-quit") + .label(tcode_ui::tr!("quit.confirm")) + .on_click(|_, _, cx| { + quit(cx); + }), + ), + ) + }); + window.activate_window(); + }); + + if let Err(error) = spawned { + fail(&view, error.to_string(), cx); + return; + } + cx.spawn(async move |cx| { + while let Ok(mut update) = received.recv().await { + // Only the latest progress is drawn; `Done` is always the last. + while let Ok(next) = received.try_recv() { + update = next; + } + match update { + Update::Progress(progress) => view.update(cx, |view, cx| { + if let Status::Running(shown) = &mut view.status { + *shown = Some(progress); + cx.notify(); + } + }), + Update::Done(result) => { + cx.update(|cx| finish(result, &view, window, start, cx)); + return; + } + } + } + }) + .detach(); +} + +fn finish( + result: std::io::Result, + view: &Entity, + window: AnyWindowHandle, + start: impl FnOnce(&mut App) -> std::io::Result<()>, + cx: &mut App, +) { + let quitting = matches!(view.read(cx).status, Status::Quitting); + match result { + Err(error) => { + log::error!("migration failed: {error}"); + fail(view, error.to_string(), cx); + if quitting { + cx.quit(); + } + } + Ok(Migration::Cancelled) => cx.quit(), + Ok(Migration::Completed) if quitting => cx.quit(), + Ok(Migration::Completed) => { + cx.remove_global::(); + match start(cx) { + Ok(()) => { + let _ = window.update(cx, |_, window, _| window.remove_window()); + } + Err(error) => { + log::error!("the local host did not start after the migration: {error}"); + cx.set_global(Running(view.clone())); + fail(view, error.to_string(), cx); + } + } + } + } +} + +/// Show `reason` in the dialog; the process exits non-zero once it quits. +fn fail(view: &Entity, reason: String, cx: &mut App) { + // gpui ends a quit with status 0; this observer is the last code that + // runs before it. + cx.on_app_quit(|_| async { std::process::exit(1) }).detach(); + view.update(cx, |view, cx| { + view.status = Status::Failed(reason); + cx.notify(); + }); +} + +/// Quit while the migration window is up. Returns whether it was. +pub(crate) fn quit(cx: &mut App) -> bool { + let Some(Running(view)) = cx.try_global::() else { + return false; + }; + let view = view.clone(); + view.update(cx, |view, cx| match view.status { + Status::Running(_) => { + view.cancel.store(true, Ordering::Relaxed); + view.status = Status::Quitting; + cx.notify(); + } + Status::Quitting => {} + Status::Failed(_) => cx.quit(), + }); + true +} + +impl Render for MigrationView { + fn render(&mut self, _: &mut Window, cx: &mut Context) -> impl IntoElement { + let theme = cx.theme(); + let (percent, line, failed) = match &self.status { + Status::Running(progress) => ( + progress.as_ref().map_or(0., overall_percent), + progress.as_ref().map_or_else( + || phase_line(MigrationPhase::Scanning, None), + |progress| phase_line(progress.phase, Some(progress)), + ), + false, + ), + Status::Quitting => (0., tcode_ui::tr!("migration.quitting").into_owned(), false), + Status::Failed(reason) => ( + 0., + tcode_ui::tr!("migration.failed", reason = reason.clone()).into_owned(), + true, + ), + }; + let quitting = matches!(self.status, Status::Quitting); + div() + .flex() + .flex_col() + .gap_3() + .text_size(px(13.)) + .child( + div() + .text_color(theme.muted_foreground) + .child(tcode_ui::tr!("migration.description")), + ) + .when(!failed, |column| { + column.child( + Progress::new("migration-progress") + .value(percent) + .loading(quitting), + ) + }) + .child( + div() + .text_color(if failed { + theme.danger + } else { + theme.muted_foreground + }) + .child(line), + ) + } +} + +/// Import and verification each go through every log once, but import +/// writes and syncs each chunk while verification only reads it back, which +/// takes a fraction of the time. The phases around them take a moment. +fn overall_percent(progress: &MigrationProgress) -> f32 { + let fraction = |done: u64, total: u64| { + if total == 0 { + 1. + } else { + (done as f32 / total as f32).min(1.) + } + }; + let phase = fraction(progress.bytes_done, progress.bytes_total); + match progress.phase { + MigrationPhase::Scanning => 0., + MigrationPhase::Importing => phase * 85., + MigrationPhase::Verifying => 85. + phase * 15., + MigrationPhase::Publishing | MigrationPhase::Archiving => 100., + } +} + +fn phase_line(phase: MigrationPhase, progress: Option<&MigrationProgress>) -> String { + let counts = |key: &str| { + let progress = progress.copied().unwrap_or(MigrationProgress { + phase, + threads_done: 0, + threads_total: 0, + bytes_done: 0, + bytes_total: 0, + }); + tcode_ui::tr!( + key, + done = progress.threads_done, + total = progress.threads_total, + bytes = tcode_ui::format_size(progress.bytes_done as usize), + total_bytes = tcode_ui::format_size(progress.bytes_total as usize) + ) + .into_owned() + }; + match phase { + MigrationPhase::Scanning => tcode_ui::tr!("migration.scanning").into_owned(), + MigrationPhase::Importing => counts("migration.importing"), + MigrationPhase::Verifying => counts("migration.verifying"), + MigrationPhase::Publishing => tcode_ui::tr!("migration.publishing").into_owned(), + MigrationPhase::Archiving => tcode_ui::tr!("migration.archiving").into_owned(), + } +} diff --git a/crates/core/src/project.rs b/crates/core/src/project.rs index e6b4ad2a8..b726f1816 100644 --- a/crates/core/src/project.rs +++ b/crates/core/src/project.rs @@ -470,8 +470,9 @@ pub fn order_sessions_with_children(sessions: Vec) -> Vec`; the store loader tolerates both. +/// Every project and session in the store. Also the shape of the legacy +/// `sessions.json` the store migrates from; older files were a bare +/// `Vec`, and the migration tolerates both. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct IndexFile { #[serde(default)] diff --git a/crates/headless/src/main.rs b/crates/headless/src/main.rs index ed2492e32..f6e401cd3 100644 --- a/crates/headless/src/main.rs +++ b/crates/headless/src/main.rs @@ -1,13 +1,14 @@ use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::Duration; +use std::sync::atomic::AtomicBool; +use std::time::{Duration, Instant}; use qrcode::QrCode; use qrcode::render::unicode::Dense1x2; use tcode_client::pairing::{PairInvite, pair_url, parse_pair_url}; use tcode_runtime::pipe::{HostServices, spawn_host}; -use tcode_services::store::SessionStore; +use tcode_services::store::{Migration, MigrationPhase, MigrationProgress, SessionStore}; use tcode_traverse::browser::{BrowserConfig, StaticBundle, check_bind, serve, set_password}; use tcode_traverse::identity::write_private; use tcode_traverse::lan::DEFAULT_PORT; @@ -35,6 +36,11 @@ const STATIC_BUNDLE: Option = None; const DEFAULT_BROWSER_LISTEN: &str = "127.0.0.1:47420"; /// The current invitation, for `pair` to print; absent while none is valid. const INVITATION_FILE: &str = "invitation.json"; +/// How often a running migration prints its progress. +const MIGRATION_REPORT_INTERVAL: Duration = Duration::from_secs(1); + +/// Set by SIGINT or SIGTERM: cancels a startup migration, then stops `serve`. +static INTERRUPTED: AtomicBool = AtomicBool::new(false); fn main() { env_logger::init(); @@ -115,6 +121,10 @@ fn serve_command(args: &[String]) -> Result<(), String> { } // Nothing else starts for a bind the listener would refuse anyway. check_bind(browser_listen, &remote_data_dir).map_err(|error| error.to_string())?; + install_interrupt_handler(); + if migrate_store(&store)? == Migration::Cancelled { + return Ok(()); + } let mut services = HostServices { background_startup_probes: true, ai_title_generation: true, @@ -230,6 +240,59 @@ fn serve_command(args: &[String]) -> Result<(), String> { Ok(()) } +/// Move an older build's threads into `tcode.db` before the host starts, +/// printing progress; an interrupt cancels it and leaves them as they were. +fn migrate_store(store: &SessionStore) -> Result { + let needed = store + .needs_migration() + .map_err(|error| format!("could not open session store: {error}"))?; + if !needed { + return Ok(Migration::Completed); + } + println!( + "Migrating threads into {}; the original files are kept in legacy/. Ctrl-C cancels.", + store.root().join("tcode.db").display() + ); + let mut last: Option<(MigrationPhase, Instant)> = None; + let outcome = store + .migrate( + |progress| { + let due = last.is_none_or(|(phase, at)| { + phase != progress.phase || at.elapsed() >= MIGRATION_REPORT_INTERVAL + }); + if due { + println!("{}", migration_line(&progress)); + last = Some((progress.phase, Instant::now())); + } + }, + &INTERRUPTED, + ) + .map_err(|error| format!("migration failed: {error}"))?; + match outcome { + Migration::Completed => println!("Migration complete"), + Migration::Cancelled => println!("Migration cancelled; no file was changed"), + } + Ok(outcome) +} + +fn migration_line(progress: &MigrationProgress) -> String { + let phase = match progress.phase { + MigrationPhase::Scanning => "scanning", + MigrationPhase::Importing => "importing", + MigrationPhase::Verifying => "verifying", + MigrationPhase::Publishing => "publishing", + MigrationPhase::Archiving => "archiving", + }; + let mib = |bytes: u64| bytes as f64 / (1024. * 1024.); + format!( + "{phase}: {}/{} threads, {:.1}/{:.1} MiB", + progress.threads_done, + progress.threads_total, + mib(progress.bytes_done), + mib(progress.bytes_total) + ) +} + fn set_password_command(args: &[String]) -> Result<(), String> { let revoke = args.iter().any(|arg| arg == "--revoke-tokens"); let values: Vec<_> = args @@ -351,26 +414,33 @@ fn reject_unknown_options(args: &[String], options_with_values: &[&str]) -> Resu } #[cfg(unix)] -fn wait_for_interrupt() { - use std::sync::atomic::{AtomicBool, Ordering}; - use std::time::Duration; - - static INTERRUPTED: AtomicBool = AtomicBool::new(false); - +fn install_interrupt_handler() { type SignalHandler = extern "C" fn(i32); unsafe extern "C" { fn signal(signal: i32, handler: SignalHandler) -> SignalHandler; } extern "C" fn handle_interrupt(_: i32) { - INTERRUPTED.store(true, Ordering::Relaxed); + INTERRUPTED.store(true, std::sync::atomic::Ordering::Relaxed); } const SIGINT: i32 = 2; - // SAFETY: installs a process-global handler with the C ABI expected by + const SIGTERM: i32 = 15; + // SAFETY: installs process-global handlers with the C ABI expected by // signal(3); the handler performs only a lock-free atomic store. unsafe { signal(SIGINT, handle_interrupt); + signal(SIGTERM, handle_interrupt); } - while !INTERRUPTED.load(Ordering::Relaxed) { +} + +/// Console interrupts keep their default effect: a migration they end is +/// discarded by the next start, and the sources are untouched until it +/// publishes. +#[cfg(not(unix))] +fn install_interrupt_handler() {} + +#[cfg(unix)] +fn wait_for_interrupt() { + while !INTERRUPTED.load(std::sync::atomic::Ordering::Relaxed) { std::thread::sleep(Duration::from_millis(100)); } } diff --git a/crates/headless/tests/migration.rs b/crates/headless/tests/migration.rs new file mode 100644 index 000000000..3a8c47fc7 --- /dev/null +++ b/crates/headless/tests/migration.rs @@ -0,0 +1,97 @@ +//! `tcode-headless serve` runs the one-time migration into `tcode.db` before +//! anything else starts, and an interrupt cancels it. + +#![cfg(unix)] + +use std::collections::BTreeMap; +use std::fs; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +/// Every file in `root` with its bytes. +fn snapshot(root: &Path) -> BTreeMap> { + fs::read_dir(root) + .unwrap() + .map(|entry| { + let entry = entry.unwrap(); + let bytes = if entry.file_type().unwrap().is_dir() { + b"".to_vec() + } else { + fs::read(entry.path()).unwrap() + }; + (entry.file_name().into_string().unwrap(), bytes) + }) + .collect() +} + +#[test] +fn sigint_cancels_the_startup_migration_and_leaves_every_source_as_it_was() { + let root: PathBuf = std::env::temp_dir().join(format!( + "tcode-headless-migration-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&root).unwrap(); + fs::write( + root.join("sessions.json"), + r#"{"projects":[],"sessions":[{"id":"log-0","title":"Zero","provider":"codex","cwd":"/work","created_at":1,"updated_at":1}]}"#, + ) + .unwrap(); + // Enough that the migration is still importing well after it starts. + let line = format!( + "{{\"ts\":1,\"event\":{{\"type\":\"turn_started\",\"turn_id\":\"{}\"}}}}\n", + "x".repeat(2000) + ); + for index in 0..24 { + fs::write(root.join(format!("log-{index}.jsonl")), line.repeat(1000)).unwrap(); + } + let sources = snapshot(&root); + + let mut serve = tcode_services::process::command(env!("CARGO_BIN_EXE_tcode-headless")) + .args(["serve", "--traverse", "off", "--data-dir"]) + .arg(&root) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut output = BufReader::new(serve.stdout.take().unwrap()).lines(); + let mut printed = Vec::new(); + loop { + let line = output + .next() + .expect("serve exited before the migration imported anything") + .unwrap(); + let importing = line.starts_with("importing:"); + printed.push(line); + if importing { + break; + } + } + let interrupted = tcode_services::process::command("kill") + .args(["-INT", &serve.id().to_string()]) + .status() + .unwrap(); + assert!(interrupted.success()); + printed.extend(output.map(Result::unwrap)); + let status = serve.wait().unwrap(); + + assert!(status.success(), "{status}: {printed:#?}"); + assert!( + printed + .iter() + .any(|line| line.starts_with("Migration cancelled")), + "{printed:#?}" + ); + assert!( + !printed.iter().any(|line| line.starts_with("Machine id")), + "the host started after the cancel: {printed:#?}" + ); + let mut after = snapshot(&root); + // The data dir's ownership lock, which every start creates. + assert!(after.remove("tcode.lock").is_some()); + assert_eq!(after, sources); + fs::remove_dir_all(&root).unwrap(); +} diff --git a/crates/runtime/src/app/approvals.rs b/crates/runtime/src/app/approvals.rs index 736477b72..ba8fbe9e2 100644 --- a/crates/runtime/src/app/approvals.rs +++ b/crates/runtime/src/app/approvals.rs @@ -100,7 +100,7 @@ mod tests { fn state(name: &str) -> (TestStore, AppState) { let store = TestStore::new(name); - let state = AppState::new((*store).clone()); + let state = AppState::new((*store).clone()).unwrap(); (store, state) } diff --git a/crates/runtime/src/app/command_validation.rs b/crates/runtime/src/app/command_validation.rs index 2e35b6bcf..7fad0bcee 100644 --- a/crates/runtime/src/app/command_validation.rs +++ b/crates/runtime/src/app/command_validation.rs @@ -298,7 +298,7 @@ mod tests { #[test] fn writes_blocked_by_host_state_receive_rejected_acks() { let store = TestStore::new("write-preconditions"); - let mut state = AppState::new((*store).clone()); + let mut state = AppState::new((*store).clone()).unwrap(); let mut context = TestAppContext::default(); let mut cx = context.host_cx(); let id = state.start_draft("fixture".into(), std::env::temp_dir(), &mut cx); @@ -417,7 +417,7 @@ mod tests { #[test] fn provider_channel_rejection_is_not_acknowledged_as_success() { let store = TestStore::new("provider-write-rejection"); - let mut state = AppState::new((*store).clone()); + let mut state = AppState::new((*store).clone()).unwrap(); let mut context = TestAppContext::default(); let mut cx = context.host_cx(); let id = state.start_draft("fixture".into(), std::env::temp_dir(), &mut cx); diff --git a/crates/runtime/src/app/events.rs b/crates/runtime/src/app/events.rs index 5bf72eec9..73bfbac99 100644 --- a/crates/runtime/src/app/events.rs +++ b/crates/runtime/src/app/events.rs @@ -95,7 +95,7 @@ impl AppState { // Provider commands/skills are session metadata for the composer menus — // stored on the live session and in a per-provider cache, never folded - // into the timeline or the persisted JSONL log. Parked sessions still + // into the timeline or the persisted event log. Parked sessions still // receive provider updates, so update/cache those too. AgentEvent::ProviderCommands { commands } => { let cache_key = self.resident_mut(session_id).map(|resident| { @@ -499,7 +499,7 @@ impl AppState { } } - /// Append to JSONL + fold into the matching active or background timeline. + /// Append to the event log + fold into the matching active or background timeline. /// The same wall-clock timestamp is persisted and folded exactly once so /// the on-disk log and the loaded timeline agree. pub(super) fn record_event(&mut self, session_id: &str, event: &AgentEvent, cx: &mut HostCx) { @@ -519,10 +519,17 @@ impl AppState { ServerEvent::SessionEvent(history::wire_record(&record).into_owned()), cx, ); - self.event_records - .entry(session_id.to_string()) - .or_insert_with(|| SessionLog::load(&self.store, session_id)) - .push(record); + // A log that cannot be read is not cached: an empty stand-in would + // replace the conversation. The append is still queued. + match self.event_records.entry(session_id.to_string()) { + std::collections::hash_map::Entry::Occupied(mut log) => log.get_mut().push(record), + std::collections::hash_map::Entry::Vacant(slot) => { + match SessionLog::load(&self.store, session_id) { + Ok(log) => slot.insert(log).push(record), + Err(error) => log::error!("cannot load the event log of {session_id}: {error}"), + } + } + } self.enqueue_store_write( StoreWrite::AppendEvent { id: session_id.to_string(), @@ -645,9 +652,13 @@ impl AppState { attachments, )); } - let timeline = Timeline::fold_events( - records.unwrap_or_else(|| store.read_events(&read_id)), - ); + let records = match records { + Some(records) => records, + None => store + .read_events(&read_id) + .map_err(|error| RuntimeError::External(error.to_string()))?, + }; + let timeline = Timeline::fold_events(records); let (source, paths) = title_regeneration_context(&timeline); if source.is_empty() { return Err(RuntimeError::TitleGenerationEmpty); diff --git a/crates/runtime/src/app/history.rs b/crates/runtime/src/app/history.rs index 67b949e02..143776c38 100644 --- a/crates/runtime/src/app/history.rs +++ b/crates/runtime/src/app/history.rs @@ -207,8 +207,8 @@ fn wire_window( } /// The complete event log of one session, held in memory while the session -/// is resident so history windows cost the page rather than a re-parse of the -/// JSONL, plus the fold that decides where each window may start. +/// is resident so history windows cost the page rather than a re-read of the +/// stored log, plus the fold that decides where each window may start. /// /// Memory policy: [`AppState::event_records`] holds a log for every live or /// parked session (bounded by the resident LRU) and for nothing else. A log @@ -216,7 +216,7 @@ fn wire_window( /// appends in this process, and is dropped once the session leaves residency /// and the store writer has flushed every append queued from it /// ([`AppState::release_stale_session_logs`]): until then the log, not the -/// JSONL, is the whole conversation. +/// store, is the whole conversation. #[derive(Clone)] pub(super) struct SessionLog { records: Vec, @@ -231,8 +231,8 @@ pub(super) struct SessionLog { } impl SessionLog { - pub(super) fn load(store: &SessionStore, session_id: &str) -> Self { - Self::from_records(store.read_events(session_id)) + pub(super) fn load(store: &SessionStore, session_id: &str) -> std::io::Result { + Ok(Self::from_records(store.read_events(session_id)?)) } pub(super) fn from_records(records: impl IntoIterator) -> Self { @@ -284,21 +284,31 @@ impl AppState { /// The session's log, cached for the resident session it belongs to. A /// non-resident session is read cold and not retained, so paging it never /// grows the cache. - fn history_log(&mut self, session_id: &str) -> std::borrow::Cow<'_, SessionLog> { + fn history_log( + &mut self, + session_id: &str, + ) -> Result, tcode_protocol::ProtocolError> { if !self.event_records.contains_key(session_id) { - let log = SessionLog::load(&self.store, session_id); + let log = SessionLog::load(&self.store, session_id).map_err(|error| { + tcode_protocol::ProtocolError { + code: "history_unavailable".into(), + message: format!("could not read the history of {session_id}: {error}"), + } + })?; if self.resident(session_id).is_none() { - return std::borrow::Cow::Owned(log); + return Ok(std::borrow::Cow::Owned(log)); } self.event_records.insert(session_id.to_string(), log); } - std::borrow::Cow::Borrowed(&self.event_records[session_id]) + Ok(std::borrow::Cow::Borrowed(&self.event_records[session_id])) } /// Queue a store-writer barrier for every cached log whose session left - /// residency, and drop the log once the barrier echoes: a cold read after - /// that sees every append the log had accepted. Appends that race the - /// barrier re-arm it. + /// residency, and drop the log once the barrier confirms every earlier + /// write committed: a cold read after that sees every append the log had + /// accepted. Appends that race the barrier re-arm it. A barrier that + /// reports a failed write, or a writer that is gone, keeps the log: it is + /// then the only complete copy of the conversation. pub(super) fn release_stale_session_logs(&mut self, cx: &mut HostCx) { let stale: Vec<(String, usize)> = self .event_records @@ -315,13 +325,21 @@ impl AppState { self.enqueue_store_write(StoreWrite::Flush(completion), cx); let host_cx = cx.clone(); HostCx::spawn_detached(cx, async move { - // A stopped writer has dropped its queue as well; nothing - // later can still land in the JSONL. - let _ = completed.recv().await; + let flushed_ok = match completed.recv().await { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(error), + Err(_) => Err("the session store writer has stopped".to_owned()), + }; host_cx.enqueue(move |state, cx| { let Some(log) = state.event_records.get_mut(&session_id) else { return; }; + if let Err(error) = flushed_ok { + // The barrier stays armed, so the log is never offered + // for release again. + log::warn!("keeping the in-memory history of {session_id}: {error}"); + return; + } log.release_barrier = None; let appended_since = log.records.len() != flushed; if state.resident(&session_id).is_some() { @@ -344,7 +362,10 @@ impl AppState { let Topic::SessionEvents { session_id } = &subscription.topic else { unreachable!() }; - let log = self.history_log(session_id); + let log = match self.history_log(session_id) { + Ok(log) => log, + Err(error) => return ServerEvent::SessionHistoryError(error), + }; let records = log.records(); let total = records.len(); let total_turns = log.fold().turns.len() as u64; @@ -388,7 +409,7 @@ impl AppState { before: u64, limit: u32, ) -> Result { - let log = self.history_log(session_id); + let log = self.history_log(session_id)?; let records = log.records(); let end = before.min(records.len() as u64) as usize; let count = (limit as usize).clamp(1, SESSION_HISTORY_RECORDS); @@ -418,7 +439,7 @@ impl AppState { session_id: &str, item_id: &str, ) -> Result { - let log = self.history_log(session_id); + let log = self.history_log(session_id)?; let output = log .fold() .entries diff --git a/crates/runtime/src/app/lifecycle.rs b/crates/runtime/src/app/lifecycle.rs index 2d01492d7..51a1cd982 100644 --- a/crates/runtime/src/app/lifecycle.rs +++ b/crates/runtime/src/app/lifecycle.rs @@ -325,8 +325,8 @@ impl AppState { cx, ); // Reflect the upsert in memory instead of reloading the whole index - // from disk: `persist_meta` runs on every turn, - // where re-reading and re-parsing a large sessions.json stalls the UI. + // from the store: `persist_meta` runs on every turn, where re-reading + // and re-parsing every session's meta would stall the mailbox. // `sessions` stays newest-first, matching `load_index`'s order. self.upsert_session_in_memory(meta.clone()); } diff --git a/crates/runtime/src/app/mod.rs b/crates/runtime/src/app/mod.rs index c33c2c649..62cbf4d20 100644 --- a/crates/runtime/src/app/mod.rs +++ b/crates/runtime/src/app/mod.rs @@ -260,7 +260,7 @@ use providers::{ }; pub use sessions::ResidentSessions; pub(crate) use snapshots::DomainDiff; -use store_write::{StoreWrite, run_store_write}; +use store_write::{StoreWrite, StoreWriteFailure, StoreWriter}; /// The result of a provider version check. #[derive(Debug, Clone, Default)] @@ -310,8 +310,12 @@ pub struct AppState { settings_store: SettingsStore, store_writes: smol::channel::Sender, store_write_receiver: Option>, - store_write_failures: smol::channel::Sender>, - store_write_failure_receiver: Option>>, + store_write_failures: smol::channel::Sender, + store_write_failure_receiver: Option>, + /// The store-writer thread, joined by [`AppState::close_store`]. + store_writer: Option>, + /// Set once the store failed and every thread was stopped. + store_failed: bool, pub sessions: Vec, pub projects: Vec, pub residents: ResidentSessions, @@ -436,16 +440,18 @@ pub(crate) fn startup_collapsed_threads(sessions: &[SessionMeta]) -> Vec } impl AppState { - pub fn new(store: SessionStore) -> Self { + pub fn new(store: SessionStore) -> std::io::Result { Self::with_ai_titles(store, false) } - pub(crate) fn with_ai_titles(store: SessionStore, ai_title_generation_enabled: bool) -> Self { - // Load + migrate once and persist so derived project ids stay stable. - let file = store.read_file(); - if let Err(err) = store.persist_index(&file) { - log::warn!("failed to persist migrated session index: {err}"); - } + /// Take ownership of the data dir and load its threads. Fails while + /// another host owns the directory, or when its database cannot be opened. + pub(crate) fn with_ai_titles( + store: SessionStore, + ai_title_generation_enabled: bool, + ) -> std::io::Result { + store.open()?; + let file = store.read_file()?; let mut sessions = file.sessions; sessions.sort_by_key(|b| std::cmp::Reverse(b.updated_at)); let projects = file.projects; @@ -488,13 +494,15 @@ impl AppState { let (store_writes, store_write_receiver) = smol::channel::unbounded(); let (store_write_failures, store_write_failure_receiver) = smol::channel::unbounded(); let session_search = Arc::new(std::sync::Mutex::new(SessionSearch::new(store.clone()))); - Self { + Ok(Self { store, settings_store, store_writes, store_write_receiver: Some(store_write_receiver), store_write_failures, store_write_failure_receiver: Some(store_write_failure_receiver), + store_writer: None, + store_failed: false, sessions, projects, residents: ResidentSessions::default(), @@ -540,7 +548,7 @@ impl AppState { external_imports: HashMap::new(), next_import_run_id: 1, session_search, - } + }) } #[cfg(any(test, feature = "test-support"))] @@ -550,27 +558,32 @@ impl AppState { fn start_store_writer(&mut self, cx: &mut HostCx) { if let Some(writes) = self.store_write_receiver.take() { - let store = self.store.clone(); - let settings_store = self.settings_store.clone(); - let terminal_preferences_path = self.terminal_preferences_path.clone(); - let failures = self.store_write_failures.clone(); - HostCx::spawn_detached(cx, async move { - while let Ok(write) = writes.recv().await { - if let Some(failure) = - run_store_write(&store, &settings_store, &terminal_preferences_path, write) - { - let _ = failures.send(failure).await; - } + let writer = StoreWriter::new( + self.store.clone(), + self.settings_store.clone(), + self.terminal_preferences_path.clone(), + self.store_write_failures.clone(), + ); + // Its own thread: every commit blocks on the database's fsync. + match std::thread::Builder::new() + .name("tcode-store-writer".into()) + .spawn(move || writer.run(writes)) + { + Ok(handle) => self.store_writer = Some(handle), + Err(error) => { + log::error!("could not start the session store writer: {error}"); + self.store_writes.close(); } - }); + } } if let Some(failures) = self.store_write_failure_receiver.take() { let host_cx = cx.clone(); HostCx::spawn_detached(cx, async move { while let Ok(failure) = failures.recv().await { host_cx.enqueue(move |state, cx| match failure { - Ok(error) => state.report_error(error, cx), - Err(message) => log::warn!("{message}"), + StoreWriteFailure::Error(error) => state.report_error(error, cx), + StoreWriteFailure::Warning(message) => log::warn!("{message}"), + StoreWriteFailure::StoreFailed(reason) => state.store_failed(&reason, cx), }); } }); @@ -579,9 +592,38 @@ impl AppState { fn enqueue_store_write(&mut self, write: StoreWrite, cx: &mut HostCx) { self.start_store_writer(cx); - if self.store_writes.try_send(write).is_err() { + if let Err(rejected) = self.store_writes.try_send(write) { log::error!("session store writer stopped before accepting a write"); + if let Some(StoreWriteFailure::Error(error)) = rejected + .into_inner() + .reject("the session store writer has stopped") + { + self.report_error(error, cx); + } + } + } + + /// The store stopped serving: stop every provider so nothing keeps + /// producing events that can no longer be saved. The failed writes were + /// reported with the reason. + fn store_failed(&mut self, reason: &str, cx: &mut HostCx) { + if std::mem::replace(&mut self.store_failed, true) { + return; + } + log::error!("session store failed ({reason}); stopping every thread"); + self.shutdown_all(cx); + } + + /// Drain the store writer, then checkpoint and close the database. Only + /// after the host loop has stopped, so nothing enqueues behind it. + pub(crate) fn close_store(&mut self) -> std::io::Result<()> { + self.store_writes.close(); + if let Some(writer) = self.store_writer.take() + && writer.join().is_err() + { + log::error!("the session store writer panicked"); } + self.store.close() } fn enqueue_settings(&mut self, settings: &Settings, cx: &mut HostCx) { diff --git a/crates/runtime/src/app/orchestrate.rs b/crates/runtime/src/app/orchestrate.rs index 10b94465b..db631f61e 100644 --- a/crates/runtime/src/app/orchestrate.rs +++ b/crates/runtime/src/app/orchestrate.rs @@ -875,18 +875,23 @@ impl AppState { unloaded .into_iter() .map(|id| { - let timeline = Timeline::fold_events(store.read_events(&id)); - (id, timeline) + let events = store + .read_events(&id) + .map_err(|error| format!("could not read thread {id}: {error}"))?; + Ok((id, Timeline::fold_events(events))) }) - .collect::>() + .collect::, String>>() }) .await; - let result = host_cx - .enqueue_and_wait(move |state, _| { - state.orchestrate_status_json(&children, &timelines) - }) - .await - .map_err(|_| "tcode orchestrator is not available".to_string()); + let result = match timelines { + Ok(timelines) => host_cx + .enqueue_and_wait(move |state, _| { + state.orchestrate_status_json(&children, &timelines) + }) + .await + .map_err(|_| "tcode orchestrator is not available".to_string()), + Err(error) => Err(error), + }; let _ = reply.send(result).await; }); } @@ -915,15 +920,18 @@ impl AppState { HostCx::spawn_detached(cx, async move { let read_id = thread_id.clone(); let timeline = host_cx - .unblock(move || Timeline::fold_events(store.read_events(&read_id))) + .unblock(move || store.read_events(&read_id).map(Timeline::fold_events)) .await; - let result = host_cx - .enqueue_and_wait(move |state, _| { - let timeline = state.loaded_child_timeline(&thread_id).unwrap_or(&timeline); - state.orchestrate_result_json(&meta, timeline) - }) - .await - .unwrap_or_else(|_| Err("tcode orchestrator is not available".to_string())); + let result = match timeline { + Ok(timeline) => host_cx + .enqueue_and_wait(move |state, _| { + let timeline = state.loaded_child_timeline(&thread_id).unwrap_or(&timeline); + state.orchestrate_result_json(&meta, timeline) + }) + .await + .unwrap_or_else(|_| Err("tcode orchestrator is not available".to_string())), + Err(error) => Err(format!("could not read thread {thread_id}: {error}")), + }; let _ = reply.send(result).await; }); } @@ -1047,14 +1055,43 @@ impl AppState { // attention. let auto_archive = child.archive_on_complete && matches!(status, TurnStatus::Completed); let result_max_chars = child.result_max_chars; + // The cached log holds appends the store writer may not have committed + // yet; without one, the store is read only after a barrier. + let source = match self.event_records.get(&child_id) { + Some(log) => Ok(log.fold().clone()), + None => Err(self.store_write_barrier(cx)), + }; let store = self.store.clone(); let host_cx = cx.clone(); HostCx::spawn_detached(cx, async move { let read_id = child_id.clone(); - let timeline = host_cx - .unblock(move || Timeline::fold_events(store.read_events(&read_id))) - .await; + let timeline = match source { + Ok(timeline) => Ok(timeline), + Err(barrier) => match barrier.recv().await { + Ok(Ok(())) => { + host_cx + .unblock(move || store.read_events(&read_id).map(Timeline::fold_events)) + .await + } + Ok(Err(error)) => Err(std::io::Error::other(error)), + Err(_) => Err(std::io::Error::other( + "the session store writer has stopped", + )), + }, + }; host_cx.enqueue(move |state, cx| { + let timeline = match timeline { + Ok(timeline) => timeline, + Err(error) => { + state.report_error( + RuntimeError::External(format!( + "could not read thread {child_id} to report its completion: {error}" + )), + cx, + ); + return; + } + }; let child_still_exists = state.sessions.iter().any(|meta| { meta.id == child_id && meta.parent_session_id.as_deref() == Some(parent_id.as_str()) diff --git a/crates/runtime/src/app/sessions.rs b/crates/runtime/src/app/sessions.rs index eb4355575..6dfc96db9 100644 --- a/crates/runtime/src/app/sessions.rs +++ b/crates/runtime/src/app/sessions.rs @@ -402,7 +402,13 @@ impl AppState { /// Reload sessions written by the external-history importer and expand its /// project group. fn finish_external_import(&mut self, project_id: &str, cx: &mut HostCx) { - self.sessions = self.store.load_index(); + match self.store.load_index() { + Ok(sessions) => self.sessions = sessions, + Err(error) => self.report_error( + RuntimeError::External(format!("could not reload the imported threads: {error}")), + cx, + ), + } if self .settings .collapsed_projects @@ -437,10 +443,17 @@ impl AppState { let store = self.store.clone(); let host_cx = cx.clone(); cx.spawn_background(async move { - barrier.recv().await.map_err(|error| ProtocolError { - code: "store_barrier_closed".into(), - message: format!("session-store flush failed: {error}"), - })?; + barrier + .recv() + .await + .map_err(|error| ProtocolError { + code: "store_barrier_closed".into(), + message: format!("session-store flush failed: {error}"), + })? + .map_err(|message| ProtocolError { + code: "store_flush_failed".into(), + message, + })?; let suggested_name = export::export_file_name(&meta.title, format); let bytes = host_cx .unblock(move || export::render_thread(&store, &meta, format)) @@ -923,9 +936,9 @@ impl AppState { let fork_id = fork.id.clone(); let (completion, completed) = smol::channel::bounded(1); self.enqueue_store_write( - StoreWrite::CloneEvents { + StoreWrite::Fork { src: source.id, - dst: fork.id.clone(), + meta: Box::new(fork.clone()), completion, }, cx, @@ -938,13 +951,6 @@ impl AppState { .unwrap_or_else(|_| Err("session store writer stopped".into())); host_cx.enqueue(move |state, cx| match result { Ok(()) => { - state.enqueue_store_write( - StoreWrite::UpsertMeta { - meta: Box::new(fork.clone()), - initial: true, - }, - cx, - ); state.upsert_session_in_memory(fork.clone()); state.select_session(&fork.id, cx); if let Some(snapshot) = @@ -959,7 +965,7 @@ impl AppState { } } Err(error) => { - state.report_error(RuntimeError::PersistEvent { error }, cx); + state.report_error(RuntimeError::PersistSession { error }, cx); } }); }); @@ -967,7 +973,7 @@ impl AppState { } /// Permanently delete a thread: stop the provider, close its terminal, - /// delete meta + JSONL, and (when `remove_worktree`) remove the git worktree + /// delete meta + event log, and (when `remove_worktree`) remove the git worktree /// it was the last user of. pub fn delete_session(&mut self, session_id: &str, remove_worktree: bool, cx: &mut HostCx) { self.clear_approvals(session_id); @@ -1448,19 +1454,32 @@ impl AppState { }; // A cached log is the whole conversation, including appends whose // disk writes are still queued, so the timeline derives from it and - // never from a JSONL that can lag the store writer. A session opened + // never from the store, which lags the writes still queued. A session opened // for a client loads its log here, on the mailbox, because the // snapshot that answers the subscription needs it in the same turn; // a background session parses off the mailbox and caches on completion. - let cached = match target { - TimelineLoadTarget::Active { .. } => Some( - self.event_records - .entry(session_id.clone()) - .or_insert_with(|| SessionLog::load(&self.store, &session_id)), - ), - TimelineLoadTarget::Background => self.event_records.get_mut(&session_id), + if matches!(target, TimelineLoadTarget::Active { .. }) + && !self.event_records.contains_key(&session_id) + { + match SessionLog::load(&self.store, &session_id) { + Ok(log) => { + self.event_records.insert(session_id.clone(), log); + } + Err(error) => { + self.report_error( + RuntimeError::External(format!( + "could not load thread {session_id}: {error}" + )), + cx, + ); + return; + } + } } - .map(|log| (log.records().len(), log.fold().clone())); + let cached = self + .event_records + .get(&session_id) + .map(|log| (log.records().len(), log.fold().clone())); let store = self.store.clone(); let host_cx = cx.clone(); HostCx::spawn_detached(cx, async move { @@ -1468,10 +1487,13 @@ impl AppState { let (timeline, folded, loaded, git_branch) = { let (mut timeline, folded, loaded) = match cached { Some((folded, fold)) => (fold, folded, None), - None => { - let log = SessionLog::load(&store, &read_id); - (log.fold().clone(), log.records().len(), Some(log)) - } + None => match SessionLog::load(&store, &read_id) { + Ok(log) => (log.fold().clone(), log.records().len(), Some(log)), + Err(error) => { + log::error!("could not load thread {read_id}: {error}"); + return; + } + }, }; let (mark_idle, load_branch) = match target { TimelineLoadTarget::Active { mark_idle } => (mark_idle, true), @@ -1524,7 +1546,7 @@ impl AppState { } /// Adopt a subscribed session, including an uncommitted parked draft. Stored - /// sessions replay their JSONL; providers start lazily on the next send. + /// sessions replay their event log; providers start lazily on the next send. pub fn select_session(&mut self, session_id: &str, cx: &mut HostCx) { if self.residents.live.contains_key(session_id) { return; diff --git a/crates/runtime/src/app/snapshots.rs b/crates/runtime/src/app/snapshots.rs index 975e02a35..ed23b7503 100644 --- a/crates/runtime/src/app/snapshots.rs +++ b/crates/runtime/src/app/snapshots.rs @@ -459,8 +459,12 @@ impl AppState { } /// Enqueue a FIFO barrier used by the application quit hook. The returned - /// receiver resolves only after every earlier store write has completed. - pub fn store_write_barrier(&mut self, cx: &mut HostCx) -> smol::channel::Receiver<()> { + /// receiver resolves only after every earlier store write has committed, + /// with the store failure that lost one if any did. + pub fn store_write_barrier( + &mut self, + cx: &mut HostCx, + ) -> smol::channel::Receiver> { let (completion, barrier) = smol::channel::bounded(1); self.enqueue_store_write(StoreWrite::Flush(completion), cx); barrier diff --git a/crates/runtime/src/app/store_write.rs b/crates/runtime/src/app/store_write.rs index 9d16c1c81..a6dbfc112 100644 --- a/crates/runtime/src/app/store_write.rs +++ b/crates/runtime/src/app/store_write.rs @@ -1,4 +1,11 @@ use super::*; +use tcode_services::store::Mutation; + +/// A drain of the queue is committed in transactions of at most this many +/// writes and about this many bytes; streamed deltas then share a commit +/// instead of each paying for one. +const MAX_BATCH_WRITES: usize = 256; +const MAX_BATCH_BYTES: usize = 4 << 20; pub(super) enum StoreWrite { AppendEvent { @@ -13,9 +20,11 @@ pub(super) enum StoreWrite { UpsertProject(Project), RemoveSession(String), RemoveProject(String), - CloneEvents { + /// The source's event log copied to a new thread, committed together with + /// the new thread's meta. + Fork { src: String, - dst: String, + meta: Box, completion: smol::channel::Sender>, }, SaveCommands { @@ -31,7 +40,102 @@ pub(super) enum StoreWrite { value: Option, }, ClearProfileSecrets(String), - Flush(smol::channel::Sender<()>), + /// Answered once every write queued before it has committed, or with the + /// first store failure since the writer started: a write that failed is + /// never certified by a later flush. + Flush(smol::channel::Sender>), +} + +/// What the writer reports back to the host. +pub(super) enum StoreWriteFailure { + Error(RuntimeError), + Warning(String), + /// The store stopped serving after a panic or a broken connection; the + /// failed writes are reported separately. + StoreFailed(String), +} + +/// One store write inside a batch: how its failure is reported, and who waits +/// for its commit. +struct Member { + failure: fn(String) -> RuntimeError, + completion: Option>>, +} + +#[derive(Default)] +struct Batch { + mutations: Vec, + members: Vec, + bytes: usize, +} + +impl StoreWrite { + /// The store changes a write makes, or `None` for a write that is not a + /// database change. + fn mutations(&self) -> Option, String>> { + Some(Ok(match self { + StoreWrite::AppendEvent { id, ts, event } => { + match Mutation::append_event(id, *ts, event) { + Ok(mutation) => vec![mutation], + Err(error) => return Some(Err(error.to_string())), + } + } + StoreWrite::UpsertMeta { meta, .. } => vec![Mutation::upsert_meta((**meta).clone())], + StoreWrite::UpsertProject(project) => vec![Mutation::upsert_project(project.clone())], + StoreWrite::RemoveSession(id) => vec![Mutation::remove_session(id)], + StoreWrite::RemoveProject(id) => vec![Mutation::remove_project(id)], + StoreWrite::Fork { src, meta, .. } => vec![ + Mutation::clone_events(src, &meta.id), + Mutation::upsert_meta((**meta).clone()), + ], + _ => return None, + })) + } + + fn member(self) -> Member { + let (failure, completion): (fn(String) -> RuntimeError, _) = match self { + StoreWrite::AppendEvent { .. } => (|error| RuntimeError::PersistEvent { error }, None), + StoreWrite::UpsertMeta { initial: true, .. } => { + (|error| RuntimeError::PersistSession { error }, None) + } + StoreWrite::UpsertMeta { initial: false, .. } => { + (|error| RuntimeError::PersistSessionIndex { error }, None) + } + StoreWrite::UpsertProject(_) => (|error| RuntimeError::PersistProject { error }, None), + StoreWrite::RemoveSession(_) => (|error| RuntimeError::DeleteSession { error }, None), + StoreWrite::RemoveProject(_) => (|error| RuntimeError::DeleteProject { error }, None), + StoreWrite::Fork { completion, .. } => ( + |error| RuntimeError::PersistSession { error }, + Some(completion), + ), + StoreWrite::Flush(completion) => ( + |error| RuntimeError::PersistEvent { error }, + Some(completion), + ), + StoreWrite::SaveCommands { .. } + | StoreWrite::WriteTerminalUi(_) + | StoreWrite::WriteSettings(_) + | StoreWrite::SetProfileSecret { .. } + | StoreWrite::ClearProfileSecrets(_) => { + (|error| RuntimeError::PersistSettings { error }, None) + } + }; + Member { + failure, + completion, + } + } + + /// Fail a write the writer never accepted: its waiter hears why, and + /// anything else is reported like a failed write. + pub(super) fn reject(self, reason: &str) -> Option { + let reports = !matches!(self, StoreWrite::Flush(_) | StoreWrite::Fork { .. }); + let member = self.member(); + if let Some(completion) = member.completion { + let _ = completion.try_send(Err(reason.to_owned())); + } + reports.then(|| StoreWriteFailure::Error((member.failure)(reason.to_owned()))) + } } pub(super) fn atomic_write(path: PathBuf, bytes: Vec) -> std::io::Result<()> { @@ -40,104 +144,177 @@ pub(super) fn atomic_write(path: PathBuf, bytes: Vec) -> std::io::Result<()> fs::rename(tmp, path) } -pub(super) fn run_store_write( - store: &SessionStore, - settings_store: &SettingsStore, - terminal_preferences_path: &Path, - write: StoreWrite, -) -> Option> { - match write { - StoreWrite::AppendEvent { id, ts, event } => { - store.append_event(&id, ts, &event).err().map(|err| { - Ok(RuntimeError::PersistEvent { - error: err.to_string(), - }) - }) +pub(super) struct StoreWriter { + pub(super) store: SessionStore, + pub(super) settings_store: SettingsStore, + pub(super) terminal_preferences_path: PathBuf, + pub(super) failures: smol::channel::Sender, + /// The first failed database write, until the writer stops. + unresolved: Option, +} + +impl StoreWriter { + pub(super) fn new( + store: SessionStore, + settings_store: SettingsStore, + terminal_preferences_path: PathBuf, + failures: smol::channel::Sender, + ) -> Self { + Self { + store, + settings_store, + terminal_preferences_path, + failures, + unresolved: None, + } + } + + /// Serve `writes` until every sender is gone and the queue is drained. + pub(super) fn run(mut self, writes: smol::channel::Receiver) { + while let Ok(first) = writes.recv_blocking() { + let mut queue = vec![first]; + while queue.len() < MAX_BATCH_WRITES { + match writes.try_recv() { + Ok(write) => queue.push(write), + Err(_) => break, + } + } + self.process(queue); } - StoreWrite::UpsertMeta { meta, initial } => store.upsert_meta(&meta).err().map(|err| { - if initial { - Ok(RuntimeError::PersistSession { - error: err.to_string(), - }) - } else { - Ok(RuntimeError::PersistSessionIndex { - error: err.to_string(), - }) + } + + /// Database writes accumulate into one transaction; any other write is an + /// ordered boundary that commits what came before it first. + fn process(&mut self, queue: Vec) { + let mut batch = Batch::default(); + for write in queue { + match write.mutations() { + Some(Ok(mutations)) => { + batch.bytes += mutations.iter().map(Mutation::payload_len).sum::(); + batch.mutations.extend(mutations); + batch.members.push(write.member()); + if batch.bytes >= MAX_BATCH_BYTES { + self.commit(&mut batch); + } + } + Some(Err(error)) => { + let member = write.member(); + self.unresolved.get_or_insert_with(|| error.clone()); + self.report(StoreWriteFailure::Error((member.failure)(error))); + } + None => { + self.commit(&mut batch); + if let Some(failure) = self.run_file_write(write) { + self.report(failure); + } + } } - }), - StoreWrite::UpsertProject(project) => store.upsert_project(&project).err().map(|err| { - Ok(RuntimeError::PersistProject { - error: err.to_string(), - }) - }), - StoreWrite::RemoveSession(id) => store.remove_session(&id).err().map(|err| { - Ok(RuntimeError::DeleteSession { - error: err.to_string(), - }) - }), - StoreWrite::RemoveProject(id) => store.remove_project(&id).err().map(|err| { - Ok(RuntimeError::DeleteProject { + } + self.commit(&mut batch); + } + + fn commit(&mut self, batch: &mut Batch) { + let Batch { + mutations, members, .. + } = std::mem::take(batch); + if members.is_empty() { + return; + } + match self.store.apply(&mutations) { + Ok(()) => { + for member in members { + if let Some(completion) = member.completion { + let _ = completion.try_send(Ok(())); + } + } + } + // The whole batch failed together; it is reported, not replayed. + Err(error) => { + let error = error.to_string(); + self.unresolved.get_or_insert_with(|| error.clone()); + for member in members { + match member.completion { + // The waiter reports its own failure. + Some(completion) => { + let _ = completion.try_send(Err(error.clone())); + } + None => { + self.report(StoreWriteFailure::Error((member.failure)(error.clone()))) + } + } + } + if self.store.is_failed() { + self.report(StoreWriteFailure::StoreFailed(error)); + } + } + } + } + + fn report(&self, failure: StoreWriteFailure) { + let _ = self.failures.send_blocking(failure); + } + + fn run_file_write(&mut self, write: StoreWrite) -> Option { + let settings_failure = |err: std::io::Error| { + StoreWriteFailure::Error(RuntimeError::PersistSettings { error: err.to_string(), }) - }), - StoreWrite::CloneEvents { - src, - dst, - completion, - } => { - let result = store - .clone_events(&src, &dst) - .map_err(|err| err.to_string()); - let _ = completion.try_send(result); - None - } - StoreWrite::SaveCommands { - provider, - acp_agent_id, - commands, - } => store - .save_commands(provider, acp_agent_id.as_deref(), &commands) - .err() - .map(|err| { - Err(format!( - "failed to persist {provider:?} command cache: {err}" - )) - }), - StoreWrite::WriteTerminalUi(bytes) => { - atomic_write(terminal_preferences_path.to_path_buf(), bytes) + }; + match write { + StoreWrite::SaveCommands { + provider, + acp_agent_id, + commands, + } => self + .store + .save_commands(provider, acp_agent_id.as_deref(), &commands) .err() - .map(|err| Err(format!("failed to persist terminal UI state: {err}"))) - } - StoreWrite::WriteSettings(bytes) => atomic_write(store.root().join("settings.json"), bytes) - .err() - .map(|err| { - Ok(RuntimeError::PersistSettings { - error: err.to_string(), - }) - }), - StoreWrite::SetProfileSecret { - profile_id, - key, - value, - } => settings_store - .set_profile_secret(&profile_id, &key, value.as_deref()) - .err() - .map(|err| { - Ok(RuntimeError::PersistSettings { - error: err.to_string(), - }) - }), - StoreWrite::ClearProfileSecrets(profile_id) => settings_store - .clear_profile_secrets(&profile_id) - .err() - .map(|err| { - Ok(RuntimeError::PersistSettings { - error: err.to_string(), - }) - }), - StoreWrite::Flush(completion) => { - let _ = completion.try_send(()); - None + .map(|err| { + StoreWriteFailure::Warning(format!( + "failed to persist {provider:?} command cache: {err}" + )) + }), + StoreWrite::WriteTerminalUi(bytes) => { + atomic_write(self.terminal_preferences_path.clone(), bytes) + .err() + .map(|err| { + StoreWriteFailure::Warning(format!( + "failed to persist terminal UI state: {err}" + )) + }) + } + StoreWrite::WriteSettings(bytes) => { + atomic_write(self.store.root().join("settings.json"), bytes) + .err() + .map(settings_failure) + } + StoreWrite::SetProfileSecret { + profile_id, + key, + value, + } => self + .settings_store + .set_profile_secret(&profile_id, &key, value.as_deref()) + .err() + .map(settings_failure), + StoreWrite::ClearProfileSecrets(profile_id) => self + .settings_store + .clear_profile_secrets(&profile_id) + .err() + .map(settings_failure), + StoreWrite::Flush(completion) => { + let _ = completion.try_send(match &self.unresolved { + None => Ok(()), + Some(error) => Err(format!("an earlier session-store write failed: {error}")), + }); + None + } + StoreWrite::AppendEvent { .. } + | StoreWrite::UpsertMeta { .. } + | StoreWrite::UpsertProject(_) + | StoreWrite::RemoveSession(_) + | StoreWrite::RemoveProject(_) + | StoreWrite::Fork { .. } => unreachable!("database writes are batched"), } } } diff --git a/crates/runtime/src/app/test_support.rs b/crates/runtime/src/app/test_support.rs index e4f8857a5..a05eda36c 100644 --- a/crates/runtime/src/app/test_support.rs +++ b/crates/runtime/src/app/test_support.rs @@ -241,7 +241,7 @@ impl std::ops::DerefMut for TestClientState { impl TestClientState { pub(super) fn new(store: SessionStore) -> Self { Self { - host: AppState::new(store), + host: AppState::new(store).unwrap(), selected: None, } } diff --git a/crates/runtime/src/app/tests.rs b/crates/runtime/src/app/tests.rs index 4707b0c6d..f9e587ea8 100644 --- a/crates/runtime/src/app/tests.rs +++ b/crates/runtime/src/app/tests.rs @@ -174,6 +174,7 @@ fn provider_native_subagent_events_create_and_feed_read_only_mirror_session() { let persisted = state .store .load_index() + .unwrap() .into_iter() .find(|meta| meta.id == mirror.id) .unwrap(); @@ -183,7 +184,7 @@ fn provider_native_subagent_events_create_and_feed_read_only_mirror_session() { serde_json::json!("low") ); assert_eq!(persisted.interaction_mode, InteractionMode::Plan); - let mirror_events = state.store.read_events(&mirror.id); + let mirror_events = state.store.read_events(&mirror.id).unwrap(); assert!(mirror_events.iter().any(|stored| matches!( &stored.event, AgentEvent::ItemCompleted(ThreadItem { @@ -192,7 +193,7 @@ fn provider_native_subagent_events_create_and_feed_read_only_mirror_session() { .. }) if id == "child-answer" ))); - let parent_events = state.store.read_events("parent"); + let parent_events = state.store.read_events("parent").unwrap(); assert!(parent_events.iter().any(|stored| matches!( &stored.event, AgentEvent::ItemCompleted(ThreadItem { @@ -311,7 +312,7 @@ fn assert_native_mirror_turn_lifecycle(evict: bool, late: bool, end: MirrorEnd, TurnStatus::Completed }; state.update(cx, |state, cx| { - let events = state.store.read_events(&mirror_id); + let events = state.store.read_events(&mirror_id).unwrap(); let mut open = false; let mut boundaries = 0; let mut items = 0; @@ -614,7 +615,7 @@ fn nested_subagent_items_open_a_titled_mirror_under_the_child_mirror_and_close_i }); cx.run_until_parked(); state.update(cx, |state, _| { - let events = state.store.read_events(&grandchild_id); + let events = state.store.read_events(&grandchild_id).unwrap(); assert!(matches!( &events.first().unwrap().event, AgentEvent::TurnStarted { turn_id } if turn_id == "toolu_grandchild" @@ -627,22 +628,18 @@ fn nested_subagent_items_open_a_titled_mirror_under_the_child_mirror_and_close_i } )); assert!(matches!( - &state.store.read_events(&second_id).last().unwrap().event, + &state.store.read_events(&second_id).unwrap().last().unwrap().event, AgentEvent::TurnCompleted { turn_id, status: TurnStatus::Interrupted, .. } if turn_id == "toolu_grandchild_2" )); - assert!( - state - .store - .read_events(&child_id) - .iter() - .all(|stored| match &stored.event { - AgentEvent::ItemStarted(item) - | AgentEvent::ItemUpdated(item) - | AgentEvent::ItemCompleted(item) => item.parent_item_id.is_none(), - _ => true, - }) - ); + assert!(state.store.read_events(&child_id).unwrap().iter().all( + |stored| match &stored.event { + AgentEvent::ItemStarted(item) + | AgentEvent::ItemUpdated(item) + | AgentEvent::ItemCompleted(item) => item.parent_item_id.is_none(), + _ => true, + } + )); }); } @@ -687,7 +684,7 @@ fn unknown_parent_mirror_closes_when_the_parent_session_closes() { }); cx.run_until_parked(); state.update(cx, |state, _| { - let events = state.store.read_events(&mirror_id); + let events = state.store.read_events(&mirror_id).unwrap(); assert!(matches!( &events.last().unwrap().event, AgentEvent::TurnCompleted { turn_id, status: TurnStatus::Interrupted, .. } @@ -753,7 +750,7 @@ fn loading_a_mirror_with_an_open_turn_and_no_live_parent_ends_it() { mirror.timeline.last_turn_status, Some(TurnStatus::Interrupted) ); - let events = state.store.read_events("mirror"); + let events = state.store.read_events("mirror").unwrap(); assert_eq!(events.len(), 3); assert!(matches!( &events[2].event, @@ -773,7 +770,7 @@ fn loading_a_mirror_with_an_open_turn_and_no_live_parent_ends_it() { }); cx.run_until_parked(); state.update(cx, |state, _| { - assert_eq!(state.store.read_events("mirror").len(), 3); + assert_eq!(state.store.read_events("mirror").unwrap().len(), 3); }); } @@ -1075,6 +1072,7 @@ fn scripted_provider_connects_command_launch_and_agent_event_paths() { !state .store .load_index() + .unwrap() .iter() .any(|meta| meta.id == session_id) ); @@ -1143,6 +1141,7 @@ fn scripted_provider_connects_command_launch_and_agent_event_paths() { state .store .load_index() + .unwrap() .into_iter() .find(|meta| meta.id == session_id) .unwrap(), @@ -1324,7 +1323,7 @@ fn archive_and_unarchive_apply_exact_timestamp_cascades() { ); }); cx.run_until_parked(); - let persisted = store.load_index(); + let persisted = store.load_index().unwrap(); assert!( persisted .iter() @@ -1666,7 +1665,11 @@ fn title_regeneration_uses_stored_history_and_preserves_intervening_changes() { .find(|meta| meta.id == id) .map(|meta| meta.title.as_str()); assert_eq!(title, expected, "{outcome}"); - let persisted = store.load_index().into_iter().find(|meta| meta.id == id); + let persisted = store + .load_index() + .unwrap() + .into_iter() + .find(|meta| meta.id == id); assert_eq!(persisted.as_ref().map(|meta| meta.title.as_str()), expected); if !matches!(outcome, "renamed" | "deleted") { assert_eq!(state.sessions[0].updated_at, 123, "{outcome}"); @@ -2416,7 +2419,7 @@ fn orchestrate_turn_records_context_and_runs_with_collaboration_disabled() { }); cx.run_until_parked(); state.update(cx, |state, _| { - let events = state.store.read_events("orchestrator"); + let events = state.store.read_events("orchestrator").unwrap(); let recorded = events .iter() .find_map(|stored| match &stored.event { @@ -2907,7 +2910,7 @@ fn draft_defaults_follow_project_history_then_global_history_without_persisting_ vec![] }; assert_eq!(draft.meta.option_selections, expected, "{name}"); - assert!(state.store.load_index().is_empty(), "{name}"); + assert!(state.store.load_index().unwrap().is_empty(), "{name}"); }); } } @@ -2952,7 +2955,7 @@ fn draft_model_selection_switches_to_the_rows_explicit_provider() { assert_eq!(draft.meta.model.as_deref(), Some("claude-fable-5-1")); assert!(draft.meta.acp_agent_id.is_none()); assert!(draft.meta.option_selections.is_empty()); - assert!(state.store.load_index().is_empty()); + assert!(state.store.load_index().unwrap().is_empty()); }); } @@ -4152,7 +4155,7 @@ fn steering_parked_orchestrator_callback_uses_recorded_id() { }); cx.run_until_parked(); state.update(cx, |state, _| { - let timeline = Timeline::fold_events(state.store.read_events("parent")); + let timeline = Timeline::fold_events(state.store.read_events("parent").unwrap()); assert!(timeline.entries.iter().any(|entry| matches!( &entry.content, EntryContent::Steer { @@ -5285,15 +5288,22 @@ fn unaccepted_send_survives_eof_and_is_delivered_once_after_resume() { let active = state.selected_session().unwrap(); assert_eq!(active.queue.len(), 1); assert_eq!(active.delivery_in_flight, Some(delivery_id)); - assert!(!state.store.read_events(&session_id).iter().any(|stored| { - matches!( - &stored.event, - AgentEvent::ItemCompleted(ThreadItem { - content: ItemContent::UserMessage { text, .. }, - .. - }) if text == "survive the eof race" - ) - })); + assert!( + !state + .store + .read_events(&session_id) + .unwrap() + .iter() + .any(|stored| { + matches!( + &stored.event, + AgentEvent::ItemCompleted(ThreadItem { + content: ItemContent::UserMessage { text, .. }, + .. + }) if text == "survive the eof race" + ) + }) + ); // EOF wins before the first actor writes, so no TurnAccepted exists. state.on_event( @@ -5343,6 +5353,7 @@ fn unaccepted_send_survives_eof_and_is_delivered_once_after_resume() { let delivered = state .store .read_events(&session_id) + .unwrap() .iter() .filter(|stored| { matches!( @@ -5449,7 +5460,7 @@ fn select_session_readopts_idle_resident_without_changing_recency_or_shutdown() assert!(actor.try_recv().is_err(), "re-adoption sent Shutdown"); }); cx.run_until_parked(); - assert_eq!(test_store.load_index()[0].updated_at, updated_at); + assert_eq!(test_store.load_index().unwrap()[0].updated_at, updated_at); } #[test] @@ -5666,7 +5677,7 @@ fn fork_thread_clones_timeline_and_provider_cursor() { serde_json::json!({"thread_id": "native-empty-source"}), )); store.upsert_meta(&empty_source).unwrap(); - assert!(!root.join(format!("{}.jsonl", empty_source.id)).exists()); + assert!(store.read_event_log(&empty_source.id).unwrap().is_empty()); let state = cx.new_entity(TestClientState::new(store)); state.update(cx, |state, cx| state.fork_thread(&source.id, cx)); @@ -5688,10 +5699,10 @@ fn fork_thread_clones_timeline_and_provider_cursor() { assert_eq!(fork.cwd, source.cwd); assert_eq!(fork.worktree, None); assert!(!active.timeline.turn_running); - assert_eq!(state.store.read_events(&fork.id).len(), 1); + assert_eq!(state.store.read_events(&fork.id).unwrap().len(), 1); assert_eq!( - std::fs::read(root.join(format!("{}.jsonl", fork.id))).unwrap(), - std::fs::read(root.join(format!("{}.jsonl", source.id))).unwrap() + state.store.read_event_log(&fork.id).unwrap(), + state.store.read_event_log(&source.id).unwrap() ); }); @@ -5718,11 +5729,18 @@ fn fork_thread_clones_timeline_and_provider_cursor() { state .store .load_index() + .unwrap() .iter() .any(|meta| meta.id == fork.id) ); - assert!(!root.join(format!("{}.jsonl", empty_source.id)).exists()); - assert!(!root.join(format!("{}.jsonl", fork.id)).exists()); + assert!( + state + .store + .read_event_log(&empty_source.id) + .unwrap() + .is_empty() + ); + assert!(state.store.read_event_log(&fork.id).unwrap().is_empty()); }); } @@ -5739,6 +5757,14 @@ fn store_writer_flush_persists_ordered_events_metadata_and_secrets_for_reopening state.record_event(&id, &persisted_assistant_event("first"), cx); state.set_profile_secret("profile", "ANTHROPIC_API_KEY", Some("writer-secret"), cx); state.record_event(&id, &persisted_assistant_event("second"), cx); + // More than one batch's worth, so the flush waits for several commits. + for index in 0..600 { + state.record_event( + &id, + &persisted_assistant_event(&format!("bulk {index}")), + cx, + ); + } }); cx.run_until_parked(); @@ -5746,13 +5772,14 @@ fn store_writer_flush_persists_ordered_events_metadata_and_secrets_for_reopening assert_eq!( fresh .load_index() + .unwrap() .iter() .find(|meta| meta.id == id) .unwrap() .title, "persisted by writer" ); - let events = fresh.read_events(&id); + let events = fresh.read_events(&id).unwrap(); let texts: Vec<_> = events .iter() .filter_map(|stored| match &stored.event { @@ -5763,7 +5790,9 @@ fn store_writer_flush_persists_ordered_events_metadata_and_secrets_for_reopening _ => None, }) .collect(); - assert_eq!(texts, ["first", "second"]); + let bulk: Vec<_> = (0..600).map(|index| format!("bulk {index}")).collect(); + assert_eq!(texts[..2], ["first", "second"]); + assert_eq!(texts[2..], bulk); let settings = SettingsStore::new(test_store.root().clone()); assert_eq!( settings @@ -5774,6 +5803,75 @@ fn store_writer_flush_persists_ordered_events_metadata_and_secrets_for_reopening ); } +/// A write the store cannot commit is reported once per write, every later +/// flush says the store lost something, and the in-memory history that holds +/// the unsaved records is kept instead of being released. +#[test] +fn failed_store_writes_are_reported_and_keep_the_unsaved_history() { + let cx = &mut TestAppContext::default(); + let test_store = TestStore::new("tcode-writer-failure"); + let state = cx.new_entity(TestClientState::new((*test_store).clone())); + let (commands, _receiver) = smol::channel::unbounded(); + let parked = live_session(ProviderKind::Codex, commands); + let id = parked.meta.id.clone(); + state.update(cx, |state, cx| { + state.residents.parked.insert(id.clone(), parked); + state.record_event(&id, &persisted_assistant_event("saved"), cx); + }); + cx.run_until_parked(); + cx.drain_outgoing(); + + // The database stops accepting writes underneath the host. + test_store.close().unwrap(); + state.update(cx, |state, cx| { + for text in ["unsaved one", "unsaved two"] { + state.record_event(&id, &persisted_assistant_event(text), cx); + } + }); + cx.run_until_parked(); + let failures = cx + .drain_outgoing() + .into_iter() + .filter(|message| { + matches!( + message, + HostMessage::Event(EventEnvelope { + event: ServerEvent::Runtime(RuntimeEvent::Error( + RuntimeError::PersistEvent { .. } + )), + .. + }) + ) + }) + .count(); + assert_eq!(failures, 2); + + // The session leaves residency; its log is the only copy of the unsaved + // records, so the release barrier's failure keeps it. + state.update(cx, |state, cx| { + state.residents.parked.remove(&id); + state.release_stale_session_logs(cx); + }); + cx.run_until_parked(); + let barrier = state.update(cx, |state, cx| state.store_write_barrier(cx)); + cx.run_until_parked(); + assert!(barrier.try_recv().unwrap().is_err()); + state.read(|state| { + let texts: Vec<_> = state.event_records[&id] + .records() + .iter() + .filter_map(|record| match &record.event { + AgentEvent::ItemCompleted(ThreadItem { + content: ItemContent::AssistantMessage { text }, + .. + }) => Some(text.as_str()), + _ => None, + }) + .collect(); + assert_eq!(texts, ["saved", "unsaved one", "unsaved two"]); + }); +} + /// A project's unsent draft is keyed by the project, so deleting the project /// drops the draft's parked terminal workspace and its persisted terminal /// preferences instead of leaving them behind forever. @@ -6210,7 +6308,7 @@ fn stop_then_new_thread_keeps_the_first_message_visible() { cx.run_until_parked(); state.update(cx, |state, _| { // And it is durable: a replay of the JSONL shows the same thing. - let replayed = Timeline::fold_events(state.store.read_events(&id_b)); + let replayed = Timeline::fold_events(state.store.read_events(&id_b).unwrap()); assert!(replayed.entries.iter().any( |e| matches!(&e.content, EntryContent::Item(ItemContent::UserMessage { text, .. }) if text == "second message") )); @@ -6232,7 +6330,7 @@ fn submitted_queue_head_cannot_leak_delivery_after_turn_completion() { state.update(cx, |state, cx| { state.store.upsert_meta(&session.meta).unwrap(); - state.sessions = state.store.load_index(); + state.sessions = state.store.load_index().unwrap(); state.install_selected(session); state.send_turn(&id, "finish this task".into(), Vec::new(), cx); let delivery_id = match commands.try_recv() { @@ -6321,6 +6419,7 @@ fn event_stream_end_closes_only_its_own_live_provider_in_any_residency() { cx.run_until_parked(); let closes = store .read_events("session") + .unwrap() .iter() .filter(|stored| matches!(stored.event, AgentEvent::SessionClosed { .. })) .count(); @@ -6431,7 +6530,7 @@ fn switching_threads_parks_a_working_session_instead_of_killing_it() { // A live session with a running turn (the overnight workflow). state.store.upsert_meta(&session.meta).unwrap(); - state.sessions = state.store.load_index(); + state.sessions = state.store.load_index().unwrap(); state.install_selected(session); state.send_turn(&id_a, "run the long migration".into(), Vec::new(), cx); state.send_turn(&id_a, "queued follow-up".into(), Vec::new(), cx); @@ -6559,7 +6658,7 @@ fn drained_parked_session_stays_resident() { state.update(cx, |state, cx| { state.store.upsert_meta(&session.meta).unwrap(); - state.sessions = state.store.load_index(); + state.sessions = state.store.load_index().unwrap(); state.install_selected(session); state.send_turn(&id, "one last thing".into(), Vec::new(), cx); let delivery_id = match commands.try_recv() { @@ -7410,7 +7509,7 @@ fn persist_streamed_turns(store: &SessionStore, id: &str, turns: u64) -> Vec> 7; + seed ^= seed << 17; + seed % range + }; + let (mut checkpointed_runs, mut acknowledged, mut unacknowledged) = (0, 0, 0); + for run in 0..runs { + let root = std::env::temp_dir().join(format!("tcode-crash-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root).unwrap(); + let mut child = tcode_services::process::command(std::env::current_exe().unwrap()) + .args([ + "--exact", + "app::tests::store_writer_crash_child", + "--ignored", + "--nocapture", + ]) + .env(CRASH_CHILD_DIR, &root) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let stdout = std::io::BufReader::new(child.stdout.take().unwrap()); + let (lines, received) = std::sync::mpsc::channel(); + let reader = std::thread::spawn(move || { + for line in stdout.lines().map_while(Result::ok) { + if let Some((_, report)) = line.split_once(CRASH_ACK) { + let _ = lines.send(report.to_owned()); + } + } + }); + let session = received + .recv_timeout(Duration::from_secs(30)) + .unwrap() + .strip_prefix("session ") + .unwrap() + .to_owned(); + std::thread::sleep(Duration::from_millis(200 + random(2_800))); + child.kill().unwrap(); + child.wait().unwrap(); + reader.join().unwrap(); + let acked = received + .try_iter() + .filter_map(|report| report.parse::().ok()) + .max(); + + // Preserve the files exactly as the kill left them, before anything + // opens them again. + let preserved = root.join("preserved"); + std::fs::create_dir_all(&preserved).unwrap(); + for name in ["tcode.db", "tcode.db-wal"] { + if root.join(name).exists() { + std::fs::copy(root.join(name), preserved.join(name)).unwrap(); + } + } + let main_file = std::fs::metadata(root.join("tcode.db")).unwrap().len(); + // A fresh database is a handful of pages; anything larger was + // written there by a checkpoint while records streamed. + if main_file > 64 * 1024 { + checkpointed_runs += 1; + } + + let store = SessionStore::open_at(root.clone()).unwrap(); + let log = store.read_event_log(&session).unwrap(); + let rows = log + .split_inclusive(|byte| *byte == b'\n') + .collect::>(); + assert!( + rows.iter().all(|row| row.ends_with(b"\n")), + "run {run}: torn row" + ); + let events = store.read_events(&session).unwrap(); + assert_eq!(events.len(), rows.len(), "run {run}: a row does not parse"); + let indices: Vec = events + .iter() + .map(|stored| match &stored.event { + AgentEvent::ItemCompleted(ThreadItem { + content: ItemContent::AssistantMessage { text }, + .. + }) => text.split(' ').next().unwrap().parse().unwrap(), + other => panic!("run {run}: unexpected record {other:?}"), + }) + .collect(); + let expected: Vec = (0..indices.len() as u64).collect(); + assert_eq!( + indices, expected, + "run {run}: records are not a gapless prefix" + ); + if let Some(acked) = acked { + assert!( + indices.len() as u64 > acked, + "run {run}: acknowledged record {acked} lost ({} survived)", + indices.len() + ); + acknowledged += acked + 1; + unacknowledged += indices.len() as u64 - (acked + 1); + } + store.close().unwrap(); + drop(store); + + let check = tcode_services::process::command("sqlite3") + .arg(preserved.join("tcode.db")) + .arg("PRAGMA integrity_check;") + .output() + .unwrap(); + assert_eq!( + String::from_utf8_lossy(&check.stdout).trim(), + "ok", + "run {run}: sqlite3 integrity_check: {}", + String::from_utf8_lossy(&check.stderr) + ); + println!( + "run {run}: acked {acked:?}, survived {}, main file {main_file} bytes", + indices.len() + ); + std::fs::remove_dir_all(&root).unwrap(); + } + println!( + "{runs} kills: {acknowledged} acknowledged records all survived, {unacknowledged} \ + committed but unacknowledged, {checkpointed_runs} runs crossed an auto-checkpoint" + ); + assert!(checkpointed_runs > 0, "no run crossed an auto-checkpoint"); +} diff --git a/crates/runtime/src/pipe.rs b/crates/runtime/src/pipe.rs index 1f90717fc..49a37a1aa 100644 --- a/crates/runtime/src/pipe.rs +++ b/crates/runtime/src/pipe.rs @@ -120,7 +120,13 @@ pub fn spawn_host(store: SessionStore, mut services: HostServices) -> std::io::R std::thread::Builder::new() .name("tcode-host".into()) .spawn(move || { - let mut state = AppState::with_ai_titles(store, services.ai_title_generation); + let mut state = match AppState::with_ai_titles(store, services.ai_title_generation) { + Ok(state) => state, + Err(error) => { + let _ = ready_tx.send(Err(error)); + return; + } + }; if let Some((url, tokens)) = preview_registration { state.attach_preview_mcp(url, tokens); } @@ -143,8 +149,11 @@ pub fn spawn_host(store: SessionStore, mut services: HostServices) -> std::io::R state.refresh_provider_status(&mut cx); } state.sync_terminal_handles(); - let _ = ready_tx.send(()); - smol::block_on(host_loop(state, cx, client_rx, mailbox_rx)); + let _ = ready_tx.send(Ok(())); + let mut state = smol::block_on(host_loop(state, cx, client_rx, mailbox_rx)); + if let Err(error) = state.close_store() { + log::error!("could not close the session store cleanly: {error}"); + } let _ = stopped_tx.send_blocking(()); })?; ready_rx.recv().map_err(|error| { @@ -152,7 +161,7 @@ pub fn spawn_host(store: SessionStore, mut services: HostServices) -> std::io::R std::io::ErrorKind::BrokenPipe, format!("host failed during startup: {error}"), ) - })?; + })??; Ok(SpawnedHost { to_host: client_tx, @@ -169,7 +178,7 @@ async fn host_loop( mut cx: HostCx, client: smol::channel::Receiver, mailbox: smol::channel::Receiver, -) { +) -> AppState { let mut domain_diff = DomainDiff::new(&state); loop { enum Input { @@ -200,6 +209,7 @@ async fn host_loop( state.reap_terminal_projections(); domain_diff.emit_changes(&state, &mut cx); } + state } fn malformed_message_id(line: &str) -> Option { @@ -247,14 +257,17 @@ pub(crate) fn handle_client_message(state: &mut AppState, cx: &mut HostCx, messa CommandOutcome::StoreBarrier(barrier) => { let response_cx = cx.clone(); cx.spawn_detached(async move { - let result = barrier - .recv() - .await - .map(|()| CommandResponse::Unit) - .map_err(|error| ProtocolError { + let result = match barrier.recv().await { + Ok(Ok(())) => Ok(CommandResponse::Unit), + Ok(Err(message)) => Err(ProtocolError { + code: "store_flush_failed".into(), + message, + }), + Err(error) => Err(ProtocolError { code: "store_barrier_closed".into(), message: error.to_string(), - }); + }), + }; response_cx.send_message(HostMessage::Ack { id, result }); }); } @@ -293,7 +306,7 @@ pub(crate) fn handle_client_message(state: &mut AppState, cx: &mut HostCx, messa enum CommandOutcome { Immediate(Result), - StoreBarrier(smol::channel::Receiver<()>), + StoreBarrier(smol::channel::Receiver>), } fn dispatch_command(app: &mut AppState, cx: &mut HostCx, command: Command) -> CommandOutcome { @@ -1016,6 +1029,8 @@ mod tests { .expect("append event"); store.upsert_meta(&meta).expect("write meta"); let event_log = store.read_event_log(&meta.id).expect("read event log"); + let index = store.load_index().expect("read index"); + let reader = store.clone(); let host = spawn_host(store, HostServices::default()).expect("spawn host"); let before = tree_snapshot(&data_root); @@ -1058,6 +1073,8 @@ mod tests { before, "rendering an export must not write anything on the host" ); + assert_eq!(reader.read_event_log(&meta.id).unwrap(), event_log); + assert_eq!(reader.load_index().unwrap(), index); let missing = smol::block_on(link.query(Query::RenderThreadExport { session_id: "nope".into(), @@ -1219,6 +1236,15 @@ mod tests { }; for entry in entries.flatten() { let path = entry.path(); + // The live store's own files are checked through the store + // above; Windows locks them while the host runs, so they + // cannot be read here. + if matches!( + entry.file_name().to_str(), + Some("tcode.db" | "tcode.db-wal" | "tcode.db-shm" | "tcode.lock") + ) { + continue; + } match entry.metadata() { Ok(metadata) if metadata.is_dir() => walk(&path, out), Ok(_) => { @@ -1259,6 +1285,9 @@ mod tests { .to_string(), ) .unwrap(); + store + .migrate(|_| {}, &std::sync::atomic::AtomicBool::new(false)) + .unwrap(); let host = spawn_host(store.clone(), HostServices::default()).unwrap(); let link = host.link(); link.subscribe(Subscription { @@ -1316,13 +1345,15 @@ mod tests { let first = project.icon_path.clone().unwrap(); assert!(first.starts_with(root.join("project-icons"))); host.shutdown_blocking().unwrap(); + // Each restart opens the data dir afresh, as a new process would. + let reopen = || SessionStore::open_at(root.clone()).unwrap(); assert_eq!( - store.read_file().projects[0].icon_path.as_ref(), + reopen().read_file().unwrap().projects[0].icon_path.as_ref(), Some(&first) ); // Removing the source must not break the saved custom copy. std::fs::remove_file(&logo).unwrap(); - let host = spawn_host(store.clone(), HostServices::default()).unwrap(); + let host = spawn_host(reopen(), HostServices::default()).unwrap(); let link = host.link(); assert!(matches!( smol::block_on(link.query(Query::ReadProjectIcon { @@ -1338,11 +1369,14 @@ mod tests { }) .unwrap(); host.shutdown_blocking().unwrap(); - let second = store.read_file().projects[0].icon_path.clone().unwrap(); + let second = reopen().read_file().unwrap().projects[0] + .icon_path + .clone() + .unwrap(); assert_ne!(first, second); assert!(!first.exists()); std::fs::write(&logo, include_bytes!("../../../assets/icons/app/tcode.png")).unwrap(); - let host = spawn_host(store.clone(), HostServices::default()).unwrap(); + let host = spawn_host(reopen(), HostServices::default()).unwrap(); let link = host.link(); link.command_blocking(Command::SetProjectIcon { project_id: "p".into(), @@ -1358,12 +1392,198 @@ mod tests { QueryResponse::FileBytes(_) )); host.shutdown_blocking().unwrap(); - assert!(store.read_file().projects[0].icon_path.is_none()); + assert!( + reopen().read_file().unwrap().projects[0] + .icon_path + .is_none() + ); assert!(!second.exists()); assert!(logo.exists()); std::fs::remove_dir_all(root).unwrap(); } + const LEGACY_START_DIR: &str = "TCODE_LEGACY_START_DIR"; + const LEGACY_START_REPORT: &str = "legacy start: "; + + fn legacy_events() -> [(Option, agent::AgentEvent); 5] { + use agent::{AgentEvent, ItemContent, ThreadItem, TurnStatus}; + + let item = |id: &str, content| { + AgentEvent::ItemCompleted(ThreadItem { + id: id.into(), + parent_item_id: None, + content, + }) + }; + [ + ( + None, + AgentEvent::TurnStarted { + turn_id: "t1".into(), + }, + ), + ( + None, + item( + "user", + ItemContent::UserMessage { + text: "What changed?".into(), + context_len: None, + attachments: vec![], + }, + ), + ), + ( + Some(1_000), + item( + "assistant", + ItemContent::AssistantMessage { + text: "The parser.".into(), + }, + ), + ), + ( + Some(2_000), + AgentEvent::TurnCompleted { + turn_id: "t1".into(), + status: TurnStatus::Completed, + usage: None, + }, + ), + ( + Some(3_000), + AgentEvent::TurnStarted { + turn_id: "t2".into(), + }, + ), + ] + } + + /// A host started on a data dir an older build wrote moves the JSON index + /// and the JSONL logs into its database once, and serves every thread's + /// timeline exactly as the logs fold, before and after a restart. + /// + /// Each start is its own process, as the app's are. A restart inside one + /// process can find the data dir still locked: a stopped host's detached + /// work (here a git status refresh) may be inside a spawn, and the child's + /// copy of the process's descriptors holds the lock until it execs. + #[test] + fn legacy_files_migrated_at_startup_serve_the_same_timelines() { + let data_root = + std::env::temp_dir().join(format!("tcode-host-migration-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&data_root).unwrap(); + let events = legacy_events(); + let line = |(ts, event): &(Option, agent::AgentEvent)| match ts { + None => serde_json::to_string(event).unwrap(), + Some(ts) => serde_json::json!({"ts": ts, "event": event}).to_string(), + }; + // Legacy bare events, a CRLF line, a blank and an unparseable line, + // and a last line without its newline. + let log = format!( + "{}\n{}\n\n{{torn\n{}\r\n{}\n{}", + line(&events[0]), + line(&events[1]), + line(&events[2]), + line(&events[3]), + line(&events[4]), + ); + std::fs::write(data_root.join("legacy-thread.jsonl"), &log).unwrap(); + std::fs::write( + data_root.join("sessions.json"), + serde_json::json!({ + "projects": [{"id": "p", "name": "Project", "root": "/work", "created_at": 1}], + "sessions": [{ + "id": "legacy-thread", "title": "Legacy", "provider": "codex", + "cwd": "/work", "project_id": "p", "created_at": 1, "updated_at": 2 + }] + }) + .to_string(), + ) + .unwrap(); + + for start in ["migrating", "restarted"] { + let output = tcode_services::process::command(std::env::current_exe().unwrap()) + .args([ + "--exact", + "pipe::tests::legacy_files_start_process", + "--ignored", + "--nocapture", + ]) + .env(LEGACY_START_DIR, &data_root) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "{start}: {stdout}{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + stdout.contains(&format!("{LEGACY_START_REPORT}{start}")), + "{start}: {stdout}" + ); + assert!(!data_root.join("sessions.json").exists(), "{start}"); + assert!(!data_root.join("legacy-thread.jsonl").exists(), "{start}"); + assert_eq!( + std::fs::read_to_string(data_root.join("legacy/legacy-thread.jsonl")).unwrap(), + log, + "{start}" + ); + } + std::fs::remove_dir_all(data_root).unwrap(); + } + + /// One start of the app: migrate if an older build's files are there, + /// serve the legacy thread's timeline, and stop. + #[test] + #[ignore = "a start of legacy_files_migrated_at_startup_serve_the_same_timelines, which runs it"] + fn legacy_files_start_process() { + use tcode_core::session::{StoredEvent, Timeline}; + + let data_root = std::path::PathBuf::from(std::env::var_os(LEGACY_START_DIR).unwrap()); + let store = SessionStore::open_at(data_root).unwrap(); + let migrating = store.needs_migration().unwrap(); + if migrating { + store + .migrate(|_| {}, &std::sync::atomic::AtomicBool::new(false)) + .unwrap(); + } + let host = spawn_host(store, HostServices::default()).unwrap(); + let link = host.link(); + let stream = link.events(); + let topic = Topic::SessionEvents { + session_id: "legacy-thread".into(), + }; + link.subscribe(Subscription { + after: None, + topic: topic.clone(), + }) + .unwrap(); + let snapshot = next_event(&stream, |event| { + event.topic == topic && matches!(event.event, ServerEvent::SessionSnapshot { .. }) + }); + let ServerEvent::SessionSnapshot { records, total, .. } = snapshot.event else { + unreachable!("filtered to snapshots") + }; + assert_eq!(total, 5); + assert_eq!( + format!("{:?}", Timeline::fold_events(records)), + format!( + "{:?}", + Timeline::fold_events(legacy_events().into_iter().map(|(ts, event)| { + StoredEvent { + ts, + event, + elided: None, + } + })) + ) + ); + host.shutdown_blocking().unwrap(); + let start = if migrating { "migrating" } else { "restarted" }; + println!("{LEGACY_START_REPORT}{start}"); + } + #[test] fn dedicated_host_round_trips_commands_queries_events_and_quit_barrier() { let data_root = diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index 28b627235..c5a04bc33 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -14,12 +14,16 @@ base64 = "0.23" chrono = "0.4" dirs = "7" flate2 = "1" +futures-lite = "2" log = "0.4" image = { version = "0.25.10", default-features = false, features = ["png", "jpeg", "gif", "webp", "bmp", "tiff", "ico"] } serde = { version = "1", features = ["derive"] } serde_json = "1" smol = "2" tar = "0.4" +# Default features add a global allocator (mimalloc) and full-text search +# (tantivy, zstd); the store needs neither. +turso = { version = "=0.8.1", default-features = false } ureq = { version = "2", default-features = false, features = ["tls", "gzip"] } zip = { version = "2", default-features = false, features = ["deflate-flate2", "flate2"] } diff --git a/crates/services/src/export.rs b/crates/services/src/export.rs index 98f09d5e4..e69aa1a62 100644 --- a/crates/services/src/export.rs +++ b/crates/services/src/export.rs @@ -392,7 +392,7 @@ mod tests { ImportOutcome::Imported ); - let restored = destination.load_index().pop().unwrap(); + let restored = destination.load_index().unwrap().pop().unwrap(); assert_ne!(restored.id, meta.id); assert_eq!(restored.cwd, project.root); assert_eq!(restored.project_id.as_deref(), Some(project.id.as_str())); @@ -404,8 +404,9 @@ mod tests { destination.read_event_log(&restored.id).unwrap(), source.read_event_log(&meta.id).unwrap() ); - let source_timeline = Timeline::fold_events(source.read_events(&meta.id)); - let restored_timeline = Timeline::fold_events(destination.read_events(&restored.id)); + let source_timeline = Timeline::fold_events(source.read_events(&meta.id).unwrap()); + let restored_timeline = + Timeline::fold_events(destination.read_events(&restored.id).unwrap()); let summarize = |timeline: &Timeline| { timeline .entries diff --git a/crates/services/src/import/mod.rs b/crates/services/src/import/mod.rs index 59a33c448..2954c6938 100644 --- a/crates/services/src/import/mod.rs +++ b/crates/services/src/import/mod.rs @@ -11,7 +11,7 @@ use agent::{AgentEvent, ProviderKind, ResumeCursor, ThreadItem, TurnStatus}; use serde_json::json; use crate::export::{ReadExportError, TcodeThreadExport, read_tcode_export}; -use crate::store::SessionStore; +use crate::store::{Mutation, SessionStore}; use tcode_core::project::{Project, SessionMeta}; pub use tcode_protocol::{ExternalThread, RecentDir, SourceTool}; @@ -233,12 +233,19 @@ pub fn import_thread( )); } + // The transcript and its meta commit together, so a failure never leaves + // an orphan log behind. + let mut mutations = Vec::with_capacity(events.len() + 1); for (ts, event) in &events { - if let Err(err) = store.append_event(&meta.id, *ts, event) { - return ImportOutcome::Failed(format!("failed to write imported events: {err}")); + match Mutation::append_event(&meta.id, *ts, event) { + Ok(mutation) => mutations.push(mutation), + Err(err) => { + return ImportOutcome::Failed(format!("failed to encode imported events: {err}")); + } } } - if let Err(err) = store.upsert_meta(&meta) { + mutations.push(Mutation::upsert_meta(meta)); + if let Err(err) = store.apply(&mutations) { return ImportOutcome::Failed(format!("failed to write imported session: {err}")); } existing.insert(converted.external_id); @@ -269,10 +276,11 @@ fn import_tcode_thread( meta.parent_session_id = None; meta.imported_from = Some(external_id.clone()); - if let Err(error) = store.write_event_log(&meta.id, &export.event_log) { - return ImportOutcome::Failed(format!("failed to write imported events: {error}")); - } - if let Err(error) = store.upsert_meta(&meta) { + let mutations = [ + Mutation::replace_event_log(&meta.id, export.event_log), + Mutation::upsert_meta(meta), + ]; + if let Err(error) = store.apply(&mutations) { return ImportOutcome::Failed(format!("failed to write imported session: {error}")); } existing.insert(external_id); diff --git a/crates/services/src/import/tests.rs b/crates/services/src/import/tests.rs index f5f4f3875..e93dff8f0 100644 --- a/crates/services/src/import/tests.rs +++ b/crates/services/src/import/tests.rs @@ -267,7 +267,7 @@ fn import_is_idempotent_and_replays_into_timeline() { title_hint: None, last_active_ms: 0, }; - let mut existing = existing_external_ids(&store.load_index()); + let mut existing = existing_external_ids(&store.load_index().unwrap()); assert_eq!( import_thread(&store, &project, &thread, &mut existing), ImportOutcome::Imported @@ -277,7 +277,7 @@ fn import_is_idempotent_and_replays_into_timeline() { ImportOutcome::SkippedDuplicate ); - let index = store.load_index(); + let index = store.load_index().unwrap(); assert_eq!(index.len(), 1); let meta = &index[0]; assert_eq!(meta.project_id.as_deref(), Some("project-1")); @@ -286,7 +286,7 @@ fn import_is_idempotent_and_replays_into_timeline() { meta.resume_cursor.as_ref().unwrap().0["session_id"], "session-1" ); - let timeline = Timeline::fold_events(store.read_events(&meta.id)); + let timeline = Timeline::fold_events(store.read_events(&meta.id).unwrap()); assert!(timeline.entries.iter().any( |entry| matches!(&entry.content, EntryContent::Item(ItemContent::UserMessage { text, .. }) if text == "Imported question") )); @@ -295,12 +295,12 @@ fn import_is_idempotent_and_replays_into_timeline() { let mut imported = meta.clone(); imported.resume_cursor = None; store.upsert_meta(&imported).unwrap(); - let mut existing = existing_external_ids(&store.load_index()); + let mut existing = existing_external_ids(&store.load_index().unwrap()); assert_eq!( import_thread(&store, &project, &thread, &mut existing), ImportOutcome::SkippedDuplicate ); - assert_eq!(store.load_index().len(), 1); + assert_eq!(store.load_index().unwrap().len(), 1); for (index, (provider, source, external_id, cursor, lines)) in [ (ProviderKind::ClaudeCode, SourceTool::ClaudeCode, "claude:native-claude", @@ -321,9 +321,9 @@ fn import_is_idempotent_and_replays_into_timeline() { let thread = ExternalThread { source, file, external_id: external_id.into(), title_hint: None, last_active_ms: 0, }; - let mut existing = existing_external_ids(&store.load_index()); + let mut existing = existing_external_ids(&store.load_index().unwrap()); assert_eq!(import_thread(&store, &project, &thread, &mut existing), ImportOutcome::SkippedDuplicate, "{external_id}"); - assert_eq!(store.load_index().len(), index + 2); + assert_eq!(store.load_index().unwrap().len(), index + 2); } } diff --git a/crates/services/src/relaunch.rs b/crates/services/src/relaunch.rs index cce8a3420..ed1f49edc 100644 --- a/crates/services/src/relaunch.rs +++ b/crates/services/src/relaunch.rs @@ -72,6 +72,12 @@ pub fn clear(data_dir: &Path) -> std::io::Result<()> { } } +/// Whether a marker is waiting, without consuming it: the next instance is +/// already starting while the one that wrote it is still closing. +pub fn pending(data_dir: &Path) -> bool { + marker_path(data_dir).is_file() +} + /// Read the marker and delete it (consume-once). Returns `None` when absent or /// unparsable; the file is removed either way so a corrupt marker can't wedge /// every future launch into a relaunch loop. diff --git a/crates/services/src/session_search.rs b/crates/services/src/session_search.rs index 73bd30491..5a4a656bd 100644 --- a/crates/services/src/session_search.rs +++ b/crates/services/src/session_search.rs @@ -1,12 +1,11 @@ //! Lazy full-text search over persisted session event logs. //! -//! The index is deliberately in-memory: each session is parsed only when its -//! JSONL file's length or modification time changes. This keeps the append-only -//! persistence format authoritative and avoids a second on-disk database. +//! The index is deliberately in-memory: each session is parsed again only after +//! the store reports a committed change to its event log +//! ([`SessionStore::event_generation`]). The store stays authoritative and the +//! cache dies with it. use std::collections::{HashMap, HashSet}; -use std::fs; -use std::time::SystemTime; use agent::ItemContent; use tcode_core::project::SessionMeta; @@ -23,19 +22,13 @@ pub struct SearchableEntry { pub text: String, } -#[derive(Debug, Clone, PartialEq, Eq)] -struct FileFingerprint { - len: u64, - modified: Option, -} - #[derive(Debug, Clone)] struct CachedSession { - fingerprint: FileFingerprint, + generation: u64, entries: Vec, } -/// Incremental, file-freshness-based content index for one [`SessionStore`]. +/// Incremental content index for one [`SessionStore`]. pub struct SessionSearch { store: SessionStore, cache: HashMap, @@ -91,34 +84,29 @@ impl SessionSearch { } fn refresh(&mut self, meta: &SessionMeta) { - let path = self.store.root().join(format!("{}.jsonl", meta.id)); - let fingerprint = match fs::metadata(path) { - Ok(metadata) => FileFingerprint { - len: metadata.len(), - modified: metadata.modified().ok(), - }, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => FileFingerprint { - len: 0, - modified: None, - }, - Err(error) => { - log::warn!("cannot inspect session log {}: {error}", meta.id); - return; - } - }; + // Read before the events: a write that commits during the read moves + // the generation on, so the next search parses again. + let generation = self.store.event_generation(&meta.id); if self .cache .get(&meta.id) - .is_some_and(|cached| cached.fingerprint == fingerprint) + .is_some_and(|cached| cached.generation == generation) { return; } - let entries = extract_searchable_entries(&self.store.read_events(&meta.id)); + let events = match self.store.read_events(&meta.id) { + Ok(events) => events, + Err(error) => { + log::warn!("cannot search session {}: {error}", meta.id); + self.cache.remove(&meta.id); + return; + } + }; self.cache.insert( meta.id.clone(), CachedSession { - fingerprint, - entries, + generation, + entries: extract_searchable_entries(&events), }, ); } @@ -249,6 +237,8 @@ mod tests { use serde_json::json; use tcode_core::project::SessionMeta; + use crate::store::Mutation; + use super::*; fn completed(id: &str, content: ItemContent) -> StoredEvent { @@ -380,8 +370,18 @@ mod tests { assert_eq!(updated.len(), 5); assert_eq!(updated[4].entry_id, "next"); assert_eq!(updated[4].turn, 1); - fs::remove_file(root.join(format!("{}.jsonl", meta.id))).unwrap(); + // A replaced log of the same shape (line count, length) is still seen + // as changed. + let replaced = String::from_utf8(store.read_event_log(&meta.id).unwrap()) + .unwrap() + .replace("auth.rs", "xyzw.rs"); + store + .apply(&[Mutation::replace_event_log(&meta.id, replaced.into_bytes())]) + .unwrap(); assert!(search.search(sessions, "auth.rs", 10).is_empty()); + assert_eq!(search.search(sessions, "xyzw.rs", 10).len(), 5); + store.remove_session(&meta.id).unwrap(); + assert!(search.search(sessions, "xyzw.rs", 10).is_empty()); for (text, query, expected) in [ (" 中文\nCAFÉ\t😀 ", " café ", Some("中文 CAFÉ 😀")), ("İstanbul", "i\u{307}stan", Some("İstanbul")), @@ -430,6 +430,6 @@ mod tests { assert!(hits[0].snippet.contains("AUTH.rs")); assert!(hits[0].snippet.chars().count() <= 142); assert!(hits[0].snippet.starts_with('…') && hits[0].snippet.ends_with('…')); - fs::remove_dir_all(root).unwrap(); + std::fs::remove_dir_all(root).unwrap(); } } diff --git a/crates/services/src/store.rs b/crates/services/src/store.rs deleted file mode 100644 index 3e9498d14..000000000 --- a/crates/services/src/store.rs +++ /dev/null @@ -1,661 +0,0 @@ -//! On-disk persistence for tcode sessions. -//! -//! Layout (under the platform data dir, e.g. `~/Library/Application Support/tcode/`): -//! * `sessions.json` — an [`IndexFile`] containing projects and sessions. -//! * `.jsonl` — append-only `{ ts, event }` records. -//! -//! Replay accepts timestamped records and legacy bare [`AgentEvent`] lines, -//! then folds [`StoredEvent`]s into a [`tcode_core::session::Timeline`]. - -use std::fs::{self, File, OpenOptions}; -use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; -use std::path::PathBuf; - -use agent::{AgentEvent, ModelSpec, ProviderCommand, ProviderKind}; -use serde::{Deserialize, Serialize}; - -use tcode_core::project::{IndexFile, Project, SessionMeta, migrate_index}; -use tcode_core::session::StoredEvent; - -/// On-disk envelope wrapping each event with its record time. Kept private: -/// callers deal in [`StoredEvent`] (which tolerates the legacy bare form). -#[derive(Serialize, Deserialize)] -struct EventEnvelope { - ts: u64, - event: AgentEvent, -} - -/// Cheap, cloneable handle to the on-disk data directory. -#[derive(Debug, Clone)] -pub struct SessionStore { - root: PathBuf, - /// Full event-log parses, shared by every clone of this store so tests can - /// prove a caller served history from memory. - #[cfg(any(test, feature = "test-support"))] - event_reads: std::sync::Arc, -} - -impl SessionStore { - /// Open (creating if needed) the store under the platform data dir, or under - /// `TCODE_DATA_DIR` when it is set — which gives a throwaway profile (its own - /// sessions, settings and installed ACP agents) for demos and screenshots. - pub fn open_default() -> std::io::Result { - let root = match std::env::var_os("TCODE_DATA_DIR") { - Some(dir) => PathBuf::from(dir), - None => dirs::data_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join("tcode"), - }; - Self::open_at(root) - } - - pub fn open_at(root: PathBuf) -> std::io::Result { - fs::create_dir_all(&root)?; - Ok(Self { - root, - #[cfg(any(test, feature = "test-support"))] - event_reads: Default::default(), - }) - } - - /// How many times [`SessionStore::read_events`] parsed a log through this - /// store or any of its clones. - #[cfg(any(test, feature = "test-support"))] - pub fn event_reads(&self) -> usize { - self.event_reads.load(std::sync::atomic::Ordering::Relaxed) - } - - pub fn root(&self) -> &PathBuf { - &self.root - } - - fn index_path(&self) -> PathBuf { - self.root.join("sessions.json") - } - - fn events_path(&self, id: &str) -> PathBuf { - self.root.join(format!("{id}.jsonl")) - } - - /// Read the native event-log bytes without parsing or normalizing them. - pub fn read_event_log(&self, id: &str) -> std::io::Result> { - match fs::read(self.events_path(id)) { - Ok(bytes) => Ok(bytes), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), - Err(error) => Err(error), - } - } - - /// Atomically install a complete native event log for a session. - pub fn write_event_log(&self, id: &str, bytes: &[u8]) -> std::io::Result<()> { - let destination = self.events_path(id); - let temporary = destination.with_extension("jsonl.tmp"); - fs::write(&temporary, bytes)?; - fs::rename(temporary, destination) - } - - fn models_path(&self, provider: ProviderKind) -> PathBuf { - let name = match provider { - ProviderKind::Codex => "codex", - ProviderKind::ClaudeCode => "claude", - ProviderKind::Pi => "pi", - ProviderKind::OpenCode => "opencode", - ProviderKind::Cursor => "cursor", - ProviderKind::Grok => "grok", - // ACP agents publish their models over the wire at session start - // (`AgentEvent::ProviderOptions`), so there is no catalog to cache. - ProviderKind::Acp => "acp", - }; - self.root.join(format!("models-{name}.json")) - } - - fn commands_path(&self, provider: ProviderKind, acp_agent_id: Option<&str>) -> Option { - let name = match provider { - ProviderKind::Codex => "codex".to_string(), - ProviderKind::ClaudeCode => "claude".to_string(), - ProviderKind::Pi => "pi".to_string(), - ProviderKind::OpenCode => "opencode".to_string(), - ProviderKind::Cursor => "cursor".to_string(), - ProviderKind::Grok => "grok".to_string(), - ProviderKind::Acp => { - let id = acp_agent_id?; - // Registry ids are external input and may contain path separators. - // Hex keeps the filename reversible and collision-free without - // allowing an id to escape the data directory. - let encoded = id - .as_bytes() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::(); - format!("acp-{encoded}") - } - }; - Some(self.root.join(format!("commands-{name}.json"))) - } - - /// Load the last-fetched model catalog for `provider` so the picker is - /// instant offline. Empty when never fetched / unreadable. - pub fn load_models(&self, provider: ProviderKind) -> Vec { - let Ok(bytes) = fs::read(self.models_path(provider)) else { - return Vec::new(); - }; - serde_json::from_slice(&bytes).unwrap_or_default() - } - - /// Persist the freshly fetched model catalog for `provider`. - pub fn save_models(&self, provider: ProviderKind, models: &[ModelSpec]) -> std::io::Result<()> { - let path = self.models_path(provider); - let tmp = path.with_extension("json.tmp"); - let data = serde_json::to_vec_pretty(models) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - fs::write(&tmp, data)?; - fs::rename(&tmp, path) - } - - /// Load the most recently reported command/skill list for a native provider - /// or one specific ACP agent. Empty when missing, unreadable, or when an ACP - /// agent id was not supplied. - pub fn load_commands( - &self, - provider: ProviderKind, - acp_agent_id: Option<&str>, - ) -> Vec { - let Some(path) = self.commands_path(provider, acp_agent_id) else { - return Vec::new(); - }; - let Ok(bytes) = fs::read(path) else { - return Vec::new(); - }; - serde_json::from_slice(&bytes).unwrap_or_default() - } - - /// Atomically persist the complete command/skill list reported by a native - /// provider or one specific ACP agent. Empty lists are meaningful: they - /// replace a stale non-empty cache. - pub fn save_commands( - &self, - provider: ProviderKind, - acp_agent_id: Option<&str>, - commands: &[ProviderCommand], - ) -> std::io::Result<()> { - let path = self.commands_path(provider, acp_agent_id).ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "ACP command cache requires an agent id", - ) - })?; - let tmp = path.with_extension("json.tmp"); - let data = serde_json::to_vec_pretty(commands) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - fs::write(&tmp, data)?; - fs::rename(&tmp, path) - } - - /// Load the whole index file (projects + sessions), tolerating the old - /// bare-array schema and deriving implicit projects for orphan sessions. - pub fn read_file(&self) -> IndexFile { - let path = self.index_path(); - let Ok(bytes) = fs::read(&path) else { - return IndexFile::default(); - }; - // Current schema is an object; the legacy schema was a bare array. - let parsed = serde_json::from_slice::(&bytes).or_else(|_| { - serde_json::from_slice::>(&bytes).map(|sessions| IndexFile { - projects: Vec::new(), - sessions, - }) - }); - match parsed { - Ok(file) => migrate_index(file), - Err(err) => { - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or(0); - let corrupt_path = self.root.join(format!("sessions.json.corrupt-{timestamp}")); - match fs::rename(&path, &corrupt_path) { - Ok(()) => log::warn!( - "failed to parse sessions.json: {err}; preserved it as {}", - corrupt_path.display() - ), - Err(rename_err) => log::warn!( - "failed to parse sessions.json: {err}; failed to preserve corrupt index: {rename_err}" - ), - } - IndexFile::default() - } - } - } - - /// Load the session index (newest first). Empty if missing / unreadable. - pub fn load_index(&self) -> Vec { - let mut metas = self.read_file().sessions; - metas.sort_by_key(|b| std::cmp::Reverse(b.updated_at)); - metas - } - - /// Insert or replace a meta in the index (by id), then persist. - pub fn upsert_meta(&self, meta: &SessionMeta) -> std::io::Result<()> { - let mut file = self.read_file(); - if let Some(existing) = file.sessions.iter_mut().find(|m| m.id == meta.id) { - *existing = meta.clone(); - } else { - file.sessions.push(meta.clone()); - } - self.persist_index(&file) - } - - /// Insert or replace a project, then remove its previous managed icon. - pub fn upsert_project(&self, project: &Project) -> std::io::Result<()> { - let mut file = self.read_file(); - let mut previous = None; - if let Some(existing) = file.projects.iter_mut().find(|p| p.id == project.id) { - if existing.icon_path != project.icon_path { - previous = existing.icon_path.clone(); - } - *existing = project.clone(); - } else { - file.projects.push(project.clone()); - } - self.persist_index(&file)?; - self.remove_project_icon(previous); - Ok(()) - } - - /// Remove a project and its managed icon. Sessions are removed separately. - pub fn remove_project(&self, id: &str) -> std::io::Result<()> { - let mut file = self.read_file(); - let icon = file - .projects - .iter() - .find(|p| p.id == id) - .and_then(|p| p.icon_path.clone()); - file.projects.retain(|project| project.id != id); - self.persist_index(&file)?; - self.remove_project_icon(icon); - Ok(()) - } - - fn remove_project_icon(&self, path: Option) { - if let Some(path) = path - && path.parent() == Some(self.root.join("project-icons").as_path()) - && let Err(error) = fs::remove_file(&path) - && error.kind() != std::io::ErrorKind::NotFound - { - log::warn!("could not remove project icon {}: {error}", path.display()); - } - } - - /// Persist a whole index file atomically (also flushes migration on startup). - pub fn persist_index(&self, file: &IndexFile) -> std::io::Result<()> { - let tmp = self.index_path().with_extension("json.tmp"); - let data = serde_json::to_vec_pretty(file) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - fs::write(&tmp, data)?; - fs::rename(&tmp, self.index_path()) - } - - /// Append one event to the session's JSONL log, wrapped in a timestamped - /// envelope (`{"ts": , "event": {…}}`). - pub fn append_event(&self, id: &str, ts: u64, event: &AgentEvent) -> std::io::Result<()> { - let envelope = EventEnvelope { - ts, - event: event.clone(), - }; - let line = serde_json::to_string(&envelope) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - let mut file: File = OpenOptions::new() - .create(true) - .read(true) - .append(true) - .open(self.events_path(id))?; - let len = file.metadata()?.len(); - if len > 0 { - file.seek(SeekFrom::End(-1))?; - let mut last = [0_u8; 1]; - file.read_exact(&mut last)?; - if last[0] != b'\n' { - file.write_all(b"\n")?; - } - } - file.write_all(line.as_bytes())?; - file.write_all(b"\n") - } - - /// Read and parse every persisted event for a session (skipping bad lines). - /// - /// Each line is tolerantly parsed as either a timestamped envelope - /// (`{"ts":…,"event":…}`) or a legacy bare event (`{"type":…}`), so logs - /// written before the envelope format still replay (with `ts == None`). - pub fn read_events(&self, id: &str) -> Vec { - #[cfg(any(test, feature = "test-support"))] - self.event_reads - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let path = self.events_path(id); - let Ok(file) = File::open(&path) else { - return Vec::new(); - }; - let mut events = Vec::new(); - for line in BufReader::new(file).lines() { - let Ok(line) = line else { break }; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - match parse_stored_line(trimmed) { - Ok(stored) => events.push(stored), - Err(err) => log::warn!("skipping unparseable event in {id}.jsonl: {err}"), - } - } - events - } - - /// Atomically clone one session's append-only event log. A missing source - /// is an empty transcript and therefore succeeds without creating a file. - pub fn clone_events(&self, src_id: &str, dst_id: &str) -> std::io::Result<()> { - let src = self.events_path(src_id); - let data = match fs::read(src) { - Ok(data) => data, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(err) => return Err(err), - }; - let dst = self.events_path(dst_id); - let tmp = dst.with_extension("jsonl.tmp"); - fs::write(&tmp, data)?; - fs::rename(tmp, dst) - } - - /// Remove a session from the index and delete its event log. - pub fn remove_session(&self, id: &str) -> std::io::Result<()> { - let mut file = self.read_file(); - file.sessions.retain(|meta| meta.id != id); - self.persist_index(&file)?; - match fs::remove_file(self.events_path(id)) { - Ok(()) => Ok(()), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(err) => Err(err), - } - } -} - -/// Parse one JSONL line into a [`StoredEvent`], accepting both the timestamped -/// envelope and the legacy bare-event form. Envelope is tried first; a bare -/// event lacks the `ts`/`event` keys so it can't masquerade as one, and an -/// envelope lacks the top-level `type` tag so it can't parse as a bare event. -pub(crate) fn parse_stored_line(line: &str) -> Result { - match serde_json::from_str::(line) { - Ok(envelope) => Ok(StoredEvent { - ts: Some(envelope.ts), - event: envelope.event, - elided: None, - }), - Err(_envelope_err) => match serde_json::from_str::(line) { - Ok(event) => Ok(StoredEvent { - ts: None, - event, - elided: None, - }), - // Both forms failed: the line is genuinely corrupt. The bare-event - // error is the more informative one (the envelope attempt always - // fails on a bare event merely because `ts` is missing). - Err(bare_err) => Err(bare_err), - }, - } -} - -pub use tcode_core::project::now_secs; - -/// Current wall-clock time in unix milliseconds (used for event envelopes). -pub fn now_millis() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - use agent::{ProviderCommandKind, TurnStatus}; - - fn temp_root() -> PathBuf { - let mut p = std::env::temp_dir(); - p.push(format!("tcode-store-test-{}", uuid::Uuid::new_v4())); - p - } - - #[test] - fn removing_project_cleans_only_managed_icons_after_persisting() { - let root = - std::env::temp_dir().join(format!("tcode-icon-cleanup-{}", uuid::Uuid::new_v4())); - let store = SessionStore::open_at(root.clone()).unwrap(); - fs::create_dir(root.join("project-icons")).unwrap(); - let mut project = Project::from_root(root.join("project")); - for (path, managed) in [ - (root.join("project-icons/icon.png"), true), - (root.join("original.png"), false), - ] { - fs::write(&path, b"stored image").unwrap(); - project.icon_path = Some(path.clone()); - store.upsert_project(&project).unwrap(); - // A failed index replacement must retain the image still referenced on disk. - fs::create_dir(root.join("sessions.json.tmp")).unwrap(); - assert!(store.remove_project(&project.id).is_err()); - assert!(path.exists()); - assert_eq!( - store.read_file().projects[0].icon_path.as_ref(), - Some(&path) - ); - fs::remove_dir(root.join("sessions.json.tmp")).unwrap(); - store.remove_project(&project.id).unwrap(); - assert!(store.read_file().projects.is_empty()); - assert_eq!(path.exists(), !managed); - } - fs::remove_dir_all(root).unwrap(); - } - - #[test] - fn session_index_upserts_orders_and_removes_only_the_selected_session() { - let store = SessionStore::open_at(temp_root()).unwrap(); - let mut a = SessionMeta::new(ProviderKind::Codex, PathBuf::from("/a"), None); - a.updated_at = 100; - let mut b = SessionMeta::new(ProviderKind::ClaudeCode, PathBuf::from("/b"), None); - b.updated_at = 200; - store.upsert_meta(&a).unwrap(); - store.upsert_meta(&b).unwrap(); - - let index = store.load_index(); - assert_eq!(index.len(), 2); - // newest first - assert_eq!(index[0].id, b.id); - assert_eq!(index[1].id, a.id); - - // upsert replaces - let mut a2 = a.clone(); - a2.title = "renamed".into(); - store.upsert_meta(&a2).unwrap(); - let index = store.load_index(); - assert_eq!(index.len(), 2); - assert_eq!( - index.iter().find(|m| m.id == a.id).unwrap().title, - "renamed" - ); - store - .append_event( - &a.id, - 1, - &AgentEvent::TurnStarted { - turn_id: "turn-a".into(), - }, - ) - .unwrap(); - store - .append_event( - &b.id, - 2, - &AgentEvent::TurnStarted { - turn_id: "turn-b".into(), - }, - ) - .unwrap(); - store.remove_session(&a.id).unwrap(); - store.remove_session(&a.id).unwrap(); - let remaining = store.load_index(); - assert_eq!(remaining.len(), 1); - assert_eq!(remaining[0].id, b.id); - assert!(!store.events_path(&a.id).exists()); - assert!(store.events_path(&b.id).exists()); - let _ = fs::remove_dir_all(store.root()); - } - - #[test] - fn command_cache_roundtrips_per_provider_and_acp_agent() { - let root = temp_root(); - let store = SessionStore::open_at(root.clone()).unwrap(); - let native = vec![ProviderCommand { - name: "review".into(), - description: Some("Review the current changes".into()), - kind: ProviderCommandKind::Command, - }]; - let acp = vec![ProviderCommand { - name: "browser".into(), - description: None, - kind: ProviderCommandKind::Skill, - }]; - store - .save_commands(ProviderKind::ClaudeCode, None, &native) - .unwrap(); - store - .save_commands(ProviderKind::Acp, Some("vendor/agent"), &acp) - .unwrap(); - - // Reopen the store to prove the values come from disk, not memory. - let reopened = SessionStore::open_at(root.clone()).unwrap(); - assert_eq!( - reopened.load_commands(ProviderKind::ClaudeCode, None), - native - ); - assert_eq!( - reopened.load_commands(ProviderKind::Acp, Some("vendor/agent")), - acp - ); - assert!( - reopened - .load_commands(ProviderKind::Acp, Some("different-agent")) - .is_empty() - ); - assert!(root.join("commands-claude.json").is_file()); - assert!( - root.join("commands-acp-76656e646f722f6167656e74.json") - .is_file() - ); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn append_recovers_a_mixed_legacy_log_and_writes_the_current_envelope() { - let store = SessionStore::open_at(temp_root()).unwrap(); - let id = "mixed"; - fs::write(store.events_path(id), concat!( - "{\"type\":\"turn_started\",\"turn_id\":\"legacy\"}\n", - "{\"ts\":2000,\"event\":{\"type\":\"turn_completed\",\"turn_id\":\"legacy\",\"status\":\"completed\",\"usage\":null}}\n", - "\n{not valid json}\n{\"type\":\"turn_started" - )).unwrap(); - store - .append_event( - id, - 3000, - &AgentEvent::TurnStarted { - turn_id: "next".into(), - }, - ) - .unwrap(); - let reopened = SessionStore::open_at(store.root().clone()).unwrap(); - let events = reopened.read_events(id); - assert_eq!(events.len(), 3); - assert_eq!( - events.iter().map(|event| event.ts).collect::>(), - [None, Some(2000), Some(3000)] - ); - assert!( - matches!(&events[0].event, AgentEvent::TurnStarted { turn_id } if turn_id == "legacy") - ); - assert!( - matches!(&events[1].event, AgentEvent::TurnCompleted { turn_id, status: TurnStatus::Completed, .. } if turn_id == "legacy") - ); - assert!( - matches!(&events[2].event, AgentEvent::TurnStarted { turn_id } if turn_id == "next") - ); - let raw = fs::read_to_string(store.events_path(id)).unwrap(); - assert_eq!( - raw.lines().last().unwrap(), - r#"{"ts":3000,"event":{"type":"turn_started","turn_id":"next"}}"# - ); - fs::remove_dir_all(store.root()).unwrap(); - } - - #[test] - fn corrupt_index_is_preserved_before_returning_empty() { - let store = SessionStore::open_at(temp_root()).unwrap(); - let corrupt_bytes = b"not valid session json"; - fs::write(store.index_path(), corrupt_bytes).unwrap(); - - assert!(store.load_index().is_empty()); - assert!(!store.index_path().exists()); - let backups: Vec<_> = fs::read_dir(store.root()) - .unwrap() - .filter_map(Result::ok) - .filter(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with("sessions.json.corrupt-") - }) - .collect(); - assert_eq!(backups.len(), 1); - assert_eq!(fs::read(backups[0].path()).unwrap(), corrupt_bytes); - let _ = fs::remove_dir_all(store.root()); - } - - #[test] - fn legacy_bare_array_index_loads_and_derives_projects() { - let store = SessionStore::open_at(temp_root()).unwrap(); - // Old-format file: a bare JSON array with no project_id fields. - let legacy = serde_json::json!([ - { - "id": "s1", "title": "One", "provider": "claude_code", - "cwd": "/work/alpha", "created_at": 1, "updated_at": 10 - }, - { - "id": "s2", "title": "Two", "provider": "codex", - "cwd": "/work/alpha", "created_at": 2, "updated_at": 20 - }, - { - "id": "s3", "title": "Three", "provider": "codex", - "cwd": "/work/beta", "created_at": 3, "updated_at": 30 - } - ]); - fs::write(store.index_path(), legacy.to_string()).unwrap(); - - let file = store.read_file(); - // Two distinct roots -> two derived projects, deduped by root. - assert_eq!(file.projects.len(), 2); - let alpha = file - .projects - .iter() - .find(|p| p.root == std::path::Path::new("/work/alpha")) - .unwrap(); - assert_eq!(alpha.name, "alpha"); - // Both alpha sessions share the same derived project. - let s1 = file.sessions.iter().find(|s| s.id == "s1").unwrap(); - let s2 = file.sessions.iter().find(|s| s.id == "s2").unwrap(); - assert_eq!(s1.project_id, Some(alpha.id.clone())); - assert_eq!(s2.project_id, s1.project_id); - let s3 = file.sessions.iter().find(|s| s.id == "s3").unwrap(); - assert_ne!(s3.project_id, s1.project_id); - let migrated_again = migrate_index(file.clone()); - assert_eq!(migrated_again.projects, file.projects); - assert_eq!(migrated_again.sessions, file.sessions); - let _ = fs::remove_dir_all(store.root()); - } -} diff --git a/crates/services/src/store/db.rs b/crates/services/src/store/db.rs new file mode 100644 index 000000000..a50c38da8 --- /dev/null +++ b/crates/services/src/store/db.rs @@ -0,0 +1,371 @@ +//! The Turso database file behind [`super::SessionStore`]: opening it, the +//! single writer connection, reader connections, and the checkpoint that +//! leaves the main file self-contained. +//! +//! Turso's API is async but drives its own I/O, so every call is run to +//! completion with `block_on` on the calling (blocking) thread. + +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use futures_lite::future::block_on; +use turso::{Builder, Connection, Database, Row, Value}; + +/// `PRAGMA user_version` of a completed store. A staged database keeps 0 +/// until its last write, so a file at the final name with 0 was never +/// finished, and a larger value was written by a newer Tcode. +pub(super) const SCHEMA_VERSION: i64 = 1; + +const SCHEMA: &str = " +CREATE TABLE projects (id TEXT PRIMARY KEY, body BLOB NOT NULL); +CREATE TABLE sessions (id TEXT PRIMARY KEY, body BLOB NOT NULL); +CREATE TABLE events ( + session_id TEXT NOT NULL, + position INTEGER NOT NULL, + line BLOB NOT NULL, + PRIMARY KEY (session_id, position) +); +"; + +/// A failure after which the connection or database can no longer be +/// trusted; the store stops serving every handle once it sees one. +#[derive(Debug)] +pub(super) struct Broken(pub(super) String); + +impl std::fmt::Display for Broken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for Broken {} + +pub(super) fn is_broken(error: &io::Error) -> bool { + error.get_ref().is_some_and(|inner| inner.is::()) +} + +pub(super) struct Db { + path: PathBuf, + database: Database, + /// Every write goes through this one connection, one transaction at a + /// time. Turso's `busy_timeout` spins the CPU, so a second writer is never + /// allowed to contend instead. + writer: Mutex, + /// Idle reader connections. Readers see the last committed snapshot and + /// never wait for the writer. + readers: Mutex>, +} + +impl Db { + /// Open an existing database file, or create an empty one with the schema + /// when `create` is set (only ever on the staging path). + pub(super) fn open(path: &Path, create: bool) -> io::Result { + if !create && !path.is_file() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("{} does not exist", path.display()), + )); + } + let location = path.to_str().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a UTF-8 path", path.display()), + ) + })?; + let database = block_on(Builder::new_local(location).build()) + .map_err(|error| open_error(path, error))?; + let writer = database + .connect() + .map_err(|error| sql_error(path, "connect", error))?; + let db = Self { + path: path.to_path_buf(), + database, + writer: Mutex::new(writer), + readers: Mutex::new(Vec::new()), + }; + { + let writer = db.writer()?; + // FULL syncs the WAL on every commit; on Apple platforms only + // F_FULLFSYNC (`fullfsync`) also flushes the drive's write cache, + // so a committed batch survives power loss. Both are read back + // because Turso accepts and ignores values it does not implement. + db.execute(&writer, "PRAGMA synchronous = FULL", ())?; + db.execute(&writer, "PRAGMA fullfsync = ON", ())?; + let synchronous = db.query_integer(&writer, "PRAGMA synchronous")?; + if synchronous != Some(2) { + return Err(io::Error::other(format!( + "{}: PRAGMA synchronous is {synchronous:?}, expected FULL", + path.display() + ))); + } + #[cfg(target_vendor = "apple")] + if db.query_integer(&writer, "PRAGMA fullfsync")? != Some(1) { + return Err(io::Error::other(format!( + "{}: PRAGMA fullfsync did not take effect", + path.display() + ))); + } + if create { + block_on(writer.execute_batch(SCHEMA)) + .map_err(|error| sql_error(path, "create schema", error))?; + } + } + Ok(db) + } + + fn writer(&self) -> io::Result> { + self.writer + .lock() + .map_err(|_| io::Error::other(Broken("the store writer panicked".into()))) + } + + /// Run `body` in one IMMEDIATE transaction on the writer connection and + /// commit it. Any failure rolls back explicitly; a connection whose + /// rollback cannot be confirmed is reported [`Broken`] and never reused. + pub(super) fn write( + &self, + body: impl FnOnce(&Self, &Connection) -> io::Result, + ) -> io::Result { + let writer = self.writer()?; + self.execute(&writer, "BEGIN IMMEDIATE", ())?; + let result = body(self, &writer) + .and_then(|value| self.execute(&writer, "COMMIT", ()).map(|_| value)); + if let Err(error) = result { + self.roll_back(&writer)?; + return Err(error); + } + result + } + + fn roll_back(&self, writer: &Connection) -> io::Result<()> { + let autocommit = |connection: &Connection| { + connection + .is_autocommit() + .map_err(|error| sql_error(&self.path, "inspect transaction", error)) + }; + if !autocommit(writer)? { + block_on(writer.execute("ROLLBACK", ())).map_err(|error| { + io::Error::other(Broken(format!( + "{}: rollback failed: {error}", + self.path.display() + ))) + })?; + } + if !autocommit(writer)? { + return Err(io::Error::other(Broken(format!( + "{}: a transaction is still open after rollback", + self.path.display() + )))); + } + Ok(()) + } + + /// Run `body` on an idle reader connection. A connection that saw an + /// error is dropped rather than returned to the pool. + pub(super) fn read( + &self, + body: impl FnOnce(&Self, &Connection) -> io::Result, + ) -> io::Result { + let pooled = self + .readers + .lock() + .ok() + .and_then(|mut readers| readers.pop()); + let connection = match pooled { + Some(connection) => connection, + None => self + .database + .connect() + .map_err(|error| sql_error(&self.path, "connect", error))?, + }; + let result = body(self, &connection); + if result.is_ok() + && let Ok(mut readers) = self.readers.lock() + { + readers.push(connection); + } + result + } + + /// Execute one statement to completion; its statement is finished before + /// this returns, so the next statement never meets an active one. + pub(super) fn execute( + &self, + connection: &Connection, + sql: &str, + params: impl turso::IntoParams, + ) -> io::Result { + block_on(async { + // Cached: migration and batches run the same few statements for + // every line. + let mut statement = connection.prepare_cached(sql).await?; + statement.execute(params).await + }) + .map_err(|error| sql_error(&self.path, sql, error)) + } + + /// Run a query and hand every row to `each`, draining the statement so it + /// is finished before this returns. + pub(super) fn query( + &self, + connection: &Connection, + sql: &str, + params: impl turso::IntoParams, + mut each: impl FnMut(&Row) -> io::Result<()>, + ) -> io::Result<()> { + block_on(async { + let mut rows = connection + .prepare_cached(sql) + .await + .map_err(|error| sql_error(&self.path, sql, error))? + .query(params) + .await + .map_err(|error| sql_error(&self.path, sql, error))?; + while let Some(row) = rows + .next() + .await + .map_err(|error| sql_error(&self.path, sql, error))? + { + each(&row)?; + } + Ok(()) + }) + } + + fn query_integer(&self, connection: &Connection, sql: &str) -> io::Result> { + let mut value = None; + self.query(connection, sql, (), |row| { + value = Some(integer(row, 0)?); + Ok(()) + })?; + Ok(value) + } + + pub(super) fn user_version(&self) -> io::Result { + let writer = self.writer()?; + Ok(self + .query_integer(&writer, "PRAGMA user_version")? + .unwrap_or(0)) + } + + /// Copy every committed frame into the main file and truncate the WAL, + /// failing unless both are confirmed: after this the main file alone is + /// the database. + pub(super) fn checkpoint(&self) -> io::Result<()> { + let writer = self.writer()?; + let mut busy = None; + self.query(&writer, "PRAGMA wal_checkpoint(TRUNCATE)", (), |row| { + busy = Some(integer(row, 0)?); + Ok(()) + })?; + if busy != Some(0) { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "{}: checkpoint did not complete (busy = {busy:?})", + self.path.display() + ), + )); + } + let wal = wal_path(&self.path); + match std::fs::metadata(&wal) { + Ok(metadata) if metadata.len() > 0 => Err(io::Error::other(format!( + "{} still holds {} bytes after a truncating checkpoint", + wal.display(), + metadata.len() + ))), + Ok(_) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } + } +} + +pub(super) fn wal_path(path: &Path) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push("-wal"); + PathBuf::from(name) +} + +pub(super) fn integer(row: &Row, index: usize) -> io::Result { + match row.get_value(index) { + Ok(Value::Integer(value)) => Ok(value), + Ok(other) => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected an integer column, found {other:?}"), + )), + Err(error) => Err(io::Error::new( + io::ErrorKind::InvalidData, + error.to_string(), + )), + } +} + +pub(super) fn blob(row: &Row, index: usize) -> io::Result> { + match row.get_value(index) { + Ok(Value::Blob(bytes)) => Ok(bytes), + Ok(other) => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected a blob column, found {other:?}"), + )), + Err(error) => Err(io::Error::new( + io::ErrorKind::InvalidData, + error.to_string(), + )), + } +} + +pub(super) fn text(row: &Row, index: usize) -> io::Result { + match row.get_value(index) { + Ok(Value::Text(text)) => Ok(text), + Ok(other) => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected a text column, found {other:?}"), + )), + Err(error) => Err(io::Error::new( + io::ErrorKind::InvalidData, + error.to_string(), + )), + } +} + +fn open_error(path: &Path, error: turso::Error) -> io::Error { + let message = error.to_string(); + // Turso holds an exclusive lock on the file for as long as a process has + // it open; the message is the only signal its error carries. + if message.contains("locked by another process") { + return io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "{} is in use by another Tcode process: {message}", + path.display() + ), + ); + } + match error { + turso::Error::IoError(kind, operation) => io::Error::new( + kind, + format!("could not open {} ({operation}): {message}", path.display()), + ), + _ => io::Error::new( + io::ErrorKind::InvalidData, + format!( + "could not open the session database {}: {message}. The file is kept as it \ + is; it can be examined with the sqlite3 command-line tool.", + path.display() + ), + ), + } +} + +pub(super) fn sql_error(path: &Path, operation: &str, error: turso::Error) -> io::Error { + let kind = match &error { + turso::Error::IoError(kind, _) => *kind, + turso::Error::DatabaseFull(_) => io::ErrorKind::StorageFull, + turso::Error::Busy(_) | turso::Error::BusySnapshot(_) => io::ErrorKind::ResourceBusy, + turso::Error::Corrupt(_) | turso::Error::NotAdb(_) => io::ErrorKind::InvalidData, + _ => io::ErrorKind::Other, + }; + io::Error::new(kind, format!("{}: {operation}: {error}", path.display())) +} diff --git a/crates/services/src/store/migrate.rs b/crates/services/src/store/migrate.rs new file mode 100644 index 000000000..aaa99107a --- /dev/null +++ b/crates/services/src/store/migrate.rs @@ -0,0 +1,749 @@ +//! Creating `tcode.db`, once: from the JSON index (`sessions.json`) and the +//! per-thread JSONL logs older builds wrote, or empty on a fresh install. +//! +//! The database is built and verified under a staging name and published by +//! a rename, so a file at the final name is always complete: +//! +//! 1. `tcode.db.migrating` is created with the schema and `user_version` 0, +//! and the index is imported. +//! 2. Every `*.jsonl` in the data dir (orphans included) is imported in +//! bounded transactions. +//! 3. Each log is read back and compared byte for byte with its file, then +//! the metadata field for field; `user_version` is set to 1, the WAL is +//! checkpointed and truncated, every handle is dropped, and a fresh open +//! re-counts what was committed. +//! 4. The staged file is synced, the list of sources to archive is written to +//! `tcode.db.archive`, and the staged file is renamed to `tcode.db`; the +//! directory is synced around both. +//! 5. The sources move into `legacy/`, and the archive list is removed. +//! +//! Nothing before step 4's rename writes, renames or removes a source: the +//! steps before it only read them. A migration cancelled or failed before +//! the rename, or a start after one interrupted there, discards the staging +//! files and leaves no `tcode.db`, so the next run begins again from the +//! untouched sources. Cancellation is honoured only up to the rename; after +//! it the next start finishes step 5 from the list. + +use std::collections::HashSet; +use std::fs::{self, File}; +use std::io::{self, BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; + +use tcode_core::project::{IndexFile, SessionMeta, migrate_index}; + +use super::db::{Db, SCHEMA_VERSION, blob, integer, wal_path}; +use super::{DB_FILE, insert_segments, read_index}; + +const STAGING_FILE: &str = "tcode.db.migrating"; +const ARCHIVE_LIST: &str = "tcode.db.archive"; +const LEGACY_INDEX: &str = "sessions.json"; +const LEGACY_DIR: &str = "legacy"; +/// A thread's log is committed in transactions of about this size; one +/// transaction for a multi-hundred-megabyte log holds every dirty page in +/// memory and doubles the WAL. +pub(super) const CHUNK_BYTES: usize = 8 << 20; +/// At most this many rows, and about [`CHUNK_BYTES`], are compared per read +/// when verifying, so a reader never pins the WAL or memory for a whole log. +const VERIFY_ROWS: i64 = 4096; + +/// Where a running migration is. Each of the importing and verifying phases +/// goes through every log once, so their counts start again from zero; +/// publishing and archiving report everything done. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MigrationProgress { + pub phase: MigrationPhase, + pub threads_done: usize, + pub threads_total: usize, + pub bytes_done: u64, + pub bytes_total: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MigrationPhase { + /// Listing the logs and importing the index; the totals are not known + /// until it ends. + Scanning, + Importing, + Verifying, + /// Past the last point a cancellation is honoured. + Publishing, + Archiving, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Migration { + /// `tcode.db` is complete and the sources are in `legacy/`. + Completed, + /// The staging files are gone, no `tcode.db` exists and every source is + /// where it was. + Cancelled, +} + +/// Whether the data dir holds an older build's files and no `tcode.db`. +pub(super) fn needed(root: &Path) -> io::Result { + Ok(!root.join(DB_FILE).exists() && !legacy_sources(root)?.is_empty()) +} + +/// Make sure a complete `tcode.db` exists for opening, creating an empty one +/// on a fresh install, and finish an archival a previous start left half +/// done. A data dir that still needs migrating is refused: that runs only +/// through [`run`], which reports progress and can be cancelled. Called with +/// the data dir's ownership lock held. +pub(super) fn prepare(root: &Path) -> io::Result<()> { + if needed(root)? { + return Err(io::Error::other(format!( + "{} holds sessions.json or *.jsonl files from an older Tcode that have not been \ + migrated into {DB_FILE} yet", + root.display() + ))); + } + run(root, &mut |_| {}, &AtomicBool::new(false)).map(drop) +} + +/// Migrate the older build's files into `tcode.db`, or create an empty one, +/// reporting progress after every chunk and every log. `cancel` is checked +/// at the same points until the staged database is complete; once it is set +/// there, the staging files are removed and nothing else is changed. Called +/// with the data dir's ownership lock held. +pub(super) fn run( + root: &Path, + report: &mut dyn FnMut(MigrationProgress), + cancel: &AtomicBool, +) -> io::Result { + if root.join(DB_FILE).exists() { + archive_sources(root)?; + return Ok(Migration::Completed); + } + remove_staging(root)?; + let started = Instant::now(); + let mut progress = Progress { + report, + cancel, + current: MigrationProgress { + phase: MigrationPhase::Scanning, + threads_done: 0, + threads_total: 0, + bytes_done: 0, + bytes_total: 0, + }, + }; + (progress.report)(progress.current); + let summary = match build_staging(root, &mut progress) { + Ok(summary) => summary, + Err(error) => { + // `build_staging` has dropped every handle on the staging files. + if let Err(cleanup) = remove_staging(root) { + log::warn!("could not discard the staged database: {cleanup}"); + } + if error.get_ref().is_some_and(|inner| inner.is::()) { + log::info!( + "migration into {} cancelled; the sources are unchanged", + root.join(DB_FILE).display() + ); + return Ok(Migration::Cancelled); + } + return Err(error); + } + }; + progress.finish(MigrationPhase::Publishing); + publish(root, &summary.sources)?; + log::info!( + "migrated {} thread(s) ({} line(s), {} blank, {} bytes) and {} session(s) in {} \ + project(s) into {} in {:.1}s", + summary.logs, + summary.lines, + summary.blank_lines, + summary.bytes, + summary.sessions, + summary.projects, + root.join(DB_FILE).display(), + started.elapsed().as_secs_f64() + ); + progress.finish(MigrationPhase::Archiving); + archive_sources(root)?; + Ok(Migration::Completed) +} + +#[derive(Debug)] +struct Cancelled; + +impl std::fmt::Display for Cancelled { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("the migration was cancelled") + } +} + +impl std::error::Error for Cancelled {} + +struct Progress<'a> { + report: &'a mut dyn FnMut(MigrationProgress), + cancel: &'a AtomicBool, + current: MigrationProgress, +} + +impl Progress<'_> { + fn check(&self) -> io::Result<()> { + if self.cancel.load(Ordering::Relaxed) { + return Err(io::Error::other(Cancelled)); + } + Ok(()) + } + + /// Begin a cancellable phase over `threads` logs of `bytes` in all. + fn start(&mut self, phase: MigrationPhase, threads: usize, bytes: u64) -> io::Result<()> { + self.check()?; + self.current = MigrationProgress { + phase, + threads_done: 0, + threads_total: threads, + bytes_done: 0, + bytes_total: bytes, + }; + (self.report)(self.current); + Ok(()) + } + + fn advance(&mut self, threads: usize, bytes: u64) -> io::Result<()> { + self.current.threads_done += threads; + self.current.bytes_done += bytes; + (self.report)(self.current); + self.check() + } + + /// Enter a phase that runs to the end whatever `cancel` says. + fn finish(&mut self, phase: MigrationPhase) { + self.current.phase = phase; + self.current.threads_done = self.current.threads_total; + self.current.bytes_done = self.current.bytes_total; + (self.report)(self.current); + } +} + +/// An older build that ran after the migration writes JSON again; this build +/// never reads it back. +pub(super) fn warn_about_stray_sources(root: &Path) { + let Ok(sources) = legacy_sources(root) else { + return; + }; + if !sources.is_empty() { + log::warn!( + "ignoring {} file(s) in {} left by an older Tcode build (sessions.json / *.jsonl); \ + threads are read only from {DB_FILE}", + sources.len(), + root.display() + ); + } +} + +struct Summary { + sources: Vec, + projects: usize, + sessions: usize, + logs: usize, + lines: u64, + blank_lines: u64, + bytes: u64, +} + +/// Every staging file a previous run may have left. Never the final name's. +fn remove_staging(root: &Path) -> io::Result<()> { + let staging = root.join(STAGING_FILE); + for path in [ + wal_path(&staging), + sidecar(&staging, "-shm"), + sidecar(&staging, "-journal"), + staging, + root.join(ARCHIVE_LIST), + ] { + match fs::remove_file(&path) { + Ok(()) => log::warn!("discarded {} from an unfinished migration", path.display()), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(()) +} + +fn sidecar(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push(suffix); + PathBuf::from(name) +} + +/// `sessions.json` and every `.jsonl` directly in the data dir. +fn legacy_sources(root: &Path) -> io::Result> { + let mut names = Vec::new(); + for entry in fs::read_dir(root)? { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if name == LEGACY_INDEX + || Path::new(&name) + .extension() + .is_some_and(|ext| ext == "jsonl") + { + names.push(name); + } + } + names.sort(); + Ok(names) +} + +fn build_staging(root: &Path, progress: &mut Progress) -> io::Result { + let sources = legacy_sources(root)?; + let mut logs = Vec::new(); + let mut bytes_total = 0; + for name in &sources { + if let Some(id) = name.strip_suffix(".jsonl") { + bytes_total += fs::metadata(root.join(name))?.len(); + logs.push((id, root.join(name))); + } + } + let index = migrate_index(read_legacy_index(root)?); + reject_duplicates(&index)?; + let staging = root.join(STAGING_FILE); + let db = Db::open(&staging, true)?; + db.write(|db, connection| { + for project in &index.projects { + let body = serde_json::to_vec(project).map_err(io::Error::other)?; + db.execute( + connection, + "INSERT INTO projects (id, body) VALUES (?1, ?2)", + (project.id.as_str(), body), + )?; + } + for session in &index.sessions { + let body = serde_json::to_vec(session).map_err(io::Error::other)?; + db.execute( + connection, + "INSERT INTO sessions (id, body) VALUES (?1, ?2)", + (session.id.as_str(), body), + )?; + } + Ok(()) + })?; + + let mut summary = Summary { + sources: sources.clone(), + projects: index.projects.len(), + sessions: index.sessions.len(), + logs: 0, + lines: 0, + blank_lines: 0, + bytes: 0, + }; + progress.start(MigrationPhase::Importing, logs.len(), bytes_total)?; + let mut tallies = Vec::new(); + for (id, path) in &logs { + let tally = import_log(&db, id, path, progress)?; + summary.logs += 1; + summary.lines += tally.lines; + summary.blank_lines += tally.blank_lines; + summary.bytes += tally.bytes; + tallies.push(((*id).to_owned(), tally)); + } + + progress.start(MigrationPhase::Verifying, logs.len(), bytes_total)?; + for ((id, path), (_, tally)) in logs.iter().zip(&tallies) { + verify_log(&db, id, path, tally, progress)?; + log::info!( + "migrated {}: {} line(s), {} blank, {} bytes, verified byte-identical", + path.display(), + tally.lines, + tally.blank_lines, + tally.bytes + ); + } + let stored = db.read(read_index)?; + verify_index(&index, &stored)?; + progress.check()?; + db.write(|db, connection| { + db.execute( + connection, + &format!("PRAGMA user_version = {SCHEMA_VERSION}"), + (), + ) + .map(drop) + })?; + db.checkpoint()?; + drop(db); + + progress.check()?; + + // A fresh open sees only what reached the main file. + let reopened = Db::open(&staging, false)?; + if reopened.user_version()? != SCHEMA_VERSION { + return Err(io::Error::other(format!( + "{} lost its completion marker after the checkpoint", + staging.display() + ))); + } + // Per thread, so each count is an index range scan rather than one sort + // over every row in the database. + let mut committed_total = 0; + for (id, tally) in &tallies { + let (lines, bytes) = reopened.read(|db, connection| { + let mut counts = (0, 0); + db.query( + connection, + "SELECT COUNT(*), COALESCE(SUM(length(line)), 0) FROM events WHERE session_id = ?1", + (id.as_str(),), + |row| { + counts = (integer(row, 0)? as u64, integer(row, 1)? as u64); + Ok(()) + }, + )?; + Ok(counts) + })?; + if (lines, bytes) != (tally.lines, tally.bytes) { + return Err(io::Error::other(format!( + "{}: thread {id} holds {lines} line(s) / {bytes} bytes after reopening, the \ + source {} / {}", + staging.display(), + tally.lines, + tally.bytes + ))); + } + committed_total += lines; + } + let mut stored_total = 0; + reopened.read(|db, connection| { + db.query(connection, "SELECT COUNT(*) FROM events", (), |row| { + stored_total = integer(row, 0)? as u64; + Ok(()) + }) + })?; + if stored_total != committed_total { + return Err(io::Error::other(format!( + "{}: {stored_total} event rows after reopening, {committed_total} migrated", + staging.display() + ))); + } + let stored = reopened.read(read_index)?; + verify_index(&index, &stored)?; + drop(reopened); + let wal = wal_path(&staging); + match fs::metadata(&wal) { + Ok(metadata) if metadata.len() > 0 => { + return Err(io::Error::other(format!( + "{} holds {} bytes after the staged database was closed", + wal.display(), + metadata.len() + ))); + } + Ok(_) => fs::remove_file(&wal)?, + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + // The last point a cancellation is honoured: what follows publishes. + progress.check()?; + sync_file(&staging)?; + Ok(summary) +} + +/// Today's tolerance: the object schema or the legacy bare array. An +/// unparseable file is left as it is, to be archived into `legacy/` with the +/// logs, which still migrate. +fn read_legacy_index(root: &Path) -> io::Result { + let path = root.join(LEGACY_INDEX); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(IndexFile::default()), + Err(error) => return Err(error), + }; + let parsed = serde_json::from_slice::(&bytes).or_else(|_| { + serde_json::from_slice::>(&bytes).map(|sessions| IndexFile { + projects: Vec::new(), + sessions, + }) + }); + Ok(parsed.unwrap_or_else(|error| { + log::warn!( + "failed to parse {LEGACY_INDEX}: {error}; migrating the event logs without it, and \ + keeping it in {LEGACY_DIR}/" + ); + IndexFile::default() + })) +} + +/// Upserting by id would silently keep only the last of two entries. +fn reject_duplicates(index: &IndexFile) -> io::Result<()> { + let mut seen = HashSet::new(); + for id in index.projects.iter().map(|project| &project.id) { + if !seen.insert(("project", id)) { + return Err(duplicate("project", id)); + } + } + for id in index.sessions.iter().map(|session| &session.id) { + if !seen.insert(("session", id)) { + return Err(duplicate("session", id)); + } + } + Ok(()) +} + +fn duplicate(kind: &str, id: &str) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{LEGACY_INDEX} lists {kind} {id} more than once; nothing was migrated and the file \ + is unchanged" + ), + ) +} + +fn verify_index(expected: &IndexFile, stored: &IndexFile) -> io::Result<()> { + if stored.projects != expected.projects { + return Err(io::Error::other( + "the migrated projects differ from sessions.json when read back", + )); + } + if stored.sessions != expected.sessions { + let id = expected + .sessions + .iter() + .zip(&stored.sessions) + .find(|(expected, stored)| expected != stored) + .map(|(expected, _)| expected.id.as_str()) + .unwrap_or("(count)"); + return Err(io::Error::other(format!( + "the migrated session {id} differs from sessions.json when read back" + ))); + } + Ok(()) +} + +#[derive(Default)] +struct Tally { + lines: u64, + blank_lines: u64, + bytes: u64, +} + +/// Every raw segment of `path`: each line with its `\n`, an unterminated last +/// line as it is. +fn read_segments(path: &Path, mut each: impl FnMut(Vec) -> io::Result<()>) -> io::Result<()> { + let mut reader = BufReader::with_capacity(1 << 20, File::open(path)?); + loop { + let mut segment = Vec::new(); + if reader.read_until(b'\n', &mut segment)? == 0 { + return Ok(()); + } + each(segment)?; + } +} + +fn import_log(db: &Db, id: &str, path: &Path, progress: &mut Progress) -> io::Result { + let mut tally = Tally::default(); + let mut pending: Vec> = Vec::new(); + let mut pending_bytes = 0; + let mut position = 0; + let flush = |pending: &mut Vec>, position: &mut i64| -> io::Result<()> { + if !pending.is_empty() { + *position = db.write(|db, connection| { + insert_segments(db, connection, id, *position, pending.iter()) + })?; + pending.clear(); + } + Ok(()) + }; + read_segments(path, |segment| { + tally.lines += 1; + tally.bytes += segment.len() as u64; + if segment.trim_ascii().is_empty() { + tally.blank_lines += 1; + } + pending_bytes += segment.len(); + pending.push(segment); + if pending_bytes >= CHUNK_BYTES { + flush(&mut pending, &mut position)?; + progress.advance(0, pending_bytes as u64)?; + pending_bytes = 0; + } + Ok(()) + })?; + flush(&mut pending, &mut position)?; + progress.advance(1, pending_bytes as u64)?; + Ok(tally) +} + +/// Compare the committed rows with the file again, segment for segment. +fn verify_log( + db: &Db, + id: &str, + path: &Path, + tally: &Tally, + progress: &mut Progress, +) -> io::Result<()> { + let mismatch = |position: i64| { + io::Error::other(format!( + "{}: line {} differs from the migrated copy", + path.display(), + position + 1 + )) + }; + let mut page: Vec> = Vec::new(); + let mut page_start = 0_i64; + let mut position = 0_i64; + let mut compared = 0; + read_segments(path, |segment| { + if position - page_start == page.len() as i64 { + if compared > 0 { + progress.advance(0, compared)?; + compared = 0; + } + page_start = position; + page = db.read(|db, connection| { + // Sizes first, so a page holds about CHUNK_BYTES however long + // the lines are, and at least one line. + let mut last = position - 1; + let mut bytes = 0; + db.query( + connection, + "SELECT position, length(line) FROM events \ + WHERE session_id = ?1 AND position >= ?2 ORDER BY position LIMIT ?3", + (id, position, VERIFY_ROWS), + |row| { + if last < position || bytes < CHUNK_BYTES as i64 { + last = integer(row, 0)?; + bytes += integer(row, 1)?; + } + Ok(()) + }, + )?; + let mut rows = Vec::new(); + db.query( + connection, + "SELECT position, line FROM events \ + WHERE session_id = ?1 AND position BETWEEN ?2 AND ?3 ORDER BY position", + (id, position, last), + |row| { + let stored = integer(row, 0)?; + if stored != position + rows.len() as i64 { + return Err(mismatch(position + rows.len() as i64)); + } + rows.push(blob(row, 1)?); + Ok(()) + }, + )?; + Ok(rows) + })?; + } + let stored = page + .get((position - page_start) as usize) + .ok_or_else(|| mismatch(position))?; + if *stored != segment { + return Err(mismatch(position)); + } + compared += segment.len() as u64; + position += 1; + Ok(()) + })?; + let mut extra = 0; + db.read(|db, connection| { + db.query( + connection, + "SELECT COUNT(*) FROM events WHERE session_id = ?1 AND position >= ?2", + (id, position), + |row| { + extra = integer(row, 0)?; + Ok(()) + }, + ) + })?; + if extra != 0 || position as u64 != tally.lines { + return Err(mismatch(position)); + } + progress.advance(1, compared) +} + +fn publish(root: &Path, sources: &[String]) -> io::Result<()> { + let list = root.join(ARCHIVE_LIST); + if !sources.is_empty() { + let temporary = sidecar(&list, ".tmp"); + fs::write( + &temporary, + serde_json::to_vec_pretty(sources).map_err(io::Error::other)?, + )?; + sync_file(&temporary)?; + fs::rename(&temporary, &list)?; + } + sync_dir(root)?; + fs::rename(root.join(STAGING_FILE), root.join(DB_FILE))?; + sync_dir(root) +} + +/// Move the sources named by `tcode.db.archive` into `legacy/`. Safe to repeat: +/// a moved file is skipped, and a name already taken in `legacy/` keeps its +/// source where it is. +fn archive_sources(root: &Path) -> io::Result<()> { + let list = root.join(ARCHIVE_LIST); + let names: Vec = match fs::read(&list) { + Ok(bytes) => serde_json::from_slice(&bytes).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("{} is unreadable: {error}", list.display()), + ) + })?, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + let legacy = root.join(LEGACY_DIR); + fs::create_dir_all(&legacy)?; + let mut moved = 0; + for name in &names { + // The list is written by this module, but a name is still never + // allowed to leave the data dir. + if Path::new(name).file_name() != Some(std::ffi::OsStr::new(name)) { + continue; + } + let source = root.join(name); + let destination = legacy.join(name); + if !source.exists() { + continue; + } + if destination.exists() { + log::warn!( + "kept {} in place: {} already exists", + source.display(), + destination.display() + ); + continue; + } + fs::rename(&source, &destination)?; + moved += 1; + } + sync_dir(&legacy)?; + sync_dir(root)?; + fs::remove_file(&list)?; + sync_dir(root)?; + log::info!( + "moved {moved} migrated source file(s) into {}", + legacy.display() + ); + Ok(()) +} + +/// Windows flushes only a handle opened for writing. +fn sync_file(path: &Path) -> io::Result<()> { + fs::OpenOptions::new().write(true).open(path)?.sync_all() +} + +/// Make a directory's entries (a rename, a new file) durable. +#[cfg(unix)] +fn sync_dir(dir: &Path) -> io::Result<()> { + File::open(dir)?.sync_all() +} + +/// NTFS journals renames itself, and std cannot open a directory handle on +/// Windows. +#[cfg(not(unix))] +fn sync_dir(_dir: &Path) -> io::Result<()> { + Ok(()) +} diff --git a/crates/services/src/store/mod.rs b/crates/services/src/store/mod.rs new file mode 100644 index 000000000..e0f7b9553 --- /dev/null +++ b/crates/services/src/store/mod.rs @@ -0,0 +1,1135 @@ +//! Persistence for tcode threads. +//! +//! Projects, thread metadata and every thread's event log live in one Turso +//! database, `tcode.db`, in the platform data dir (e.g. +//! `~/Library/Application Support/tcode/`): +//! * `projects` and `sessions` hold each [`Project`] / [`SessionMeta`] as the +//! JSON serde produces for it. +//! * `events` holds each thread's log as raw byte segments, one per line +//! including its `\n`, densely numbered from 0: the bytes of a `{ ts, event }` +//! record, or whatever a migrated or imported log contained. +//! +//! Model and command caches stay as JSON files beside it. +//! +//! Replay accepts timestamped records and legacy bare [`AgentEvent`] lines, +//! then folds [`StoredEvent`]s into a [`tcode_core::session::Timeline`]. + +mod db; +mod migrate; +#[cfg(test)] +mod tests; + +pub use migrate::{Migration, MigrationPhase, MigrationProgress}; + +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, LazyLock, Mutex, MutexGuard, Weak}; +use std::time::{Duration, Instant}; + +use agent::{AgentEvent, ModelSpec, ProviderCommand, ProviderKind}; +use serde::{Deserialize, Serialize}; + +use tcode_core::project::{IndexFile, Project, SessionMeta}; +use tcode_core::session::StoredEvent; + +use db::{Broken, Db, SCHEMA_VERSION, blob, integer, is_broken}; + +const DB_FILE: &str = "tcode.db"; +/// Held with an OS file lock for as long as a host owns the data dir, which +/// covers the migration as well as the open database. It is never deleted: +/// unlinking a lock file while another process waits on it would let two +/// owners in. +const LOCK_FILE: &str = "tcode.lock"; +/// A macOS permission relaunch starts the new instance while the old one is +/// still closing its store. +const RELAUNCH_WAIT: Duration = Duration::from_secs(15); + +/// On-disk envelope wrapping each event with its record time. Kept private: +/// callers deal in [`StoredEvent`] (which tolerates the legacy bare form). +#[derive(Serialize, Deserialize)] +struct EventEnvelope { + ts: u64, + event: AgentEvent, +} + +#[derive(Serialize)] +struct EventEnvelopeRef<'a> { + ts: u64, + event: &'a AgentEvent, +} + +/// Cheap, cloneable handle to the data directory. Every handle for the same +/// directory in this process shares one database, writer and lifecycle. +#[derive(Clone)] +pub struct SessionStore { + root: PathBuf, + shared: Arc, +} + +impl std::fmt::Debug for SessionStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SessionStore") + .field("root", &self.root) + .finish_non_exhaustive() + } +} + +struct Shared { + state: Mutex, + /// Signalled whenever an operation finishes, for [`SessionStore::close`]. + idle: Condvar, + /// Per-session event-log generation, advanced after every committed + /// change to that log; see [`SessionStore::event_generation`]. + generations: Mutex>, + next_generation: AtomicU64, + /// Full event-log parses, shared by every clone of this store so tests can + /// prove a caller served history from memory. + #[cfg(any(test, feature = "test-support"))] + event_reads: std::sync::atomic::AtomicUsize, +} + +enum State { + /// Opened on first use, so a handle that only needs the directory (a + /// subcommand, a settings file) never takes the data dir. + Unopened, + /// The data dir is owned, for a migration and then the database, but the + /// database is not open yet. + Owned(File), + /// [`SessionStore::migrate`] holds the ownership lock while it runs. + Migrating, + Open(Live), + /// [`SessionStore::close`] is waiting for in-flight operations. + Closing(Live), + Closed, + /// A panic or an unrecoverable connection: nothing is served again, and + /// the ownership lock stays held while any handle remains, so no other + /// process opens a database this one may have left mid-operation. + Failed { + reason: String, + _ownership: Option, + }, +} + +struct Live { + db: Arc, + ownership: File, + in_flight: usize, +} + +/// Every open handle by canonical data dir, so a second +/// [`SessionStore::open_at`] in one process shares the first one's database +/// instead of contending for its locks. +static STORES: LazyLock>>> = LazyLock::new(Default::default); + +/// One change to the store. [`SessionStore::apply`] commits a sequence of them +/// in a single transaction. +#[derive(Debug, Clone)] +pub struct Mutation(Op); + +#[derive(Debug, Clone)] +enum Op { + AppendEvent { session_id: String, line: Vec }, + ReplaceEventLog { session_id: String, bytes: Vec }, + CloneEvents { src: String, dst: String }, + UpsertMeta(Box), + UpsertProject(Box), + RemoveSession(String), + RemoveProject(String), +} + +impl Mutation { + /// Append one event, wrapped in a timestamped envelope + /// (`{"ts": , "event": {…}}`). + pub fn append_event(session_id: &str, ts: u64, event: &AgentEvent) -> io::Result { + let mut line = serde_json::to_vec(&EventEnvelopeRef { ts, event }).map_err(invalid_data)?; + line.push(b'\n'); + Ok(Self(Op::AppendEvent { + session_id: session_id.to_owned(), + line, + })) + } + + /// Replace a session's whole native event log with `bytes`, kept exactly. + pub fn replace_event_log(session_id: &str, bytes: Vec) -> Self { + Self(Op::ReplaceEventLog { + session_id: session_id.to_owned(), + bytes, + }) + } + + /// Give `dst` a copy of `src`'s event log. A missing source is an empty + /// transcript. + pub fn clone_events(src: &str, dst: &str) -> Self { + Self(Op::CloneEvents { + src: src.to_owned(), + dst: dst.to_owned(), + }) + } + + /// Insert or replace a meta (by id). + pub fn upsert_meta(meta: SessionMeta) -> Self { + Self(Op::UpsertMeta(Box::new(meta))) + } + + /// Insert or replace a project; its previous managed icon is removed once + /// the change has committed. + pub fn upsert_project(project: Project) -> Self { + Self(Op::UpsertProject(Box::new(project))) + } + + /// Remove a session's meta and its event log. + pub fn remove_session(id: &str) -> Self { + Self(Op::RemoveSession(id.to_owned())) + } + + /// Remove a project; its managed icon is removed once the change has + /// committed. Sessions are removed separately. + pub fn remove_project(id: &str) -> Self { + Self(Op::RemoveProject(id.to_owned())) + } + + /// Approximate bytes this change writes, for bounding a batch. + pub fn payload_len(&self) -> usize { + match &self.0 { + Op::AppendEvent { line, .. } => line.len(), + Op::ReplaceEventLog { bytes, .. } => bytes.len(), + _ => 0, + } + } + + /// The session whose event log this change rewrites, if any. + fn event_log(&self) -> Option<&str> { + match &self.0 { + Op::AppendEvent { session_id, .. } | Op::ReplaceEventLog { session_id, .. } => { + Some(session_id) + } + Op::CloneEvents { dst, .. } => Some(dst), + Op::RemoveSession(id) => Some(id), + Op::UpsertMeta(_) | Op::UpsertProject(_) | Op::RemoveProject(_) => None, + } + } +} + +impl SessionStore { + /// Open (creating if needed) the store under the platform data dir, or under + /// `TCODE_DATA_DIR` when it is set — which gives a throwaway profile (its own + /// sessions, settings and installed ACP agents) for demos and screenshots. + pub fn open_default() -> io::Result { + let root = match std::env::var_os("TCODE_DATA_DIR") { + Some(dir) => PathBuf::from(dir), + None => dirs::data_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join("tcode"), + }; + Self::open_at(root) + } + + /// A handle to `root`, created if needed. The database is not opened until + /// [`SessionStore::open`] or the first operation that needs it. + pub fn open_at(root: PathBuf) -> io::Result { + fs::create_dir_all(&root)?; + let key = fs::canonicalize(&root)?; + let mut stores = STORES + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + stores.retain(|_, shared| shared.strong_count() > 0); + let shared = match stores.get(&key).and_then(Weak::upgrade) { + Some(shared) if !shared.finished() => shared, + _ => { + let shared = Arc::new(Shared { + state: Mutex::new(State::Unopened), + idle: Condvar::new(), + generations: Mutex::default(), + next_generation: AtomicU64::new(1), + #[cfg(any(test, feature = "test-support"))] + event_reads: Default::default(), + }); + stores.insert(key, Arc::downgrade(&shared)); + shared + } + }; + Ok(Self { root, shared }) + } + + /// How many times [`SessionStore::read_events`] parsed a log through this + /// store or any of its clones. + #[cfg(any(test, feature = "test-support"))] + pub fn event_reads(&self) -> usize { + self.shared + .event_reads + .load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn root(&self) -> &PathBuf { + &self.root + } + + /// Take ownership of the data dir and open its database now. A data dir + /// that still needs migrating ([`SessionStore::needs_migration`]) is + /// refused. Fails with + /// [`io::ErrorKind::ResourceBusy`] while another host owns the directory. + pub fn open(&self) -> io::Result<()> { + self.run("open", |_| Ok(())) + } + + /// Take ownership of the data dir, keeping it for the database opened + /// later, and say whether it holds the JSON index or JSONL logs of an + /// older build that [`SessionStore::migrate`] must move into `tcode.db` + /// before the store opens. Fails with [`io::ErrorKind::ResourceBusy`] + /// while another host owns the directory. + pub fn needs_migration(&self) -> io::Result { + let mut state = self.shared.lock_state()?; + match &*state { + State::Unopened => *state = State::Owned(acquire_ownership(&self.root)?), + State::Owned(_) => {} + State::Open(_) => return Ok(false), + other => return Err(unavailable(other, "check for a migration")), + } + migrate::needed(&self.root) + } + + /// Migrate an older build's files into `tcode.db` under the data dir's + /// ownership lock, which the store keeps for the database afterwards. + /// `progress` is called after every chunk of about 8 MiB and every log; + /// `cancel` is checked at the same points and between phases until the + /// migrated database is complete. A cancelled migration removes its + /// staging files and changes nothing else; a later one starts over. + pub fn migrate( + &self, + mut progress: impl FnMut(MigrationProgress), + cancel: &AtomicBool, + ) -> io::Result { + let ownership = { + let mut state = self.shared.lock_state()?; + match std::mem::replace(&mut *state, State::Migrating) { + State::Unopened => match acquire_ownership(&self.root) { + Ok(ownership) => ownership, + Err(error) => { + *state = State::Unopened; + return Err(error); + } + }, + State::Owned(ownership) => ownership, + State::Open(live) => { + *state = State::Open(live); + return Ok(Migration::Completed); + } + other => { + let error = unavailable(&other, "migrate"); + *state = other; + return Err(error); + } + } + }; + let outcome = catch_unwind(AssertUnwindSafe(|| { + migrate::run(&self.root, &mut progress, cancel) + })); + let mut state = self.shared.lock_state()?; + let result = match outcome { + Ok(result) => { + *state = State::Owned(ownership); + result + } + Err(panic) => { + let reason = format!( + "session store panicked while migrating {}: {}", + self.root.display(), + panic_message(panic.as_ref()) + ); + log::error!("{reason}"); + *state = State::Failed { + reason: reason.clone(), + _ownership: Some(ownership), + }; + Err(io::Error::other(reason)) + } + }; + self.shared.idle.notify_all(); + result + } + + /// Whether the store stopped serving after a panic or a broken connection. + pub fn is_failed(&self) -> bool { + matches!( + *self + .shared + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + State::Failed { .. } + ) + } + + /// Wait for every in-flight operation and a running migration, checkpoint + /// the WAL into the main file and release the database and the data dir. + /// Every handle of this store fails afterwards; a store that already + /// failed is not checkpointed. + pub fn close(&self) -> io::Result<()> { + let mut state = self.shared.lock_state()?; + loop { + match std::mem::replace(&mut *state, State::Closed) { + State::Unopened | State::Closed | State::Owned(_) => return Ok(()), + State::Migrating => { + *state = State::Migrating; + state = self + .shared + .idle + .wait(state) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + } + State::Open(live) => *state = State::Closing(live), + State::Closing(live) if live.in_flight > 0 => { + *state = State::Closing(live); + state = self + .shared + .idle + .wait(state) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + } + State::Closing(live) => { + drop(state); + let Live { db, ownership, .. } = live; + let result = match Arc::try_unwrap(db) { + Ok(db) => catch_unwind(AssertUnwindSafe(|| db.checkpoint())) + .unwrap_or_else(|panic| { + Err(io::Error::other(format!( + "session store panicked during the closing checkpoint: {}", + panic_message(panic.as_ref()) + ))) + }), + Err(_) => Err(io::Error::other( + "session store is still referenced after its last operation", + )), + }; + drop(ownership); + return result; + } + failed @ State::Failed { .. } => { + let error = failed_error(&failed); + *state = failed; + return Err(error); + } + } + } + } + + /// Run `body` against the open database. A panic inside it (Turso asserts + /// its invariants by panicking) becomes an error naming `operation` and + /// fails the store for every handle. + fn run(&self, operation: &str, body: impl FnOnce(&Db) -> io::Result) -> io::Result { + let db = self.acquire(operation)?; + let outcome = catch_unwind(AssertUnwindSafe(|| body(&db))); + drop(db); + let result = match outcome { + Ok(Err(error)) if is_broken(&error) => { + self.shared.fail(format!("{operation}: {error}")); + Err(error) + } + Ok(result) => result, + Err(panic) => { + let reason = format!( + "session store panicked during {operation}: {}", + panic_message(panic.as_ref()) + ); + log::error!("{reason}"); + self.shared.fail(reason.clone()); + Err(io::Error::other(reason)) + } + }; + self.shared.release(); + result + } + + fn acquire(&self, operation: &str) -> io::Result> { + let mut state = self.shared.lock_state()?; + let ownership = match std::mem::replace(&mut *state, State::Unopened) { + State::Unopened => Some(acquire_ownership(&self.root)?), + State::Owned(ownership) => Some(ownership), + other => { + *state = other; + None + } + }; + if let Some(ownership) = ownership { + // Dropping the ownership on failure lets the next attempt start over. + match catch_unwind(AssertUnwindSafe(|| open_live(&self.root, ownership))) { + Ok(Ok(live)) => *state = State::Open(live), + Ok(Err(error)) => return Err(error), + Err(panic) => { + let reason = format!( + "session store panicked while opening {}: {}", + self.root.display(), + panic_message(panic.as_ref()) + ); + log::error!("{reason}"); + *state = State::Failed { + reason: reason.clone(), + _ownership: None, + }; + return Err(io::Error::other(reason)); + } + } + } + match &mut *state { + State::Open(live) => { + live.in_flight += 1; + Ok(live.db.clone()) + } + other => Err(unavailable(other, operation)), + } + } + + /// Commit `mutations` in one transaction, then run their deferred side + /// effects. Either every change is committed or none is. + pub fn apply(&self, mutations: &[Mutation]) -> io::Result<()> { + if mutations.is_empty() { + return Ok(()); + } + let icons = self.run("write", |db| { + db.write(|db, connection| { + let mut icons = Vec::new(); + for mutation in mutations { + apply_op(db, connection, &mutation.0, &mut icons)?; + } + Ok(icons) + }) + })?; + { + let mut generations = self + .shared + .generations + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + for id in mutations.iter().filter_map(Mutation::event_log) { + let generation = self.shared.next_generation.fetch_add(1, Ordering::Relaxed); + generations.insert(id.to_owned(), generation); + } + } + for icon in icons { + self.remove_project_icon(icon); + } + Ok(()) + } + + /// A value that changes whenever a committed write changes `id`'s event + /// log through this store, so a cache of something derived from the log + /// can tell it is stale. 0 means unchanged since the store was opened. + pub fn event_generation(&self, id: &str) -> u64 { + self.shared + .generations + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(id) + .copied() + .unwrap_or(0) + } + + /// Read a native event log back as the exact bytes it holds. + pub fn read_event_log(&self, id: &str) -> io::Result> { + self.run("read event log", |db| { + db.read(|db, connection| { + let mut bytes = Vec::new(); + db.query( + connection, + "SELECT line FROM events WHERE session_id = ?1 ORDER BY position", + (id,), + |row| { + bytes.extend_from_slice(&blob(row, 0)?); + Ok(()) + }, + )?; + Ok(bytes) + }) + }) + } + + fn models_path(&self, provider: ProviderKind) -> PathBuf { + let name = match provider { + ProviderKind::Codex => "codex", + ProviderKind::ClaudeCode => "claude", + ProviderKind::Pi => "pi", + ProviderKind::OpenCode => "opencode", + ProviderKind::Cursor => "cursor", + ProviderKind::Grok => "grok", + // ACP agents publish their models over the wire at session start + // (`AgentEvent::ProviderOptions`), so there is no catalog to cache. + ProviderKind::Acp => "acp", + }; + self.root.join(format!("models-{name}.json")) + } + + fn commands_path(&self, provider: ProviderKind, acp_agent_id: Option<&str>) -> Option { + let name = match provider { + ProviderKind::Codex => "codex".to_string(), + ProviderKind::ClaudeCode => "claude".to_string(), + ProviderKind::Pi => "pi".to_string(), + ProviderKind::OpenCode => "opencode".to_string(), + ProviderKind::Cursor => "cursor".to_string(), + ProviderKind::Grok => "grok".to_string(), + ProviderKind::Acp => { + let id = acp_agent_id?; + // Registry ids are external input and may contain path separators. + // Hex keeps the filename reversible and collision-free without + // allowing an id to escape the data directory. + let encoded = id + .as_bytes() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + format!("acp-{encoded}") + } + }; + Some(self.root.join(format!("commands-{name}.json"))) + } + + /// Load the last-fetched model catalog for `provider` so the picker is + /// instant offline. Empty when never fetched / unreadable. + pub fn load_models(&self, provider: ProviderKind) -> Vec { + let Ok(bytes) = fs::read(self.models_path(provider)) else { + return Vec::new(); + }; + serde_json::from_slice(&bytes).unwrap_or_default() + } + + /// Persist the freshly fetched model catalog for `provider`. + pub fn save_models(&self, provider: ProviderKind, models: &[ModelSpec]) -> io::Result<()> { + let path = self.models_path(provider); + let tmp = path.with_extension("json.tmp"); + let data = serde_json::to_vec_pretty(models).map_err(invalid_data)?; + fs::write(&tmp, data)?; + fs::rename(&tmp, path) + } + + /// Load the most recently reported command/skill list for a native provider + /// or one specific ACP agent. Empty when missing, unreadable, or when an ACP + /// agent id was not supplied. + pub fn load_commands( + &self, + provider: ProviderKind, + acp_agent_id: Option<&str>, + ) -> Vec { + let Some(path) = self.commands_path(provider, acp_agent_id) else { + return Vec::new(); + }; + let Ok(bytes) = fs::read(path) else { + return Vec::new(); + }; + serde_json::from_slice(&bytes).unwrap_or_default() + } + + /// Atomically persist the complete command/skill list reported by a native + /// provider or one specific ACP agent. Empty lists are meaningful: they + /// replace a stale non-empty cache. + pub fn save_commands( + &self, + provider: ProviderKind, + acp_agent_id: Option<&str>, + commands: &[ProviderCommand], + ) -> io::Result<()> { + let path = self.commands_path(provider, acp_agent_id).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "ACP command cache requires an agent id", + ) + })?; + let tmp = path.with_extension("json.tmp"); + let data = serde_json::to_vec_pretty(commands).map_err(invalid_data)?; + fs::write(&tmp, data)?; + fs::rename(&tmp, path) + } + + /// Load every project and session, in the order they were first stored. + /// A row this build cannot decode is skipped with a warning and left in + /// the database untouched. + pub fn read_file(&self) -> io::Result { + self.run("read index", |db| { + db.read(|db, connection| { + // One read transaction, so projects and sessions come from the + // same snapshot. + db.execute(connection, "BEGIN", ())?; + let file = read_index(db, connection); + db.execute(connection, "COMMIT", ())?; + file + }) + }) + } + + /// Load the session index (newest first). + pub fn load_index(&self) -> io::Result> { + let mut metas = self.read_file()?.sessions; + metas.sort_by_key(|b| std::cmp::Reverse(b.updated_at)); + Ok(metas) + } + + /// Insert or replace a meta in the index (by id). + pub fn upsert_meta(&self, meta: &SessionMeta) -> io::Result<()> { + self.apply(&[Mutation::upsert_meta(meta.clone())]) + } + + /// Insert or replace a project, then remove its previous managed icon. + pub fn upsert_project(&self, project: &Project) -> io::Result<()> { + self.apply(&[Mutation::upsert_project(project.clone())]) + } + + /// Remove a project and its managed icon. Sessions are removed separately. + pub fn remove_project(&self, id: &str) -> io::Result<()> { + self.apply(&[Mutation::remove_project(id)]) + } + + fn remove_project_icon(&self, path: Option) { + if let Some(path) = path + && path.parent() == Some(self.root.join("project-icons").as_path()) + && let Err(error) = fs::remove_file(&path) + && error.kind() != io::ErrorKind::NotFound + { + log::warn!("could not remove project icon {}: {error}", path.display()); + } + } + + /// Append one event to the session's log, wrapped in a timestamped + /// envelope (`{"ts": , "event": {…}}`). + pub fn append_event(&self, id: &str, ts: u64, event: &AgentEvent) -> io::Result<()> { + self.apply(&[Mutation::append_event(id, ts, event)?]) + } + + /// Read and parse every persisted event for a session, skipping lines that + /// do not parse. + /// + /// Each line is tolerantly parsed as either a timestamped envelope + /// (`{"ts":…,"event":…}`) or a legacy bare event (`{"type":…}`), so logs + /// written before the envelope format still replay (with `ts == None`). + pub fn read_events(&self, id: &str) -> io::Result> { + #[cfg(any(test, feature = "test-support"))] + self.shared + .event_reads + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.run("read events", |db| { + db.read(|db, connection| { + let mut events = Vec::new(); + db.query( + connection, + "SELECT position, line FROM events WHERE session_id = ?1 ORDER BY position", + (id,), + |row| { + let position = integer(row, 0)?; + let line = blob(row, 1)?; + let Ok(line) = std::str::from_utf8(&line) else { + log::warn!("skipping event {position} of {id}: not UTF-8"); + return Ok(()); + }; + let trimmed = line.trim(); + if trimmed.is_empty() { + return Ok(()); + } + match parse_stored_line(trimmed) { + Ok(stored) => events.push(stored), + Err(err) => { + log::warn!("skipping unparseable event {position} of {id}: {err}") + } + } + Ok(()) + }, + )?; + Ok(events) + }) + }) + } + + /// Remove a session from the index and delete its event log. + pub fn remove_session(&self, id: &str) -> io::Result<()> { + self.apply(&[Mutation::remove_session(id)]) + } +} + +impl Shared { + fn lock_state(&self) -> io::Result> { + // The state is replaced whole under the lock, so a poisoned guard + // still holds a consistent value. + Ok(self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner())) + } + + fn finished(&self) -> bool { + matches!( + *self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + State::Closed | State::Failed { .. } + ) + } + + fn release(&self) { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let State::Open(live) | State::Closing(live) = &mut *state { + live.in_flight -= 1; + } + self.idle.notify_all(); + } + + fn fail(&self, reason: String) { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let ownership = match std::mem::replace(&mut *state, State::Closed) { + State::Open(live) | State::Closing(live) => Some(live.ownership), + State::Owned(ownership) => Some(ownership), + State::Failed { _ownership, .. } => _ownership, + State::Unopened | State::Migrating | State::Closed => None, + }; + *state = State::Failed { + reason, + _ownership: ownership, + }; + self.idle.notify_all(); + } +} + +fn failed_error(state: &State) -> io::Error { + let State::Failed { reason, .. } = state else { + unreachable!("only called for a failed store") + }; + io::Error::other(Broken(format!( + "the session store stopped after an internal failure ({reason}); restart Tcode" + ))) +} + +/// Why a store in `state` cannot `operation` now. +fn unavailable(state: &State, operation: &str) -> io::Error { + match state { + State::Migrating => io::Error::new( + io::ErrorKind::ResourceBusy, + format!("cannot {operation}: the data directory is being migrated"), + ), + failed @ State::Failed { .. } => failed_error(failed), + _ => io::Error::new( + io::ErrorKind::BrokenPipe, + format!("cannot {operation}: the session store is closed"), + ), + } +} + +fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { + panic + .downcast_ref::<&str>() + .map(|message| (*message).to_owned()) + .or_else(|| panic.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".into()) +} + +/// Bring `tcode.db` up to date in the data dir `ownership` holds, and open it. +fn open_live(root: &Path, ownership: File) -> io::Result { + migrate::prepare(root)?; + let path = root.join(DB_FILE); + let db = Db::open(&path, false)?; + match db.user_version()? { + SCHEMA_VERSION => {} + 0 => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} was never completed (user_version 0). It is kept as it is and can be \ + examined with the sqlite3 command-line tool.", + path.display() + ), + )); + } + version => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} has schema version {version}, written by a newer Tcode; this build \ + understands version {SCHEMA_VERSION}", + path.display() + ), + )); + } + } + migrate::warn_about_stray_sources(root); + Ok(Live { + db: Arc::new(db), + ownership, + in_flight: 0, + }) +} + +fn acquire_ownership(root: &Path) -> io::Result { + let path = root.join(LOCK_FILE); + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&path)?; + let mut deadline = None; + let mut delay = Duration::from_millis(20); + loop { + match file.try_lock() { + Ok(()) => return Ok(file), + Err(fs::TryLockError::Error(error)) => return Err(error), + Err(fs::TryLockError::WouldBlock) => { + let deadline = *deadline.get_or_insert_with(|| { + // Peek only: the marker is consumed by the host after it + // owns the data dir. + crate::relaunch::pending(root).then(|| Instant::now() + RELAUNCH_WAIT) + }); + match deadline { + Some(deadline) if Instant::now() < deadline => { + std::thread::sleep(delay); + delay = (delay * 2).min(Duration::from_millis(250)); + } + _ => { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + format!( + "another Tcode host is already using the data directory {}; \ + quit it first, or set TCODE_DATA_DIR to use another directory", + root.display() + ), + )); + } + } + } + } + } +} + +fn read_index(db: &Db, connection: &turso::Connection) -> io::Result { + fn rows( + db: &Db, + connection: &turso::Connection, + table: &str, + ) -> io::Result> { + let mut values = Vec::new(); + db.query( + connection, + &format!("SELECT id, body FROM {table} ORDER BY rowid"), + (), + |row| { + let body = blob(row, 1)?; + match serde_json::from_slice(&body) { + Ok(value) => values.push(value), + Err(error) => log::warn!( + "skipping {table} row {:?} this build cannot read: {error}", + db::text(row, 0).unwrap_or_default() + ), + } + Ok(()) + }, + )?; + Ok(values) + } + Ok(IndexFile { + projects: rows(db, connection, "projects")?, + sessions: rows(db, connection, "sessions")?, + }) +} + +/// The raw segments of a native log: every line with its `\n`, and an +/// unterminated last line as it is. +fn segments(bytes: &[u8]) -> impl Iterator { + bytes.split_inclusive(|byte| *byte == b'\n') +} + +fn insert_segments( + db: &Db, + connection: &turso::Connection, + session_id: &str, + first: i64, + lines: impl IntoIterator>, +) -> io::Result { + let mut position = first; + for line in lines { + db.execute( + connection, + "INSERT INTO events (session_id, position, line) VALUES (?1, ?2, ?3)", + (session_id, position, line.as_ref()), + )?; + position += 1; + } + Ok(position) +} + +fn apply_op( + db: &Db, + connection: &turso::Connection, + op: &Op, + icons: &mut Vec>, +) -> io::Result<()> { + match op { + Op::AppendEvent { session_id, line } => { + let mut last = None; + db.query( + connection, + "SELECT position, substr(line, -1) = x'0a' FROM events \ + WHERE session_id = ?1 ORDER BY position DESC LIMIT 1", + (session_id.as_str(),), + |row| { + last = Some((integer(row, 0)?, integer(row, 1)? != 0)); + Ok(()) + }, + )?; + let next = match last { + None => 0, + Some((position, terminated)) => { + if !terminated { + // A log imported without a final newline: terminate + // its last line so the new record starts a line. + let mut unterminated = Vec::new(); + db.query( + connection, + "SELECT line FROM events WHERE session_id = ?1 AND position = ?2", + (session_id.as_str(), position), + |row| { + unterminated = blob(row, 0)?; + Ok(()) + }, + )?; + unterminated.push(b'\n'); + db.execute( + connection, + "UPDATE events SET line = ?3 WHERE session_id = ?1 AND position = ?2", + (session_id.as_str(), position, unterminated), + )?; + } + position + 1 + } + }; + insert_segments(db, connection, session_id, next, [line])?; + } + Op::ReplaceEventLog { session_id, bytes } => { + db.execute( + connection, + "DELETE FROM events WHERE session_id = ?1", + (session_id.as_str(),), + )?; + insert_segments(db, connection, session_id, 0, segments(bytes))?; + } + Op::CloneEvents { src, dst } => { + db.execute( + connection, + "DELETE FROM events WHERE session_id = ?1", + (dst.as_str(),), + )?; + db.execute( + connection, + "INSERT INTO events (session_id, position, line) \ + SELECT ?2, position, line FROM events WHERE session_id = ?1", + (src.as_str(), dst.as_str()), + )?; + } + Op::UpsertMeta(meta) => { + let body = serde_json::to_vec(meta.as_ref()).map_err(invalid_data)?; + db.execute( + connection, + "INSERT INTO sessions (id, body) VALUES (?1, ?2) \ + ON CONFLICT (id) DO UPDATE SET body = excluded.body", + (meta.id.as_str(), body), + )?; + } + Op::UpsertProject(project) => { + if let Some(previous) = stored_project(db, connection, &project.id)? + && previous.icon_path != project.icon_path + { + icons.push(previous.icon_path); + } + let body = serde_json::to_vec(project.as_ref()).map_err(invalid_data)?; + db.execute( + connection, + "INSERT INTO projects (id, body) VALUES (?1, ?2) \ + ON CONFLICT (id) DO UPDATE SET body = excluded.body", + (project.id.as_str(), body), + )?; + } + Op::RemoveSession(id) => { + db.execute( + connection, + "DELETE FROM events WHERE session_id = ?1", + (id.as_str(),), + )?; + db.execute( + connection, + "DELETE FROM sessions WHERE id = ?1", + (id.as_str(),), + )?; + } + Op::RemoveProject(id) => { + if let Some(previous) = stored_project(db, connection, id)? { + icons.push(previous.icon_path); + } + db.execute( + connection, + "DELETE FROM projects WHERE id = ?1", + (id.as_str(),), + )?; + } + } + Ok(()) +} + +fn stored_project( + db: &Db, + connection: &turso::Connection, + id: &str, +) -> io::Result> { + let mut project = None; + db.query( + connection, + "SELECT body FROM projects WHERE id = ?1", + (id,), + |row| { + project = serde_json::from_slice(&blob(row, 0)?).ok(); + Ok(()) + }, + )?; + Ok(project) +} + +fn invalid_data(error: impl Into>) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, error) +} + +/// Parse one JSONL line into a [`StoredEvent`], accepting both the timestamped +/// envelope and the legacy bare-event form. Envelope is tried first; a bare +/// event lacks the `ts`/`event` keys so it can't masquerade as one, and an +/// envelope lacks the top-level `type` tag so it can't parse as a bare event. +pub(crate) fn parse_stored_line(line: &str) -> Result { + match serde_json::from_str::(line) { + Ok(envelope) => Ok(StoredEvent { + ts: Some(envelope.ts), + event: envelope.event, + elided: None, + }), + Err(_envelope_err) => match serde_json::from_str::(line) { + Ok(event) => Ok(StoredEvent { + ts: None, + event, + elided: None, + }), + // Both forms failed: the line is genuinely corrupt. The bare-event + // error is the more informative one (the envelope attempt always + // fails on a bare event merely because `ts` is missing). + Err(bare_err) => Err(bare_err), + }, + } +} + +pub use tcode_core::project::now_secs; + +/// Current wall-clock time in unix milliseconds (used for event envelopes). +pub fn now_millis() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} diff --git a/crates/services/src/store/tests.rs b/crates/services/src/store/tests.rs new file mode 100644 index 000000000..19db868a1 --- /dev/null +++ b/crates/services/src/store/tests.rs @@ -0,0 +1,956 @@ +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::Path; + +use agent::{ApprovalMode, ProviderCommand, ProviderCommandKind, TurnStatus}; + +use std::sync::atomic::AtomicBool; + +use super::*; + +/// A data dir that is removed when the test ends. +struct DataDir(PathBuf); + +impl DataDir { + fn new() -> Self { + let path = std::env::temp_dir().join(format!("tcode-store-test-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(&path).unwrap(); + Self(path) + } + + fn path(&self) -> &Path { + &self.0 + } + + fn store(&self) -> SessionStore { + SessionStore::open_at(self.0.clone()).unwrap() + } + + /// A store over this data dir after the migration a start runs first. + fn migrated(&self) -> SessionStore { + let store = self.store(); + assert_eq!(migrate(&store).unwrap(), Migration::Completed); + store + } +} + +fn migrate(store: &SessionStore) -> io::Result { + store.migrate(|_| {}, &AtomicBool::new(false)) +} + +impl Drop for DataDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// A log as older builds wrote it, with every irregularity a real one can +/// hold: a legacy bare event, CRLF, a blank line, an unparseable line, a line +/// that is not UTF-8, and a torn last line without its newline. +const MIXED_LOG: &[u8] = b"{\"type\":\"turn_started\",\"turn_id\":\"legacy\"}\n\ +{\"ts\":2000,\"event\":{\"type\":\"turn_completed\",\"turn_id\":\"legacy\",\"status\":\"completed\",\"usage\":null}}\r\n\ +\n\ +{not valid json}\n\ +\xff\xfe{\"type\":\"turn_started\",\"turn_id\":\"not-utf8\"}\n\ +{\"ts\":2500,\"event\":{\"type\":\"turn_started\",\"turn_id\":\"second\"}}\n\ +{\"type\":\"turn_started"; + +const ORPHAN_LOG: &[u8] = + b"{\"ts\":1,\"event\":{\"type\":\"turn_started\",\"turn_id\":\"orphan\"}}\n"; + +/// `sessions.json` in the current object schema, with one session that +/// carries fields older builds wrote and this one dropped, and one that has +/// no project yet. +fn legacy_index() -> serde_json::Value { + serde_json::json!({ + "projects": [ + {"id": "p1", "name": "alpha", "root": "/work/alpha", "created_at": 1} + ], + "sessions": [ + { + "id": "mixed", "title": "Mixed", "provider": "codex", "cwd": "/work/alpha", + "project_id": "p1", "created_at": 1, "updated_at": 30, + "forked_from": "source", + "checkpoints": [{"turn": 2, "commit": "deadbeef", "event_offset": 7}] + }, + { + "id": "empty", "title": "Empty", "provider": "claude_code", + "cwd": "/work/beta", "created_at": 2, "updated_at": 20 + } + ] + }) +} + +fn write_legacy_fixture(root: &Path) { + fs::write(root.join("sessions.json"), legacy_index().to_string()).unwrap(); + fs::write(root.join("mixed.jsonl"), MIXED_LOG).unwrap(); + fs::write(root.join("empty.jsonl"), b"").unwrap(); + fs::write(root.join("orphan.jsonl"), ORPHAN_LOG).unwrap(); +} + +const LEGACY_FILES: [&str; 4] = [ + "sessions.json", + "mixed.jsonl", + "empty.jsonl", + "orphan.jsonl", +]; + +fn turn_ids(events: &[StoredEvent]) -> Vec<(Option, String)> { + events + .iter() + .map(|stored| { + let id = match &stored.event { + AgentEvent::TurnStarted { turn_id } => format!("started {turn_id}"), + AgentEvent::TurnCompleted { + turn_id, + status: TurnStatus::Completed, + .. + } => format!("completed {turn_id}"), + other => format!("{other:?}"), + }; + (stored.ts, id) + }) + .collect() +} + +/// What the fixture must read back as, wherever it was migrated. +fn assert_fixture_migrated(store: &SessionStore, root: &Path) { + let file = store.read_file().unwrap(); + assert_eq!(file.projects.len(), 2); + assert_eq!(file.projects[0].id, "p1"); + let derived = &file.projects[1]; + assert_eq!(derived.root, Path::new("/work/beta")); + assert_eq!(derived.name, "beta"); + assert_eq!( + file.sessions + .iter() + .map(|meta| (meta.id.as_str(), meta.project_id.as_deref())) + .collect::>(), + [("mixed", Some("p1")), ("empty", Some(derived.id.as_str()))] + ); + let mixed = &file.sessions[0]; + assert_eq!(mixed.title, "Mixed"); + assert_eq!(mixed.approval_mode, ApprovalMode::FullAccess); + assert_eq!(mixed.archived_at, None); + + assert_eq!(store.read_event_log("mixed").unwrap(), MIXED_LOG); + assert_eq!(store.read_event_log("orphan").unwrap(), ORPHAN_LOG); + assert!(store.read_event_log("empty").unwrap().is_empty()); + assert_eq!( + turn_ids(&store.read_events("mixed").unwrap()), + [ + (None, "started legacy".to_owned()), + (Some(2000), "completed legacy".to_owned()), + (Some(2500), "started second".to_owned()), + ] + ); + assert_eq!( + turn_ids(&store.read_events("orphan").unwrap()), + [(Some(1), "started orphan".to_owned())] + ); + + for name in LEGACY_FILES { + assert!(!root.join(name).exists(), "{name} left in the data dir"); + } + assert_eq!( + fs::read(root.join("legacy/sessions.json")).unwrap(), + legacy_index().to_string().as_bytes() + ); + assert_eq!( + fs::read(root.join("legacy/mixed.jsonl")).unwrap(), + MIXED_LOG + ); + assert_eq!( + fs::read(root.join("legacy/orphan.jsonl")).unwrap(), + ORPHAN_LOG + ); + assert!( + fs::read(root.join("legacy/empty.jsonl")) + .unwrap() + .is_empty() + ); + for leftover in [ + "tcode.db.migrating", + "tcode.db.migrating-wal", + "tcode.db.archive", + ] { + assert!(!root.join(leftover).exists(), "{leftover} left behind"); + } +} + +#[test] +fn first_start_migrates_the_json_index_and_every_log_byte_for_byte() { + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + let store = dir.store(); + assert!(store.needs_migration().unwrap()); + let error = store.open().unwrap_err(); + assert!(error.to_string().contains("not been migrated"), "{error}"); + assert_eq!(migrate(&store).unwrap(), Migration::Completed); + assert!(!store.needs_migration().unwrap()); + store.open().unwrap(); + assert_fixture_migrated(&store, dir.path()); + + // Appending to a log that lost its final newline terminates that line + // first, as appending to the file did. + store + .append_event( + "mixed", + 3000, + &AgentEvent::TurnStarted { + turn_id: "next".into(), + }, + ) + .unwrap(); + let mut expected = MIXED_LOG.to_vec(); + expected.extend_from_slice( + b"\n{\"ts\":3000,\"event\":{\"type\":\"turn_started\",\"turn_id\":\"next\"}}\n", + ); + assert_eq!(store.read_event_log("mixed").unwrap(), expected); + store.close().unwrap(); + assert!(store.read_file().is_err(), "a closed store serves nothing"); + + // A later start opens the database as it is: the data dir now holds only + // what an older build would write, and that is ignored. + fs::write(dir.path().join("sessions.json"), b"[]").unwrap(); + fs::write(dir.path().join("mixed.jsonl"), b"").unwrap(); + let reopened = dir.store(); + assert!(!reopened.needs_migration().unwrap()); + assert_eq!(reopened.read_event_log("mixed").unwrap(), expected); + assert_eq!(reopened.read_file().unwrap().sessions.len(), 2); + assert_eq!(fs::read(dir.path().join("sessions.json")).unwrap(), b"[]"); + assert_eq!( + fs::read(dir.path().join("legacy/mixed.jsonl")).unwrap(), + MIXED_LOG + ); +} + +#[test] +fn legacy_bare_array_index_migrates_with_one_derived_project_per_root() { + let dir = DataDir::new(); + // Old-format file: a bare JSON array with no project_id fields. + let legacy = serde_json::json!([ + { + "id": "s1", "title": "One", "provider": "claude_code", + "cwd": "/work/alpha", "created_at": 1, "updated_at": 10 + }, + { + "id": "s2", "title": "Two", "provider": "codex", + "cwd": "/work/alpha", "created_at": 2, "updated_at": 20 + }, + { + "id": "s3", "title": "Three", "provider": "codex", + "cwd": "/work/beta", "created_at": 3, "updated_at": 30 + } + ]); + fs::write(dir.path().join("sessions.json"), legacy.to_string()).unwrap(); + + let file = dir.migrated().read_file().unwrap(); + // Two distinct roots -> two derived projects, deduped by root. + assert_eq!(file.projects.len(), 2); + let alpha = file + .projects + .iter() + .find(|p| p.root == Path::new("/work/alpha")) + .unwrap(); + assert_eq!(alpha.name, "alpha"); + let project_of = |id: &str| { + file.sessions + .iter() + .find(|session| session.id == id) + .unwrap() + .project_id + .clone() + }; + assert_eq!(project_of("s1"), Some(alpha.id.clone())); + assert_eq!(project_of("s2"), project_of("s1")); + assert_ne!(project_of("s3"), project_of("s1")); + // Newest first. + assert_eq!( + dir.store() + .load_index() + .unwrap() + .iter() + .map(|meta| meta.id.as_str()) + .collect::>(), + ["s3", "s2", "s1"] + ); +} + +#[test] +fn an_unparseable_index_is_archived_unchanged_and_the_logs_still_migrate() { + let dir = DataDir::new(); + let corrupt = b"not valid session json"; + fs::write(dir.path().join("sessions.json"), corrupt).unwrap(); + fs::write(dir.path().join("orphan.jsonl"), ORPHAN_LOG).unwrap(); + let store = dir.migrated(); + + let file = store.read_file().unwrap(); + assert!(file.projects.is_empty() && file.sessions.is_empty()); + assert_eq!(store.read_event_log("orphan").unwrap(), ORPHAN_LOG); + assert!(!dir.path().join("sessions.json").exists()); + assert_eq!( + fs::read(dir.path().join("legacy/sessions.json")).unwrap(), + corrupt + ); +} + +#[test] +fn duplicate_index_ids_fail_the_migration_and_leave_the_sources_alone() { + let dir = DataDir::new(); + let session = serde_json::json!({ + "id": "twice", "title": "One", "provider": "codex", + "cwd": "/work", "created_at": 1, "updated_at": 1 + }); + let index = serde_json::json!({"projects": [], "sessions": [session, session]}).to_string(); + fs::write(dir.path().join("sessions.json"), &index).unwrap(); + fs::write(dir.path().join("twice.jsonl"), ORPHAN_LOG).unwrap(); + + let error = migrate(&dir.store()).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("twice"), "{error}"); + assert!(!dir.path().join(DB_FILE).exists()); + assert!(!dir.path().join(LEGACY_DIR_NAME).exists()); + assert_eq!( + fs::read(dir.path().join("sessions.json")).unwrap(), + index.as_bytes() + ); + assert_eq!( + fs::read(dir.path().join("twice.jsonl")).unwrap(), + ORPHAN_LOG + ); +} + +const LEGACY_DIR_NAME: &str = "legacy"; + +/// Put the migrated sources back where an interruption would have left them. +fn restore_sources(root: &Path) { + for name in LEGACY_FILES { + fs::rename(root.join("legacy").join(name), root.join(name)).unwrap(); + } +} + +fn archive_list(names: &[&str]) -> Vec { + serde_json::to_vec(names).unwrap() +} + +#[test] +fn a_start_after_an_interrupted_migration_discards_the_staging_files_and_starts_over() { + // Killed while building: a torn staging file and WAL, no final database. + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + fs::write(dir.path().join("tcode.db.migrating"), b"partial").unwrap(); + fs::write(dir.path().join("tcode.db.migrating-wal"), b"torn frames").unwrap(); + let store = dir.migrated(); + assert_fixture_migrated(&store, dir.path()); + store.close().unwrap(); + + // Killed after the staged database was complete and the archive list was + // written, but before the rename was durable: the sources are all still + // in the data dir. + fs::rename( + dir.path().join(DB_FILE), + dir.path().join("tcode.db.migrating"), + ) + .unwrap(); + restore_sources(dir.path()); + fs::write( + dir.path().join("tcode.db.archive"), + archive_list(&LEGACY_FILES), + ) + .unwrap(); + let store = dir.migrated(); + assert_fixture_migrated(&store, dir.path()); +} + +/// Every file in `root`, by name, with its bytes; `legacy/` as a marker. +/// The store's own files are recorded by name only: Windows locks the whole +/// of a file the store holds, so even the empty `tcode.lock` cannot be read. +fn snapshot(root: &Path) -> std::collections::BTreeMap> { + fs::read_dir(root) + .unwrap() + .map(|entry| { + let entry = entry.unwrap(); + let name = entry.file_name().into_string().unwrap(); + let bytes = if entry.file_type().unwrap().is_dir() { + b"".to_vec() + } else if name == LOCK_FILE || name.starts_with(DB_FILE) { + b"".to_vec() + } else { + fs::read(entry.path()).unwrap() + }; + (name, bytes) + }) + .collect() +} + +#[test] +fn a_cancelled_migration_changes_no_source_and_the_next_one_completes() { + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + // Longer than one chunk, so a cancel can land inside a log. + let bulk_line = format!( + "{{\"ts\":1,\"event\":{{\"type\":\"turn_started\",\"turn_id\":\"{}\"}}}}\n", + "x".repeat(4000) + ); + let bulk = bulk_line.repeat(2600); + assert!(bulk.len() > migrate::CHUNK_BYTES); + fs::write(dir.path().join("bulk.jsonl"), &bulk).unwrap(); + let store = dir.store(); + assert!(store.needs_migration().unwrap()); + let mut sources = snapshot(dir.path()); + // Owned for the whole start, and kept from the first check. + sources.remove(LOCK_FILE); + + type Stop = fn(&MigrationProgress) -> bool; + let stops: [(&str, Stop); 4] = [ + ("scanning", |progress| { + progress.phase == MigrationPhase::Scanning + }), + ("inside a log", |progress| { + progress.phase == MigrationPhase::Importing + && progress.bytes_done > 0 + && progress.threads_done == 0 + }), + ("between logs", |progress| { + progress.phase == MigrationPhase::Importing && progress.threads_done == 2 + }), + ("verifying", |progress| { + progress.phase == MigrationPhase::Verifying && progress.threads_done == 1 + }), + ]; + for (point, stop) in stops { + let cancel = AtomicBool::new(false); + let mut seen = None; + let outcome = store + .migrate( + |progress| { + if !cancel.load(Ordering::Relaxed) && stop(&progress) { + seen = Some(progress); + cancel.store(true, Ordering::Relaxed); + } + }, + &cancel, + ) + .unwrap(); + assert_eq!(outcome, Migration::Cancelled, "{point}"); + assert!(seen.is_some(), "{point}: never reached"); + let mut after = snapshot(dir.path()); + assert!(after.remove(LOCK_FILE).is_some()); + assert_eq!(after, sources, "{point}"); + assert!(store.needs_migration().unwrap(), "{point}"); + } + + let mut reports = Vec::new(); + let outcome = store + .migrate(|progress| reports.push(progress), &AtomicBool::new(false)) + .unwrap(); + assert_eq!(outcome, Migration::Completed); + let mut phases = reports + .iter() + .map(|progress| progress.phase) + .collect::>(); + phases.dedup(); + assert_eq!( + phases, + [ + MigrationPhase::Scanning, + MigrationPhase::Importing, + MigrationPhase::Verifying, + MigrationPhase::Publishing, + MigrationPhase::Archiving + ] + ); + let log_bytes = (MIXED_LOG.len() + ORPHAN_LOG.len() + bulk.len()) as u64; + for phase in [MigrationPhase::Importing, MigrationPhase::Verifying] { + let last = reports + .iter() + .rfind(|progress| progress.phase == phase) + .unwrap(); + assert_eq!( + ( + last.threads_done, + last.threads_total, + last.bytes_done, + last.bytes_total + ), + (4, 4, log_bytes, log_bytes), + "{phase:?}" + ); + } + let last = reports.last().unwrap(); + assert_eq!( + (last.threads_done, last.bytes_done), + (last.threads_total, last.bytes_total) + ); + assert_eq!(store.read_event_log("bulk").unwrap(), bulk.as_bytes()); + assert_fixture_migrated(&store, dir.path()); +} + +#[test] +fn a_start_after_an_interrupted_archival_finishes_it_without_overwriting() { + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + dir.migrated().close().unwrap(); + + // Killed after the rename while moving the sources: two are still in the + // data dir. One of them also has a copy in legacy/ already, which must + // survive untouched while the source stays where it is. + fs::rename( + dir.path().join("legacy/mixed.jsonl"), + dir.path().join("mixed.jsonl"), + ) + .unwrap(); + fs::copy( + dir.path().join("legacy/orphan.jsonl"), + dir.path().join("orphan.jsonl"), + ) + .unwrap(); + fs::write(dir.path().join("legacy/orphan.jsonl"), b"older copy").unwrap(); + fs::write( + dir.path().join("tcode.db.archive"), + archive_list(&LEGACY_FILES), + ) + .unwrap(); + + let store = dir.store(); + store.open().unwrap(); + assert!(!dir.path().join("tcode.db.archive").exists()); + assert!(!dir.path().join("mixed.jsonl").exists()); + assert_eq!( + fs::read(dir.path().join("legacy/mixed.jsonl")).unwrap(), + MIXED_LOG + ); + assert_eq!( + fs::read(dir.path().join("orphan.jsonl")).unwrap(), + ORPHAN_LOG + ); + assert_eq!( + fs::read(dir.path().join("legacy/orphan.jsonl")).unwrap(), + b"older copy" + ); + assert_eq!(store.read_event_log("mixed").unwrap(), MIXED_LOG); +} + +#[test] +fn a_fresh_data_dir_gets_a_completed_database_through_the_staged_path() { + let dir = DataDir::new(); + let store = dir.store(); + assert!(store.load_index().unwrap().is_empty()); + store.close().unwrap(); + let db = db::Db::open(&dir.path().join(DB_FILE), false).unwrap(); + assert_eq!(db.user_version().unwrap(), SCHEMA_VERSION); + drop(db); + assert!(!dir.path().join("legacy").exists()); + assert!(!dir.path().join("tcode.db.migrating").exists()); +} + +#[test] +fn an_unfinished_newer_or_corrupt_database_is_refused_and_left_untouched() { + let set_version = |root: &Path, version: i64| { + let db = db::Db::open(&root.join(DB_FILE), false).unwrap(); + db.write(|db, connection| { + db.execute(connection, &format!("PRAGMA user_version = {version}"), ()) + .map(drop) + }) + .unwrap(); + db.checkpoint().unwrap(); + }; + for (version, expected) in [(0, "never completed"), (2, "newer Tcode")] { + let dir = DataDir::new(); + dir.store() + .upsert_meta(&SessionMeta::new(ProviderKind::Codex, "/w".into(), None)) + .unwrap(); + dir.store().close().unwrap(); + set_version(dir.path(), version); + let before = fs::read(dir.path().join(DB_FILE)).unwrap(); + let error = dir.store().open().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains(expected), "{error}"); + assert_eq!(fs::read(dir.path().join(DB_FILE)).unwrap(), before); + } + + let dir = DataDir::new(); + let garbage = vec![0x5a_u8; 8192]; + fs::write(dir.path().join(DB_FILE), &garbage).unwrap(); + let error = dir.store().open().unwrap_err(); + assert!(error.to_string().contains("sqlite3"), "{error}"); + assert!(error.to_string().contains(DB_FILE), "{error}"); + assert_eq!(fs::read(dir.path().join(DB_FILE)).unwrap(), garbage); +} + +#[test] +fn session_index_upserts_orders_and_removes_only_the_selected_session() { + let dir = DataDir::new(); + let store = dir.store(); + let mut a = SessionMeta::new(ProviderKind::Codex, PathBuf::from("/a"), None); + a.updated_at = 100; + let mut b = SessionMeta::new(ProviderKind::ClaudeCode, PathBuf::from("/b"), None); + b.updated_at = 200; + store.upsert_meta(&a).unwrap(); + store.upsert_meta(&b).unwrap(); + + let index = store.load_index().unwrap(); + assert_eq!(index.len(), 2); + // newest first + assert_eq!(index[0].id, b.id); + assert_eq!(index[1].id, a.id); + + // upsert replaces + let mut a2 = a.clone(); + a2.title = "renamed".into(); + store.upsert_meta(&a2).unwrap(); + let index = store.load_index().unwrap(); + assert_eq!(index.len(), 2); + assert_eq!( + index.iter().find(|m| m.id == a.id).unwrap().title, + "renamed" + ); + for (meta, ts) in [(&a, 1), (&b, 2)] { + store + .append_event( + &meta.id, + ts, + &AgentEvent::TurnStarted { + turn_id: format!("turn-{ts}"), + }, + ) + .unwrap(); + } + store.remove_session(&a.id).unwrap(); + store.remove_session(&a.id).unwrap(); + let remaining = store.load_index().unwrap(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].id, b.id); + assert!(store.read_event_log(&a.id).unwrap().is_empty()); + assert_eq!( + store.read_event_log(&b.id).unwrap(), + b"{\"ts\":2,\"event\":{\"type\":\"turn_started\",\"turn_id\":\"turn-2\"}}\n" + ); +} + +#[test] +fn a_cloned_or_imported_log_keeps_its_exact_bytes() { + let dir = DataDir::new(); + let store = dir.store(); + let fork = SessionMeta::new(ProviderKind::Codex, PathBuf::from("/w"), None); + store + .apply(&[ + Mutation::replace_event_log("source", MIXED_LOG.to_vec()), + Mutation::clone_events("source", &fork.id), + Mutation::upsert_meta(fork.clone()), + Mutation::clone_events("missing", "empty-fork"), + ]) + .unwrap(); + assert_eq!(store.read_event_log("source").unwrap(), MIXED_LOG); + assert_eq!(store.read_event_log(&fork.id).unwrap(), MIXED_LOG); + assert!(store.read_event_log("empty-fork").unwrap().is_empty()); + assert_eq!(store.load_index().unwrap(), [fork]); +} + +#[test] +fn removing_project_cleans_only_managed_icons() { + let dir = DataDir::new(); + let root = dir.path(); + let store = dir.store(); + fs::create_dir(root.join("project-icons")).unwrap(); + let mut project = Project::from_root(root.join("project")); + for (path, managed) in [ + (root.join("project-icons/icon.png"), true), + (root.join("original.png"), false), + ] { + fs::write(&path, b"stored image").unwrap(); + project.icon_path = Some(path.clone()); + store.upsert_project(&project).unwrap(); + assert_eq!( + store.read_file().unwrap().projects[0].icon_path.as_ref(), + Some(&path) + ); + store.remove_project(&project.id).unwrap(); + assert!(store.read_file().unwrap().projects.is_empty()); + assert_eq!(path.exists(), !managed); + } +} + +#[test] +fn command_cache_roundtrips_per_provider_and_acp_agent() { + let dir = DataDir::new(); + let root = dir.path().to_path_buf(); + let store = dir.store(); + let native = vec![ProviderCommand { + name: "review".into(), + description: Some("Review the current changes".into()), + kind: ProviderCommandKind::Command, + }]; + let acp = vec![ProviderCommand { + name: "browser".into(), + description: None, + kind: ProviderCommandKind::Skill, + }]; + store + .save_commands(ProviderKind::ClaudeCode, None, &native) + .unwrap(); + store + .save_commands(ProviderKind::Acp, Some("vendor/agent"), &acp) + .unwrap(); + + // Reopen the store to prove the values come from disk, not memory. + let reopened = SessionStore::open_at(root.clone()).unwrap(); + assert_eq!( + reopened.load_commands(ProviderKind::ClaudeCode, None), + native + ); + assert_eq!( + reopened.load_commands(ProviderKind::Acp, Some("vendor/agent")), + acp + ); + assert!( + reopened + .load_commands(ProviderKind::Acp, Some("different-agent")) + .is_empty() + ); + assert!(root.join("commands-claude.json").is_file()); + assert!( + root.join("commands-acp-76656e646f722f6167656e74.json") + .is_file() + ); +} + +/// This test binary, re-run as another process that opens `root` and holds +/// it until its stdin closes. It reports `OPEN` or `ERR `. +struct Child { + process: std::process::Child, + stdout: BufReader, + report: String, +} + +impl Child { + fn spawn(root: &Path) -> std::process::Child { + crate::process::command(std::env::current_exe().unwrap()) + .args([ + "--exact", + "store::tests::store_owner_process", + "--ignored", + "--nocapture", + "--test-threads=1", + ]) + .env("TCODE_STORE_OWNER_DIR", root) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap() + } + + /// Wait for the child to say whether it owns the data dir. + fn report(mut process: std::process::Child) -> Self { + let mut stdout = BufReader::new(process.stdout.take().unwrap()); + // libtest prints the test's name on the same line first. + let mut line = String::new(); + let report = loop { + line.clear(); + assert!( + stdout.read_line(&mut line).unwrap() > 0, + "the child exited without reporting" + ); + if let Some((_, report)) = line.trim_end().split_once(OWNER_REPORT) { + break report.to_owned(); + } + }; + Self { + process, + stdout, + report, + } + } + + /// Close the child's stdin, which makes it close the store and exit. + fn release(mut self) { + drop(self.process.stdin.take()); + std::io::copy(&mut self.stdout, &mut std::io::sink()).unwrap(); + assert!(self.process.wait().unwrap().success()); + } +} + +const OWNER_REPORT: &str = "store owner: "; + +#[test] +#[ignore = "the other process of the ownership tests below, which run it"] +fn store_owner_process() { + let root = PathBuf::from(std::env::var_os("TCODE_STORE_OWNER_DIR").unwrap()); + let store = SessionStore::open_at(root).unwrap(); + let opened = store.needs_migration().and_then(|needed| { + if needed { + migrate(&store)?; + } + store.open() + }); + match opened { + Ok(()) => println!("{OWNER_REPORT}OPEN"), + Err(error) => println!("{OWNER_REPORT}ERR {:?} {error}", error.kind()), + } + std::io::stdout().flush().unwrap(); + let _ = std::io::stdin().read(&mut [0_u8; 1]); + store.close().unwrap(); +} + +#[test] +fn a_second_process_is_refused_and_a_relaunch_waits_for_the_owner_to_exit() { + let dir = DataDir::new(); + let owner = Child::report(Child::spawn(dir.path())); + assert_eq!(owner.report, "OPEN"); + + let error = dir.store().open().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::ResourceBusy); + assert!( + error + .to_string() + .contains(&dir.path().display().to_string()), + "{error}" + ); + + // A permission relaunch starts while the old instance is still closing. + crate::relaunch::write( + dir.path(), + &crate::relaunch::RelaunchMarker { + reopen_settings: "computer_use".into(), + active_session: None, + }, + ) + .unwrap(); + let waiting = std::thread::spawn({ + let store = dir.store(); + move || store.open() + }); + std::thread::sleep(Duration::from_millis(300)); + assert!(!waiting.is_finished(), "the relaunch waits for the owner"); + owner.release(); + waiting.join().unwrap().unwrap(); + // Only peeked: the host consumes the marker once it owns the data dir. + assert!(crate::relaunch::pending(dir.path())); +} + +#[test] +fn two_simultaneous_first_launches_migrate_once() { + let dir = DataDir::new(); + write_legacy_fixture(dir.path()); + let (first, second) = (Child::spawn(dir.path()), Child::spawn(dir.path())); + let (first, second) = (Child::report(first), Child::report(second)); + let mut reports = [first.report.clone(), second.report.clone()]; + reports.sort(); + assert_eq!(reports[1], "OPEN", "{reports:?}"); + assert!(reports[0].starts_with("ERR ResourceBusy"), "{reports:?}"); + first.release(); + second.release(); + + let store = dir.store(); + assert_fixture_migrated(&store, dir.path()); +} + +/// Kill the migration at random points — while logs are copied, around the +/// checkpoint, the rename, the directory syncs and the archival — and restart +/// it: there is either no `tcode.db` and every source is where it was, or a +/// complete one, and the finished migration always holds every source byte +/// for byte. `TCODE_MIGRATION_KILLS` sets the number of kills. +#[test] +#[ignore = "spawns and SIGKILLs child processes over a large fixture; run deliberately"] +fn migration_survives_sigkill() { + let runs: usize = std::env::var("TCODE_MIGRATION_KILLS") + .ok() + .and_then(|runs| runs.parse().ok()) + .unwrap_or(20); + let originals = DataDir::new(); + write_legacy_fixture(originals.path()); + let line = format!( + "{{\"ts\":1,\"event\":{{\"type\":\"turn_started\",\"turn_id\":\"{}\"}}}}\n", + "x".repeat(1000) + ); + for index in 0..24 { + fs::write( + originals.path().join(format!("bulk-{index}.jsonl")), + line.repeat(300 + index * 20), + ) + .unwrap(); + } + let names: Vec = fs::read_dir(originals.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect(); + let mut seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 + | 1; + // Kills spread over the time an uninterrupted migration takes. + let calibration = DataDir::new(); + for name in &names { + fs::copy(originals.path().join(name), calibration.path().join(name)).unwrap(); + } + let started = Instant::now(); + let full = Child::report(Child::spawn(calibration.path())); + let window = started.elapsed().as_millis() as u64; + assert_eq!(full.report, "OPEN"); + full.release(); + let mut outcomes = HashMap::<&str, usize>::new(); + for run in 0..runs { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let dir = DataDir::new(); + for name in &names { + fs::copy(originals.path().join(name), dir.path().join(name)).unwrap(); + } + let mut owner = Child::spawn(dir.path()); + std::thread::sleep(Duration::from_millis(seed % (window + window / 10))); + owner.kill().unwrap(); + owner.wait().unwrap(); + + // What the kill left: no database and untouched sources, or a + // completed database and every source in the data dir or legacy/. + let source = |name: &str| { + [dir.path().join(name), dir.path().join("legacy").join(name)] + .into_iter() + .find(|path| path.exists()) + }; + let outcome = if dir.path().join(DB_FILE).exists() { + let db = db::Db::open(&dir.path().join(DB_FILE), false).unwrap(); + assert_eq!(db.user_version().unwrap(), SCHEMA_VERSION, "run {run}"); + drop(db); + if dir.path().join("tcode.db.archive").exists() { + "published, archival pending" + } else { + "complete" + } + } else { + for name in &names { + assert!( + dir.path().join(name).exists(), + "run {run}: {name} moved early" + ); + } + if dir.path().join("tcode.db.migrating").exists() { + "staging" + } else { + "not started" + } + }; + *outcomes.entry(outcome).or_default() += 1; + for name in &names { + assert_eq!( + fs::read(source(name).unwrap_or_else(|| panic!("run {run}: {name} lost"))).unwrap(), + fs::read(originals.path().join(name)).unwrap(), + "run {run}: {name}" + ); + } + + let store = dir.migrated(); + for name in names.iter().filter_map(|name| name.strip_suffix(".jsonl")) { + assert_eq!( + store.read_event_log(name).unwrap(), + fs::read(originals.path().join(format!("{name}.jsonl"))).unwrap(), + "run {run}: {name}" + ); + } + assert_eq!(store.read_file().unwrap().sessions.len(), 2, "run {run}"); + store.close().unwrap(); + } + println!("{runs} kills over a {window} ms migration: {outcomes:?}"); +} diff --git a/crates/ui/src/lib.rs b/crates/ui/src/lib.rs index a3255293b..12f45e376 100644 --- a/crates/ui/src/lib.rs +++ b/crates/ui/src/lib.rs @@ -73,9 +73,13 @@ pub use i18n::{ LANGUAGE_ENGLISH, LANGUAGE_SIMPLIFIED_CHINESE, apply_locale, resolve_locale, set_locale, translate, translate_with_args, }; -pub use run::{ShellOptions, THEME_JSON, flattened_theme_json, last_host_target, run_shell}; +pub use run::{ + ShellOptions, THEME_JSON, flattened_theme_json, init_client, last_host_target, + open_client_window, run_shell, +}; pub(crate) use shell::window_drag_area; pub use shell::{AppShell, Quit, ShellSetup, TogglePalette, handle_back}; +pub use thread_export::format_size; pub use window_seam::force_mobile_layout; pub use window_state::{OpenThread, WindowState}; diff --git a/crates/ui/src/run.rs b/crates/ui/src/run.rs index 15e6bfc7c..3dd836c3d 100644 --- a/crates/ui/src/run.rs +++ b/crates/ui/src/run.rs @@ -11,7 +11,8 @@ use std::cell::RefCell; use std::rc::Rc; use gpui::{ - AnyWindowHandle, App, AppContext as _, Entity, KeyBinding, SharedString, WindowOptions, + AnyWindowHandle, App, AppContext as _, Entity, KeyBinding, Render, SharedString, Window, + WindowOptions, }; use tcode_client::host::ClientHost; @@ -99,15 +100,17 @@ impl ShellOptions { } } -/// Open this client's window on the shared shell. -/// -/// Returns the window and its shell, so bootstrap can keep doing whatever is -/// genuinely its own (a smoke run, a launch flag, a platform Back callback). -pub fn run_shell( - cx: &mut App, - host: Rc, - options: ShellOptions, -) -> (AnyWindowHandle, Entity) { +/// Process-wide setup every window of this client needs: locale, fonts, +/// theme, markdown and keybindings. [`run_shell`] runs it; a bootstrap that +/// opens a window of its own before the shell runs it first, and the second +/// call does nothing. +pub fn init_client(cx: &mut App, host: &dyn ClientHost, options: &ShellOptions) { + struct Initialized; + impl gpui::Global for Initialized {} + if cx.has_global::() { + return; + } + cx.set_global(Initialized); // Browser bootstrap supplies its Fetch client; native image URLs need an HTTP client too. #[cfg(not(target_family = "wasm"))] { @@ -127,7 +130,7 @@ pub fn run_shell( override_locale => override_locale, }); cx.text_system() - .add_fonts(options.fonts) + .add_fonts(options.fonts.clone()) .expect("failed to register bundled application fonts"); theme::init_with_json(&options.theme_json, cx); crate::markdown::init(cx); @@ -139,6 +142,50 @@ pub fn run_shell( cx.bind_keys([KeyBinding::new("secondary-k", TogglePalette, None)]); #[cfg(target_os = "macos")] cx.bind_keys([KeyBinding::new("cmd-q", crate::shell::Quit, None)]); +} + +/// Open a window of this client around the view `build` returns: titled, in +/// the system appearance, on the platform material its options ask for, and +/// with the overlay host every dialog and toast needs. +pub fn open_client_window( + cx: &mut App, + options: WindowOptions, + title: SharedString, + build: impl FnOnce(&mut Window, &mut App) -> Entity + 'static, +) -> AnyWindowHandle { + #[cfg(target_os = "macos")] + let window_background = options.window_background; + let window = cx + .open_window(options, move |window, cx| { + window.set_window_title(&title); + theme::sync_system_appearance(Some(window), cx); + let view = build(window, cx); + cx.new(|cx| gpui_base::Root::new(view, window, cx)) + }) + .expect("failed to open the tcode window"); + // A transparent macOS window gets its blur from a stock semantic material + // rather than GPUI's `Blurred` path; see `macos_backdrop`. + #[cfg(target_os = "macos")] + if window_background == gpui::WindowBackgroundAppearance::Transparent { + use crate::theme::ActiveTheme as _; + let _ = window.update(cx, |_, window, cx| { + crate::macos_backdrop::install(window, cx); + theme::change_mode(cx.theme().mode, Some(window), cx); + }); + } + window.into() +} + +/// Open this client's window on the shared shell. +/// +/// Returns the window and its shell, so bootstrap can keep doing whatever is +/// genuinely its own (a smoke run, a launch flag, a platform Back callback). +pub fn run_shell( + cx: &mut App, + host: Rc, + options: ShellOptions, +) -> (AnyWindowHandle, Entity) { + init_client(cx, host.as_ref(), &options); if options.activate { cx.activate(true); } @@ -163,36 +210,19 @@ pub fn run_shell( }, )); let captured = mounted.clone(); - #[cfg(target_os = "macos")] - let window_background = options.window.window_background; - let window: AnyWindowHandle = cx - .open_window(options.window, move |window, cx| { - window.set_window_title(&title); - theme::sync_system_appearance(Some(window), cx); - let window_state = cx.new(|_| WindowState::new(false)); - let shell = cx.new(|cx| { - AppShell::new( - window_state, - setup.borrow_mut().take().unwrap_or_default(), - window, - cx, - ) - }); - *captured.borrow_mut() = Some(shell.clone()); - cx.new(|cx| gpui_base::Root::new(shell, window, cx)) - }) - .expect("failed to open the tcode window") - .into(); - // A transparent macOS window gets its blur from a stock semantic material - // rather than GPUI's `Blurred` path; see `macos_backdrop`. - #[cfg(target_os = "macos")] - if window_background == gpui::WindowBackgroundAppearance::Transparent { - use crate::theme::ActiveTheme as _; - let _ = window.update(cx, |_, window, cx| { - crate::macos_backdrop::install(window, cx); - theme::change_mode(cx.theme().mode, Some(window), cx); + let window = open_client_window(cx, options.window, title, move |window, cx| { + let window_state = cx.new(|_| WindowState::new(false)); + let shell = cx.new(|cx| { + AppShell::new( + window_state, + setup.borrow_mut().take().unwrap_or_default(), + window, + cx, + ) }); - } + *captured.borrow_mut() = Some(shell.clone()); + shell + }); let shell = mounted .borrow_mut() .take() diff --git a/crates/ui/src/thread_export.rs b/crates/ui/src/thread_export.rs index 7b8bb09aa..04585f564 100644 --- a/crates/ui/src/thread_export.rs +++ b/crates/ui/src/thread_export.rs @@ -228,12 +228,14 @@ fn save_to_disk( ) { } -pub(crate) fn format_size(bytes: usize) -> String { +pub fn format_size(bytes: usize) -> String { if bytes < 1024 { format!("{bytes} B") } else if bytes < 1024 * 1024 { format!("{:.1} KB", bytes as f64 / 1024.) - } else { + } else if bytes < 1024 * 1024 * 1024 { format!("{:.1} MB", bytes as f64 / (1024. * 1024.)) + } else { + format!("{:.1} GB", bytes as f64 / (1024. * 1024. * 1024.)) } } diff --git a/locales/en.yml b/locales/en.yml index 788ee4b12..263cdb092 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -218,6 +218,17 @@ quit: title: "Quit Tcode?" description: "%{count} session(s) are still working. Quitting will stop them." confirm: "Quit" + startup_failed: "Tcode could not start" +migration: + title: "Upgrading thread storage" + description: "Tcode is copying your threads into a new database, tcode.db. The original files are kept in the data folder's legacy/ directory." + scanning: "Looking for threads…" + importing: "Copying threads: %{done} of %{total} · %{bytes} of %{total_bytes}" + verifying: "Checking the copy: %{done} of %{total} · %{bytes} of %{total_bytes}" + publishing: "Finishing…" + archiving: "Moving the original files into legacy/…" + quitting: "Stopping. The original files are unchanged." + failed: "The upgrade could not finish: %{reason}" settings: title: "Settings" sections: "All settings" diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index 35cda3304..b565a9ee6 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -218,6 +218,17 @@ quit: title: "退出 Tcode？" description: "仍有 %{count} 个会话正在工作。退出将停止这些会话。" confirm: "退出" + startup_failed: "Tcode 无法启动" +migration: + title: "正在升级对话存储" + description: "Tcode 正在把你的对话复制到新的数据库 tcode.db，原始文件会保留在数据目录的 legacy/ 文件夹中。" + scanning: "正在查找对话…" + importing: "正在复制对话：%{done}/%{total} · %{bytes}/%{total_bytes}" + verifying: "正在校验副本：%{done}/%{total} · %{bytes}/%{total_bytes}" + publishing: "即将完成…" + archiving: "正在把原始文件移入 legacy/…" + quitting: "正在停止，原始文件未被改动。" + failed: "升级未能完成：%{reason}" settings: title: "设置" sections: "全部设置"