Skip to content

Update convo prune logic #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jagerman merged 10 commits into from
Feb 10, 2026
Merged

Update convo prune logic #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1c3fde3
Update convo prune logic
Feb 6, 2026
062b54f
Update gpg
Feb 9, 2026
330221f
Fix tests
Feb 9, 2026
e044978
Remove unused include
Feb 9, 2026
9c63b7c
Formatting
Feb 9, 2026
afa4e6e
More formatting
Feb 9, 2026
25c450e
sys_time template
Feb 9, 2026
0a0ab47
Update clang and llvm version
Feb 9, 2026
bd34a3e
Update to use virtual load method, and pro_base
Feb 10, 2026
4cb4102
Simplify is_stale
Feb 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .drone.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,8 +354,8 @@ local static_build(name,
debian_build('Debian sid', docker_base + 'debian-sid'),
debian_build('Debian sid/Debug', docker_base + 'debian-sid', build_type='Debug'),
debian_build('Debian testing', docker_base + 'debian-testing'),
clang(17),
full_llvm(17),
clang(19),
full_llvm(19),
debian_build('Debian stable (i386)', docker_base + 'debian-stable/i386'),
debian_build('Debian 12', docker_base + 'debian-bookworm'),
debian_build('Ubuntu latest', docker_base + 'ubuntu-rolling'),
Expand Down
31 changes: 19 additions & 12 deletions include/session/config/convo_info_volatile.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,22 +74,36 @@ namespace convo {
int64_t last_read = 0;
bool unread = false;

virtual ~base() = default;

protected:
void load(const dict& info_dict);
virtual void load(const dict& info_dict);
friend class session::config::val_loader;
friend class session::config::ConvoInfoVolatile;
};

struct one_to_one : base {
std::string session_id; // in hex
base() = default;
base(int64_t last_read, bool unread) : last_read(last_read), unread(unread) {}
};

struct pro_base : base {
/// Hash of the generation index set by the Session Pro Backend
std::optional<array_uc32> pro_gen_index_hash;

/// Unix epoch timestamp to which this proof's entitlement to Session Pro features is valid
/// to
std::chrono::sys_time<std::chrono::milliseconds> pro_expiry_unix_ts{};

protected:
using base::base;

void load(const dict& info_dict) override;
friend class session::config::val_loader;
friend class session::config::ConvoInfoVolatile;
};

struct one_to_one : pro_base {
std::string session_id; // in hex

/// API: convo_info_volatile/one_to_one::one_to_one
///
/// Constructs an empty one_to_one from a session_id. Session ID can be either bytes (33)
Expand Down Expand Up @@ -168,17 +182,10 @@ namespace convo {
void into(convo_info_volatile_legacy_group& c) const; // Into c struct
};

struct blinded_one_to_one : base {
struct blinded_one_to_one : pro_base {
std::string blinded_session_id; // in hex
bool legacy_blinding;

/// Hash of the generation index set by the Session Pro Backend
std::optional<array_uc32> pro_gen_index_hash;

/// Unix epoch timestamp to which this proof's entitlement to Session Pro features is valid
/// to
std::chrono::sys_time<std::chrono::milliseconds> pro_expiry_unix_ts{};

/// API: convo_info_volatile/blinded_one_to_one::blinded_one_to_one
///
/// Constructs an empty blinded_one_to_one from a blinded_session_id. Session ID can be
Expand Down
97 changes: 51 additions & 46 deletions src/config/convo_info_volatile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ namespace convo {
check_session_id(session_id);
}
one_to_one::one_to_one(const convo_info_volatile_1to1& c) :
base{c.last_read, c.unread}, session_id{c.session_id, 66} {
pro_base(c.last_read, c.unread), session_id{c.session_id, 66} {
if (c.has_pro_gen_index_hash) {
pro_gen_index_hash.emplace();
std::memcpy(
Expand Down Expand Up @@ -62,7 +62,7 @@ namespace convo {

community::community(const convo_info_volatile_community& c) :
config::community{c.base_url, c.room, std::span<const unsigned char>{c.pubkey, 32}},
base{c.last_read, c.unread} {}
base(c.last_read, c.unread) {}

void community::into(convo_info_volatile_community& c) const {
static_assert(sizeof(c.base_url) == BASE_URL_MAX_LENGTH + 1);
Expand All @@ -81,7 +81,7 @@ namespace convo {
check_session_id(id, "03");
}
group::group(const convo_info_volatile_group& c) :
base{c.last_read, c.unread}, id{c.group_id, 66} {}
base(c.last_read, c.unread), id{c.group_id, 66} {}

void group::into(convo_info_volatile_group& c) const {
std::memcpy(c.group_id, id.c_str(), 67);
Expand All @@ -96,7 +96,7 @@ namespace convo {
check_session_id(id);
}
legacy_group::legacy_group(const convo_info_volatile_legacy_group& c) :
base{c.last_read, c.unread}, id{c.group_id, 66} {}
base(c.last_read, c.unread), id{c.group_id, 66} {}

void legacy_group::into(convo_info_volatile_legacy_group& c) const {
std::memcpy(c.group_id, id.data(), 67);
Expand All @@ -123,7 +123,7 @@ namespace convo {
"Invalid blinded ID: Expected '15' or '25' prefix; got " + blinded_session_id};
}
blinded_one_to_one::blinded_one_to_one(const convo_info_volatile_blinded_1to1& c) :
base{c.last_read, c.unread},
pro_base(c.last_read, c.unread),
blinded_session_id{c.blinded_session_id, 66},
legacy_blinding{c.legacy_blinding} {
if (c.has_pro_gen_index_hash) {
Expand Down Expand Up @@ -156,6 +156,23 @@ namespace convo {
}
}

void pro_base::load(const dict& info_dict) {
base::load(info_dict);

auto pro_expiry = int_or_0(info_dict, "e");
std::optional<std::vector<unsigned char>> maybe_pro_gen_index_hash =
maybe_vector(info_dict, "g");
if (pro_expiry > 0 && maybe_pro_gen_index_hash && maybe_pro_gen_index_hash->size() == 32) {
pro_expiry_unix_ts = std::chrono::sys_time<std::chrono::milliseconds>(
std::chrono::milliseconds(pro_expiry));
pro_gen_index_hash.emplace();
std::memcpy(
pro_gen_index_hash->data(),
maybe_pro_gen_index_hash->data(),
pro_gen_index_hash->size());
}
}

void base::load(const dict& info_dict) {
last_read = int_or_0(info_dict, "r");
unread = (bool)int_or_0(info_dict, "u");
Expand All @@ -179,20 +196,6 @@ std::optional<convo::one_to_one> ConvoInfoVolatile::get_1to1(std::string_view pu

auto result = std::make_optional<convo::one_to_one>(std::string{pubkey_hex});
result->load(*info_dict);

auto pro_expiry = int_or_0(*info_dict, "e");
std::optional<std::vector<unsigned char>> maybe_pro_gen_index_hash =
maybe_vector(*info_dict, "g");
if (pro_expiry > 0 && maybe_pro_gen_index_hash && maybe_pro_gen_index_hash->size() == 32) {
result->pro_expiry_unix_ts = std::chrono::sys_time<std::chrono::milliseconds>(
std::chrono::milliseconds(pro_expiry));
result->pro_gen_index_hash.emplace();
std::memcpy(
result->pro_gen_index_hash->data(),
maybe_pro_gen_index_hash->data(),
result->pro_gen_index_hash->size());
}

return result;
}

Expand Down Expand Up @@ -318,20 +321,6 @@ std::optional<convo::blinded_one_to_one> ConvoInfoVolatile::get_blinded_1to1(

auto result = std::make_optional<convo::blinded_one_to_one>(std::string{pubkey_hex});
result->load(*info_dict);

auto pro_expiry = int_or_0(*info_dict, "e");
std::optional<std::vector<unsigned char>> maybe_pro_gen_index_hash =
maybe_vector(*info_dict, "g");
if (pro_expiry > 0 && maybe_pro_gen_index_hash && maybe_pro_gen_index_hash->size() == 32) {
result->pro_expiry_unix_ts = std::chrono::sys_time<std::chrono::milliseconds>(
std::chrono::milliseconds(pro_expiry));
result->pro_gen_index_hash.emplace();
std::memcpy(
result->pro_gen_index_hash->data(),
maybe_pro_gen_index_hash->data(),
result->pro_gen_index_hash->size());
}

return result;
}

Expand All @@ -347,11 +336,6 @@ void ConvoInfoVolatile::set(const convo::one_to_one& c) {
auto info = data["1"][session_id_to_bytes(c.session_id)];
set_base(c, info);

// If the base values weren't stored it means the data was too old so the record shouldn't be
// added
if (auto* d = info.dict(); !d || d->empty())
return;

auto pro_expiry = c.pro_expiry_unix_ts.time_since_epoch().count();
if (pro_expiry > 0 && c.pro_gen_index_hash) {
set_nonzero_int(info["e"], pro_expiry);
Expand All @@ -375,28 +359,54 @@ void ConvoInfoVolatile::set_base(const convo::base& c, DictFieldProxy& info) {
set_flag(info["u"], c.unread);
}

template <std::derived_from<convo::base> C>
static bool is_stale(const C& c, int64_t cutoff_ms) {
if (c.unread)
return false;
if constexpr (std::derived_from<C, convo::pro_base>)
if (c.pro_gen_index_hash.has_value() &&
c.pro_expiry_unix_ts.time_since_epoch().count() >= cutoff_ms)
return false;
return c.last_read < cutoff_ms;
}

void ConvoInfoVolatile::prune_stale(std::chrono::milliseconds prune) {
const int64_t cutoff = std::chrono::duration_cast<std::chrono::milliseconds>(
(std::chrono::system_clock::now() - prune).time_since_epoch())
.count();

std::vector<std::string> stale;
for (auto it = begin_1to1(); it != end(); ++it)
if (!it->unread && it->last_read < cutoff)
if (is_stale(*it, cutoff))
stale.push_back(it->session_id);

for (const auto& sid : stale)
erase_1to1(sid);

stale.clear();
for (auto it = begin_legacy_groups(); it != end(); ++it)
if (!it->unread && it->last_read < cutoff)
if (is_stale(*it, cutoff))
stale.push_back(it->id);
for (const auto& id : stale)
erase_legacy_group(id);

stale.clear();
for (auto it = begin_blinded_1to1(); it != end(); ++it)
if (is_stale(*it, cutoff))
stale.push_back(it->blinded_session_id);
for (const auto& id : stale)
erase_blinded_1to1(id);

stale.clear();
for (auto it = begin_groups(); it != end(); ++it)
if (is_stale(*it, cutoff))
stale.push_back(it->id);
for (const auto& id : stale)
erase_group(id);

std::vector<std::pair<std::string, std::string>> stale_comms;
for (auto it = begin_communities(); it != end(); ++it)
if (!it->unread && it->last_read < cutoff)
if (is_stale(*it, cutoff))
stale_comms.emplace_back(it->base_url(), it->room());
for (const auto& [base, room] : stale_comms)
erase_community(base, room);
Expand Down Expand Up @@ -433,11 +443,6 @@ void ConvoInfoVolatile::set(const convo::blinded_one_to_one& c) {
auto info = data["b"][pubkey];
set_base(c, info);

// If the base values weren't stored it means the data was too old so the record shouldn't be
// added
if (auto* d = info.dict(); !d || d->empty())
return;

set_nonzero_int(info["y"], c.legacy_blinding);

auto pro_expiry = c.pro_expiry_unix_ts.time_since_epoch().count();
Expand Down
42 changes: 28 additions & 14 deletions tests/test_config_convo_info_volatile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -614,19 +614,22 @@ TEST_CASE("Conversation pruning", "[config][conversations][pruning]") {
(now - days_ago * 24h).time_since_epoch())
.count();
};

for (int i = 0; i <= 65; i++) {
if (i % 3 == 0) {
auto c = convos.get_or_construct_1to1(some_session_id(i));
c.last_read = unix_timestamp(i);
if (i % 5 == 0)
c.unread = true;

c.pro_expiry_unix_ts =
std::chrono::time_point_cast<std::chrono::milliseconds>(now + 24h);
if (i % 7 == 0) {
c.pro_expiry_unix_ts = std::chrono::sys_time<std::chrono::milliseconds>{
std::chrono::milliseconds{unix_timestamp(i)}};

session::array_uc32 hash{};
std::fill(hash.begin(), hash.end(), static_cast<uint8_t>(i % 256));
c.pro_gen_index_hash = hash;
session::array_uc32 hash{};
std::fill(hash.begin(), hash.end(), static_cast<uint8_t>(i % 256));
c.pro_gen_index_hash = hash;
}

convos.set(c);
} else if (i % 3 == 1) {
Expand All @@ -645,18 +648,21 @@ TEST_CASE("Conversation pruning", "[config][conversations][pruning]") {
}
}

// 0, 3, 6, ..., 30 == 11 not-too-old last_read entries
// 45, 60 have unread flags
CHECK(convos.size_1to1() == 11 + 2);
// (15, 30, 45, 60) == 4 have `unread` set.
// (21, 42, 63) == 3 have pro proof.
// (0) == 1 has both unread and pro proof.
// (3, 6, 9, 12, 18, 24, 27) == 7 are recent enough.
CHECK(convos.size_1to1() == 4 + 3 + 1 + 7);
// 1, 4, 7, ..., 28 == 10 last_read's
// 40, 55 = 2 unread flags
CHECK(convos.size_legacy_groups() == 10 + 2);
// 2, 5, 8, ..., 29 == 10 last_read's
// 35, 50, 65 = 3 unread flags
CHECK(convos.size_communities() == 10 + 3);
// 31 (0-30) were recent enough to be kept
// 5 more (35, 40, 45, 50, 55) have `unread` set.
CHECK(convos.size() == 38);
// 6 more (35, 40, 45, 50, 55, 60) have `unread` set.
// 3 more (21, 42, 63) have pro proof.
CHECK(convos.size() == 40);

// Now we deliberately set some values in the internals that are too old to see that they get
// properly pruned when we push. (This is only for testing, clients should never touch the
Expand All @@ -671,16 +677,24 @@ TEST_CASE("Conversation pruning", "[config][conversations][pruning]") {
convos.data["1"][oxenc::from_hex(some_session_id(84))]["r"] = unix_timestamp(46);
convos.data["1"][oxenc::from_hex(some_session_id(85))]["r"] = unix_timestamp(1000);

CHECK(convos.size_1to1() == 19);
// 6 additional 1-to-1s got added unconditionally
CHECK(convos.size_1to1() == 21);
int count = 0;
for (auto it = convos.begin_1to1(); it != convos.end(); it++) {
count++;
}
CHECK(count == 19);
CHECK(count == 21);

CHECK(convos.size() == 44);
CHECK(convos.size() == 46);

// Push and confirm the pruned
auto [seqno, push_data, obs] = convos.push();
CHECK(convos.size() == 41);

// The push should have pruned these:
// 63 - where pro proof is too old
// 83, 84, 85 - where last_read is too old
CHECK(convos.size_1to1() == 17);
CHECK(convos.size() == 42);
}

TEST_CASE("Conversation dump/load state bug", "[config][conversations][dump-load]") {
Expand Down
Binary file modified utils/deb.oxen.io.gpg
View file
Open in desktop
Binary file not shown.