Skip to content

Commit

Permalink
conversations.c: store internaldates as nanseconds since epoch
Browse files Browse the repository at this point in the history 
store that (encoded) value as J(MAPID) key that maps to GUID of email
  • Loading branch information
@ksmurchison
ksmurchison committed Jan 16, 2025
1 parent f9aa2a1 commit 4bcfc06
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 29 deletions.
59 changes: 43 additions & 16 deletions imap/conversations.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -962,8 +962,8 @@ static void conv_to_buf(conversation_t *conv, struct buf *buf, int flagcount)
nn = dlist_newlist(n, "THREAD");
dlist_setguid(nn, "GUID", &thread->guid);
dlist_setnum32(nn, "EXISTS", thread->exists);
dlist_setnum32(nn, "INTERNALDATE", thread->internaldate);
dlist_setnum32(nn, "CREATEDMODSEQ", thread->createdmodseq);
dlist_setnum64(nn, "INTERNALDATE", thread->internaldate);
dlist_setnum64(nn, "CREATEDMODSEQ", thread->createdmodseq);
}

dlist_setnum64(dl, "CREATEDMODSEQ", conv->createdmodseq);
Expand Down Expand Up @@ -1490,7 +1490,7 @@ int _saxconvparse(int type, struct dlistsax_data *d)
return 0;

case 2:
rock->thread->internaldate = atol(d->data);
rock->thread->internaldate = atoll(d->data);
rock->substate = 3;
return 0;

Expand Down Expand Up @@ -1775,8 +1775,12 @@ static int _thread_datesort(const void **a, const void **b)
{
const conv_thread_t *ta = (const conv_thread_t *)*a;
const conv_thread_t *tb = (const conv_thread_t *)*b;
struct timespec ta_internaldate, tb_internaldate;

int r = (ta->internaldate - tb->internaldate);
TIMESPEC_FROM_NANOSEC(&ta_internaldate, ta->internaldate);
TIMESPEC_FROM_NANOSEC(&tb_internaldate, tb->internaldate);

int r = (ta_internaldate.tv_sec - tb_internaldate.tv_sec);
if (r < 0) return -1;
if (r > 0) return 1;

Expand Down Expand Up @@ -1817,7 +1821,7 @@ static void conversations_thread_sort(conversation_t *conv)

static void conversation_update_thread(conversation_t *conv,
const struct message_guid *guid,
time_t internaldate,
uint64_t internaldate,
modseq_t createdmodseq,
int delta_exists)
{
Expand Down Expand Up @@ -1963,7 +1967,7 @@ static int _guid_one(struct guid_foreach_rock *frock,
conversation_id_t basecid,
uint32_t system_flags,
uint32_t internal_flags,
time_t internaldate,
uint64_t internaldate,
char version)
{
const char *p, *err;
Expand Down Expand Up @@ -2071,7 +2075,7 @@ static int _guid_cb(void *rock,
conversation_id_t basecid = 0;
uint32_t system_flags = 0;
uint32_t internal_flags = 0;
time_t internaldate = 0;
uint64_t internaldate = 0;
char version = 0;
if (datalen >= 16) {
const char *p = data;
Expand Down Expand Up @@ -2114,7 +2118,7 @@ static int _guid_cb(void *rock,
internal_flags = ntohl(*((bit32*)p));
p += 4;
/* internaldate*/
internaldate = (time_t) ntohll(*((bit64*)p));
internaldate = ntohll(*((bit64*)p));
p += 8;
/* basecid */
basecid = ntohll(*((bit64*)p));
Expand Down Expand Up @@ -2228,7 +2232,7 @@ static int conversations_guid_setitem(struct conversations_state *state,
conversation_id_t basecid,
uint32_t system_flags,
uint32_t internal_flags,
time_t internaldate,
uint64_t internaldate,
int add)
{
struct buf key = BUF_INITIALIZER;
Expand Down Expand Up @@ -2271,15 +2275,15 @@ static int conversations_guid_setitem(struct conversations_state *state,
buf_appendbit64(&val, cid);
buf_appendbit32(&val, system_flags);
buf_appendbit32(&val, internal_flags);
buf_appendbit64(&val, (bit64)internaldate);
buf_appendbit64(&val, internaldate);
}
/* When bumping the G value version, make sure to update _guid_cb */
else {
buf_putc(&val, 0x80 | CONV_GUIDREC_VERSION);
buf_appendbit64(&val, cid);
buf_appendbit32(&val, system_flags);
buf_appendbit32(&val, internal_flags);
buf_appendbit64(&val, (bit64)internaldate);
buf_appendbit64(&val, internaldate);
buf_appendbit64(&val, basecid == cid ? 0 : basecid);
}

Expand All @@ -2302,7 +2306,7 @@ static int _guid_addbody(struct conversations_state *state,
conversation_id_t cid,
conversation_id_t basecid,
uint32_t system_flags, uint32_t internal_flags,
time_t internaldate,
uint64_t internaldate,
struct body *body,
const char *base, int add)
{
Expand Down Expand Up @@ -2357,15 +2361,37 @@ static int conversations_set_guid(struct conversations_state *state,
buf_printf(&item, "%d:%u", folder, record->uid);
const char *base = buf_cstring(&item);

r = conversations_guid_setitem(state, message_guid_encode(&record->guid),
const char *guidrep = message_guid_encode(&record->guid);
uint64_t internaldate = TIMESPEC_TO_NANOSEC(&record->internaldate);
r = conversations_guid_setitem(state, guidrep,
base, record->cid, record->basecid,
record->system_flags,
record->internal_flags,
record->internaldate.tv_sec,
internaldate,
add);
if (!r) {
struct buf key = BUF_INITIALIZER;

/* Build J key */
buf_setcstr(&key, "J");
NANOSEC_TO_JMAPID(&key, internaldate);

if (add) {
/* Insert J record: jidrep -> guidrep */
r = cyrusdb_store(state->db, buf_base(&key), buf_len(&key),
guidrep, strlen(guidrep),
&state->txn);
}
else {
/* Remove J record */
r = cyrusdb_delete(state->db, buf_base(&key), buf_len(&key),
&state->txn, /*force*/1);
}
}
if (!r) r = _guid_addbody(state, record->cid, record->basecid,
record->system_flags, record->internal_flags,
record->internaldate.tv_sec, body, base, add);
internaldate,
body, base, add);

message_free_body(body);
free(body);
Expand Down Expand Up @@ -2518,6 +2544,7 @@ EXPORTED int conversations_update_record(struct conversations_state *cstate,
if (new) {
if (!old || old->system_flags != new->system_flags ||
old->internal_flags != new->internal_flags ||
old->internaldate.tv_nsec != new->internaldate.tv_nsec ||
old->internaldate.tv_sec != new->internaldate.tv_sec) {
r = conversations_set_guid(cstate, mailbox, new, /*add*/1);
if (r) goto done;
Expand Down Expand Up @@ -2634,7 +2661,7 @@ EXPORTED int conversations_update_record(struct conversations_state *cstate,

conversation_update_thread(conv,
&record->guid,
record->internaldate.tv_sec,
TIMESPEC_TO_NANOSEC(&record->internaldate),
record->createdmodseq,
delta_exists);

Expand Down
8 changes: 4 additions & 4 deletions imap/conversations.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ struct conv_thread {
conv_thread_t *next;
struct message_guid guid;
uint32_t exists;
time_t internaldate;
uint64_t internaldate; // nanoseconds since epoch
modseq_t createdmodseq;
};

Expand All @@ -138,7 +138,7 @@ struct conv_folder {
uint32_t prev_exists;
};

#define CONV_GUIDREC_VERSION 3 // (must be <= 127)
#define CONV_GUIDREC_VERSION 4 // (must be <= 127)
#define CONV_GUIDREC_BYNAME_VERSION 1 // last folders byname version

struct conv_guidrec {
Expand All @@ -148,11 +148,11 @@ struct conv_guidrec {
uint32_t uid;
const char *part;
conversation_id_t cid;
conversation_id_t basecid;
conversation_id_t basecid; // if version >= 3
char version;
uint32_t system_flags; // if version >= 1
uint32_t internal_flags; // if version >= 1
time_t internaldate; // if version >= 1
uint64_t internaldate; // if version >= 4 (nanoseconds since epoch)
};

struct conv_sender {
Expand Down
16 changes: 10 additions & 6 deletions imap/jmap_mail.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3235,9 +3235,9 @@ static int emailsearch_is_mutable(struct emailsearch *search)
struct guidsearch_match {
char guidrep[MESSAGE_GUID_SIZE*2+1];
uint32_t system_flags;
uint32_t internaldate;
uint64_t internaldate; // nanoseconds since epoch
conversation_id_t cid;
bitvector_t folders; // only set if numfolders > 0
bitvector_t folders; // only set if numfolders > 0
};

static void guidsearch_match_init(struct guidsearch_match *match,
Expand All @@ -3261,10 +3261,14 @@ static int guidsearch_match_cmp QSORT_R_COMPAR_ARGS(const void *va,
while (sort->key != SORT_SEQUENCE) {
int ret;
switch (sort->key) {
case SORT_ARRIVAL:
ret = a->internaldate < b->internaldate ? -1 :
a->internaldate > b->internaldate ? 1 : 0;
case SORT_ARRIVAL: {
struct timespec a_internaldate, b_internaldate;
TIMESPEC_FROM_NANOSEC(&a_internaldate, a->internaldate);
TIMESPEC_FROM_NANOSEC(&b_internaldate, b->internaldate);
ret = a_internaldate.tv_sec < b_internaldate.tv_sec ? -1 :
a_internaldate.tv_sec > b_internaldate.tv_sec ? 1 : 0;
break;
}
case SORT_GUID:
ret = memcmp(a->guidrep, b->guidrep, MESSAGE_GUID_SIZE*2);
break;
Expand Down Expand Up @@ -13223,7 +13227,7 @@ static void _email_bulkupdate_exec_setflags(struct email_bulkupdate *bulk)

if (update->received_at) {
/* Write internaldate (Email/copy only) */
struct timespec internaldate = { 0 };
struct timespec internaldate = { 0 };
time_from_iso8601(update->received_at, &internaldate.tv_sec);
r = msgrecord_set_internaldate(mrw, &internaldate);
}
Expand Down
11 changes: 8 additions & 3 deletions lib/util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,14 @@ extern const unsigned char convert_to_uppercase[256];

#define TIMESPEC_TO_NANOSEC(ts) ((ts)->tv_sec * 1000000000 + (ts)->tv_nsec)

#define TIMESPEC_FROM_NANOSEC(ts, nanosec) { \
(ts)->tv_sec = nanosec / 1000000000; \
(ts)->tv_nsec = nanosec % 1000000000; \
#define TIMESPEC_FROM_NANOSEC(ts, nanosec) { \
(ts)->tv_sec = (nanosec) / 1000000000; \
(ts)->tv_nsec = (nanosec) % 1000000000; \
}

#define NANOSEC_TO_JMAPID(buf, nanosec) { \
uint64_t u64 = htonll(UINT64_MAX - (nanosec)); \
charset_encode(buf, (const char *) &u64, 8, ENCODING_BASE64JMAPID); \
}

typedef struct keyvalue {
Expand Down

0 comments on commit 4bcfc06

Please sign in to comment.