From 6468a01d432b6389ad0a64ccf0bee6923fe9b250 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 25 Aug 2009 13:41:23 +1000 Subject: [PATCH] Add "version 11" mailbox header with crc32 fields Adds CRC32 support to index header and record structures, and supporting code to actually calculate the values! --- imap/append.c | 2 + imap/mailbox.c | 118 +++++++++++++++++++++++++++++++++++++++----------- imap/mailbox.h | 17 +++++-- imap/mbexamine.c | 4 +- imap/message.c | 17 ++++++- imap/reconstruct.c | 22 ++++++++- imap/sync_commit.c | 77 +++++++++++++++++++++++++++------ imap/sync_server.c | 1 + imap/sync_support.c | 2 + imap/sync_support.h | 1 + imap/unexpunge.c | 11 +++++ 11 files changed, 220 insertions(+), 52 deletions(-) diff --git a/imap/append.c b/imap/append.c index 9cacf65..0b56f03 100644 --- a/imap/append.c +++ b/imap/append.c @@ -926,6 +926,8 @@ int append_copy(struct mailbox *mailbox, message_index[msg].content_offset = copymsg[msg].header_size; message_index[msg].content_lines = copymsg[msg].content_lines; message_index[msg].cache_version = copymsg[msg].cache_version; + message_index[msg].cache_crc = + crc32_buf(copymsg[msg].cache_begin, copymsg[msg].cache_len); n = retry_write(append_mailbox->cache_fd, copymsg[msg].cache_begin, copymsg[msg].cache_len); diff --git a/imap/mailbox.c b/imap/mailbox.c index e651cf5..db4d12d 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -72,6 +72,7 @@ #include "acl.h" #include "assert.h" +#include "crc32.h" #include "exitcodes.h" #include "global.h" #include "imap_err.h" @@ -297,7 +298,6 @@ unsigned mailbox_cached_header_inline(const char *text) return BIT32_MAX; } -/* returns the length of the parsed record, if it's valid */ unsigned cache_parserecord(const char *map_base, unsigned map_size, unsigned cache_offset, cacherecord *rec) { unsigned cache_ent, offset; @@ -333,9 +333,6 @@ unsigned cache_parserecord(const char *map_base, unsigned map_size, unsigned cac unsigned mailbox_cacherecord_offset(struct mailbox *mailbox, unsigned cache_offset, cacherecord *rec) { - unsigned cache_ent, offset; - const char *cacheitem; - return cache_parserecord(mailbox->cache_base, mailbox->cache_size, cache_offset, rec); } @@ -343,6 +340,8 @@ unsigned mailbox_cacherecord_index(struct mailbox *mailbox, unsigned msgno, cach { unsigned cache_offset; const char *p; + unsigned len; + uint32_t crc, newcrc; assert((msgno > 0) && (msgno <= mailbox->exists)); @@ -350,8 +349,21 @@ unsigned mailbox_cacherecord_index(struct mailbox *mailbox, unsigned msgno, cach ((msgno-1) * mailbox->record_size)); cache_offset = ntohl(*((bit32 *)(p+OFFSET_CACHE_OFFSET))); + crc = ntohl(*((bit32 *)(p+OFFSET_CACHE_CRC))); + + len = mailbox_cacherecord_offset(mailbox, cache_offset, rec); + + /* skip checksum check if crc in index is zero (i.e upgraded mailbox or rare case) */ + if (len && crc) { + newcrc = crc32_buf(mailbox->cache_base + cache_offset, len); + if (newcrc != crc) { + syslog(LOG_ERR, "IOERROR: invalid cache checksum for %s record %u (%lu != %lu) over len %d", + mailbox->name, msgno, newcrc, crc, len); + /* len = 0; */ + } + } - return mailbox_cacherecord_offset(mailbox, cache_offset, rec); + return len; } /* function to be used for notification of mailbox changes/updates */ @@ -933,12 +945,15 @@ int mailbox_read_index_header(struct mailbox *mailbox) mailbox->record_size = ntohl(*((bit32 *)(mailbox->index_base+OFFSET_RECORD_SIZE))); - if ((mailbox->start_offset < OFFSET_HIGHESTMODSEQ+4) || + if ((mailbox->start_offset < OFFSET_HEADER_SIZE) || (mailbox->record_size < INDEX_RECORD_SIZE) || (mailbox->minor_version < MAILBOX_MINOR_VERSION)) { if (mailbox_upgrade_index(mailbox)) return IMAP_IOERROR; + syslog(LOG_INFO, "Index upgrade: %s (%d -> %d)", mailbox->name, + mailbox->minor_version, MAILBOX_MINOR_VERSION); + /* things might have been changed out from under us. reread */ return mailbox_open_index(mailbox); } @@ -1007,6 +1022,7 @@ int mailbox_read_index_record_from_mapped(struct mailbox *mailbox, { unsigned long offset; unsigned const char *buf; + uint32_t crc; int n; offset = mailbox->start_offset + (msgno-1) * mailbox->record_size; @@ -1039,6 +1055,16 @@ int mailbox_read_index_record_from_mapped(struct mailbox *mailbox, #else record->modseq = ntohl(*((bit32 *)(buf+OFFSET_MODSEQ))); #endif + record->cache_crc = ntohl(*((bit32 *)(buf+OFFSET_CACHE_CRC))); + record->record_crc = ntohl(*((bit32 *)(buf+OFFSET_RECORD_CRC))); + + crc = crc32_buf(buf, OFFSET_RECORD_CRC); + if (crc != record->record_crc) { + syslog(LOG_ERR, + "IOERROR: index record %u for %s crc mismatch (%d != %d)", + msgno, mailbox->name, crc, record->record_crc); + } + return 0; } @@ -1386,7 +1412,10 @@ int mailbox_write_index_header(struct mailbox *mailbox) *((bit32 *)(buf+OFFSET_SPARE1)) = htonl(0); /* RESERVED */ *((bit32 *)(buf+OFFSET_SPARE2)) = htonl(0); /* RESERVED */ *((bit32 *)(buf+OFFSET_SPARE3)) = htonl(0); /* RESERVED */ - *((bit32 *)(buf+OFFSET_SPARE4)) = htonl(0); /* RESERVED */ + + /* Update checksum */ + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); if (mailbox->start_offset < header_size) header_size = mailbox->start_offset; @@ -1434,6 +1463,11 @@ void mailbox_index_record_to_buf(struct index_record *record, *((bit32 *)(buf+OFFSET_MODSEQ_64)) = htonl(0); *((bit32 *)(buf+OFFSET_MODSEQ)) = htonl(record->modseq); #endif + *((bit32 *)(buf+OFFSET_CACHE_CRC)) = htonl(record->cache_crc); + + /* calculate the checksum */ + *((bit32 *)(buf+OFFSET_RECORD_CRC)) = + htonl(crc32_buf(buf, OFFSET_RECORD_CRC)); } /* @@ -1676,7 +1710,10 @@ static void mailbox_upgrade_index_work(struct mailbox *mailbox, *((bit32 *)(headerbuf+OFFSET_SPARE1)) = htonl(0); /* RESERVED */ *((bit32 *)(headerbuf+OFFSET_SPARE2)) = htonl(0); /* RESERVED */ *((bit32 *)(headerbuf+OFFSET_SPARE3)) = htonl(0); /* RESERVED */ - *((bit32 *)(headerbuf+OFFSET_SPARE4)) = htonl(0); /* RESERVED */ + + /* calculate the checksum */ + *((bit32 *)(headerbuf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(headerbuf, OFFSET_HEADER_CRC)); /* Write new header */ fwrite(headerbuf, 1, INDEX_HEADER_SIZE, newindex); @@ -1759,6 +1796,13 @@ static void mailbox_upgrade_index_work(struct mailbox *mailbox, #endif } + /* we can't calculate the crc32 of the cache records here + * because it's just too expensive to open and scan everything, + * but we will checksum the record itself */ + + *((bit32 *)(recordbuf+OFFSET_RECORD_CRC)) = + htonl(crc32_buf(recordbuf, OFFSET_RECORD_CRC)); + fwrite(recordbuf, INDEX_RECORD_SIZE, 1, newindex); } @@ -1768,6 +1812,10 @@ static void mailbox_upgrade_index_work(struct mailbox *mailbox, *((bit32 *)(headerbuf+OFFSET_ANSWERED)) = htonl(numansweredflag); *((bit32 *)(headerbuf+OFFSET_FLAGGED)) = htonl(numflaggedflag); + /* recalculate the checksum */ + *((bit32 *)(headerbuf+OFFSET_HEADER_CRC)) + = htonl(crc32_buf(headerbuf, OFFSET_HEADER_CRC)); + rewind(newindex); fwrite(headerbuf, INDEX_HEADER_SIZE, 1, newindex); } @@ -1999,6 +2047,9 @@ static int process_records(struct mailbox *mailbox, FILE *newindex, * For two-phase, we should sort by UID. */ *((bit32 *)(buf+OFFSET_LAST_UPDATED)) = htonl(now); + /* recalculate the checksum */ + *((bit32 *)(buf+OFFSET_RECORD_CRC)) = + htonl(crc32_buf(buf, OFFSET_RECORD_CRC)); n = retry_write(expunge_fd, buf, mailbox->record_size); if (n != mailbox->record_size) { syslog(LOG_ERR, @@ -2023,23 +2074,7 @@ static int process_records(struct mailbox *mailbox, FILE *newindex, continue; } - /* Fix up cache file offset */ - *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = - htonl(*new_cache_total_size); - if (newindex) fwrite(buf, 1, mailbox->record_size, newindex); - - /* Compute size of this record */ - cacheitembegin = cacheitem = mailbox->cache_base + cache_offset; - if (cache_offset >= mailbox->cache_size) { - syslog(LOG_ERR, - "IOERROR: reading cache record for %s:" - " initial bogus offset %d of %d for %u/%lu; mailbox needs a reconstruct", - mailbox->name, - (int) (cacheitem - mailbox->cache_base), - (int) mailbox->cache_size, - msgno, exists); - return IMAP_IOERROR; - } + cacheitembegin = mailbox->cache_base + cache_offset; cache_record_size = mailbox_cacherecord_offset(mailbox, cache_offset, 0); if (!cache_record_size) { @@ -2048,10 +2083,21 @@ static int process_records(struct mailbox *mailbox, FILE *newindex, mailbox->name, msgno); return IMAP_IOERROR; } + + /* Fix up cache file offset */ + *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = + htonl(*new_cache_total_size); *new_cache_total_size += cache_record_size; + /* recalculate the checksums */ + *((bit32 *)(buf+OFFSET_CACHE_CRC)) = + htonl(crc32_buf(cacheitembegin, cache_record_size)); + *((bit32 *)(buf+OFFSET_RECORD_CRC)) = + htonl(crc32_buf(buf, OFFSET_RECORD_CRC)); + /* fwrite will automatically call write() in a sane way */ fwrite(cacheitembegin, 1, cache_record_size, newcache); + if (newindex) fwrite(buf, 1, mailbox->record_size, newindex); } else if (newindex) { /* Keep this message, but just update the index record */ @@ -2106,7 +2152,11 @@ static int process_records(struct mailbox *mailbox, FILE *newindex, if (mailbox->start_offset < INDEX_HEADER_SIZE) { *((bit32 *)(buf+OFFSET_START_OFFSET)) = htonl(INDEX_HEADER_SIZE); } - + + /* Update checksum */ + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); + /* Write out new index header */ rewind(newindex); fwrite(buf, 1, mailbox->start_offset, newindex); @@ -2269,6 +2319,10 @@ int mailbox_expunge(struct mailbox *mailbox, fwrite(buf, 1, sizeof(bit32), newcache); } + /* Update checksum */ + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); + /* Write out new index header */ fwrite(buf, 1, mailbox->start_offset, newindex); @@ -2314,6 +2368,10 @@ int mailbox_expunge(struct mailbox *mailbox, *((bit32 *)(buf+OFFSET_LEAKED_CACHE)) = htonl(0); + /* Update checksum */ + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); + n = retry_write(expunge_fd, buf, mailbox->start_offset); /* Ensure everything made it to disk */ @@ -2413,6 +2471,10 @@ int mailbox_expunge(struct mailbox *mailbox, /* Update Generation Number */ *((bit32 *)buf+OFFSET_GENERATION_NO) = htonl(mailbox->generation_no+1); + /* Update checksum */ + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); + /* Write out new expunge index header */ fwrite(buf, 1, mailbox->start_offset, newexpungeindex); } @@ -2473,6 +2535,10 @@ int mailbox_expunge(struct mailbox *mailbox, *((bit32 *)(buf+OFFSET_START_OFFSET)) = htonl(INDEX_HEADER_SIZE); } + /* Update checksum */ + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); + /* Write out new cyrus.expunge header */ lseek(expunge_fd, 0, SEEK_SET); n = retry_write(expunge_fd, buf, mailbox->start_offset); diff --git a/imap/mailbox.h b/imap/mailbox.h index 83b102b..fdbf5f3 100644 --- a/imap/mailbox.h +++ b/imap/mailbox.h @@ -90,7 +90,7 @@ typedef unsigned long int modseq_t; #define MAILBOX_FORMAT_NORMAL 0 #define MAILBOX_FORMAT_NETNEWS 1 -#define MAILBOX_MINOR_VERSION 10 +#define MAILBOX_MINOR_VERSION 11 #define MAILBOX_CACHE_MINOR_VERSION 2 #define FNAME_HEADER "/cyrus.header" @@ -166,7 +166,8 @@ struct mailbox { * this case, just add a new field, and optionally add some more * spares. */ - unsigned long spares[4]; + unsigned long spares[3]; + unsigned long header_crc; struct quota quota; @@ -192,6 +193,8 @@ struct index_record { unsigned long cache_version; struct message_guid guid; modseq_t modseq; + unsigned long cache_crc; + unsigned long record_crc; }; /* Offsets of index/expunge header fields @@ -222,7 +225,7 @@ struct index_record { #define OFFSET_SPARE1 80 /* record size remains the same */ #define OFFSET_SPARE2 84 /* (see note above about spares) */ #define OFFSET_SPARE3 88 -#define OFFSET_SPARE4 92 +#define OFFSET_HEADER_CRC 92 /* includes all zero for the spares! */ /* Offsets of index_record fields in index/expunge file * @@ -245,9 +248,11 @@ struct index_record { #define OFFSET_MESSAGE_GUID OFFSET_CACHE_VERSION+sizeof(bit32) #define OFFSET_MODSEQ_64 (OFFSET_MESSAGE_GUID+MESSAGE_GUID_SIZE) /* CONDSTORE (64-bit modseq) */ #define OFFSET_MODSEQ (OFFSET_MODSEQ_64+sizeof(bit32)) /* CONDSTORE (32-bit modseq) */ +#define OFFSET_CACHE_CRC (OFFSET_MODSEQ+sizeof(bit32)) /* CRC32 of cache record */ +#define OFFSET_RECORD_CRC (OFFSET_CACHE_CRC+sizeof(bit32)) /* CRC32 of index record */ -#define INDEX_HEADER_SIZE (OFFSET_SPARE4+sizeof(bit32)) -#define INDEX_RECORD_SIZE (OFFSET_MODSEQ+sizeof(bit32)) +#define INDEX_HEADER_SIZE (OFFSET_HEADER_CRC+sizeof(bit32)) +#define INDEX_RECORD_SIZE (OFFSET_RECORD_CRC+sizeof(bit32)) /* Number of fields in an individual message's cache record */ #define NUM_CACHE_FIELDS 10 @@ -343,7 +348,9 @@ enum { unsigned mailbox_cached_header(const char *s); unsigned mailbox_cached_header_inline(const char *text); +unsigned cache_parserecord_crc(const char *map_base, unsigned map_size, unsigned cache_offset, cacherecord *rec, uint32_t crc); unsigned cache_parserecord(const char *map_base, unsigned map_size, unsigned cache_offset, cacherecord *rec); +unsigned mailbox_cacherecord_offset_crc(struct mailbox *mailbox, unsigned cache_offset, cacherecord *rec, uint32_t crc); unsigned mailbox_cacherecord_offset(struct mailbox *mailbox, unsigned cache_offset, cacherecord *rec); unsigned mailbox_cacherecord_index(struct mailbox *mailbox, unsigned msgno, cacherecord *rec); diff --git a/imap/mbexamine.c b/imap/mbexamine.c index 9e43bcf..ea35e06 100644 --- a/imap/mbexamine.c +++ b/imap/mbexamine.c @@ -129,8 +129,8 @@ int main(int argc, char **argv) } /* Ensure we're up-to-date on the index file format */ - assert(INDEX_HEADER_SIZE == (OFFSET_SPARE4+4)); - assert(INDEX_RECORD_SIZE == (OFFSET_MODSEQ+4)); + assert(INDEX_HEADER_SIZE == (OFFSET_HEADER_CRC+4)); + assert(INDEX_RECORD_SIZE == (OFFSET_RECORD_CRC+4)); while ((opt = getopt(argc, argv, "C:u:s:q")) != EOF) { switch (opt) { diff --git a/imap/message.c b/imap/message.c index 811125d..fbad530 100644 --- a/imap/message.c +++ b/imap/message.c @@ -204,7 +204,8 @@ static char *message_getline P((char *s, unsigned n, struct msg *msg)); static int message_pendingboundary P((const char *s, char **boundaries, int *boundaryct)); -static int message_write_cache P((int outfd, struct body *body)); +static int message_write_cache P((int outfd, struct body *body, + uint32_t *crcptr)); static void message_write_envelope P((struct ibuf *ibuf, struct body *body)); static void message_write_body P((struct ibuf *ibuf, struct body *body, @@ -547,6 +548,7 @@ char **messageid; { int n; enum enum_value config_guidmode = config_getenum(IMAPOPT_GUID_MODE); + uint32_t crc; if (config_getenum(IMAPOPT_INTERNALDATE_HEURISTIC) == IMAP_ENUM_INTERNALDATE_HEURISTIC_RECEIVEDHEADER) { @@ -564,7 +566,9 @@ char **messageid; message_index->cache_version = MAILBOX_CACHE_MINOR_VERSION; - n = message_write_cache(cache_fd, body); + n = message_write_cache(cache_fd, body, &crc); + message_index->cache_crc = crc; + if (n == -1) { syslog(LOG_ERR, "IOERROR: appending cache for %s: %m", cache_name); @@ -2097,9 +2101,10 @@ int *boundaryct; * to 'outfile'. */ static int -message_write_cache(outfd, body) +message_write_cache(outfd, body, crcptr) int outfd; struct body *body; +uint32_t *crcptr; { struct ibuf section, envelope, bodystructure, oldbody; struct ibuf from, to, cc, bcc, subject; @@ -2152,6 +2157,12 @@ struct body *body; message_ibuf_iov(&iov[8], &bcc); message_ibuf_iov(&iov[9], &subject); + /* find the CRC32 of the record... + * NOTE: have to do this before the retry_writev because + * it's destructive to the .l values */ + if (crcptr) + *crcptr = crc32_iovec(iov, 10); + n = retry_writev(outfd, iov, 10); message_ibuf_free(&envelope); diff --git a/imap/reconstruct.c b/imap/reconstruct.c index 2e5f1a2..46cc895 100644 --- a/imap/reconstruct.c +++ b/imap/reconstruct.c @@ -59,6 +59,9 @@ #include #include #include +#ifdef HAVE_ZLIB +#include +#endif #if HAVE_DIRENT_H # include @@ -80,6 +83,7 @@ #include "acl.h" #include "assert.h" #include "bsearch.h" +#include "crc32.h" #include "imparse.h" #include "global.h" #include "exitcodes.h" @@ -161,8 +165,8 @@ int main(int argc, char **argv) } /* Ensure we're up-to-date on the index file format */ - assert(INDEX_HEADER_SIZE == (OFFSET_SPARE4+4)); - assert(INDEX_RECORD_SIZE == (OFFSET_MODSEQ+4)); + assert(INDEX_HEADER_SIZE == (OFFSET_HEADER_CRC+4)); + assert(INDEX_RECORD_SIZE == (OFFSET_RECORD_CRC+4)); while ((opt = getopt(argc, argv, "C:kp:rmfsxgG")) != EOF) { switch (opt) { @@ -660,7 +664,9 @@ static void reconstruct_counts_tobuf(unsigned char *buf, *((bit32 *)(buf+OFFSET_SPARE1)) = htonl(0); /* RESERVED */ *((bit32 *)(buf+OFFSET_SPARE2)) = htonl(0); /* RESERVED */ *((bit32 *)(buf+OFFSET_SPARE3)) = htonl(0); /* RESERVED */ - *((bit32 *)(buf+OFFSET_SPARE4)) = htonl(0); /* RESERVED */ + + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); } /* ---------------------------------------------------------------------- */ @@ -673,6 +679,8 @@ static int reconstruct_header_isvalid(const char *index_base, unsigned long start_offset; unsigned long record_size; unsigned long exists; + unsigned long header_crc; + bit32 crc; if ((index_base == NULL) || (index_len < INDEX_HEADER_SIZE)) return(0); @@ -682,8 +690,12 @@ static int reconstruct_header_isvalid(const char *index_base, start_offset = ntohl(*((bit32 *)(index_base+OFFSET_START_OFFSET))); record_size = ntohl(*((bit32 *)(index_base+OFFSET_RECORD_SIZE))); exists = ntohl(*((bit32 *)(index_base+OFFSET_EXISTS))); + header_crc = ntohl(*((bit32 *)(index_base+OFFSET_HEADER_CRC))); + + crc = crc32_buf(index_base, OFFSET_HEADER_CRC); if ((format != 0) || + (header_crc != crc) || (minor_version == 0) || (minor_version > MAILBOX_MINOR_VERSION) || (start_offset == 0) || (start_offset > INDEX_HEADER_SIZE) || (record_size == 0) || (record_size > INDEX_RECORD_SIZE) || @@ -761,6 +773,8 @@ int reconstruct_read_index_record(const char *name, #else record->modseq = ntohl(*((bit32 *)(buf+OFFSET_MODSEQ))); #endif + record->cache_crc = ntohl(*((bit32 *)(buf+OFFSET_CACHE_CRC))); + record->record_crc = ntohl(*((bit32 *)(buf+OFFSET_RECORD_CRC))); return 0; } @@ -1008,6 +1022,8 @@ int reconstruct(char *name, struct discovered *found) /* Create placeholder space for index/cache/expunge headers */ memset(buf, 0, sizeof(buf)); *((bit32 *)(buf+OFFSET_GENERATION_NO)) = htonl(mailbox.generation_no + 1); + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); fwrite(buf, 1, INDEX_HEADER_SIZE, newindex); fwrite(buf, 1, INDEX_HEADER_SIZE, newexpunge); retry_write(newcache_fd, buf, sizeof(bit32)); diff --git a/imap/sync_commit.c b/imap/sync_commit.c index 969d496..f85023c 100644 --- a/imap/sync_commit.c +++ b/imap/sync_commit.c @@ -170,6 +170,9 @@ static void sync_counts_write(unsigned char *buf, struct sync_counts *c, *((bit32 *)(buf+OFFSET_HIGHESTMODSEQ_64)) = htonl(0); *((bit32 *)(buf+OFFSET_HIGHESTMODSEQ)) = htonl(c->newhighestmodseq); #endif + + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); } /* ---------------------------------------------------------------------- */ @@ -271,7 +274,9 @@ static void sync_write_header(struct mailbox *mailbox, unsigned long n; memcpy(buf, index_base, mailbox->start_offset); - *((bit32 *)buf+OFFSET_GENERATION_NO) = htonl(mailbox->generation_no+1); + *((bit32 *)(buf+OFFSET_GENERATION_NO)) = htonl(mailbox->generation_no+1); + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); fwrite(buf, 1, mailbox->start_offset, file); /* Grow the index header if necessary */ @@ -314,6 +319,7 @@ static void sync_make_index_record(struct index_record *p, p->content_lines = message->content_lines; p->cache_version = message->cache_version; + p->cache_crc = message->cache_crc; message_guid_copy(&p->guid, &item->guid); p->modseq = item->modseq; @@ -332,11 +338,6 @@ static unsigned long sync_cacheitem_size(const char *cacheitem) return(cacheitem - cacheitembegin); /* Compute size of this record */ } -static void sync_cacheitem_write(const char *cacheitem, FILE *newcache) -{ - fwrite(cacheitem, 1, sync_cacheitem_size(cacheitem), newcache); -} - /* ====================================================================== */ /* sync_combine_commit() is a three way combine on an open mailbox, the @@ -497,7 +498,9 @@ static int sync_combine_commit(struct mailbox *mailbox, if (expunge_base) sync_write_header(mailbox, expunge_base, newexpunge); /* XXX OFFSET_GENERATION_NO can only be zero */ - *((bit32 *)buf+OFFSET_GENERATION_NO) = htonl(mailbox->generation_no+1); + *((bit32 *)(buf+OFFSET_GENERATION_NO)) = htonl(mailbox->generation_no+1); + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); fwrite(buf, 1, sizeof(bit32), newcache); /* Copy messages into target mailfolder (blat existing messages: @@ -527,6 +530,8 @@ static int sync_combine_commit(struct mailbox *mailbox, while (!r) { int have_index = (index_msgno <= mailbox->exists); int have_expunge = (expunge_msgno <= expunge_exists); + const char *cacheitem; + unsigned cacheitem_size; if (!item && !have_index && !have_expunge) break; @@ -536,10 +541,20 @@ static int sync_combine_commit(struct mailbox *mailbox, (!item || (expunge_record.uid < item->uid))) { /* Add expunged item to cyrus.expunge, cyrus.cache */ mailbox_index_record_to_buf(&expunge_record, buf); + + /* handle the cache item */ + cacheitem = mailbox->cache_base + expunge_record.cache_offset; + cacheitem_size = sync_cacheitem_size(cacheitem); *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(ftell(newcache)); + /* XXX - not strictly needed, it shouldn't have changed */ + *((bit32 *)(buf+OFFSET_CACHE_CRC)) = + htonl(crc32_buf(cacheitem, cacheitem_size)); + + /* update the CRC */ + *((bit32 *)(buf+OFFSET_RECORD_CRC)) = + htonl(crc32_buf(buf, OFFSET_RECORD_CRC)); - sync_cacheitem_write - (mailbox->cache_base+expunge_record.cache_offset, newcache); + fwrite(cacheitem, 1, cacheitem_size, newcache); fwrite(buf, 1, mailbox->record_size, newexpunge); sync_counts_update(&expunge, &expunge_record); @@ -551,10 +566,20 @@ static int sync_combine_commit(struct mailbox *mailbox, ((item == NULL) || (index_record.uid < item->uid))) { /* Add existing item from mailbox to cyrus.index, cyrus.cache */ mailbox_index_record_to_buf(&index_record, buf); + + /* handle the cache item */ + cacheitem = mailbox->cache_base + index_record.cache_offset; + cacheitem_size = sync_cacheitem_size(cacheitem); *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(ftell(newcache)); + /* XXX - not strictly needed, it shouldn't have changed */ + *((bit32 *)(buf+OFFSET_CACHE_CRC)) = + htonl(crc32_buf(cacheitem, cacheitem_size)); + + /* update the CRC */ + *((bit32 *)(buf+OFFSET_RECORD_CRC)) = + htonl(crc32_buf(buf, OFFSET_RECORD_CRC)); - sync_cacheitem_write - (mailbox->cache_base+index_record.cache_offset, newcache); + fwrite(cacheitem, 1, cacheitem_size, newcache); fwrite(buf, 1, mailbox->record_size, newindex); sync_counts_update(&index, &index_record); @@ -571,10 +596,17 @@ static int sync_combine_commit(struct mailbox *mailbox, /* Use item from upload list (may replace existing msg) */ sync_make_index_record(&tmp_record, item); mailbox_index_record_to_buf(&tmp_record, buf); + + /* handle the cache item */ + cacheitem = message_list->cache_base + tmp_record.cache_offset; + cacheitem_size = sync_cacheitem_size(cacheitem); *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(ftell(newcache)); - sync_cacheitem_write - (message_list->cache_base+tmp_record.cache_offset, newcache); + /* update the CRC */ + *((bit32 *)(buf+OFFSET_RECORD_CRC)) = + htonl(crc32_buf(buf, OFFSET_RECORD_CRC)); + + fwrite(cacheitem, 1, cacheitem_size, newcache); fwrite(buf, 1, mailbox->record_size, newindex); sync_counts_update(&index, &tmp_record); @@ -766,6 +798,11 @@ static int sync_append_commit(struct mailbox *mailbox, *((bit32 *)(record+OFFSET_MODSEQ_64)) = htonl(0); *((bit32 *)(record+OFFSET_MODSEQ)) = htonl(item->modseq); #endif + *((bit32 *)(record+OFFSET_CACHE_CRC)) = htonl(message->cache_crc); + + /* calculate crc32 of the index record */ + *((bit32 *)(record+OFFSET_RECORD_CRC)) + = htonl(crc32_buf(record, OFFSET_RECORD_CRC)); if (item->modseq > highestmodseq) highestmodseq = item->modseq; @@ -876,6 +913,10 @@ static int sync_append_commit(struct mailbox *mailbox, } #endif + /* Update checksum */ + *((bit32 *)(hbuf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(hbuf, OFFSET_HEADER_CRC)); + /* And write it back out */ lseek(mailbox->index_fd, 0L, SEEK_SET); @@ -1012,6 +1053,10 @@ int sync_uidlast_commit(struct mailbox *mailbox, /* Fix up last_append time */ *((bit32 *)(hbuf+OFFSET_LAST_APPENDDATE)) = htonl(last_appenddate); + /* Update checksum */ + *((bit32 *)(hbuf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(hbuf, OFFSET_HEADER_CRC)); + /* And write it back out */ lseek(mailbox->index_fd, 0L, SEEK_SET); @@ -1054,6 +1099,9 @@ int sync_uidvalidity_commit(struct mailbox *mailbox, /* Fix up uidvalidity */ *((bit32 *)(hbuf+OFFSET_UIDVALIDITY)) = htonl(uidvalidity); + *((bit32 *)(hbuf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(hbuf, OFFSET_HEADER_CRC)); + /* And write it back out */ lseek(mailbox->index_fd, 0L, SEEK_SET); @@ -1174,6 +1222,9 @@ int sync_highestmodseq_commit(struct mailbox *mailbox, *((bit32 *)(hbuf+OFFSET_HIGHESTMODSEQ)) = htonl(newhighestmodseq); #endif + *((bit32 *)(hbuf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(hbuf, OFFSET_HEADER_CRC)); + /* And write it back out */ lseek(mailbox->index_fd, 0L, SEEK_SET); diff --git a/imap/sync_server.c b/imap/sync_server.c index b50416d..1c8bda7 100644 --- a/imap/sync_server.c +++ b/imap/sync_server.c @@ -1522,6 +1522,7 @@ static void cmd_reserve(char *mailbox_name, message->content_lines = record.content_lines; message->cache_version = record.cache_version; message->cache_size = cache_size; + message->cache_crc = record.cache_crc; sync_message_list_cache(message_list, (char *)(m.cache_base+record.cache_offset), diff --git a/imap/sync_support.c b/imap/sync_support.c index 1147942..0d413a6 100644 --- a/imap/sync_support.c +++ b/imap/sync_support.c @@ -1302,6 +1302,7 @@ int sync_getcache(struct protstream *input, struct protstream *output, } message->cache_offset = sync_message_list_cache_offset(list); message->cache_size = cache_size; + message->cache_crc = crc32_buf(cache_entry, cache_size); sync_message_list_cache(list, cache_entry, cache_size); return(0); @@ -1423,6 +1424,7 @@ int sync_getsimple(struct protstream *input, struct protstream *output, message->cache_version = record.cache_version; message->cache_size = lseek(list->cache_fd, 0, SEEK_CUR) - record.cache_offset; + message->cache_crc = record.cache_crc; fclose(file); return(r); diff --git a/imap/sync_support.h b/imap/sync_support.h index 30bb164..6c15d5e 100644 --- a/imap/sync_support.h +++ b/imap/sync_support.h @@ -282,6 +282,7 @@ struct sync_message { unsigned long cache_size; unsigned long content_lines; unsigned long cache_version; + uint32_t cache_crc; struct message_guid guid; char stagename[100]; diff --git a/imap/unexpunge.c b/imap/unexpunge.c index 213fc42..5429d07 100644 --- a/imap/unexpunge.c +++ b/imap/unexpunge.c @@ -261,6 +261,9 @@ int restore_expunged(struct mailbox *mailbox, /* Write record to cyrus.index */ *((bit32 *)(buf+OFFSET_LAST_UPDATED)) = htonl(now); + /* Update checksum */ + *((bit32 *)(buf+OFFSET_RECORD_CRC)) = + htonl(crc32_buf(buf, OFFSET_RECORD_CRC)); fwrite(buf, 1, mailbox->record_size, newindex); } else { @@ -313,6 +316,10 @@ int restore_expunged(struct mailbox *mailbox, *((bit32 *)(buf+OFFSET_QUOTA_MAILBOX_USED)) = htonl(newquotaused); #endif + /* Update checksum */ + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); + /* Write out new index header */ rewind(newindex); fwrite(buf, 1, mailbox->start_offset, newindex); @@ -357,6 +364,10 @@ int restore_expunged(struct mailbox *mailbox, *((bit32 *)(buf+OFFSET_QUOTA_MAILBOX_USED)) = htonl(newquotaused); #endif + /* Update checksum */ + *((bit32 *)(buf+OFFSET_HEADER_CRC)) = + htonl(crc32_buf(buf, OFFSET_HEADER_CRC)); + /* Write out new expunge index header */ rewind(newexpungeindex); fwrite(buf, 1, mailbox->start_offset, newexpungeindex); -- 1.5.6.5