From 264cb38c72f1ae410e253b62d780c716ea9768cb Mon Sep 17 00:00:00 2001 From: rswindell <> Date: Tue, 24 Jul 2018 05:15:55 +0000 Subject: [PATCH] Singificantlly improve message scan pointer initialization (e.g. for new users) and adjusting scan ptrs by date or set to "Last msg" while online: - speed (on my system, reduced from 12 to < 2 seconds) This involved 2 enhancements: check msgbase index file timestamp (fast) before opening any files. When opening the msgbase, just open the index file (no other msgbase files) - required new smblib function: smb_open_index. - fixed off-by-one when setting by date (when setting the new-scan ptr to Jan-1-2018, you want that first message posted on/after Jan-1-2018 to be found as "new", not the first message posted *after* that date. - re-added the fixmsgptrs() call in putmsgptrs(), inadvertently removed in the .ixb -> .ini ptr storage update in rev 1.192 of userdat.c. --- src/sbbs3/getmsg.cpp | 11 ++++++----- src/sbbs3/msgdate.c | 8 ++++++++ src/sbbs3/sbbs.h | 3 +++ src/sbbs3/scansubs.cpp | 14 ++++++++++++-- src/sbbs3/userdat.c | 38 +++++++++++++++++++++++--------------- src/smblib/smblib.c | 21 +++++++++++---------- src/smblib/smblib.h | 1 + 7 files changed, 64 insertions(+), 32 deletions(-) diff --git a/src/sbbs3/getmsg.cpp b/src/sbbs3/getmsg.cpp index 3fcac400d3..db56cb9cfc 100644 --- a/src/sbbs3/getmsg.cpp +++ b/src/sbbs3/getmsg.cpp @@ -328,14 +328,15 @@ ulong sbbs_t::getmsgnum(uint subnum, time_t t) SAFEPRINTF2(smb.file,"%s%s",cfg.sub[subnum]->data_dir,cfg.sub[subnum]->code); smb.retry_time=cfg.smb_retry_time; smb.subnum=subnum; - if((i=smb_open(&smb)) != SMB_SUCCESS) { + if((i=smb_open_index(&smb)) != SMB_SUCCESS) { errormsg(WHERE,ERR_OPEN,smb.file,i,smb.last_error); - return(0); + return 0; } - smb_getmsgidx_by_time(&smb, &idx, t); - + int result = smb_getmsgidx_by_time(&smb, &idx, t); smb_close(&smb); - return idx.number; + if(result >= SMB_SUCCESS) + return idx.number - 1; + return ~0; } /****************************************************************************/ diff --git a/src/sbbs3/msgdate.c b/src/sbbs3/msgdate.c index b8790e43ff..b0ba4cbafc 100644 --- a/src/sbbs3/msgdate.c +++ b/src/sbbs3/msgdate.c @@ -174,3 +174,11 @@ when_t DLLCALL rfc822date(char* date) return(when); } + +BOOL DLLCALL newmsgs(smb_t* smb, time_t t) +{ + char index_fname[MAX_PATH + 1]; + + SAFEPRINTF(index_fname, "%s.sid", smb->file); + return fdate(index_fname) > t; +} diff --git a/src/sbbs3/sbbs.h b/src/sbbs3/sbbs.h index 35899fab52..61bcf7c4d4 100644 --- a/src/sbbs3/sbbs.h +++ b/src/sbbs3/sbbs.h @@ -1106,8 +1106,11 @@ extern "C" { DLLEXPORT char * DLLCALL seconds_to_str(uint, char*); DLLEXPORT char * DLLCALL hhmmtostr(scfg_t* cfg, struct tm* tm, char* str); DLLEXPORT char * DLLCALL timestr(scfg_t* cfg, time32_t intime, char* str); + + /* msgdate.c */ DLLEXPORT when_t DLLCALL rfc822date(char* p); DLLEXPORT char * DLLCALL msgdate(when_t when, char* buf); + DLLEXPORT BOOL DLLCALL newmsgs(smb_t*, time_t); /* load_cfg.c */ DLLEXPORT BOOL DLLCALL load_cfg(scfg_t* cfg, char* text[], BOOL prep, char* error); diff --git a/src/sbbs3/scansubs.cpp b/src/sbbs3/scansubs.cpp index dd22361083..83ff1c748c 100644 --- a/src/sbbs3/scansubs.cpp +++ b/src/sbbs3/scansubs.cpp @@ -301,6 +301,10 @@ void sbbs_t::new_scan_ptr_cfg() for(j=0;j<usrsubs[i] && online;j++) { progress(text[LoadingMsgPtrs], subs++, total_subs, 10); checkline(); + if(s == 0) { + subscan[usrsub[i][j]].ptr = ~0; + continue; + } getlastmsg(usrsub[i][j],&l,0); if(s>(long)l) subscan[usrsub[i][j]].ptr=0; @@ -358,6 +362,10 @@ void sbbs_t::new_scan_ptr_cfg() for(j=0;j<usrsubs[i] && online;j++) { progress(text[LoadingMsgPtrs], j, usrsubs[i], 10); checkline(); + if(s == 0) { + subscan[usrsub[i][j]].ptr = ~0; + continue; + } getlastmsg(usrsub[i][j],&l,0); if(s>(long)l) subscan[usrsub[i][j]].ptr=0; @@ -382,8 +390,10 @@ void sbbs_t::new_scan_ptr_cfg() } continue; } - if(s=='L') - s=0; + if(s=='L') { + subscan[usrsub[i][j]].ptr = ~0; + continue; + } if(s) s&=~0x80000000L; getlastmsg(usrsub[i][j],&l,0); diff --git a/src/sbbs3/userdat.c b/src/sbbs3/userdat.c index 93469b36ea..a2cdd7f0c7 100644 --- a/src/sbbs3/userdat.c +++ b/src/sbbs3/userdat.c @@ -3111,6 +3111,7 @@ BOOL DLLCALL putmsgptrs(scfg_t* cfg, user_t* user, subscan_t* subscan) if(user->number==0 || (user->rest&FLAG('G'))) /* Guest */ return(TRUE); + fixmsgptrs(cfg, subscan); SAFEPRINTF2(path,"%suser/%4.4u.subs", cfg->data_dir, user->number); FILE* fp = fnopen(NULL, path, O_RDWR|O_CREAT|O_TEXT); if (fp == NULL) @@ -3153,21 +3154,25 @@ BOOL DLLCALL initmsgptrs(scfg_t* cfg, subscan_t* subscan, unsigned days, void (* for(i=0;i<cfg->total_subs;i++) { if(progress != NULL) progress(cbdata, i, cfg->total_subs); - if(days == 0) { - /* This value will be "fixed" (changed to the last msg) when saving */ - subscan[i].ptr = ~0; + /* This value will be "fixed" (changed to the last msg) when saving */ + subscan[i].ptr = ~0; + if(days == 0) continue; - } ZERO_VAR(smb); SAFEPRINTF2(smb.file,"%s%s",cfg->sub[i]->data_dir,cfg->sub[i]->code); + + if(!newmsgs(&smb, t)) + continue; + smb.retry_time=cfg->smb_retry_time; smb.subnum=i; - if(smb_open(&smb) != SMB_SUCCESS) + if(smb_open_index(&smb) != SMB_SUCCESS) continue; - if(days == 0) - subscan[i].ptr = smb.status.last_msg; - else if(smb_getmsgidx_by_time(&smb, &idx, t) >= SMB_SUCCESS) - subscan[i].ptr = idx.number; + memset(&idx, 0, sizeof(idx)); + smb_getlastidx(&smb, &idx); + subscan[i].ptr = idx.number; + if(smb_getmsgidx_by_time(&smb, &idx, t) >= SMB_SUCCESS) + subscan[i].ptr = idx.number - 1; smb_close(&smb); } if(progress != NULL) @@ -3187,18 +3192,21 @@ BOOL DLLCALL fixmsgptrs(scfg_t* cfg, subscan_t* subscan) for(i=0;i<cfg->total_subs;i++) { if(subscan[i].ptr == 0) continue; - if(subscan[i].sav_ptr == subscan[i].ptr) + if(subscan[i].ptr < ~0 && subscan[i].sav_ptr == subscan[i].ptr) continue; ZERO_VAR(smb); SAFEPRINTF2(smb.file,"%s%s",cfg->sub[i]->data_dir,cfg->sub[i]->code); smb.retry_time=cfg->smb_retry_time; smb.subnum=i; - if(smb_open(&smb) != SMB_SUCCESS) + if(smb_open_index(&smb) != SMB_SUCCESS) continue; - if(subscan[i].ptr > smb.status.last_msg) - subscan[i].ptr = smb.status.last_msg; - if(subscan[i].last > smb.status.last_msg) - subscan[i].last = smb.status.last_msg; + idxrec_t idx; + memset(&idx, 0xff, sizeof(idx)); + smb_getlastidx(&smb, &idx); + if(subscan[i].ptr > idx.number) + subscan[i].ptr = idx.number; + if(subscan[i].last > idx.number) + subscan[i].last = idx.number; smb_close(&smb); } return TRUE; diff --git a/src/smblib/smblib.c b/src/smblib/smblib.c index 65749ff8ff..89e335906e 100644 --- a/src/smblib/smblib.c +++ b/src/smblib/smblib.c @@ -150,12 +150,17 @@ int SMBCALL smb_open(smb_t* smb) if((i=smb_open_fp(smb,&smb->sdt_fp,SH_DENYNO))!=SMB_SUCCESS) return(i); - if((i=smb_open_fp(smb,&smb->sid_fp,SH_DENYNO))!=SMB_SUCCESS) - return(i); + if((i=smb_open_index(smb)) != SMB_SUCCESS) + return i; return(SMB_SUCCESS); } +int SMBCALL smb_open_index(smb_t* smb) +{ + return smb_open_fp(smb, &smb->sid_fp, SH_DENYNO); +} + /****************************************************************************/ /* Closes the currently open message base */ /****************************************************************************/ @@ -652,6 +657,8 @@ int SMBCALL smb_getlastidx(smb_t* smb, idxrec_t *idx) /****************************************************************************/ /* Finds index of last message imported at or after specified time */ +/* If you want the message base locked during this operation, the caller */ +/* must call smb_locksmbhdr() before, smb_unlocksmbhdr() after. */ /* Returns >= 0 on success, negative (SMB_* error) on failure. */ /****************************************************************************/ long SMBCALL smb_getmsgidx_by_time(smb_t* smb, idxrec_t* match, time_t t) @@ -674,15 +681,10 @@ long SMBCALL smb_getmsgidx_by_time(smb_t* smb, idxrec_t* match, time_t t) if(!total) /* Empty base */ return SMB_ERR_NOT_FOUND; - if((result=smb_locksmbhdr(smb)) != SMB_SUCCESS) - return result; - if((result=smb_getlastidx(smb, &idx)) != SMB_SUCCESS) { - smb_unlocksmbhdr(smb); return result; } if((time_t)idx.time < t) { - smb_unlocksmbhdr(smb); return SMB_ERR_NOT_FOUND; } @@ -695,9 +697,9 @@ long SMBCALL smb_getmsgidx_by_time(smb_t* smb, idxrec_t* match, time_t t) while(bot <= top) { long idx_offset = (bot + top) / 2; if(fseek(smb->sid_fp, idx_offset * sizeof(idxrec_t), SEEK_SET) != 0) - break; + return SMB_ERR_SEEK; if(fread(&idx, 1, sizeof(idx), smb->sid_fp) != sizeof(idxrec_t)) - break; + return SMB_ERR_READ; if((time_t)idx.time < t) { bot = idx_offset + 1; continue; @@ -710,7 +712,6 @@ long SMBCALL smb_getmsgidx_by_time(smb_t* smb, idxrec_t* match, time_t t) } break; } - smb_unlocksmbhdr(smb); return match_offset; } diff --git a/src/smblib/smblib.h b/src/smblib/smblib.h index d9f793c9d8..eb58628b09 100644 --- a/src/smblib/smblib.h +++ b/src/smblib/smblib.h @@ -131,6 +131,7 @@ extern "C" { SMBEXPORT int SMBCALL smb_ver(void); SMBEXPORT char* SMBCALL smb_lib_ver(void); SMBEXPORT int SMBCALL smb_open(smb_t* smb); +SMBEXPORT int SMBCALL smb_open_index(smb_t* smb); SMBEXPORT void SMBCALL smb_close(smb_t* smb); SMBEXPORT int SMBCALL smb_initsmbhdr(smb_t* smb); SMBEXPORT int SMBCALL smb_create(smb_t* smb); -- GitLab