Synchronet now requires the libarchive development package (e.g. libarchive-dev on Debian-based Linux distros, libarchive.org for more info) to build successfully.

Commit b81a0c95 authored by rswindell's avatar rswindell

There appears to be an issue with QWKnet messages being crossed-up where

a message is posted to a different conference than the original sub-board with
completely different header information. I suspect this has something to do
HEADERS.DAT creation or import - not sure. So I added a "Conference"
headers.dat field for *messages* (it already existed for votes) and use that
value to confirm that the message header at the associated offset value in the
QWK packet has the same conference number as the section in the headers.dat
file. This is really just a sanity check and will only catch messages that were
mistakening cross-posted (to a different conference) - *but* it log errors and
save the bad QWK or REP file for me to examine more closely and see what's
going on - and the message won't be imported (just "lost", which is also bad).

So added more QWK import success/error checking and logging (especially for QWK
packets since REP importing already had a lot of stats covered).

Another check would be to store the original message number in the headers.dat
file as well and use that to confirm that the headers.dat section is the
correct match for the QWK message at that offset. I did not implement this
check, yet. The conference number check seems like it'll catch most of the bad
msgs and lead me to the root-cause.
parent 249faa11
......@@ -228,6 +228,8 @@ ulong sbbs_t::msgtoqwk(smbmsg_t* msg, FILE *qwk_fp, long mode, smb_t* smb
if(msg->hfield[i].type==RFC822HEADER)
fprintf(hdrs,"%s\n",truncsp_lines((char*)msg->hfield_dat[i]));
fprintf(hdrs, "Conference: %u\n", conf);
/* Blank line: */
fprintf(hdrs,"\n");
}
......
......@@ -1046,7 +1046,8 @@ uint sbbs_t::resolve_qwkconf(uint n, int hubnum)
return usrsub[j][k];
}
bool sbbs_t::qwk_voting(str_list_t* ini, long offset, smb_net_type_t net_type, const char* qnet_id, int hubnum)
bool sbbs_t::qwk_voting(str_list_t* ini, long offset, smb_net_type_t net_type, const char* qnet_id
, uint confnum, int hubnum)
{
char* section;
char location[128];
......@@ -1064,7 +1065,7 @@ bool sbbs_t::qwk_voting(str_list_t* ini, long offset, smb_net_type_t net_type, c
strListFree(&section_list);
return false;
}
result = qwk_vote(*ini, section, net_type, qnet_id, hubnum);
result = qwk_vote(*ini, section, net_type, qnet_id, confnum, hubnum);
iniRemoveSection(ini, section);
iniRemoveSection(ini, location);
strListFree(&section_list);
......@@ -1076,22 +1077,36 @@ void sbbs_t::qwk_handle_remaining_votes(str_list_t* ini, smb_net_type_t net_type
str_list_t section_list = iniGetSectionList(*ini, /* prefix: */NULL);
for(int i=0; section_list != NULL && section_list[i] != NULL; i++)
qwk_vote(*ini, section_list[i], net_type, qnet_id, hubnum);
qwk_vote(*ini, section_list[i], net_type, qnet_id, /* confnum: */0, hubnum);
strListFree(&section_list);
}
bool sbbs_t::qwk_vote(str_list_t ini, const char* section, smb_net_type_t net_type, const char* qnet_id, int hubnum)
bool sbbs_t::qwk_vote(str_list_t ini, const char* section, smb_net_type_t net_type, const char* qnet_id, uint confnum, int hubnum)
{
char* p;
int result;
smb_t smb;
ZERO_VAR(smb);
ulong n = iniGetLongInt(ini, section, "Conference", 0);
if(confnum == 0)
confnum = n;
else if(n != confnum) {
char info[128];
SAFEPRINTF(info, "expected: %u", confnum);
errormsg(WHERE, ERR_CHK, "conference number", n, info);
return false;
}
smb.subnum = resolve_qwkconf(iniGetInteger(ini, section, "Conference", 0), hubnum);
if(smb.subnum == INVALID_SUB)
smb.subnum = resolve_qwkconf(confnum, hubnum);
if(smb.subnum == INVALID_SUB) {
errormsg(WHERE, ERR_CHK, "conference number", confnum, "invalid");
return false;
if(cfg.sub[smb.subnum]->misc&SUB_NOVOTING)
}
if(cfg.sub[smb.subnum]->misc&SUB_NOVOTING) {
errormsg(WHERE, ERR_CHK, "conference number", confnum, "voting not allowed");
return false;
}
if((result = smb_open_sub(&cfg, &smb, smb.subnum)) != SMB_SUCCESS) {
errormsg(WHERE, ERR_OPEN, smb.file, 0, smb.last_error);
return false;
......
......@@ -39,7 +39,7 @@
#include "qwk.h"
#include "utf8.h"
static void qwk_parse_header_list(ulong confnum, smbmsg_t* msg, str_list_t* headers, bool parse_sender_hfields, bool parse_recipient_hfields)
static bool qwk_parse_header_list(sbbs_t* sbbs, ulong confnum, smbmsg_t* msg, str_list_t* headers, bool parse_sender_hfields, bool parse_recipient_hfields)
{
char* p;
char zone[32];
......@@ -48,6 +48,13 @@ static void qwk_parse_header_list(ulong confnum, smbmsg_t* msg, str_list_t* head
uint16_t net_type;
uint16_t hfield_type;
if((p = iniPopKey(headers,ROOT_SECTION,"Conference",value)) != NULL) {
if(confnum > 0 && confnum != strtoul(value, NULL, 0)) {
sbbs->errormsg(WHERE, ERR_CHK, "Conference number", confnum, value);
return false;
}
}
if((p=iniPopKey(headers,ROOT_SECTION,"WhenWritten",value))!=NULL) {
xpDateTime_t dt=isoDateTimeStr_parse(p);
......@@ -158,9 +165,11 @@ static void qwk_parse_header_list(ulong confnum, smbmsg_t* msg, str_list_t* head
for(i=0;(*headers)[i]!=NULL;i++)
if((*headers)[i][0])
smb_hfield_str(msg,RFC822HEADER,(*headers)[i]);
return true;
}
void sbbs_t::qwk_new_msg(ulong confnum, smbmsg_t* msg, char* hdrblk, long offset, str_list_t all_headers, bool parse_sender_hfields)
bool sbbs_t::qwk_new_msg(ulong confnum, smbmsg_t* msg, char* hdrblk, long offset, str_list_t all_headers, bool parse_sender_hfields)
{
char str[128];
char to[128];
......@@ -170,6 +179,10 @@ void sbbs_t::qwk_new_msg(ulong confnum, smbmsg_t* msg, char* hdrblk, long offset
sprintf(str,"%lx",offset);
msg_headers=iniGetSection(all_headers,str);
if(msg_headers == NULL && all_headers != NULL) {
errormsg(WHERE, ERR_CHK, "missing header section", offset);
return false;
}
memset(msg,0,sizeof(smbmsg_t)); /* Initialize message header */
msg->hdr.version=smb_ver();
......@@ -177,8 +190,10 @@ void sbbs_t::qwk_new_msg(ulong confnum, smbmsg_t* msg, char* hdrblk, long offset
sprintf(to,"%25.25s",(char *)hdrblk+21); /* To user */
truncsp(to);
if(msg_headers!=NULL)
qwk_parse_header_list(confnum, msg, &msg_headers, parse_sender_hfields, stricmp(to,"NETMAIL")!=0);
if(msg_headers!=NULL) {
if(!qwk_parse_header_list(this, confnum, msg, &msg_headers, parse_sender_hfields, stricmp(to,"NETMAIL") != 0))
return false;
}
/* Parse the QWK message header: */
if(msg->hdr.when_written.time==0) {
......
......@@ -976,9 +976,9 @@ public:
void qwksetptr(uint subnum, char *buf, int reset);
void qwkcfgline(char *buf,uint subnum);
int set_qwk_flag(ulong flag);
uint resolve_qwkconf(uint n, int hubnum=-1);
bool qwk_vote(str_list_t ini, const char* section, smb_net_type_t, const char* qnet_id, int hubnum);
bool qwk_voting(str_list_t* ini, long offset, smb_net_type_t, const char* qnet_id, int hubnum = -1);
uint resolve_qwkconf(uint confnum, int hubnum=-1);
bool qwk_vote(str_list_t ini, const char* section, smb_net_type_t, const char* qnet_id, uint confnum, int hubnum);
bool qwk_voting(str_list_t* ini, long offset, smb_net_type_t, const char* qnet_id, uint confnum, int hubnum = -1);
void qwk_handle_remaining_votes(str_list_t* ini, smb_net_type_t, const char* qnet_id, int hubnum = -1);
/* pack_qwk.cpp */
......@@ -997,7 +997,7 @@ public:
ulong msgtoqwk(smbmsg_t* msg, FILE *qwk_fp, long mode, smb_t*, int conf, FILE* hdrs_dat, FILE* voting_dat = NULL);
/* qwktomsg.cpp */
void qwk_new_msg(ulong confnum, smbmsg_t* msg, char* hdrblk, long offset, str_list_t headers, bool parse_sender_hfields);
bool qwk_new_msg(ulong confnum, smbmsg_t* msg, char* hdrblk, long offset, str_list_t headers, bool parse_sender_hfields);
bool qwk_import_msg(FILE *qwk_fp, char *hdrblk, ulong blocks, char fromhub, smb_t*
,uint touser, smbmsg_t* msg);
......
......@@ -63,6 +63,7 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
ulong t;
ulong msgs=0;
ulong tmsgs=0;
ulong errors=0;
time_t start;
time_t startsub;
DIR* dir;
......@@ -148,23 +149,29 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
eprintf(LOG_NOTICE,"!Invalid QWK message status (%02X) at offset %lu in %s"
,block[0], l, packet);
blocks=1;
errors++;
continue;
}
sprintf(tmp,"%.6s",block+116);
blocks=atoi(tmp); /* i = number of blocks */
n=(uint)block[123]|(((uint)block[124])<<8); /* conference number */
if(blocks<2) {
if(block[0] == 'V' && blocks == 1 && voting != NULL) { /* VOTING DATA */
qwk_voting(&voting, l, NET_QWK, cfg.qhub[hubnum]->id, hubnum);
if(!qwk_voting(&voting, l, NET_QWK, cfg.qhub[hubnum]->id, n, hubnum))
errors++;
continue;
}
eprintf(LOG_NOTICE,"!Invalid number of QWK blocks (%d) at offset %lu in %s"
,blocks, l+116, packet);
errors++;
blocks=1;
continue;
}
n=(uint)block[123]|(((uint)block[124])<<8); /* conference number */
qwk_new_msg(n, &msg,(char*)block,/* offset: */l,headers,/* parse_sender_hfields: */true);
if(!qwk_new_msg(n, &msg,(char*)block,/* offset: */l,headers,/* parse_sender_hfields: */true)) {
errors++;
continue;
}
if(cfg.max_qwkmsgage && msg.hdr.when_written.time < (uint32_t)now
&& (now-msg.hdr.when_written.time)/(24*60*60) > cfg.max_qwkmsgage) {
......@@ -231,6 +238,7 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
if((k=smb_open(&smb))!=0) {
errormsg(WHERE,ERR_OPEN,smb.file,k,smb.last_error);
smb_stack(&smb,SMB_STACK_POP);
errors++;
continue;
}
if(!filelength(fileno(smb.shd_fp))) {
......@@ -242,6 +250,7 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
smb_close(&smb);
errormsg(WHERE,ERR_CREATE,smb.file,k,smb.last_error);
smb_stack(&smb,SMB_STACK_POP);
errors++;
continue;
}
}
......@@ -249,12 +258,14 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
smb_close(&smb);
errormsg(WHERE,ERR_LOCK,smb.file,k,smb.last_error);
smb_stack(&smb,SMB_STACK_POP);
errors++;
continue;
}
if((k=smb_getstatus(&smb))!=0) {
smb_close(&smb);
errormsg(WHERE,ERR_READ,smb.file,k,smb.last_error);
smb_stack(&smb,SMB_STACK_POP);
errors++;
continue;
}
smb_unlocksmbhdr(&smb);
......@@ -263,7 +274,8 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
SAFEPRINTF(str,text[UserSentYouMail],msg.from);
putsmsg(&cfg,usernum,str);
tmsgs++;
}
} else
errors++;
smb_close(&smb);
smb_stack(&smb,SMB_STACK_POP);
continue;
......@@ -275,6 +287,7 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
if((j = resolve_qwkconf(n, hubnum)) == INVALID_SUB) { /* ignore messages for subs not in config */
eprintf(LOG_NOTICE,"!Message from %s on UNKNOWN QWK CONFERENCE NUMBER: %u"
,cfg.qhub[hubnum]->id, n);
errors++;
continue;
}
......@@ -305,6 +318,7 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
smb.subnum=j;
if((k=smb_open(&smb))!=0) {
errormsg(WHERE,ERR_OPEN,smb.file,k,smb.last_error);
errors++;
continue;
}
if(!filelength(fileno(smb.shd_fp))) {
......@@ -315,17 +329,20 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
if((k=smb_create(&smb))!=0) {
smb_close(&smb);
errormsg(WHERE,ERR_CREATE,smb.file,k,smb.last_error);
errors++;
continue;
}
}
if((k=smb_locksmbhdr(&smb))!=0) {
smb_close(&smb);
errormsg(WHERE,ERR_LOCK,smb.file,k,smb.last_error);
errors++;
continue;
}
if((k=smb_getstatus(&smb))!=0) {
smb_close(&smb);
errormsg(WHERE,ERR_READ,smb.file,k,smb.last_error);
errors++;
continue;
}
smb_unlocksmbhdr(&smb);
......@@ -336,7 +353,8 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
signal_sub_sem(&cfg,j);
msgs++;
tmsgs++;
}
} else
errors++;
}
if(lastsub != INVALID_SUB) {
log_qwk_import_stats(msgs, startsub);
......@@ -400,12 +418,12 @@ bool sbbs_t::unpack_qwk(char *packet,uint hubnum)
if(t<1)
t=1;
eprintf(LOG_INFO,"Finished Importing QWK Network Packet from %s: "
"(%lu msgs) in %lu seconds (%lu msgs/sec)"
,cfg.qhub[hubnum]->id, tmsgs, t, tmsgs/t);
"(%lu msgs) in %lu seconds (%lu msgs/sec), %lu errors"
,cfg.qhub[hubnum]->id, tmsgs, t, tmsgs/t, errors);
/* trigger timed event with internal code of 'qnet-qwk' to run */
sprintf(str,"%sqnet-qwk.now",cfg.data_dir);
ftouch(str);
}
delfiles(cfg.temp_dir,ALLFILES);
return(true);
return errors == 0;
}
......@@ -55,6 +55,7 @@ bool sbbs_t::unpack_rep(char* repfile)
long l,size,misc;
ulong n;
ulong ex;
ulong tmsgs = 0;
ulong errors = 0;
node_t node;
FILE* rep;
......@@ -191,9 +192,10 @@ bool sbbs_t::unpack_rep(char* repfile)
}
sprintf(tmp,"%.6s",block+116);
blocks=atoi(tmp); /* i = number of blocks */
long confnum = atol((char *)block+1);
if(blocks<2) {
if(block[0] == 'V' && blocks == 1 && voting != NULL) { /* VOTING DATA */
if(!qwk_voting(&voting, l, (useron.rest&FLAG('Q')) ? NET_QWK : NET_NONE, /* QWKnet ID : */useron.alias))
if(!qwk_voting(&voting, l, (useron.rest&FLAG('Q')) ? NET_QWK : NET_NONE, /* QWKnet ID : */useron.alias, confnum))
errors++;
continue;
}
......@@ -206,9 +208,10 @@ bool sbbs_t::unpack_rep(char* repfile)
continue;
}
long confnum = atol((char *)block+1);
qwk_new_msg(confnum, &msg, block, /* offset: */l, headers, /* parse_sender_hfields: */useron.rest&FLAG('Q') ? true:false);
if(!qwk_new_msg(confnum, &msg, block, /* offset: */l, headers, /* parse_sender_hfields: */useron.rest&FLAG('Q') ? true:false)) {
errors++;
continue;
}
if(cfg.max_qwkmsgage && msg.hdr.when_written.time < (uint32_t)now
&& (now-msg.hdr.when_written.time)/(24*60*60) > cfg.max_qwkmsgage) {
......@@ -345,7 +348,6 @@ bool sbbs_t::unpack_rep(char* repfile)
if(qwk_import_msg(rep, block, blocks
,/* fromhub: */0, &smb, /* touser: */usernum, &msg)) {
if(usernum==1) {
useron.fbacks++;
logon_fbacks++;
......@@ -383,7 +385,9 @@ bool sbbs_t::unpack_rep(char* repfile)
SAFEPRINTF(str,text[UserSentYouMail],msg.from);
putsmsg(&cfg,usernum,str);
}
}
tmsgs++;
} else
errors++;
smb_close(&smb);
} /* end of email */
......@@ -545,7 +549,9 @@ bool sbbs_t::unpack_rep(char* repfile)
logline("P+",str);
if(!(useron.rest&FLAG('Q')))
user_event(EVENT_POST);
}
tmsgs++;
} else
errors++;
} /* end of public message */
}
......@@ -660,7 +666,7 @@ bool sbbs_t::unpack_rep(char* repfile)
/**********************************************/
autohangup();
} else
lprintf(LOG_INFO, "Unpacking completed: %s", rep_fname);
lprintf(LOG_INFO, "Unpacking completed: %s (%lu msgs, %lu errors)", rep_fname, tmsgs, errors);
return errors == 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment