talons

commit 9729bf0e0d39ca60ef9717d7f059e4dfdf7fd60d
parent 126ff4dcbc912d839ba9e0340857d22abf71e0ef
Author: Andrej Kacian <ticho@claws-mail.org>
Date:   Mon, 23 Oct 2017 13:49:15 +0200

Fetch XOVER and XHDR data in batches of 5000.

This avoids long stalls, which sometimes confuse libetpan.
It also allows implementing some sort of progressbar in UI.

Diffstat:
M
src/etpan/nntp-thread.c
 | 
124
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------
M
src/news.c
 | 
4
+++-

2 files changed, 103 insertions(+), 25 deletions(-)
diff --git a/src/etpan/nntp-thread.c b/src/etpan/nntp-thread.c
@@ -51,6 +51,8 @@
 
 #define DISABLE_LOG_DURING_LOGIN
 
+#define NNTP_BATCH_SIZE 4999
+
 static struct etpan_thread_manager * thread_manager = NULL;
 static chash * nntp_hash = NULL;
 static chash * session_hash = NULL;
@@ -822,26 +824,62 @@ static void xover_run(struct etpan_thread_op * op)
 	}
 	
 	result->error = r;
-	debug_print("nntp xover run - end %i\n", r);
+	debug_print("nntp xover run %d-%d - end %i\n",
+			param->beg, param->end, r);
 }
 
 int nntp_threaded_xover(Folder * folder, guint32 beg, guint32 end, struct newsnntp_xover_resp_item **single_result, clist **multiple_result)
 {
 	struct xover_param param;
 	struct xover_result result;
-	
-	debug_print("nntp xover - begin\n");
-	
-	param.nntp = get_nntp(folder);
-	param.beg = beg;
-	param.end = end;
-	param.result = single_result;
-	param.msglist = multiple_result;
-
-	threaded_run(folder, &param, &result, xover_run);
+	clist *l = NULL, *h = NULL;
+	guint32 cbeg = 0, cend = 0;
+
+	debug_print("nntp xover - begin (%d-%d)\n", beg, end);
+
+	h = clist_new();
+
+	/* Request the overview in batches of NNTP_BATCH_SIZE, to prevent
+	 * long stalls or libetpan choking on too large server response,
+	 * and to allow updating any progress indicators while we work. */
+	cbeg = beg;
+	while (cbeg <= end && cend <= end) {
+		cend = cbeg + NNTP_BATCH_SIZE;
+		if (cend > end)
+			cend = end;
+
+		param.nntp = get_nntp(folder);
+		param.beg = cbeg;
+		param.end = cend;
+		param.result = single_result;
+		param.msglist = &l;
+
+		threaded_run(folder, &param, &result, xover_run);
+
+		/* Handle errors */
+		if (result.error != NEWSNNTP_NO_ERROR) {
+			log_warning(LOG_PROTOCOL, _("couldn't get xover range\n"));
+			debug_print("couldn't get xover for %d-%d\n", cbeg, cend);
+			if (l != NULL)
+				clist_free(l);
+			return result.error;
+		}
+
+		/* Append the new data (l) to list of results (h). */
+		if (l != NULL) {
+			debug_print("total items so far %d, items this batch %d\n",
+					clist_count(h), clist_count(l));
+			clist_concat(h, l);
+			l = NULL;
+		}
+
+		cbeg += NNTP_BATCH_SIZE + 1;
+	}
 	
 	debug_print("nntp xover - end\n");
-	
+
+	*multiple_result = h;
+
 	return result.error;
 }
 
@@ -875,26 +913,64 @@ static void xhdr_run(struct etpan_thread_op * op)
 	}
 	
 	result->error = r;
-	debug_print("nntp xhdr run - end %i\n", r);
+	debug_print("nntp xhdr '%s %d-%d' run - end %i\n",
+			param->header, param->beg, param->end, r);
 }
 
 int nntp_threaded_xhdr(Folder * folder, const char *header, guint32 beg, guint32 end, clist **hdrlist)
 {
 	struct xhdr_param param;
 	struct xhdr_result result;
+	clist *l = NULL;
+	clist *h = *hdrlist;
+	guint32 cbeg = 0, cend = 0;
+
+	debug_print("nntp xhdr %s - begin (%d-%d)\n", header, beg, end);
+
+	if (h == NULL)
+		h = clist_new();
+
+	/* Request the headers in batches of NNTP_BATCH_SIZE, to prevent
+	 * long stalls or libetpan choking on too large server response,
+	 * and to allow updating any progress indicators while we work. */
+	cbeg = beg;
+	while (cbeg <= end && cend <= end) {
+		cend = cbeg + NNTP_BATCH_SIZE;
+		if (cend > end)
+			cend = end;
+
+		param.nntp = get_nntp(folder);
+		param.header = header;
+		param.beg = cbeg;
+		param.end = cend;
+		param.hdrlist = &l;
+
+		threaded_run(folder, &param, &result, xhdr_run);
+
+		/* Handle errors */
+		if (result.error != NEWSNNTP_NO_ERROR) {
+			log_warning(LOG_PROTOCOL, _("couldn't get xhdr range\n"));
+			debug_print("couldn't get xhdr %s %d-%d\n",	header, cbeg, cend);
+			if (l != NULL)
+				clist_free(l);
+			return result.error;
+		}
+
+		/* Append the new data (l) to list of results (h). */
+		if (l != NULL) {
+			debug_print("total items so far %d, items this batch %d\n",
+					clist_count(h), clist_count(l));
+			clist_concat(h, l);
+			l = NULL;
+		}
+
+		cbeg += NNTP_BATCH_SIZE + 1;
+	}
 	
-	debug_print("nntp xhdr - begin\n");
-	
-	param.nntp = get_nntp(folder);
-	param.header = header;
-	param.beg = beg;
-	param.end = end;
-	param.hdrlist = hdrlist;
+	debug_print("nntp xhdr %s - end (%d-%d)\n", header, beg, end);
+
+	*hdrlist = h;
 
-	threaded_run(folder, &param, &result, xhdr_run);
-	
-	debug_print("nntp xhdr - end\n");
-	
 	return result.error;
 }
 
diff --git a/src/news.c b/src/news.c
@@ -1149,6 +1149,7 @@ static void news_get_extra_fields(NewsSession *session, FolderItem *item, GSList
 		}
 	}
 	newsnntp_xhdr_free(hdrlist);
+	hdrlist = NULL;
 	
 /* To */
 	ok = nntp_threaded_xhdr(item->folder, "to", first, last, &hdrlist);
@@ -1173,6 +1174,7 @@ static void news_get_extra_fields(NewsSession *session, FolderItem *item, GSList
 		}
 	}
 	newsnntp_xhdr_free(hdrlist);
+	hdrlist = NULL;
 	
 /* Cc */
 	ok = nntp_threaded_xhdr(item->folder, "cc", first, last, &hdrlist);
@@ -1197,7 +1199,7 @@ static void news_get_extra_fields(NewsSession *session, FolderItem *item, GSList
 		}
 	}
 	newsnntp_xhdr_free(hdrlist);
-	
+	hdrlist = NULL;
 
 	g_hash_table_destroy(hash_table);
 	news_folder_unlock(NEWS_FOLDER(item->folder));