commit df3833d369d125250f30a1ea1f20f9dca0f21a2a
parent 0cc52c0d67e5121b107ec5cc360140c797f25221
Author: Paul <paul@claws-mail.org>
Date: Mon, 7 Apr 2025 12:59:31 +0100
postpone auto-updates if Retry-After: given
Sometimes when a feed returns an HTTP error code due to having received
too many requests in too short a period of time (especially 403 or 429),
they specify a safe interval after which the request could be retried
using the Retry-After: HTTP header. Parse the value and skip automatic
updates if it's set to a date in the future.
When a manual update succeeds, clear the retry_after value, making it
possible to unstuck automatic updates in case a Retry-After: value was
accidentally set too far in the future.
patch by Ivan Krylov
Diffstat:
7 files changed, 51 insertions(+), 2 deletions(-)
diff --git a/src/plugins/rssyl/libfeed/feed.c b/src/plugins/rssyl/libfeed/feed.c
@@ -22,6 +22,7 @@
#include <glib.h>
#include <curl/curl.h>
#include <expat.h>
+#include <procheader.h>
#include "feed.h"
#include "../rssyl_prefs.h"
@@ -53,6 +54,7 @@ Feed *feed_new(gchar *url)
feed->cookies_path = NULL;
feed->last_modified = NULL;
feed->etag = NULL;
+ feed->retry_after = 0;
feed->ssl_verify_peer = TRUE;
feed->cacert_file = NULL;
@@ -374,6 +376,21 @@ guint feed_update(Feed *feed, const gchar *user_agent)
feed_set_last_modified(feed,
ret == CURLHE_OK ? header->value
: NULL);
+ ret = curl_easy_header(eh, "Retry-After", 0,
+ CURLH_HEADER, -1,
+ &header);
+ if (ret == CURLHE_OK && *header->value) {
+ /* Retry-After: either delay >= 0 seconds or HTTP Date */
+ char * end = NULL;
+ unsigned long long seconds = strtoull(header->value, &end, 10);
+ if (!*end && seconds > 0) { /* parse successful */
+ feed->retry_after = time(NULL) + seconds;
+ } else { /* will set to 0 if fails to parse */
+ feed->retry_after = procheader_date_parse(NULL,
+ header->value,
+ 0);
+ }
+ }
}
cleanup:
diff --git a/src/plugins/rssyl/libfeed/feed.h b/src/plugins/rssyl/libfeed/feed.h
@@ -61,6 +61,7 @@ struct _Feed {
gchar *cacert_file;
gchar *last_modified;
gchar *etag;
+ time_t retry_after;
GSList *items;
};
diff --git a/src/plugins/rssyl/rssyl.c b/src/plugins/rssyl/rssyl.c
@@ -36,9 +36,11 @@
#include <prefs_toolbar.h>
#include <utils.h>
#include <file-utils.h>
+#include <procheader.h>
/* Local includes */
#include "libfeed/feeditem.h"
+#include "libfeed/date.h"
#include "rssyl.h"
#include "rssyl_deleted.h"
#include "rssyl_gtk.h"
@@ -343,6 +345,10 @@ static void rssyl_item_set_xml(Folder *folder, FolderItem *item, XMLTag *tag)
g_free(ritem->etag);
ritem->etag = g_strdup(attr->value);
}
+ /* (time_t) Retry-After header */
+ if( !strcmp(attr->name, "retry_after")) {
+ ritem->retry_after = procheader_date_parse(NULL, attr->value, 0);
+ }
}
}
@@ -408,6 +414,13 @@ static XMLTag *rssyl_item_get_xml(Folder *folder, FolderItem *item)
/* (str) ETag header */
if( ri->etag != NULL )
xml_tag_add_attr(tag, xml_attr_new("etag", ri->etag));
+ /* (time_t) Retry-After */
+ time_t now = time(NULL);
+ if (ri->retry_after > now) {
+ tmp = createRFC822Date(&now);
+ xml_tag_add_attr(tag, xml_attr_new("retry_after", tmp));
+ g_free(tmp);
+ }
return tag;
}
@@ -478,6 +491,7 @@ static FolderItem *rssyl_item_new(Folder *folder)
ritem->specific_user_agent = NULL;
ritem->last_modified = NULL;
ritem->etag = NULL;
+ ritem->retry_after = 0;
return (FolderItem *)ritem;
}
@@ -1024,6 +1038,7 @@ static void rssyl_copy_private_data(Folder *folder, FolderItem *oldi,
g_free(newitem->specific_user_agent);
newitem->specific_user_agent = g_strdup(olditem->specific_user_agent);
}
+ newitem->retry_after = olditem->retry_after;
/* ETag, Last-Modified */
if (olditem->etag != NULL) {
diff --git a/src/plugins/rssyl/rssyl.h b/src/plugins/rssyl/rssyl.h
@@ -72,6 +72,7 @@ struct _RFolderItem {
time_t last_update;
gchar *last_modified;
gchar *etag;
+ time_t retry_after;
gboolean use_default_user_agent;
gchar *specific_user_agent;
diff --git a/src/plugins/rssyl/rssyl_feed.c b/src/plugins/rssyl/rssyl_feed.c
@@ -1,6 +1,6 @@
/*
* Claws Mail -- a GTK based, lightweight, and fast e-mail client
- * Copyright (C) 2006-2023 the Claws Mail Team and Andrej Kacian <andrej@kacian.sk>
+ * Copyright (C) 2006-2025 the Claws Mail Team and Andrej Kacian <andrej@kacian.sk>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -85,6 +85,11 @@ gboolean rssyl_refresh_timeout_cb(gpointer data)
if (prefs_common_get_prefs()->work_offline) {
debug_print("RSSyl: %s: skipping update of %s (%d), we are offline\n",
tmpdate, ctx->ritem->url, ctx->ritem->refresh_id);
+ } else if (ctx->ritem->retry_after > tt) {
+ gchar *whenretry = createRFC822Date(&ctx->ritem->retry_after);
+ debug_print("RSSyl: %s: skipping update of %s (%d): Retry-After = %s\n",
+ tmpdate, ctx->ritem->url, ctx->ritem->refresh_id, whenretry);
+ g_free(whenretry);
} else {
debug_print("RSSyl: %s: updating %s (%d)\n",
tmpdate, ctx->ritem->url, ctx->ritem->refresh_id);
diff --git a/src/plugins/rssyl/rssyl_feed.h b/src/plugins/rssyl/rssyl_feed.h
@@ -31,6 +31,7 @@
#define RSSYL_LOG_NOT_MODIFIED _("RSSyl: Feed not modified: %s\n")
#define RSSYL_LOG_UPDATED _("RSSyl: Feed update finished: %s\n")
#define RSSYL_LOG_ERROR_FETCH _("RSSyl: Error fetching feed at '%s': %s\n")
+#define RSSYL_LOG_RETRY_AFTER _("RSSyl: Told to update feed at '%s' after %s\n")
#define RSSYL_LOG_ERROR_NOFEED _("RSSyl: No valid feed found at '%s'\n")
#define RSSYL_LOG_ERROR_PROC _("RSSyl: Couldn't process feed at '%s'\n")
#define RSSYL_LOG_ABORTED_EXITING _("RSSyl: Application is exiting, couldn't finish updating feed at '%s'\n")
diff --git a/src/plugins/rssyl/rssyl_update_feed.c b/src/plugins/rssyl/rssyl_update_feed.c
@@ -37,6 +37,7 @@
/* Local includes */
#include "libfeed/feed.h"
+#include "libfeed/date.h"
#include "rssyl.h"
#include "rssyl_deleted.h"
#include "rssyl_feed.h"
@@ -145,7 +146,7 @@ void rssyl_fetch_feed(RFetchCtx *ctx, RSSylVerboseFlags verbose)
} else if( ctx->response_code == FEED_ERR_UNAUTH ) {
debug_print("RSSyl: URL authorization type is unknown\n");
ctx->error = g_strdup("Unknown value for URL authorization type");
- } else if( ctx->response_code >= 400 && ctx->response_code < 500 ) {
+ } else if( ctx->response_code >= 400 && ctx->response_code <= 599 ) {
switch( ctx->response_code ) {
case 401:
ctx->error = g_strdup(_("401 (Authorisation required)"));
@@ -304,12 +305,20 @@ gboolean rssyl_update_feed(RFolderItem *ritem, RSSylVerboseFlags verbose)
ctx->success ? "TRUE" : "FALSE");
if (!ctx->success) {
+ ritem->retry_after = ctx->feed->retry_after;
+ if (ritem->retry_after) {
+ gchar *tmp = createRFC822Date(&ritem->retry_after);
+ log_print(LOG_PROTOCOL, RSSYL_LOG_RETRY_AFTER, ritem->url, tmp);
+ g_free(tmp);
+ }
feed_free(ctx->feed);
g_free(ctx->error);
g_free(ctx);
STATUSBAR_POP(mainwin);
return FALSE;
}
+ /* Successful reply means we don't have to throttle auto-updates */
+ ritem->retry_after = 0;
g_free(ritem->etag);
gchar *etag = feed_get_etag(ctx->feed);
