From 42ea24f2f41ce04588aa929e5ffdaf1dbeb1a700 Mon Sep 17 00:00:00 2001 From: Kevin Decherf Date: Wed, 9 Dec 2020 22:13:09 +0100 Subject: Remove LastModified-based cursor when updating feeds We remove the call to readSince() as some feeds push new articles with pubDate prior to the lastModified time stored for these feeds (e.g. lemonde.fr). As we go through all items of a feed again and again, we prevent the constant update of an item's lastModified timestamp by keeping the previous one if its fingerprint does not change. Fixes #921 Signed-off-by: Kevin Decherf --- lib/Fetcher/FeedFetcher.php | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) (limited to 'lib/Fetcher/FeedFetcher.php') diff --git a/lib/Fetcher/FeedFetcher.php b/lib/Fetcher/FeedFetcher.php index fa6d9a346..b9526165d 100755 --- a/lib/Fetcher/FeedFetcher.php +++ b/lib/Fetcher/FeedFetcher.php @@ -111,21 +111,7 @@ class FeedFetcher implements IFeedFetcher } $url = $url2->getNormalizedURL(); $this->reader->resetFilters(); - if (empty($lastModified) || !is_string($lastModified)) { - $resource = $this->reader->read($url); - } else { - $resource = $this->reader->readSince($url, new DateTime($lastModified)); - } - - $response = $resource->getResponse(); - if (!$response->isModified()) { - $this->logger->debug('Feed {url} was not modified since last fetch. old: {old}, new: {new}', [ - 'url' => $url, - 'old' => print_r($lastModified, true), - 'new' => print_r($response->getLastModified(), true), - ]); - return [null, []]; - } + $resource = $this->reader->read($url); $location = $resource->getUrl(); $parsedFeed = $resource->getFeed(); -- cgit v1.2.3