Remove LastModified-based cursor when updating feeds

We remove the call to readSince() as some feeds push new articles with
pubDate prior to the lastModified time stored for these feeds (e.g.
lemonde.fr). As we go through all items of a feed again and again, we
prevent the constant update of an item's lastModified timestamp by
keeping the previous one if its fingerprint does not change.

Fixes #921

Signed-off-by: Kevin Decherf <kevin@kdecherf.com>

3 files changed, 11 insertions, 16 deletions

diff --git a/lib/Db/NewsMapperV2.php b/lib/Db/NewsMapperV2.php
index 708ef4f73..93500c7c9 100644
--- a/lib/Db/NewsMapperV2.php
+++ b/lib/Db/NewsMapperV2.php
@@ -58,7 +58,9 @@ abstract class NewsMapperV2 extends QBMapper
 
     public function update(Entity $entity): Entity
     {
-        $entity->setLastModified($this->time->getMicroTime());
+        if ([] !== $entity->getUpdatedFields()) {
+            $entity->setLastModified($this->time->getMicroTime());
+        }
         return parent::update($entity);
     }
 
diff --git a/lib/Fetcher/FeedFetcher.php b/lib/Fetcher/FeedFetcher.php
index fa6d9a346..b9526165d 100755
--- a/lib/Fetcher/FeedFetcher.php
+++ b/lib/Fetcher/FeedFetcher.php
@@ -111,21 +111,7 @@ class FeedFetcher implements IFeedFetcher
         }
         $url = $url2->getNormalizedURL();
         $this->reader->resetFilters();
-        if (empty($lastModified) || !is_string($lastModified)) {
-            $resource = $this->reader->read($url);
-        } else {
-            $resource = $this->reader->readSince($url, new DateTime($lastModified));
-        }
-
-        $response = $resource->getResponse();
-        if (!$response->isModified()) {
-            $this->logger->debug('Feed {url} was not modified since last fetch. old: {old}, new: {new}', [
-                 'url' => $url,
-                 'old' => print_r($lastModified, true),
-                 'new' => print_r($response->getLastModified(), true),
-            ]);
-            return [null, []];
-        }
+        $resource = $this->reader->read($url);
 
         $location     = $resource->getUrl();
         $parsedFeed   = $resource->getFeed();
diff --git a/lib/Service/ItemServiceV2.php b/lib/Service/ItemServiceV2.php
index d0093a07e..0a04fd8ad 100644
--- a/lib/Service/ItemServiceV2.php
+++ b/lib/Service/ItemServiceV2.php
@@ -87,6 +87,13 @@ class ItemServiceV2 extends Service
                  ->setStarred($db_item->isStarred())
                  ->setId($db_item->getId());
 
+            $item->generateSearchIndex();
+            // We don't want to update the database record if there is no
+            // change in the fetched item
+            if ($db_item->getFingerprint() === $item->getFingerprint()) {
+                $item->resetUpdatedFields();
+            }
+
             $this->mapper->update($item);
         } catch (DoesNotExistException $exception) {
             $this->mapper->insert($item);