Add feed autodiscovery

Issue GH-415

Signed-off-by: Sean Molenaar <sean@seanmolenaar.eu>

12 files changed, 190 insertions, 97 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 31c48b268..1f42f0015 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 ### Changed
 
 - Update feed-io to v4.7.9
+- Feed autodiscovery
 - Drop support before nextcloud 20 #794
 - Move to modern SQL syntax #750
 - Add management commands #804 #750
diff --git a/composer.lock b/composer.lock
index 5078effa1..5205b96fc 100644
--- a/composer.lock
+++ b/composer.lock
@@ -826,16 +826,16 @@
         },
         {
             "name": "phpstan/phpstan",
-            "version": "0.12.46",
+            "version": "0.12.47",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpstan/phpstan.git",
-                "reference": "9419738e20f0c49757be05d22969c1c44c1dff3b"
+                "reference": "74325a6ae15378db0df71b969ded245378d2e058"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpstan/phpstan/zipball/9419738e20f0c49757be05d22969c1c44c1dff3b",
-                "reference": "9419738e20f0c49757be05d22969c1c44c1dff3b",
+                "url": "https://api.github.com/repos/phpstan/phpstan/zipball/74325a6ae15378db0df71b969ded245378d2e058",
+                "reference": "74325a6ae15378db0df71b969ded245378d2e058",
                 "shasum": ""
             },
             "require": {
@@ -878,7 +878,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2020-09-28T09:48:55+00:00"
+            "time": "2020-09-29T13:35:39+00:00"
         },
         {
             "name": "phpunit/php-code-coverage",
diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php
index f6a5dcdd2..88cba4752 100644
--- a/lib/AppInfo/Application.php
+++ b/lib/AppInfo/Application.php
@@ -13,6 +13,7 @@
 
 namespace OCA\News\AppInfo;
 
+use FeedIo\Explorer;
 use FeedIo\FeedIo;
 use HTMLPurifier;
 use HTMLPurifier_Config;
@@ -138,6 +139,11 @@ class Application extends App implements IBootstrap
             return new FeedIo($config->getClient(), $c->get(LoggerInterface::class));
         });
 
+        $context->registerService(Explorer::class, function (ContainerInterface $c): Explorer {
+            $config = $c->get(FetcherConfig::class);
+            return new Explorer($config->getClient(), $c->get(LoggerInterface::class));
+        });
+
         $context->registerService(Favicon::class, function (ContainerInterface $c): Favicon {
             $favicon = new Favicon();
             $favicon->cache(['dir' => $c->get(ITempManager::class)->getTempBaseDir()]);
diff --git a/lib/Command/Config/FeedAdd.php b/lib/Command/Config/FeedAdd.php
index d21f448c0..35bac6480 100644
--- a/lib/Command/Config/FeedAdd.php
+++ b/lib/Command/Config/FeedAdd.php
@@ -63,7 +63,7 @@ class FeedAdd extends Command
         $password = $input->getOption('password');
 
         $feed = $this->feedService->create($user, $url, $folder, $full_text, $title, $username, $password);
-        $this->feedService->fetch($feed, true);
+        $this->feedService->fetch($feed);
 
         $output->writeln(json_encode($feed->toAPI(), JSON_PRETTY_PRINT));
 
diff --git a/lib/Command/Config/FeedList.php b/lib/Command/Config/FeedList.php
index 57e14d339..0c547868c 100644
--- a/lib/Command/Config/FeedList.php
+++ b/lib/Command/Config/FeedList.php
@@ -57,5 +57,7 @@ class FeedList extends Command
         }
 
         $output->writeln(json_encode($this->serialize($feeds), JSON_PRETTY_PRINT));
+
+        return 0;
     }
 }
diff --git a/lib/Command/Config/FolderList.php b/lib/Command/Config/FolderList.php
index 7a2d33ab5..26c61f5cf 100644
--- a/lib/Command/Config/FolderList.php
+++ b/lib/Command/Config/FolderList.php
@@ -58,5 +58,7 @@ class FolderList extends Command
         }
 
         $output->writeln(json_encode($this->serialize($folders), JSON_PRETTY_PRINT));
+
+        return 0;
     }
 }
diff --git a/lib/Db/Folder.php b/lib/Db/Folder.php
index eb3389546..ec9d27477 100644
--- a/lib/Db/Folder.php
+++ b/lib/Db/Folder.php
@@ -95,61 +95,75 @@ class Folder extends Entity implements IAPI, \JsonSerializable
         );
     }
 
-    public function setDeletedAt(?int $deletedAt = null): void
+    public function setDeletedAt(?int $deletedAt = null): self
     {
         if ($this->deletedAt !== $deletedAt) {
             $this->deletedAt = $deletedAt;
             $this->markFieldUpdated('deletedAt');
         }
+
+        return $this;
     }
 
-    public function setId(int $id): void
+    public function setId(int $id): self
     {
         if ($this->id !== $id) {
             $this->id = $id;
             $this->markFieldUpdated('id');
         }
+
+        return $this;
     }
 
-    public function setLastModified(?string $lastModified = null): void
+    public function setLastModified(?string $lastModified = null): self
     {
 
         if ($this->lastModified !== $lastModified) {
             $this->lastModified = $lastModified;
             $this->markFieldUpdated('lastModified');
         }
+
+        return $this;
     }
 
-    public function setName(string $name): void
+    public function setName(string $name): self
     {
         if ($this->name !== $name) {
             $this->name = $name;
             $this->markFieldUpdated('name');
         }
+
+        return $this;
     }
 
-    public function setOpened(bool $opened): void
+    public function setOpened(bool $opened): self
     {
         if ($this->opened !== $opened) {
             $this->opened = $opened;
             $this->markFieldUpdated('opened');
         }
+
+        return $this;
     }
 
-    public function setParentId(int $parentId = 0): void
+    public function setParentId(int $parentId = 0): self
     {
         if ($this->parentId !== $parentId) {
             $this->parentId = $parentId;
             $this->markFieldUpdated('parentId');
         }
+
+        return $this;
     }
 
-    public function setUserId(string $userId): void
+    public function setUserId(string $userId): self
     {
         if ($this->userId !== $userId) {
             $this->userId = $userId;
             $this->markFieldUpdated('userId');
         }
+
+        return $this;
     }
 
     public function toAPI(): array
diff --git a/lib/Db/Item.php b/lib/Db/Item.php
index 322d12fed..d7efdd14b 100644
--- a/lib/Db/Item.php
+++ b/lib/Db/Item.php
@@ -299,7 +299,7 @@ class Item extends Entity implements IAPI, \JsonSerializable
         ];
     }
 
-    public function setAuthor(string $author = null): void
+    public function setAuthor(string $author = null): self
     {
         $author = strip_tags($author);
 
@@ -307,9 +307,11 @@ class Item extends Entity implements IAPI, \JsonSerializable
             $this->author = $author;
             $this->markFieldUpdated('author');
         }
+
+        return $this;
     }
 
-    public function setBody(string $body = null): void
+    public function setBody(string $body = null): self
     {
         // FIXME: this should not happen if the target="_blank" is already
         // on the link
@@ -319,129 +321,161 @@ class Item extends Entity implements IAPI, \JsonSerializable
             $this->body = $body;
             $this->markFieldUpdated('body');
         }
+
+        return $this;
     }
 
-    public function setContentHash(string $contentHash = null): void
+    public function setContentHash(string $contentHash = null): self
     {
         if ($this->contentHash !== $contentHash) {
             $this->contentHash = $contentHash;
             $this->markFieldUpdated('contentHash');
         }
+
+        return $this;
     }
 
-    public function setEnclosureLink(string $enclosureLink = null): void
+    public function setEnclosureLink(string $enclosureLink = null): self
     {
         if ($this->enclosureLink !== $enclosureLink) {
             $this->enclosureLink = $enclosureLink;
             $this->markFieldUpdated('enclosureLink');
         }
+
+        return $this;
     }
 
-    public function setEnclosureMime(string $enclosureMime = null): void
+    public function setEnclosureMime(string $enclosureMime = null): self
     {
         if ($this->enclosureMime !== $enclosureMime) {
             $this->enclosureMime = $enclosureMime;
             $this->markFieldUpdated('enclosureMime');
         }
+
+        return $this;
     }
 
-    public function setMediaThumbnail(string $mediaThumbnail = null): void
+    public function setMediaThumbnail(string $mediaThumbnail = null): self
     {
         if ($this->mediaThumbnail !== $mediaThumbnail) {
             $this->mediaThumbnail = $mediaThumbnail;
             $this->markFieldUpdated('mediaThumbnail');
         }
+
+        return $this;
     }
 
-    public function setMediaDescription(string $mediaDescription = null): void
+    public function setMediaDescription(string $mediaDescription = null): self
     {
         if ($this->mediaDescription !== $mediaDescription) {
             $this->mediaDescription = $mediaDescription;
             $this->markFieldUpdated('mediaDescription');
         }
+
+        return $this;
     }
 
-    public function setFeedId(int $feedId): void
+    public function setFeedId(int $feedId): self
     {
         if ($this->feedId !== $feedId) {
             $this->feedId = $feedId;
             $this->markFieldUpdated('feedId');
         }
+
+        return $this;
     }
 
-    public function setFingerprint(string $fingerprint = null): void
+    public function setFingerprint(string $fingerprint = null): self
     {
         if ($this->fingerprint !== $fingerprint) {
             $this->fingerprint = $fingerprint;
             $this->markFieldUpdated('fingerprint');
         }
+
+        return $this;
     }
 
-    public function setGuid(string $guid): void
+    public function setGuid(string $guid): self
     {
         if ($this->guid !== $guid) {
             $this->guid = $guid;
             $this->markFieldUpdated('guid');
         }
+
+        return $this;
     }
 
-    public function setGuidHash(string $guidHash): void
+    public function setGuidHash(string $guidHash): self
     {
         if ($this->guidHash !== $guidHash) {
             $this->guidHash = $guidHash;
             $this->markFieldUpdated('guidHash');
         }
+
+        return $this;
     }
 
-    public function setId(int $id): void
+    public function setId(int $id): self
     {
         if ($this->id !== $id) {
             $this->id = $id;
             $this->markFieldUpdated('id');
         }
+
+        return $this;
     }
 
-    public function setLastModified(string $lastModified = null): void
+    public function setLastModified(string $lastModified = null): self
     {
         if ($this->lastModified !== $lastModified) {
             $this->lastModified = $lastModified;
             $this->markFieldUpdated('lastModified');
         }
+
+        return $this;
     }
 
-    public function setPubDate(int $pubDate = null): void
+    public function setPubDate(int $pubDate = null): self
     {
         if ($this->pubDate !== $pubDate) {
             $this->pubDate = $pubDate;
             $this->markFieldUpdated('pubDate');
         }
+
+        return $this;
     }
 
-    public function setRtl(bool $rtl): void
+    public function setRtl(bool $rtl): self
     {
         if ($this->rtl !== $rtl) {
             $this->rtl = $rtl;
             $this->markFieldUpdated('rtl');
         }
+
+        return $this;
     }
 
-    public function setSearchIndex(string $searchIndex = null): void
+    public function setSearchIndex(string $searchIndex = null): self
     {
         if ($this->searchIndex !== $searchIndex) {
             $this->searchIndex = $searchIndex;
             $this->markFieldUpdated('searchIndex');
         }
+
+        return $this;
     }
 
-    public function setStarred(bool $starred): void
+    public function setStarred(bool $starred): self
     {
         if ($this->starred !== $starred) {
             $this->starred = $starred;
             $this->markFieldUpdated('starred');
         }
+
+        return $this;
     }
 
-    public function setTitle(string $title = null): void
+    public function setTitle(string $title = null): self
     {
         $title = strip_tags($title);
 
@@ -449,25 +483,31 @@ class Item extends Entity implements IAPI, \JsonSerializable
             $this->title = $title;
             $this->markFieldUpdated('title');
         }
+
+        return $this;
     }
 
-    public function setUnread(bool $unread): void
+    public function setUnread(bool $unread): self
     {
         if ($this->unread !== $unread) {
             $this->unread = $unread;
             $this->markFieldUpdated('unread');
         }
+
+        return $this;
     }
 
-    public function setUpdatedDate(int $updatedDate = null): void
+    public function setUpdatedDate(int $updatedDate = null): self
     {
         if ($this->updatedDate !== $updatedDate) {
             $this->updatedDate = $updatedDate;
             $this->markFieldUpdated('updatedDate');
         }
+
+        return $this;
     }
 
-    public function setUrl(string $url = null): void
+    public function setUrl(string $url = null): self
     {
         $url = trim($url);
         if ((strpos($url, 'http') === 0 || strpos($url, 'magnet') === 0)
@@ -476,6 +516,8 @@ class Item extends Entity implements IAPI, \JsonSerializable
             $this->url = $url;
             $this->markFieldUpdated('url');
         }
+
+        return $this;
     }
 
     public function toAPI(): array
diff --git a/lib/Fetcher/FeedFetcher.php b/lib/Fetcher/FeedFetcher.php
index 5a84e2170..ec7902f23 100755
--- a/lib/Fetcher/FeedFetcher.php
+++ b/lib/Fetcher/FeedFetcher.php
@@ -20,7 +20,6 @@ use FeedIo\FeedInterface;
 use FeedIo\FeedIo;
 
 use Net_URL2;
-use OCA\News\Utility\PsrLogger;
 use OCP\IL10N;
 
 use OCA\News\Db\Item;
@@ -33,27 +32,50 @@ use SimpleXMLElement;
 class FeedFetcher implements IFeedFetcher
 {
 
+    /**
+     * @var Favicon
+     */
     private $faviconFactory;
+
+    /**
+     * @var FeedIo
+     */
     private $reader;
+
+    /**
+     * @var Scraper
+     */
+    private $scraper;
+
+    /**
+     * @var IL10N
+     */
     private $l10n;
+
+    /**
+     * @var Time
+     */
     private $time;
+
+    /**
+     * @var LoggerInterface
+     */
     private $logger;
-    private $scraper;
 
     public function __construct(
         FeedIo $fetcher,
         Favicon $favicon,
+        Scraper $scraper,
         IL10N $l10n,
         Time $time,
-        LoggerInterface $logger,
-        Scraper $scraper
+        LoggerInterface $logger
     ) {
         $this->reader         = $fetcher;
         $this->faviconFactory = $favicon;
+        $this->scraper        = $scraper;
         $this->l10n           = $l10n;
         $this->time           = $time;
         $this->logger         = $logger;
-        $this->scraper        = $scraper;
     }
 
 
diff --git a/lib/Service/FeedServiceV2.php b/lib/Service/FeedServiceV2.php
index 55137c357..41fb41e89 100644
--- a/lib/Service/FeedServiceV2.php
+++ b/lib/Service/FeedServiceV2.php
@@ -13,6 +13,7 @@
 
 namespace OCA\News\Service;
 
+use FeedIo\Explorer;
 use FeedIo\Reader\ReadErrorException;
 use HTMLPurifier;
 
@@ -58,6 +59,11 @@ class FeedServiceV2 extends Service
      * @var HTMLPurifier
      */
     protected $purifier;
+    /**
+     * Feed Explorer
+     * @var Explorer
+     */
+    protected $explorer;
 
     /**
      * FeedService constructor.
@@ -65,6 +71,7 @@ class FeedServiceV2 extends Service
      * @param FeedMapperV2    $mapper      DB layer for feeds
      * @param FeedFetcher     $feedFetcher FeedIO interface
      * @param ItemServiceV2   $itemService Service to manage items
+     * @param Explorer        $explorer    Feed Explorer
      * @param HTMLPurifier    $purifier    HTML Purifier
      * @param LoggerInterface $logger      Logger
      */
@@ -72,6 +79,7 @@ class FeedServiceV2 extends Service
         FeedMapperV2 $mapper,
         FeedFetcher $feedFetcher,
         ItemServiceV2 $itemService,
+        Explorer $explorer,
         HTMLPurifier $purifier,
         LoggerInterface $logger
     ) {
@@ -79,19 +87,21 @@ class FeedServiceV2 extends Service
 
         $this->feedFetcher = $feedFetcher;
         $this->itemService = $itemService;
-        $this->purifier = $purifier;
+        $this->explorer    = $explorer;
+        $this->purifier    = $purifier;
     }
 
     /**
      * Finds all feeds of a user
      *
-     * @param  string $userId the name of the user
+     * @param string $userId the name/ID of the user
+     * @param array  $params Filter parameters
      *
      * @return Feed[]
      */
     public function findAllForUser(string $userId, array $params = []): array
     {
-        return $this->mapper->findAllFromUser($userId);
+        return $this->mapper->findAllFromUser($userId, $params);
     }
 
     /**
@@ -170,14 +180,15 @@ class FeedServiceV2 extends Service
     /**
      * Creates a new feed
      *
-     * @param string      $userId   Feed owner
-     * @param string      $feedUrl  Feed URL
-     * @param int         $folderId Target folder, defaults to root
-     * @param string|null $title    The OPML feed title
-     * @param string|null $user     Basic auth username, if set
-     * @param string|null $password Basic auth password if username is set
+     * @param string      $userId    Feed owner
+     * @param string      $feedUrl   Feed URL
+     * @param int         $folderId  Target folder, defaults to root
+     * @param bool        $full_text Scrape the feed for full text
+     * @param string|null $title     The feed title
+     * @param string|null $user      Basic auth username, if set
+     * @param string|null $password  Basic auth password if username is set
      *
-     * @return Feed the newly created feed
+     * @return Feed|Entity
      *
      * @throws ServiceConflictException The feed already exists
      * @throws ServiceNotFoundException The url points to an invalid feed
@@ -190,53 +201,56 @@ class FeedServiceV2 extends Service
         ?string $title = null,
         ?string $user = null,
         ?string $password = null
-    ): Feed {
+    ): Entity {
         if ($this->existsForUser($userId, $feedUrl)) {
             throw new ServiceConflictException('Feed with this URL exists');
         }
 
+        $feeds = $this->explorer->discover($feedUrl);
+        if ($feeds !== []) {
+            $feedUrl = array_shift($feeds);
+        }
+
         try {
             /**
              * @var Feed   $feed
              * @var Item[] $items
              */
-            list($feed, $items) = $this->feedFetcher->fetch($feedUrl, true, $full_text, false, $user, $password);
-            if ($feed === null) {
-                throw new ServiceNotFoundException('Failed to fetch feed');
-            }
-
-            $feed->setFolderId($folderId)
-                ->setUserId($userId)
-                ->setArticlesPerUpdate(count($items));
+            list($feed, $items) = $this->feedFetcher->fetch($feedUrl, true, '0', $full_text, $user, $password);
+        } catch (ReadErrorException $ex) {
+            $this->logger->debug($ex->getMessage());
+            throw new ServiceNotFoundException($ex->getMessage());
+        }
 
-            if (!is_null($title)) {
-                $feed->setTitle($title);
-            }
+        if ($feed === null) {
+            throw new ServiceNotFoundException('Failed to fetch feed');
+        }
 
-            if (!is_null($user)) {
-                $feed->setBasicAuthUser($user)
-                    ->setBasicAuthUser($password);
-            }
+        $feed->setFolderId($folderId)
+            ->setUserId($userId)
+            ->setArticlesPerUpdate(count($items));
 
-            $feed = $this->mapper->insert($feed);
+        if (!is_null($title)) {
+            $feed->setTitle($title);
+        }
 
-            return $feed;
-        } catch (ReadErrorException $ex) {
-            $this->logger->debug($ex->getMessage());
-            throw new ServiceNotFoundException($ex->getMessage());
+        if (!is_null($user)) {
+            $feed->setBasicAuthUser($user)
+                ->setBasicAuthUser($password);
         }
+
+        return $this->mapper->insert($feed);
     }
 
 
     /**
      * Update a feed
      *
-     * @param  Feed   $feed   Feed item
-     * @param  bool   $force  update even if the article exists already
+     * @param Feed $feed Feed item
      *
      * @return Feed|Entity Database feed entity
      */
-    public function fetch(Feed $feed, bool $force = false)
+    public function fetch(Feed $feed)
     {
         if ($feed->getPreventUpdate() === true) {
             return $feed;
@@ -277,27 +291,16 @@ class FeedServiceV2 extends Service
                 $feed->setArticlesPerUpdate($itemCount);
             }
 
-            $feed->setHttpLastModified($fetchedFeed->getHttpLastModified());
-            $feed->setHttpEtag($fetchedFeed->getHttpEtag());
-            $feed->setLocation($fetchedFeed->getLocation());
+            $feed->setHttpLastModified($fetchedFeed->getHttpLastModified())
+                 ->setHttpEtag($fetchedFeed->getHttpEtag())
+                 ->setLocation($fetchedFeed->getLocation());
 
             // insert items in reverse order because the first one is
             // usually the newest item
             for ($i = $itemCount - 1; $i >= 0; $i--) {
                 $item = $items[$i];
-                $item->setFeedId($feed->getId());
-
-