From e5283611412d4b3305ee7d549da00ca24c0e35df Mon Sep 17 00:00:00 2001 From: Sean Molenaar Date: Tue, 29 Sep 2020 16:14:53 +0200 Subject: Add feed autodiscovery Issue GH-415 Signed-off-by: Sean Molenaar --- CHANGELOG.md | 1 + composer.lock | 10 ++-- lib/AppInfo/Application.php | 6 ++ lib/Command/Config/FeedAdd.php | 2 +- lib/Command/Config/FeedList.php | 2 + lib/Command/Config/FolderList.php | 2 + lib/Db/Folder.php | 28 ++++++--- lib/Db/Item.php | 84 ++++++++++++++++++++------- lib/Fetcher/FeedFetcher.php | 32 ++++++++-- lib/Service/FeedServiceV2.php | 103 +++++++++++++++++---------------- lib/Service/FolderServiceV2.php | 13 +++-- tests/Unit/Fetcher/FeedFetcherTest.php | 4 +- 12 files changed, 190 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31c48b268..1f42f0015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Changed - Update feed-io to v4.7.9 +- Feed autodiscovery - Drop support before nextcloud 20 #794 - Move to modern SQL syntax #750 - Add management commands #804 #750 diff --git a/composer.lock b/composer.lock index 5078effa1..5205b96fc 100644 --- a/composer.lock +++ b/composer.lock @@ -826,16 +826,16 @@ }, { "name": "phpstan/phpstan", - "version": "0.12.46", + "version": "0.12.47", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "9419738e20f0c49757be05d22969c1c44c1dff3b" + "reference": "74325a6ae15378db0df71b969ded245378d2e058" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/9419738e20f0c49757be05d22969c1c44c1dff3b", - "reference": "9419738e20f0c49757be05d22969c1c44c1dff3b", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/74325a6ae15378db0df71b969ded245378d2e058", + "reference": "74325a6ae15378db0df71b969ded245378d2e058", "shasum": "" }, "require": { @@ -878,7 +878,7 @@ "type": "tidelift" } ], - "time": "2020-09-28T09:48:55+00:00" + "time": "2020-09-29T13:35:39+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index f6a5dcdd2..88cba4752 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -13,6 +13,7 @@ namespace OCA\News\AppInfo; +use FeedIo\Explorer; use FeedIo\FeedIo; use HTMLPurifier; use HTMLPurifier_Config; @@ -138,6 +139,11 @@ class Application extends App implements IBootstrap return new FeedIo($config->getClient(), $c->get(LoggerInterface::class)); }); + $context->registerService(Explorer::class, function (ContainerInterface $c): Explorer { + $config = $c->get(FetcherConfig::class); + return new Explorer($config->getClient(), $c->get(LoggerInterface::class)); + }); + $context->registerService(Favicon::class, function (ContainerInterface $c): Favicon { $favicon = new Favicon(); $favicon->cache(['dir' => $c->get(ITempManager::class)->getTempBaseDir()]); diff --git a/lib/Command/Config/FeedAdd.php b/lib/Command/Config/FeedAdd.php index d21f448c0..35bac6480 100644 --- a/lib/Command/Config/FeedAdd.php +++ b/lib/Command/Config/FeedAdd.php @@ -63,7 +63,7 @@ class FeedAdd extends Command $password = $input->getOption('password'); $feed = $this->feedService->create($user, $url, $folder, $full_text, $title, $username, $password); - $this->feedService->fetch($feed, true); + $this->feedService->fetch($feed); $output->writeln(json_encode($feed->toAPI(), JSON_PRETTY_PRINT)); diff --git a/lib/Command/Config/FeedList.php b/lib/Command/Config/FeedList.php index 57e14d339..0c547868c 100644 --- a/lib/Command/Config/FeedList.php +++ b/lib/Command/Config/FeedList.php @@ -57,5 +57,7 @@ class FeedList extends Command } $output->writeln(json_encode($this->serialize($feeds), JSON_PRETTY_PRINT)); + + return 0; } } diff --git a/lib/Command/Config/FolderList.php b/lib/Command/Config/FolderList.php index 7a2d33ab5..26c61f5cf 100644 --- a/lib/Command/Config/FolderList.php +++ b/lib/Command/Config/FolderList.php @@ -58,5 +58,7 @@ class FolderList extends Command } $output->writeln(json_encode($this->serialize($folders), JSON_PRETTY_PRINT)); + + return 0; } } diff --git a/lib/Db/Folder.php b/lib/Db/Folder.php index eb3389546..ec9d27477 100644 --- a/lib/Db/Folder.php +++ b/lib/Db/Folder.php @@ -95,61 +95,75 @@ class Folder extends Entity implements IAPI, \JsonSerializable ); } - public function setDeletedAt(?int $deletedAt = null): void + public function setDeletedAt(?int $deletedAt = null): self { if ($this->deletedAt !== $deletedAt) { $this->deletedAt = $deletedAt; $this->markFieldUpdated('deletedAt'); } + + return $this; } - public function setId(int $id): void + public function setId(int $id): self { if ($this->id !== $id) { $this->id = $id; $this->markFieldUpdated('id'); } + + return $this; } - public function setLastModified(?string $lastModified = null): void + public function setLastModified(?string $lastModified = null): self { if ($this->lastModified !== $lastModified) { $this->lastModified = $lastModified; $this->markFieldUpdated('lastModified'); } + + return $this; } - public function setName(string $name): void + public function setName(string $name): self { if ($this->name !== $name) { $this->name = $name; $this->markFieldUpdated('name'); } + + return $this; } - public function setOpened(bool $opened): void + public function setOpened(bool $opened): self { if ($this->opened !== $opened) { $this->opened = $opened; $this->markFieldUpdated('opened'); } + + return $this; } - public function setParentId(int $parentId = 0): void + public function setParentId(int $parentId = 0): self { if ($this->parentId !== $parentId) { $this->parentId = $parentId; $this->markFieldUpdated('parentId'); } + + return $this; } - public function setUserId(string $userId): void + public function setUserId(string $userId): self { if ($this->userId !== $userId) { $this->userId = $userId; $this->markFieldUpdated('userId'); } + + return $this; } public function toAPI(): array diff --git a/lib/Db/Item.php b/lib/Db/Item.php index 322d12fed..d7efdd14b 100644 --- a/lib/Db/Item.php +++ b/lib/Db/Item.php @@ -299,7 +299,7 @@ class Item extends Entity implements IAPI, \JsonSerializable ]; } - public function setAuthor(string $author = null): void + public function setAuthor(string $author = null): self { $author = strip_tags($author); @@ -307,9 +307,11 @@ class Item extends Entity implements IAPI, \JsonSerializable $this->author = $author; $this->markFieldUpdated('author'); } + + return $this; } - public function setBody(string $body = null): void + public function setBody(string $body = null): self { // FIXME: this should not happen if the target="_blank" is already // on the link @@ -319,129 +321,161 @@ class Item extends Entity implements IAPI, \JsonSerializable $this->body = $body; $this->markFieldUpdated('body'); } + + return $this; } - public function setContentHash(string $contentHash = null): void + public function setContentHash(string $contentHash = null): self { if ($this->contentHash !== $contentHash) { $this->contentHash = $contentHash; $this->markFieldUpdated('contentHash'); } + + return $this; } - public function setEnclosureLink(string $enclosureLink = null): void + public function setEnclosureLink(string $enclosureLink = null): self { if ($this->enclosureLink !== $enclosureLink) { $this->enclosureLink = $enclosureLink; $this->markFieldUpdated('enclosureLink'); } + + return $this; } - public function setEnclosureMime(string $enclosureMime = null): void + public function setEnclosureMime(string $enclosureMime = null): self { if ($this->enclosureMime !== $enclosureMime) { $this->enclosureMime = $enclosureMime; $this->markFieldUpdated('enclosureMime'); } + + return $this; } - public function setMediaThumbnail(string $mediaThumbnail = null): void + public function setMediaThumbnail(string $mediaThumbnail = null): self { if ($this->mediaThumbnail !== $mediaThumbnail) { $this->mediaThumbnail = $mediaThumbnail; $this->markFieldUpdated('mediaThumbnail'); } + + return $this; } - public function setMediaDescription(string $mediaDescription = null): void + public function setMediaDescription(string $mediaDescription = null): self { if ($this->mediaDescription !== $mediaDescription) { $this->mediaDescription = $mediaDescription; $this->markFieldUpdated('mediaDescription'); } + + return $this; } - public function setFeedId(int $feedId): void + public function setFeedId(int $feedId): self { if ($this->feedId !== $feedId) { $this->feedId = $feedId; $this->markFieldUpdated('feedId'); } + + return $this; } - public function setFingerprint(string $fingerprint = null): void + public function setFingerprint(string $fingerprint = null): self { if ($this->fingerprint !== $fingerprint) { $this->fingerprint = $fingerprint; $this->markFieldUpdated('fingerprint'); } + + return $this; } - public function setGuid(string $guid): void + public function setGuid(string $guid): self { if ($this->guid !== $guid) { $this->guid = $guid; $this->markFieldUpdated('guid'); } + + return $this; } - public function setGuidHash(string $guidHash): void + public function setGuidHash(string $guidHash): self { if ($this->guidHash !== $guidHash) { $this->guidHash = $guidHash; $this->markFieldUpdated('guidHash'); } + + return $this; } - public function setId(int $id): void + public function setId(int $id): self { if ($this->id !== $id) { $this->id = $id; $this->markFieldUpdated('id'); } + + return $this; } - public function setLastModified(string $lastModified = null): void + public function setLastModified(string $lastModified = null): self { if ($this->lastModified !== $lastModified) { $this->lastModified = $lastModified; $this->markFieldUpdated('lastModified'); } + + return $this; } - public function setPubDate(int $pubDate = null): void + public function setPubDate(int $pubDate = null): self { if ($this->pubDate !== $pubDate) { $this->pubDate = $pubDate; $this->markFieldUpdated('pubDate'); } + + return $this; } - public function setRtl(bool $rtl): void + public function setRtl(bool $rtl): self { if ($this->rtl !== $rtl) { $this->rtl = $rtl; $this->markFieldUpdated('rtl'); } + + return $this; } - public function setSearchIndex(string $searchIndex = null): void + public function setSearchIndex(string $searchIndex = null): self { if ($this->searchIndex !== $searchIndex) { $this->searchIndex = $searchIndex; $this->markFieldUpdated('searchIndex'); } + + return $this; } - public function setStarred(bool $starred): void + public function setStarred(bool $starred): self { if ($this->starred !== $starred) { $this->starred = $starred; $this->markFieldUpdated('starred'); } + + return $this; } - public function setTitle(string $title = null): void + public function setTitle(string $title = null): self { $title = strip_tags($title); @@ -449,25 +483,31 @@ class Item extends Entity implements IAPI, \JsonSerializable $this->title = $title; $this->markFieldUpdated('title'); } + + return $this; } - public function setUnread(bool $unread): void + public function setUnread(bool $unread): self { if ($this->unread !== $unread) { $this->unread = $unread; $this->markFieldUpdated('unread'); } + + return $this; } - public function setUpdatedDate(int $updatedDate = null): void + public function setUpdatedDate(int $updatedDate = null): self { if ($this->updatedDate !== $updatedDate) { $this->updatedDate = $updatedDate; $this->markFieldUpdated('updatedDate'); } + + return $this; } - public function setUrl(string $url = null): void + public function setUrl(string $url = null): self { $url = trim($url); if ((strpos($url, 'http') === 0 || strpos($url, 'magnet') === 0) @@ -476,6 +516,8 @@ class Item extends Entity implements IAPI, \JsonSerializable $this->url = $url; $this->markFieldUpdated('url'); } + + return $this; } public function toAPI(): array diff --git a/lib/Fetcher/FeedFetcher.php b/lib/Fetcher/FeedFetcher.php index 5a84e2170..ec7902f23 100755 --- a/lib/Fetcher/FeedFetcher.php +++ b/lib/Fetcher/FeedFetcher.php @@ -20,7 +20,6 @@ use FeedIo\FeedInterface; use FeedIo\FeedIo; use Net_URL2; -use OCA\News\Utility\PsrLogger; use OCP\IL10N; use OCA\News\Db\Item; @@ -33,27 +32,50 @@ use SimpleXMLElement; class FeedFetcher implements IFeedFetcher { + /** + * @var Favicon + */ private $faviconFactory; + + /** + * @var FeedIo + */ private $reader; + + /** + * @var Scraper + */ + private $scraper; + + /** + * @var IL10N + */ private $l10n; + + /** + * @var Time + */ private $time; + + /** + * @var LoggerInterface + */ private $logger; - private $scraper; public function __construct( FeedIo $fetcher, Favicon $favicon, + Scraper $scraper, IL10N $l10n, Time $time, - LoggerInterface $logger, - Scraper $scraper + LoggerInterface $logger ) { $this->reader = $fetcher; $this->faviconFactory = $favicon; + $this->scraper = $scraper; $this->l10n = $l10n; $this->time = $time; $this->logger = $logger; - $this->scraper = $scraper; } diff --git a/lib/Service/FeedServiceV2.php b/lib/Service/FeedServiceV2.php index 55137c357..41fb41e89 100644 --- a/lib/Service/FeedServiceV2.php +++ b/lib/Service/FeedServiceV2.php @@ -13,6 +13,7 @@ namespace OCA\News\Service; +use FeedIo\Explorer; use FeedIo\Reader\ReadErrorException; use HTMLPurifier; @@ -58,6 +59,11 @@ class FeedServiceV2 extends Service * @var HTMLPurifier */ protected $purifier; + /** + * Feed Explorer + * @var Explorer + */ + protected $explorer; /** * FeedService constructor. @@ -65,6 +71,7 @@ class FeedServiceV2 extends Service * @param FeedMapperV2 $mapper DB layer for feeds * @param FeedFetcher $feedFetcher FeedIO interface * @param ItemServiceV2 $itemService Service to manage items + * @param Explorer $explorer Feed Explorer * @param HTMLPurifier $purifier HTML Purifier * @param LoggerInterface $logger Logger */ @@ -72,6 +79,7 @@ class FeedServiceV2 extends Service FeedMapperV2 $mapper, FeedFetcher $feedFetcher, ItemServiceV2 $itemService, + Explorer $explorer, HTMLPurifier $purifier, LoggerInterface $logger ) { @@ -79,19 +87,21 @@ class FeedServiceV2 extends Service $this->feedFetcher = $feedFetcher; $this->itemService = $itemService; - $this->purifier = $purifier; + $this->explorer = $explorer; + $this->purifier = $purifier; } /** * Finds all feeds of a user * - * @param string $userId the name of the user + * @param string $userId the name/ID of the user + * @param array $params Filter parameters * * @return Feed[] */ public function findAllForUser(string $userId, array $params = []): array { - return $this->mapper->findAllFromUser($userId); + return $this->mapper->findAllFromUser($userId, $params); } /** @@ -170,14 +180,15 @@ class FeedServiceV2 extends Service /** * Creates a new feed * - * @param string $userId Feed owner - * @param string $feedUrl Feed URL - * @param int $folderId Target folder, defaults to root - * @param string|null $title The OPML feed title - * @param string|null $user Basic auth username, if set - * @param string|null $password Basic auth password if username is set + * @param string $userId Feed owner + * @param string $feedUrl Feed URL + * @param int $folderId Target folder, defaults to root + * @param bool $full_text Scrape the feed for full text + * @param string|null $title The feed title + * @param string|null $user Basic auth username, if set + * @param string|null $password Basic auth password if username is set * - * @return Feed the newly created feed + * @return Feed|Entity * * @throws ServiceConflictException The feed already exists * @throws ServiceNotFoundException The url points to an invalid feed @@ -190,53 +201,56 @@ class FeedServiceV2 extends Service ?string $title = null, ?string $user = null, ?string $password = null - ): Feed { + ): Entity { if ($this->existsForUser($userId, $feedUrl)) { throw new ServiceConflictException('Feed with this URL exists'); } + $feeds = $this->explorer->discover($feedUrl); + if ($feeds !== []) { + $feedUrl = array_shift($feeds); + } + try { /** * @var Feed $feed * @var Item[] $items */ - list($feed, $items) = $this->feedFetcher->fetch($feedUrl, true, $full_text, false, $user, $password); - if ($feed === null) { - throw new ServiceNotFoundException('Failed to fetch feed'); - } - - $feed->setFolderId($folderId) - ->setUserId($userId) - ->setArticlesPerUpdate(count($items)); + list($feed, $items) = $this->feedFetcher->fetch($feedUrl, true, '0', $full_text, $user, $password); + } catch (ReadErrorException $ex) { + $this->logger->debug($ex->getMessage()); + throw new ServiceNotFoundException($ex->getMessage()); + } - if (!is_null($title)) { - $feed->setTitle($title); - } + if ($feed === null) { + throw new ServiceNotFoundException('Failed to fetch feed'); + } - if (!is_null($user)) { - $feed->setBasicAuthUser($user) - ->setBasicAuthUser($password); - } + $feed->setFolderId($folderId) + ->setUserId($userId) + ->setArticlesPerUpdate(count($items)); - $feed = $this->mapper->insert($feed); + if (!is_null($title)) { + $feed->setTitle($title); + } - return $feed; - } catch (ReadErrorException $ex) { - $this->logger->debug($ex->getMessage()); - throw new ServiceNotFoundException($ex->getMessage()); + if (!is_null($user)) { + $feed->setBasicAuthUser($user) + ->setBasicAuthUser($password); } + + return $this->mapper->insert($feed); } /** * Update a feed * - * @param Feed $feed Feed item - * @param bool $force update even if the article exists already + * @param Feed $feed Feed item * * @return Feed|Entity Database feed entity */ - public function fetch(Feed $feed, bool $force = false) + public function fetch(Feed $feed) { if ($feed->getPreventUpdate() === true) { return $feed; @@ -277,27 +291,16 @@ class FeedServiceV2 extends Service $feed->setArticlesPerUpdate($itemCount); } - $feed->setHttpLastModified($fetchedFeed->getHttpLastModified()); - $feed->setHttpEtag($fetchedFeed->getHttpEtag()); - $feed->setLocation($fetchedFeed->getLocation()); + $feed->setHttpLastModified($fetchedFeed->getHttpLastModified()) + ->setHttpEtag($fetchedFeed->getHttpEtag()) + ->setLocation($fetchedFeed->getLocation()); // insert items in reverse order because the first one is // usually the newest item for ($i = $itemCount - 1; $i >= 0; $i--) { $item = $items[$i]; - $item->setFeedId($feed->getId()); - - $item->setTitle($item->getTitle()); - $item->setUrl($item->getUrl()); - $item->setAuthor($item->getAuthor()); - $item->setSearchIndex($item->getSearchIndex()); - $item->setRtl($item->getRtl()); - $item->setLastModified($item->getLastModified()); - $item->setPubDate($item->getPubDate()); - $item->setUpdatedDate($item->getUpdatedDate()); - $item->setEnclosureMime($item->getEnclosureMime()); - $item->setEnclosureLink($item->getEnclosureLink()); - $item->setBody($this->purifier->purify($item->getBody())); + $item->setFeedId($feed->getId()) + ->setBody($this->purifier->purify($item->getBody())); // update modes: 0 nothing, 1 set unread if ($feed->getUpdateMode() === 1) { diff --git a/lib/Service/FolderServiceV2.php b/lib/Service/FolderServiceV2.php index b1290eca2..5d3f149ce 100644 --- a/lib/Service/FolderServiceV2.php +++ b/lib/Service/FolderServiceV2.php @@ -43,13 +43,14 @@ class FolderServiceV2 extends Service /** * Finds all folders of a user * - * @param string $userId the name of the user + * @param string $userId The name/ID of the user + * @param array $params Filter parameters * * @return Folder[] */ public function findAllForUser(string $userId, array $params = []): array { - return $this->mapper->findAllFromUser($userId); + return $this->mapper->findAllFromUser($userId, $params); } /** @@ -60,7 +61,7 @@ class FolderServiceV2 extends Service public function findAllForUserRecursive(string $userId): array { $folders = $this->findAllForUser($userId); - foreach ($folders as &$folder) { + foreach ($folders as $folder) { $feeds = $this->feedService->findAllFromFolder($folder->getId()); $folder->feeds = $feeds; } @@ -81,9 +82,9 @@ class FolderServiceV2 extends Service public function create(string $userId, string $name, int $parent = 0): void { $folder = new Folder(); - $folder->setUserId($userId); - $folder->setName($name); - $folder->setParentId($parent); + $folder->setUserId($userId) + ->setName($name) + ->setParentId($parent); $this->mapper->insert($folder); } diff --git a/tests/Unit/Fetcher/FeedFetcherTest.php b/tests/Unit/Fetcher/FeedFetcherTest.php index 38a7c2f61..c52ccd62d 100644 --- a/tests/Unit/Fetcher/FeedFetcherTest.php +++ b/tests/Unit/Fetcher/FeedFetcherTest.php @@ -175,10 +175,10 @@ class FeedFetcherTest extends TestCase $this->fetcher = new FeedFetcher( $this->reader, $this->favicon, + $this->scraper, $this->l10n, $timeFactory, - $this->logger, - $this->scraper + $this->logger ); $this->url = 'http://tests/'; -- cgit v1.2.3