Reimplement full-text scraping (#563)

Add readability.php scraper
Fixes #482
Signed-off-by: Gioele Falcetti <thegio.f@gmail.com>

3 files changed, 28 insertions, 20 deletions

diff --git a/tests/Unit/Fetcher/FeedFetcherTest.php b/tests/Unit/Fetcher/FeedFetcherTest.php
index 397007876..0bae8023f 100644
--- a/tests/Unit/Fetcher/FeedFetcherTest.php
+++ b/tests/Unit/Fetcher/FeedFetcherTest.php
@@ -21,6 +21,7 @@ use FeedIo\FeedInterface;
 use OC\L10N\L10N;
 use \OCA\News\Db\Feed;
 use \OCA\News\Db\Item;
+use OCA\News\Scraper\Scraper;
 use OCA\News\Fetcher\FeedFetcher;
 use OCA\News\Utility\PsrLogger;
 
@@ -163,12 +164,16 @@ class FeedFetcherTest extends TestCase
         $this->logger = $this->getMockBuilder(PsrLogger::class)
             ->disableOriginalConstructor()
             ->getMock();
+        $this->scraper = $this->getMockBuilder(Scraper::class)
+            ->disableOriginalConstructor()
+            ->getMock();
         $this->fetcher = new FeedFetcher(
             $this->reader,
             $this->favicon,
             $this->l10n,
             $timeFactory,
-            $this->logger
+            $this->logger,
+            $this->scraper
         );
         $this->url = 'http://tests/';
 
@@ -220,7 +225,7 @@ class FeedFetcherTest extends TestCase
             ->with(
                 'Feed {url} was not modified since last fetch. old: {old}, new: {new}'
             );
-        $result = $this->fetcher->fetch($this->url, false, '@0', null, null);
+        $result = $this->fetcher->fetch($this->url, false, '@0', false, null, null);
 
         $this->assertSame([null, []], $result);
     }
@@ -234,7 +239,7 @@ class FeedFetcherTest extends TestCase
         $item = $this->createItem();
         $feed = $this->createFeed();
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        $result = $this->fetcher->fetch($this->url, false, '0', null, null);
+        $result = $this->fetcher->fetch($this->url, false, '0', false, null, null);
 
         $this->assertEquals([$feed, [$item]], $result);
     }
@@ -285,7 +290,7 @@ class FeedFetcherTest extends TestCase
         $item = $this->createItem();
         $feed = $this->createFeed();
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        $result = $this->fetcher->fetch($this->url, false, '0', null, null);
+        $result = $this->fetcher->fetch($this->url, false, '0', false, null, null);
 
         $this->assertEquals([$feed, [$item]], $result);
 
@@ -302,7 +307,7 @@ class FeedFetcherTest extends TestCase
         $item = $this->createItem();
         $feed = $this->createFeed();
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        $result = $this->fetcher->fetch($this->url, false, '@1553118393', null, null);
+        $result = $this->fetcher->fetch($this->url, false, '@1553118393', false, null, null);
 
         $this->assertEquals([$feed, [$item]], $result);
     }
@@ -316,7 +321,7 @@ class FeedFetcherTest extends TestCase
         $item = $this->createItem();
         $feed = $this->createFeed('de-DE', false, 'http://account%40email.com:F9sEU%2ARt%25%3AKFK8HMHT%26@tests/');
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        $result = $this->fetcher->fetch($this->url, false, '@1553118393', 'account@email.com', 'F9sEU*Rt%:KFK8HMHT&');
+        $result = $this->fetcher->fetch($this->url, false, '@1553118393', false, 'account@email.com', 'F9sEU*Rt%:KFK8HMHT&');
 
         $this->assertEquals([$feed, [$item]], $result);
     }
@@ -330,7 +335,7 @@ class FeedFetcherTest extends TestCase
         $item = $this->createItem('audio/ogg');
         $feed = $this->createFeed();
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        $result = $this->fetcher->fetch($this->url, false, '@1553118393', null, null);
+        $result = $this->fetcher->fetch($this->url, false, '@1553118393', false, null, null);
 
         $this->assertEquals([$feed, [$item]], $result);
     }
@@ -344,7 +349,7 @@ class FeedFetcherTest extends TestCase
         $item = $this->createItem('video/ogg');
         $feed = $this->createFeed();
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        $result = $this->fetcher->fetch($this->url, false, '@1553118393', null, null);
+        $result = $this->fetcher->fetch($this->url, false, '@1553118393', false, null, null);
 
         $this->assertEquals([$feed, [$item]], $result);
     }
@@ -359,7 +364,7 @@ class FeedFetcherTest extends TestCase
         $feed = $this->createFeed('de-DE', true);
         $item = $this->createItem();
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        $result = $this->fetcher->fetch($this->url, true, '@1553118393', null, null);
+        $result = $this->fetcher->fetch($this->url, true, '@1553118393', false, null, null);
 
         $this->assertEquals([$feed, [$item]], $result);
     }
@@ -378,7 +383,7 @@ class FeedFetcherTest extends TestCase
 
         $item = $this->createItem();
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        $result = $this->fetcher->fetch($this->url, false, '@1553118393', null, null);
+        $result = $this->fetcher->fetch($this->url, false, '@1553118393', false, null, null);
 
         $this->assertEquals([$feed, [$item]], $result);
     }
@@ -392,7 +397,7 @@ class FeedFetcherTest extends TestCase
         $this->createFeed('he-IL');
         $this->createItem();
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        list($feed, $items) = $this->fetcher->fetch($this->url, false, '@1553118393', null, null);
+        list($feed, $items) = $this->fetcher->fetch($this->url, false, '@1553118393', false, null, null);
         $this->assertTrue($items[0]->getRtl());
     }
 
@@ -418,7 +423,7 @@ class FeedFetcherTest extends TestCase
 
 
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        list($feed, $items) = $this->fetcher->fetch($this->url, false, '@1553118393', null, null);
+        list($feed, $items) = $this->fetcher->fetch($this->url, false, '@1553118393', false, null, null);
         $this->assertSame($items[0]->getPubDate(), 1522180229);
     }
 
@@ -444,7 +449,7 @@ class FeedFetcherTest extends TestCase
 
 
         $this->mockIterator($this->feed_mock, [$this->item_mock]);
-        list($feed, $items) = $this->fetcher->fetch($this->url, false, '@1553118393', null, null);
+        list($feed, $items) = $this->fetcher->fetch($this->url, false, '@1553118393', false, null, null);
         $this->assertSame($items[0]->getPubDate(), 1519761029);
     }
 
diff --git a/tests/Unit/Fetcher/YoutubeFetcherTest.php b/tests/Unit/Fetcher/YoutubeFetcherTest.php
index 80f50b9d6..1c0c4c819 100644
--- a/tests/Unit/Fetcher/YoutubeFetcherTest.php
+++ b/tests/Unit/Fetcher/YoutubeFetcherTest.php
@@ -35,7 +35,7 @@ class YoutubeFetcherTest extends TestCase
      */
     private $feedFetcher;
 
-    public function setUp() 
+    public function setUp()
     {
         $this->feedFetcher = $this->getMockBuilder(FeedFetcher::class)
             ->disableOriginalConstructor()
@@ -44,26 +44,27 @@ class YoutubeFetcherTest extends TestCase
     }
 
 
-    public function testCanHandleFails() 
+    public function testCanHandleFails()
     {
         $url = 'http://youtube.com';
         $this->assertFalse($this->fetcher->canHandle($url));
     }
 
 
-    public function testCanHandle() 
+    public function testCanHandle()
     {
         $url = 'http://youtube.com/test/?test=a&list=b&b=c';
         $this->assertTrue($this->fetcher->canHandle($url));
     }
 
 
-    public function testPlaylistUrl() 
+    public function testPlaylistUrl()
     {
         $url = 'http://youtube.com/something/weird?a=b&list=sobo3&c=1';
         $transformedUrl = 'http://gdata.youtube.com/feeds/api/playlists/sobo3';
         $favicon = true;
         $modified = 3;
+        $fullTextEnabled = false;
         $user = 5;
         $password = 5;
         $feed = new Feed();
@@ -76,13 +77,14 @@ class YoutubeFetcherTest extends TestCase
                 $this->equalTo($transformedUrl),
                 $this->equalTo($favicon),
                 $this->equalTo($modified),
+                $this->equalTo($fullTextEnabled),
                 $this->equalTo($user)
             )
             ->will($this->returnValue($result));
-        $feed = $this->fetcher->fetch($url, $favicon, $modified, $user, $password);
+        $feed = $this->fetcher->fetch($url, $favicon, $modified, $fullTextEnabled, $user, $password);
 
         $this->assertEquals($url, $result[0]->getUrl());
     }
 
 
-}
\ No newline at end of file
+}
diff --git a/tests/Unit/Service/FeedServiceTest.php b/tests/Unit/Service/FeedServiceTest.php
index 149b3cdaf..1b2f4e5a4 100644
--- a/tests/Unit/Service/FeedServiceTest.php
+++ b/tests/Unit/Service/FeedServiceTest.php
@@ -657,7 +657,8 @@ class FeedServiceTest extends TestCase
             ->with(
                 $this->equalTo($feed->getUrl()),
                 $this->equalTo(false),
-                $this->equalTo($feed->getHttpLastModified())
+                $this->equalTo($feed->getHttpLastModified()),
+                $this->equalTo($feed->getFullTextEnabled())
             )
             ->will($this->throwException($ex));