serverside full text

remove enhancers

add full text client side implementation

fix bugs and tests for full text feed

7 files changed, 167 insertions, 417 deletions

diff --git a/tests/unit/articleenhancer/EnhancerTest.php b/tests/unit/articleenhancer/EnhancerTest.php
deleted file mode 100644
index f7dc2246e..000000000
--- a/tests/unit/articleenhancer/EnhancerTest.php
+++ /dev/null
@@ -1,102 +0,0 @@
-<?php
-/**
- * ownCloud - News
- *
- * This file is licensed under the Affero General Public License version 3 or
- * later. See the COPYING file.
- *
- * @author Alessandro Cosentino <cosenal@gmail.com>
- * @author Bernhard Posselt <dev@bernhard-posselt.com>
- * @copyright Alessandro Cosentino 2012
- * @copyright Bernhard Posselt 2012, 2014
- */
-
-namespace OCA\News\ArticleEnhancer;
-
-use \OCA\News\Db\Item;
-
-
-class AddEnhancer implements ArticleEnhancer {
-    public function enhance(Item $item) {
-        $body = $item->getBody();
-        $item->setBody($body += 1);
-        return $item;
-    }
-}
-
-
-class EnhancerTest extends \PHPUnit_Framework_TestCase {
-
-    private $enhancer;
-    private $articleEnhancer;
-    private $articleEnhancer2;
-
-    protected function setUp(){
-        $this->enhancer = new Enhancer();
-        $this->articleEnhancer = $this->getMockBuilder(
-            '\OCA\News\ArticleEnhancer\ArticleEnhancer')
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->enhancer->registerEnhancer('test.com', $this->articleEnhancer);
-    }
-
-
-    public function testEnhanceSetsCorrectHash(){
-        $item = new Item();
-        $item->setUrl('hi');
-        $urls = [
-            'https://test.com',
-            'https://www.test.com',
-            'https://test.com/',
-            'http://test.com',
-            'http://test.com/',
-            'http://www.test.com'
-        ];
-        $count = count($urls);
-        for ($i=0; $i < $count; $i++) {
-            $this->articleEnhancer->expects($this->at($i))
-                ->method('enhance')
-                ->with($this->equalTo($item))
-                ->will($this->returnValue($item));
-        }
-
-        for ($i=0; $i < $count; $i++) {
-            $url = $urls[$i];
-            $result = $this->enhancer->enhance($item, $url);
-            $this->assertEquals($item, $result);
-        }
-
-    }
-
-
-    public function testNotMatchShouldJustReturnItem() {
-        $item = new Item();
-        $item->setUrl('hi');
-
-        $url = 'https://tests.com';
-        $this->articleEnhancer->expects($this->never())
-                ->method('enhance');
-
-        $result = $this->enhancer->enhance($item, $url);
-        $this->assertEquals($item, $result);
-    }
-
-
-    public function testGlobalEnhancer() {
-        $this->enhancer->registerGlobalEnhancer(
-            new AddEnhancer()
-        );
-
-        $this->enhancer->registerGlobalEnhancer(
-            new AddEnhancer()
-        );
-
-        $item = new Item();
-        $item->setBody(1);
-
-        $result = $this->enhancer->enhance($item, 'test');
-
-        $this->assertEquals(3, $result->getBody());
-    }
-
-}
\ No newline at end of file
diff --git a/tests/unit/articleenhancer/RegexArticleEnhancerTest.php b/tests/unit/articleenhancer/RegexArticleEnhancerTest.php
deleted file mode 100644
index 018a596fa..000000000
--- a/tests/unit/articleenhancer/RegexArticleEnhancerTest.php
+++ /dev/null
@@ -1,35 +0,0 @@
-<?php
-/**
- * ownCloud - News
- *
- * This file is licensed under the Affero General Public License version 3 or
- * later. See the COPYING file.
- *
- * @author Alessandro Cosentino <cosenal@gmail.com>
- * @author Bernhard Posselt <dev@bernhard-posselt.com>
- * @copyright Alessandro Cosentino 2012
- * @copyright Bernhard Posselt 2012, 2014
- */
-
-namespace OCA\News\ArticleEnhancer;
-
-use \OCA\News\Db\Item;
-
-
-class RegexArticleEnhancerTest extends \PHPUnit_Framework_TestCase {
-
-
-    public function testRegexEnhancer() {
-        $item = new Item();
-        $item->setBody('atests is a nice thing');
-        $item->setUrl('http://john.com');
-        $regex = ["%tes(ts)%" => "heho$1tests"];
-
-        $regexEnhancer = new RegexArticleEnhancer('%john.com%', $regex);
-        $item = $regexEnhancer->enhance($item);
-
-        $this->assertEquals('ahehotstests is a nice thing', $item->getBody());
-    }
-
-
-}
\ No newline at end of file
diff --git a/tests/unit/articleenhancer/XPathArticleEnhancerTest.php b/tests/unit/articleenhancer/XPathArticleEnhancerTest.php
deleted file mode 100644
index 77c5ef2e7..000000000
--- a/tests/unit/articleenhancer/XPathArticleEnhancerTest.php
+++ /dev/null
@@ -1,239 +0,0 @@
-<?php
-/**
- * ownCloud - News
- *
- * This file is licensed under the Affero General Public License version 3 or
- * later. See the COPYING file.
- *
- * @author Alessandro Cosentino <cosenal@gmail.com>
- * @author Bernhard Posselt <dev@bernhard-posselt.com>
- * @copyright Alessandro Cosentino 2012
- * @copyright Bernhard Posselt 2012, 2014
- */
-
-namespace OCA\News\ArticleEnhancer;
-
-use \OCA\News\Db\Item;
-
-
-class XPathArticleEnhancerTest extends \PHPUnit_Framework_TestCase {
-
-    private $testEnhancer;
-    private $client;
-    private $clientFactory;
-
-    protected function setUp() {
-        $this->timeout = 30;
-        $this->clientFactory = $this
-            ->getMockBuilder('\OCA\News\Utility\PicoFeedClientFactory')
-            ->disableOriginalConstructor()
-            ->getMock();
-        $this->client = $this
-            ->getMockBuilder('\PicoFeed\Client\Client')
-            ->disableOriginalConstructor()
-            ->getMock();
-
-        $this->testEnhancer = new XPathArticleEnhancer(
-            $this->clientFactory,
-            [
-                '/explosm.net\/comics/' =>
-                    '//*[@id=\'maincontent\']/div[2]/div/span',
-                '/explosm.net\/shorts/' => '//*[@id=\'maincontent\']/div/div',
-                '/explosm.net\/all/' => '//body/*',
-                '/themerepublic.net/' => '//*[@class=\'post hentry\']'
-            ]
-        );
-        $this->userAgent = 'Mozilla/5.0 AppleWebKit';
-    }
-
-    private function setUpFile($body, $encoding, $url) {
-        $this->clientFactory->expects($this->once())
-            ->method('build')
-            ->will($this->returnValue($this->client));
-        $this->client->expects($this->once())
-            ->method('execute')
-            ->with($this->equalTo($url));
-        $this->client->expects($this->once())
-            ->method('setUserAgent')
-            ->with($this->equalTo($this->userAgent));
-        $this->client->expects($this->once())
-            ->method('getContent')
-            ->will($this->returnValue($body));
-    }
-
-
-    public function testDoesNotModifiyNotMatchingResults() {
-        $item = new Item();
-        $item->setUrl('http://explosm.net');
-        $this->assertEquals($item, $this->testEnhancer->enhance($item));
-    }
-
-
-    public function testDoesModifiyArticlesThatMatch() {
-        $encoding = 'utf-8';
-        $body = '<html>
-            <body>
-                <div id="maincontent">
-                    <div>nooo</div>
-                    <div><div><span>hiho</span></div></div>
-                </div>
-            </body>
-        </html>';
-        $item = new Item();
-        $item->setUrl('https://www.explosm.net/comics/312');
-        $item->setBody('Hello thar');
-
-        $this->setUpFile($body, $encoding, $item->getUrl());
-
-        $result = $this->testEnhancer->enhance($item);
-        $this->assertEquals('<div><span>hiho</span></div>', $result->getBody());
-    }
-
-
-    public function testDoesModifiyAllArticlesThatMatch() {
-        $encoding = 'utf-8';
-        $body = '<html>
-            <body>
-                <div id="maincontent">
-                    <div>nooo<div>hiho</div></div>
-                    <div><div>rawr</div></div>
-                </div>
-            </body>
-        </html>';
-        $item = new Item();
-        $item->setUrl('https://www.explosm.net/shorts/312');
-        $item->setBody('Hello thar');
-
-        $this->setUpFile($body, $encoding, $item->getUrl());
-
-        $result = $this->testEnhancer->enhance($item);
-        $this->assertEquals('<div><div>hiho</div><div>rawr</div></div>',
-            $result->getBody());
-    }
-
-
-    public function testModificationHandlesEmptyResults() {
-        $encoding = 'utf-8';
-        $body = '<html>
-            <body>
-                <div id="maincontent">
-                </div>
-            </body>
-        </html>';
-        $item = new Item();
-        $item->setUrl('https://www.explosm.net/comics/312');
-        $item->setBody('Hello thar');
-
-        $this->setUpFile($body, $encoding, $item->getUrl());
-
-        $result = $this->testEnhancer->enhance($item);
-        $this->assertEquals('Hello thar', $result->getBody());
-    }
-
-
-    public function testModificationDoesNotBreakOnEmptyDom() {
-        $encoding = 'utf-8';
-        $body = '';
-        $item = new Item();
-        $item->setUrl('https://www.explosm.net/comics/312');
-        $item->setBody('Hello thar');
-
-        $this->setUpFile($body, $encoding, $item->getUrl());
-
-        $result = $this->testEnhancer->enhance($item);
-        $this->assertEquals('Hello thar', $result->getBody());
-    }
-
-
-    public function testModificationDoesNotBreakOnBrokenDom() {
-        $encoding = 'utf-8';
-        $body = '<html/><p>
-            <body>
-                <div id="maincontent">
-                </div>
-            </body>
-        </html>';
-        $item = new Item();
-        $item->setUrl('https://www.explosm.net/comics/312');
-        $item->setBody('Hello thar');
-
-        $this->setUpFile($body, $encoding, $item->getUrl());
-
-        $result = $this->testEnhancer->enhance($item);
-        $this->assertEquals('Hello thar', $result->getBody());
-    }
-
-
-    public function testTransformRelativeUrls() {
-        $encoding = 'utf-8';
-        $body = '<html>
-            <body>
-                <a href="../a/relative/url.html?a=1#b">link</a>
-                <a href="b/relative/url.html">link2</a>
-                <img src="/another/relative/link.jpg"></img>
-            </body>
-        </html>';
-        $item = new Item();
-        $item->setUrl('https://www.explosm.net/all/312');
-        $item->setBody('Hello thar');
-
-        $this->setUpFile($body, $encoding, $item->getUrl());
-
-        $result = $this->testEnhancer->enhance($item);
-        $this->assertEquals('<div>' .
-            '<a target="_blank" rel="noreferrer" ' .
-                'href="https://www.explosm.net/a/relative/url.html?a=1#b">' .
-                'link</a>' .
-            '<a target="_blank" rel="noreferrer" ' .
-                'href="https://www.explosm.net/all/b/relative/url.html">' .
-                'link2</a>' .
-            '<img src="https://www.explosm.net/another/relative/link.jpg">' .
-            '</div>', $result->getBody());
-    }
-
-    public function testTransformRelativeUrlSpecials() {
-        $encoding = 'utf-8';
-        $body = '<html>
-            <body>
-                <img src="relative/url.png?a=1&b=2">
-            </body>
-        </html>';
-        $item = new Item();
-        $item->setUrl('https://username:secret@www.explosm.net/all/312');
-        $item->setBody('Hello thar');
-
-        $this->setUpFile($body, $encoding, $item->getUrl());
-
-        $result = $this->testEnhancer->enhance($item);
-        $this->assertEquals(
-            '<div><img src="' .
-            'https://username:secret@www.explosm.net' .
-            '/all/relative/url.png?a=1&amp;b=2"></div>',
-            $result->getBody());
-    }
-
-    public function testDontTransformAbsoluteUrlsAndMails() {
-        $encoding = 'utf-8';
-        $body = '<html>
-            <body>
-                <img src="http://www.url.com/absolute/url.png">
-                <a href="mailto:test@testsite.com">mail</a>
-            </body>
-        </html>';
-        $item = new Item();
-        $item->setUrl('https://www.explosm.net/all/312');
-        $item->setBody('Hello thar');
-
-        $this->setUpFile($body, $encoding, $item->getUrl());
-
-        $result = $this->testEnhancer->enhance($item);
-        $this->assertEquals(
-            '<div>' .
-            '<img src="http://www.url.com/absolute/url.png">' .
-            '<a target="_blank" rel="noreferrer" href="mailto:test@testsite.com">mail</a>' .
-            '</div>',
-            $result->getBody()
-        );
-    }
-
-}
diff --git a/tests/unit/controller/FeedControllerTest.php b/tests/unit/controller/FeedControllerTest.php
index 813508cef..ceed1c0b2 100644
--- a/tests/unit/controller/FeedControllerTest.php
+++ b/tests/unit/controller/FeedControllerTest.php
@@ -13,12 +13,12 @@
 
 namespace OCA\News\Controller;
 
-use \OCP\AppFramework\Http;
+use OCP\AppFramework\Http;
 
-use \OCA\News\Db\Feed;
-use \OCA\News\Db\FeedType;
-use \OCA\News\Service\ServiceNotFoundException;
-use \OCA\News\Service\ServiceConflictException;
+use OCA\News\Db\Feed;
+use OCA\News\Db\FeedType;
+use OCA\News\Service\ServiceNotFoundException;
+use OCA\News\Service\ServiceConflictException;
 
 
 class FeedControllerTest extends \PHPUnit_Framework_TestCase {
@@ -519,6 +519,17 @@ class FeedControllerTest extends \PHPUnit_Framework_TestCase {
     }
 
 
+    public function testEnableFullText() {
+        $this->feedService->expects($this->once())
+            ->method('enableFullText')
+            ->with($this->equalTo(4),
+                    $this->equalTo(true),
+                    $this->equalTo($this->user))
+            ->will($this->returnValue(1));
+
+        $this->controller->enableFullText(4, true);
+    }
+
     public function testOrderingDoesNotExist(){
         $msg = 'hehe';
 
diff --git a/tests/unit/fetcher/FeedFetcherTest.php b/tests/unit/fetcher/FeedFetcherTest.php
index e1a3997ac..2c743500a 100644
--- a/tests/unit/fetcher/FeedFetcherTest.php
+++ b/tests/unit/fetcher/FeedFetcherTest.php
@@ -319,6 +319,17 @@ class FeedFetcherTest extends \PHPUnit_Framework_TestCase {
         $this->assertEquals([$feed, [$item]], $result);
     }
 
+    public function testFullText() {
+        $this->setUpReader($this->url);
+
+        $feed = $this->createFeed();
+        $item = $this->createItem();
+        $this->parser->expects($this->once())
+            ->method('enableContentGrabber');
+        $this->expectFeed('getItems', [$this->item]);
+        $this->fetcher->fetch($this->url, false, null, null, true);
+    }
+
 
     public function testNoFavicon() {
         $this->setUpReader($this->url);
diff --git a/tests/unit/fetcher/FetcherTest.php b/tests/unit/fetcher/FetcherTest.php
index 1ab81fc1e..5b9efc920 100644
--- a/tests/unit/fetcher/FetcherTest.php
+++ b/tests/unit/fetcher/FetcherTest.php
@@ -44,9 +44,16 @@ class FetcherTest extends \PHPUnit_Framework_TestCase {
             ->method('canHandle')
             ->with($this->equalTo($url))
             ->will($this->returnValue(true));
+        $mockFetcher->expects($this->once())
+            ->method('fetch')
+            ->with($this->equalTo($url),
+                   $this->equalTo(true),
+                   $this->equalTo(1),
+                   $this->equalTo(2),
+                   $this->equalTo(3));
         $this->fetcher->registerFetcher($mockFetcher);
 
-        $this->fetcher->fetch($url);
+        $this->fetcher->fetch($url, true, 1, 2, 3);
     }
 
 
@@ -127,4 +134,4 @@ class FetcherTest extends \PHPUnit_Framework_TestCase {
     }
 
 
-}
\ No newline at end of file
+}
diff --git a/tests/unit/service/FeedServiceTest.php b/tests/unit/service/FeedServiceTest.php
index 8b0d78c52..ae85ed5a7 100644
--- a/tests/unit/service/FeedServiceTest.php
+++ b/tests/unit/service/FeedServiceTest.php
@@ -14,12 +14,12 @@
 
 namespace OCA\News\Service;
 
-use \OCP\AppFramework\Db\DoesNotExistException;
+use OCP\AppFramework\Db\DoesNotExistException;
 
-use \OCA\News\Db\Feed;
-use \OCA\News\Db\Item;
-use \OCA\News\Fetcher\Fetcher;
-use \OCA\News\Fetcher\FetcherException;
+use OCA\News\Db\Feed;
+use OCA\News\Db\Item;
+use OCA\News\Fetcher\Fetcher;
+use OCA\News\Fetcher\FetcherException;
 
 
 class FeedServiceTest extends \PHPUnit_Framework_TestCase {
@@ -34,7 +34,6 @@ class FeedServiceTest extends \PHPUnit_Framework_TestCase {
     private $time;
     private $importParser;
     private $autoPurgeMinimumInterval;
-    private $enhancer;
     private $purifier;
     private $l10n;
     private $logger;
@@ -69,11 +68,6 @@ class FeedServiceTest extends \PHPUnit_Framework_TestCase {
             ->getMockBuilder('\OCA\News\Db\ItemMapper')
             ->disableOriginalConstructor()
             ->getMock();
-        $this->enhancer = $this
-            ->getMockBuilder('\OCA\News\ArticleEnhancer\Enhancer')
-            ->disableOriginalConstructor()
-            ->getMock();
-
         $this->purifier = $this
             ->getMockBuilder('\HTMLPurifier')
             ->disableOriginalConstructor()
@@ -88,8 +82,7 @@ class FeedServiceTest extends \PHPUnit_Framework_TestCase {
 
         $this->feedService = new FeedService($this->feedMapper,
             $this->fetcher, $this->itemMapper, $this->logger, $this->l10n,
-            $timeFactory, $config, $this->enhancer, $this->purifier,
-            $this->loggerParams);
+            $timeFactory, $config, $this->purifier, $this->loggerParams);
         $this->user = 'jack';
     }
 
@@ -158,11 +151,6 @@ class FeedServiceTest extends \PHPUnit_Framework_TestCase {
                 $this->equalTo($item2->getFeedId()),
                 $this->equalTo($this->user))
             ->will($this->throwException($ex));
-        $this->enhancer->expects($this->at(0))
-            ->method('enhance')
-            ->with($this->equalTo($return[1][1]),
-                $this->equalTo($url))
-            ->will($this->returnValue($return[1][1]));
         $this->purifier->expects($this->at(0))
             ->method('purify')
             ->with($this->equalTo($return[1][1]->getBody()))
@@ -177,11 +165,6 @@ class FeedServiceTest extends \PHPUnit_Framework_TestCase {
                 $this->equalTo($item1->getFeedId()),
                 $this->equalTo($this->user))
             ->will($this->throwException($ex));
-        $this->enhancer->expects($this->at(1))
-            ->method('enhance')
-            ->with($this->equalTo($return[1][0]),
-                $this->equalTo($url))
-            ->will($this->returnValue($return[1][0]));
         $this->purifier->expects($this->at(1))
             ->method('purify')
             ->with($this->equalTo($return[1][0]->getBody()))
@@ -235,11 +218,6 @@ class FeedServiceTest extends \PHPUnit_Framework_TestCase {
                 $this->equalTo($item2->getFeedId()),
                 $this->equalTo($this->user))
             ->will($this->throwException($ex));
-        $this->enhancer->expects($this->at(0))
-            ->method('enhance')
-            ->with($this->equalTo($return[1][1]),
-                $this->equalTo($url))
-            ->will($this->returnValue($return[1][1]));
         $this->purifier->expects($this->at(0))
             ->method('purify')
             ->with($this->equalTo($return[1][1]->getBody()))
@@ -304,11 +282,6 @@ class FeedServiceTest extends \PHPUnit_Framework_TestCase {
                     $this->equalTo($items[0]->getFeedId()),
                     $this->equalTo($this->user))
             ->will($this->throwException($ex));
-        $this->enhancer->expects($this->at(0))
-            ->method('enhance')
-            ->with($this->equalTo($items[0]),
-                $this->equalTo($feed->getUrl()))
-            ->will($this->returnValue($items[0]));
         $this->purifier->expects($this->at(0))
             ->method('purify')
             ->with($this->equalTo($items[0]->getBody()))
@@ -328,6 +301,68 @@ class FeedServiceTest extends \PHPUnit_Framework_TestCase {
         $this->assertEquals($return, $feed);
     }
 
+    public function testForceUpdateUpdatesEntry(){
+        $feed = new Feed();
+        $feed->setId(3);
+        $feed->setArticlesPerUpdate(1);
+        $feed->setLink('http://test');
+        $feed->setUrl('http://test');
+        $feed->setUrlHash('yo');
+        $feed->setLastModified(3);
+        $feed->setEtag(4);
+
+        $item = new Item();
+        $item->setGuidHash(md5('hi'));
+        $item->setFeedId(3);
+        $items = [$item];
+
+        $ex = new DoesNotExistException('hi');
+
+        $fetchReturn = [$feed, $items];
+
+        $this->feedMapper->expects($this->at(0))
+            ->method('find')
+            ->with($this->equalTo($feed->getId()),
+                    $this->equalTo($this->user))
+            ->will($this->returnValue($feed));
+        $this->fetcher->expects($this->once())
+            ->method('fetch')
+            ->with(
+                $this->equalTo('http://test'),
+                $this->equalTo(false),
+                $this->equalTo(3),
+                $this->equalTo(4)
+            )
+            ->will($this->returnValue($fetchReturn));
+        $this->feedMapper->expects($this->at(1))
+            ->method('update')
+            ->with($this->equalTo($feed));
+        $this->itemMapper->expects($this->once())
+            ->method('findByGuidHash')
+            ->with($this->equalTo($items[0]->getGuidHash()),
+                    $this->equalTo($items[0]->getFeedId()),
+                    $this->equalTo($this->user))
+            ->will($this->returnValue($items[0]));
+        $this->purifier->expects($this->at(0))
+            ->method('purify')
+            ->with($this->equalTo($items[0]->getBody()))
+            ->will($this->returnValue($items[0]->getBody()));
+        $this->itemMapper->expects($this->once())
+            ->method('update')
+            ->with($this->equalTo($items[0]));
+
+        $this->feedMapper->expects($this->at(2))
+            ->method('find')
+            ->with($feed->getId(), $this->user)
+            ->will($this->returnValue($feed));
+
+
+        $return = $this->feedService->update($feed->getId(), $this->user, true);
+
+        $this->assertEquals($return, $feed);
+    }
+
+
 
     public function testUpdateUpdatesArticlesPerFeedCount() {
         $feed = new Feed();
@@ -404,6 +439,38 @@ class FeedServiceTest extends \PHPUnit_Framework_TestCase {
     }
 
 
+    public function testUpdatePassesFullText() {
+        $feed = new Feed();
+        $feed->setId(3);
+        $feed->setUrl('https://goo.com');
+        $feed->setEtag('abc');
+        $feed->setLastModified(123);
+        $feed->setFullTextEnabled(true);
+
+        $ex = new DoesNotExistException('');
+
+        $this->feedMapper->expects($this->at(0))
+            ->method('find')
+            ->with($this->equalTo($feed->getId()),
+                    $this->equalTo($this->user))
+            ->will($this->returnValue($feed));
+
+        $this->fetcher->expects($this->once())
+            ->method('fetch')
+