diff options
-rw-r--r-- | CHANGELOG | 1 | ||||
-rw-r--r-- | README.rst | 6 | ||||
-rw-r--r-- | app/news.php | 3 | ||||
-rw-r--r-- | articleenhancer/xpatharticleenhancer.php | 39 | ||||
-rw-r--r-- | fetcher/feedfetcher.php | 12 | ||||
-rw-r--r-- | tests/unit/articleenhancer/XPathArticleEnhancerTest.php | 106 | ||||
-rw-r--r-- | tests/unit/fetcher/FeedFetcherTest.php | 67 | ||||
-rw-r--r-- | tests/unit/utility/ConfigTest.php | 8 | ||||
-rw-r--r-- | tests/unit/utility/FaviconFetcherTest.php | 102 | ||||
-rw-r--r-- | utility/config.php | 14 | ||||
-rw-r--r-- | utility/faviconfetcher.php | 22 | ||||
-rw-r--r-- | utility/simplepieapifactory.php | 6 |
12 files changed, 348 insertions, 38 deletions
@@ -5,6 +5,7 @@ owncloud-news (2.001) * Require ownCloud 6.0.3 * Remove html tags from feed titles * Port to built in core App Framework and thus removing the need to install the App Framework +* Add experimental webproxy support owncloud-news (1.808) * Also focus article area when clicking on all unread link diff --git a/README.rst b/README.rst index 7b60fba95..e78cb3761 100644 --- a/README.rst +++ b/README.rst @@ -178,7 +178,11 @@ The configuration is in **INI** format and looks like this: * **simplePieCacheDuration**: Amount of seconds to cache feeds * **feedFetcherTimeout**: Maximum number of seconds to wait for an RSS or Atom feed to load. If a feed takes longer than that number of seconds to update, the update will be aborted * **useCronUpdates**: To use a custom update/cron script you need to disable the cronjob which is run by ownCloud by default by setting this to false - +* **proxHost**: The adress of the proxy. Leave empty if no proxy should be used +* **proxyPort**: The proxy port, defaults to 8080 +* **proxyAuth**: The proxy auth details. Provide these in HTTP basic auth format: + + user:password diff --git a/app/news.php b/app/news.php index 18891f2c8..259163904 100644 --- a/app/news.php +++ b/app/news.php @@ -361,7 +361,8 @@ class News extends App { $container->registerService('FaviconFetcher', function($c) { return new FaviconFetcher( - $c->query('SimplePieAPIFactory') + $c->query('SimplePieAPIFactory'), + $c->query('Config') ); }); diff --git a/articleenhancer/xpatharticleenhancer.php b/articleenhancer/xpatharticleenhancer.php index 50dc402cc..0d7589fec 100644 --- a/articleenhancer/xpatharticleenhancer.php +++ b/articleenhancer/xpatharticleenhancer.php @@ -36,6 +36,7 @@ class XPathArticleEnhancer implements ArticleEnhancer { private $feedRegex; private $fileFactory; private $maximumTimeout; + private $config; /** @@ -50,6 +51,7 @@ class XPathArticleEnhancer implements ArticleEnhancer { $this->regexXPathPair = $regexXPathPair; $this->fileFactory = $fileFactory; $this->maximumTimeout = $config->getFeedFetcherTimeout(); + $this->config = $config; } @@ -58,13 +60,7 @@ class XPathArticleEnhancer implements ArticleEnhancer { foreach($this->regexXPathPair as $regex => $search) { if(preg_match($regex, $item->getUrl())) { - $file = $this->fileFactory->getFile( - $item->getUrl(), - $this->maximumTimeout, - 5, - null, - "Mozilla/5.0 AppleWebKit" - ); + $file = $this->getFile($item->getUrl()); // convert encoding by detecting charset from header $contentType = $file->headers['content-type']; @@ -98,6 +94,35 @@ class XPathArticleEnhancer implements ArticleEnhancer { } + private function getFile($url) { + if(trim($this->config->getProxyHost()) === '') { + return $this->fileFactory->getFile( + $url, + $this->maximumTimeout, + 5, + null, + "Mozilla/5.0 AppleWebKit", + false, + null, + null, + null + ); + } else { + return $this->fileFactory->getFile( + $url, + $this->maximumTimeout, + 5, + null, + "Mozilla/5.0 AppleWebKit", + false, + $this->config->getProxyHost(), + $this->config->getProxyPort(), + $this->config->getProxyAuth() + ); + } + } + + /** * Method which converts all relative "href" and "src" URLs of * a HTML snippet with their absolute equivalent diff --git a/fetcher/feedfetcher.php b/fetcher/feedfetcher.php index c00a8c859..6808465c2 100644 --- a/fetcher/feedfetcher.php +++ b/fetcher/feedfetcher.php @@ -42,6 +42,9 @@ class FeedFetcher implements IFeedFetcher { private $simplePieFactory; private $fetchTimeout; private $time; + private $proxyHost; + private $proxyPort; + private $proxyAuth; public function __construct(API $api, SimplePieAPIFactory $simplePieFactory, @@ -56,6 +59,9 @@ class FeedFetcher implements IFeedFetcher { $this->faviconFetcher = $faviconFetcher; $this->simplePieFactory = $simplePieFactory; $this->time = $time; + $this->proxyHost = $config->getProxyHost(); + $this->proxyPort = $config->getProxyPort(); + $this->proxyAuth = $config->getProxyAuth(); } @@ -83,6 +89,12 @@ class FeedFetcher implements IFeedFetcher { $simplePie->set_cache_location($this->cacheDirectory); $simplePie->set_cache_duration($this->cacheDuration); + if(trim($this->proxyHost) !== '') { + $simplePie->set_proxyhost($this->proxyHost); + $simplePie->set_proxyport($this->proxyPort); + $simplePie->set_proxyuserpwd($this->proxyAuth); + } + if (!$simplePie->init()) { throw new FetcherException('Could not initialize simple pie on feed with url ' . $url); } diff --git a/tests/unit/articleenhancer/XPathArticleEnhancerTest.php b/tests/unit/articleenhancer/XPathArticleEnhancerTest.php index a600d6739..dbd752518 100644 --- a/tests/unit/articleenhancer/XPathArticleEnhancerTest.php +++ b/tests/unit/articleenhancer/XPathArticleEnhancerTest.php @@ -38,17 +38,32 @@ class XPathArticleEnhancerTest extends \OCA\News\Utility\TestUtility { private $redirects; private $headers; private $userAgent; + private $proxyHost; + private $proxyPort; + private $proxyAuth; protected function setUp() { $this->timeout = 30; $this->fileFactory = $this->getMockBuilder('\OCA\News\Utility\SimplePieAPIFactory') ->disableOriginalConstructor() ->getMock(); - $config = $this->getMockBuilder( + $this->proxyHost = 'test'; + $this->proxyPort = 3; + $this->proxyAuth = 'hi'; + $this->config = $this->getMockBuilder( '\OCA\News\Utility\Config') ->disableOriginalConstructor() ->getMock(); - $config->expects($this->any()) + $this->config->expects($this->any()) + ->method('getProxyHost') + ->will($this->returnValue('')); + $this->config->expects($this->any()) + ->method('getProxyAuth') + ->will($this->returnValue($this->proxyAuth)); + $this->config->expects($this->any()) + ->method('getProxyPort') + ->will($this->returnValue($this->proxyPort)); + $this->config->expects($this->any()) ->method('getFeedFetcherTimeout') ->will($this->returnValue($this->timeout)); @@ -60,7 +75,7 @@ class XPathArticleEnhancerTest extends \OCA\News\Utility\TestUtility { '/explosm.net\/all/' => '//body/*', '/themerepublic.net/' => '//*[@class=\'post hentry\']' ), - $config + $this->config ); $this->redirects = 5; $this->headers = null; @@ -68,6 +83,91 @@ class XPathArticleEnhancerTest extends \OCA\News\Utility\TestUtility { } + public function testXPathUsesNoProxy() { + $file = new \stdClass; + $file->headers = array("content-type"=>"text/html; charset=utf-8"); + $file->body = ''; + $item = new Item(); + $item->setUrl('https://www.explosm.net/comics/312'); + $item->setBody('Hello thar'); + + $this->fileFactory->expects($this->once()) + ->method('getFile') + ->with($this->equalTo($item->getUrl()), + $this->equalTo($this->timeout), + $this->equalTo($this->redirects), + $this->equalTo($this->headers), + $this->equalTo($this->userAgent), + $this->equalTo(false), + $this->equalTo(null), + $this->equalTo(null), + $this->equalTo(null)) + ->will($this->returnValue($file)); + + $result = $this->testEnhancer->enhance($item); + $this->assertEquals('Hello thar', $result->getBody()); + } + + + public function testXPathUsesProxy() { + $this->config = $this->getMockBuilder( + '\OCA\News\Utility\Config') + ->disableOriginalConstructor() + ->getMock(); + $this->config->expects($this->any()) + ->method('getProxyHost') + ->will($this->returnValue($this->proxyHost)); + $this->config->expects($this->any()) + ->method('getProxyAuth') + ->will($this->returnValue($this->proxyAuth)); + $this->config->expects($this->any()) + ->method('getProxyPort') + ->will($this->returnValue($this->proxyPort)); + $this->config->expects($this->any()) + ->method('getFeedFetcherTimeout') + ->will($this->returnValue($this->timeout)); + + $this->testEnhancer = new XPathArticleEnhancer( + $this->fileFactory, + array( + '/explosm.net\/comics/' => '//*[@id=\'maincontent\']/div[2]/div/span', + '/explosm.net\/shorts/' => '//*[@id=\'maincontent\']/div/div', + '/explosm.net\/all/' => '//body/*', + '/themerepublic.net/' => '//*[@class=\'post hentry\']' + ), + $this->config + ); + + $file = new \stdClass; + $file->headers = array("content-type"=>"text/html; charset=utf-8"); + $file->body = ''; + $item = new Item(); + $item->setUrl('https://www.explosm.net/comics/312'); + $item->setBody('Hello thar'); + + $this->config->expects($this->any()) + ->method('getProxyHost') + ->will($this->returnValue($this->proxyHost)); + + $this->fileFactory->expects($this->once()) + ->method('getFile') + ->with($this->equalTo($item->getUrl()), + $this->equalTo($this->timeout), + $this->equalTo($this->redirects), + $this->equalTo($this->headers), + $this->equalTo($this->userAgent), + $this->equalTo(false), + $this->equalTo($this->proxyHost), + $this->equalTo($this->proxyPort), + $this->equalTo($this->proxyAuth)) + ->will($this->returnValue($file)); + + $result = $this->testEnhancer->enhance($item); + $this->assertEquals('Hello thar', $result->getBody()); + } + + + public function testDoesNotModifiyNotMatchingResults() { $item = new Item(); $item->setUrl('http://explosm.net'); diff --git a/tests/unit/fetcher/FeedFetcherTest.php b/tests/unit/fetcher/FeedFetcherTest.php index ff076d11c..392d0972c 100644 --- a/tests/unit/fetcher/FeedFetcherTest.php +++ b/tests/unit/fetcher/FeedFetcherTest.php @@ -44,6 +44,10 @@ class FeedFetcherTest extends \OCA\News\Utility\TestUtility { private $item; private $purifier; private $fetchTimeout; + private $proxyHost; + private $getProxyPort; + private $proxyAuth; + private $config; // items private $permalink; @@ -62,10 +66,23 @@ class FeedFetcherTest extends \OCA\News\Utility\TestUtility { private $webFavicon; protected function setUp(){ - $this->core = $this->getMockBuilder( - '\SimplePie_Core') - ->disableOriginalConstructor() - ->getMock(); + $this->core = $this->getMock( + '\SimplePie_Core', array( + 'set_timeout', + 'set_feed_url', + 'enable_cache', + 'set_stupidly_fast', + 'set_cache_location', + 'set_cache_duration', + 'set_proxyhost', + 'set_proxyport', + 'set_proxyuserpwd', + 'init', + 'get_permalink', + 'get_items', + 'get_title', + 'get_image_url' + )); $this->coreFactory = $this->getMockBuilder( '\OCA\News\Utility\SimplePieAPIFactory') ->disableOriginalConstructor() @@ -88,15 +105,27 @@ class FeedFetcherTest extends \OCA\News\Utility\TestUtility { ->will($this->returnValue($this->time)); $this->cacheDuration = 100; $this->cacheDirectory = 'dir/'; + $this->proxyHost = 'test'; + $this->proxyPort = 30; + $this->proxyAuth = 'hi'; $this->fetchTimeout = 40; - $config = $this->getMockBuilder( + $this->config = $this->getMockBuilder( '\OCA\News\Utility\Config') ->disableOriginalConstructor() ->getMock(); - $config->expects($this->any()) + $this->config->expects($this->any()) ->method('getSimplePieCacheDuration') ->will($this->returnValue($this->cacheDuration)); - $config->expects($this->any()) + $this->config->expects($this->any()) + ->method('getProxyHost') + ->will($this->returnValue($this->proxyHost)); + $this->config->expects($this->any()) + ->method('getProxyAuth') + ->will($this->returnValue($this->proxyAuth)); + $this->config->expects($this->any()) + ->method('getProxyPort') + ->will($this->returnValue($this->proxyPort)); + $this->config->expects($this->any()) ->method('getFeedFetcherTimeout') ->will($this->returnValue($this->fetchTimeout)); $this->fetcher = new FeedFetcher($this->getAPIMock(), @@ -104,7 +133,7 @@ class FeedFetcherTest extends \OCA\News\Utility\TestUtility { $this->faviconFetcher, $timeFactory, $this->cacheDirectory, - $config); + $this->config); $this->url = 'http://tests'; $this->permalink = 'http://permalink'; @@ -131,6 +160,19 @@ class FeedFetcherTest extends \OCA\News\Utility\TestUtility { } + public function testDoesNotUseProxyIfNotEnabled() { + $this->config->expects($this->any()) + ->method('getProxyHost') + ->will($this->returnValue('')); + $this->core->expects($this->never()) + ->method('set_proxyhost'); + $this->core->expects($this->never()) + ->method('set_proxyport'); + $this->core->expects($this->never()) + ->method('set_proxyuserpwd'); + } + + public function testFetchThrowsExceptionWhenInitFailed() { $this->core->expects($this->once()) ->method('set_feed_url') @@ -145,6 +187,15 @@ class FeedFetcherTest extends \OCA\News\Utility\TestUtility { ->method('set_cache_location') ->with($this->equalTo($this->cacheDirectory)); $this->core->expects($this->once()) + ->method('set_proxyhost') + ->with($this->equalTo($this->proxyHost)); + $this->core->expects($this->once()) + ->method('set_proxyport') + ->with($this->equalTo($this->proxyPort)); + $this->core->expects($this->once()) + ->method('set_proxyuserpwd') + ->with($this->equalTo($this->proxyAuth)); + $this->core->expects($this->once()) ->method('set_stupidly_fast') ->with($this->equalTo(true)); $this->core->expects($this->once()) diff --git a/tests/unit/utility/ConfigTest.php b/tests/unit/utility/ConfigTest.php index d3f91b975..9cea0acf0 100644 --- a/tests/unit/utility/ConfigTest.php +++ b/tests/unit/utility/ConfigTest.php @@ -57,7 +57,7 @@ class ConfigFetcherTest extends \OCA\News\Utility\TestUtility { $this->assertEquals(true, $this->config->getUseCronUpdates()); $this->assertEquals(8080, $this->config->getProxyPort()); $this->assertEquals('', $this->config->getProxyHost()); - $this->assertEquals('', $this->config->getProxyPassword()); + $this->assertEquals('', $this->config->getProxyAuth()); } @@ -124,11 +124,11 @@ class ConfigFetcherTest extends \OCA\News\Utility\TestUtility { "useCronUpdates = true\n" . "proxyHost = yo man\n" . "proxyPort = 12\n" . - "proxyPassword = this is a test"; + "proxyAuth = this is a test"; $this->config->setAutoPurgeCount(3); $this->config->setProxyHost("yo man"); $this->config->setProxyPort(12); - $this->config->setProxyPassword("this is a test"); + $this->config->setProxyAuth("this is a test"); $this->fileSystem->expects($this->once()) ->method('file_put_contents') @@ -154,7 +154,7 @@ class ConfigFetcherTest extends \OCA\News\Utility\TestUtility { "useCronUpdates = false\n" . "proxyHost = \n" . "proxyPort = 8080\n" . - "proxyPassword = "; + "proxyAuth = "; $this->fileSystem->expects($this->once()) ->method('file_put_contents') diff --git a/tests/unit/utility/FaviconFetcherTest.php b/tests/unit/utility/FaviconFetcherTest.php index 4f11c49e1..875fcf11d 100644 --- a/tests/unit/utility/FaviconFetcherTest.php +++ b/tests/unit/utility/FaviconFetcherTest.php @@ -33,14 +33,33 @@ class FaviconFetcherTest extends \PHPUnit_Framework_TestCase { private $fetcher; private $fileFactory; private $png; + private $proxyHost; + private $proxyPort; + private $proxyAuth; protected function setUp(){ $this->png = "\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"; + $this->proxyHost = 'test'; + $this->proxyPort = 3; + $this->proxyAuth = 'hi'; $this->fileFactory = $this->getMockBuilder( '\OCA\News\Utility\SimplePieAPIFactory') ->disableOriginalConstructor() ->getMock(); - $this->fetcher = new FaviconFetcher($this->fileFactory); + $this->config = $this->getMockBuilder( + '\OCA\News\Utility\Config') + ->disableOriginalConstructor() + ->getMock(); + $this->config->expects($this->any()) + ->method('getProxyHost') + ->will($this->returnValue('')); + $this->config->expects($this->any()) + ->method('getProxyAuth') + ->will($this->returnValue($this->proxyAuth)); + $this->config->expects($this->any()) + ->method('getProxyPort') + ->will($this->returnValue($this->proxyPort)); + $this->fetcher = new FaviconFetcher($this->fileFactory, $this->config); } @@ -78,6 +97,87 @@ class FaviconFetcherTest extends \PHPUnit_Framework_TestCase { } + public function testProxySettingsAreUsed() { + $this->config = $this->getMockBuilder( + '\OCA\News\Utility\Config') + ->disableOriginalConstructor() + ->getMock(); + $this->config->expects($this->any()) + ->method('getProxyHost') + ->will($this->returnValue($this->proxyHost)); + $this->config->expects($this->any()) + ->method('getProxyAuth') + ->will($this->returnValue($this->proxyAuth)); + $this->config->expects($this->any()) + ->method('getProxyPort') + ->will($this->returnValue($this->proxyPort)); + $this->fetcher = new FaviconFetcher($this->fileFactory, $this->config); + + $faviconPath = "/owncloud/core/img/favicon.png"; + $html = $this->getFaviconHTML($faviconPath); + + $url = 'http://google.com'; + $pageMock = $this->getFileMock($html); + $pngMock = $this->getFileMock($this->png); + + $this->fileFactory->expects($this->at(0)) + ->method('getFile') + ->with($this->equalTo('http://google.com')) + ->will($this->returnValue($pageMock)); + + $this->fileFactory->expects($this->at(1)) + ->method('getFile') + ->with($this->equalTo( + 'http://google.com/owncloud/core/img/favicon.png'), + $this->equalTo(10), + $this->equalTo(5), + $this->equalTo(null), + $this->equalTo(null), + $this->equalTo(false), + $this->equalTo($this->proxyHost), + $this->equalTo($this->proxyPort), + $this->equalTo($this->proxyAuth)) + ->will($this->returnValue($pngMock)); + + $favicon = $this->fetcher->fetch($url); + + $this->assertEquals('http://google.com/owncloud/core/img/favicon.png', $favicon); + } + + + public function testNoProxySettingsAreUsed() { + $faviconPath = "/owncloud/core/img/favicon.png"; + $html = $this->getFaviconHTML($faviconPath); + + $url = 'http://google.com'; + $pageMock = $this->getFileMock($html); + $pngMock = $this->getFileMock($this->png); + + $this->fileFactory->expects($this->at(0)) + ->method('getFile') + ->with($this->equalTo('http://google.com')) + ->will($this->returnValue($pageMock)); + + $this->fileFactory->expects($this->at(1)) + ->method('getFile') + ->with($this->equalTo( + 'http://google.com/owncloud/core/img/favicon.png'), + $this->equalTo(10), + $this->equalTo(5), + $this->equalTo(null), + $this->equalTo(null), + $this->equalTo(false), + $this->equalTo(null), + $this->equalTo(null), + $this->equalTo(null)) + ->will($this->returnValue($pngMock)); + + $favicon = $this->fetcher->fetch($url); + + $this->assertEquals('http://google.com/owncloud/core/img/favicon.png', $favicon); + } + + public function testFetchFaviconFaviconDotIcoHttp(){ $url = ' sub.google.com '; $mock = $this->getFileMock($this->png); diff --git a/utility/config.php b/utility/config.php index 92d0b9daf..0fa636ea8 100644 --- a/utility/config.php +++ b/utility/config.php @@ -41,7 +41,7 @@ class Config { private $useCronUpdates; // turn off updates run by owncloud cronjob private $proxyHost; private $proxyPort; - private $proxyPassword; + private $proxyAuth; private $api; @@ -55,7 +55,7 @@ class Config { $this->api = $api; $this->proxyHost = ''; $this->proxyPort = 8080; - $this->proxyPassword = ''; + $this->proxyAuth = ''; } public function getProxyPort() { @@ -66,8 +66,8 @@ class Config { return $this->proxyHost; } - public function getProxyPassword() { - return $this->proxyPassword; + public function getProxyAuth() { + return $this->proxyAuth; } public function getAutoPurgeMinimumInterval() { @@ -128,8 +128,8 @@ class Config { $this->proxyHost = $value; } - public function setProxyPassword($value) { - $this->proxyPassword = $value; + public function setProxyAuth($value) { + $this->proxyAuth = $value; } @@ -172,7 +172,7 @@ class Config { "useCronUpdates = " . var_export($this->useCronUpdates, true) . "\n" . "proxyHost = " . $this->proxyHost . "\n" . "proxyPort = " . $this->proxyPort . "\n" . - "proxyPassword = " . $this->proxyPassword; + "proxyAuth = " . $this->proxyAuth; ; $this->fileSystem->file_put_contents($configPath, $ini); diff --git a/utility/faviconfetcher.php b/utility/faviconfetcher.php index d5d270a49..ae5d8c184 100644 --- a/utility/faviconfetcher.php +++ b/utility/faviconfetcher.php @@ -28,15 +28,16 @@ namespace OCA\News\Utility; class FaviconFetcher { private $apiFactory; - + private $config; /** * Inject a factory to build a simplepie file object. This is needed because * the file object contains logic in its constructor which makes it * impossible to inject and test */ - public function __construct(SimplePieAPIFactory $apiFactory) { + public function __construct(SimplePieAPIFactory $apiFactory, Config $config) { $this->apiFactory = $apiFactory; + $this->config = $config; } @@ -80,7 +81,7 @@ class FaviconFetcher { return null; } - $file = $this->apiFactory->getFile($url); + $file = $this->getFile($url); if($file->body !== '') { $document = new \DOMDocument(); @@ -99,6 +100,19 @@ class FaviconFetcher { } } + + private function getFile($url) { + if(trim($this->config->getProxyHost()) === '') { + return $this->apiFactory->getFile($url, 10, 5, null, null, false, + null, null, null); + } else { + return $this->apiFactory->getFile($url, 10, 5, null, null, false, + $this->config->getProxyHost(), + $this->config->getProxyPort(), + $this->config->getProxyAuth()); + } + } + /** * Test if the file is an image @@ -111,7 +125,7 @@ class FaviconFetcher { return false; } - $file = $this->apiFactory->getFile($url); + $file = $this->getFile($url); $sniffer = new \SimplePie_Content_Type_Sniffer($file); return $sniffer->image() !== false; } diff --git a/utility/simplepieapifactory.php b/utility/simplepieapifactory.php index 5014838d1..65f482095 100644 --- a/utility/simplepieapifactory.php +++ b/utility/simplepieapifactory.php @@ -34,10 +34,12 @@ class SimplePieAPIFactory { * @return SimplePie_File a new object */ public function getFile($url, $timeout=10, $redirects=5, $headers=null, - $useragent=null, $force_fsockopen=false) { + $useragent=null, $force_fsockopen=false, + $proxyHost=null, $proxyPort=null, $proxyAuth=null) { return new \SimplePie_File($url, $timeout, $redirects, $headers, - $useragent, $force_fsockopen); + $useragent, $force_fsockopen, $proxyHost, $proxyPort, + $proxyAuth); } |