diff options
author | Bernhard Posselt <dev@bernhard-posselt.com> | 2015-11-21 11:07:43 +0100 |
---|---|---|
committer | Bernhard Posselt <dev@bernhard-posselt.com> | 2015-11-21 11:08:04 +0100 |
commit | f99a84fd78688fa4132fa8e9ca9bb9243036a572 (patch) | |
tree | 0aa58fc6e4efd4e05a577f4da197bff7fdc358ba /vendor/fguillot/picofeed/lib | |
parent | 12b3f840b194ea762afe4f4d25b336b4f3e383b0 (diff) |
update picofeed
Diffstat (limited to 'vendor/fguillot/picofeed/lib')
11 files changed, 144 insertions, 47 deletions
diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Client/Client.php b/vendor/fguillot/picofeed/lib/PicoFeed/Client/Client.php index 3c64ce5fb..24c619cdf 100644 --- a/vendor/fguillot/picofeed/lib/PicoFeed/Client/Client.php +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Client/Client.php @@ -496,7 +496,7 @@ abstract class Client } /** - * Set the mximum number of HTTP redirections. + * Set the maximum number of HTTP redirections. * * @param int $max Maximum * diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Client/Curl.php b/vendor/fguillot/picofeed/lib/PicoFeed/Client/Curl.php index b07685fcb..82ea87a44 100644 --- a/vendor/fguillot/picofeed/lib/PicoFeed/Client/Curl.php +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Client/Curl.php @@ -89,27 +89,6 @@ class Curl extends Client } /** - * cURL callback to passthrough the HTTP status header to the client. - * - * @param resource $ch cURL handler - * @param string $buffer Header line - * - * @return int Length of the buffer - */ - public function passthroughHeaders($ch, $buffer) - { - list($status, $headers) = HttpHeaders::parse(array($buffer)); - - if ($status !== 0) { - header(':', true, $status); - } elseif (isset($headers['Content-Type'])) { - header($buffer); - } - - return $this->readHeaders($ch, $buffer); - } - - /** * cURL callback to passthrough the HTTP body to the client. * * If the function return -1, curl stop to read the HTTP response @@ -121,9 +100,27 @@ class Curl extends Client */ public function passthroughBody($ch, $buffer) { + // do it only at the beginning of a transmission + if ($this->body_length === 0) { + list($status, $headers) = HttpHeaders::parse(explode("\n", $this->response_headers[$this->response_headers_count - 1])); + + if ($this->isRedirection($status)) { + return $this->handleRedirection($headers['Location']); + } + + header($status); + + if (isset($headers['Content-Type'])) { + header('Content-Type:' .$headers['Content-Type']); + } + } + + $length = strlen($buffer); + $this->body_length += $length; + echo $buffer; - return strlen($buffer); + return $length; } /** @@ -207,7 +204,6 @@ class Curl extends Client if ($this->isPassthroughEnabled()) { $write_function = 'passthroughBody'; - $header_function = 'passthroughHeaders'; } curl_setopt($ch, CURLOPT_WRITEFUNCTION, array($this, $write_function)); @@ -285,17 +281,15 @@ class Curl extends Client /** * Do the HTTP request. * - * @param bool $follow_location Flag used when there is an open_basedir restriction - * * @return array HTTP response ['body' => ..., 'status' => ..., 'headers' => ...] */ - public function doRequest($follow_location = true) + public function doRequest() { $this->executeContext(); list($status, $headers) = HttpHeaders::parse(explode("\n", $this->response_headers[$this->response_headers_count - 1])); - if ($follow_location && $this->isRedirection($status)) { + if ($this->isRedirection($status)) { return $this->handleRedirection($headers['Location']); } @@ -307,7 +301,7 @@ class Curl extends Client } /** - * Handle manually redirections when there is an open base dir restriction. + * Handle HTTP redirects * * @param string $location Redirected URL * @@ -330,7 +324,7 @@ class Curl extends Client throw new MaxRedirectException('Maximum number of redirections reached'); } - $result = $this->doRequest(false); + $result = $this->doRequest(); if ($this->isRedirection($result['status'])) { $this->url = Url::resolve($result['headers']['Location'], $this->url); diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/bizjournals.com.php b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/bizjournals.com.php index 44a644a91..6d8a5a91c 100644 --- a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/bizjournals.com.php +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/bizjournals.com.php @@ -5,9 +5,8 @@ return array( '%.*%' => array( 'test_url' => 'http://www.bizjournals.com/milwaukee/news/2015/09/30/bucks-will-hike-prices-on-best-seats-at-new-arena.html', 'body' => array( - '//p[@class="media__caption"]', - '//figure/div/a/img', - '//p[@class="content__segment"]', + '//figure/div/a/img', + '//p[@class="content__segment"]', ), ), ), diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/cnet.com.php b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/cnet.com.php index fdffefd35..402196840 100644 --- a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/cnet.com.php +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/cnet.com.php @@ -2,6 +2,23 @@ return array( 'grabber' => array( + '%^/products.*%' => array( + 'test_url' => 'http://www.cnet.com/products/fibaro-flood-sensor/#ftag=CADf328eec', + 'body' => array( + '//li[contains(@class,"slide first"] || //figure[contains(@class,(promoFigure))]', + '//div[@class="quickInfo"]', + '//div[@class="col-6 ratings"]', + '//div[@id="editorReview"]', + ), + 'strip' => array( + '//script', + '//a[@class="clickToEnlarge"]', + '//div[@section="topSharebar"]', + '//div[contains(@class,"related")]', + '//div[contains(@class,"ad-")]', + '//div[@section="shortcodeGallery"]', + ), + ), '%.*%' => array( 'test_url' => 'http://cnet.com.feedsportal.com/c/34938/f/645093/s/4a340866/sc/28/l/0L0Scnet0N0Cnews0Cman0Eclaims0Eonline0Epsychic0Emade0Ehim0Ebuy0E10Emillion0Epowerball0Ewinning0Eticket0C0Tftag0FCAD590Aa51e/story01.htm', 'body' => array( @@ -9,12 +26,12 @@ return array( '//div[@itemprop="articleBody"]', ), 'strip' => array( - '//script', - '//a[@class="clickToEnlarge"]', - '//div[@section="topSharebar"]', - '//div[contains(@class,"related")]', - '//div[contains(@class,"ad-")]', - '//div[@section="shortcodeGallery"]', + '//script', + '//a[@class="clickToEnlarge"]', + '//div[@section="topSharebar"]', + '//div[contains(@class,"related")]', + '//div[contains(@class,"ad-")]', + '//div[@section="shortcodeGallery"]', ), ), ), diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/nature.com.php b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/nature.com.php new file mode 100644 index 000000000..1dad8b66f --- /dev/null +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/nature.com.php @@ -0,0 +1,20 @@ +<?php +return array( + 'grabber' => array( + 'http://dx.doi.org/10.1038.*%' => array( + 'test_url' => 'http://dx.doi.org/10.1038/525184a', + 'body' => array( + '//div[@class="content "]', + ), + 'strip' => array() + ), + '%.*%' => array( + 'test_url' => 'http://www.nature.com/doifinder/10.1038/nature.2015.18340', + 'body' => array( + '//div[contains(@class,"main-content")]', + ), + 'strip' => array() + ), + ) +); + diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/networkworld.com.php b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/networkworld.com.php index dbb63a28a..94f346ee6 100644 --- a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/networkworld.com.php +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/networkworld.com.php @@ -7,10 +7,13 @@ return array( 'body' => array( '//figure/img[@class="hero-img"]', '//section[@class="deck"]', - '//div[@itemprop="articleBody"]', + '//div[@itemprop="articleBody"] || //div[@itemprop="reviewBody"]', + '//div[@class="carousel-inside-crop"]', ), 'strip' => array( '//aside', + '//div[@class="credit"]', + '//div[@class="view-large"]', ), ), ), diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/news.sciencemag.org.php b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/news.sciencemag.org.php new file mode 100644 index 000000000..9b572ef4b --- /dev/null +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/news.sciencemag.org.php @@ -0,0 +1,18 @@ +<?php +return array( + 'grabber' => array( + '%.*%' => array( + 'test_url' => 'http://news.sciencemag.org/biology/2015/09/genetic-engineering-turns-common-plant-cancer-fighter', + 'body' => array( + '//div[@class="content"]', + ), + 'strip' => array( + '//h1[@class="snews-article__headline"]', + '//div[contains(@class,"easy_social_box")]', + '//div[@class="author-teaser"]', + '//div[@class="article-byline"]', + ), + ), + ) +); + diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/retractionwatch.com.php b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/retractionwatch.com.php new file mode 100644 index 000000000..b97c73eda --- /dev/null +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/retractionwatch.com.php @@ -0,0 +1,18 @@ +<?php +return array( + 'grabber' => array( + '%.*%' => array( + 'test_url' => 'http://retractionwatch.com/2015/11/12/psychologist-jens-forster-settles-case-by-agreeing-to-2-retractions/', + 'body' => array( + '//*[@class="main"]', + '//*[@class="entry-content"]', + ), + 'strip' => array( + '//*[contains(@class, "sharedaddy")]', + '//*[contains(@class, "jp-relatedposts")]', + '//p[@class="p1"]', + ) + ) + ) +); + diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/thelocal.se.php b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/thelocal.se.php new file mode 100644 index 000000000..964850fa4 --- /dev/null +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/thelocal.se.php @@ -0,0 +1,21 @@ +<?php +return array( + 'grabber' => array( + '%.*%' => array( + 'test_url' => 'http://www.thelocal.se/20151018/swedish-moderates-tighten-focus-on-begging-ban', + 'body' => array( + '//article', + ), + 'strip' => array( + '//p[@id="mobile-signature"]', + '//article/div[4]', + '//article/ul[1]', + '//div[@class="clr"]', + '//p[@class="small"]', + '//p[@style="font-weight: bold; font-size: 14px;"]', + '//div[@class="author"]', + ) + ) + ) +); + diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/upi.com.php b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/upi.com.php index 413a1720b..5d78f8b93 100644 --- a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/upi.com.php +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/upi.com.php @@ -6,16 +6,10 @@ return array( 'test_url' => 'http://www.upi.com/Top_News/US/2015/09/26/Tech-giants-Hollywood-stars-among-guests-at-state-dinner-for-Chinas-Xi-Jinping/4541443281006/', 'body' => array( '//div[@class="img"]', - '//div[@class="st_text_c"]', + '//div/article[@itemprop="articleBody"]', ), 'strip' => array( '//div[@align="center"]', - '//div[@class="ad_slot"]', - '//div[@class="ipara"]', - '//div[@class="st_embed"]', - '//div[contains(@styel,"font-size"]', - '//ul', - '//style[@type="text/css"]', ), ), ), diff --git a/vendor/fguillot/picofeed/lib/PicoFeed/Rules/www.geekculture.com.php b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/www.geekculture.com.php new file mode 100644 index 000000000..7f03a1df3 --- /dev/null +++ b/vendor/fguillot/picofeed/lib/PicoFeed/Rules/www.geekculture.com.php @@ -0,0 +1,13 @@ +<?php +return array( + 'grabber' => array( + '%.*%' => array( + 'test_url' => 'http://www.geekculture.com/joyoftech/joyarchives/2180.html', + 'body' => array( + '//p[contains(@class,"Maintext")][2]/img', + ), + 'strip' => array(), + ), + ), +); + |