diff options
Diffstat (limited to '3rdparty/fguillot/picofeed/lib/PicoFeed/Client')
12 files changed, 1942 insertions, 0 deletions
diff --git a/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/Client.php b/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/Client.php new file mode 100644 index 000000000..7328b2c75 --- /dev/null +++ b/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/Client.php @@ -0,0 +1,540 @@ +<?php + +namespace PicoFeed\Client; + +use LogicException; +use PicoFeed\Logging\Logging; + +/** + * Client class + * + * @author Frederic Guillot + * @package client + */ +abstract class Client +{ + /** + * Flag that say if the resource have been modified + * + * @access private + * @var bool + */ + private $is_modified = true; + + /** + * HTTP encoding + * + * @access private + * @var string + */ + private $encoding = ''; + + /** + * HTTP Etag header + * + * @access protected + * @var string + */ + protected $etag = ''; + + /** + * HTTP Last-Modified header + * + * @access protected + * @var string + */ + protected $last_modified = ''; + + /** + * Proxy hostname + * + * @access protected + * @var string + */ + protected $proxy_hostname = ''; + + /** + * Proxy port + * + * @access protected + * @var integer + */ + protected $proxy_port = 3128; + + /** + * Proxy username + * + * @access protected + * @var string + */ + protected $proxy_username = ''; + + /** + * Proxy password + * + * @access protected + * @var string + */ + protected $proxy_password = ''; + + /** + * Client connection timeout + * + * @access protected + * @var integer + */ + protected $timeout = 10; + + /** + * User-agent + * + * @access protected + * @var string + */ + protected $user_agent = 'PicoFeed (https://github.com/fguillot/picoFeed)'; + + /** + * Real URL used (can be changed after a HTTP redirect) + * + * @access protected + * @var string + */ + protected $url = ''; + + /** + * Page/Feed content + * + * @access protected + * @var string + */ + protected $content = ''; + + /** + * Number maximum of HTTP redirections to avoid infinite loops + * + * @access protected + * @var integer + */ + protected $max_redirects = 5; + + /** + * Maximum size of the HTTP body response + * + * @access protected + * @var integer + */ + protected $max_body_size = 2097152; // 2MB + + /** + * Do the HTTP request + * + * @abstract + * @access public + * @return array + */ + abstract public function doRequest(); + + /** + * Get client instance: curl or stream driver + * + * @static + * @access public + * @return \PicoFeed\Client\Client + */ + public static function getInstance() + { + if (function_exists('curl_init')) { + return new Curl; + } + else if (ini_get('allow_url_fopen')) { + return new Stream; + } + + throw new LogicException('You must have "allow_url_fopen=1" or curl extension installed'); + } + + /** + * Perform the HTTP request + * + * @access public + * @param string $url URL + * @return Client + */ + public function execute($url = '') + { + if ($url !== '') { + $this->url = $url; + } + + Logging::setMessage(get_called_class().' Fetch URL: '.$this->url); + Logging::setMessage(get_called_class().' Etag provided: '.$this->etag); + Logging::setMessage(get_called_class().' Last-Modified provided: '.$this->last_modified); + + $response = $this->doRequest(); + + $this->handleNotModifiedResponse($response); + $this->handleNotFoundResponse($response); + $this->handleNormalResponse($response); + + return $this; + } + + /** + * Handle not modified response + * + * @access public + * @param array $response Client response + */ + public function handleNotModifiedResponse(array $response) + { + if ($response['status'] == 304) { + $this->is_modified = false; + } + else if ($response['status'] == 200) { + + $etag = $this->getHeader($response, 'ETag'); + $last_modified = $this->getHeader($response, 'Last-Modified'); + + if ($this->isPropertyEquals('etag', $etag) || $this->isPropertyEquals('last_modified', $last_modified)) { + $this->is_modified = false; + } + + $this->etag = $etag; + $this->last_modified = $last_modified; + } + + if ($this->is_modified === false) { + Logging::setMessage(get_called_class().' Resource not modified'); + } + } + + /** + * Handle not found response + * + * @access public + * @param array $response Client response + */ + public function handleNotFoundResponse(array $response) + { + if ($response['status'] == 404) { + throw new InvalidUrlException('Resource not found'); + } + } + + /** + * Handle normal response + * + * @access public + * @param array $response Client response + */ + public function handleNormalResponse(array $response) + { + if ($response['status'] == 200) { + $this->content = $response['body']; + $this->encoding = $this->findCharset($response); + } + } + + /** + * Check if a class property equals to a value + * + * @access public + * @param string $property Class property + * @param string $value Value + * @return boolean + */ + private function isPropertyEquals($property, $value) + { + return $this->$property && $this->$property === $value; + } + + /** + * Find charset from response headers + * + * @access public + * @param array $response Client response + */ + public function findCharset(array $response) + { + $result = explode('charset=', strtolower($this->getHeader($response, 'Content-Type'))); + return isset($result[1]) ? $result[1] : ''; + } + + /** + * Get header value from a client response + * + * @access public + * @param array $response Client response + * @param string $header Header name + * @return string + */ + public function getHeader(array $response, $header) + { + return isset($response['headers'][$header]) ? $response['headers'][$header] : ''; + } + + /** + * Parse HTTP headers + * + * @access public + * @param array $lines List of headers + * @return array + */ + public function parseHeaders(array $lines) + { + $status = 200; + $headers = array(); + + foreach ($lines as $line) { + + if (strpos($line, 'HTTP') === 0) { + $status = (int) substr($line, 9, 3); + } + else if (strpos($line, ':') !== false) { + + @list($name, $value) = explode(': ', $line); + if ($value) $headers[trim($name)] = trim($value); + } + } + + Logging::setMessage(get_called_class().' HTTP status code: '.$status); + + foreach ($headers as $name => $value) { + Logging::setMessage(get_called_class().' HTTP header: '.$name.' => '.$value); + } + + return array($status, $headers); + } + + /** + * Set the Last-Modified HTTP header + * + * @access public + * @param string $last_modified Header value + * @return \PicoFeed\Client\Client + */ + public function setLastModified($last_modified) + { + $this->last_modified = $last_modified; + return $this; + } + + /** + * Get the value of the Last-Modified HTTP header + * + * @access public + * @return string + */ + public function getLastModified() + { + return $this->last_modified; + } + + /** + * Set the value of the Etag HTTP header + * + * @access public + * @param string $etag Etag HTTP header value + * @return \PicoFeed\Client\Client + */ + public function setEtag($etag) + { + $this->etag = $etag; + return $this; + } + + /** + * Get the Etag HTTP header value + * + * @access public + * @return string + */ + public function getEtag() + { + return $this->etag; + } + + /** + * Get the final url value + * + * @access public + * @return string + */ + public function getUrl() + { + return $this->url; + } + + /** + * Set the url + * + * @access public + * @return string + * @return \PicoFeed\Client\Client + */ + public function setUrl($url) + { + $this->url = $url; + return $this; + } + + /** + * Get the body of the HTTP response + * + * @access public + * @return string + */ + public function getContent() + { + return $this->content; + } + + /** + * Get the encoding value from HTTP headers + * + * @access public + * @return string + */ + public function getEncoding() + { + return $this->encoding; + } + + /** + * Return true if the remote resource has changed + * + * @access public + * @return bool + */ + public function isModified() + { + return $this->is_modified; + } + + /** + * Set connection timeout + * + * @access public + * @param integer $timeout Connection timeout + * @return \PicoFeed\Client\Client + */ + public function setTimeout($timeout) + { + $this->timeout = $timeout ?: $this->timeout; + return $this; + } + + /** + * Set a custom user agent + * + * @access public + * @param string $user_agent User Agent + * @return \PicoFeed\Client\Client + */ + public function setUserAgent($user_agent) + { + $this->user_agent = $user_agent ?: $this->user_agent; + return $this; + } + + /** + * Set the mximum number of HTTP redirections + * + * @access public + * @param integer $max Maximum + * @return \PicoFeed\Client\Client + */ + public function setMaxRedirections($max) + { + $this->max_redirects = $max ?: $this->max_redirects; + return $this; + } + + /** + * Set the maximum size of the HTTP body + * + * @access public + * @param integer $max Maximum + * @return \PicoFeed\Client\Client + */ + public function setMaxBodySize($max) + { + $this->max_body_size = $max ?: $this->max_body_size; + return $this; + } + + /** + * Set the proxy hostname + * + * @access public + * @param string $hostname Proxy hostname + * @return \PicoFeed\Client\Client + */ + public function setProxyHostname($hostname) + { + $this->proxy_hostname = $hostname ?: $this->proxy_hostname; + return $this; + } + + /** + * Set the proxy port + * + * @access public + * @param integer $port Proxy port + * @return \PicoFeed\Client\Client + */ + public function setProxyPort($port) + { + $this->proxy_port = $port ?: $this->proxy_port; + return $this; + } + + /** + * Set the proxy username + * + * @access public + * @param string $username Proxy username + * @return \PicoFeed\Client\Client + */ + public function setProxyUsername($username) + { + $this->proxy_username = $username ?: $this->proxy_username; + return $this; + } + + /** + * Set the proxy password + * + * @access public + * @param string $password Password + * @return \PicoFeed\Client\Client + */ + public function setProxyPassword($password) + { + $this->proxy_password = $password ?: $this->proxy_password; + return $this; + } + + /** + * Set config object + * + * @access public + * @param \PicoFeed\Config\Config $config Config instance + * @return \PicoFeed\Config\Config + */ + public function setConfig($config) + { + if ($config !== null) { + $this->setTimeout($config->getGrabberTimeout()); + $this->setUserAgent($config->getGrabberUserAgent()); + $this->setMaxRedirections($config->getMaxRedirections()); + $this->setMaxBodySize($config->getMaxBodySize()); + $this->setProxyHostname($config->getProxyHostname()); + $this->setProxyPort($config->getProxyPort()); + $this->setProxyUsername($config->getProxyUsername()); + $this->setProxyPassword($config->getProxyPassword()); + } + + return $this; + } +} diff --git a/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/ClientException.php b/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/ClientException.php new file mode 100644 index 000000000..0e27452ed --- /dev/null +++ b/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/ClientException.php @@ -0,0 +1,16 @@ +<?php + +namespace PicoFeed\Client; + +use PicoFeed\PicoFeedException; + + +/** + * ClientException Exception + * + * @author Frederic Guillot + * @package Client + */ +abstract class ClientException extends PicoFeedException +{ +} diff --git a/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/Curl.php b/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/Curl.php new file mode 100644 index 000000000..9cf3eb6f4 --- /dev/null +++ b/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/Curl.php @@ -0,0 +1,324 @@ +<?php + +namespace PicoFeed\Client; + +use PicoFeed\Logging\Logging; + +/** + * cURL HTTP client + * + * @author Frederic Guillot + * @package Client + */ +class Curl extends Client +{ + /** + * HTTP response body + * + * @access private + * @var string + */ + private $body = ''; + + /** + * Body size + * + * @access private + * @var integer + */ + private $body_length = 0; + + /** + * HTTP response headers + * + * @access private + * @var array + */ + private $headers = array(); + + /** + * Counter on the number of header received + * + * @access private + * @var integer + */ + private $headers_counter = 0; + + /** + * cURL callback to read the HTTP body + * + * If the function return -1, curl stop to read the HTTP response + * + * @access public + * @param resource $ch cURL handler + * @param string $buffer Chunk of data + * @return integer Length of the buffer + */ + public function readBody($ch, $buffer) + { + $length = strlen($buffer); + $this->body_length += $length; + + if ($this->body_length > $this->max_body_size) { + return -1; + } + + $this->body .= $buffer; + + return $length; + } + + /** + * cURL callback to read HTTP headers + * + * @access public + * @param resource $ch cURL handler + * @param string $buffer Header line + * @return integer Length of the buffer + */ + public function readHeaders($ch, $buffer) + { + $length = strlen($buffer); + + if ($buffer === "\r\n") { + $this->headers_counter++; + } + else { + + if (! isset($this->headers[$this->headers_counter])) { + $this->headers[$this->headers_counter] = ''; + } + + $this->headers[$this->headers_counter] .= $buffer; + } + + return $length; + } + + /** + * Prepare HTTP headers + * + * @access private + * @return array + */ + private function prepareHeaders() + { + $headers = array( + 'Connection: close', + 'User-Agent: '.$this->user_agent, + ); + + if ($this->etag) { + $headers[] = 'If-None-Match: '.$this->etag; + } + + if ($this->last_modified) { + $headers[] = 'If-Modified-Since: '.$this->last_modified; + } + + return $headers; + } + + /** + * Prepare curl proxy context + * + * @access private + * @return resource + */ + private function prepareProxyContext($ch) + { + if ($this->proxy_hostname) { + + Logging::setMessage(get_called_class().' Proxy: '.$this->proxy_hostname.':'.$this->proxy_port); + + curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxy_port); + curl_setopt($ch, CURLOPT_PROXYTYPE, 'HTTP'); + curl_setopt($ch, CURLOPT_PROXY, $this->proxy_hostname); + + if ($this->proxy_username) { + Logging::setMessage(get_called_class().' Proxy credentials: Yes'); + curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxy_username.':'.$this->proxy_password); + } + else { + Logging::setMessage(get_called_class().' Proxy credentials: No'); + } + } + + return $ch; + } + + /** + * Prepare curl context + * + * @access private + * @return resource + */ + private function prepareContext() + { + $ch = curl_init(); + + curl_setopt($ch, CURLOPT_URL, $this->url); + curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $this->timeout); + curl_setopt($ch, CURLOPT_TIMEOUT, $this->timeout); + curl_setopt($ch, CURLOPT_HTTPHEADER, $this->prepareHeaders()); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, ini_get('open_basedir') === ''); + curl_setopt($ch, CURLOPT_MAXREDIRS, $this->max_redirects); + curl_setopt($ch, CURLOPT_ENCODING, ''); + curl_setopt($ch, CURLOPT_WRITEFUNCTION, array($this, 'readBody')); + curl_setopt($ch, CURLOPT_HEADERFUNCTION, array($this, 'readHeaders')); + curl_setopt($ch, CURLOPT_COOKIEJAR, 'php://memory'); + curl_setopt($ch, CURLOPT_COOKIEFILE, 'php://memory'); + + $ch = $this->prepareProxyContext($ch); + + return $ch; + } + + /** + * Execute curl context + * + * @access private + */ + private function executeContext() + { + $ch = $this->prepareContext(); + curl_exec($ch); + + Logging::setMessage(get_called_class().' cURL total time: '.curl_getinfo($ch, CURLINFO_TOTAL_TIME)); + Logging::setMessage(get_called_class().' cURL dns lookup time: '.curl_getinfo($ch, CURLINFO_NAMELOOKUP_TIME)); + Logging::setMessage(get_called_class().' cURL connect time: '.curl_getinfo($ch, CURLINFO_CONNECT_TIME)); + Logging::setMessage(get_called_class().' cURL speed download: '.curl_getinfo($ch, CURLINFO_SPEED_DOWNLOAD)); + Logging::setMessage(get_called_class().' cURL effective url: '.curl_getinfo($ch, CURLINFO_EFFECTIVE_URL)); + + $curl_errno = curl_errno($ch); + + if ($curl_errno) { + Logging::setMessage(get_called_class().' cURL error: '.curl_error($ch)); + curl_close($ch); + + $this->handleError($curl_errno); + } + + curl_close($ch); + } + + /** + * Do the HTTP request + * + * @access public + * @param bool $follow_location Flag used when there is an open_basedir restriction + * @return array HTTP response ['body' => ..., 'status' => ..., 'headers' => ...] + */ + public function doRequest($follow_location = true) + { + $this->executeContext(); + + list($status, $headers) = $this->parseHeaders(explode("\r\n", $this->headers[$this->headers_counter - 1])); + + // When resticted with open_basedir + if ($this->needToHandleRedirection($follow_location, $status)) { + return $this->handleRedirection($headers['Location']); + } + + return array( + 'status' => $status, + 'body' => $this->body, + 'headers' => $headers + ); + } + + /** + * Check if the redirection have to be handled manually + * + * @access private + * @param boolean $follow_location Flag + * @param integer $status HTTP status code + * @return boolean + */ + private function needToHandleRedirection($follow_location, $status) + { + return $follow_location && ini_get('open_basedir') !== '' && ($status == 301 || $status == 302); + } + + /** + * Handle manually redirections when there is an open base dir restriction + * + * @access private + * @param string $location Redirected URL + * @return boolean|array + */ + private function handleRedirection($location) + { + $nb_redirects = 0; + $this->url = $location; + $this->body = ''; + $this->body_length = 0; + $this->headers = array(); + $this->headers_counter = 0; + + while (true) { + + $nb_redirects++; + + if ($nb_redirects >= $this->max_redirects) { + return false; + } + + $result = $this->doRequest(false); + + if ($result['status'] == 301 || $result['status'] == 302) { + $this->url = $result['headers']['Location']; + $this->body = ''; + $this->body_length = 0; + $this->headers = array(); + $this->headers_counter = 0; + } + else { + return $result; + } + } + + return false; + } + + /** + * Handle cURL errors (throw individual exceptions) + * + * We don't use constants because they are not necessary always available + * (depends of the version of libcurl linked to php) + * + * @see http://curl.haxx.se/libcurl/c/libcurl-errors.html + * @access private + * @param integer $errno cURL error code + */ + private function handleError($errno) + { + switch ($errno) { + case 78: // CURLE_REMOTE_FILE_NOT_FOUND + throw new InvalidUrlException('Resource not found'); + case 6: // CURLE_COULDNT_RESOLVE_HOST + throw new InvalidUrlException('Unable to resolve hostname'); + case 7: // CURLE_COULDNT_CONNECT + throw new InvalidUrlException('Unable to connect to the remote host'); + case 28: // CURLE_OPERATION_TIMEDOUT + throw new TimeoutException('Operation timeout'); + case 35: // CURLE_SSL_CONNECT_ERROR + case 51: // CURLE_PEER_FAILED_VERIFICATION + case 58: // CURLE_SSL_CERTPROBLEM + case 60: // CURLE_SSL_CACERT + case 59: // CURLE_SSL_CIPHER + case 64: // CURLE_USE_SSL_FAILED + case 66: // CURLE_SSL_ENGINE_INITFAILED + case 77: // CURLE_SSL_CACERT_BADFILE + case 83: // CURLE_SSL_ISSUER_ERROR + throw new InvalidCertificateException('Invalid SSL certificate'); + case 47: // CURLE_TOO_MANY_REDIRECTS + throw new MaxRedirectException('Maximum number of redirections reached'); + case 63: // CURLE_FILESIZE_EXCEEDED + throw new MaxSizeException('Maximum response size exceeded'); + default: + throw new InvalidUrlException('Unable to fetch the URL'); + } + } +} diff --git a/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/Favicon.php b/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/Favicon.php new file mode 100644 index 000000000..b6d3b6d26 --- /dev/null +++ b/3rdparty/fguillot/picofeed/lib/PicoFeed/Client/Favicon.php @@ -0,0 +1,170 @@ +<?php + +namespace PicoFeed\Client; + +use DOMXpath; + +use PicoFeed\Config\Config; +use PicoFeed\Logging\Logging; +use PicoFeed\Parser\XmlParser; + +/** + * Favicon class + * + * https://en.wikipedia.org/wiki/Favicon + * + * @author Frederic Guillot + * @package Client + */ +class Favicon +{ + /** + * Config class instance + * + * @access private + * @var \PicoFeed\Config\Config + */ + private $config; + + /** + * Icon content + * + * @access private + * @var string + */ + private $content = ''; + + /** + * Constructor + * + * @access public + * @param \PicoFeed\Config\Config $config Config class instance + */ + public function __construct(Config $config = null) + { + $this->config = $config ?: new Config; + } + + /** + * Get the icon file content (available only after the download) + * + * @access public + * @return string + */ + public function getContent() + { + return $this->content; + } + + /** + * Download and check if a resource exists + * + * @access public + * @param string $url URL + * @return string Resource content + */ + public function download($url) + { + try { + + Logging::setMessage(get_called_class().' Download => '.$url); + + $client = Client::getInstance(); + $client->setConfig($this->config); + $client->execute($url); + + return $client->getContent(); + } + catch (ClientException $e) { + return ''; + } + } + + /** + * Check if a remote file exists + * + * @access public + * @param string $url URL + * @return boolean + */ + public function exists($url) + { + return $this->download($url) !== ''; + } + + /** + * Get the icon link for a website + * + * @access public + * @param string $website_link URL + * @return string + */ + public function find($website_link) + { + $website = new Url($website_link); + + $icons = $this->extract($this->download($website->getBaseUrl('/'))); + $icons[] = $website->getBaseUrl('/favicon.ico'); + + foreach ($icons as $icon_link) { + + $icon_link = $this->convertLink($website, new Url($icon_link)); + $this->content = $this->download($icon_link); + + if ($this->content !== '') { + return $icon_link; + } + } + + return ''; + } + + /** + * Convert icon links to absolute url + * + * @access public + * @param \PicoFeed\Client\Url $website Website url + * @param \PicoFeed\Client\Url $icon Icon url + * @return string + */ + public function convertLink(Url $website, Url $icon) + |