summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--app/controllers/api/push_controller.rb2
-rw-r--r--app/controllers/authorize_follow_controller.rb2
-rw-r--r--app/lib/tag_manager.rb8
-rw-r--r--app/models/account.rb31
-rw-r--r--app/models/domain_block.rb8
-rw-r--r--app/models/media_attachment.rb2
-rw-r--r--app/services/fetch_link_card_service.rb2
-rw-r--r--app/services/fetch_remote_account_service.rb2
-rw-r--r--app/services/fetch_remote_status_service.rb2
-rw-r--r--app/services/follow_remote_account_service.rb2
-rw-r--r--app/services/process_feed_service.rb6
-rw-r--r--app/services/process_interaction_service.rb2
-rw-r--r--app/services/pubsubhubbub/subscribe_service.rb2
-rw-r--r--app/validators/url_validator.rb2
-rw-r--r--app/workers/pubsubhubbub/delivery_worker.rb2
-rw-r--r--spec/fixtures/requests/idn.txt483
-rw-r--r--spec/services/fetch_link_card_service_spec.rb14
17 files changed, 546 insertions, 26 deletions
diff --git a/app/controllers/api/push_controller.rb b/app/controllers/api/push_controller.rb
index f2ddfd969ee..bc547d6530b 100644
--- a/app/controllers/api/push_controller.rb
+++ b/app/controllers/api/push_controller.rb
@@ -26,7 +26,7 @@ class Api::PushController < ApiController
def topic_to_account(topic_url)
return if topic_url.blank?
- uri = Addressable::URI.parse(topic_url)
+ uri = Addressable::URI.parse(topic_url).normalize
params = Rails.application.routes.recognize_path(uri.path)
domain = uri.host + (uri.port ? ":#{uri.port}" : '')
diff --git a/app/controllers/authorize_follow_controller.rb b/app/controllers/authorize_follow_controller.rb
index c98a5f45f5d..9b28a9455a0 100644
--- a/app/controllers/authorize_follow_controller.rb
+++ b/app/controllers/authorize_follow_controller.rb
@@ -6,7 +6,7 @@ class AuthorizeFollowController < ApplicationController
before_action :authenticate_user!
def new
- uri = Addressable::URI.parse(acct_param)
+ uri = Addressable::URI.parse(acct_param).normalize
if uri.path && %w(http https).include?(uri.scheme)
set_account_from_url
diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb
index f26c943d256..3bddfba7c28 100644
--- a/app/lib/tag_manager.rb
+++ b/app/lib/tag_manager.rb
@@ -64,6 +64,12 @@ class TagManager
domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.local_domain).zero?
end
+ def normalize_domain(domain)
+ uri = Addressable::URI.new
+ uri.host = domain
+ uri.normalize.host
+ end
+
def same_acct?(canonical, needle)
return true if canonical.casecmp(needle).zero?
username, domain = needle.split('@')
@@ -71,7 +77,7 @@ class TagManager
end
def local_url?(url)
- uri = Addressable::URI.parse(url)
+ uri = Addressable::URI.parse(url).normalize
domain = uri.host + (uri.port ? ":#{uri.port}" : '')
TagManager.instance.local_domain?(domain)
end
diff --git a/app/models/account.rb b/app/models/account.rb
index 084b17f43dc..eebcf90b8f6 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -182,22 +182,22 @@ class Account < ApplicationRecord
end
def avatar_remote_url=(url)
- parsed_url = URI.parse(url)
+ parsed_url = Addressable::URI.parse(url).normalize
return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty? || self[:avatar_remote_url] == url
- self.avatar = parsed_url
+ self.avatar = URI.parse(parsed_url.to_s)
self[:avatar_remote_url] = url
rescue OpenURI::HTTPError => e
Rails.logger.debug "Error fetching remote avatar: #{e}"
end
def header_remote_url=(url)
- parsed_url = URI.parse(url)
+ parsed_url = Addressable::URI.parse(url).normalize
return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty? || self[:header_remote_url] == url
- self.header = parsed_url
+ self.header = URI.parse(parsed_url.to_s)
self[:header_remote_url] = url
rescue OpenURI::HTTPError => e
Rails.logger.debug "Error fetching remote header: #{e}"
@@ -331,16 +331,25 @@ class Account < ApplicationRecord
end
end
- before_create do
- if local?
- keypair = OpenSSL::PKey::RSA.new(Rails.env.test? ? 1024 : 2048)
- self.private_key = keypair.to_pem
- self.public_key = keypair.public_key.to_pem
- end
- end
+ before_create :generate_keys
+ before_validation :normalize_domain
private
+ def generate_keys
+ return unless local?
+
+ keypair = OpenSSL::PKey::RSA.new(Rails.env.test? ? 1024 : 2048)
+ self.private_key = keypair.to_pem
+ self.public_key = keypair.public_key.to_pem
+ end
+
+ def normalize_domain
+ return if local?
+
+ self.domain = TagManager.instance.normalize_domain(domain)
+ end
+
def set_file_extensions
unless avatar.blank?
extension = Paperclip::Interpolations.content_type_extension(avatar, :original)
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index baf5c3973bd..6c2ba70f769 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -13,4 +13,12 @@ class DomainBlock < ApplicationRecord
def self.blocked?(domain)
where(domain: domain, severity: :suspend).exists?
end
+
+ before_validation :normalize_domain
+
+ private
+
+ def normalize_domain
+ self.domain = TagManager.instance.normalize_domain(domain)
+ end
end
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 181e01674f3..a43c76c77ce 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -42,7 +42,7 @@ class MediaAttachment < ApplicationRecord
end
def file_remote_url=(url)
- self.file = URI.parse(url)
+ self.file = URI.parse(Addressable::URI.parse(url).normalize.to_s)
end
def to_param
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 31f9be194eb..f74a0d34dc2 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -19,7 +19,7 @@ class FetchLinkCardService < BaseService
card.title = meta_property(page, 'og:title') || page.at_xpath('//title')&.content
card.description = meta_property(page, 'og:description') || meta_property(page, 'description')
- card.image = URI.parse(meta_property(page, 'og:image')) if meta_property(page, 'og:image')
+ card.image = URI.parse(Addressable::URI.parse(meta_property(page, 'og:image')).normalize.to_s) if meta_property(page, 'og:image')
return if card.title.blank?
diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb
index 50ffc47c66b..7c2fb0ef12f 100644
--- a/app/services/fetch_remote_account_service.rb
+++ b/app/services/fetch_remote_account_service.rb
@@ -21,7 +21,7 @@ class FetchRemoteAccountService < BaseService
email = xml.at_xpath('//xmlns:author/xmlns:email').try(:content)
if email.nil?
- url_parts = Addressable::URI.parse(url)
+ url_parts = Addressable::URI.parse(url).normalize
username = xml.at_xpath('//xmlns:author/xmlns:name').try(:content)
domain = url_parts.host
else
diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb
index e2d18572312..c666961ad28 100644
--- a/app/services/fetch_remote_status_service.rb
+++ b/app/services/fetch_remote_status_service.rb
@@ -31,7 +31,7 @@ class FetchRemoteStatusService < BaseService
end
def extract_author(url, xml)
- url_parts = Addressable::URI.parse(url)
+ url_parts = Addressable::URI.parse(url).normalize
username = xml.at_xpath('//xmlns:author/xmlns:name').try(:content)
domain = url_parts.host
diff --git a/app/services/follow_remote_account_service.rb b/app/services/follow_remote_account_service.rb
index 2d7414bc0ba..fbf983093ab 100644
--- a/app/services/follow_remote_account_service.rb
+++ b/app/services/follow_remote_account_service.rb
@@ -73,7 +73,7 @@ class FollowRemoteAccountService < BaseService
end
def get_feed(url)
- response = http_client.get(Addressable::URI.parse(url))
+ response = http_client.get(Addressable::URI.parse(url).normalize)
[response.to_s, Nokogiri::XML(response)]
end
diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb
index 98d92f6300c..d002b9130e4 100644
--- a/app/services/process_feed_service.rb
+++ b/app/services/process_feed_service.rb
@@ -174,7 +174,7 @@ class ProcessFeedService < BaseService
end
def account_from_href(href)
- url = Addressable::URI.parse(href)
+ url = Addressable::URI.parse(href).normalize
if TagManager.instance.web_domain?(url.host)
Account.find_local(url.path.gsub('/users/', ''))
@@ -195,7 +195,7 @@ class ProcessFeedService < BaseService
next unless link['href']
media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href'])
- parsed_url = URI.parse(link['href'])
+ parsed_url = Addressable::URI.parse(link['href']).normalize
next if !%w[http https].include?(parsed_url.scheme) || parsed_url.host.empty?
@@ -271,7 +271,7 @@ class ProcessFeedService < BaseService
def acct(xml = @xml)
username = xml.at_xpath('./xmlns:author/xmlns:name', xmlns: TagManager::XMLNS).content
url = xml.at_xpath('./xmlns:author/xmlns:uri', xmlns: TagManager::XMLNS).content
- domain = Addressable::URI.parse(url).host
+ domain = Addressable::URI.parse(url).normalize.host
"#{username}@#{domain}"
end
diff --git a/app/services/process_interaction_service.rb b/app/services/process_interaction_service.rb
index 805ca5a2735..410e805d345 100644
--- a/app/services/process_interaction_service.rb
+++ b/app/services/process_interaction_service.rb
@@ -14,7 +14,7 @@ class ProcessInteractionService < BaseService
username = xml.at_xpath('/xmlns:entry/xmlns:author/xmlns:name', xmlns: TagManager::XMLNS).content
url = xml.at_xpath('/xmlns:entry/xmlns:author/xmlns:uri', xmlns: TagManager::XMLNS).content
- domain = Addressable::URI.parse(url).host
+ domain = Addressable::URI.parse(url).normalize.host
account = Account.find_by(username: username, domain: domain)
if account.nil?
diff --git a/app/services/pubsubhubbub/subscribe_service.rb b/app/services/pubsubhubbub/subscribe_service.rb
index bf36e3fa6f5..3642b4eca10 100644
--- a/app/services/pubsubhubbub/subscribe_service.rb
+++ b/app/services/pubsubhubbub/subscribe_service.rb
@@ -4,7 +4,7 @@ class Pubsubhubbub::SubscribeService < BaseService
def call(account, callback, secret, lease_seconds)
return ['Invalid topic URL', 422] if account.nil?
return ['Invalid callback URL', 422] unless !callback.blank? && callback =~ /\A#{URI.regexp(%w(http https))}\z/
- return ['Callback URL not allowed', 403] if DomainBlock.blocked?(Addressable::URI.parse(callback).host)
+ return ['Callback URL not allowed', 403] if DomainBlock.blocked?(Addressable::URI.parse(callback).normalize.host)
subscription = Subscription.where(account: account, callback_url: callback).first_or_create!(account: account, callback_url: callback)
Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'subscribe', secret, lease_seconds)
diff --git a/app/validators/url_validator.rb b/app/validators/url_validator.rb
index 4a5c4ef3fff..f39560d90ac 100644
--- a/app/validators/url_validator.rb
+++ b/app/validators/url_validator.rb
@@ -8,7 +8,7 @@ class UrlValidator < ActiveModel::EachValidator
private
def compliant?(url)
- parsed_url = Addressable::URI.parse(url)
+ parsed_url = Addressable::URI.parse(url).normalize
!parsed_url.nil? && %w(http https).include?(parsed_url.scheme) && parsed_url.host
end
end
diff --git a/app/workers/pubsubhubbub/delivery_worker.rb b/app/workers/pubsubhubbub/delivery_worker.rb
index b440f7986e6..f645b1e24d0 100644
--- a/app/workers/pubsubhubbub/delivery_worker.rb
+++ b/app/workers/pubsubhubbub/delivery_worker.rb
@@ -13,7 +13,7 @@ class Pubsubhubbub::DeliveryWorker
def perform(subscription_id, payload)
subscription = Subscription.find(subscription_id)
headers = {}
- host = Addressable::URI.parse(subscription.callback_url).host
+ host = Addressable::URI.parse(subscription.callback_url).normalize.host
return if DomainBlock.blocked?(host)
diff --git a/spec/fixtures/requests/idn.txt b/spec/fixtures/requests/idn.txt
new file mode 100644
index 00000000000..3c76c59c05c
--- /dev/null
+++ b/spec/fixtures/requests/idn.txt
@@ -0,0 +1,483 @@
+HTTP/1.1 200 OK
+Server: nginx
+Date: Sun, 23 Apr 2017 19:37:13 GMT
+Content-Type: text/html
+Content-Length: 38111
+Last-Modified: Wed, 20 Jul 2016 02:50:52 GMT
+Connection: keep-alive
+Accept-Ranges: bytes
+
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta charset="UTF-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge, chrome=1" />
+ <meta name="viewport" content="initial-scale=1, maximum-scale=1, minimum-scale=1, user-scalable=no">
+ <script>
+ var _hmt = _hmt || [];
+ (function() {
+ var hm = document.createElement("script");
+ hm.src = "http://hm.baidu.com/hm.js?746c3f6346fae8612933e5921803ee32";
+ var s = document.getElementsByTagName("script")[0];
+ s.parentNode.insertBefore(hm, s);
+ })();
+
+ </script>
+
+
+ <link rel="stylesheet" type="text/css" href="css/common.css"/>
+ <script src="js/jquery-1.11.1.min.js" type="text/javascript" charset="utf-8"></script>
+ <script src="js/common.js" type="text/javascript" charset="utf-8"></script>
+ <script src="js/carousel.js" type="text/javascript" charset="utf-8"></script>
+ <title>中国域名网站</title>
+
+ </head>
+ <body>
+ <div class="head-tips" id="headTip">
+ <span class="close" id="headtips-close"><img src="css/img/close.png" alt="" /></span>
+ </div>
+ <div class="banner-bg"></div>
+ <div class="container">
+ <div class="banner">
+ <img src="css/img/banner.png" alt="" />
+ </div>
+ <div class="nav">
+ <h1>名站导航</h1>
+ <div class="left-btn" id="pre">
+ <img src="css/img/arrow-left.png" alt="" />
+ </div>
+ <div class="carousel">
+ <ul class="carousel-content">
+ <li>
+ <a href="http://中央电视台.中国" target="_blank">
+ <img src="css/img/p10.png" alt="" />
+ <p>中央电视台.中国</p>
+ </a><a href="http://平安北京.中国" target="_blank" class="mt-4">
+ <img src="css/img/p5.png" alt="" />
+ <p>平安北京.中国</p>
+ </a>
+ </li>
+ <li>
+ <a href="http://人民网.中国" target="_blank">
+ <img src="css/img/p6.png" alt="" />
+ <p>人民网.中国</p>
+ </a><a href="http://招商银行.中国" target="_blank" class="mt-4">
+ <img src="css/img/p8.png" alt="" />
+ <p>招商银行.中国</p>
+ </a>
+ </li>
+ <li>
+ <a href="http://必胜客宅急送.中国" target="_blank">
+ <img src="css/img/p1.png" alt="" />
+ <p>必胜客宅急送.中国</p>
+ </a><a href="http://创业咖啡.中国" target="_blank" class="mt-4">
+ <img src="css/img/p2.png" alt="" />
+ <p>创业咖啡.中国</p>
+ </a>
+ </li>
+ <li>
+ <a href="http://中国移动.中国" target="_blank">
+ <img src="css/img/p9.png" alt="" />
+ <p>中国移动.中国</p>
+ </a><a href="http://海盟.中国" target="_blank" class="mt-4">
+ <img src="css/img/p3.png" alt="" />
+ <p>海盟.中国</p>
+ </a>
+ </li>
+ <li>
+ <a href="http://艺龙.中国" target="_blank">
+ <img src="css/img/p7.png" alt="" />
+ <p>艺龙.中国</p>
+ </a><a href="http://和讯.中国" target="_blank" class="mt-4">
+ <img src="css/img/p4.png" alt="" />
+ <p>和讯.中国</p>
+ </a>
+ </li>
+ </ul>
+ </div>
+ <div class="right-btn" id="next">
+ <img src="css/img/arrow-right.png" alt="" />
+ </div>
+ </div>
+ </div>
+ <div class="all-url">
+ <div class="container">
+ <h1>网址大全</h1>
+ <ul class="url">
+ <li><a href="http://人民网.中国" target="_blank">人民网.中国</a></li>
+ <li><a href="http://新华网.中国" target="_blank">新华网.中国</a></li>
+ <li><a href="http://中央电视台.中国" target="_blank">中央电视台.中国</a></li>
+ <li><a href="http://光明网.中国" target="_blank">光明网.中国</a></li>
+ <li><a href="http://平安北京.中国" target="_blank">平安北京.中国</a></li>
+ <li><a href="http://联想微博.中国" target="_blank">联想微博.中国</a></li>
+ <li><a href="http://首都网警.中国" target="_blank">首都网警.中国</a></li>
+ <li><a href="http://北京消防.中国" target="_blank">北京消防.中国</a></li>
+ <li><a href="http://海淀公安.中国" target="_blank">海淀公安.中国</a></li>
+ <li><a href="http://通州警方.中国" target="_blank">通州警方.中国</a></li>
+ <li><a href="http://门头沟禁毒.中国" target="_blank">门头沟禁毒.中国</a></li>
+ <li><a href="http://西部数码.中国" target="_blank">西部数码.中国</a></li>
+ <li><a href="http://中央电视台.中国" target="_blank">中央电视台.中国</a></li>
+ <li><a href="http://中国移动.中国" target="_blank">中国移动.中国</a></li>
+ <li><a href="http://必胜宅急送.中国" target="_blank">必胜宅急送.中国</a></li>
+ <li><a href="http://老正兴.中国" target="_blank">老正兴.中国</a></li>
+ <li><a href="http://广州酒家.中国" target="_blank">广州酒家.中国</a></li>
+ <li><a href="http://格力.中国" target="_blank">格力.中国</a></li>
+ <li><a href="http://福建金爵.中国" target="_blank">福建金爵.中国</a></li>
+ <li><a href="http://和信房产.中国" target="_blank">和信房产.中国</a></li>
+ <li><a href="http://金爵房地产.中国" target="_blank">金爵房地产.中国</a></li>
+ <li><a href="http://联泰地产.中国" target="_blank">联泰地产.中国</a></li>
+ <li><a href="http://鲁商置业.中国" target="_blank">鲁商置业.中国</a></li>
+ <li><a href="http://鲁商置业股份.中国" target="_blank">鲁商置业股份.中国</a></li>
+ <li><a href="http://美佳华.中国" target="_blank">美佳华.中国</a></li>
+ <li><a href="http://金世纪工程.中国" target="_blank">金世纪工程.中国</a></li>
+ <li><a href="http://金世纪集团.中国" target="_blank">金世纪集团.中国</a></li>
+ <li><a href="http://深圳金世纪.中国" target="_blank">深圳金世纪.中国</a></li>
+ <li><a href="http://总部基地.中国" target="_blank">总部基地.中国</a></li>
+ <li><a href="http://德律风.中国" target="_blank">德律风.中国</a></li>
+ <li><a href="http://德律风物业.中国" target="_blank">德律风物业.中国</a></li>
+ <li><a href="http://柯林.中国" target="_blank">柯林.中国</a></li>
+ <li><a href="http://上海德律风物业.中国" target="_blank">上海德律风物业.中国</a></li>
+ <li><a href="http://广东海印集团股份.中国" target="_blank">广东海印集团股份.中国</a></li>
+ <li><a href="http://广东海印集团股份有限公司.中国" target="_blank">广东海印集团股份有限公司.中国</a></li>
+ <li><a href="http://艺龙.中国" target="_blank">艺龙.中国</a></li>
+ <li><a href="http://北京旅游信息网.中国" target="_blank">北京旅游信息网.中国</a></li>
+ <li><a href="http://北京故宫博物院.中国" target="_blank">北京故宫博物院.中国</a></li>
+ <li><a href="http://旅行张家界.中国" target="_blank">旅行张家界.中国</a></li>
+ <li><a href="http://张家界旅游.中国" target="_blank">张家界旅游.中国</a></li>
+ <li><a href="http://广州市旅游局.中国" target="_blank">广州市旅游局.中国</a></li>
+ <li><a href="http://旅游在线.中国" target="_blank">旅游在线.中国</a></li>
+ <li><a href="http://威海旅游集散中心.中国" target="_blank">威海旅游集散中心.中国</a></li>
+ <li><a href="http://锦州旅游.中国" target="_blank">锦州旅游.中国</a></li>
+ <li><a href="http://金牛湖风景旅游度假区.中国" target="_blank">金牛湖风景旅游度假区.中国</a></li>
+ <li><a href="http://环球旅行社.中国" target="_blank">环球旅行社.中国</a></li>
+ <li><a href="http://养鹿场.中国" target="_blank">养鹿场.中国</a></li>
+ <li><a href="http://东瀛游.中国" target="_blank">东瀛游.中国</a></li>
+ <li><a href="http://东瀛游旅行社.中国" target="_blank">东瀛游旅行社.中国</a></li>
+ <li><a href="http://桂林游.中国" target="_blank">桂林游.中国</a></li>
+ <li><a href="http://桂林之旅.中国" target="_blank">桂林之旅.中国</a></li>
+ <li><a href="http://美国环球旅行社.中国" target="_blank">美国环球旅行社.中国</a></li>
+ <li><a href="http://东天目山.中国" target="_blank">东天目山.中国</a></li>
+ <li><a href="http://凤山寺.中国" target="_blank">凤山寺.中国</a></li>
+ <li><a href="http://黄沙古渡.中国" target="_blank">黄沙古渡.中国</a></li>
+ <li><a href