gearheads
/
mastodon
Archived
2
0
Fork 0
Code Releases 4 Activity

Ignore low-confidence CharlockHolmes guesses when parsing link cards (#9510) 

* Add failing test for windows-1251 link cards

* Ignore low-confidence CharlockHolmes guesses

Fixes #9466

* Fix no method error when charlock holmes cannot detect charset
gh/stable
ThibG 2018-12-17 19:19:45 +01:00 committed by Eugen Rochko
parent 4ede51743e
commit e709b8da0d
3 changed files with 30 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/services/fetch_link_card_service.rb
View File

@ -137,7 +137,8 @@ class FetchLinkCardService < BaseService
detector.strip_tags = true detector.strip_tags = true
guess = detector.detect(@html, @html_charset) guess = detector.detect(@html, @html_charset)
page = Nokogiri::HTML(@html, nil, guess&.fetch(:encoding, nil)) encoding = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
page = Nokogiri::HTML(@html, nil, encoding)
player_url = meta_property(page, 'twitter:player') player_url = meta_property(page, 'twitter:player')
if player_url && !bad_url?(Addressable::URI.parse(player_url)) if player_url && !bad_url?(Addressable::URI.parse(player_url))

17
spec/fixtures/requests/windows-1251.txt vendored 100644
View File

@ -0,0 +1,17 @@
HTTP/1.1 200 OK
server: nginx
date: Wed, 12 Dec 2018 13:14:03 GMT
content-type: text/html
content-length: 190
accept-ranges: bytes
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=windows-1251" />
<title>ñýìïë òåêñò</title>
</head>
<body>
<p>ñýìïë òåêñò</p>
</body>
</html>

11
spec/services/fetch_link_card_service_spec.rb
View File

@ -17,6 +17,8 @@ RSpec.describe FetchLinkCardService, type: :service do
stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404) stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404)
stub_request(:head, 'http://example.com/test-').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) stub_request(:head, 'http://example.com/test-').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt')) stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt'))
stub_request(:head, 'http://example.com/windows-1251').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
subject.call(status) subject.call(status)
end end
@ -57,6 +59,15 @@ RSpec.describe FetchLinkCardService, type: :service do
end end
end end
context do
let(:status) { Fabricate(:status, text: 'Check out http://example.com/windows-1251') }
it 'works with windows-1251' do
expect(a_request(:get, 'http://example.com/windows-1251')).to have_been_made.at_least_once
expect(status.preview_cards.first.title).to eq('сэмпл текст')
end
end
context do context do
let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') } let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') }