Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 24 additions & 3 deletions lib/twingly/url/extended.rb
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,17 @@ def original_url_without_blacklisted_parameters
url.to_s
end

# Parsed here instead of with Addressable::URI#query_values, as the urlhash depends on the result
# and has to stay the same when Addressable changes how it reads a query.
def query_values
addressable_uri.query_values
return if query.nil?

query.split("&").each_with_object({}) do |pair, values|
next if pair.empty?

name, value = pair.split("=", 2)
values[decode_query_name(name)] = decode_query_value(value)
end
end

def query
Expand All @@ -105,13 +114,25 @@ def normalized_path
end

def normalized_query
without_blacklisted_query_parameters(addressable_uri.query_values)
without_blacklisted_query_parameters(query_values)
end

def normalize_addressable_url
addressable_uri.normalize
end

# A literal '+' in a name is a plus
def decode_query_name(name)
Addressable::URI.unencode_component(name)
end

# A literal '+' in a value is a space
def decode_query_value(value)
return if value.nil?

Addressable::URI.unencode_component(value.tr("+", " "))
end

def without_blacklisted_query_parameters(query_values)
return if query_values.nil?

Expand All @@ -128,7 +149,7 @@ def without_blacklisted_query_string(query)
kept = query.split("&").reject do |pair|
next true if pair.empty?

name = Addressable::URI.unencode_component(pair.split("=", 2).first)
name = decode_query_name(pair.split("=", 2).first)
BLACKLISTED_QUERY_PARAMETERS.include?(name)
end

Expand Down
47 changes: 47 additions & 0 deletions spec/lib/twingly/url/extended_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,14 @@
it { is_expected.to eq(expected) }
end

# The urlhash is calculated from this URL, changing how '+' is treated changes the urlhash
context "keeps a literal '+' in a query parameter name as a plus, but rewrites it as a space in the value" do
let(:url) { "https://example.com/?a+b=c+d" }
let(:expected) { "https://www.example.com/?a%2Bb=c%20d" }

it { is_expected.to eq(expected) }
end

context "with addressable_normalize: true" do
subject { described_class.parse(url).normalized(addressable_normalize: true).to_s }

Expand Down Expand Up @@ -240,6 +248,22 @@
end
end

describe "#query_values" do
subject { described_class.parse(url).query_values }

context "keeps a literal '+' in a name, but reads it as a space in a value" do
let(:url) { "https://example.com/?a+b=c+d" }

it { is_expected.to eq("a+b" => "c d") }
end

context "reads '%2B' as a plus in both names and values" do
let(:url) { "https://example.com/?a%2Bb=c%2Bd" }

it { is_expected.to eq("a+b" => "c+d") }
end
end

describe "#callable_url" do
it "is the addressable-normalized URL without blacklisted parameters" do
url = described_class.parse("https://Example.com/p%c3%a4ge;jsessionid=X?k=1&utm_source=a&k=2")
Expand Down Expand Up @@ -316,6 +340,16 @@
legacy_urlhash: "14653629529287702089")
end

it "keeps a literal '+' in a query parameter name as '%2B' in normalized_url, so the urlhash stays the same" do
url = "https://example.com/?a+b=c"

expect(described_class.normalize_and_calculate_urlhash(url)).to have_attributes(url: url,
callable_url: url,
normalized_url: "//www.example.com/?a%2Bb=c",
urlhash: "12381321737887489877",
legacy_urlhash: "8901784836407991551")
end

it "calculates the legacy urlhash from the normalized URL with its scheme kept" do
result = described_class.normalize_and_calculate_urlhash("https://example.com/blog")

Expand Down Expand Up @@ -402,6 +436,19 @@ def callable_query(url)
expect(Addressable::URI.unencode_component(callable)).to eq("tag=c++")
end

it "rewrites a literal '+' in a name as '%2B' in normalized_url, so it decodes to a plus, but keeps it in callable_url" do
url = "https://example.com/?a+b=c"

normalized = normalized_query(url)
callable = callable_query(url)

expect(normalized).to eq("a%2Bb=c")
expect(callable).to eq("a+b=c")

expect(Addressable::URI.unencode_component(normalized)).to eq("a+b=c")
expect(Addressable::URI.unencode_component(callable)).to eq("a+b=c")
end

it "percent-encodes raw reserved characters in values in normalized_url, but keeps them in callable_url" do
url = "https://example.com/?path=https://x/y"

Expand Down
Loading