Add Extended Twingly::URL hasher

.github/workflows/ci.yml

@@ -12,8 +12,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { ruby: '2.6' }
-          - { ruby: '2.7' }
           - { ruby: '3.0' }
           - { ruby: '3.1' }
           - { ruby: '3.2' }

.ruby-version

@@ -0,0 +1 @@
+3.3.7

Gemfile

@@ -3,3 +3,4 @@
 source 'https://rubygems.org/'
 
 gemspec
+gem 'public_suffix', '5.1.1'

README.md

@@ -12,6 +12,9 @@ Twingly URL tools.
     * `Twingly::URL::Hasher.autopingdb_hash(url)` - SHA256 64-bit signed, native endian digest
 * `twingly/url/utilities` - Utilities to work with URLs
     * `Twingly::URL::Utilities.extract_valid_urls` - Returns Array of valid `Twingly::URL`
+* `twingly/url/extended` - Extended normalization and hashing, stripping blacklisted query and matrix parameters as well as the fragment.
+    * `Twingly::URL::Extended.parse` - Like `Twingly::URL.parse`, with the extended normalization
+    * `Twingly::URL::Extended.normalize_and_calculate_urlhash(url)` - Returns a `HashResult` struct with `url` (original URL minus blacklisted parameters), `normalized_url` (scheme-less), `urlhash` (`documentdb_hash` of the normalized URL, as a `String`) and `legacy_urlhash` (same digest with the scheme kept)
 
 ## Getting Started
 

lib/twingly/url/extended.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "twingly/url"
+require "twingly/url/hasher"
+
+module Twingly
+  class URL
+    # The purpose of the twingly-url gems is to replicate the normalization happening in our legacy .NET code.
+    # As both twingly-url and the legacy .NET system are used in production, we can't change these things in the
+    # gem just yet, which is why this extension was created here.
+    class Extended < Twingly::URL
+      # Use the same expression as we use in Zambezi: https://github.com/twingly/zambezi/blob/d997871aea199256ddc56762c975b6e961d5a2d1/lib/post_document.rb#L12
+      # This way the urlhashes here will be the same as the one we have in Elasticsearch
+      PROTOCOL_EXPRESSION = /\Ahttps?:/i
+
+      HashResult = Struct.new(:url, :normalized_url, :urlhash, :legacy_urlhash)
+
+      # These parameters are just used for tracking, and should therefore not be included in the normalized URL
+      # See https://en.wikipedia.org/wiki/UTM_parameters
+      BLACKLISTED_QUERY_PARAMETERS = %w[
+        utm_source
+        utm_medium
+        utm_campaign
+        utm_term
+        utm_content
+        b_source
+        b_medium
+        b_campaign
+        session
+        PHPSESSID
+        cb
+      ].freeze
+
+      BLACKLISTED_MATRIX_PARAMETERS = %w[
+        jsessionid
+        jsessionid_jboss
+        JSESSIONID_B2BCH
+        JSESSIONID_kookbnagWEB
+      ].freeze
+
+      # Precompiled regex matching any blacklisted matrix parameter
+      # Matches: ;paramname=value where value ends at ; / ? or #
+      MATRIX_PARAM_REGEX = %r{;(?:#{BLACKLISTED_MATRIX_PARAMETERS.map { |p| Regexp.escape(p) }.join('|')})=[^;/?#]*}i
+
+      # Taken from twingly-url, with the addition of normalizing the query and fragment components
+      # See https://github.com/twingly/twingly-url/blob/e20f5fce077d93e89ef8520961be453c90cfec8c/lib/twingly/url.rb#L185-L193
+      def normalized # rubocop:disable Metrics/AbcSize
+        normalized_url = addressable_uri.dup
+
+        normalized_url.scheme       = normalized_scheme
+        normalized_url.host         = normalized_host
+        normalized_url.path         = normalized_path
+        normalized_url.query_values = normalized_query
+        normalized_url.fragment     = normalized_fragment
+
+        # This is a bit ugly, remove when Twingly::URL is updated to
+        # to handle this.
+        public_suffix_domain = get_public_suffix_domain(normalized_url.host)
+        self.class.send(:new, normalized_url, public_suffix_domain)
+      end
+
+      def original_url_without_blacklisted_parameters
+        url = addressable_uri.dup
+        url.path = without_blacklisted_matrix_parameters(addressable_uri.path)
+        url.query_values = without_blacklisted_query_parameters(addressable_uri.query_values)
+        url.to_s
+      end
+
+      def query_values
+        addressable_uri.query_values
+      end
+
+      def query
+        addressable_uri.query
+      end
+
+      private
+
+      # copied the way public_suffix_domain is calculated
+      # from Twingly::URL to be able calculate it without calling the parse method.
+      def get_public_suffix_domain(host)
+        public_suffix_domain = PublicSuffix.parse(host, list: CUSTOM_PSL, default_rule: nil)
+        raise Twingly::URL::Error::ParseError if public_suffix_domain.nil?
+        raise Twingly::URL::Error::ParseError if public_suffix_domain.sld.nil?
+
+        public_suffix_domain
+      end
+
+      def normalized_path
+        addressable_uri.path = without_blacklisted_matrix_parameters(addressable_uri.path)
+
+        super
+      end
+
+      def normalized_query
+        without_blacklisted_query_parameters(addressable_uri.query_values)
+      end
+
+      def without_blacklisted_query_parameters(query_values)
+        return if query_values.nil?
+
+        values_without_blacklisted_params = query_values.except(*BLACKLISTED_QUERY_PARAMETERS)
+
+        return if values_without_blacklisted_params.empty?
+
+        values_without_blacklisted_params
+      end
+
+      def without_blacklisted_matrix_parameters(path)
+        # No need to run gsub if there are no matrix parameters
+        return path unless path.include?(";")
+
+        path.gsub(MATRIX_PARAM_REGEX, "")
+      end
+
+      def normalized_fragment
+        nil
+      end
+
+      def self.normalize_and_calculate_urlhash(url)
+        return empty_result if url.to_s.strip.empty?
+
+        twingly_url = if url.is_a?(Extended)
+                        url
+                      else
+                        Extended.parse(url)
+                      end
+
+        return empty_result unless twingly_url.valid?
+
+        original_url                  = twingly_url.original_url_without_blacklisted_parameters
+        normalized_url                = twingly_url.normalized.to_s
+        normalized_url_without_scheme = remove_scheme(normalized_url)
+        urlhash                       = calculate_urlhash(normalized_url_without_scheme)
+        legacy_urlhash                = calculate_urlhash(normalized_url)
+
+        HashResult.new(original_url, normalized_url_without_scheme, urlhash, legacy_urlhash)
+      end
+
+      def self.empty_result
+        HashResult.new(nil, nil, nil, nil)
+      end
+
+      def self.remove_scheme(url)
+        url.to_s.sub(PROTOCOL_EXPRESSION, "")
+      end
+
+      def self.calculate_urlhash(url)
+        Twingly::URL::Hasher.documentdb_hash(url).to_s
+      end
+
+      private_class_method :empty_result
+      private_class_method :calculate_urlhash
+    end
+  end
+end

lib/twingly/version.rb

@@ -2,6 +2,6 @@
 
 module Twingly
   class URL
-    VERSION = "7.0.1"
+    VERSION = "7.1.0"
   end
 end

spec/lib/twingly/url/extended_spec.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+require "twingly/url/extended"
+
+RSpec.describe Twingly::URL::Extended do
+  describe "#normalized" do
+    subject { described_class.parse(url).normalized.to_s }
+
+    context "removes both matrix and query parameters" do
+      let(:url)      { "https://example.com/path;jsessionid=ABC.123?id=1&utm_source=google&name=test" }
+      let(:expected) { "https://www.example.com/path?id=1&name=test" }
+
+      it { is_expected.to eq(expected) }
+    end
+
+    context "applies parent path normalization after removing matrix parameters" do
+      let(:url)      { "https://example.com/article/;jsessionid=XYZ" }
+      let(:expected) { "https://www.example.com/article" }
+
+      it { is_expected.to eq(expected) }
+    end
+
+    context "removes the fragment" do
+      let(:url)      { "https://example.com/#foo" }
+      let(:expected) { "https://www.example.com/" }
+
+      it { is_expected.to eq(expected) }
+    end
+
+    context "removes the UTM parameters" do
+      let(:url)      { "https://example.com/?baz=qux&#{utm_query_parameters}&foo=bar" }
+      let(:expected) { "https://www.example.com/?baz=qux&foo=bar" }
+
+      let(:utm_query_parameters) do
+        [
+          "utm_source=foo",
+          "utm_medium=bar",
+          "utm_campaign=baz",
+          "utm_term=qux",
+          "utm_content=quux",
+        ].join("&")
+      end
+
+      it { is_expected.to eq(expected) }
+    end
+
+    context "removes both the UTM parameters and the ending '?' when the URL only contains UTM query parameters" do
+      let(:url)      { "https://example.com/?utm_source=foo&utm_medium=bar" }
+      let(:expected) { "https://www.example.com/" }
+
+      it { is_expected.to eq(expected) }
+    end
+
+    context "removes both the UTM parameters and the fragment" do
+      let(:url)      { "https://example.com/?baz=qux&utm_source=123&foo=bar#quux#something" }
+      let(:expected) { "https://www.example.com/?baz=qux&foo=bar" }
+
+      it { is_expected.to eq(expected) }
+    end
+
+    context "sorts the query parameters" do
+      let(:url)      { "https://example.com/?foo=bar&baz=qux&asd=123" }
+      let(:expected) { "https://www.example.com/?asd=123&baz=qux&foo=bar" }
+
+      it { is_expected.to eq(expected) }
+    end
+
+    context "returns a normalized URL without query parameters" do
+      let(:url)      { "https://example.com/blog" }
+      let(:expected) { "https://www.example.com/blog" }
+
+      it { is_expected.to eq(expected) }
+    end
+
+    context "keeps the embedded URL percent-encoded in the query" do
+      let(:url)      { "https://example.com/login?next=https://example.com/account" }
+      let(:expected) { "https://www.example.com/login?next=https%3A%2F%2Fexample.com%2Faccount" }
+
+      it { is_expected.to eq(expected) }
+    end
+  end
+
+  describe "#original_url_without_blacklisted_parameters" do
+    subject { described_class.parse(url).original_url_without_blacklisted_parameters }
+
+    context "removes the UTM parameters" do
+      let(:url)      { "https://example.com/?baz=qux&#{utm_query_parameters}&foo=bar" }
+      let(:expected) { "https://example.com/?baz=qux&foo=bar" }
+
+      let(:utm_query_parameters) do
+        [
+          "utm_source=foo",
+          "utm_medium=bar",
+          "utm_campaign=baz",
+          "utm_term=qux",
+          "utm_content=quux",
+        ].join("&")
+      end
+
+      it { is_expected.to eq(expected) }
+    end
+
+    context "removes both the UTM parameters and the ending '?' when the URL only contains UTM query parameters" do
+      let(:url)      { "https://example.com/?utm_source=foo&utm_medium=bar" }
+      let(:expected) { "https://example.com/" }
+
+      it { is_expected.to eq(expected) }
+    end
+  end
+
+  describe ".normalize_and_calculate_urlhash" do
+    let(:url) { "https://example.com" }
+
+    it "produces same hash when blacklisted query parameters differ" do
+      url_with = "https://example.com/page?id=1&session=x&PHPSESSID=y&utm_source=z&cb=w"
+      url_without = "https://example.com/page?id=1"
+
+      result_with = described_class.normalize_and_calculate_urlhash(url_with)
+      result_without = described_class.normalize_and_calculate_urlhash(url_without)
+
+      expect(result_with.urlhash).to eq(result_without.urlhash)
+      expect(result_with.legacy_urlhash).to eq(result_without.legacy_urlhash)
+    end
+
+    it "produces same hash when blacklisted matrix parameters differ" do
+      url_with = "https://example.com/path;jsessionid=ABC.123;allowed_matrix_param=yes;JSESSIONID_B2BCH=XYZ"
+      url_without = "https://example.com/path;allowed_matrix_param=yes"
+
+      result_with = described_class.normalize_and_calculate_urlhash(url_with)
+      result_without = described_class.normalize_and_calculate_urlhash(url_without)
+
+      expect(result_with.urlhash).to eq(result_without.urlhash)
+      expect(result_with.legacy_urlhash).to eq(result_without.legacy_urlhash)
+    end
+
+    it "returns both original url, normalized url, urlhash and legacy_urlhash" do
+      expect(described_class.normalize_and_calculate_urlhash(url)).to have_attributes(url: url,
+                                                                 normalized_url: "//www.example.com/",
+                                                                 urlhash:        "1119909257551956256",
+                                                                 legacy_urlhash: "14653629529287702089")
+    end
+
+    it "calculates the legacy urlhash from the normalized URL with its scheme kept" do
+      result = described_class.normalize_and_calculate_urlhash("https://example.com/blog")
+
+      expect(result.legacy_urlhash)
+        .to eq(Twingly::URL::Hasher.documentdb_hash("https:#{result.normalized_url}").to_s)
+    end
+
+    ["", nil].each do |empty_value|
+      context "when url is #{empty_value.inspect}" do
+        let(:url) { empty_value }
+
+        it "returns a result where all attributes are set to nil" do
+          expect(described_class.normalize_and_calculate_urlhash(url)).to have_attributes(url:            nil,
+                                                                     normalized_url: nil,
+                                                                     urlhash:        nil,
+                                                                     legacy_urlhash: nil)
+        end
+      end
+    end
+
+    context "with an invalid URL" do
+      let(:url) { "http:// example.com? hello # there" }
+
+      it "returns a result where all attributes are set to nil" do
+        expect(described_class.normalize_and_calculate_urlhash(url)).to have_attributes(url:            nil,
+                                                                   normalized_url: nil,
+                                                                   urlhash:        nil,
+                                                                   legacy_urlhash: nil)
+      end
+    end
+  end
+
+end

twingly-url.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.summary     = "Ruby library for URL handling"
   s.description = "Twingly URL tools"
   s.license     = "MIT"
-  s.required_ruby_version = ">= 2.6"
+  s.required_ruby_version = ">= 3.0"
 
   s.add_dependency "addressable", "~> 2.6"
   s.add_dependency "public_suffix", ">= 3.0.1", "< 8"