Update Deno documentation scraper

.gitattributes

@@ -0,0 +1,24 @@
+# Auto detect text files and perform LF normalization
+* text=auto
+
+# Explicitly declare text files
+*.md text diff=markdown
+*.txt text
+*.csv text
+*.yml text
+*.yaml text
+*.json text
+*.xml text
+*.html text diff=html
+*.css text diff=css
+
+# Denote binary files
+*.png binary
+*.jpg binary
+*.jpeg binary
+*.gif binary
+*.ico binary
+*.pdf binary
+*.zip binary
+*.gz binary
+*.tar binary

lib/docs/filters/deno/clean_html.rb

@@ -2,30 +2,25 @@ module Docs
   class Deno
     class CleanHtmlFilter < Filter
       def call
-        if result[:path].start_with?('api/deno/')
-          @doc = at_css('main[id!="content"] article', 'main[id!="content"]')
-        else
-          @doc = at_css('main article .markdown-body')
-        end
+        @doc = at_css('main#content article', 'article') || doc
+
+        css('.anchor-link', '.breadcrumbs', '.copy-page-split', '.copyButton',
+            '.docNodeKindIcon', '.header-anchor', 'a > svg',
+            'nav[aria-label="Breadcrumb"]',
+            'nav[aria-label="Previous and next page"]').remove
 
-        if at_css('.text-2xl')
-          doc.prepend_child at_css('.text-2xl').remove
-          at_css('.text-2xl').name = 'h1'
+        css('details > summary').each do |node|
+          node.parent.remove if node.content.strip == 'On this page'
         end
 
-        css('code').each do |node|
-          if node['class']
-            lang = node['class'][/language-(\w+)/, 1]
-          end
-          node['data-language'] = lang || 'ts'
-          node.remove_attribute('class')
-          if node.parent.name == 'div'
-            node.content = node.content.strip
-          end
+        css('h1, h2, h3, h4, h5, h6').each do |node|
+          node.css('a.anchor[aria-label="Anchor"]').remove
         end
 
-        css('a.header-anchor').remove()
-        css('.breadcrumbs').remove()
+        css('pre > code').each do |node|
+          language = node['class'].to_s[/\blanguage-([\w-]+)/, 1]
+          node.parent['data-language'] = language if language
+        end
 
         doc
       end

lib/docs/filters/deno/entries.rb

@@ -1,25 +1,48 @@
 module Docs
   class Deno
     class EntriesFilter < Docs::EntriesFilter
+      TYPES_BY_PATH = {
+        'api'     => 'API',
+        'runtime' => 'Runtime',
+      }
 
       def get_name
-        if result[:path].start_with?('api/deno/')
-          at_css('main[id!="content"]')['id'][/\Asymbol_([.\w]+)/, 1]
-        else
-          at_css('main article h1').content
-        end
+        name = at_css('h1')
+        name ? name.content.strip : slug.split('/').last
       end
 
       def get_type
-        if result[:path].start_with?('api/deno/')
-          'API'
-        elsif result[:path].start_with?('runtime/reference/cli')
-          'CLI'
-        else
-          at_css('main article nav ul :first span').content
-        end
+        TYPES_BY_PATH[slug.split('/').first] || 'Guide'
       end
 
+      # Multi-symbol API pages open with a `#symbol-index` listing every symbol
+      # that is then documented in full further down the same page. Index those
+      # symbols individually, then drop the listing: once each symbol has its
+      # own entry linking to its section, the listing just repeats the page.
+      def additional_entries
+        index = at_css('#symbol-index')
+        return [] unless index
+
+        entries = index.css('.namespaceItem > .namespaceItemContent > a[href*="#"]')
+          .each_with_object([]) do |link, acc|
+            target, fragment = link['href'].split('#', 2)
+            next if fragment.blank?
+            # Symbols re-exported from another module link to the page that
+            # documents them; let that page own the entry instead of repeating it.
+            next unless target.blank? || target == File.basename(path)
+            symbol = link.content.strip
+            next if symbol.empty? || acc.any? { |entry| entry.first == symbol }
+            acc << [symbol, fragment]
+          end
+
+        index.remove
+        # The same symbol name is documented by several modules (`Socket` by
+        # net, dgram and process); qualify each one with the page it belongs to.
+        # For `node:` modules the page title is the import name, so
+        # `Profiler.CoverageRange` becomes `Profiler.CoverageRange (inspector)`.
+        entries.map! { |symbol, fragment| ["#{symbol} (#{name})", fragment] } if name.present?
+        entries
+      end
     end
   end
 end

lib/docs/scrapers/deno.rb

@@ -2,39 +2,64 @@ module Docs
   class Deno < UrlScraper
     self.name = 'Deno'
     self.type = 'simple'
+    self.base_url = 'https://docs.deno.com/'
+    self.root_path = 'api'
+    self.initial_paths = %w(
+      api/deno
+      runtime
+      runtime/fundamentals
+      runtime/reference
+    )
     self.links = {
       home: 'https://deno.com/',
       code: 'https://github.com/denoland/deno'
     }
 
-    # https://github.com/denoland/manual/blob/main/LICENSE
-    # https://github.com/denoland/deno/blob/main/LICENSE.md
+    html_filters.push 'deno/clean_html', 'deno/entries'
+
+    options[:root_title] = 'Deno'
+    options[:title] = false
+    options[:follow_links] = true
+    options[:only_patterns] = [
+      /\Aapi\//,
+      /\Aruntime\//,
+    ]
+    options[:skip_patterns] = [
+      /\Ablog\//,
+      /\Adeploy\//,
+      /\Asubhosting\//,
+      # Aggregate listings that repeat every symbol already documented on the
+      # individual module pages.
+      /\Aapi\/\w+\/all_symbols/,
+      # Per-symbol URLs now 301 to a fragment of the module page
+      # (api/web/~/Blob -> api/web/file/#Blob). Because the stored path comes
+      # from the requested URL, crawling them would file a second copy of the
+      # whole module page under every symbol it documents.
+      /\Aapi\/\w+\/~\//,
+    ]
+    # docs.deno.com links to both `api/node/buffer` and `api/node/buffer/`;
+    # without this each page is crawled twice, once as `…/buffer` and once as
+    # `…/buffer/index`.
+    options[:trailing_slash] = false
+
     options[:attribution] = <<-HTML
-      &copy; 2018–2025 the Deno authors<br>
+      &copy; 2018&ndash;2025 the Deno authors<br>
       Licensed under the MIT License.
     HTML
 
-
-    html_filters.push 'deno/entries', 'deno/clean_html'
+    # ── Versions ──────────────────────────────────────────────────────
 
     version '2' do
-      self.release = '2.4.4'
-      self.base_url = 'https://docs.deno.com/'
-      self.root_path = 'runtime'
-      options[:only_patterns] = [/\Aruntime/, /\Aapi\/deno\/~/, /\Adeploy/, /\Asubhosting/]
-      options[:skip_patterns] = [
-        /\Aruntime\/manual/,
-        /\Aapi\/deno\/.+\.prototype\z/, # all prototype pages get redirected to the main page
-        /\Aapi\/deno\/~\/Deno\.jupyter\.MediaBundle.+/, # docs unavailable
-        /\Aapi\/deno\/~\/Deno\.OpMetrics/, # deprecated in deno 2
-      ]
-      options[:trailing_slash] = false
+      self.release = '2.9.6'
     end
 
     version '1' do
-      self.release = '1.27.0'
+      self.release = '1.46.3'
+      self.base_url = 'https://docs.deno.com/api/'
     end
 
+    # ── Latest version lookup ─────────────────────────────────────────
+
     def get_latest_version(opts)
       get_latest_github_release('denoland', 'deno', opts)
     end

lib/tasks/sprites.thor

@@ -222,7 +222,7 @@ class SpritesCLI < Thor
     scss_erb_files.each do |erb_path|
       scss_path = erb_path.gsub('.erb', '')
       File.open(scss_path, 'w') do |f|
-        f.write(ERB.new(File.open(erb_path).read).result)
+        f.write(ERB.new(File.read(erb_path)).result)
         logger.info("Compiling #{erb_path} to #{scss_path}")
       end
     end

test/files/deno_api.html

@@ -0,0 +1,59 @@
+<!doctype html>
+<html>
+  <body>
+    <nav>Site navigation</nav>
+    <main id="content">
+      <div class="w-full">
+        <article>
+          <div class="ddoc markdown-body">
+            <ul class="breadcrumbs"><li>Deno</li><li>Network</li></ul>
+            <div class="multiSymbolPage">
+              <h1>Network</h1>
+              <p>Networking APIs.</p>
+              <section class="mt-8" id="symbol-index">
+                <div class="mb-6">
+                  <h2>Functions</h2>
+                  <div class="namespaceSection">
+                    <div id="namespace_deno_connect" class="namespaceItem">
+                      <div class="docNodeKindIcon"><div title="Function">f</div></div>
+                      <div class="namespaceItemContent">
+                        <a href="network#Deno.connect" title="Deno.connect">Deno.connect</a>
+                        <div class="namespaceItemContentDoc">Connects to a TCP listener.</div>
+                        <ul class="namespaceItemContentSubItems">
+                          <li><a href="network#Deno.connect.property_addr">addr</a></li>
+                        </ul>
+                      </div>
+                    </div>
+                  </div>
+                </div>
+                <div class="mb-6">
+                  <h2>Classes</h2>
+                  <div class="namespaceSection">
+                    <div id="namespace_blob" class="namespaceItem">
+                      <div class="docNodeKindIcon"><div title="Class">c</div></div>
+                      <div class="namespaceItemContent">
+                        <a href="../../web/file/file#Blob" title="Blob">Blob</a>
+                        <div class="namespaceItemContentDoc">Re-exported from another module.</div>
+                      </div>
+                    </div>
+                  </div>
+                </div>
+              </section>
+              <article class="symbolGroup" id="Deno.connect">
+                <h2 class="anchorable-heading">Deno.connect <a href="#Deno.connect" class="anchor-link" aria-label="Anchor" tabindex="-1">#</a></h2>
+                <h3 class="anchorable-heading">Parameters <a href="#Deno.connect.parameters" class="anchor-link" aria-label="Anchor" tabindex="-1">#</a></h3>
+                <code>options<a href="#Deno.connect.options" class="anchor-link" aria-label="Anchor" tabindex="-1">#</a>: ConnectOptions</code>
+                <div class="relative">
+                  <pre><code class="highlight language-ts">await Deno.connect();</code></pre>
+                  <button class="copyButton" title="Copy">Copy</button>
+                </div>
+              </article>
+            </div>
+          </div>
+        </article>
+        <section id="feedback-section">Did you find what you needed?</section>
+      </div>
+    </main>
+    <footer>Site footer</footer>
+  </body>
+</html>

test/files/deno_runtime.html

@@ -0,0 +1,36 @@
+<!doctype html>
+<html>
+  <body>
+    <main id="content">
+      <div class="w-full">
+        <article>
+          <details><summary>On this page</summary><a href="#overview">Overview</a></details>
+          <div class="markdown-body">
+            <header>
+              <nav aria-label="Breadcrumb">Reference / Standard library</nav>
+              <div class="copy-page-split"><button>Copy page</button></div>
+            </header>
+            <h1>@std/fmt</h1>
+            <h2 id="overview"><a class="anchor" aria-label="Anchor" href="#overview">#</a>Overview</h2>
+            <p>Provides utilities for formatting text.</p>
+            <div class="relative">
+              <pre><code class="highlight language-js">console.log('formatted');</code></pre>
+              <button class="copyButton" title="Copy">Copy</button>
+            </div>
+            <h2>Runtime compatibility</h2>
+            <p>Works with Deno.</p>
+            <h2>Add to your project</h2>
+            <pre><code class="highlight language-sh">deno add jsr:@std/fmt</code></pre>
+            <pre><code class="highlight language-jsonc">{"imports": {}}</code></pre>
+            <a href="https://jsr.io/@std/fmt/doc">See all symbols</a>
+          </div>
+          <nav aria-label="Previous and next page">Previous Next</nav>
+        </article>
+        <section id="feedback-section">
+          <h2>Did you find what you needed?</h2>
+          <a href="https://github.com/denoland/docs">Edit this page</a>
+        </section>
+      </div>
+    </main>
+  </body>
+</html>