diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..979a9c17ae --- /dev/null +++ b/.gitattributes @@ -0,0 +1,24 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Explicitly declare text files +*.md text diff=markdown +*.txt text +*.csv text +*.yml text +*.yaml text +*.json text +*.xml text +*.html text diff=html +*.css text diff=css + +# Denote binary files +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.pdf binary +*.zip binary +*.gz binary +*.tar binary diff --git a/lib/docs/filters/deno/clean_html.rb b/lib/docs/filters/deno/clean_html.rb index c111a125bf..7821a5469d 100644 --- a/lib/docs/filters/deno/clean_html.rb +++ b/lib/docs/filters/deno/clean_html.rb @@ -2,30 +2,25 @@ module Docs class Deno class CleanHtmlFilter < Filter def call - if result[:path].start_with?('api/deno/') - @doc = at_css('main[id!="content"] article', 'main[id!="content"]') - else - @doc = at_css('main article .markdown-body') - end + @doc = at_css('main#content article', 'article') || doc + + css('.anchor-link', '.breadcrumbs', '.copy-page-split', '.copyButton', + '.docNodeKindIcon', '.header-anchor', 'a > svg', + 'nav[aria-label="Breadcrumb"]', + 'nav[aria-label="Previous and next page"]').remove - if at_css('.text-2xl') - doc.prepend_child at_css('.text-2xl').remove - at_css('.text-2xl').name = 'h1' + css('details > summary').each do |node| + node.parent.remove if node.content.strip == 'On this page' end - css('code').each do |node| - if node['class'] - lang = node['class'][/language-(\w+)/, 1] - end - node['data-language'] = lang || 'ts' - node.remove_attribute('class') - if node.parent.name == 'div' - node.content = node.content.strip - end + css('h1, h2, h3, h4, h5, h6').each do |node| + node.css('a.anchor[aria-label="Anchor"]').remove end - css('a.header-anchor').remove() - css('.breadcrumbs').remove() + css('pre > code').each do |node| + language = node['class'].to_s[/\blanguage-([\w-]+)/, 1] + node.parent['data-language'] = language if language + end doc end diff --git a/lib/docs/filters/deno/entries.rb b/lib/docs/filters/deno/entries.rb index 512dd4d16b..209aad45f8 100644 --- a/lib/docs/filters/deno/entries.rb +++ b/lib/docs/filters/deno/entries.rb @@ -1,25 +1,48 @@ module Docs class Deno class EntriesFilter < Docs::EntriesFilter + TYPES_BY_PATH = { + 'api' => 'API', + 'runtime' => 'Runtime', + } def get_name - if result[:path].start_with?('api/deno/') - at_css('main[id!="content"]')['id'][/\Asymbol_([.\w]+)/, 1] - else - at_css('main article h1').content - end + name = at_css('h1') + name ? name.content.strip : slug.split('/').last end def get_type - if result[:path].start_with?('api/deno/') - 'API' - elsif result[:path].start_with?('runtime/reference/cli') - 'CLI' - else - at_css('main article nav ul :first span').content - end + TYPES_BY_PATH[slug.split('/').first] || 'Guide' end + # Multi-symbol API pages open with a `#symbol-index` listing every symbol + # that is then documented in full further down the same page. Index those + # symbols individually, then drop the listing: once each symbol has its + # own entry linking to its section, the listing just repeats the page. + def additional_entries + index = at_css('#symbol-index') + return [] unless index + + entries = index.css('.namespaceItem > .namespaceItemContent > a[href*="#"]') + .each_with_object([]) do |link, acc| + target, fragment = link['href'].split('#', 2) + next if fragment.blank? + # Symbols re-exported from another module link to the page that + # documents them; let that page own the entry instead of repeating it. + next unless target.blank? || target == File.basename(path) + symbol = link.content.strip + next if symbol.empty? || acc.any? { |entry| entry.first == symbol } + acc << [symbol, fragment] + end + + index.remove + # The same symbol name is documented by several modules (`Socket` by + # net, dgram and process); qualify each one with the page it belongs to. + # For `node:` modules the page title is the import name, so + # `Profiler.CoverageRange` becomes `Profiler.CoverageRange (inspector)`. + entries.map! { |symbol, fragment| ["#{symbol} (#{name})", fragment] } if name.present? + entries + end end end end diff --git a/lib/docs/scrapers/deno.rb b/lib/docs/scrapers/deno.rb index 4dfb34564f..3230ade10b 100644 --- a/lib/docs/scrapers/deno.rb +++ b/lib/docs/scrapers/deno.rb @@ -2,39 +2,64 @@ module Docs class Deno < UrlScraper self.name = 'Deno' self.type = 'simple' + self.base_url = 'https://docs.deno.com/' + self.root_path = 'api' + self.initial_paths = %w( + api/deno + runtime + runtime/fundamentals + runtime/reference + ) self.links = { home: 'https://deno.com/', code: 'https://github.com/denoland/deno' } - # https://github.com/denoland/manual/blob/main/LICENSE - # https://github.com/denoland/deno/blob/main/LICENSE.md + html_filters.push 'deno/clean_html', 'deno/entries' + + options[:root_title] = 'Deno' + options[:title] = false + options[:follow_links] = true + options[:only_patterns] = [ + /\Aapi\//, + /\Aruntime\//, + ] + options[:skip_patterns] = [ + /\Ablog\//, + /\Adeploy\//, + /\Asubhosting\//, + # Aggregate listings that repeat every symbol already documented on the + # individual module pages. + /\Aapi\/\w+\/all_symbols/, + # Per-symbol URLs now 301 to a fragment of the module page + # (api/web/~/Blob -> api/web/file/#Blob). Because the stored path comes + # from the requested URL, crawling them would file a second copy of the + # whole module page under every symbol it documents. + /\Aapi\/\w+\/~\//, + ] + # docs.deno.com links to both `api/node/buffer` and `api/node/buffer/`; + # without this each page is crawled twice, once as `…/buffer` and once as + # `…/buffer/index`. + options[:trailing_slash] = false + options[:attribution] = <<-HTML - © 2018–2025 the Deno authors
+ © 2018–2025 the Deno authors
Licensed under the MIT License. HTML - - html_filters.push 'deno/entries', 'deno/clean_html' + # ── Versions ────────────────────────────────────────────────────── version '2' do - self.release = '2.4.4' - self.base_url = 'https://docs.deno.com/' - self.root_path = 'runtime' - options[:only_patterns] = [/\Aruntime/, /\Aapi\/deno\/~/, /\Adeploy/, /\Asubhosting/] - options[:skip_patterns] = [ - /\Aruntime\/manual/, - /\Aapi\/deno\/.+\.prototype\z/, # all prototype pages get redirected to the main page - /\Aapi\/deno\/~\/Deno\.jupyter\.MediaBundle.+/, # docs unavailable - /\Aapi\/deno\/~\/Deno\.OpMetrics/, # deprecated in deno 2 - ] - options[:trailing_slash] = false + self.release = '2.9.6' end version '1' do - self.release = '1.27.0' + self.release = '1.46.3' + self.base_url = 'https://docs.deno.com/api/' end + # ── Latest version lookup ───────────────────────────────────────── + def get_latest_version(opts) get_latest_github_release('denoland', 'deno', opts) end diff --git a/lib/tasks/sprites.thor b/lib/tasks/sprites.thor index 54df982f07..d12d314585 100644 --- a/lib/tasks/sprites.thor +++ b/lib/tasks/sprites.thor @@ -222,7 +222,7 @@ class SpritesCLI < Thor scss_erb_files.each do |erb_path| scss_path = erb_path.gsub('.erb', '') File.open(scss_path, 'w') do |f| - f.write(ERB.new(File.open(erb_path).read).result) + f.write(ERB.new(File.read(erb_path)).result) logger.info("Compiling #{erb_path} to #{scss_path}") end end diff --git a/test/files/deno_api.html b/test/files/deno_api.html new file mode 100644 index 0000000000..55dea0770d --- /dev/null +++ b/test/files/deno_api.html @@ -0,0 +1,59 @@ + + + + +
+
+
+
+
  • Deno
  • Network
+
+

Network

+

Networking APIs.

+
+
+

Functions

+
+
+
f
+
+ Deno.connect +
Connects to a TCP listener.
+ +
+
+
+
+
+

Classes

+
+
+
c
+
+ Blob +
Re-exported from another module.
+
+
+
+
+
+
+

Deno.connect #

+

Parameters #

+ options#: ConnectOptions +
+ 
await Deno.connect();
+ +
+
+
+
+
+
Did you find what you needed?
+
+
+ + + diff --git a/test/files/deno_runtime.html b/test/files/deno_runtime.html new file mode 100644 index 0000000000..96bec01d0e --- /dev/null +++ b/test/files/deno_runtime.html @@ -0,0 +1,36 @@ + + + +
+
+
+
On this pageOverview
+
+
+ +
+
+

@std/fmt

+

#Overview

+

Provides utilities for formatting text.

+
+ 
console.log('formatted');
+ +
+

Runtime compatibility

+

Works with Deno.

+

Add to your project

+ 
deno add jsr:@std/fmt
+ 
{"imports": {}}
+ See all symbols +
+ +
+
+

Did you find what you needed?

+ Edit this page +
+
+
+ + diff --git a/test/lib/docs/scrapers/deno_test.rb b/test/lib/docs/scrapers/deno_test.rb new file mode 100644 index 0000000000..b24b90c46f --- /dev/null +++ b/test/lib/docs/scrapers/deno_test.rb @@ -0,0 +1,228 @@ +require_relative '../../../test_helper' +require_relative '../../../../lib/docs' + +class DenoScraperTest < Minitest::Test + def setup + @scraper_class = Docs::Deno + end + + def test_scraper_name + assert_equal 'Deno', @scraper_class.name + end + + def test_scraper_type + assert_equal 'simple', @scraper_class.type + end + + def test_base_url + assert_equal 'https://docs.deno.com/', @scraper_class.base_url + end + + def test_root_path + assert_equal 'api', @scraper_class.root_path + end + + def test_initial_paths_present + assert_kind_of Array, @scraper_class.initial_paths + refute_empty @scraper_class.initial_paths + assert_includes @scraper_class.initial_paths, 'api/deno' + assert_includes @scraper_class.initial_paths, 'runtime' + end + + def test_links_defined + links = @scraper_class.links + assert_kind_of Hash, links + assert links.key?(:home) + assert links.key?(:code) + assert_match %r{\Ahttps://}, links[:home] + assert_match %r{github\.com}, links[:code] + end + + def test_only_patterns_defined + patterns = @scraper_class.options[:only_patterns] + assert_kind_of Array, patterns + refute_empty patterns + assert patterns.any? { |p| p.is_a?(Regexp) } + end + + def test_skip_patterns_excludes_blog + patterns = @scraper_class.options[:skip_patterns] + assert_kind_of Array, patterns + assert patterns.any? { |p| 'blog/foo' =~ p } + end + + def test_skip_patterns_excludes_deploy + patterns = @scraper_class.options[:skip_patterns] + assert patterns.any? { |p| 'deploy/docs' =~ p } + end + + # Aggregate pages repeat every symbol documented on the module pages. + def test_skip_patterns_excludes_all_symbols_pages + patterns = @scraper_class.options[:skip_patterns] + %w(api/node/all_symbols api/deno/all_symbols api/web/all_symbols).each do |path| + assert patterns.any? { |p| path =~ p }, "expected #{path} to be skipped" + end + end + + # These 301 to a fragment of the module page, so crawling them would file a + # duplicate copy of that page under every symbol it documents. + def test_skip_patterns_excludes_per_symbol_redirect_urls + patterns = @scraper_class.options[:skip_patterns] + %w(api/web/~/Blob api/deno/~/Deno.connect api/node/~/SlowBuffer).each do |path| + assert patterns.any? { |p| path =~ p }, "expected #{path} to be skipped" + end + end + + # Without this the site's two link forms (`api/node/buffer` and + # `api/node/buffer/`) are crawled as two pages with identical content. + def test_trailing_slashes_are_normalized_away + assert_equal false, @scraper_class.options[:trailing_slash] + end + + def test_attribution_present + attribution = @scraper_class.options[:attribution] + assert_kind_of String, attribution + refute_empty attribution.strip + assert_match(/Deno/, attribution) + end + + def test_has_versions + versions = @scraper_class.versions + refute_nil versions + refute_empty versions + end + + def test_inherits_from_url_scraper + assert @scraper_class < Docs::UrlScraper + end +end + +class DenoEntriesFilterTest < Minitest::Test + def test_extracts_api_entry_from_content_heading + entry = filter_fixture('deno_api.html', 'api/deno/network').first + + assert_equal 'Network', entry.name + assert_equal 'API', entry.type + end + + def test_extracts_symbol_entries_from_multi_symbol_api_page + entries = filter_fixture('deno_api.html', 'api/deno/network') + symbol = entries.find { |entry| entry.name == 'Deno.connect (Network)' } + + refute_nil symbol + assert_equal 'api/deno/network#Deno.connect', symbol.path + assert_equal 'API', symbol.type + end + + # `Socket` is documented by net, dgram and process; the module name is what + # tells the three entries apart. For node: modules it is the import name. + def test_qualifies_symbol_entries_with_the_module_name + names = filter_fixture('deno_api.html', 'api/deno/network').map(&:name) + + assert_includes names, 'Deno.connect (Network)' + refute_includes names, 'Deno.connect' + end + + # The listing repeats every symbol documented in full below it. + def test_removes_the_symbol_index_after_extracting_entries + html = File.read(File.join(DenoCleanHtmlFilterTest::FIXTURES_PATH, 'deno_api.html')) + doc = Docs::Parser.new(html).html + context = { base_url: Docs::URL.parse('https://docs.deno.com/'), + url: Docs::URL.parse('https://docs.deno.com/api/deno/network'), + root_path: 'api' } + result = { path: 'api/deno/network' } + content = Docs::Deno::CleanHtmlFilter.new(doc, context, result).call + content = Docs::Deno::EntriesFilter.new(content, context, result).call + + assert_nil content.at_css('#symbol-index') + refute_empty content.css('.symbolGroup') + assert_includes content.text, 'Deno.connect' + end + + # Blob is documented on api/web/file; that page owns the entry. + def test_skips_symbols_re_exported_from_another_page + names = filter_fixture('deno_api.html', 'api/deno/network').map(&:name) + + refute_includes names, 'Blob' + refute_includes names, 'Blob (Network)' + end + + def test_skips_namespace_sub_item_links + names = filter_fixture('deno_api.html', 'api/deno/network').map(&:name) + + refute_includes names, 'addr' + refute_includes names, 'addr (Network)' + end + + def test_runtime_page_has_no_symbol_entries + entries = filter_fixture('deno_runtime.html', 'runtime/reference/std/fmt') + + assert_equal 1, entries.size + end + + def test_extracts_runtime_entry_from_content_heading + entry = filter_fixture('deno_runtime.html', 'runtime/reference/std/fmt').first + + assert_equal '@std/fmt', entry.name + assert_equal 'Runtime', entry.type + end + + private + + def filter_fixture(name, path) + html = File.read(File.join(DenoCleanHtmlFilterTest::FIXTURES_PATH, name)) + doc = Docs::Parser.new(html).html + context = { + base_url: Docs::URL.parse('https://docs.deno.com/'), + url: Docs::URL.parse("https://docs.deno.com/#{path}"), + root_path: 'api', + } + result = { path: path } + content = Docs::Deno::CleanHtmlFilter.new(doc, context, result).call + Docs::Deno::EntriesFilter.new(content, context, result).call + result[:entries] + end +end + +class DenoCleanHtmlFilterTest < Minitest::Test + FIXTURES_PATH = File.expand_path('../../../files', __dir__) + + def test_cleans_api_page_chrome_and_preserves_content + output = filter_fixture('deno_api.html') + + assert_equal 'article', output.name + assert_equal 'Network', output.at_css('h1').content + assert_includes output.text, 'Deno.connect' + refute_includes output.text, 'Site navigation' + refute_includes output.text, 'Did you find what you needed?' + assert_empty output.css('.breadcrumbs, .docNodeKindIcon, .copyButton') + assert_empty output.css('.anchor-link') + # The markers sit inside headings and code, where a stray "#" corrupts the text. + assert_equal 'Deno.connect', output.at_css('.symbolGroup h2').content.strip + assert_equal 'options: ConnectOptions', output.at_css('code').content.strip + assert_equal 'ts', output.at_css('pre')['data-language'] + end + + def test_cleans_runtime_page_chrome_and_preserves_content + output = filter_fixture('deno_runtime.html') + + assert_equal '@std/fmt', output.at_css('h1').content + assert_includes output.text, 'Runtime compatibility' + assert_includes output.text, 'Add to your project' + assert_includes output.text, 'See all symbols' + refute_includes output.text, 'On this page' + refute_includes output.text, 'Copy page' + refute_includes output.text, 'Did you find what you needed?' + refute_includes output.text, 'Edit this page' + assert_empty output.css('.copyButton, a.anchor, nav') + assert_equal %w(js sh jsonc), output.css('pre').map { |node| node['data-language'] } + end + + private + + def filter_fixture(name) + html = File.read(File.join(FIXTURES_PATH, name)) + doc = Docs::Parser.new(html).html + Docs::Deno::CleanHtmlFilter.new(doc, {}, {}).call + end +end