Skip to content

Commit d17aafb

Browse files
committed
fix: clean Deno scraper output
Remove source-site navigation and controls that leaked into generated API and runtime pages, while preserving the source language metadata DevDocs uses for syntax highlighting. Add representative regression fixtures for the reviewer-reported page shapes. Test Plan: - Ran the focused Deno suite: 17 runs, 73 assertions, 0 failures. - Ran the remaining Ruby suite excluding two independently failing Windows-only baseline files: 665 runs, 927 assertions, 0 failures. - Compared old and new filters on current Network and @std/fmt source pages; all measured chrome counts dropped to zero while TypeScript, JavaScript, and shell language metadata was preserved. - Ran Ruby syntax checks and git diff --check.
1 parent a320029 commit d17aafb

4 files changed

Lines changed: 153 additions & 9 deletions

File tree

‎lib/docs/filters/deno/clean_html.rb‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,24 @@ module Docs
22
class Deno
33
class CleanHtmlFilter < Filter
44
def call
5-
@doc = at_css('main, article, [role="main"], .markdown-body') || doc
5+
@doc = at_css('main#content article', 'article') || doc
66

7-
css('nav, footer, .sidebar, .breadcrumb, .toc,
8-
.page-nav, .edit-link, .header-anchor, script, style').remove
7+
css('.breadcrumbs', '.copy-page-split', '.copyButton',
8+
'.docNodeKindIcon', '.header-anchor', 'a > svg',
9+
'nav[aria-label="Breadcrumb"]',
10+
'nav[aria-label="Previous and next page"]').remove
11+
12+
css('details > summary').each do |node|
13+
node.parent.remove if node.content.strip == 'On this page'
14+
end
15+
16+
css('h1, h2, h3, h4, h5, h6').each do |node|
17+
node.css('a.anchor[aria-label="Anchor"]').remove
18+
end
919

1020
css('pre > code').each do |node|
11-
if node['class']
12-
lang = node['class'][/language-(\w+)/, 1]
13-
node.parent['data-language'] = lang if lang
14-
end
15-
node.parent['data-language'] ||= 'typescript'
21+
language = node['class'].to_s[/\blanguage-([\w-]+)/, 1]
22+
node.parent['data-language'] = language if language
1623
end
1724

1825
doc

‎test/files/deno_api.html‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<!doctype html>
2+
<html>
3+
<body>
4+
<nav>Site navigation</nav>
5+
<main id="content">
6+
<div class="w-full">
7+
<article>
8+
<div class="markdown-body">
9+
<ul class="breadcrumbs"><li>Deno</li><li>Network</li></ul>
10+
<div class="multiSymbolPage">
11+
<h1>Network</h1>
12+
<p>Networking APIs.</p>
13+
<div class="namespaceItem">
14+
<div class="docNodeKindIcon"><div title="Function">f</div></div>
15+
<a href="#Deno.connect">Deno.connect</a>
16+
</div>
17+
<div class="relative">
18+
<pre><code class="highlight language-ts">await Deno.connect();</code></pre>
19+
<button class="copyButton" title="Copy">Copy</button>
20+
</div>
21+
</div>
22+
</div>
23+
</article>
24+
<section id="feedback-section">Did you find what you needed?</section>
25+
</div>
26+
</main>
27+
<footer>Site footer</footer>
28+
</body>
29+
</html>

‎test/files/deno_runtime.html‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<!doctype html>
2+
<html>
3+
<body>
4+
<main id="content">
5+
<div class="w-full">
6+
<article>
7+
<details><summary>On this page</summary><a href="#overview">Overview</a></details>
8+
<div class="markdown-body">
9+
<header>
10+
<nav aria-label="Breadcrumb">Reference / Standard library</nav>
11+
<div class="copy-page-split"><button>Copy page</button></div>
12+
</header>
13+
<h1>@std/fmt</h1>
14+
<h2 id="overview"><a class="anchor" aria-label="Anchor" href="#overview">#</a>Overview</h2>
15+
<p>Provides utilities for formatting text.</p>
16+
<div class="relative">
17+
<pre><code class="highlight language-js">console.log('formatted');</code></pre>
18+
<button class="copyButton" title="Copy">Copy</button>
19+
</div>
20+
<h2>Runtime compatibility</h2>
21+
<p>Works with Deno.</p>
22+
<h2>Add to your project</h2>
23+
<pre><code class="highlight language-sh">deno add jsr:@std/fmt</code></pre>
24+
<pre><code class="highlight language-jsonc">{"imports": {}}</code></pre>
25+
<a href="https://jsr.io/@std/fmt/doc">See all symbols</a>
26+
</div>
27+
<nav aria-label="Previous and next page">Previous Next</nav>
28+
</article>
29+
<section id="feedback-section">
30+
<h2>Did you find what you needed?</h2>
31+
<a href="https://github.com/denoland/docs">Edit this page</a>
32+
</section>
33+
</div>
34+
</main>
35+
</body>
36+
</html>

‎test/lib/docs/scrapers/deno_test.rb‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
require_relative '../../test_helper'
1+
require_relative '../../../test_helper'
2+
require_relative '../../../../lib/docs'
23

34
class DenoScraperTest < Minitest::Test
45
def setup
@@ -82,3 +83,74 @@ def test_inherits_from_url_scraper
8283
assert @scraper_class < Docs::UrlScraper
8384
end
8485
end
86+
87+
class DenoEntriesFilterTest < Minitest::Test
88+
def test_extracts_api_entry_from_content_heading
89+
entry = filter_fixture('deno_api.html', 'api/deno/network/').first
90+
91+
assert_equal 'Network', entry.name
92+
assert_equal 'API', entry.type
93+
end
94+
95+
def test_extracts_runtime_entry_from_content_heading
96+
entry = filter_fixture('deno_runtime.html', 'runtime/reference/std/fmt/').first
97+
98+
assert_equal '@std/fmt', entry.name
99+
assert_equal 'Runtime', entry.type
100+
end
101+
102+
private
103+
104+
def filter_fixture(name, path)
105+
html = File.read(File.join(DenoCleanHtmlFilterTest::FIXTURES_PATH, name))
106+
doc = Docs::Parser.new(html).html
107+
context = {
108+
base_url: Docs::URL.parse('https://docs.deno.com/'),
109+
url: Docs::URL.parse("https://docs.deno.com/#{path}"),
110+
root_path: 'api/',
111+
}
112+
result = { path: path }
113+
content = Docs::Deno::CleanHtmlFilter.new(doc, context, result).call
114+
Docs::Deno::EntriesFilter.new(content, context, result).call
115+
result[:entries]
116+
end
117+
end
118+
119+
class DenoCleanHtmlFilterTest < Minitest::Test
120+
FIXTURES_PATH = File.expand_path('../../../files', __dir__)
121+
122+
def test_cleans_api_page_chrome_and_preserves_content
123+
output = filter_fixture('deno_api.html')
124+
125+
assert_equal 'article', output.name
126+
assert_equal 'Network', output.at_css('h1').content
127+
assert_includes output.text, 'Deno.connect'
128+
refute_includes output.text, 'Site navigation'
129+
refute_includes output.text, 'Did you find what you needed?'
130+
assert_empty output.css('.breadcrumbs, .docNodeKindIcon, .copyButton')
131+
assert_equal 'ts', output.at_css('pre')['data-language']
132+
end
133+
134+
def test_cleans_runtime_page_chrome_and_preserves_content
135+
output = filter_fixture('deno_runtime.html')
136+
137+
assert_equal '@std/fmt', output.at_css('h1').content
138+
assert_includes output.text, 'Runtime compatibility'
139+
assert_includes output.text, 'Add to your project'
140+
assert_includes output.text, 'See all symbols'
141+
refute_includes output.text, 'On this page'
142+
refute_includes output.text, 'Copy page'
143+
refute_includes output.text, 'Did you find what you needed?'
144+
refute_includes output.text, 'Edit this page'
145+
assert_empty output.css('.copyButton, a.anchor, nav')
146+
assert_equal %w(js sh jsonc), output.css('pre').map { |node| node['data-language'] }
147+
end
148+
149+
private
150+
151+
def filter_fixture(name)
152+
html = File.read(File.join(FIXTURES_PATH, name))
153+
doc = Docs::Parser.new(html).html
154+
Docs::Deno::CleanHtmlFilter.new(doc, {}, {}).call
155+
end
156+
end

0 commit comments

Comments
 (0)