diff --git a/.gitignore b/.gitignore index c0a2e8c85..bf5f80cee 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,6 @@ static/_headers # Build dir for format-rawhtml.ts (npm run format:rawhtml) dist/ + +# Claude Code personal instructions +CLAUDE.local.md diff --git a/CLAUDE.md b/CLAUDE.md index f83f5e235..a8adacc55 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,7 +64,7 @@ Each release note is its own file, `content//release-notes/YYYY-MM-DD-< `hugo.toml` defines the sidebar navigation menus (`menus.geoip`, `menus.minfraud`, `menus.general`). Menu structure lives here, not in frontmatter. -Hugo generates both HTML and Markdown output formats for most pages (configured via `[outputs]` and `[outputFormats]`). The home page and the release note listings have no Markdown output. +Hugo generates both HTML and Markdown output formats for every page (configured via `[outputs]` and `[outputFormats]`). Markdown templates live in `layouts/_default/*.md`, `layouts/index.md`, and `layouts/partials/markdown/`. Hugo also builds `llms.txt` from the home page (`layouts/index.llms.txt`), and `build.sh` concatenates every page's Markdown into `llms-full.txt`. ### Shortcodes diff --git a/README.md b/README.md index d8ecd98ef..6cc5328b5 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,23 @@ when files change. hugo server ``` +#### Markdown for AI Agents + +Every page is also built as Markdown at `index.md`. Each HTML page +points to it with `` and a "View as +Markdown" link. Three site-wide files help an agent start: + +- `/llms.txt`: an index of the site, built by Hugo from the sidebar menus + (`layouts/index.llms.txt`). +- `/llms-full.txt`: every page's Markdown in one file, built after Hugo by + `bin/build-llms-full.ts`. `hugo server` does not serve it. The script fails + the build when a page's Markdown holds a link inside an HTML block, + which never renders, or an unrendered shortcode or Prettier marker. +- `/robots.txt`: built by Hugo from `layouts/robots.txt`. + +The Markdown files and the two `llms` files carry `X-Robots-Tag: noindex`, so +search engines index only the HTML pages. + #### Cloudflare Pages HTTP Headers Configuration The `static/_headers` file is automatically generated from diff --git a/assets/scss/_page.scss b/assets/scss/_page.scss index 495d5127c..0e328c987 100644 --- a/assets/scss/_page.scss +++ b/assets/scss/_page.scss @@ -76,6 +76,15 @@ margin-top: 0; } +// Matches the 36px height of the copy button beside it +.page__markdown-link { + display: inline-block; + font-size: 0.875rem; + line-height: 36px; + margin: 0 0 1rem 0.5rem; + vertical-align: top; +} + // Copy Markdown button styles .copy-markdown-btn { align-items: center; diff --git a/bin/_headers.config.ts b/bin/_headers.config.ts index 481c94fb9..ffe676285 100644 --- a/bin/_headers.config.ts +++ b/bin/_headers.config.ts @@ -13,6 +13,20 @@ interface HeadersConfig { const config: HeadersConfig = { paths: [ + // The Markdown outputs are for AI agents. Keep them out of search + // results, where they would duplicate the HTML pages. + { + pattern: '/*.md', + headers: { 'X-Robots-Tag': ['noindex'] }, + }, + { + pattern: '/llms.txt', + headers: { 'X-Robots-Tag': ['noindex'] }, + }, + { + pattern: '/llms-full.txt', + headers: { 'X-Robots-Tag': ['noindex'] }, + }, { pattern: '/*', headers: { diff --git a/bin/build-llms-full.ts b/bin/build-llms-full.ts new file mode 100644 index 000000000..9c1fe98d6 --- /dev/null +++ b/bin/build-llms-full.ts @@ -0,0 +1,61 @@ +/** + * Concatenate the Markdown output of every page into public/llms-full.txt, + * one fetch for the whole site. See https://llmstxt.org/. + * + * Runs after Hugo. Fails when a page holds Markdown that a reader would see + * broken: a link inside an HTML block, which never renders, or source + * markup that a template failed to turn into Markdown. + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +const PUBLIC_DIR = 'public'; +const OUTPUT = path.join(PUBLIC_DIR, 'llms-full.txt'); +const BROKEN = [ + { + // Only a block-level tag starts an HTML block. Inline tags such as + // leave Markdown parsing on, and a blank line ends the block. + pattern: + /<(?:blockquote|dd|details|div|dl|dt|h[1-6]|li|ol|p|section|summary|table|tbody|td|th|thead|tr|ul)(?:\s[^>]*)?>[^\S\n]*\n?[^\S\n]*\[[^\]]+\]\(/i, + reason: 'Markdown link inside an HTML block never renders', + }, + { + pattern: /\{\{[<%]/, + reason: 'Hugo shortcode left unrendered', + }, + { + pattern: /\n*` "" $body -}} +{{- $body | strings.TrimSpace -}} diff --git a/layouts/partials/markdown/front-matter.md b/layouts/partials/markdown/front-matter.md new file mode 100644 index 000000000..61e73ca9f --- /dev/null +++ b/layouts/partials/markdown/front-matter.md @@ -0,0 +1,11 @@ +{{- /* Front matter and H1 for a page's Markdown output. The HTML output has + neither: its page template renders the title. */ -}} +--- +title: {{ .Title | jsonify (dict "noHTMLEscape" true) }} +{{- with .Description }} +description: {{ . | jsonify (dict "noHTMLEscape" true) }} +{{- end }} +url: {{ (.OutputFormats.Get "HTML").Permalink }} +--- + +# {{ .Title }} diff --git a/layouts/partials/markdown/schema-row.md b/layouts/partials/markdown/schema-row.md new file mode 100644 index 000000000..646a17f00 --- /dev/null +++ b/layouts/partials/markdown/schema-row.md @@ -0,0 +1,13 @@ +{{- /* One API field as a headed entry, not a table row: a description can + hold paragraphs, lists, and tables, which a Markdown table cell cannot. + A request field has no .services, because no service "returns" it. */ -}} +{{- if and (isset . "services") (not .services) -}} + {{- errorf "Field %q is available in no service" .key -}} +{{- end -}} +#### `{{ .key }}` + +Type: {{ .type }}.{{ with .services }} Available in: {{ delimit . ", " }}.{{ end }} + +{{/* The content indents the description by two spaces under the shortcode + call. Strip the indent so block syntax inside it renders. */}} +{{ replaceRE `(?m)^ ` "" .inner | strings.TrimSpace }} diff --git a/layouts/partials/markdown/site-index.md b/layouts/partials/markdown/site-index.md new file mode 100644 index 000000000..c522a89de --- /dev/null +++ b/layouts/partials/markdown/site-index.md @@ -0,0 +1,16 @@ +{{- /* Every page, grouped by top-level section and ordered by path, so a + section page comes before its children. Single release notes are + left out: there are hundreds, and each listing page links them. */ -}} +{{- range site.Sections -}} +## {{ .Title }} + +{{ range sort (where site.Pages "Section" .Section) "Path" -}} +{{- if and .IsPage (eq .CurrentSection.Type "release-note") }}{{ continue }}{{ end -}} +- [{{ .LinkTitle }}]({{ (.OutputFormats.Get "MARKDOWN").Permalink }}) +{{ end }} +{{- end }} +## General + +{{ range site.Home.RegularPages -}} +- [{{ .LinkTitle }}]({{ (.OutputFormats.Get "MARKDOWN").Permalink }}) +{{ end -}} diff --git a/layouts/partials/page.html b/layouts/partials/page.html index 1259329bd..6660f02a1 100644 --- a/layouts/partials/page.html +++ b/layouts/partials/page.html @@ -7,6 +7,9 @@

{{ .Title }}

{{- if and .IsPage (eq .CurrentSection.Type "release-note") (not .Date.IsZero) }} {{ .Date | time.Format ":date_long" }} {{- end }} + {{ with .OutputFormats.Get "MARKDOWN" }} + View as Markdown + {{ end }} {{ partial "release-note-age-notice.html" . }} {{ if eq .Type "has-toc" }}
diff --git a/layouts/robots.txt b/layouts/robots.txt new file mode 100644 index 000000000..6b4c001c4 --- /dev/null +++ b/layouts/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: {{ "sitemap.xml" | absURL }} diff --git a/layouts/shortcodes/alert.md b/layouts/shortcodes/alert.md index a2b97a678..1328cd9fa 100644 --- a/layouts/shortcodes/alert.md +++ b/layouts/shortcodes/alert.md @@ -1,17 +1,3 @@ -{{- if len .Params | eq 1 -}} -{{- if eq (index .Params 0) "warning" -}} -> **⚠️ Warning** -> -{{ .Inner }} -{{- else if eq (index .Params 0) "info" -}} -> **ℹ️ Info** -> -{{ .Inner }} -{{- else -}} -> **Note** -> -{{ .Inner }} -{{- end -}} -{{- else -}} -> {{ .Inner }} -{{- end -}} \ No newline at end of file +{{- $kind := "" -}} +{{- with .Params }}{{ $kind = index . 0 }}{{ end -}} +{{ partial "markdown/alert.md" (dict "kind" $kind "inner" .Inner) }} diff --git a/layouts/shortcodes/geoip-schema-row.md b/layouts/shortcodes/geoip-schema-row.md index 99974c4ba..f12c53868 100644 --- a/layouts/shortcodes/geoip-schema-row.md +++ b/layouts/shortcodes/geoip-schema-row.md @@ -1,28 +1,11 @@ -{{- $key := .Get "key" -}} -{{- $country := .Get "country" -}} -{{- $city := .Get "city" -}} -{{- $insights := .Get "insights" -}} -{{- $hasCountry := eq $country "true" -}} -{{- $hasCity := eq $city "true" -}} -{{- $hasInsights := eq $insights "true" -}} - - - {{ $key }} - - - {{ .Get "valueType" }} - - - {{ .Inner | .Page.RenderString }} - {{ with .Get "valueTypeNote" }} -

- {{ . }} -

- {{ end }} -
- {{ if $hasCountry }}✓{{ else }}✗{{ end }} GeoIP Country - {{ if $hasCity }}✓{{ else }}✗{{ end }} GeoIP City Plus - {{ if $hasInsights }}✓{{ else }}✗{{ end }} GeoIP Insights -
- - +{{- $type := .Get "valueType" -}} +{{- with .Get "valueTypeNote" }}{{ $type = printf "%s (%s)" $type . }}{{ end -}} +{{- $services := slice -}} +{{- if eq (.Get "country") "true" }}{{ $services = $services | append "GeoIP Country" }}{{ end -}} +{{- if eq (.Get "city") "true" }}{{ $services = $services | append "GeoIP City Plus" }}{{ end -}} +{{- if eq (.Get "insights") "true" }}{{ $services = $services | append "GeoIP Insights" }}{{ end -}} +{{- /* This comment ends without a trim marker on purpose. The content + indents each shortcode call by two spaces. The newline after the + comment moves those spaces onto a blank line, so the heading starts at + column one. */}} +{{ partial "markdown/schema-row.md" (dict "key" (.Get "key") "type" $type "services" $services "inner" .Inner) }} diff --git a/layouts/shortcodes/minfraud-schema-row.md b/layouts/shortcodes/minfraud-schema-row.md index 13a56ea4c..32105fdbc 100644 --- a/layouts/shortcodes/minfraud-schema-row.md +++ b/layouts/shortcodes/minfraud-schema-row.md @@ -1,31 +1,15 @@ -{{- $key := .Get "key" -}} -{{- $type := .Get "type" -}} -{{- $score := .Get "score" -}} -{{- $insights := .Get "insights" -}} -{{- $factors := .Get "factors" -}} -{{- $hasScore := eq $score "true" -}} -{{- $hasInsights := eq $insights "true" -}} -{{- $hasFactors := eq $factors "true" -}} - - - {{ $key }} - - - {{ .Get "valueType" }} - - - {{ .Inner | .Page.RenderString }} - {{ with .Get "valueTypeNote" }} -

- {{ . }} -

- {{ end }} - {{ if eq $type "response" }} -
- {{ if $hasScore }}✓{{ else }}✗{{ end }} minFraud Score - {{ if $hasInsights }}✓{{ else }}✗{{ end }} minFraud Insights - {{ if $hasFactors }}✓{{ else }}✗{{ end }} minFraud Factors -
- {{ end }} - - +{{- $type := .Get "valueType" -}} +{{- with .Get "valueTypeNote" }}{{ $type = printf "%s (%s)" $type . }}{{ end -}} +{{- $row := dict "key" (.Get "key") "type" $type "inner" .Inner -}} +{{- if eq (.Get "type") "response" -}} + {{- $services := slice -}} + {{- if eq (.Get "score") "true" }}{{ $services = $services | append "minFraud Score" }}{{ end -}} + {{- if eq (.Get "insights") "true" }}{{ $services = $services | append "minFraud Insights" }}{{ end -}} + {{- if eq (.Get "factors") "true" }}{{ $services = $services | append "minFraud Factors" }}{{ end -}} + {{- $row = merge $row (dict "services" $services) -}} +{{- end -}} +{{- /* This comment ends without a trim marker on purpose. The content + indents each shortcode call by two spaces. The newline after the + comment moves those spaces onto a blank line, so the heading starts at + column one. */}} +{{ partial "markdown/schema-row.md" $row }} diff --git a/layouts/shortcodes/schema-table.md b/layouts/shortcodes/schema-table.md index b897fc698..df8950393 100644 --- a/layouts/shortcodes/schema-table.md +++ b/layouts/shortcodes/schema-table.md @@ -1,12 +1,5 @@ - - - - - - - - - - {{- .Inner -}} - -
KeyValue TypeDescription
\ No newline at end of file +{{- /* Rows render as headed entries, not a table: a description can hold + paragraphs, lists, and tables, which a Markdown table cell cannot. The + source indents each row call, which leaves whitespace-only lines + between rows. */ -}} +{{ replaceRE `\n\s*\n` "\n\n" .Inner | strings.TrimSpace }} diff --git a/layouts/shortcodes/snippet.md b/layouts/shortcodes/snippet.md index 7286415fd..9933ceb13 100644 --- a/layouts/shortcodes/snippet.md +++ b/layouts/shortcodes/snippet.md @@ -13,4 +13,14 @@ {{- errorf "The %q shortcode was unable to find %q. See %s" .Name $asset .Position}} {{- end }} -{{- $r.Content -}} \ No newline at end of file +{{- /* The HTML shortcode passes the snippet through RenderString, which + also runs the shortcodes inside it. Here that would turn Markdown into + HTML, so the snippet is used as is and its alert shortcodes stay + literal. Render them the way the alert shortcode does. */ -}} +{{- $md := $r.Content -}} +{{- range findRE `(?s)\{\{<\s*alert\s*\w*\s*>\}\}.*?\{\{\}\}` $md -}} + {{- $kind := replaceRE `(?s)^\{\{<\s*alert\s*(\w*)\s*>\}\}.*` "$1" . -}} + {{- $inner := replaceRE `(?s)^\{\{<[^>]*>\}\}(.*)\{\{\}\}$` "$1" . -}} + {{- $md = replace $md . (partial "markdown/alert.md" (dict "kind" $kind "inner" $inner)) -}} +{{- end -}} +{{- $md -}} \ No newline at end of file diff --git a/package.json b/package.json index f43c2b8b7..2714eba37 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ }, "scripts": { "build:headers": "node --experimental-strip-types bin/generate-headers.ts", + "build:llms-full": "node --experimental-strip-types bin/build-llms-full.ts", "fix": "run-p fix:*", "fix:scripts": "pnpm run lint:scripts --fix", "fix:styles": "pnpm run lint:styles --fix", diff --git a/stopwords.txt b/stopwords.txt index 9ab803b4d..27c5cca1d 100644 --- a/stopwords.txt +++ b/stopwords.txt @@ -73,6 +73,7 @@ isset JCOM jgeoip Jong +jsonify Jönköping jsonl Kalifornien @@ -81,6 +82,7 @@ libmaxminddb libvmod lifecycleScope Linkem +llms Ltda Luhansk Magento