dev site is more agent friendly

.gitignore

@@ -83,3 +83,6 @@ static/_headers
 
 # Build dir for format-rawhtml.ts (npm run format:rawhtml)
 dist/
+
+# Claude Code personal instructions
+CLAUDE.local.md

CLAUDE.md

@@ -64,7 +64,7 @@ Each release note is its own file, `content/<product>/release-notes/YYYY-MM-DD-<
 
 `hugo.toml` defines the sidebar navigation menus (`menus.geoip`, `menus.minfraud`, `menus.general`). Menu structure lives here, not in frontmatter.
 
-Hugo generates both HTML and Markdown output formats for most pages (configured via `[outputs]` and `[outputFormats]`). The home page and the release note listings have no Markdown output.
+Hugo generates both HTML and Markdown output formats for every page (configured via `[outputs]` and `[outputFormats]`). Markdown templates live in `layouts/_default/*.md`, `layouts/index.md`, and `layouts/partials/markdown/`. Hugo also builds `llms.txt` from the home page (`layouts/index.llms.txt`), and `build.sh` concatenates every page's Markdown into `llms-full.txt`.
 
 ### Shortcodes
 

README.md

@@ -64,6 +64,23 @@ when files change.
 hugo server
 ```
 
+#### Markdown for AI Agents
+
+Every page is also built as Markdown at `<page URL>index.md`. Each HTML page
+points to it with `<link rel="alternate" type="text/markdown">` and a "View as
+Markdown" link. Three site-wide files help an agent start:
+
+- `/llms.txt`: an index of the site, built by Hugo from the sidebar menus
+  (`layouts/index.llms.txt`).
+- `/llms-full.txt`: every page's Markdown in one file, built after Hugo by
+  `bin/build-llms-full.ts`. `hugo server` does not serve it. The script fails
+  the build when a page's Markdown holds a link inside an HTML block,
+  which never renders, or an unrendered shortcode or Prettier marker.
+- `/robots.txt`: built by Hugo from `layouts/robots.txt`.
+
+The Markdown files and the two `llms` files carry `X-Robots-Tag: noindex`, so
+search engines index only the HTML pages.
+
 #### Cloudflare Pages HTTP Headers Configuration
 
 The `static/_headers` file is automatically generated from

assets/scss/_page.scss

@@ -76,6 +76,15 @@
   margin-top: 0;
 }
 
+// Matches the 36px height of the copy button beside it
+.page__markdown-link {
+  display: inline-block;
+  font-size: 0.875rem;
+  line-height: 36px;
+  margin: 0 0 1rem 0.5rem;
+  vertical-align: top;
+}
+
 // Copy Markdown button styles
 .copy-markdown-btn {
   align-items: center;

bin/_headers.config.ts

@@ -13,6 +13,20 @@ interface HeadersConfig {
 
 const config: HeadersConfig = {
   paths: [
+    // The Markdown outputs are for AI agents. Keep them out of search
+    // results, where they would duplicate the HTML pages.
+    {
+      pattern: '/*.md',
+      headers: { 'X-Robots-Tag': ['noindex'] },
+    },
+    {
+      pattern: '/llms.txt',
+      headers: { 'X-Robots-Tag': ['noindex'] },
+    },
+    {
+      pattern: '/llms-full.txt',
+      headers: { 'X-Robots-Tag': ['noindex'] },
+    },
     {
       pattern: '/*',
       headers: {

bin/build-llms-full.ts

@@ -0,0 +1,61 @@
+/**
+ * Concatenate the Markdown output of every page into public/llms-full.txt,
+ * one fetch for the whole site. See https://llmstxt.org/.
+ *
+ * Runs after Hugo. Fails when a page holds Markdown that a reader would see
+ * broken: a link inside an HTML block, which never renders, or source
+ * markup that a template failed to turn into Markdown.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+const PUBLIC_DIR = 'public';
+const OUTPUT = path.join(PUBLIC_DIR, 'llms-full.txt');
+const BROKEN = [
+  {
+    // Only a block-level tag starts an HTML block. Inline tags such as <em>
+    // leave Markdown parsing on, and a blank line ends the block.
+    pattern:
+      /<(?:blockquote|dd|details|div|dl|dt|h[1-6]|li|ol|p|section|summary|table|tbody|td|th|thead|tr|ul)(?:\s[^>]*)?>[^\S\n]*\n?[^\S\n]*\[[^\]]+\]\(/i,
+    reason: 'Markdown link inside an HTML block never renders',
+  },
+  {
+    pattern: /\{\{[<%]/,
+    reason: 'Hugo shortcode left unrendered',
+  },
+  {
+    pattern: /<!-- prettier-ignore/,
+    reason: 'Prettier marker left in output',
+  },
+];
+
+// Order by directory, so a section page comes before its children. The home
+// page's directory is "./", and "." sorts before every letter.
+const dir = (file: string) => path.dirname(file) + '/';
+const pages = fs
+  .readdirSync(PUBLIC_DIR, { recursive: true, encoding: 'utf8' })
+  .filter((file) => path.basename(file) === 'index.md')
+  .sort((a, b) => (dir(a) < dir(b) ? -1 : dir(a) > dir(b) ? 1 : 0));
+
+if (pages.length === 0) {
+  console.error(`❌ No index.md files under ${PUBLIC_DIR}. Run hugo first.`);
+  process.exit(1);
+}
+
+const parts: string[] = [];
+for (const page of pages) {
+  const file = path.join(PUBLIC_DIR, page);
+  const markdown = fs.readFileSync(file, 'utf8');
+  for (const { pattern, reason } of BROKEN) {
+    const broken = markdown.match(pattern);
+    if (broken) {
+      console.error(`❌ ${file}: ${reason}: ${broken[0]}`);
+      process.exit(1);
+    }
+  }
+  parts.push(markdown.trimEnd());
+}
+
+fs.writeFileSync(OUTPUT, parts.join('\n\n') + '\n');
+console.log(`Wrote ${OUTPUT} from ${pages.length} pages.`);

build.sh

@@ -5,4 +5,7 @@ set -eu
 # Generate _headers file from TypeScript configuration
 pnpm run build:headers
 
-hugo --gc --minify -b "$CF_PAGES_URL"
+hugo --gc --minify --cleanDestinationDir -b "$CF_PAGES_URL"
+
+# Reads public/, so it must run after hugo
+pnpm run build:llms-full

content/geoip/release-notes/_index.md

@@ -1,7 +1,7 @@
 +++
 title = 'GeoIP Release Notes'
 type = 'release-note'
-outputs = ['html', 'rss', 'anchors']
+outputs = ['html', 'rss', 'anchors', 'markdown']
 +++
 
 {{< alert info >}}

content/maxmind-server-ip-addresses.md

@@ -2,7 +2,7 @@
 draft = false
 layout = 'mm-ips'
 title = 'MaxMind Server IP Addresses'
-outputs = ['rss', 'html']
+outputs = ['rss', 'html', 'markdown']
 _comment = 'json feed is handled by module mounts.  see hugo.toml'
 +++
 

content/minfraud/release-notes/_index.md

@@ -1,7 +1,7 @@
 +++
 title = 'minFraud Release Notes'
 type = 'release-note'
-outputs = ['html', 'rss', 'anchors']
+outputs = ['html', 'rss', 'anchors', 'markdown']
 +++
 
 {{< alert info >}}

content/minfraud/track-devices/android.md

@@ -1,6 +1,7 @@
 +++
 draft = false
 title = 'Android'
+linkTitle = 'Android Device Tracking'
 +++
 
 {{< alert warning >}} The MaxMind Device SDK for Android is currently in beta.

content/minfraud/track-devices/ios.md

@@ -1,6 +1,7 @@
 +++
 draft = false
 title = 'iOS'
+linkTitle = 'iOS Device Tracking'
 +++
 
 {{< alert warning >}} The MaxMind Device SDK for iOS is currently in beta.

content/minfraud/track-devices/web.md

@@ -1,6 +1,7 @@
 +++
 draft = false
 title = 'Web'
+linkTitle = 'Web Device Tracking'
 +++
 
 The Device Tracking Add-On is JavaScript code for you to add to your website. It

cspell.json

@@ -18,5 +18,6 @@
     ],
     "ignoreRegExpList": [
         "/^legacy_anchor = .*$/gm"
-    ]
+    ],
+    "useGitignore": true
 }

hugo.toml

@@ -2,6 +2,7 @@
 baseurl = "/"
 locale = "en-us"
 title = 'MaxMind'
+enableRobotsTXT = true
 
 [markup]
   [markup.goldmark]
@@ -296,17 +297,23 @@ title = 'MaxMind'
     target = "static/maxmind-server-ip-addresses.json"
 
 [outputs]
-  home = ["HTML"]
+  home = ["HTML", "MARKDOWN", "LLMS"]
   page = ["HTML", "MARKDOWN"]
   section = ["HTML", "MARKDOWN"]
-  taxonomy = ["HTML", "MARKDOWN"]
-  term = ["HTML", "MARKDOWN"]
+  taxonomy = ["HTML"]
+  term = ["HTML"]
 
 [outputFormats]
   [outputFormats.MARKDOWN]
     mediaType = "text/markdown"
     baseName = "index"
     notAlternative = true
+  # llms.txt: a Markdown index of the site for AI agents. See
+  # https://llmstxt.org/.
+  [outputFormats.LLMS]
+    mediaType = "text/plain"
+    baseName = "llms"
+    notAlternative = true
 
   # Legacy anchor map for the release note listing; see release-note-anchors.ts.
   [outputFormats.ANCHORS]

layouts/_default/list.md

@@ -1,6 +1,8 @@
-{{- partial "clean-alerts.md" .RenderShortcodes -}}
+{{ partial "markdown/front-matter.md" . }}
+{{ with partial "markdown/body.md" . }}{{ . }}
 
-{{ range .Pages }}
-## [{{ .Title }}]({{ .RelPermalink }})
-{{ .Summary }}
-{{ end }}
+{{ end }}## Pages in this section
+
+{{ range .Pages -}}
+- [{{ .LinkTitle }}]({{ (.OutputFormats.Get "MARKDOWN").Permalink }})
+{{ end -}}

layouts/_default/single.md

@@ -1,7 +1,6 @@
-{{- if and .IsPage (eq .CurrentSection.Type "release-note") (not .Date.IsZero) -}}
-# {{ .Title }}
-
+{{ partial "markdown/front-matter.md" . }}
+{{- if and .IsPage (eq .CurrentSection.Type "release-note") (not .Date.IsZero) }}
 _{{ .Date | time.Format ":date_long" }}_
 {{ partial "release-note-age-notice.md" . }}
-{{- end -}}
-{{- partial "clean-alerts.md" .RenderShortcodes -}}
+{{- end }}
+{{ partial "markdown/body.md" . }}

layouts/index.llms.txt

@@ -0,0 +1,14 @@
+# MaxMind Developer Portal
+
+> Documentation for MaxMind's GeoIP and GeoLite IP intelligence databases and
+> web services, and for the minFraud fraud detection web services.
+
+Every page has a Markdown version at the page URL followed by `index.md`, for
+example {{ "geoip/docs/web-services/responses/index.md" | absURL }}. The whole
+site as one Markdown file is at {{ "llms-full.txt" | absURL }}.
+
+The GeoIP web services are GeoIP Country, GeoIP City Plus, and GeoIP Insights.
+The minFraud web services are minFraud Score, minFraud Insights, and minFraud
+Factors. Each field in the API reference names the services that return it.
+
+{{ partial "markdown/site-index.md" . | safeHTML -}}

layouts/index.md

@@ -0,0 +1,7 @@
+{{ partial "markdown/front-matter.md" . }}
+MaxMind provides IP intelligence and online fraud detection. This site documents
+the GeoIP and GeoLite databases and web services, and the minFraud web services.
+Every page has a Markdown version at the page URL followed by `index.md`. The
+whole site as one Markdown file is at [llms-full.txt]({{ "llms-full.txt" | absURL }}).
+
+{{ partial "markdown/site-index.md" . -}}

layouts/partials/markdown/alert.md

@@ -0,0 +1,10 @@
+{{- /* An alert as a blockquote. Every line gets the quote marker, because a
+       blank line without one would end the quote after the label. */ -}}
+{{- $labels := dict "warning" "⚠️ Warning" "info" "ℹ️ Info" "danger" "🚫 Danger" -}}
+{{- with .kind }}
+{{- $label := index $labels . -}}
+{{- if not $label }}{{ errorf "Unknown alert kind %q" . }}{{ end }}
+> **{{ $label }}**
+>
+{{ end -}}
+{{ replaceRE `(?m)^` "> " (strings.TrimSpace .inner) }}

layouts/partials/markdown/body.md

@@ -0,0 +1,6 @@
+{{- /* Page content with shortcodes rendered by their .md templates. The
+       Prettier markers exist for the source formatter and mean nothing to a
+       reader of the output. */ -}}
+{{- $body := .RenderShortcodes -}}
+{{- $body = replaceRE `<!-- prettier-ignore-(start|end) -->\n*` "" $body -}}
+{{- $body | strings.TrimSpace -}}

layouts/partials/markdown/front-matter.md

@@ -0,0 +1,11 @@
+{{- /* Front matter and H1 for a page's Markdown output. The HTML output has
+       neither: its page template renders the title. */ -}}
+---
+title: {{ .Title | jsonify (dict "noHTMLEscape" true) }}
+{{- with .Description }}
+description: {{ . | jsonify (dict "noHTMLEscape" true) }}
+{{- end }}
+url: {{ (.OutputFormats.Get "HTML").Permalink }}
+---
+
+# {{ .Title }}

layouts/partials/markdown/schema-row.md

@@ -0,0 +1,13 @@
+{{- /* One API field as a headed entry, not a table row: a description can
+       hold paragraphs, lists, and tables, which a Markdown table cell cannot.
+       A request field has no .services, because no service "returns" it. */ -}}
+{{- if and (isset . "services") (not .services) -}}
+  {{- errorf "Field %q is available in no service" .key -}}
+{{- end -}}
+#### `{{ .key }}`
+
+Type: {{ .type }}.{{ with .services }} Available in: {{ delimit . ", " }}.{{ end }}
+
+{{/* The content indents the description by two spaces under the shortcode
+       call. Strip the indent so block syntax inside it renders. */}}
+{{ replaceRE `(?m)^  ` "" .inner | strings.TrimSpace }}

layouts/partials/markdown/site-index.md

@@ -0,0 +1,16 @@
+{{- /* Every page, grouped by top-level section and ordered by path, so a
+       section page comes before its children. Single release notes are
+       left out: there are hundreds, and each listing page links them. */ -}}
+{{- range site.Sections -}}
+## {{ .Title }}
+
+{{ range sort (where site.Pages "Section" .Section) "Path" -}}
+{{- if and .IsPage (eq .CurrentSection.Type "release-note") }}{{ continue }}{{ end -}}
+- [{{ .LinkTitle }}]({{ (.OutputFormats.Get "MARKDOWN").Permalink }})
+{{ end }}
+{{- end }}
+## General
+
+{{ range site.Home.RegularPages -}}
+- [{{ .LinkTitle }}]({{ (.OutputFormats.Get "MARKDOWN").Permalink }})
+{{ end -}}

layouts/partials/page.html

@@ -7,6 +7,9 @@ <h1 class="page__title">{{ .Title }}</h1>
     {{- if and .IsPage (eq .CurrentSection.Type "release-note") (not .Date.IsZero) }}
     <em class="release-note__date">{{ .Date | time.Format ":date_long" }}</em>
     {{- end }}
+    {{ with .OutputFormats.Get "MARKDOWN" }}
+    <a class="page__markdown-link" href="{{ .RelPermalink }}">View as Markdown</a>
+    {{ end }}
     {{ partial "release-note-age-notice.html" . }}
     {{ if eq .Type "has-toc" }}
     <div class="page__toc">

layouts/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: {{ "sitemap.xml" | absURL }}

layouts/shortcodes/alert.md

@@ -1,17 +1,3 @@
-{{- if len .Params | eq 1 -}}
-{{- if eq (index .Params 0) "warning" -}}
-> **⚠️ Warning**
-> 
-{{ .Inner }}
-{{- else if eq (index .Params 0) "info" -}}
-> **ℹ️ Info**
-> 
-{{ .Inner }}
-{{- else -}}
-> **Note**
-> 
-{{ .Inner }}
-{{- end -}}
-{{- else -}}
-> {{ .Inner }}
-{{- end -}}
+{{- $kind := "" -}}
+{{- with .Params }}{{ $kind = index . 0 }}{{ end -}}
+{{ partial "markdown/alert.md" (dict "kind" $kind "inner" .Inner) }}