diff --git a/composer.json b/composer.json
index a989234..1bfc468 100644
--- a/composer.json
+++ b/composer.json
@@ -10,6 +10,7 @@
"erusev/parsedown": "^1.0",
"symfony/console": "^8",
"symfony/dotenv": "^8",
+ "symfony/filesystem": "^8",
"symfony/flex": "^2",
"symfony/framework-bundle": "^8",
"symfony/http-client": "^8",
diff --git a/composer.lock b/composer.lock
index 8736bed..dd0d438 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
- "content-hash": "54d95b5f60c6c0a946e8f49a272d3e48",
+ "content-hash": "29afd44a707bd6883159f536422531d7",
"packages": [
{
"name": "composer/ca-bundle",
diff --git a/src/Controller/DocsController.php b/src/Controller/DocsController.php
index 9eef09a..4577755 100644
--- a/src/Controller/DocsController.php
+++ b/src/Controller/DocsController.php
@@ -3,87 +3,88 @@
namespace App\Controller;
use Composer\Pcre\Preg;
+use Symfony\Component\Filesystem\Path;
use Symfony\Component\Finder\Finder;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\Routing\Attribute\Route;
+use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
use Symfony\Component\HttpFoundation\RedirectResponse;
+/**
+ * @phpstan-type Page array{page: string, metadata: array|null, contents: string}
+ */
class DocsController extends AbstractController
{
+ /** old page => [new page, fragment] */
+ private const array MOVED_PAGES = [
+ 'articles/handling-private-packages-with-satis.md' => ['articles/handling-private-packages.md', null],
+ 'articles/http-basic-authentication.md' => ['articles/authentication-for-private-packages.md', 'http-basic'],
+ 'faqs/why-can\'t-composer-load-repositories-recursively.md' => ['faqs/why-cant-composer-load-repositories-recursively.md', null],
+ ];
+
#[Route("/doc/", name: "docs")]
public function index(string $docDir): Response
{
- $scan = function ($dir, $prefix = '') {
- $finder = new Finder();
- $finder->files()
- ->in($dir)
- ->sortByName()
- ->depth(0);
-
- $filenames = [];
-
- foreach ($finder as $file) {
- $filename = basename($file->getPathname(), '.md');
- $url = $this->generateUrl(
- 'docs.view',
- ['page' => $prefix.$filename.'.md']
- );
-
- $metadata = null;
- $contents = file_get_contents($file->getPathname());
- assert(is_string($contents));
- if (Preg::isMatchStrictGroups('{^}s', $contents, $match)) {
- Preg::matchAllStrictGroups('{^ *(?P\w+): *(?P.*)}m', $match[1], $matches);
- $metadata = [];
- foreach ($matches['keyword'] as $key => $keyword) {
- $metadata[$keyword] = $matches['value'][$key];
- }
- }
-
- if (Preg::isMatch('{^(\n+)?# (.+?)\n}s', $contents, $match)) {
- $displayName = $match[2];
- } else {
- $displayName = Preg::replace('{^\d{2} }', '', ucwords(str_replace('-', ' ', $filename)));
- }
- $filenames[$displayName] = ['link' => $url, 'metadata' => $metadata];
- }
-
- return $filenames;
- };
-
- $book = $scan($docDir);
- $articles = $scan($docDir.'/articles', 'articles/');
- $faqs = $scan($docDir.'/faqs', 'faqs/');
+ $sections = $this->listSections($docDir);
return $this->render('doc.list.html.twig', [
- 'book' => $book,
- 'articles' => $articles,
- 'faqs' => $faqs,
+ 'book' => $sections['Book'],
+ 'articles' => $sections['Articles'],
+ 'faqs' => $sections['FAQs'],
'page' => 'docs'
]);
}
- #[Route("/doc/{page}", name: "docs.view", requirements: ["page" => "[a-z0-9/\'-]+\.md"])]
- public function viewPage(string $page, string $docDir, \Parsedown $parsedown): Response
+ #[Route("/llms.txt", name: "llms_txt")]
+ public function llmsTxt(string $docDir): Response
+ {
+ return $this->render('llms.txt.twig', [
+ 'sections' => $this->listSections($docDir),
+ ], $this->createTextResponse('text/plain'));
+ }
+
+ #[Route("/llms-full.txt", name: "llms_full_txt")]
+ public function llmsFullTxt(string $docDir): Response
+ {
+ $pages = [];
+ foreach ($this->listSections($docDir) as $section) {
+ foreach ($section as $data) {
+ $url = $this->generateUrl('docs.view', ['page' => $data['page']], UrlGeneratorInterface::ABSOLUTE_URL);
+ $pages[] = 'Source: '.$url."\n\n".$this->toMarkdown($data['page'], $data['contents']);
+ }
+ }
+
+ return $this->createTextResponse('text/plain')->setContent(implode("\n\n---\n\n", $pages));
+ }
+
+ #[Route("/doc/{page}.md", name: "docs.markdown", requirements: ["page" => "[a-z0-9/\'-]+\.md"])]
+ public function viewMarkdown(string $page, string $docDir): Response
{
$filename = $docDir.'/'.$page;
if (!file_exists($filename)) {
- if ($page === 'articles/handling-private-packages-with-satis.md') {
- return new RedirectResponse('/doc/articles/handling-private-packages.md', 301);
- }
+ return $this->redirectMovedPage($page, 'docs.markdown');
+ }
- if ($page === 'articles/http-basic-authentication.md') {
- return new RedirectResponse('/doc/articles/authentication-for-private-packages.md#http-basic', 301);
- }
+ $contents = file_get_contents($filename);
+ assert(is_string($contents));
+ $canonical = $this->generateUrl('docs.view', ['page' => $page], UrlGeneratorInterface::ABSOLUTE_URL);
- if ($page === 'faqs/why-can\'t-composer-load-repositories-recursively.md') {
- return new RedirectResponse('/doc/faqs/why-cant-composer-load-repositories-recursively.md', 301);
- }
+ $response = $this->createTextResponse('text/markdown')->setContent($this->toMarkdown($page, $contents));
+ $response->headers->set('Link', '<'.$canonical.'>; rel="canonical"');
- throw $this->createNotFoundException('Requested page was not found.');
+ return $response;
+ }
+
+ #[Route("/doc/{page}", name: "docs.view", requirements: ["page" => "[a-z0-9/\'-]+\.md"])]
+ public function viewPage(string $page, string $docDir, \Parsedown $parsedown): Response
+ {
+ $filename = $docDir.'/'.$page;
+
+ if (!file_exists($filename)) {
+ return $this->redirectMovedPage($page, 'docs.view');
}
$contents = file_get_contents($filename);
@@ -237,4 +238,120 @@ public function viewPage(string $page, string $docDir, \Parsedown $parsedown): R
'title' => $firstTitle
]);
}
+
+ private function redirectMovedPage(string $page, string $route): RedirectResponse
+ {
+ if (!isset(self::MOVED_PAGES[$page])) {
+ throw $this->createNotFoundException('Requested page was not found.');
+ }
+
+ [$target, $fragment] = self::MOVED_PAGES[$page];
+ $params = ['page' => $target];
+ if ($fragment !== null) {
+ $params['_fragment'] = $fragment;
+ }
+
+ return $this->redirectToRoute($route, $params, 301);
+ }
+
+ private function createTextResponse(string $type): Response
+ {
+ $response = new Response('', 200, ['Content-Type' => $type.'; charset=UTF-8']);
+ $response->setPublic();
+ $response->setMaxAge(3600);
+
+ return $response;
+ }
+
+ /**
+ * @return array{Book: array, Articles: array, FAQs: array}
+ */
+ private function listSections(string $docDir): array
+ {
+ return [
+ 'Book' => $this->listPages($docDir),
+ 'Articles' => $this->listPages($docDir.'/articles', 'articles/'),
+ 'FAQs' => $this->listPages($docDir.'/faqs', 'faqs/'),
+ ];
+ }
+
+ /**
+ * @return array keyed by display name
+ */
+ private function listPages(string $dir, string $prefix = ''): array
+ {
+ $finder = new Finder();
+ $finder->files()
+ ->in($dir)
+ ->sortByName()
+ ->depth(0);
+
+ $pages = [];
+
+ foreach ($finder as $file) {
+ $filename = basename($file->getPathname(), '.md');
+ $page = $prefix.$filename.'.md';
+
+ $metadata = null;
+ $contents = file_get_contents($file->getPathname());
+ assert(is_string($contents));
+ if (Preg::isMatchStrictGroups('{^}s', $contents, $match)) {
+ Preg::matchAllStrictGroups('{^ *(?P\w+): *(?P.*)}m', $match[1], $matches);
+ $metadata = [];
+ foreach ($matches['keyword'] as $key => $keyword) {
+ $metadata[$keyword] = $matches['value'][$key];
+ }
+ }
+
+ if (Preg::isMatch('{^(\n+)?# (.+?)\n}s', $contents, $match)) {
+ $displayName = $match[2];
+ } else {
+ $displayName = Preg::replace('{^\d{2} }', '', ucwords(str_replace('-', ' ', $filename)));
+ }
+ $pages[$displayName] = [
+ 'page' => $page,
+ 'metadata' => $metadata,
+ 'contents' => $contents,
+ ];
+ }
+
+ return $pages;
+ }
+
+ /**
+ * Returns the page source without its metadata comment, with doc links pointing to the absolute
+ * markdown URLs so they still resolve when served outside /doc/ or concatenated in llms-full.txt
+ */
+ private function toMarkdown(string $page, string $contents): string
+ {
+ $contents = Preg::replace('{^\n+}s', '', $contents);
+
+ // code fences and spans are skipped, then inline [text](target) and reference [n]: target
+ // links are rewritten, except external, absolute and non-.md ones
+ return Preg::replaceCallback(
+ '{(?:^ *(`{3,}|~{3,})[\s\S]*?(?:^ *\1[`~]* *$|\z)|(`+)[\s\S]*?\2)(*SKIP)(*FAIL)'
+ .'|(\]\(|^ {0,3}\[[^\]]+\]:[ \t]*)(?!\w+:|/)([a-z0-9/._\'-]+\.md(?=[)#\s]|$))?(#[^)\s]*)?}m',
+ function (array $match) use ($page): string {
+ $target = $match[4] ?? '';
+ $fragment = $match[5] ?? '';
+ if ($target === '' && $fragment === '') {
+ return (string) $match[0];
+ }
+
+ $resolved = $page;
+ if ($target !== '') {
+ $dir = dirname($page);
+ $resolved = Path::canonicalize(($dir === '.' ? '' : $dir.'/').$target);
+ }
+
+ // also rejects targets climbing out of the doc dir, which canonicalize leaves as ../
+ if (!Preg::isMatch('{^[a-z0-9/\'-]+\.md$}', $resolved)) {
+ return (string) $match[0];
+ }
+
+ return $match[3].$this->generateUrl('docs.markdown', ['page' => $resolved], UrlGeneratorInterface::ABSOLUTE_URL).$fragment;
+ },
+ $contents
+ );
+ }
}
diff --git a/templates/doc.list.html.twig b/templates/doc.list.html.twig
index 1736424..590ca81 100644
--- a/templates/doc.list.html.twig
+++ b/templates/doc.list.html.twig
@@ -35,6 +35,14 @@
+ For LLMs
+ The documentation is also available as markdown: see
+ llms.txt
+ for an index, or
+ llms-full.txt
+ for all of it in a single file. Every page can also be fetched as markdown by appending .md to its URL.
+
+
Translations
The Composer documentation is also available in other languages:
@@ -49,7 +57,7 @@
{% macro renderFile(filename, data) %}
-
{{ filename }}
diff --git a/templates/doc.show.html.twig b/templates/doc.show.html.twig
index ee7df81..4736d6e 100644
--- a/templates/doc.show.html.twig
+++ b/templates/doc.show.html.twig
@@ -4,6 +4,10 @@
{% block title %}{% if title is not empty %}{{ title }} - {% endif %}{{ parent() }}{% endblock %}
+{% block head %}
+
+{% endblock %}
+
{% block content %}
diff --git a/templates/layout.html.twig b/templates/layout.html.twig
index bc7024b..a0d0b80 100644
--- a/templates/layout.html.twig
+++ b/templates/layout.html.twig
@@ -8,6 +8,7 @@
+ {% block head %}{% endblock %}
diff --git a/templates/llms.txt.twig b/templates/llms.txt.twig
new file mode 100644
index 0000000..22b2f29
--- /dev/null
+++ b/templates/llms.txt.twig
@@ -0,0 +1,19 @@
+# Composer
+
+> Composer is a dependency manager for PHP. You declare the libraries your project depends on in composer.json, and Composer installs and updates them for you. Packages are published on Packagist (https://packagist.org/).
+
+The links below point to the markdown source of each documentation page. All of it is also available as a single file at {{ url('llms_full_txt') }}
+{% for section, pages in sections %}
+
+## {{ section }}
+
+{% for title, data in pages %}
+- [{{ title }}]({{ url('docs.markdown', {page: data.page}) }}){% if data.metadata.tagline is defined %}: {{ data.metadata.tagline }}{% endif %}
+
+{% endfor %}
+{% endfor %}
+
+## Optional
+
+- [composer.json schema]({{ url('schema') }}): JSON schema for validating composer.json files
+- [Changelog](https://raw.githubusercontent.com/composer/composer/main/CHANGELOG.md): changes in each Composer release
diff --git a/tests/Controller/DocsControllerTest.php b/tests/Controller/DocsControllerTest.php
index bc388be..8830f63 100644
--- a/tests/Controller/DocsControllerTest.php
+++ b/tests/Controller/DocsControllerTest.php
@@ -20,6 +20,8 @@ public function testIndex(): void
self::assertStringContainsString('Why can\'t Composer load repositories recursively?', $text);
self::assertCount(1, $crawler->filter('a[href="/doc/articles/handling-private-packages.md"]'));
self::assertCount(1, $crawler->filter('a[href="/doc/faqs/why-cant-composer-load-repositories-recursively.md"]'));
+ self::assertCount(1, $crawler->filter('a[href="/llms.txt"]'));
+ self::assertCount(1, $crawler->filter('a[href="/llms-full.txt"]'));
}
public function testViewPage(): void
@@ -80,4 +82,94 @@ public function testMissingPage(): void
self::assertResponseStatusCodeSame(404);
}
+
+ public function testViewPageLinksToMarkdown(): void
+ {
+ static::createClient()->request('GET', '/doc/01-basic-usage.md');
+
+ self::assertSelectorExists('link[rel="alternate"][type="text/markdown"][href="/doc/01-basic-usage.md.md"]');
+ }
+
+ public function testMarkdown(): void
+ {
+ $client = static::createClient();
+ $client->request('GET', '/doc/01-basic-usage.md.md');
+
+ self::assertResponseIsSuccessful();
+ self::assertResponseHeaderSame('Content-Type', 'text/markdown; charset=UTF-8');
+ self::assertResponseHeaderSame('Link', '
; rel="canonical"');
+
+ $content = self::getContent($client);
+ // metadata comment is stripped and relative links are made absolute
+ self::assertStringStartsWith("# Basic usage\n", $content);
+ self::assertStringContainsString('[config docs](http://localhost/doc/06-config.md.md)', $content);
+ self::assertStringContainsString('[Libraries](http://localhost/doc/02-libraries.md.md)', $content);
+ self::assertStringContainsString('[README](https://example.org/README.md)', $content);
+ self::assertStringContainsString('[1]: http://localhost/doc/articles/handling-private-packages.md.md#setup', $content);
+ self::assertStringContainsString('[below](http://localhost/doc/01-basic-usage.md.md#details)', $content);
+ self::assertStringContainsString('`^[a-z0-9]([_.-]?[a-z0-9]+)*$`', $content);
+ self::assertStringContainsString(' [2]: http://localhost/doc/02-libraries.md.md#intro', $content);
+ self::assertStringContainsString('[notes](README.md.txt), [mdx](guide.mdx), [absolute](/doc/04-schema.md),', $content);
+ self::assertStringContainsString('[outside](../../04-schema.md) and `[inline](02-libraries.md)`.', $content);
+ self::assertStringContainsString("```text\n[fenced](02-libraries.md)\n[3]: 02-libraries.md\n```", $content);
+ self::assertResponseHeaderSame('Cache-Control', 'max-age=3600, public');
+ }
+
+ #[DataProvider('provideMovedPages')]
+ public function testMovedMarkdownPagesRedirect(string $path, string $target): void
+ {
+ static::createClient()->request('GET', str_replace('.md', '.md.md', $path));
+
+ self::assertResponseStatusCodeSame(301);
+ self::assertResponseRedirects(str_replace('.md', '.md.md', $target));
+ }
+
+ public function testMarkdownResolvesParentLinks(): void
+ {
+ $client = static::createClient();
+ $client->request('GET', '/doc/faqs/why-cant-composer-load-repositories-recursively.md.md');
+
+ self::assertResponseIsSuccessful();
+ self::assertStringContainsString('[schema](http://localhost/doc/04-schema.md.md#repositories)', self::getContent($client));
+ }
+
+ public function testMissingMarkdown(): void
+ {
+ static::createClient()->request('GET', '/doc/99-nope.md.md');
+
+ self::assertResponseStatusCodeSame(404);
+ }
+
+ public function testLlmsTxt(): void
+ {
+ $client = static::createClient();
+ $client->request('GET', '/llms.txt');
+
+ self::assertResponseIsSuccessful();
+ self::assertResponseHeaderSame('Content-Type', 'text/plain; charset=UTF-8');
+
+ $content = self::getContent($client);
+ self::assertStringStartsWith("# Composer\n", $content);
+ self::assertStringContainsString("## Book\n\n- [Introduction](http://localhost/doc/00-intro.md.md)\n- [Basic usage](http://localhost/doc/01-basic-usage.md.md): Learn the basics\n", $content);
+ self::assertStringContainsString('- [Handling private packages](http://localhost/doc/articles/handling-private-packages.md.md)', $content);
+ self::assertStringContainsString('- [Why can\'t Composer load repositories recursively?](http://localhost/doc/faqs/why-cant-composer-load-repositories-recursively.md.md)', $content);
+ self::assertStringContainsString('http://localhost/llms-full.txt', $content);
+ self::assertResponseHeaderSame('Cache-Control', 'max-age=3600, public');
+ }
+
+ public function testLlmsFullTxt(): void
+ {
+ $client = static::createClient();
+ $client->request('GET', '/llms-full.txt');
+
+ self::assertResponseIsSuccessful();
+ self::assertResponseHeaderSame('Content-Type', 'text/plain; charset=UTF-8');
+
+ $content = self::getContent($client);
+ self::assertStringStartsWith("Source: http://localhost/doc/00-intro.md\n\n# Introduction\n", $content);
+ self::assertStringContainsString("\n\n---\n\nSource: http://localhost/doc/01-basic-usage.md\n\n# Basic usage\n", $content);
+ self::assertStringContainsString('Source: http://localhost/doc/faqs/why-cant-composer-load-repositories-recursively.md', $content);
+ self::assertStringNotContainsString('tagline:', $content);
+ self::assertResponseHeaderSame('Cache-Control', 'max-age=3600, public');
+ }
}
diff --git a/tests/Fixtures/composer/doc/01-basic-usage.md b/tests/Fixtures/composer/doc/01-basic-usage.md
index da2130c..271c26d 100644
--- a/tests/Fixtures/composer/doc/01-basic-usage.md
+++ b/tests/Fixtures/composer/doc/01-basic-usage.md
@@ -14,6 +14,21 @@
See the [config docs](06-config.md).
+More in the [README](https://example.org/README.md), the [setup guide][1] and [below](#details).
+
+Names match `^[a-z0-9]([_.-]?[a-z0-9]+)*$`.
+
+[1]: articles/handling-private-packages.md#setup
+ [2]: 02-libraries.md#intro
+
+Left as is: [notes](README.md.txt), [mdx](guide.mdx), [absolute](/doc/04-schema.md),
+[outside](../../04-schema.md) and `[inline](02-libraries.md)`.
+
+```text
+[fenced](02-libraries.md)
+[3]: 02-libraries.md
+```
+
```json
{"require": {}}
```
diff --git a/tests/Fixtures/composer/doc/faqs/why-cant-composer-load-repositories-recursively.md b/tests/Fixtures/composer/doc/faqs/why-cant-composer-load-repositories-recursively.md
index f0f7994..d07381d 100644
--- a/tests/Fixtures/composer/doc/faqs/why-cant-composer-load-repositories-recursively.md
+++ b/tests/Fixtures/composer/doc/faqs/why-cant-composer-load-repositories-recursively.md
@@ -1,3 +1,3 @@
# Why can't Composer load repositories recursively?
-Text.
+Text, see the [schema](../04-schema.md#repositories).