diff --git a/composer.json b/composer.json index a989234..1bfc468 100644 --- a/composer.json +++ b/composer.json @@ -10,6 +10,7 @@ "erusev/parsedown": "^1.0", "symfony/console": "^8", "symfony/dotenv": "^8", + "symfony/filesystem": "^8", "symfony/flex": "^2", "symfony/framework-bundle": "^8", "symfony/http-client": "^8", diff --git a/composer.lock b/composer.lock index 8736bed..dd0d438 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "54d95b5f60c6c0a946e8f49a272d3e48", + "content-hash": "29afd44a707bd6883159f536422531d7", "packages": [ { "name": "composer/ca-bundle", diff --git a/src/Controller/DocsController.php b/src/Controller/DocsController.php index 9eef09a..4577755 100644 --- a/src/Controller/DocsController.php +++ b/src/Controller/DocsController.php @@ -3,87 +3,88 @@ namespace App\Controller; use Composer\Pcre\Preg; +use Symfony\Component\Filesystem\Path; use Symfony\Component\Finder\Finder; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpFoundation\Request; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; use Symfony\Component\Routing\Attribute\Route; +use Symfony\Component\Routing\Generator\UrlGeneratorInterface; use Symfony\Component\HttpFoundation\RedirectResponse; +/** + * @phpstan-type Page array{page: string, metadata: array|null, contents: string} + */ class DocsController extends AbstractController { + /** old page => [new page, fragment] */ + private const array MOVED_PAGES = [ + 'articles/handling-private-packages-with-satis.md' => ['articles/handling-private-packages.md', null], + 'articles/http-basic-authentication.md' => ['articles/authentication-for-private-packages.md', 'http-basic'], + 'faqs/why-can\'t-composer-load-repositories-recursively.md' => ['faqs/why-cant-composer-load-repositories-recursively.md', null], + ]; + #[Route("/doc/", name: "docs")] public function index(string $docDir): Response { - $scan = function ($dir, $prefix = '') { - $finder = new Finder(); - $finder->files() - ->in($dir) - ->sortByName() - ->depth(0); - - $filenames = []; - - foreach ($finder as $file) { - $filename = basename($file->getPathname(), '.md'); - $url = $this->generateUrl( - 'docs.view', - ['page' => $prefix.$filename.'.md'] - ); - - $metadata = null; - $contents = file_get_contents($file->getPathname()); - assert(is_string($contents)); - if (Preg::isMatchStrictGroups('{^}s', $contents, $match)) { - Preg::matchAllStrictGroups('{^ *(?P\w+): *(?P.*)}m', $match[1], $matches); - $metadata = []; - foreach ($matches['keyword'] as $key => $keyword) { - $metadata[$keyword] = $matches['value'][$key]; - } - } - - if (Preg::isMatch('{^(\n+)?# (.+?)\n}s', $contents, $match)) { - $displayName = $match[2]; - } else { - $displayName = Preg::replace('{^\d{2} }', '', ucwords(str_replace('-', ' ', $filename))); - } - $filenames[$displayName] = ['link' => $url, 'metadata' => $metadata]; - } - - return $filenames; - }; - - $book = $scan($docDir); - $articles = $scan($docDir.'/articles', 'articles/'); - $faqs = $scan($docDir.'/faqs', 'faqs/'); + $sections = $this->listSections($docDir); return $this->render('doc.list.html.twig', [ - 'book' => $book, - 'articles' => $articles, - 'faqs' => $faqs, + 'book' => $sections['Book'], + 'articles' => $sections['Articles'], + 'faqs' => $sections['FAQs'], 'page' => 'docs' ]); } - #[Route("/doc/{page}", name: "docs.view", requirements: ["page" => "[a-z0-9/\'-]+\.md"])] - public function viewPage(string $page, string $docDir, \Parsedown $parsedown): Response + #[Route("/llms.txt", name: "llms_txt")] + public function llmsTxt(string $docDir): Response + { + return $this->render('llms.txt.twig', [ + 'sections' => $this->listSections($docDir), + ], $this->createTextResponse('text/plain')); + } + + #[Route("/llms-full.txt", name: "llms_full_txt")] + public function llmsFullTxt(string $docDir): Response + { + $pages = []; + foreach ($this->listSections($docDir) as $section) { + foreach ($section as $data) { + $url = $this->generateUrl('docs.view', ['page' => $data['page']], UrlGeneratorInterface::ABSOLUTE_URL); + $pages[] = 'Source: '.$url."\n\n".$this->toMarkdown($data['page'], $data['contents']); + } + } + + return $this->createTextResponse('text/plain')->setContent(implode("\n\n---\n\n", $pages)); + } + + #[Route("/doc/{page}.md", name: "docs.markdown", requirements: ["page" => "[a-z0-9/\'-]+\.md"])] + public function viewMarkdown(string $page, string $docDir): Response { $filename = $docDir.'/'.$page; if (!file_exists($filename)) { - if ($page === 'articles/handling-private-packages-with-satis.md') { - return new RedirectResponse('/doc/articles/handling-private-packages.md', 301); - } + return $this->redirectMovedPage($page, 'docs.markdown'); + } - if ($page === 'articles/http-basic-authentication.md') { - return new RedirectResponse('/doc/articles/authentication-for-private-packages.md#http-basic', 301); - } + $contents = file_get_contents($filename); + assert(is_string($contents)); + $canonical = $this->generateUrl('docs.view', ['page' => $page], UrlGeneratorInterface::ABSOLUTE_URL); - if ($page === 'faqs/why-can\'t-composer-load-repositories-recursively.md') { - return new RedirectResponse('/doc/faqs/why-cant-composer-load-repositories-recursively.md', 301); - } + $response = $this->createTextResponse('text/markdown')->setContent($this->toMarkdown($page, $contents)); + $response->headers->set('Link', '<'.$canonical.'>; rel="canonical"'); - throw $this->createNotFoundException('Requested page was not found.'); + return $response; + } + + #[Route("/doc/{page}", name: "docs.view", requirements: ["page" => "[a-z0-9/\'-]+\.md"])] + public function viewPage(string $page, string $docDir, \Parsedown $parsedown): Response + { + $filename = $docDir.'/'.$page; + + if (!file_exists($filename)) { + return $this->redirectMovedPage($page, 'docs.view'); } $contents = file_get_contents($filename); @@ -237,4 +238,120 @@ public function viewPage(string $page, string $docDir, \Parsedown $parsedown): R 'title' => $firstTitle ]); } + + private function redirectMovedPage(string $page, string $route): RedirectResponse + { + if (!isset(self::MOVED_PAGES[$page])) { + throw $this->createNotFoundException('Requested page was not found.'); + } + + [$target, $fragment] = self::MOVED_PAGES[$page]; + $params = ['page' => $target]; + if ($fragment !== null) { + $params['_fragment'] = $fragment; + } + + return $this->redirectToRoute($route, $params, 301); + } + + private function createTextResponse(string $type): Response + { + $response = new Response('', 200, ['Content-Type' => $type.'; charset=UTF-8']); + $response->setPublic(); + $response->setMaxAge(3600); + + return $response; + } + + /** + * @return array{Book: array, Articles: array, FAQs: array} + */ + private function listSections(string $docDir): array + { + return [ + 'Book' => $this->listPages($docDir), + 'Articles' => $this->listPages($docDir.'/articles', 'articles/'), + 'FAQs' => $this->listPages($docDir.'/faqs', 'faqs/'), + ]; + } + + /** + * @return array keyed by display name + */ + private function listPages(string $dir, string $prefix = ''): array + { + $finder = new Finder(); + $finder->files() + ->in($dir) + ->sortByName() + ->depth(0); + + $pages = []; + + foreach ($finder as $file) { + $filename = basename($file->getPathname(), '.md'); + $page = $prefix.$filename.'.md'; + + $metadata = null; + $contents = file_get_contents($file->getPathname()); + assert(is_string($contents)); + if (Preg::isMatchStrictGroups('{^}s', $contents, $match)) { + Preg::matchAllStrictGroups('{^ *(?P\w+): *(?P.*)}m', $match[1], $matches); + $metadata = []; + foreach ($matches['keyword'] as $key => $keyword) { + $metadata[$keyword] = $matches['value'][$key]; + } + } + + if (Preg::isMatch('{^(\n+)?# (.+?)\n}s', $contents, $match)) { + $displayName = $match[2]; + } else { + $displayName = Preg::replace('{^\d{2} }', '', ucwords(str_replace('-', ' ', $filename))); + } + $pages[$displayName] = [ + 'page' => $page, + 'metadata' => $metadata, + 'contents' => $contents, + ]; + } + + return $pages; + } + + /** + * Returns the page source without its metadata comment, with doc links pointing to the absolute + * markdown URLs so they still resolve when served outside /doc/ or concatenated in llms-full.txt + */ + private function toMarkdown(string $page, string $contents): string + { + $contents = Preg::replace('{^\n+}s', '', $contents); + + // code fences and spans are skipped, then inline [text](target) and reference [n]: target + // links are rewritten, except external, absolute and non-.md ones + return Preg::replaceCallback( + '{(?:^ *(`{3,}|~{3,})[\s\S]*?(?:^ *\1[`~]* *$|\z)|(`+)[\s\S]*?\2)(*SKIP)(*FAIL)' + .'|(\]\(|^ {0,3}\[[^\]]+\]:[ \t]*)(?!\w+:|/)([a-z0-9/._\'-]+\.md(?=[)#\s]|$))?(#[^)\s]*)?}m', + function (array $match) use ($page): string { + $target = $match[4] ?? ''; + $fragment = $match[5] ?? ''; + if ($target === '' && $fragment === '') { + return (string) $match[0]; + } + + $resolved = $page; + if ($target !== '') { + $dir = dirname($page); + $resolved = Path::canonicalize(($dir === '.' ? '' : $dir.'/').$target); + } + + // also rejects targets climbing out of the doc dir, which canonicalize leaves as ../ + if (!Preg::isMatch('{^[a-z0-9/\'-]+\.md$}', $resolved)) { + return (string) $match[0]; + } + + return $match[3].$this->generateUrl('docs.markdown', ['page' => $resolved], UrlGeneratorInterface::ABSOLUTE_URL).$fragment; + }, + $contents + ); + } } diff --git a/templates/doc.list.html.twig b/templates/doc.list.html.twig index 1736424..590ca81 100644 --- a/templates/doc.list.html.twig +++ b/templates/doc.list.html.twig @@ -35,6 +35,14 @@

+

For LLMs

+

The documentation is also available as markdown: see + llms.txt + for an index, or + llms-full.txt + for all of it in a single file. Every page can also be fetched as markdown by appending .md to its URL. +

+

Translations

The Composer documentation is also available in other languages:

    @@ -49,7 +57,7 @@ {% macro renderFile(filename, data) %}
  • - {{ filename }} diff --git a/templates/doc.show.html.twig b/templates/doc.show.html.twig index ee7df81..4736d6e 100644 --- a/templates/doc.show.html.twig +++ b/templates/doc.show.html.twig @@ -4,6 +4,10 @@ {% block title %}{% if title is not empty %}{{ title }} - {% endif %}{{ parent() }}{% endblock %} +{% block head %} + +{% endblock %} + {% block content %}