Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"erusev/parsedown": "^1.0",
"symfony/console": "^8",
"symfony/dotenv": "^8",
"symfony/filesystem": "^8",
"symfony/flex": "^2",
"symfony/framework-bundle": "^8",
"symfony/http-client": "^8",
Expand Down
2 changes: 1 addition & 1 deletion composer.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

229 changes: 173 additions & 56 deletions src/Controller/DocsController.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,87 +3,88 @@
namespace App\Controller;

use Composer\Pcre\Preg;
use Symfony\Component\Filesystem\Path;
use Symfony\Component\Finder\Finder;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\Routing\Attribute\Route;
use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
use Symfony\Component\HttpFoundation\RedirectResponse;

/**
* @phpstan-type Page array{page: string, metadata: array<string, string>|null, contents: string}
*/
class DocsController extends AbstractController
{
/** old page => [new page, fragment] */
private const array MOVED_PAGES = [
'articles/handling-private-packages-with-satis.md' => ['articles/handling-private-packages.md', null],
'articles/http-basic-authentication.md' => ['articles/authentication-for-private-packages.md', 'http-basic'],
'faqs/why-can\'t-composer-load-repositories-recursively.md' => ['faqs/why-cant-composer-load-repositories-recursively.md', null],
];

#[Route("/doc/", name: "docs")]
public function index(string $docDir): Response
{
$scan = function ($dir, $prefix = '') {
$finder = new Finder();
$finder->files()
->in($dir)
->sortByName()
->depth(0);

$filenames = [];

foreach ($finder as $file) {
$filename = basename($file->getPathname(), '.md');
$url = $this->generateUrl(
'docs.view',
['page' => $prefix.$filename.'.md']
);

$metadata = null;
$contents = file_get_contents($file->getPathname());
assert(is_string($contents));
if (Preg::isMatchStrictGroups('{^<!--(.*)-->}s', $contents, $match)) {
Preg::matchAllStrictGroups('{^ *(?P<keyword>\w+): *(?P<value>.*)}m', $match[1], $matches);
$metadata = [];
foreach ($matches['keyword'] as $key => $keyword) {
$metadata[$keyword] = $matches['value'][$key];
}
}

if (Preg::isMatch('{^(<!--.+?-->\n+)?# (.+?)\n}s', $contents, $match)) {
$displayName = $match[2];
} else {
$displayName = Preg::replace('{^\d{2} }', '', ucwords(str_replace('-', ' ', $filename)));
}
$filenames[$displayName] = ['link' => $url, 'metadata' => $metadata];
}

return $filenames;
};

$book = $scan($docDir);
$articles = $scan($docDir.'/articles', 'articles/');
$faqs = $scan($docDir.'/faqs', 'faqs/');
$sections = $this->listSections($docDir);

return $this->render('doc.list.html.twig', [
'book' => $book,
'articles' => $articles,
'faqs' => $faqs,
'book' => $sections['Book'],
'articles' => $sections['Articles'],
'faqs' => $sections['FAQs'],
'page' => 'docs'
]);
}

#[Route("/doc/{page}", name: "docs.view", requirements: ["page" => "[a-z0-9/\'-]+\.md"])]
public function viewPage(string $page, string $docDir, \Parsedown $parsedown): Response
#[Route("/llms.txt", name: "llms_txt")]
public function llmsTxt(string $docDir): Response
{
return $this->render('llms.txt.twig', [
'sections' => $this->listSections($docDir),
], $this->createTextResponse('text/plain'));
}

#[Route("/llms-full.txt", name: "llms_full_txt")]
public function llmsFullTxt(string $docDir): Response
{
$pages = [];
foreach ($this->listSections($docDir) as $section) {
foreach ($section as $data) {
$url = $this->generateUrl('docs.view', ['page' => $data['page']], UrlGeneratorInterface::ABSOLUTE_URL);
$pages[] = 'Source: '.$url."\n\n".$this->toMarkdown($data['page'], $data['contents']);
}
}

return $this->createTextResponse('text/plain')->setContent(implode("\n\n---\n\n", $pages));
}

#[Route("/doc/{page}.md", name: "docs.markdown", requirements: ["page" => "[a-z0-9/\'-]+\.md"])]
public function viewMarkdown(string $page, string $docDir): Response
{
$filename = $docDir.'/'.$page;

if (!file_exists($filename)) {
if ($page === 'articles/handling-private-packages-with-satis.md') {
return new RedirectResponse('/doc/articles/handling-private-packages.md', 301);
}
return $this->redirectMovedPage($page, 'docs.markdown');
}

if ($page === 'articles/http-basic-authentication.md') {
return new RedirectResponse('/doc/articles/authentication-for-private-packages.md#http-basic', 301);
}
$contents = file_get_contents($filename);
assert(is_string($contents));
$canonical = $this->generateUrl('docs.view', ['page' => $page], UrlGeneratorInterface::ABSOLUTE_URL);

if ($page === 'faqs/why-can\'t-composer-load-repositories-recursively.md') {
return new RedirectResponse('/doc/faqs/why-cant-composer-load-repositories-recursively.md', 301);
}
$response = $this->createTextResponse('text/markdown')->setContent($this->toMarkdown($page, $contents));
$response->headers->set('Link', '<'.$canonical.'>; rel="canonical"');

throw $this->createNotFoundException('Requested page was not found.');
return $response;
}

#[Route("/doc/{page}", name: "docs.view", requirements: ["page" => "[a-z0-9/\'-]+\.md"])]
public function viewPage(string $page, string $docDir, \Parsedown $parsedown): Response
{
$filename = $docDir.'/'.$page;

if (!file_exists($filename)) {
return $this->redirectMovedPage($page, 'docs.view');
}

$contents = file_get_contents($filename);
Expand Down Expand Up @@ -237,4 +238,120 @@ public function viewPage(string $page, string $docDir, \Parsedown $parsedown): R
'title' => $firstTitle
]);
}

private function redirectMovedPage(string $page, string $route): RedirectResponse
{
if (!isset(self::MOVED_PAGES[$page])) {
throw $this->createNotFoundException('Requested page was not found.');
}

[$target, $fragment] = self::MOVED_PAGES[$page];
$params = ['page' => $target];
if ($fragment !== null) {
$params['_fragment'] = $fragment;
}

return $this->redirectToRoute($route, $params, 301);
}

private function createTextResponse(string $type): Response
{
$response = new Response('', 200, ['Content-Type' => $type.'; charset=UTF-8']);
$response->setPublic();
$response->setMaxAge(3600);

return $response;
}

/**
* @return array{Book: array<string, Page>, Articles: array<string, Page>, FAQs: array<string, Page>}
*/
private function listSections(string $docDir): array
{
return [
'Book' => $this->listPages($docDir),
'Articles' => $this->listPages($docDir.'/articles', 'articles/'),
'FAQs' => $this->listPages($docDir.'/faqs', 'faqs/'),
];
}

/**
* @return array<string, Page> keyed by display name
*/
private function listPages(string $dir, string $prefix = ''): array
{
$finder = new Finder();
$finder->files()
->in($dir)
->sortByName()
->depth(0);

$pages = [];

foreach ($finder as $file) {
$filename = basename($file->getPathname(), '.md');
$page = $prefix.$filename.'.md';

$metadata = null;
$contents = file_get_contents($file->getPathname());
assert(is_string($contents));
if (Preg::isMatchStrictGroups('{^<!--(.*)-->}s', $contents, $match)) {
Preg::matchAllStrictGroups('{^ *(?P<keyword>\w+): *(?P<value>.*)}m', $match[1], $matches);
$metadata = [];
foreach ($matches['keyword'] as $key => $keyword) {
$metadata[$keyword] = $matches['value'][$key];
}
}

if (Preg::isMatch('{^(<!--.+?-->\n+)?# (.+?)\n}s', $contents, $match)) {
$displayName = $match[2];
} else {
$displayName = Preg::replace('{^\d{2} }', '', ucwords(str_replace('-', ' ', $filename)));
}
$pages[$displayName] = [
'page' => $page,
'metadata' => $metadata,
'contents' => $contents,
];
}

return $pages;
}

/**
* Returns the page source without its metadata comment, with doc links pointing to the absolute
* markdown URLs so they still resolve when served outside /doc/ or concatenated in llms-full.txt
*/
private function toMarkdown(string $page, string $contents): string
{
$contents = Preg::replace('{^<!--.*?-->\n+}s', '', $contents);

// code fences and spans are skipped, then inline [text](target) and reference [n]: target
// links are rewritten, except external, absolute and non-.md ones
return Preg::replaceCallback(
'{(?:^ *(`{3,}|~{3,})[\s\S]*?(?:^ *\1[`~]* *$|\z)|(`+)[\s\S]*?\2)(*SKIP)(*FAIL)'
.'|(\]\(|^ {0,3}\[[^\]]+\]:[ \t]*)(?!\w+:|/)([a-z0-9/._\'-]+\.md(?=[)#\s]|$))?(#[^)\s]*)?}m',
function (array $match) use ($page): string {
$target = $match[4] ?? '';
$fragment = $match[5] ?? '';
if ($target === '' && $fragment === '') {
return (string) $match[0];
}

$resolved = $page;
if ($target !== '') {
$dir = dirname($page);
$resolved = Path::canonicalize(($dir === '.' ? '' : $dir.'/').$target);
}

// also rejects targets climbing out of the doc dir, which canonicalize leaves as ../
if (!Preg::isMatch('{^[a-z0-9/\'-]+\.md$}', $resolved)) {
return (string) $match[0];
}

return $match[3].$this->generateUrl('docs.markdown', ['page' => $resolved], UrlGeneratorInterface::ABSOLUTE_URL).$fragment;
},
$contents
);
}
}
10 changes: 9 additions & 1 deletion templates/doc.list.html.twig
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,14 @@
</a>
</p>

<h1>For LLMs</h1>
<p>The documentation is also available as markdown: see
<a href="{{ path('llms_txt') }}" title="Index of the documentation in markdown" aria-label="Index of the documentation in markdown">llms.txt</a>
for an index, or
<a href="{{ path('llms_full_txt') }}" title="The whole documentation as a single markdown file" aria-label="The whole documentation as a single markdown file">llms-full.txt</a>
for all of it in a single file. Every page can also be fetched as markdown by appending <code>.md</code> to its URL.
</p>

<h1>Translations</h1>
<p>The Composer documentation is also available in other languages:</p>
<ul>
Expand All @@ -49,7 +57,7 @@

{% macro renderFile(filename, data) %}
<li>
<a href="{{ data.link }}" title="Go to documentation section: {{ filename }}"
<a href="{{ path('docs.view', {page: data.page}) }}" title="Go to documentation section: {{ filename }}"
aria-label="Link to documentation section: {{ filename }}">
{{ filename }}
</a>
Expand Down
4 changes: 4 additions & 0 deletions templates/doc.show.html.twig
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@

{% block title %}{% if title is not empty %}{{ title }} - {% endif %}{{ parent() }}{% endblock %}

{% block head %}
<link rel="alternate" type="text/markdown" href="{{ path('docs.markdown', {page: file}) }}">
{% endblock %}

{% block content %}
<div id="searchbar" class="clearfix">
<div id="docsearch"></div>
Expand Down
1 change: 1 addition & 0 deletions templates/layout.html.twig
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
<meta name="viewport" content="width=device-width,initial-scale=1">

<link rel="stylesheet" href="{{ app.request.basePath }}/build/app.css?v=3">
{% block head %}{% endblock %}
</head>

<body>
Expand Down
19 changes: 19 additions & 0 deletions templates/llms.txt.twig
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Composer

> Composer is a dependency manager for PHP. You declare the libraries your project depends on in composer.json, and Composer installs and updates them for you. Packages are published on Packagist (https://packagist.org/).

The links below point to the markdown source of each documentation page. All of it is also available as a single file at {{ url('llms_full_txt') }}
{% for section, pages in sections %}

## {{ section }}

{% for title, data in pages %}
- [{{ title }}]({{ url('docs.markdown', {page: data.page}) }}){% if data.metadata.tagline is defined %}: {{ data.metadata.tagline }}{% endif %}

{% endfor %}
{% endfor %}

## Optional

- [composer.json schema]({{ url('schema') }}): JSON schema for validating composer.json files
- [Changelog](https://raw.githubusercontent.com/composer/composer/main/CHANGELOG.md): changes in each Composer release
Loading
Loading