feat: obey robots.txt by default

README.md

@@ -102,17 +102,20 @@ docker run -d --name lightpanda -p 127.0.0.1:9222:9222 lightpanda/browser:nightl
 ### Dump a URL
 
 ```console
-./lightpanda fetch --obey-robots --dump html --log-format pretty  --log-level info https://demo-browser.lightpanda.io/campfire-commerce/
+./lightpanda fetch --dump html --log-format pretty  --log-level info https://demo-browser.lightpanda.io/campfire-commerce/
 ```
 
+Lightpanda fetches and obeys each site's `robots.txt` by default. Pass
+`--no-obey-robots` to opt out.
+
 You can use `--dump markdown` to convert directly into markdown.
 `--wait-until`, `--wait-ms`, `--wait-selector` and `--wait-script` are
 available to adjust waiting time before dump.
 
 ### Start a CDP server
 
 ```console
-./lightpanda serve --obey-robots --log-format pretty  --log-level info --host 127.0.0.1 --port 9222
+./lightpanda serve --log-format pretty  --log-level info --host 127.0.0.1 --port 9222
 ```
 Once the CDP server started, you can run a Puppeteer script by configuring the
 `browserWSEndpoint`.
@@ -258,7 +261,7 @@ Here are the key features we have implemented:
 - [x] Custom HTTP headers
 - [x] Proxy support
 - [x] Network interception
-- [x] Respect `robots.txt` with option `--obey-robots`
+- [x] Respect `robots.txt` by default (opt out with `--no-obey-robots`)
 
 NOTE: There are hundreds of Web APIs. Developing a browser (even just for headless mode) is a huge task. Coverage will increase over time.
 

src/Config.zig

@@ -176,7 +176,13 @@ fn caPathValidator(
 
 /// Common CLI args.
 const CommonOptions = .{
-    .{ .name = "obey_robots", .type = bool },
+    .{
+        .name = "no_obey_robots",
+        .field_name = "obey_robots",
+        .type = bool,
+        .default = true,
+        .variants = .{.{ .name = "obey_robots" }},
+    },
     .{ .name = "proxy_bearer_token", .type = ?[:0]const u8 },
     .{ .name = "http_proxy", .type = ?[:0]const u8 },
     .{ .name = "http_max_concurrent", .type = ?u8 },

src/cdp/domains/lp.zig

@@ -227,8 +227,8 @@ fn getStructuredData(cmd: anytype) !void {
 
 // Advisory `Content-Signal` robots.txt preferences for the current document's
 // host (https://contentsignals.org). `available` is false when no robots.txt
-// has been fetched for the host — note this is the case unless `obey_robots`
-// is enabled, since the robots layer is what populates the store.
+// has been fetched for the host — note this is the case when the robots layer
+// is disabled with `--no-obey-robots`, since that's what populates the store.
 fn getContentSignal(cmd: anytype) !void {
     const bc = cmd.browser_context orelse return error.NoBrowserContext;
     const frame = bc.mainFrame() orelse return error.FrameNotLoaded;

src/help.zon

@@ -387,9 +387,10 @@
     \\          The log level.
     \\          Defaults to {1s}.
     \\          Allowed values: "debug", "info", "warn", "error", "fatal".
-    \\  --obey-robots
-    \\          Fetches and obeys robots.txt of the target page.
-    \\          Defaults to false.
+    \\  --no-obey-robots
+    \\          Disables fetching and obeying robots.txt of the target page.
+    \\          Obeys robots.txt by default; --obey-robots is still accepted
+    \\          as an explicit opt-in.
     \\  --proxy-bearer-token <TOKEN>
     \\          Token sent for bearer authentication with the proxy:
     \\          Proxy-Authorization: Bearer <token>.