Merge pull request #2158 from vyos/claude/html-handling-parity

workers: html_handling none + worker index-mapping — RTD .html URL parity

6 files changed, 240 insertions, 5 deletions

diff --git a/workers/branch/src/index.ts b/workers/branch/src/index.ts
index d955d166..7cdaaf14 100644
--- a/workers/branch/src/index.ts
+++ b/workers/branch/src/index.ts
@@ -6,8 +6,18 @@ export interface Env {
 
 export type CacheClass = "page" | "asset";
 
+// Binary/media assets get the longer asset cache class, alongside .pdf and the Sphinx
+// /_static/ (theme) + /_images/ (figure) trees.
+const ASSET_EXT_RE = /\.(png|jpe?g|svg|gif|ico|woff2?|ttf|eot)$/i;
+
 export function classifyPath(path: string): CacheClass {
-  if (path.endsWith(".pdf") || path.includes("/_static/")) return "asset";
+  if (
+    path.endsWith(".pdf") ||
+    path.includes("/_static/") ||
+    path.includes("/_images/") ||
+    ASSET_EXT_RE.test(path)
+  )
+    return "asset";
   return "page"; // HTML, versions.json, sitemaps, robots/llms, pagefind index
 }
 
@@ -38,6 +48,36 @@ export function withDocsHeaders(
 export default {
   async fetch(request: Request, env: Env): Promise<Response> {
     const url = new URL(request.url);
+    // With assets html_handling "none", the runtime serves explicit .html URLs directly
+    // but does NOT map a directory URL ("/foo/") to its index.html — the worker must map
+    // trailing-slash URLs to index.html itself to preserve ReadTheDocs URL parity.
+    if (url.pathname.endsWith("/")) {
+      const mapped = new URL(url);
+      mapped.pathname = url.pathname + "index.html";
+      const resp = await env.ASSETS.fetch(new Request(mapped, request));
+      return withDocsHeaders(resp, url.pathname, env);
+    }
+    // Bare extensionless path (no trailing slash, no "." in the last segment): it may be a
+    // real directory whose slashed form RTD 301-redirects to ("/foo" → "/foo/"), or a
+    // file-like path with no matching asset (e.g. "/cli", whose real asset is "cli.html")
+    // that RTD 404s. A dot heuristic can't separate them, so probe the assets binding for
+    // "<path>/index.html": 200 → 301 to the slashed form; anything else → fall through to
+    // the exact-path fetch (404 for "/cli", matching live RTD).
+    const lastSegment = url.pathname.slice(url.pathname.lastIndexOf("/") + 1);
+    if (lastSegment !== "" && !lastSegment.includes(".")) {
+      const probe = new URL(url);
+      probe.pathname = url.pathname + "/index.html";
+      const probeResp = await env.ASSETS.fetch(new Request(probe, { method: "GET" }));
+      probeResp.body?.cancel(); // existence check only — release the probe body stream
+      if (probeResp.status === 200) {
+        const location = url.pathname + "/" + url.search; // preserve query; no fragment
+        return withDocsHeaders(
+          new Response(null, { status: 301, headers: { Location: location } }),
+          url.pathname,
+          env,
+        );
+      }
+    }
     const resp = await env.ASSETS.fetch(request);
     return withDocsHeaders(resp, url.pathname, env);
   },
diff --git a/workers/branch/test/content.test.ts b/workers/branch/test/content.test.ts
index fcf068da..94a0d5a6 100644
--- a/workers/branch/test/content.test.ts
+++ b/workers/branch/test/content.test.ts
@@ -1,5 +1,16 @@
 import { describe, it, expect } from "vitest";
 import worker, { classifyPath, cacheHeaderFor, withDocsHeaders, type Env } from "../src/index";
+// The workers pool has no real filesystem; import the branch wrangler configs as Vite `?raw`
+// assets (pattern from apex/test/manifest.test.ts) so their content is inlined at bundle time
+// for the html_handling congruence pin at the bottom of this file.
+// eslint-disable-next-line import/no-unresolved
+import wranglerRolling from "../wrangler.rolling.jsonc?raw";
+// eslint-disable-next-line import/no-unresolved
+import wranglerV15 from "../wrangler.v15.jsonc?raw";
+// eslint-disable-next-line import/no-unresolved
+import wranglerV14 from "../wrangler.v14.jsonc?raw";
+// eslint-disable-next-line import/no-unresolved
+import wranglerLegacy from "../wrangler.legacy.jsonc?raw";
 
 describe("cache classes (§3.3)", () => {
   it("HTML + config class → max-age=0, s-maxage=300", () => {
@@ -78,3 +89,187 @@ describe("default fetch entrypoint", () => {
     expect(resp.headers.get("Cache-Control")).toBe("no-store");
   });
 });
+
+describe("directory-index mapping (html_handling \"none\", §3.2.3 amended 2026-07-22)", () => {
+  // NOTE: these tests mock the ASSETS binding — they exercise the WORKER's directory
+  // mapping + bare-directory probe logic, NOT the wrangler `html_handling` config (which
+  // only exists at deploy time). The config-level backstop is the smoke gate,
+  // scripts/docs_gates/smoke.py; the raw-string congruence test at the bottom of this file
+  // is the unit-level pin against a silent revert.
+  //
+  // Assets binding stub emulating html_handling:"none": exact-path lookups only — no
+  // extension inference and no directory auto-index. A bare "/foo/" resolves only because
+  // the worker rewrites it to "/foo/index.html" first; a bare "/foo" 301s only because the
+  // worker probes "/foo/index.html" and finds it — exactly the behavior under test.
+  const ASSET_MAP: Record<string, string> = {
+    "/en/rolling/index.html": "<html>root index</html>",
+    "/en/rolling/cli.html": "<html>cli page</html>",
+    "/en/rolling/guide/index.html": "<html>guide index</html>",
+    "/en/rolling/installation/index.html": "<html>installation index</html>",
+  };
+
+  const makeAssetsEnv = (docsEnv: Env["DOCS_ENV"], seen: string[]): Env => ({
+    ASSETS: {
+      fetch: async (req: Request) => {
+        seen.push(req.url);
+        const body = ASSET_MAP[new URL(req.url).pathname];
+        return body === undefined
+          ? new Response("not found", { status: 404, headers: { "content-type": "text/html" } })
+          : new Response(body, { headers: { "content-type": "text/html" } });
+      },
+    } as unknown as Fetcher,
+    DOCS_BUILD_SHA: "testsha",
+    DOCS_ENV: docsEnv,
+  });
+
+  it("trailing-slash directory URL is mapped to index.html → 200 + index content", async () => {
+    const seen: string[] = [];
+    const resp = await worker.fetch(
+      new Request("https://docs.vyos.io/en/rolling/"),
+      makeAssetsEnv("production", seen),
+    );
+    expect(resp.status).toBe(200);
+    expect(await resp.text()).toBe("<html>root index</html>");
+    // worker rewrote "/en/rolling/" → "/en/rolling/index.html" before hitting ASSETS
+    expect(seen).toEqual(["https://docs.vyos.io/en/rolling/index.html"]);
+    // 200 still carries the build stamp + the page cache class
+    expect(resp.headers.get("X-Docs-Build")).toBe("testsha");
+    expect(resp.headers.get("Cache-Control"))
+      .toBe("public, max-age=0, s-maxage=300, must-revalidate");
+  });
+
+  it("explicit .html URL is served directly — 200, never a 3xx redirect", async () => {
+    const seen: string[] = [];
+    const resp = await worker.fetch(
+      new Request("https://docs.vyos.io/en/rolling/cli.html"),
+      makeAssetsEnv("production", seen),
+    );
+    expect(resp.status).toBe(200);
+    expect(resp.status).toBeLessThan(300); // no 307/308 for explicit .html paths
+    expect(await resp.text()).toBe("<html>cli page</html>");
+    // passed through unmodified — the worker never rewrites explicit-file paths
+    expect(seen).toEqual(["https://docs.vyos.io/en/rolling/cli.html"]);
+    expect(resp.headers.get("X-Docs-Build")).toBe("testsha");
+    expect(resp.headers.get("Cache-Control"))
+      .toBe("public, max-age=0, s-maxage=300, must-revalidate");
+  });
+
+  it("explicit nested /folder/index.html is served directly → 200", async () => {
+    const seen: string[] = [];
+    const resp = await worker.fetch(
+      new Request("https://docs.vyos.io/en/rolling/guide/index.html"),
+      makeAssetsEnv("production", seen),
+    );
+    expect(resp.status).toBe(200);
+    expect(await resp.text()).toBe("<html>guide index</html>");
+    expect(seen).toEqual(["https://docs.vyos.io/en/rolling/guide/index.html"]);
+    expect(resp.headers.get("X-Docs-Build")).toBe("testsha");
+    expect(resp.headers.get("Cache-Control"))
+      .toBe("public, max-age=0, s-maxage=300, must-revalidate");
+  });
+
+  it("extensionless file-like path (no dir behind it) probes, misses, then 404s", async () => {
+    const seen: string[] = [];
+    const resp = await worker.fetch(
+      new Request("https://docs.vyos.io/en/rolling/cli"),
+      makeAssetsEnv("production", seen),
+    );
+    expect(resp.status).toBe(404);
+    // probe "/cli/index.html" misses (no such dir) → fall through to the exact-path fetch of
+    // "/cli", which also misses (real asset is cli.html); html_handling:"none" never infers it.
+    expect(seen).toEqual([
+      "https://docs.vyos.io/en/rolling/cli/index.html",
+      "https://docs.vyos.io/en/rolling/cli",
+    ]);
+    // a 404 must never carry a cacheable page/asset class
+    expect(resp.headers.get("Cache-Control")).toBe("no-store");
+  });
+
+  it("directory mapping preserves the original query string", async () => {
+    const seen: string[] = [];
+    const resp = await worker.fetch(
+      new Request("https://docs.vyos.io/en/rolling/?q=foo"),
+      makeAssetsEnv("production", seen),
+    );
+    expect(resp.status).toBe(200);
+    expect(seen).toEqual(["https://docs.vyos.io/en/rolling/index.html?q=foo"]);
+  });
+
+  it("bare directory path (real dir behind it) → 301 to the slashed form, query preserved", async () => {
+    const seen: string[] = [];
+    const resp = await worker.fetch(
+      new Request("https://docs.vyos.io/en/rolling/installation?q=x"),
+      makeAssetsEnv("production", seen),
+    );
+    expect(resp.status).toBe(301);
+    expect(resp.headers.get("Location")).toBe("/en/rolling/installation/?q=x");
+    expect(resp.headers.get("X-Docs-Build")).toBe("testsha");
+    // only the index.html probe reached ASSETS — the redirect short-circuits the passthrough
+    expect(seen).toEqual(["https://docs.vyos.io/en/rolling/installation/index.html?q=x"]);
+  });
+
+  it("trailing-slash mapping preserves request method + conditional/range headers", async () => {
+    // Codex: pins that `new Request(mapped, request)` carries method + headers to ASSETS,
+    // so conditional-GET (304) and Range (206) still work on directory URLs.
+    let captured: Request | undefined;
+    const env: Env = {
+      ASSETS: {
+        fetch: async (req: Request) => {
+          captured = req;
+          return new Response(null, { headers: { "content-type": "text/html" } });
+        },
+      } as unknown as Fetcher,
+      DOCS_BUILD_SHA: "testsha",
+      DOCS_ENV: "production",
+    };
+    await worker.fetch(
+      new Request("https://docs.vyos.io/en/rolling/", {
+        method: "HEAD",
+        headers: { "If-None-Match": '"abc"', Range: "bytes=0-0" },
+      }),
+      env,
+    );
+    expect(captured?.url).toBe("https://docs.vyos.io/en/rolling/index.html");
+    expect(captured?.method).toBe("HEAD");
+    expect(captured?.headers.get("If-None-Match")).toBe('"abc"');
+    expect(captured?.headers.get("Range")).toBe("bytes=0-0");
+  });
+});
+
+describe("asset cache-class classification (§3.3, extended)", () => {
+  it("images, fonts, and the /_images/ + /_static/ trees get the asset class", () => {
+    for (const p of [
+      "/en/rolling/_images/diagram.png",
+      "/en/rolling/_static/fonts/roboto.woff2",
+      "/en/rolling/logo.svg",
+      "/en/rolling/photo.jpeg",
+      "/en/rolling/icon.ico",
+      "/en/rolling/vyos-documentation.pdf",
+    ]) {
+      expect(classifyPath(p)).toBe("asset");
+    }
+  });
+  it("HTML pages and data files stay the page class", () => {
+    for (const p of [
+      "/en/rolling/index.html",
+      "/en/rolling/cli.html",
+      "/en/rolling/versions.json",
+      "/en/rolling/sitemap.xml",
+    ]) {
+      expect(classifyPath(p)).toBe("page");
+    }
+  });
+});
+
+describe("wrangler config congruence — html_handling pinned to \"none\"", () => {
+  // Unit tests mock ASSETS and never exercise the real wrangler html_handling config; this
+  // raw-string assertion is the unit-level pin against a silent revert to
+  // "auto-trailing-slash" (which reintroduces the 307-on-explicit-.html smoke failure). The
+  // deploy-time smoke gate (scripts/docs_gates/smoke.py) is the runtime-level backstop.
+  it("all four branch wrangler envs set html_handling \"none\"", () => {
+    for (const raw of [wranglerRolling, wranglerV15, wranglerV14, wranglerLegacy]) {
+      expect(raw).toContain('"html_handling": "none"');
+      expect(raw).not.toContain("auto-trailing-slash");
+    }
+  });
+});
diff --git a/workers/branch/wrangler.legacy.jsonc b/workers/branch/wrangler.legacy.jsonc
index fd036ec5..aef5144e 100644
--- a/workers/branch/wrangler.legacy.jsonc
+++ b/workers/branch/wrangler.legacy.jsonc
@@ -8,7 +8,7 @@
   "assets": {
     "directory": "../../dist/assets",       // populated by CI (Task 3.2); html_handling per §3.2.3
     "binding": "ASSETS",
-    "html_handling": "auto-trailing-slash",
+    "html_handling": "none",                // "none" + worker index-mapping: RTD parity — explicit .html URLs must serve 200, never 307 (§3.2.3 amended 2026-07-22)
     "not_found_handling": "404-page",
     "run_worker_first": true
   },
diff --git a/workers/branch/wrangler.rolling.jsonc b/workers/branch/wrangler.rolling.jsonc
index 4595a06a..7a6f646c 100644
--- a/workers/branch/wrangler.rolling.jsonc
+++ b/workers/branch/wrangler.rolling.jsonc
@@ -8,7 +8,7 @@
   "assets": {
     "directory": "../../dist/assets",       // populated by CI (Task 3.2); html_handling per §3.2.3
     "binding": "ASSETS",
-    "html_handling": "auto-trailing-slash",
+    "html_handling": "none",                // "none" + worker index-mapping: RTD parity — explicit .html URLs must serve 200, never 307 (§3.2.3 amended 2026-07-22)
     "not_found_handling": "404-page",
     "run_worker_first": true
   },
diff --git a/workers/branch/wrangler.v14.jsonc b/workers/branch/wrangler.v14.jsonc
index 2c7310aa..fe758dd9 100644
--- a/workers/branch/wrangler.v14.jsonc
+++ b/workers/branch/wrangler.v14.jsonc
@@ -8,7 +8,7 @@
   "assets": {
     "directory": "../../dist/assets",       // populated by CI (Task 3.2); html_handling per §3.2.3
     "binding": "ASSETS",
-    "html_handling": "auto-trailing-slash",
+    "html_handling": "none",                // "none" + worker index-mapping: RTD parity — explicit .html URLs must serve 200, never 307 (§3.2.3 amended 2026-07-22)
     "not_found_handling": "404-page",
     "run_worker_first": true
   },
diff --git a/workers/branch/wrangler.v15.jsonc b/workers/branch/wrangler.v15.jsonc
index 45098aee..f2a0d7a4 100644
--- a/workers/branch/wrangler.v15.jsonc
+++ b/workers/branch/wrangler.v15.jsonc
@@ -8,7 +8,7 @@
   "assets": {
     "directory": "../../dist/assets",       // populated by CI (Task 3.2); html_handling per §3.2.3
     "binding": "ASSETS",
-    "html_handling": "auto-trailing-slash",
+    "html_handling": "none",                // "none" + worker index-mapping: RTD parity — explicit .html URLs must serve 200, never 307 (§3.2.3 amended 2026-07-22)
     "not_found_handling": "404-page",
     "run_worker_first": true
   },