fix: pin validated DNS addresses

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "node server.mjs",
     "dev": "node --watch server.mjs",
-    "lint": "node --check server.mjs && node --check app.js",
+    "lint": "node --check server.mjs && node --check url-fetch.mjs && node --check app.js",
     "test": "node --test test/*.test.mjs",
     "smoke": "npm test",
     "ci": "npm run lint && npm run smoke"

server.mjs

@@ -1,13 +1,11 @@
 import http from "node:http";
 import { readFile } from "node:fs/promises";
-import { lookup } from "node:dns/promises";
-import { isIP } from "node:net";
 import { extname, join, normalize } from "node:path";
 import { fileURLToPath } from "node:url";
+import { fetchPage } from "./url-fetch.mjs";
 
 const root = fileURLToPath(new URL(".", import.meta.url));
 const port = Number(process.env.PORT || 4173);
-const maxPageBytes = 1_500_000;
 
 const mimeTypes = {
   ".html": "text/html; charset=utf-8",
@@ -216,149 +214,6 @@ function termResult(input) {
   return profile ? { ...genericResult(input), ...profile } : genericResult(input);
 }
 
-const blockedIpv4Ranges = [
-  ["0.0.0.0", 8],
-  ["10.0.0.0", 8],
-  ["100.64.0.0", 10],
-  ["127.0.0.0", 8],
-  ["169.254.0.0", 16],
-  ["172.16.0.0", 12],
-  ["192.0.0.0", 24],
-  ["192.0.2.0", 24],
-  ["192.88.99.0", 24],
-  ["192.168.0.0", 16],
-  ["198.18.0.0", 15],
-  ["198.51.100.0", 24],
-  ["203.0.113.0", 24],
-  ["224.0.0.0", 4],
-  ["240.0.0.0", 4]
-];
-
-const blockedIpv6Ranges = [
-  ["64:ff9b:1::", 48],
-  ["100::", 64],
-  ["2001:2::", 48],
-  ["2001:10::", 28],
-  ["2001:db8::", 32],
-  ["fc00::", 7],
-  ["fe80::", 10],
-  ["fec0::", 10],
-  ["ff00::", 8]
-];
-
-function ipv4ToBigInt(address) {
-  const parts = address.split(".").map(Number);
-  if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return null;
-  return parts.reduce((value, part) => (value << 8n) + BigInt(part), 0n);
-}
-
-function ipv6ToBigInt(address) {
-  let normalized = address.toLowerCase().replace(/^\[|\]$/g, "").split("%")[0];
-  if (normalized.includes(".")) {
-    const separator = normalized.lastIndexOf(":");
-    const ipv4 = ipv4ToBigInt(normalized.slice(separator + 1));
-    if (separator < 0 || ipv4 === null) return null;
-    normalized = `${normalized.slice(0, separator)}:${(ipv4 >> 16n).toString(16)}:${(ipv4 & 0xffffn).toString(16)}`;
-  }
-
-  const halves = normalized.split("::");
-  if (halves.length > 2) return null;
-  const left = halves[0] ? halves[0].split(":") : [];
-  const right = halves[1] ? halves[1].split(":") : [];
-  const missing = 8 - left.length - right.length;
-  if ((halves.length === 1 && missing !== 0) || missing < 0) return null;
-  const parts = [...left, ...Array(missing).fill("0"), ...right];
-  if (parts.length !== 8 || parts.some((part) => !/^[0-9a-f]{1,4}$/.test(part))) return null;
-  return parts.reduce((value, part) => (value << 16n) + BigInt(`0x${part}`), 0n);
-}
-
-function isInRange(value, network, prefixLength, bitLength) {
-  const shift = BigInt(bitLength - prefixLength);
-  return value >> shift === network >> shift;
-}
-
-function isBlockedIpv4(address) {
-  const value = ipv4ToBigInt(address);
-  return value !== null && blockedIpv4Ranges.some(([network, prefix]) => {
-    const networkValue = ipv4ToBigInt(network);
-    return isInRange(value, networkValue, prefix, 32);
-  });
-}
-
-function embeddedIpv4(value, prefix, prefixLength) {
-  const network = ipv6ToBigInt(prefix);
-  if (!isInRange(value, network, prefixLength, 128)) return false;
-  const ipv4 = Number(value & 0xffffffffn);
-  return isBlockedIpv4([24, 16, 8, 0].map((shift) => (ipv4 >>> shift) & 255).join("."));
-}
-
-function isBlockedAddress(address) {
-  const normalized = address.toLowerCase().replace(/^\[|\]$/g, "").split("%")[0];
-  const version = isIP(normalized);
-  if (version === 4) return isBlockedIpv4(normalized);
-  if (version !== 6) return false;
-
-  const value = ipv6ToBigInt(normalized);
-  if (value === null) return true;
-  if (embeddedIpv4(value, "::", 96) || embeddedIpv4(value, "::ffff:0:0", 96) || embeddedIpv4(value, "64:ff9b::", 96)) return true;
-  return blockedIpv6Ranges.some(([network, prefix]) => isInRange(value, ipv6ToBigInt(network), prefix, 128));
-}
-
-function isBlockedHost(hostname) {
-  const host = hostname.toLowerCase().replace(/^\[|\]$/g, "");
-  if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) return true;
-  return isIP(host) > 0 && isBlockedAddress(host);
-}
-
-async function assertPublicUrl(target) {
-  const url = new URL(target);
-  if (!["http:", "https:"].includes(url.protocol) || isBlockedHost(url.hostname)) {
-    throw new Error("只支持公开的 HTTP 或 HTTPS 网页");
-  }
-  const addresses = await lookup(url.hostname, { all: true, verbatim: true });
-  if (!addresses.length || addresses.some(({ address }) => isBlockedAddress(address))) {
-    throw new Error("不能读取本机、内网或特殊用途地址");
-  }
-  return url;
-}
-
-async function fetchPage(target) {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), 9000);
-  try {
-    let url = await assertPublicUrl(target);
-    let response;
-    for (let redirects = 0; redirects <= 4; redirects += 1) {
-      response = await fetch(url, {
-        signal: controller.signal,
-        redirect: "manual",
-        headers: { "User-Agent": "QuickLearn/1.0 (local learning assistant)", Accept: "text/html, text/plain" }
-      });
-      if (![301, 302, 303, 307, 308].includes(response.status)) break;
-      const location = response.headers.get("location");
-      if (!location || redirects === 4) throw new Error("网页跳转次数过多");
-      url = await assertPublicUrl(new URL(location, url).href);
-    }
-    if (!response.ok) throw new Error(`网页返回 ${response.status}`);
-    const type = response.headers.get("content-type") || "";
-    if (!type.includes("text/html") && !type.includes("text/plain")) throw new Error("暂不支持这种页面格式");
-    if (!response.body) throw new Error("网页没有可读取的正文");
-    const reader = response.body.getReader();
-    const chunks = [];
-    let size = 0;
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      size += value.byteLength;
-      if (size > maxPageBytes) break;
-      chunks.push(value);
-    }
-    return { html: new TextDecoder().decode(Buffer.concat(chunks)), finalUrl: response.url };
-  } finally {
-    clearTimeout(timeout);
-  }
-}
-
 async function urlResult(input) {
   const parsed = new URL(input);
   try {

test/url-fetch.test.mjs

@@ -0,0 +1,155 @@
+import assert from "node:assert/strict";
+import { EventEmitter } from "node:events";
+import { Readable } from "node:stream";
+import { test } from "node:test";
+import { fetchPage, isBlockedAddress, resolvePublicUrl } from "../url-fetch.mjs";
+
+function pinnedResult(pinnedLookup, options = {}) {
+  return new Promise((resolve, reject) => {
+    pinnedLookup("rebind.test", options, (error, address, family) => {
+      if (error) reject(error);
+      else resolve(options.all ? address : { address, family });
+    });
+  });
+}
+
+function mockResponse({ statusCode = 200, headers = { "content-type": "text/html" }, remoteAddress, body = "" }) {
+  const response = Readable.from([Buffer.from(body)]);
+  response.statusCode = statusCode;
+  response.headers = headers;
+  response.socket = { remoteAddress };
+  return response;
+}
+
+test("classifies non-public address ranges without blocking public boundaries", () => {
+  for (const address of [
+    "127.0.0.1",
+    "169.254.169.254",
+    "100.64.0.1",
+    "100.127.255.255",
+    "172.31.255.255",
+    "192.168.1.1",
+    "::1",
+    "febf::1",
+    "fc00::1",
+    "fec0::1",
+    "::ffff:169.254.169.254",
+    "64:ff9b::a9fe:a9fe"
+  ]) assert.equal(isBlockedAddress(address), true, address);
+
+  for (const address of [
+    "1.1.1.1",
+    "100.63.255.255",
+    "100.128.0.1",
+    "169.253.255.255",
+    "172.15.255.255",
+    "172.32.0.1",
+    "2001:4860:4860::8888"
+  ]) assert.equal(isBlockedAddress(address), false, address);
+});
+
+test("pins the validated answer when DNS later rebinds to localhost", async () => {
+  let resolutions = 0;
+  let connectionAddresses;
+  const resolver = async () => {
+    resolutions += 1;
+    return resolutions === 1
+      ? [{ address: "93.184.216.34", family: 4 }]
+      : [{ address: "127.0.0.1", family: 4 }];
+  };
+  const requester = (_url, options, callback) => {
+    const request = new EventEmitter();
+    request.end = () => queueMicrotask(async () => {
+      try {
+        const rebound = await resolver();
+        assert.equal(isBlockedAddress(rebound[0].address), true);
+        connectionAddresses = await pinnedResult(options.lookup, { all: true });
+        callback(mockResponse({
+          remoteAddress: connectionAddresses[0].address,
+          body: "<html><title>Safe</title><p>pinned response</p></html>"
+        }));
+      } catch (error) {
+        request.emit("error", error);
+      }
+    });
+    return request;
+  };
+
+  const result = await fetchPage("http://rebind.test/guide", {
+    resolver,
+    requesters: { http: requester }
+  });
+
+  assert.deepEqual(connectionAddresses, [{ address: "93.184.216.34", family: 4 }]);
+  assert.match(result.html, /pinned response/);
+  assert.equal(result.finalUrl, "http://rebind.test/guide");
+  assert.equal(resolutions, 2);
+});
+
+test("resolves and pins each redirect destination independently", async () => {
+  const resolutions = [];
+  const connections = [];
+  const resolver = async (hostname) => {
+    resolutions.push(hostname);
+    return [{ address: hostname === "first.test" ? "93.184.216.34" : "1.1.1.1", family: 4 }];
+  };
+  const requester = (url, options, callback) => {
+    const request = new EventEmitter();
+    request.end = () => queueMicrotask(async () => {
+      try {
+        const approved = await pinnedResult(options.lookup, { all: true });
+        connections.push({ hostname: url.hostname, address: approved[0].address });
+        if (url.hostname === "first.test") {
+          callback(mockResponse({
+            statusCode: 302,
+            headers: { location: "http://second.test/final" },
+            remoteAddress: approved[0].address
+          }));
+          return;
+        }
+        callback(mockResponse({ remoteAddress: approved[0].address, body: "redirected page" }));
+      } catch (error) {
+        request.emit("error", error);
+      }
+    });
+    return request;
+  };
+
+  const result = await fetchPage("http://first.test/start", {
+    resolver,
+    requesters: { http: requester }
+  });
+
+  assert.deepEqual(resolutions, ["first.test", "second.test"]);
+  assert.deepEqual(connections, [
+    { hostname: "first.test", address: "93.184.216.34" },
+    { hostname: "second.test", address: "1.1.1.1" }
+  ]);
+  assert.equal(result.finalUrl, "http://second.test/final");
+});
+
+test("rejects a connection whose actual socket address is non-public", async () => {
+  const resolver = async () => [{ address: "93.184.216.34", family: 4 }];
+  const requester = (_url, _options, callback) => {
+    const request = new EventEmitter();
+    request.end = () => queueMicrotask(() => callback(mockResponse({ remoteAddress: "127.0.0.1" })));
+    return request;
+  };
+
+  await assert.rejects(
+    fetchPage("http://socket-check.test", { resolver, requesters: { http: requester } }),
+    /连接到的地址不是公开地址/
+  );
+});
+
+test("rejects a hostname when any DNS answer is non-public", async () => {
+  const resolver = async () => [
+    { address: "93.184.216.34", family: 4 },
+    { address: "127.0.0.1", family: 4 }
+  ];
+
+  await assert.rejects(
+    resolvePublicUrl("https://mixed.test", resolver),
+    /不能读取本机、内网或特殊用途地址/
+  );
+});