pinned response
" + })); + } catch (error) { + request.emit("error", error); + } + }); + return request; + }; + + const result = await fetchPage("http://rebind.test/guide", { + resolver, + requesters: { http: requester } + }); + + assert.deepEqual(connectionAddresses, [{ address: "93.184.216.34", family: 4 }]); + assert.match(result.html, /pinned response/); + assert.equal(result.finalUrl, "http://rebind.test/guide"); + assert.equal(resolutions, 2); +}); + +test("resolves and pins each redirect destination independently", async () => { + const resolutions = []; + const connections = []; + const resolver = async (hostname) => { + resolutions.push(hostname); + return [{ address: hostname === "first.test" ? "93.184.216.34" : "1.1.1.1", family: 4 }]; + }; + const requester = (url, options, callback) => { + const request = new EventEmitter(); + request.end = () => queueMicrotask(async () => { + try { + const approved = await pinnedResult(options.lookup, { all: true }); + connections.push({ hostname: url.hostname, address: approved[0].address }); + if (url.hostname === "first.test") { + callback(mockResponse({ + statusCode: 302, + headers: { location: "http://second.test/final" }, + remoteAddress: approved[0].address + })); + return; + } + callback(mockResponse({ remoteAddress: approved[0].address, body: "redirected page" })); + } catch (error) { + request.emit("error", error); + } + }); + return request; + }; + + const result = await fetchPage("http://first.test/start", { + resolver, + requesters: { http: requester } + }); + + assert.deepEqual(resolutions, ["first.test", "second.test"]); + assert.deepEqual(connections, [ + { hostname: "first.test", address: "93.184.216.34" }, + { hostname: "second.test", address: "1.1.1.1" } + ]); + assert.equal(result.finalUrl, "http://second.test/final"); +}); + +test("rejects a connection whose actual socket address is non-public", async () => { + const resolver = async () => [{ address: "93.184.216.34", family: 4 }]; + const requester = (_url, _options, callback) => { + const request = new EventEmitter(); + request.end = () => queueMicrotask(() => callback(mockResponse({ remoteAddress: "127.0.0.1" }))); + return request; + }; + + await assert.rejects( + fetchPage("http://socket-check.test", { resolver, requesters: { http: requester } }), + /连接到的地址不是公开地址/ + ); +}); + +test("rejects a hostname when any DNS answer is non-public", async () => { + const resolver = async () => [ + { address: "93.184.216.34", family: 4 }, + { address: "127.0.0.1", family: 4 } + ]; + + await assert.rejects( + resolvePublicUrl("https://mixed.test", resolver), + /不能读取本机、内网或特殊用途地址/ + ); +}); diff --git a/url-fetch.mjs b/url-fetch.mjs new file mode 100644 index 0000000..1b16146 --- /dev/null +++ b/url-fetch.mjs @@ -0,0 +1,207 @@ +import http from "node:http"; +import https from "node:https"; +import { lookup } from "node:dns/promises"; +import { isIP } from "node:net"; + +const maxPageBytes = 1_500_000; +const redirectStatuses = new Set([301, 302, 303, 307, 308]); + +const blockedIpv4Ranges = [ + ["0.0.0.0", 8], + ["10.0.0.0", 8], + ["100.64.0.0", 10], + ["127.0.0.0", 8], + ["169.254.0.0", 16], + ["172.16.0.0", 12], + ["192.0.0.0", 24], + ["192.0.2.0", 24], + ["192.88.99.0", 24], + ["192.168.0.0", 16], + ["198.18.0.0", 15], + ["198.51.100.0", 24], + ["203.0.113.0", 24], + ["224.0.0.0", 4], + ["240.0.0.0", 4] +]; + +const blockedIpv6Ranges = [ + ["64:ff9b:1::", 48], + ["100::", 64], + ["2001:2::", 48], + ["2001:10::", 28], + ["2001:db8::", 32], + ["fc00::", 7], + ["fe80::", 10], + ["fec0::", 10], + ["ff00::", 8] +]; + +function ipv4ToBigInt(address) { + const parts = address.split(".").map(Number); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return null; + return parts.reduce((value, part) => (value << 8n) + BigInt(part), 0n); +} + +function ipv6ToBigInt(address) { + let normalized = address.toLowerCase().replace(/^\[|\]$/g, "").split("%")[0]; + if (normalized.includes(".")) { + const separator = normalized.lastIndexOf(":"); + const ipv4 = ipv4ToBigInt(normalized.slice(separator + 1)); + if (separator < 0 || ipv4 === null) return null; + normalized = `${normalized.slice(0, separator)}:${(ipv4 >> 16n).toString(16)}:${(ipv4 & 0xffffn).toString(16)}`; + } + + const halves = normalized.split("::"); + if (halves.length > 2) return null; + const left = halves[0] ? halves[0].split(":") : []; + const right = halves[1] ? halves[1].split(":") : []; + const missing = 8 - left.length - right.length; + if ((halves.length === 1 && missing !== 0) || missing < 0) return null; + const parts = [...left, ...Array(missing).fill("0"), ...right]; + if (parts.length !== 8 || parts.some((part) => !/^[0-9a-f]{1,4}$/.test(part))) return null; + return parts.reduce((value, part) => (value << 16n) + BigInt(`0x${part}`), 0n); +} + +function isInRange(value, network, prefixLength, bitLength) { + const shift = BigInt(bitLength - prefixLength); + return value >> shift === network >> shift; +} + +function isBlockedIpv4(address) { + const value = ipv4ToBigInt(address); + return value !== null && blockedIpv4Ranges.some(([network, prefix]) => ( + isInRange(value, ipv4ToBigInt(network), prefix, 32) + )); +} + +function embeddedIpv4(value, prefix, prefixLength) { + const network = ipv6ToBigInt(prefix); + if (!isInRange(value, network, prefixLength, 128)) return false; + const ipv4 = Number(value & 0xffffffffn); + return isBlockedIpv4([24, 16, 8, 0].map((shift) => (ipv4 >>> shift) & 255).join(".")); +} + +export function isBlockedAddress(address) { + const normalized = address.toLowerCase().replace(/^\[|\]$/g, "").split("%")[0]; + const version = isIP(normalized); + if (version === 4) return isBlockedIpv4(normalized); + if (version !== 6) return true; + + const value = ipv6ToBigInt(normalized); + if (value === null) return true; + if (embeddedIpv4(value, "::", 96) || embeddedIpv4(value, "::ffff:0:0", 96) || embeddedIpv4(value, "64:ff9b::", 96)) return true; + return blockedIpv6Ranges.some(([network, prefix]) => isInRange(value, ipv6ToBigInt(network), prefix, 128)); +} + +function isBlockedHost(hostname) { + const host = hostname.toLowerCase().replace(/^\[|\]$/g, ""); + if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) return true; + return isIP(host) > 0 && isBlockedAddress(host); +} + +export async function resolvePublicUrl(target, resolver = lookup) { + const url = new URL(target); + if (!["http:", "https:"].includes(url.protocol) || isBlockedHost(url.hostname)) { + throw new Error("只支持公开的 HTTP 或 HTTPS 网页"); + } + + const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, ""); + const addresses = await resolver(hostname, { all: true, verbatim: true }); + if (!addresses.length || addresses.some(({ address }) => isBlockedAddress(address))) { + throw new Error("不能读取本机、内网或特殊用途地址"); + } + return { url, addresses }; +} + +export function createPinnedLookup(addresses) { + const approved = addresses.map(({ address, family }) => ({ address, family: Number(family) || isIP(address) })); + if (!approved.length || approved.some(({ address, family }) => !family || isBlockedAddress(address))) { + throw new Error("没有可用的公开地址"); + } + + return (_hostname, options, callback) => { + const settings = typeof options === "number" ? { family: options } : (options || {}); + const requestedFamily = Number(settings.family || 0); + const candidates = requestedFamily ? approved.filter(({ family }) => family === requestedFamily) : approved; + if (!candidates.length) { + const error = new Error("没有符合协议族的公开地址"); + error.code = "ENOTFOUND"; + queueMicrotask(() => callback(error)); + return; + } + if (settings.all) { + queueMicrotask(() => callback(null, candidates)); + return; + } + queueMicrotask(() => callback(null, candidates[0].address, candidates[0].family)); + }; +} + +async function requestPage(url, addresses, signal, requesters = {}) { + const requester = url.protocol === "https:" ? (requesters.https || https.request) : (requesters.http || http.request); + return new Promise((resolve, reject) => { + const request = requester(url, { + method: "GET", + signal, + lookup: createPinnedLookup(addresses), + headers: { "User-Agent": "QuickLearn/1.0 (local learning assistant)", Accept: "text/html, text/plain" } + }, (response) => { + const remoteAddress = response.socket?.remoteAddress; + if (!remoteAddress || isBlockedAddress(remoteAddress)) { + response.destroy(); + reject(new Error("连接到的地址不是公开地址")); + return; + } + resolve(response); + }); + request.on("error", reject); + request.end(); + }); +} + +async function readPageBody(response) { + const chunks = []; + let size = 0; + for await (const chunk of response) { + size += chunk.byteLength; + if (size > maxPageBytes) { + response.destroy(); + throw new Error("网页内容过大"); + } + chunks.push(chunk); + } + return new TextDecoder().decode(Buffer.concat(chunks)); +} + +export async function fetchPage(target, options = {}) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 9000); + const resolver = options.resolver || lookup; + try { + let resolved = await resolvePublicUrl(target, resolver); + for (let redirects = 0; redirects <= 4; redirects += 1) { + const response = await requestPage(resolved.url, resolved.addresses, controller.signal, options.requesters); + const status = response.statusCode || 0; + if (redirectStatuses.has(status)) { + const location = response.headers.location; + response.resume(); + if (!location || redirects === 4) throw new Error("网页跳转次数过多"); + resolved = await resolvePublicUrl(new URL(location, resolved.url).href, resolver); + continue; + } + if (status < 200 || status >= 300) { + response.resume(); + throw new Error(`网页返回 ${status}`); + } + const contentType = String(response.headers["content-type"] || ""); + if (!contentType.includes("text/html") && !contentType.includes("text/plain")) { + response.resume(); + throw new Error("暂不支持这种页面格式"); + } + return { html: await readPageBody(response), finalUrl: resolved.url.href }; + } + throw new Error("网页跳转次数过多"); + } finally { + clearTimeout(timeout); + } +}