Implement private IP range request host validator (#5497)

* feat: implement private IP range request host validator for server-side HTTP requests

* feat: add link-local address support

* fix: use correct config keys

* fix: if config missing use default loopback addresses

* fix: re-use resolved address for connection

* test: add unit coverage for private request filter and proxy interaction

Agent-Logs-Url: https://github.com/SillyTavern/SillyTavern/sessions/1813593e-2263-45e2-aa53-74d39515f1df

Co-authored-by: Cohee1207 <18619528+Cohee1207@users.noreply.github.com>

* test: remove request-proxy.test.js

* perf: cache resolved matches

* fix: remove unused import

* fix: use proper ipv4 loopback cidr

* fix: correct raiseError comment

* test: uses tls.connect for secure endpoints

* Implement private IP range request host validator

Agent-Logs-Url: https://github.com/SillyTavern/SillyTavern/sessions/e76ba122-136e-43ad-b4bc-ea48a01fcdda

Co-authored-by: Cohee1207 <18619528+Cohee1207@users.noreply.github.com>

* Revert "Implement private IP range request host validator"

This reverts commit 14e271470227b485b7d23caac31a237abf9f7835.

* fix: close request without sending status in CORS forwarding when headers were sent

* fix: not enabled -> disabled

* feat: add enableKeepAlive option to PrivateRequestAgent

Co-authored-by: Copilot <copilot@github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: anthropic-code-agent[bot] <242468646+Claude@users.noreply.github.com>
Co-authored-by: Copilot <copilot@github.com>

default/config.yaml

@@ -151,6 +151,26 @@ hostWhitelist:
   # - .trycloudflare.com
   hosts: []
 
+# Perform whitelist checks against server-side HTTP requests that resolve to private IP addresses.
+# This is an additional layer of security to prevent Server-Side Request Forgery (SSRF) attacks.
+# Recommended when listen mode is enabled, or if your server is accessible by untrusted users.
+privateAddressWhitelist:
+  # Enable private address whitelist to block requests to private IP ranges.
+  enabled: false
+  # If true, requests to hosts that cannot be resolved will be allowed instead of blocked.
+  allowUnresolvedHosts: false
+  # Log blocked and allowed requests to the console.
+  log:
+    # Log blocked requests to the console with a warning message
+    blockedRequests: true
+    # Log allowed requests to the console with an info message
+    allowedRequests: false
+  # List of allowed private IP ranges (in CIDR notation or wildcard format).
+  # Allows loopback IP ranges by default, but you can customize this list to fit your needs.
+  allowedRanges:
+    - '127.0.0.0/8'      # Loopback (IPv4)
+    - '::1/128'          # Loopback (IPv6)
+
 # User session timeout *in seconds* (defaults to 24 hours).
 ## Set to a positive number to expire session after a certain time of inactivity
 ## Set to 0 to expire session when the browser is closed

package-lock.json

@@ -38,6 +38,7 @@
                 "@mozilla/readability": "^0.6.0",
                 "@popperjs/core": "^2.11.8",
                 "@zeldafan0225/ai_horde": "^5.2.0",
+                "agent-base": "^7.1.3",
                 "archiver": "^7.0.1",
                 "bing-translate-api": "^4.1.0",
                 "body-parser": "^1.20.2",

package.json

@@ -29,6 +29,7 @@
         "@mozilla/readability": "^0.6.0",
         "@popperjs/core": "^2.11.8",
         "@zeldafan0225/ai_horde": "^5.2.0",
+        "agent-base": "^7.1.3",
         "archiver": "^7.0.1",
         "bing-translate-api": "^4.1.0",
         "body-parser": "^1.20.2",

src/middleware/corsProxy.js

@@ -37,6 +37,10 @@ export default async function corsProxyMiddleware(req, res) {
         // Copy over relevant response params to the proxy response
         await forwardFetchResponse(response, res);
     } catch (error) {
-        res.status(500).send('Error occurred while trying to proxy to: ' + url + ' ' + error);
+        console.error('Error in CORS proxy middleware:', error);
+        if (!res.headersSent) {
+            return res.sendStatus(500);
+        }
+        return res.end();
     }
 }

src/private-request-filter.js

@@ -0,0 +1,231 @@
+import net from 'node:net';
+import tls from 'node:tls';
+import http from 'node:http';
+import https from 'node:https';
+import dns from 'node:dns';
+import ipMatch from 'ip-matching';
+import ipRegex from 'ip-regex';
+import { Agent } from 'agent-base';
+import { color } from './util.js';
+import { filterValidIpPatterns } from './express-common.js';
+
+const LOG_HEADER = '[Private Request Filter]';
+
+/** @type {import('ip-matching').IPMatch[]} */
+const privateIpRanges = [
+    // Loopback (IPv4)
+    ipMatch.getMatch('127.0.0.0/8'),
+    // Class A private network
+    ipMatch.getMatch('10.0.0.0/8'),
+    // Class B private network
+    ipMatch.getMatch('172.16.0.0/12'),
+    // Class C private network
+    ipMatch.getMatch('192.168.0.0/16'),
+    // Link-local address (IPv4)
+    ipMatch.getMatch('169.254.0.0/16'),
+    // Loopback (IPv6)
+    ipMatch.getMatch('::1/128'),
+    // Unique local address (IPv6)
+    ipMatch.getMatch('fc00::/7'),
+    // Link-local address (IPv6)
+    ipMatch.getMatch('fe80::/10'),
+];
+
+/**
+ * Custom HTTP/HTTPS agent that blocks requests to private IP addresses unless they are explicitly allowed in the private address whitelist.
+ * This is used to prevent Server-Side Request Forgery (SSRF) attacks by ensuring that the server cannot make requests to internal services or resources that are not intended to be exposed.
+ * The agent checks if the target host resolves to a private IP address and blocks the request if it does, unless the IP address is included in the private address whitelist.
+ * The private address whitelist can contain specific IP addresses or CIDR ranges that are allowed to be accessed even if they fall within private IP ranges.
+ */
+class PrivateRequestAgent extends Agent {
+    /**
+     * List of private IP addresses or CIDR ranges to allow
+     * @type {Readonly<import('ip-matching').IPMatch[]>}
+     */
+    privateAddressWhitelist = [];
+
+    /**
+     * Whether to log blocked requests to the console
+     * @type {boolean}
+     */
+    logBlocked = true;
+
+    /**
+     * Whether to log allowed requests to the console
+     * @type {boolean}
+     */
+    logAllowed = false;
+
+    /**
+     * Whether to allow requests to hosts that cannot be resolved
+     * @type {boolean}
+     */
+    allowUnresolvedHosts = false;
+
+    /**
+     * Create a new PrivateRequestAgent instance.
+     * @param {object} options
+     * @param {string[]} options.privateAddressWhitelist List of private IP addresses or CIDR ranges to allow.
+     * @param {boolean} options.logBlocked Whether to log blocked requests to the console.
+     * @param {boolean} options.logAllowed Whether to log allowed requests to the console.
+     * @param {boolean} options.allowUnresolvedHosts Whether to allow requests to hosts that cannot be resolved.
+     * @param {boolean} options.enableKeepAlive Whether to enable HTTP/HTTPS keep-alive.
+     */
+    constructor(options = { privateAddressWhitelist: [], logBlocked: true, logAllowed: false, allowUnresolvedHosts: false, enableKeepAlive: false }) {
+        super({ keepAlive: options.enableKeepAlive });
+
+        const logEntryWarning = (entry, message) => `${color.red('Warning')}: Ignoring invalid private whitelist entry ${color.yellow(entry)} - ${message}`;
+        const whitelistArray = Array.isArray(options.privateAddressWhitelist) ? options.privateAddressWhitelist : [];
+        this.privateAddressWhitelist = Object.freeze(filterValidIpPatterns(whitelistArray, logEntryWarning).map(pattern => ipMatch.getMatch(pattern)));
+        this.allowUnresolvedHosts = options.allowUnresolvedHosts;
+        this.logBlocked = options.logBlocked;
+        this.logAllowed = options.logAllowed;
+    }
+
+    /**
+     * Check if the given address is a private IP address.
+     * @param {string} address The IP address to check.
+     * @returns {boolean} Whether the given address is a private IP address.
+     */
+    #isPrivateIp(address) {
+        return privateIpRanges.some(range => range.matches(address));
+    }
+
+    /**
+     * Check if the given address is allowed based on the private address whitelist.
+     * @param {string} address The IP address to check.
+     * @returns {boolean} Whether the given address is allowed based on the private address whitelist.
+     */
+    #isAllowedPrivateAddress(address) {
+        // Permit the request if the private IP address is in the whitelist
+        return this.privateAddressWhitelist.some(match => match.matches(address));
+    }
+
+    /**
+     * Connect method that checks if the target host resolves to a private IP address and blocks the request if it does.
+     * @param {http.ClientRequest} _req HTTP request object.
+     * @param {import('agent-base').AgentConnectOpts} options Agent connection options.
+     */
+    async connect(_req, options) {
+        /**
+         * Raise an error and log it if necessary.
+         * @param {string} message The error message.
+         * @param {boolean} [log=true] Whether to log the error to the console.
+         */
+        const raiseError = (message, log = true) => {
+            if (log) {
+                console.error(color.red(LOG_HEADER), message);
+            }
+            throw new Error(message);
+        };
+
+        /**
+         * Establish a connection to the target host using either TLS or a regular socket based on the options provided.
+         * @param {string|null} [hostOverride] Pass a host to override the one in options when connecting.
+         * @returns {net.Socket|tls.TLSSocket} A socket connected to the target host.
+         */
+        const connect = (hostOverride = null) => {
+            if (hostOverride) {
+                options.host = hostOverride;
+            }
+            if (options.secureEndpoint) {
+                return tls.connect(options);
+            } else {
+                return net.connect(options);
+            }
+        };
+
+        /**
+         * Validate the given IP address against the private address whitelist and connect if it's allowed.
+         * @param {string} ip The IP address to validate.
+         * @returns {net.Socket|tls.TLSSocket} A socket connected to the target IP address if it's allowed, otherwise an error is raised.
+         */
+        const validateIpAddress = (ip) => {
+            // Not a private IP address, allow the request
+            if (!this.#isPrivateIp(ip)) {
+                return connect(ip);
+            }
+
+            // Private IP address, check if it's allowed in the whitelist
+            if (this.#isAllowedPrivateAddress(ip)) {
+                if (this.logAllowed) {
+                    console.info(color.green(LOG_HEADER), 'Allowed request to private IP address:', color.blue(ip));
+                }
+
+                return connect(ip);
+            }
+
+            return raiseError(`Blocked request to private IP address: ${ip}`, this.logBlocked);
+        };
+
+        /**
+         * Resolve the given host to an IP address using DNS lookup.
+         * @param {string} host The host to resolve to an IP address.
+         * @returns {Promise<string>} The resolved IP address for the given host, or an empty string if the host cannot be resolved.
+         */
+        const lookupHost = async (host) => {
+            try {
+                return (await dns.promises.lookup(host)).address;
+            } catch {
+                return '';
+            }
+        };
+
+        const host = options.host;
+
+        if (!host) {
+            return raiseError('No host specified in request options', true);
+        }
+
+        const isIp = ipRegex.v4({ exact: true }).test(host) || ipRegex.v6({ exact: true }).test(host);
+
+        if (isIp) {
+            return validateIpAddress(host);
+        } else {
+            const address = await lookupHost(host);
+            if (!address) {
+                if (this.allowUnresolvedHosts) {
+                    return connect();
+                } else {
+                    return raiseError(`Unable to resolve host: ${host}. Set privateAddressWhitelist.allowUnresolvedHosts to true to bypass this check.`, true);
+                }
+            }
+
+            return validateIpAddress(address);
+        }
+    }
+}
+
+/**
+ * Initialize the private request filter by replacing the global HTTP and HTTPS agents with an instance of PrivateRequestAgent.
+ * @param {object} options Options for initializing the private request filter.
+ * @param {boolean} options.listen Whether the server is listening for incoming requests. This is used to determine whether to log a warning if the private request filter is not enabled.
+ * @param {boolean} options.enabled Whether the private request filter is enabled.
+ * @param {string[]} options.privateAddressWhitelist List of private IP addresses or CIDR ranges to allow.
+ * @param {boolean} options.logBlocked Whether to log blocked requests to the console.
+ * @param {boolean} options.logAllowed Whether to log allowed requests to the console.
+ * @param {boolean} options.allowUnresolvedHosts Whether to allow requests to hosts that cannot be resolved.
+ * @param {boolean} options.enableKeepAlive Whether to enable HTTP/HTTPS keep-alive.
+ */
+export default function initPrivateRequestFilter({ listen, enabled, privateAddressWhitelist, logBlocked, logAllowed, allowUnresolvedHosts, enableKeepAlive }) {
+    if (!enabled) {
+        if (listen) {
+            console.warn();
+            console.warn(color.yellow('Warning: listen is enabled but private request filter is disabled. This may expose your server to SSRF attacks.'));
+            console.warn(color.blue('To enable, provide trusted addresses in privateAddressWhitelist.allowedRanges and set privateAddressWhitelist.enabled to true in config.yaml and restart the server.'));
+        }
+        return;
+    }
+
+    const agent = new PrivateRequestAgent({ privateAddressWhitelist, logBlocked, logAllowed, allowUnresolvedHosts, enableKeepAlive });
+
+    http.globalAgent = agent;
+    https.globalAgent = agent;
+
+    console.info();
+    console.info(color.green(LOG_HEADER), 'Enabled');
+    if (agent.privateAddressWhitelist.length > 0) {
+        console.info(color.green(LOG_HEADER), 'Allowed private addresses:', color.blue(agent.privateAddressWhitelist.join(', ')));
+    }
+    console.info();
+}

src/request-proxy.js

@@ -14,14 +14,20 @@ const LOG_HEADER = '[Request Proxy]';
  * @property {string} url Proxy URL.
  * @property {string[]} bypass List of URLs to bypass proxy.
  * @property {boolean} enableKeepAlive Enable HTTP/HTTPS keep-alive.
+ * @property {boolean} privateRequestFilterEnabled Whether the private request filter is enabled.
  */
-export default function initRequestProxy({ enabled, url, bypass, enableKeepAlive }) {
+export default function initRequestProxy({ enabled, url, bypass, enableKeepAlive, privateRequestFilterEnabled }) {
     try {
         // No proxy is enabled, so return
         if (!enabled) {
             return;
         }
 
+        if (privateRequestFilterEnabled) {
+            console.warn(color.yellow(LOG_HEADER), 'Warning: Request proxy is enabled while private request filter is also enabled. Only URLs that BYPASS the request proxy will be checked.');
+            console.warn(color.yellow(LOG_HEADER), 'To ensure all requests are properly filtered, disable the request proxy.');
+        }
+
         if (!url) {
             console.error(color.red(LOG_HEADER), 'No proxy URL provided');
             return;
@@ -40,8 +46,11 @@ export default function initRequestProxy({ enabled, url, bypass, enableKeepAlive
             process.env.no_proxy = bypass.join(',');
         }
 
-        const proxyAgentOptions =  enableKeepAlive ? { keepAlive: true } : { keepAlive: false };
+        const httpAgent = http.globalAgent;
-        const proxyAgent = new ProxyAgent(proxyAgentOptions);
+        const httpsAgent = https.globalAgent;
+
+        const proxyAgent = new ProxyAgent({ httpAgent, httpsAgent, keepAlive: enableKeepAlive });
+
         http.globalAgent = proxyAgent;
         https.globalAgent = proxyAgent;
 

src/server-main.js

@@ -48,6 +48,7 @@ import getWhitelistMiddleware from './middleware/whitelist.js';
 import accessLoggerMiddleware, { getAccessLogPath, migrateAccessLog } from './middleware/accessLogWriter.js';
 import multerMonkeyPatch from './middleware/multerMonkeyPatch.js';
 import initRequestProxy from './request-proxy.js';
+import initPrivateRequestFilter from './private-request-filter.js';
 import cacheBuster from './middleware/cacheBuster.js';
 import corsProxyMiddleware from './middleware/corsProxy.js';
 import hostWhitelistMiddleware from './middleware/hostWhitelist.js';
@@ -333,8 +334,20 @@ async function preSetupTasks() {
         exitProcess();
     });
 
+    // Add private request filter.
+    const requestFilterOptions = {
+        listen: cliArgs.listen,
+        enabled: !!getConfigValue('privateAddressWhitelist.enabled', false, 'boolean'),
+        privateAddressWhitelist: getConfigValue('privateAddressWhitelist.allowedRanges', ['127.0.0.0/8', '::1/128']),
+        logBlocked: !!getConfigValue('privateAddressWhitelist.log.blockedRequests', true, 'boolean'),
+        logAllowed: !!getConfigValue('privateAddressWhitelist.log.allowedRequests', false, 'boolean'),
+        allowUnresolvedHosts: !!getConfigValue('privateAddressWhitelist.allowUnresolvedHosts', false, 'boolean'),
+        enableKeepAlive: cliArgs.enableKeepAlive,
+    };
+    initPrivateRequestFilter(requestFilterOptions);
+
     // Add request proxy.
-    initRequestProxy({ enabled: cliArgs.requestProxyEnabled, url: cliArgs.requestProxyUrl, bypass: cliArgs.requestProxyBypass, enableKeepAlive: cliArgs.enableKeepAlive });
+    initRequestProxy({ enabled: cliArgs.requestProxyEnabled, url: cliArgs.requestProxyUrl, bypass: cliArgs.requestProxyBypass, enableKeepAlive: cliArgs.enableKeepAlive, privateRequestFilterEnabled: requestFilterOptions.enabled });
 
     // Wait for frontend libs to compile
     await webpackMiddleware.runWebpackCompiler({ pruneCache: true });

tests/private-request-filter.test.js

@@ -0,0 +1,130 @@
+import { describe, test, expect, jest, beforeAll, beforeEach, afterAll } from '@jest/globals';
+
+const mockNetConnect = jest.fn(() => ({ type: 'net-socket' }));
+const mockTlsConnect = jest.fn(() => ({ type: 'tls-socket' }));
+const mockLookup = jest.fn();
+
+jest.unstable_mockModule('node:net', () => ({
+    default: { connect: mockNetConnect },
+}));
+
+jest.unstable_mockModule('node:tls', () => ({
+    default: { connect: mockTlsConnect },
+}));
+
+jest.unstable_mockModule('node:dns', () => ({
+    default: { promises: { lookup: mockLookup } },
+}));
+
+jest.unstable_mockModule('../src/util.js', () => ({
+    color: {
+        red: text => text,
+        green: text => text,
+        blue: text => text,
+        yellow: text => text,
+    },
+}));
+
+jest.unstable_mockModule('../src/express-common.js', () => ({
+    filterValidIpPatterns: patterns => patterns,
+}));
+
+/** @type {import('../src/private-request-filter.js').default} */
+let initPrivateRequestFilter;
+/** @type {import('node:http').default} */
+let http;
+/** @type {import('node:https').default} */
+let https;
+let originalHttpGlobalAgent;
+let originalHttpsGlobalAgent;
+
+beforeAll(async () => {
+    ({ default: initPrivateRequestFilter } = await import('../src/private-request-filter.js'));
+    ({ default: http } = await import('node:http'));
+    ({ default: https } = await import('node:https'));
+    originalHttpGlobalAgent = http.globalAgent;
+    originalHttpsGlobalAgent = https.globalAgent;
+});
+
+beforeEach(() => {
+    mockNetConnect.mockClear();
+    mockTlsConnect.mockClear();
+    mockLookup.mockReset();
+    http.globalAgent = originalHttpGlobalAgent;
+    https.globalAgent = originalHttpsGlobalAgent;
+});
+
+afterAll(() => {
+    http.globalAgent = originalHttpGlobalAgent;
+    https.globalAgent = originalHttpsGlobalAgent;
+});
+
+function initAgent({ privateAddressWhitelist = [], allowUnresolvedHosts = false } = {}) {
+    initPrivateRequestFilter({
+        listen: false,
+        enabled: true,
+        privateAddressWhitelist,
+        logBlocked: false,
+        logAllowed: false,
+        allowUnresolvedHosts,
+    });
+
+    return http.globalAgent;
+}
+
+describe('private request filter', () => {
+    test('allows direct private IP requests only when whitelisted', async () => {
+        const agent = initAgent({ privateAddressWhitelist: ['127.0.0.0/8'] });
+        await agent.connect({}, { host: '127.0.0.1', secureEndpoint: false });
+
+        expect(mockNetConnect).toHaveBeenCalledWith(expect.objectContaining({ host: '127.0.0.1' }));
+
+        const blockedAgent = initAgent({ privateAddressWhitelist: [] });
+        await expect(blockedAgent.connect({}, { host: '127.0.0.1', secureEndpoint: false }))
+            .rejects
+            .toThrow('Blocked request to private IP address: 127.0.0.1');
+    });
+
+    test('resolves hostnames and blocks when DNS returns private IP', async () => {
+        mockLookup.mockResolvedValue({ address: '192.168.1.8' });
+        const agent = initAgent();
+
+        await expect(agent.connect({}, { host: 'example.com', secureEndpoint: false }))
+            .rejects
+            .toThrow('Blocked request to private IP address: 192.168.1.8');
+        expect(mockNetConnect).not.toHaveBeenCalled();
+    });
+
+    test('connects to resolved public IP to avoid hostname re-resolution', async () => {
+        mockLookup.mockResolvedValue({ address: '93.184.216.34' });
+        const agent = initAgent();
+
+        await agent.connect({}, { host: 'example.com', secureEndpoint: false });
+
+        expect(mockLookup).toHaveBeenCalledWith('example.com');
+        expect(mockNetConnect).toHaveBeenCalledWith(expect.objectContaining({ host: '93.184.216.34' }));
+    });
+
+    test('handles unresolved hosts according to allowUnresolvedHosts setting', async () => {
+        mockLookup.mockRejectedValue(new Error('lookup failed'));
+        const blockedAgent = initAgent({ allowUnresolvedHosts: false });
+
+        await expect(blockedAgent.connect({}, { host: 'missing-host.local', secureEndpoint: false }))
+            .rejects
+            .toThrow('Unable to resolve host: missing-host.local. Set privateAddressWhitelist.allowUnresolvedHosts to true to bypass this check.');
+        expect(mockNetConnect).not.toHaveBeenCalled();
+
+        const allowedAgent = initAgent({ allowUnresolvedHosts: true });
+        await allowedAgent.connect({}, { host: 'missing-host.local', secureEndpoint: false });
+        expect(mockNetConnect).toHaveBeenCalledWith(expect.objectContaining({ host: 'missing-host.local' }));
+    });
+
+    test('uses tls.connect for secure endpoints', async () => {
+        mockLookup.mockResolvedValue({ address: '93.184.216.34' });
+        const agent = initAgent();
+        await agent.connect({}, { host: 'example.com', secureEndpoint: true });
+        expect(mockLookup).toHaveBeenCalledWith('example.com');
+        expect(mockTlsConnect).toHaveBeenCalledWith(expect.objectContaining({ host: '93.184.216.34' }));
+        expect(mockNetConnect).not.toHaveBeenCalled();
+    });
+});