Feature: allow disable ipv6 in proxy, refactor cacheFetch to use proxy (#5011)

src/utils/config/widget-helpers.js

@@ -56,21 +56,22 @@ export async function cleanWidgetGroups(widgets) {
 export async function getPrivateWidgetOptions(type, widgetIndex) {
   const widgets = await widgetsFromConfig();
 
-  const privateOptions = widgets.map((widget) => {
-    const { index, url, username, password, key, apiKey } = widget.options;
+  const privateOptions =
+    widgets.map((widget) => {
+      const { index, url, username, password, key, apiKey } = widget.options;
 
-    return {
-      type: widget.type,
-      options: {
-        index,
-        url,
-        username,
-        password,
-        key,
-        apiKey,
-      },
-    };
-  });
+      return {
+        type: widget.type,
+        options: {
+          index,
+          url,
+          username,
+          password,
+          key,
+          apiKey,
+        },
+      };
+    }) || {};
 
   return type !== undefined && widgetIndex !== undefined
     ? privateOptions.find((o) => o.type === type && o.options.index === parseInt(widgetIndex, 10))?.options

src/utils/proxy/cached-fetch.js

@@ -1,25 +0,0 @@
-import cache from "memory-cache";
-
-const defaultDuration = 5;
-
-export default async function cachedFetch(url, duration, ua) {
-  const cached = cache.get(url);
-
-  // eslint-disable-next-line no-param-reassign
-  duration = duration || defaultDuration;
-
-  if (cached) {
-    return cached;
-  }
-
-  // wrapping text in JSON.parse to handle utf-8 issues
-  const options = {};
-  if (ua) {
-    options.headers = {
-      "User-Agent": ua,
-    };
-  }
-  const data = await fetch(url, options).then((res) => res.json());
-  cache.put(url, data, duration * 1000 * 60);
-  return data;
-}

src/utils/proxy/http.js

@@ -3,6 +3,7 @@
 import { createUnzip, constants as zlibConstants } from "node:zlib";
 
 import { http, https } from "follow-redirects";
+import cache from "memory-cache";
 
 import { addCookieToJar, setCookieHeader } from "./cookie-jar";
 import { sanitizeErrorURL } from "./api-helpers";
@@ -81,20 +82,46 @@ export function httpRequest(url, params) {
   return handleRequest(http, url, params);
 }
 
+export async function cachedRequest(url, duration = 5, ua = "homepage") {
+  const cached = cache.get(url);
+
+  if (cached) {
+    return cached;
+  }
+
+  const options = {
+    headers: {
+      "User-Agent": ua,
+      Accept: "application/json",
+    },
+  };
+  let [, , data] = await httpProxy(url, options);
+  if (Buffer.isBuffer(data)) {
+    try {
+      data = JSON.parse(Buffer.from(data).toString());
+    } catch (e) {
+      logger.debug("Error parsing cachedRequest data for %s: %s %s", url, Buffer.from(data).toString(), e);
+      data = Buffer.from(data).toString();
+    }
+  }
+  cache.put(url, data, duration * 1000 * 60);
+  return data;
+}
+
 export async function httpProxy(url, params = {}) {
   const constructedUrl = new URL(url);
+  const disableIpv6 = process.env.HOMEPAGE_PROXY_DISABLE_IPV6 === "true";
+  const agentOptions = disableIpv6 ? { family: 4, autoSelectFamily: false } : {};
 
   let request = null;
   if (constructedUrl.protocol === "https:") {
     request = httpsRequest(constructedUrl, {
-      agent: new https.Agent({
-        rejectUnauthorized: false,
-      }),
+      agent: new https.Agent({ ...agentOptions, rejectUnauthorized: false }),
       ...params,
     });
   } else {
     request = httpRequest(constructedUrl, {
-      agent: new http.Agent(),
+      agent: new http.Agent(agentOptions),
       ...params,
     });
   }