increase connection timeout

Author: afeld

lib/check_urls.js

@@ -1,4 +1,12 @@
 import records from "../_data/catalogs.js";
+import { Agent } from "undici";
+
+const TIMEOUT_IN_MS = 20 * 1000;
+// connection timeout
+// https://stackoverflow.com/a/77319848/358804
+const AGENT = new Agent({ connectTimeout: TIMEOUT_IN_MS });
+// response timeout
+const SIGNAL = AbortSignal.timeout(TIMEOUT_IN_MS);
 
 const handleError = (url, error) => {
   const cause = error.cause || {};
@@ -20,12 +28,12 @@ const checkUrl = async (url) => {
         "User-Agent": "https://dataportals.org/ bot",
       },
       redirect: "follow",
-      signal: AbortSignal.timeout(20 * 1000),
+      dispatcher: AGENT,
+      signal: SIGNAL,
     });
 
-    const status = response.status;
-    if (status < 200 || status >= 300) {
-      console.warn(`${url} has status ${status}`);
+    if (!response.ok) {
+      console.warn(`${url} has status ${response.status}`);
       return false;
     }
   } catch (error) {

package-lock.json

@@ -12,7 +12,8 @@
   "dependencies": {
     "@11ty/eleventy": "^3.0.0",
     "csv-parse": "^5.6.0",
-    "markdown-it": "^14.1.0"
+    "markdown-it": "^14.1.0",
+    "undici": "^7.10.0"
   },
   "repository": {
     "type": "git",