parallel fetches + changed how it works

config/feeds.json

@@ -1,6 +1,25 @@
 {
+  "Web Development": [
+    "https://hacks.mozilla.org/feed/",
+    "https://blogasdfasdf/",
+    "https://web.dev/feed.xml",
+    "https://v8.dev/blog.atom",
+    "https://alistapart.com/main/feed/",
+    "https://css-tricks.com/feed/",
+    "https://dev.to/feed"
+  ],
   "Blogs": [
-    "https://george.mand.is/feed.json",
+    "https://george.mand.is/feed.xml",
     "https://joy.recurse.com/feed.atom"
+  ],
+  "My GitHub Projects": [
+    "https://github.com/georgemandis.atom",
+    "https://github.com/georgemandis/bubo-rss/releases.atom",
+    "https://github.com/georgemandis/konami-js/releases.atom",
+    "https://github.com/georgemandis/konami-js/commits/main.atom",
+    "https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom",
+    "https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom",
+    "https://github.com/georgemandis/remote-working-list/commits/master.atom",
+    "https://github.com/georgemandis/tweeter-totter/commits/master.atom"
   ]
 }

package-lock.json

@@ -9,6 +9,7 @@
       "version": "1.0.3",
       "license": "ISC",
       "dependencies": {
+        "chalk": "^5.0.0",
         "node-fetch": "^3.1.0",
         "nunjucks": "^3.2.0",
         "rss-parser": "^3.6.3"
@@ -519,6 +520,17 @@
         "node": ">=6"
       }
     },
+    "node_modules/chalk": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.0.tgz",
+      "integrity": "sha512-/duVOqst+luxCQRKEo4bNxinsOQtMP80ZYm7mMqzuh5PociNL0PvmHFvREJ9ueYL2TxlHjBcmLCdmocx9Vg+IQ==",
+      "engines": {
+        "node": "^12.17.0 || ^14.13 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
     "node_modules/commander": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",
@@ -2081,6 +2093,11 @@
       "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
       "dev": true
     },
+    "chalk": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.0.tgz",
+      "integrity": "sha512-/duVOqst+luxCQRKEo4bNxinsOQtMP80ZYm7mMqzuh5PociNL0PvmHFvREJ9ueYL2TxlHjBcmLCdmocx9Vg+IQ=="
+    },
     "commander": {
       "version": "5.1.0",
       "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",

package.json

@@ -7,8 +7,8 @@
   "scripts": {
     "dev": "tsc --watch",
     "clean": "rm -rf dist",
-    "build": "tsc && node dist/index.js > public/index.html",
+    "build": "tsc",
-    "build:bubo": "node dist/index.js > public/index.html"
+    "start": "node dist/index.js"    
   },
   "author": {
     "name": "George Mandis",
@@ -21,6 +21,7 @@
   },
   "license": "ISC",
   "dependencies": {
+    "chalk": "^5.0.0",
     "node-fetch": "^3.1.0",
     "nunjucks": "^3.2.0",
     "rss-parser": "^3.6.3"
@@ -35,4 +36,4 @@
     "tslib": "^2.3.1",
     "typescript": "^4.5.2"
   }
-}
+}

src/index.ts

@@ -13,53 +13,89 @@
 import fetch from "node-fetch";
 import Parser from "rss-parser";
 import { Feeds, FeedItem } from "./@types/bubo";
+import { Response } from "node-fetch";
 import { render } from "./renderer.js";
 import { getLink, getTitle, getTimestamp, parseFeed, getFeedList } from "./utilities.js";
+import { writeFile } from "fs/promises";
+import chalk from "chalk";
 
 const parser = new Parser();
 const feedList = await getFeedList();
 const contentFromAllFeeds: Feeds = {};
-const errors = [];
+const errors: unknown[] = [];
+const allFetches = [];
 
+
+const error = chalk.bold.red;
+const success = chalk.bold.green;
+const time = chalk.cyanBright.bold;
+
+
+// process each feed and its content
+const processFeed = (
+  {
+    group, feed, startTime
+  }: { group: string; feed: string, startTime: number }
+) => async (response: Response) => {
+  const body = await parseFeed(response);
+
+  // skip to the next one if this didn't work out
+  if (!body) return;
+
+  try {
+    const contents: FeedItem =
+      (typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem;
+
+    contents.feed = feed;
+    contents.title = getTitle(contents);
+    contents.link = getLink(contents);
+
+    // try to normalize date attribute naming
+    contents?.items?.forEach((item) => {
+      item.timestamp = getTimestamp(item);
+      item.title = getTitle(item);
+      item.link = getLink(item);
+    });
+    contentFromAllFeeds[group].push(contents as object);
+    console.log(`${success("Successfully fetched:")} ${feed}`, time(`(${((Date.now() - startTime) / 1000)} seconds)`));
+    return true;
+  } catch (err) {
+    console.log(`${error("Error processing:")} ${feed} (${((Date.now() - startTime) / 1000)} seconds)`);
+    errors.push(err);
+    return false;
+  }
+
+};
+
+
+// go through each group of feeds and process them
 for (const [group, feeds] of Object.entries(feedList)) {
   contentFromAllFeeds[group] = [];
 
   for (const feed of feeds) {
-    try {
+    console.log(`Fetching ${feed}...`);
-
+    const startTime = Date.now();
-      const response = await fetch(feed);
+    allFetches.push(
-      const body = await parseFeed(response);
+      fetch(feed).then(processFeed({ group, feed, startTime })).catch(err => {
-
+        console.log(error(`Error fetching ${feed} (${((Date.now() - startTime) / 1000)} seconds)`));
-      // skip to the next one if this didn't work out
+        errors.push(`Error fetching ${feed} ${err.toString()}`);
-      if (!body) continue;
+      })
-
+    );
-      const contents: FeedItem =
-        (typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem;
-
-      contents.feed = feed;
-      contents.title = getTitle(contents);
-      contents.link = getLink(contents);
-
-      // try to normalize date attribute naming
-      contents?.items?.forEach((item) => {
-        item.timestamp = getTimestamp(item);
-        item.title = getTitle(item);
-        item.link = getLink(item);
-      });
-
-      contentFromAllFeeds[group].push(contents as object);
-
-    } catch (error) {
-      errors.push(feed);
-    }
   }
 }
 
-// generate the static HTML output from our template renderer
+
-const output = render({
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
-  data: contentFromAllFeeds,
+Promise.all(allFetches).then(async () => {
-  errors: errors
+  console.log("\nDone fetching everything!");
+  // generate the static HTML output from our template renderer
+  const output = render({
+    data: contentFromAllFeeds,
+    errors: errors
+  });
+
+  // write the output to public/index.html
+  await writeFile("./public/index.html", output);
+
 });
 
-// return the rendered console and save it somewhere.
-console.log(output);