parallel fetches + changed how it works
This commit is contained in:
George Mandis
parent
d22fd1ef9d
commit
cd63571efc
4 changed files with 111 additions and 38 deletions
|
|
@ -1,6 +1,25 @@
|
|||
{
|
||||
"Web Development": [
|
||||
"https://hacks.mozilla.org/feed/",
|
||||
"https://blogasdfasdf/",
|
||||
"https://web.dev/feed.xml",
|
||||
"https://v8.dev/blog.atom",
|
||||
"https://alistapart.com/main/feed/",
|
||||
"https://css-tricks.com/feed/",
|
||||
"https://dev.to/feed"
|
||||
],
|
||||
"Blogs": [
|
||||
"https://george.mand.is/feed.json",
|
||||
"https://george.mand.is/feed.xml",
|
||||
"https://joy.recurse.com/feed.atom"
|
||||
],
|
||||
"My GitHub Projects": [
|
||||
"https://github.com/georgemandis.atom",
|
||||
"https://github.com/georgemandis/bubo-rss/releases.atom",
|
||||
"https://github.com/georgemandis/konami-js/releases.atom",
|
||||
"https://github.com/georgemandis/konami-js/commits/main.atom",
|
||||
"https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom",
|
||||
"https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom",
|
||||
"https://github.com/georgemandis/remote-working-list/commits/master.atom",
|
||||
"https://github.com/georgemandis/tweeter-totter/commits/master.atom"
|
||||
]
|
||||
}
|
||||
17
package-lock.json
generated
17
package-lock.json
generated
|
|
@ -9,6 +9,7 @@
|
|||
"version": "1.0.3",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"chalk": "^5.0.0",
|
||||
"node-fetch": "^3.1.0",
|
||||
"nunjucks": "^3.2.0",
|
||||
"rss-parser": "^3.6.3"
|
||||
|
|
@ -519,6 +520,17 @@
|
|||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/chalk": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.0.tgz",
|
||||
"integrity": "sha512-/duVOqst+luxCQRKEo4bNxinsOQtMP80ZYm7mMqzuh5PociNL0PvmHFvREJ9ueYL2TxlHjBcmLCdmocx9Vg+IQ==",
|
||||
"engines": {
|
||||
"node": "^12.17.0 || ^14.13 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/commander": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",
|
||||
|
|
@ -2081,6 +2093,11 @@
|
|||
"integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
|
||||
"dev": true
|
||||
},
|
||||
"chalk": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.0.tgz",
|
||||
"integrity": "sha512-/duVOqst+luxCQRKEo4bNxinsOQtMP80ZYm7mMqzuh5PociNL0PvmHFvREJ9ueYL2TxlHjBcmLCdmocx9Vg+IQ=="
|
||||
},
|
||||
"commander": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@
|
|||
"scripts": {
|
||||
"dev": "tsc --watch",
|
||||
"clean": "rm -rf dist",
|
||||
"build": "tsc && node dist/index.js > public/index.html",
|
||||
"build:bubo": "node dist/index.js > public/index.html"
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js"
|
||||
},
|
||||
"author": {
|
||||
"name": "George Mandis",
|
||||
|
|
@ -21,6 +21,7 @@
|
|||
},
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"chalk": "^5.0.0",
|
||||
"node-fetch": "^3.1.0",
|
||||
"nunjucks": "^3.2.0",
|
||||
"rss-parser": "^3.6.3"
|
||||
|
|
|
|||
104
src/index.ts
104
src/index.ts
|
|
@ -13,53 +13,89 @@
|
|||
import fetch from "node-fetch";
|
||||
import Parser from "rss-parser";
|
||||
import { Feeds, FeedItem } from "./@types/bubo";
|
||||
import { Response } from "node-fetch";
|
||||
import { render } from "./renderer.js";
|
||||
import { getLink, getTitle, getTimestamp, parseFeed, getFeedList } from "./utilities.js";
|
||||
import { writeFile } from "fs/promises";
|
||||
import chalk from "chalk";
|
||||
|
||||
const parser = new Parser();
|
||||
const feedList = await getFeedList();
|
||||
const contentFromAllFeeds: Feeds = {};
|
||||
const errors = [];
|
||||
const errors: unknown[] = [];
|
||||
const allFetches = [];
|
||||
|
||||
|
||||
const error = chalk.bold.red;
|
||||
const success = chalk.bold.green;
|
||||
const time = chalk.cyanBright.bold;
|
||||
|
||||
|
||||
// process each feed and its content
|
||||
const processFeed = (
|
||||
{
|
||||
group, feed, startTime
|
||||
}: { group: string; feed: string, startTime: number }
|
||||
) => async (response: Response) => {
|
||||
const body = await parseFeed(response);
|
||||
|
||||
// skip to the next one if this didn't work out
|
||||
if (!body) return;
|
||||
|
||||
try {
|
||||
const contents: FeedItem =
|
||||
(typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem;
|
||||
|
||||
contents.feed = feed;
|
||||
contents.title = getTitle(contents);
|
||||
contents.link = getLink(contents);
|
||||
|
||||
// try to normalize date attribute naming
|
||||
contents?.items?.forEach((item) => {
|
||||
item.timestamp = getTimestamp(item);
|
||||
item.title = getTitle(item);
|
||||
item.link = getLink(item);
|
||||
});
|
||||
contentFromAllFeeds[group].push(contents as object);
|
||||
console.log(`${success("Successfully fetched:")} ${feed}`, time(`(${((Date.now() - startTime) / 1000)} seconds)`));
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.log(`${error("Error processing:")} ${feed} (${((Date.now() - startTime) / 1000)} seconds)`);
|
||||
errors.push(err);
|
||||
return false;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
// go through each group of feeds and process them
|
||||
for (const [group, feeds] of Object.entries(feedList)) {
|
||||
contentFromAllFeeds[group] = [];
|
||||
|
||||
for (const feed of feeds) {
|
||||
try {
|
||||
|
||||
const response = await fetch(feed);
|
||||
const body = await parseFeed(response);
|
||||
|
||||
// skip to the next one if this didn't work out
|
||||
if (!body) continue;
|
||||
|
||||
const contents: FeedItem =
|
||||
(typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem;
|
||||
|
||||
contents.feed = feed;
|
||||
contents.title = getTitle(contents);
|
||||
contents.link = getLink(contents);
|
||||
|
||||
// try to normalize date attribute naming
|
||||
contents?.items?.forEach((item) => {
|
||||
item.timestamp = getTimestamp(item);
|
||||
item.title = getTitle(item);
|
||||
item.link = getLink(item);
|
||||
});
|
||||
|
||||
contentFromAllFeeds[group].push(contents as object);
|
||||
|
||||
} catch (error) {
|
||||
errors.push(feed);
|
||||
}
|
||||
console.log(`Fetching ${feed}...`);
|
||||
const startTime = Date.now();
|
||||
allFetches.push(
|
||||
fetch(feed).then(processFeed({ group, feed, startTime })).catch(err => {
|
||||
console.log(error(`Error fetching ${feed} (${((Date.now() - startTime) / 1000)} seconds)`));
|
||||
errors.push(`Error fetching ${feed} ${err.toString()}`);
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// generate the static HTML output from our template renderer
|
||||
const output = render({
|
||||
data: contentFromAllFeeds,
|
||||
errors: errors
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
Promise.all(allFetches).then(async () => {
|
||||
console.log("\nDone fetching everything!");
|
||||
// generate the static HTML output from our template renderer
|
||||
const output = render({
|
||||
data: contentFromAllFeeds,
|
||||
errors: errors
|
||||
});
|
||||
|
||||
// write the output to public/index.html
|
||||
await writeFile("./public/index.html", output);
|
||||
|
||||
});
|
||||
|
||||
// return the rendered console and save it somewhere.
|
||||
console.log(output);
|
||||
|
|
|
|||
Loading…
Reference in a new issue