diff --git a/config/feeds.json b/config/feeds.json index d9219e1..d6b2b10 100644 --- a/config/feeds.json +++ b/config/feeds.json @@ -1,6 +1,25 @@ { + "Web Development": [ + "https://hacks.mozilla.org/feed/", + "https://blogasdfasdf/", + "https://web.dev/feed.xml", + "https://v8.dev/blog.atom", + "https://alistapart.com/main/feed/", + "https://css-tricks.com/feed/", + "https://dev.to/feed" + ], "Blogs": [ - "https://george.mand.is/feed.json", + "https://george.mand.is/feed.xml", "https://joy.recurse.com/feed.atom" + ], + "My GitHub Projects": [ + "https://github.com/georgemandis.atom", + "https://github.com/georgemandis/bubo-rss/releases.atom", + "https://github.com/georgemandis/konami-js/releases.atom", + "https://github.com/georgemandis/konami-js/commits/main.atom", + "https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom", + "https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom", + "https://github.com/georgemandis/remote-working-list/commits/master.atom", + "https://github.com/georgemandis/tweeter-totter/commits/master.atom" ] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index b0f5c2d..7ba4603 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.3", "license": "ISC", "dependencies": { + "chalk": "^5.0.0", "node-fetch": "^3.1.0", "nunjucks": "^3.2.0", "rss-parser": "^3.6.3" @@ -519,6 +520,17 @@ "node": ">=6" } }, + "node_modules/chalk": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.0.tgz", + "integrity": "sha512-/duVOqst+luxCQRKEo4bNxinsOQtMP80ZYm7mMqzuh5PociNL0PvmHFvREJ9ueYL2TxlHjBcmLCdmocx9Vg+IQ==", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/commander": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", @@ -2081,6 +2093,11 @@ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true }, + "chalk": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.0.tgz", + "integrity": "sha512-/duVOqst+luxCQRKEo4bNxinsOQtMP80ZYm7mMqzuh5PociNL0PvmHFvREJ9ueYL2TxlHjBcmLCdmocx9Vg+IQ==" + }, "commander": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", diff --git a/package.json b/package.json index 2554dea..009b47a 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,8 @@ "scripts": { "dev": "tsc --watch", "clean": "rm -rf dist", - "build": "tsc && node dist/index.js > public/index.html", - "build:bubo": "node dist/index.js > public/index.html" + "build": "tsc", + "start": "node dist/index.js" }, "author": { "name": "George Mandis", @@ -21,6 +21,7 @@ }, "license": "ISC", "dependencies": { + "chalk": "^5.0.0", "node-fetch": "^3.1.0", "nunjucks": "^3.2.0", "rss-parser": "^3.6.3" @@ -35,4 +36,4 @@ "tslib": "^2.3.1", "typescript": "^4.5.2" } -} \ No newline at end of file +} diff --git a/src/index.ts b/src/index.ts index 2acd688..f643031 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,53 +13,89 @@ import fetch from "node-fetch"; import Parser from "rss-parser"; import { Feeds, FeedItem } from "./@types/bubo"; +import { Response } from "node-fetch"; import { render } from "./renderer.js"; import { getLink, getTitle, getTimestamp, parseFeed, getFeedList } from "./utilities.js"; +import { writeFile } from "fs/promises"; +import chalk from "chalk"; const parser = new Parser(); const feedList = await getFeedList(); const contentFromAllFeeds: Feeds = {}; -const errors = []; +const errors: unknown[] = []; +const allFetches = []; + +const error = chalk.bold.red; +const success = chalk.bold.green; +const time = chalk.cyanBright.bold; + + +// process each feed and its content +const processFeed = ( + { + group, feed, startTime + }: { group: string; feed: string, startTime: number } +) => async (response: Response) => { + const body = await parseFeed(response); + + // skip to the next one if this didn't work out + if (!body) return; + + try { + const contents: FeedItem = + (typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem; + + contents.feed = feed; + contents.title = getTitle(contents); + contents.link = getLink(contents); + + // try to normalize date attribute naming + contents?.items?.forEach((item) => { + item.timestamp = getTimestamp(item); + item.title = getTitle(item); + item.link = getLink(item); + }); + contentFromAllFeeds[group].push(contents as object); + console.log(`${success("Successfully fetched:")} ${feed}`, time(`(${((Date.now() - startTime) / 1000)} seconds)`)); + return true; + } catch (err) { + console.log(`${error("Error processing:")} ${feed} (${((Date.now() - startTime) / 1000)} seconds)`); + errors.push(err); + return false; + } + +}; + + +// go through each group of feeds and process them for (const [group, feeds] of Object.entries(feedList)) { contentFromAllFeeds[group] = []; for (const feed of feeds) { - try { - - const response = await fetch(feed); - const body = await parseFeed(response); - - // skip to the next one if this didn't work out - if (!body) continue; - - const contents: FeedItem = - (typeof body === "string" ? (await parser.parseString(body)) : body) as FeedItem; - - contents.feed = feed; - contents.title = getTitle(contents); - contents.link = getLink(contents); - - // try to normalize date attribute naming - contents?.items?.forEach((item) => { - item.timestamp = getTimestamp(item); - item.title = getTitle(item); - item.link = getLink(item); - }); - - contentFromAllFeeds[group].push(contents as object); - - } catch (error) { - errors.push(feed); - } + console.log(`Fetching ${feed}...`); + const startTime = Date.now(); + allFetches.push( + fetch(feed).then(processFeed({ group, feed, startTime })).catch(err => { + console.log(error(`Error fetching ${feed} (${((Date.now() - startTime) / 1000)} seconds)`)); + errors.push(`Error fetching ${feed} ${err.toString()}`); + }) + ); } } -// generate the static HTML output from our template renderer -const output = render({ - data: contentFromAllFeeds, - errors: errors + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +Promise.all(allFetches).then(async () => { + console.log("\nDone fetching everything!"); + // generate the static HTML output from our template renderer + const output = render({ + data: contentFromAllFeeds, + errors: errors + }); + + // write the output to public/index.html + await writeFile("./public/index.html", output); + }); -// return the rendered console and save it somewhere. -console.log(output);