gablaxy/bubo-rss
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixed formatting and output; setup idea for throttling

Browse source
This commit is contained in:
George Mandis 2021-11-28 13:12:13 -08:00
parent cd63571efc
commit 57b61e4fe1
1 changed files with 16 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/index.ts
View file

@ -23,13 +23,16 @@ const parser = new Parser();
const feedList = await getFeedList(); const feedList = await getFeedList();
const contentFromAllFeeds: Feeds = {}; const contentFromAllFeeds: Feeds = {};
const errors: unknown[] = []; const errors: unknown[] = [];
const allFetches = []; const allFetches: Promise<boolean | void | undefined>[] = [];
const MAX_CONNECTIONS = 2;
let connections = 0;
const initTime = Date.now();
const benchmark = (startTime: number) => chalk.cyanBright.bold(`(${(Date.now() - startTime) / 1000} seconds)`);
const error = chalk.bold.red; const error = chalk.bold.red;
const success = chalk.bold.green; const success = chalk.bold.green;
const time = chalk.cyanBright.bold;
// process each feed and its content // process each feed and its content
const processFeed = ( const processFeed = (
@ -37,6 +40,7 @@ const processFeed = (
group, feed, startTime group, feed, startTime
}: { group: string; feed: string, startTime: number } }: { group: string; feed: string, startTime: number }
) => async (response: Response) => { ) => async (response: Response) => {
connections--;
const body = await parseFeed(response); const body = await parseFeed(response);
// skip to the next one if this didn't work out // skip to the next one if this didn't work out
@ -57,10 +61,10 @@ const processFeed = (
item.link = getLink(item); item.link = getLink(item);
}); });
contentFromAllFeeds[group].push(contents as object); contentFromAllFeeds[group].push(contents as object);
console.log(`${success("Successfully fetched:")} ${feed}`, time(`(${((Date.now() - startTime) / 1000)} seconds)`)); console.log(`${success("Successfully fetched:")} ${feed} ${benchmark(startTime)}`);
return true; return true;
} catch (err) { } catch (err) {
console.log(`${error("Error processing:")} ${feed} (${((Date.now() - startTime) / 1000)} seconds)`); console.log(`${error("Error processing:")} ${feed} ${benchmark(startTime)}`);
errors.push(err); errors.push(err);
return false; return false;
} }
@ -69,15 +73,18 @@ const processFeed = (
// go through each group of feeds and process them // go through each group of feeds and process them
for (const [group, feeds] of Object.entries(feedList)) { for (const [group, feeds] of Object.entries(feedList)) {
contentFromAllFeeds[group] = []; contentFromAllFeeds[group] = [];
for (const feed of feeds) { for (const feed of feeds) {
console.log(`Fetching ${feed}...`); // throttle if we're exceeding MAX_CONNECTIONS
connections++;
console.log(`Fetching: ${feed}...`);
const startTime = Date.now(); const startTime = Date.now();
allFetches.push( allFetches.push(
fetch(feed).then(processFeed({ group, feed, startTime })).catch(err => { fetch(feed).then(processFeed({ group, feed, startTime })).catch(err => {
console.log(error(`Error fetching ${feed} (${((Date.now() - startTime) / 1000)} seconds)`)); console.log(error(`Error fetching ${feed} ${benchmark(startTime)}`));
errors.push(`Error fetching ${feed} ${err.toString()}`); errors.push(`Error fetching ${feed} ${err.toString()}`);
}) })
); );
@ -88,6 +95,7 @@ for (const [group, feeds] of Object.entries(feedList)) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
Promise.all(allFetches).then(async () => { Promise.all(allFetches).then(async () => {
console.log("\nDone fetching everything!"); console.log("\nDone fetching everything!");
// generate the static HTML output from our template renderer // generate the static HTML output from our template renderer
const output = render({ const output = render({
data: contentFromAllFeeds, data: contentFromAllFeeds,
@ -96,6 +104,6 @@ Promise.all(allFetches).then(async () => {
// write the output to public/index.html // write the output to public/index.html
await writeFile("./public/index.html", output); await writeFile("./public/index.html", output);
console.log(`Finished writing to output. ${benchmark(initTime)}`);
}); });