gablaxy/bubo-rss
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Init astro

Browse source
This commit is contained in:
Carter McBride 2024-08-22 18:00:58 -06:00
parent 93326d6483
commit def05390bf
17 changed files with 303 additions and 421 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.astro/settings.json Normal file
View file

@ -0,0 +1,5 @@
{
"_variables": {
"lastUpdateCheck": 1724282769525
}
}

1
.astro/types.d.ts vendored Normal file
View file

@ -0,0 +1 @@
/// <reference types="astro/client" />

6
.editorconfig Normal file
View file

@ -0,0 +1,6 @@
[*]
end_of_line = lf
insert_final_newline = true
tab_width = 2
indent_size = 2
indent_style = tab

2
.gitignore vendored
View file

@ -4,3 +4,5 @@ public/styles.css
dist/*
.DS_Store
configs/feeds.json
src-old/
.env

8
astro.config.js Normal file
View file

@ -0,0 +1,8 @@
import tailwindIntegration from "@astrojs/tailwind";
import { defineConfig } from "astro/config";
// https://astro.build/config
export default defineConfig({
output: "static",
integrations: [tailwindIntegration()],
});

29
biome.json
View file

@ -1,5 +1,5 @@
{
"$schema": "https://biomejs.dev/schemas/1.8.1/schema.json",
"$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
"organizeImports": {
"enabled": true
},
@ -8,5 +8,32 @@
"rules": {
"recommended": true
}
},
"css": {
"formatter": {
"enabled": true
},
"linter": {
"enabled": true
}
},
"vcs": {
"enabled": true,
"clientKind": "git",
"useIgnoreFile": true,
"defaultBranch": "main"
},
"overrides": [
{
"include": ["*.svelte", "*.astro", "*.vue"],
"linter": {
"rules": {
"style": {
"useConst": "off",
"useImportType": "off"
}
}
}
}
]
}

BIN
bun.lockb
View file

Binary file not shown.

9
package.json
View file

@ -6,10 +6,11 @@
"main": "src/index.ts",
"type": "module",
"scripts": {
"clean": "rm -rf dist",
"build": "NODE_ENV=production bun src/index.ts",
"check": "biome check --write ./{src,config,public} ./*.{json,js} --no-errors-on-unmatched",
"dev": "bun src/index.ts"
"dev": "astro dev",
"start": "astro dev",
"build": "astro build",
"preview": "astro preview"
},
"author": {
"name": "George Mandis",
@ -26,7 +27,9 @@
},
"license": "MIT",
"dependencies": {
"@astrojs/tailwind": "^5.1.0",
"@feelinglovelynow/get-relative-time": "^1.1.2",
"astro": "^4.14.4",
"chalk": "^5.2.0",
"node-fetch": "^3.3.1",
"nunjucks": "^3.2.4",

15
src/@types/bubo.d.ts vendored
View file

@ -1,15 +0,0 @@
export interface Feeds {
[key: string]: object[];
}
export interface FeedItem {
[key: string]: string | number | Date | boolean | FeedItem[];
items: FeedItem[];
}
//NEW WAY
export type JSONValue =
| string
| number
| boolean
| { [x: string]: JSONValue }
| Array<JSONValue>;

1
src/env.d.ts vendored Normal file
View file

@ -0,0 +1 @@
/// <reference path="../.astro/types.d.ts" />

208
src/index.ts
View file

@ -1,208 +0,0 @@
/*
* 🦉 Bubo Reader
* ====
* Dead simple feed reader (RSS + JSON) that renders an HTML
* page with links to content from feeds organized by site
*
* Code: https://github.com/georgemandis/bubo-rss
* Copyright (c) 2019 George Mandis (https://george.mand.is)
* Version: 1.0.1 (11/14/2021)
* Licensed under the MIT License (http://opensource.org/licenses/MIT)
*/
import { writeFile } from "node:fs/promises";
import chalk from "chalk";
import fetch from "node-fetch";
import type { Response } from "node-fetch";
import Parser from "rss-parser";
import type { FeedItem, Feeds } from "./@types/bubo";
import { render } from "./renderer.js";
import {
buildCSS,
getBuboInfo,
getFeedList,
getLink,
getTimestamp,
getTitle,
parseFeed,
} from "./utilities.js";
const cssInput = "./public/styles.input.css";
const cssOutput = "./public/styles.css";
const minifyCss = process.env.NODE_ENV === "production";
await buildCSS(minifyCss, cssInput, cssOutput);
const buboInfo = await getBuboInfo();
const parser = new Parser();
const feedOptions: Parameters<typeof getFeedList>[0] = {
feeds: process.env.FEEDS,
feedFilePath: process.env.FEEDS
? ""
: process.env.FEED_FILE ?? "../config/feeds.json",
};
console.log("feedOptions", JSON.stringify(feedOptions, null, 2));
const feedList = await getFeedList(feedOptions);
const feedListLength =
Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
/**
* contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end
* errors = Contains errors from parsing feeds and is also passed to template.
*/
const contentFromAllFeeds: Feeds = {};
const errors: unknown[] = [];
// benchmarking data + utility
const initTime = Date.now();
const benchmark = (startTime: number) =>
chalk.cyanBright.bold(`${(Date.now() - startTime) / 1000} seconds`);
/**
* These values are used to control throttling/batching the fetches:
* - MAX_CONNECTION = max number of fetches to contain in a batch
* - DELAY_MS = the delay in milliseconds between batches
*/
const MAX_CONNECTIONS = Number.POSITIVE_INFINITY;
const DELAY_MS = 850;
const error = chalk.bold.red;
const success = chalk.bold.green;
// keeping tally of total feeds fetched and parsed so we can compare
// to feedListLength and know when we're finished.
let completed = 0;
/**
* finishBuild
* --
* function that gets called when all the feeds are through fetching
* and we want to build the static output.
*/
const finishBuild: () => void = async () => {
completed++;
// if this isn't the last feed, just return early
if (completed !== feedListLength) return;
process.stdout.write("\nDone fetching everything!\n");
// sort all the categories and the feeds alphabetically
const sortedFeeds: Feeds = {};
const sortedKeys = Object.keys(contentFromAllFeeds).sort((a, b) =>
a.localeCompare(b),
);
for (const key of sortedKeys) {
sortedFeeds[key] = contentFromAllFeeds[key].sort((a, b) =>
a.title.localeCompare(b.title),
);
}
const yazzyUrl = process.env.YAZZY_URL;
process.stdout.write(`\nUsing yazzy url: "${yazzyUrl}"\n`);
// generate the static HTML output from our template renderer
const output = render({
data: sortedFeeds,
errors: errors,
info: buboInfo,
yazzyUrl,
});
// write the output to public/index.html
await writeFile("./public/index.html", output);
process.stdout.write(
`\nFinished writing to output:\n- ${feedListLength} feeds in ${benchmark(
initTime,
)}\n- ${errors.length} errors\n`,
);
};
/**
* processFeed
* --
* Process an individual feed and normalize its items
* @param { group, feed, startTime}
* @returns Promise<void>
*/
const processFeed =
({
group,
feed,
startTime,
}: {
group: string;
feed: string;
startTime: number;
}) =>
async (response: Response): Promise<void> => {
const body = await parseFeed(response);
//skip to the next one if this didn't work out
if (!body) return;
try {
const contents: FeedItem = (
typeof body === "string" ? await parser.parseString(body) : body
) as FeedItem;
contents.feed = feed;
contents.title = getTitle(contents);
contents.link = getLink(contents);
// try to normalize date attribute naming
for (const item of contents.items) {
item.timestamp = getTimestamp(item);
item.title = getTitle(item);
item.link = getLink(item);
const timestamp = new Date(Number.parseInt(item.timestamp));
const eightHoursAgo = new Date();
eightHoursAgo.setHours(eightHoursAgo.getHours() - 8);
item.isRecent = timestamp > eightHoursAgo;
}
contents.hasRecent = contents.items.some((item) => item.isRecent);
contentFromAllFeeds[group].push(contents as object);
process.stdout.write(
`${success("Successfully fetched:")} ${feed} - ${benchmark(startTime)}\n`,
);
} catch (err) {
process.stdout.write(
`${error("Error processing:")} ${feed} - ${benchmark(
startTime,
)}\n${err}\n`,
);
errors.push(`Error processing: ${feed}\n\t${err}`);
}
finishBuild();
};
// go through each group of feeds and process
const processFeeds = () => {
let idx = 0;
for (const [group, feeds] of Object.entries(feedList)) {
contentFromAllFeeds[group] = [];
for (const feed of feeds) {
const startTime = Date.now();
setTimeout(
() => {
process.stdout.write(`Fetching: ${feed}...\n`);
fetch(feed)
.then(processFeed({ group, feed, startTime }))
.catch((err) => {
process.stdout.write(
error(`Error fetching ${feed} ${benchmark(startTime)}\n`),
);
errors.push(`Error fetching ${feed} ${err.toString()}\n`);
finishBuild();
});
},
(idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS,
);
idx++;
}
}
};
processFeeds();

65
src/pages/index.astro Normal file
View file

@ -0,0 +1,65 @@
---
import getAllFeedItems from "../services/feeds";
const feedItems = await getAllFeedItems();
const categories = Array.from(
new Set(feedItems.contents.map((item) => item.category)),
).sort();
const categoriesSelectorCss = categories.map((c => `
#category-picker:has(#${c}:checked) ~ main ul {
> .${c}-item {
display: block;
}
> *:not(.${c}-item) {
display: none;
}
}`)).join("\n")
---
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta http-equiv="X-UA-Compatible" content="ie=edge" />
<title>Carter's RSS Feeds</title>
<link rel="icon" href="/news-emoji.svg" sizes="any" type="image/svg+xml" />
<link rel="manifest" href="/manifest.json" />
<style client:load set:html={categoriesSelectorCss}>
</style>
</head>
<body class="font-system text-base m-4">
<header>
<h1 class="text-2xl">📰 Carter's RSS Feeds</h1>
<p>
{feedItems.contents.length} item(s) | {feedItems.errors.length} error(s)
</p>
</header>
<nav id="category-picker">
<input type="radio" id="all" name="category" checked />
<label for="all">All</label>
{
categories.map((category) => (
<>
<input type="radio" id={category} name="category" />
<label for={category}>{category}</label>
</>
))
}
</nav>
<main>
<ul>
{
feedItems.contents.map((item) => (
<li class={`${item.category}-item`}>
<a href={item.link}>
<span set:html={item.title} /> | {item.feedName} |{" "}
{item.category} | {new Date(item.pubIsoDate).toDateString()}
</a>
</li>
))
}
</ul>
</main>
</body>
</html>

58
src/renderer.ts
View file

@ -1,58 +0,0 @@
/*
* Return our renderer.
* Using Nunjucks out of the box.
* https://mozilla.github.io/nunjucks/
*/
import nunjucks from "nunjucks";
const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
import { readFile } from "node:fs/promises";
import { getRelativeTime } from "@feelinglovelynow/get-relative-time";
import type { Feeds, JSONValue } from "./@types/bubo";
/**
* Global filters for my Nunjucks templates
*/
env.addFilter("relative", (dateString): string => {
const date: Date = new Date(Number.parseInt(dateString));
return !Number.isNaN(date.getTime()) ? getRelativeTime(date) : dateString;
});
env.addFilter("formatTime", (dateString): string => {
const date: Date = new Date(Number.parseInt(dateString));
return !Number.isNaN(date.getTime()) ? date.toLocaleTimeString() : dateString;
});
env.addFilter("formatDateTime", (dateString): string => {
const date: Date = new Date(Number.parseInt(dateString));
return !Number.isNaN(date.getTime()) ? date.toLocaleString() : dateString;
});
env.addGlobal("now", new Date().getTime());
// load the template
const template: string = (
await readFile(new URL("../config/template.html", import.meta.url))
).toString();
// generate the static HTML output from our template renderer
const render = ({
data,
errors,
info,
yazzyUrl,
}: {
data: Feeds;
errors: unknown[];
info?: JSONValue;
yazzyUrl?: string;
}) => {
return env.renderString(template, {
data,
errors,
info,
yazzyUrl,
});
};
export { render };

178
src/services/feeds.ts Normal file
View file

@ -0,0 +1,178 @@
import Parser from "rss-parser";
interface FeedItem {
title: string;
feedName: string;
feedLink: string;
pubIsoDate: number;
link: string;
category: string;
}
const MAX_CONNECTIONS = Number.POSITIVE_INFINITY;
const DELAY_MS = 850;
const parser = new Parser();
function readFeedCategoriesFromEnv(): Record<string, string[]> {
if (import.meta.env.FEEDS) {
return JSON.parse(import.meta.env.FEEDS);
}
throw new Error("FEEDS environment variable is not set");
}
async function getRawFeedContents(response: Response): Promise<unknown> {
const contentType = response.headers.get("content-type")?.split(";")[0];
if (!contentType) return {};
if (
[
"application/atom+xml",
"application/rss+xml",
"application/xml",
"text/xml",
"text/html",
].includes(contentType)
) {
return response.text();
}
if (["application/json", "application/feed+json"].includes(contentType)) {
return response.json();
}
return {};
}
interface RawFeedItem {
creator?: string;
title: string;
link: string;
pubDate: string;
"content:encoded"?: string;
"content:encodedSnippet"?: string;
"dc:creator"?: string;
comments?: string;
content: string;
contentSnippet: string;
guid: string;
categories: unknown[];
isoDate: string;
[other: string]: unknown;
}
interface RawFeed {
items: RawFeedItem[];
feedUrl?: string;
image?: {
link: string;
url: string;
title: string;
width: string;
height: string;
};
pagenationLinks?: {
self: string;
next: string;
};
title: string;
description: string;
generator: string;
link: string;
language?: string;
lastBuildDate?: string;
[other: string]: unknown;
}
function getTitle(item: RawFeed | RawFeedItem): string {
const titleValues: (keyof RawFeed | keyof RawFeedItem)[] = [
"title",
"url",
"link",
];
const keys = Object.keys(item);
const titleProperty = titleValues.find(
(titleValue) => keys.includes(titleValue) && item[titleValue],
);
return titleProperty ? (item[titleProperty] as string) : "";
}
function getLink(item: RawFeed | RawFeedItem): string {
const linkValues: (keyof RawFeed | keyof RawFeedItem)[] = [
"link",
"url",
"guid",
"home_page_url",
];
const keys = Object.keys(item);
const linkProperty = linkValues.find((linkValue) => keys.includes(linkValue));
return linkProperty ? (item[linkProperty] as string) : "";
}
function getTimestamp(item: RawFeedItem): number {
const dateString =
item.pubDate || item.isoDate || item.date || item.date_published;
if (!dateString || typeof dateString !== "string") {
return Date.now();
}
const timestamp = new Date(dateString).getTime();
return Number.isNaN(timestamp) ? Date.now() : timestamp;
}
async function parseFeedContents(
feedUrl: string,
category: string,
): Promise<FeedItem[]> {
console.log(`Fetching: ${feedUrl}...`);
const response = await fetch(feedUrl);
const body = await getRawFeedContents(response);
if (!body) {
throw new Error(`Failed to fetch feed: ${feedUrl}`);
}
try {
const rawFeed = (
typeof body === "string" ? await parser.parseString(body) : body
) as RawFeed;
const feedName = getTitle(rawFeed);
const feedLink = getLink(rawFeed);
const items: FeedItem[] = rawFeed.items.flatMap((item) => ({
feedName,
feedLink,
category,
title: item.title,
pubIsoDate: getTimestamp(item),
link: item.link,
}));
return items;
} catch (err) {
console.error(`Error processing: ${feedUrl}\n${err}`);
throw err;
}
}
export default async function getAllFeedItems(): Promise<{
contents: FeedItem[];
errors: Error[];
}> {
const feedCategories = readFeedCategoriesFromEnv();
const results = (
await Promise.allSettled(
Object.entries(feedCategories)
.flatMap(([category, feeds]) =>
feeds.map((feedUrl) => ({ category, feedUrl })),
)
.flatMap(({ category, feedUrl }) => {
return parseFeedContents(feedUrl, category);
}),
)
).reduce(
(acc, result) => {
if (result.status === "fulfilled") {
acc.contents.push(...result.value);
} else {
acc.errors.push(result.reason);
}
return acc;
},
{ contents: [], errors: [] },
);
results.contents.sort((a, b) => b.pubIsoDate - a.pubIsoDate);
return results;
}

113
src/utilities.ts
View file

@ -1,113 +0,0 @@
import { $ } from "bun";
/*
There's a little inconsistency with how feeds report certain things like
title, links and timestamps. These helpers try to normalize that bit and
provide an order-of-operations list of properties to look for.
Note: these are tightly-coupled to the template and a personal preference.
*/
import { readFile } from "node:fs/promises";
import type { Response } from "node-fetch";
import type { FeedItem, JSONValue } from "./@types/bubo";
export const getLink = (obj: FeedItem): string => {
const link_values: string[] = ["link", "url", "guid", "home_page_url"];
const keys: string[] = Object.keys(obj);
const link_property: string | undefined = link_values.find((link_value) =>
keys.includes(link_value),
);
return link_property ? (obj[link_property] as string) : "";
};
// fallback to URL for the title if not present
// (title -> url -> link)
export const getTitle = (obj: FeedItem): string => {
const title_values: string[] = ["title", "url", "link"];
const keys: string[] = Object.keys(obj);
// if title is empty for some reason, fall back on url or link
const title_property: string | undefined = title_values.find(
(title_value) => keys.includes(title_value) && obj[title_value],
);
return title_property ? (obj[title_property] as string) : "";
};
// More dependable way to get timestamps
export const getTimestamp = (obj: FeedItem): string => {
const dateString: string = (
obj.pubDate ||
obj.isoDate ||
obj.date ||
obj.date_published
).toString();
const timestamp: number = new Date(dateString).getTime();
return Number.isNaN(timestamp) ? dateString : timestamp.toString();
};
// parse RSS/XML or JSON feeds
export async function parseFeed(response: Response): Promise<JSONValue> {
const contentType = response.headers.get("content-type")?.split(";")[0];
if (!contentType) return {};
const rssFeed = [contentType]
.map((item) =>
[
"application/atom+xml",
"application/rss+xml",
"application/xml",
"text/xml",
"text/html", // this is kind of a gamble
].includes(item)
? response.text()
: false,
)
.filter((_) => _)[0];
const jsonFeed = [contentType]
.map((item) =>
["application/json", "application/feed+json"].includes(item)
? (response.json() as Promise<JSONValue>)
: false,
)
.filter((_) => _)[0];
return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
}
export const getFeedList = async ({
feedFilePath,
feeds,
}: { feedFilePath?: string; feeds?: string }): Promise<JSONValue> => {
if (feeds) {
return JSON.parse(feeds);
}
if (!feedFilePath) {
throw new Error("No feed list provided");
}
return JSON.parse(
(await readFile(new URL(feedFilePath, import.meta.url))).toString(),
);
};
export const getBuboInfo = async (): Promise<JSONValue> => {
return JSON.parse(
(await readFile(new URL("../package.json", import.meta.url))).toString(),
);
};
export const buildCSS = async (
minify: boolean,
input: string,
destination: string,
): Promise<void> => {
const output =
await $`bun x tailwindcss -i ${input} ${minify ? "--minify" : ""} -o ${destination}`;
if (output.exitCode !== 0) {
const err = new TextDecoder().decode(output.stderr);
throw new Error(`Building tailwind failed: ${err}`);
}
console.log(`Successfully built CSS to ${destination}`);
return;
};

2
tailwind.config.js
View file

@ -1,7 +1,7 @@
const plugin = require("tailwindcss/plugin");
/** @type {import('tailwindcss').Config} */
export default {
content: ["./config/*.html"],
content: ["./src/**/*.astro"],
theme: {
extend: {
fontFamily: {

22
tsconfig.json
View file

@ -1,23 +1,3 @@
{
"compilerOptions": {
"module": "esnext",
"forceConsistentCasingInFileNames": true,
"removeComments": true,
"strict": true,
"importHelpers": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"target": "ES2021",
"noImplicitAny": true,
"moduleResolution": "node",
"sourceMap": false,
"resolveJsonModule": true,
"outDir": "dist",
"baseUrl": ".",
"typeRoots": ["src/@types"],
"paths": {
"*": ["node_modules/*", "src/@types"]
}
},
"include": ["src/**/*"]
"extends": "astro/tsconfigs/base"
}