=> {
- const body = await parseFeed(response);
- //skip to the next one if this didn't work out
- if (!body) return;
-
- try {
- const contents: FeedItem = (
- typeof body === "string" ? await parser.parseString(body) : body
- ) as FeedItem;
-
- contents.feed = feed;
- contents.title = getTitle(contents);
- contents.link = getLink(contents);
-
- // try to normalize date attribute naming
- for (const item of contents.items) {
- item.timestamp = getTimestamp(item);
- item.title = getTitle(item);
- item.link = getLink(item);
- const timestamp = new Date(Number.parseInt(item.timestamp));
- const eightHoursAgo = new Date();
- eightHoursAgo.setHours(eightHoursAgo.getHours() - 8);
- item.isRecent = timestamp > eightHoursAgo;
- }
-
- contents.hasRecent = contents.items.some((item) => item.isRecent);
-
- contentFromAllFeeds[group].push(contents as object);
- process.stdout.write(
- `${success("Successfully fetched:")} ${feed} - ${benchmark(startTime)}\n`,
- );
- } catch (err) {
- process.stdout.write(
- `${error("Error processing:")} ${feed} - ${benchmark(
- startTime,
- )}\n${err}\n`,
- );
- errors.push(`Error processing: ${feed}\n\t${err}`);
- }
-
- finishBuild();
- };
-
-// go through each group of feeds and process
-const processFeeds = () => {
- let idx = 0;
-
- for (const [group, feeds] of Object.entries(feedList)) {
- contentFromAllFeeds[group] = [];
-
- for (const feed of feeds) {
- const startTime = Date.now();
- setTimeout(
- () => {
- process.stdout.write(`Fetching: ${feed}...\n`);
-
- fetch(feed)
- .then(processFeed({ group, feed, startTime }))
- .catch((err) => {
- process.stdout.write(
- error(`Error fetching ${feed} ${benchmark(startTime)}\n`),
- );
- errors.push(`Error fetching ${feed} ${err.toString()}\n`);
- finishBuild();
- });
- },
- (idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS,
- );
- idx++;
- }
- }
-};
-
-processFeeds();
diff --git a/src/pages/index.astro b/src/pages/index.astro
new file mode 100644
index 0000000..d860263
--- /dev/null
+++ b/src/pages/index.astro
@@ -0,0 +1,65 @@
+---
+import getAllFeedItems from "../services/feeds";
+
+const feedItems = await getAllFeedItems();
+const categories = Array.from(
+ new Set(feedItems.contents.map((item) => item.category)),
+).sort();
+const categoriesSelectorCss = categories.map((c => `
+#category-picker:has(#${c}:checked) ~ main ul {
+ > .${c}-item {
+ display: block;
+ }
+ > *:not(.${c}-item) {
+ display: none;
+ }
+}`)).join("\n")
+---
+
+
+
+
+
+
+
+ Carter's RSS Feeds
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/renderer.ts b/src/renderer.ts
deleted file mode 100644
index a038c5c..0000000
--- a/src/renderer.ts
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Return our renderer.
- * Using Nunjucks out of the box.
- * https://mozilla.github.io/nunjucks/
- */
-
-import nunjucks from "nunjucks";
-const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
-import { readFile } from "node:fs/promises";
-import { getRelativeTime } from "@feelinglovelynow/get-relative-time";
-import type { Feeds, JSONValue } from "./@types/bubo";
-
-/**
- * Global filters for my Nunjucks templates
- */
-env.addFilter("relative", (dateString): string => {
- const date: Date = new Date(Number.parseInt(dateString));
- return !Number.isNaN(date.getTime()) ? getRelativeTime(date) : dateString;
-});
-
-env.addFilter("formatTime", (dateString): string => {
- const date: Date = new Date(Number.parseInt(dateString));
- return !Number.isNaN(date.getTime()) ? date.toLocaleTimeString() : dateString;
-});
-
-env.addFilter("formatDateTime", (dateString): string => {
- const date: Date = new Date(Number.parseInt(dateString));
- return !Number.isNaN(date.getTime()) ? date.toLocaleString() : dateString;
-});
-
-env.addGlobal("now", new Date().getTime());
-
-// load the template
-const template: string = (
- await readFile(new URL("../config/template.html", import.meta.url))
-).toString();
-
-// generate the static HTML output from our template renderer
-const render = ({
- data,
- errors,
- info,
- yazzyUrl,
-}: {
- data: Feeds;
- errors: unknown[];
- info?: JSONValue;
- yazzyUrl?: string;
-}) => {
- return env.renderString(template, {
- data,
- errors,
- info,
- yazzyUrl,
- });
-};
-
-export { render };
diff --git a/src/services/feeds.ts b/src/services/feeds.ts
new file mode 100644
index 0000000..4dc3a59
--- /dev/null
+++ b/src/services/feeds.ts
@@ -0,0 +1,178 @@
+import Parser from "rss-parser";
+
+interface FeedItem {
+ title: string;
+ feedName: string;
+ feedLink: string;
+ pubIsoDate: number;
+ link: string;
+ category: string;
+}
+
+const MAX_CONNECTIONS = Number.POSITIVE_INFINITY;
+const DELAY_MS = 850;
+const parser = new Parser();
+
+function readFeedCategoriesFromEnv(): Record {
+ if (import.meta.env.FEEDS) {
+ return JSON.parse(import.meta.env.FEEDS);
+ }
+ throw new Error("FEEDS environment variable is not set");
+}
+
+async function getRawFeedContents(response: Response): Promise {
+ const contentType = response.headers.get("content-type")?.split(";")[0];
+ if (!contentType) return {};
+ if (
+ [
+ "application/atom+xml",
+ "application/rss+xml",
+ "application/xml",
+ "text/xml",
+ "text/html",
+ ].includes(contentType)
+ ) {
+ return response.text();
+ }
+ if (["application/json", "application/feed+json"].includes(contentType)) {
+ return response.json();
+ }
+ return {};
+}
+
+interface RawFeedItem {
+ creator?: string;
+ title: string;
+ link: string;
+ pubDate: string;
+ "content:encoded"?: string;
+ "content:encodedSnippet"?: string;
+ "dc:creator"?: string;
+ comments?: string;
+ content: string;
+ contentSnippet: string;
+ guid: string;
+ categories: unknown[];
+ isoDate: string;
+ [other: string]: unknown;
+}
+
+interface RawFeed {
+ items: RawFeedItem[];
+ feedUrl?: string;
+ image?: {
+ link: string;
+ url: string;
+ title: string;
+ width: string;
+ height: string;
+ };
+ pagenationLinks?: {
+ self: string;
+ next: string;
+ };
+ title: string;
+ description: string;
+ generator: string;
+ link: string;
+ language?: string;
+ lastBuildDate?: string;
+ [other: string]: unknown;
+}
+
+function getTitle(item: RawFeed | RawFeedItem): string {
+ const titleValues: (keyof RawFeed | keyof RawFeedItem)[] = [
+ "title",
+ "url",
+ "link",
+ ];
+ const keys = Object.keys(item);
+ const titleProperty = titleValues.find(
+ (titleValue) => keys.includes(titleValue) && item[titleValue],
+ );
+ return titleProperty ? (item[titleProperty] as string) : "";
+}
+
+function getLink(item: RawFeed | RawFeedItem): string {
+ const linkValues: (keyof RawFeed | keyof RawFeedItem)[] = [
+ "link",
+ "url",
+ "guid",
+ "home_page_url",
+ ];
+ const keys = Object.keys(item);
+ const linkProperty = linkValues.find((linkValue) => keys.includes(linkValue));
+ return linkProperty ? (item[linkProperty] as string) : "";
+}
+
+function getTimestamp(item: RawFeedItem): number {
+ const dateString =
+ item.pubDate || item.isoDate || item.date || item.date_published;
+ if (!dateString || typeof dateString !== "string") {
+ return Date.now();
+ }
+ const timestamp = new Date(dateString).getTime();
+ return Number.isNaN(timestamp) ? Date.now() : timestamp;
+}
+
+async function parseFeedContents(
+ feedUrl: string,
+ category: string,
+): Promise {
+ console.log(`Fetching: ${feedUrl}...`);
+ const response = await fetch(feedUrl);
+ const body = await getRawFeedContents(response);
+ if (!body) {
+ throw new Error(`Failed to fetch feed: ${feedUrl}`);
+ }
+ try {
+ const rawFeed = (
+ typeof body === "string" ? await parser.parseString(body) : body
+ ) as RawFeed;
+ const feedName = getTitle(rawFeed);
+ const feedLink = getLink(rawFeed);
+ const items: FeedItem[] = rawFeed.items.flatMap((item) => ({
+ feedName,
+ feedLink,
+ category,
+ title: item.title,
+ pubIsoDate: getTimestamp(item),
+ link: item.link,
+ }));
+ return items;
+ } catch (err) {
+ console.error(`Error processing: ${feedUrl}\n${err}`);
+ throw err;
+ }
+}
+
+export default async function getAllFeedItems(): Promise<{
+ contents: FeedItem[];
+ errors: Error[];
+}> {
+ const feedCategories = readFeedCategoriesFromEnv();
+
+ const results = (
+ await Promise.allSettled(
+ Object.entries(feedCategories)
+ .flatMap(([category, feeds]) =>
+ feeds.map((feedUrl) => ({ category, feedUrl })),
+ )
+ .flatMap(({ category, feedUrl }) => {
+ return parseFeedContents(feedUrl, category);
+ }),
+ )
+ ).reduce(
+ (acc, result) => {
+ if (result.status === "fulfilled") {
+ acc.contents.push(...result.value);
+ } else {
+ acc.errors.push(result.reason);
+ }
+ return acc;
+ },
+ { contents: [], errors: [] },
+ );
+ results.contents.sort((a, b) => b.pubIsoDate - a.pubIsoDate);
+ return results;
+}
diff --git a/src/utilities.ts b/src/utilities.ts
deleted file mode 100644
index 3d9be1a..0000000
--- a/src/utilities.ts
+++ /dev/null
@@ -1,113 +0,0 @@
-import { $ } from "bun";
-/*
- There's a little inconsistency with how feeds report certain things like
- title, links and timestamps. These helpers try to normalize that bit and
- provide an order-of-operations list of properties to look for.
-
- Note: these are tightly-coupled to the template and a personal preference.
-*/
-
-import { readFile } from "node:fs/promises";
-import type { Response } from "node-fetch";
-import type { FeedItem, JSONValue } from "./@types/bubo";
-
-export const getLink = (obj: FeedItem): string => {
- const link_values: string[] = ["link", "url", "guid", "home_page_url"];
- const keys: string[] = Object.keys(obj);
- const link_property: string | undefined = link_values.find((link_value) =>
- keys.includes(link_value),
- );
- return link_property ? (obj[link_property] as string) : "";
-};
-
-// fallback to URL for the title if not present
-// (title -> url -> link)
-export const getTitle = (obj: FeedItem): string => {
- const title_values: string[] = ["title", "url", "link"];
- const keys: string[] = Object.keys(obj);
-
- // if title is empty for some reason, fall back on url or link
- const title_property: string | undefined = title_values.find(
- (title_value) => keys.includes(title_value) && obj[title_value],
- );
- return title_property ? (obj[title_property] as string) : "";
-};
-
-// More dependable way to get timestamps
-export const getTimestamp = (obj: FeedItem): string => {
- const dateString: string = (
- obj.pubDate ||
- obj.isoDate ||
- obj.date ||
- obj.date_published
- ).toString();
- const timestamp: number = new Date(dateString).getTime();
- return Number.isNaN(timestamp) ? dateString : timestamp.toString();
-};
-
-// parse RSS/XML or JSON feeds
-export async function parseFeed(response: Response): Promise {
- const contentType = response.headers.get("content-type")?.split(";")[0];
-
- if (!contentType) return {};
-
- const rssFeed = [contentType]
- .map((item) =>
- [
- "application/atom+xml",
- "application/rss+xml",
- "application/xml",
- "text/xml",
- "text/html", // this is kind of a gamble
- ].includes(item)
- ? response.text()
- : false,
- )
- .filter((_) => _)[0];
-
- const jsonFeed = [contentType]
- .map((item) =>
- ["application/json", "application/feed+json"].includes(item)
- ? (response.json() as Promise)
- : false,
- )
- .filter((_) => _)[0];
-
- return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
-}
-
-export const getFeedList = async ({
- feedFilePath,
- feeds,
-}: { feedFilePath?: string; feeds?: string }): Promise => {
- if (feeds) {
- return JSON.parse(feeds);
- }
- if (!feedFilePath) {
- throw new Error("No feed list provided");
- }
- return JSON.parse(
- (await readFile(new URL(feedFilePath, import.meta.url))).toString(),
- );
-};
-
-export const getBuboInfo = async (): Promise => {
- return JSON.parse(
- (await readFile(new URL("../package.json", import.meta.url))).toString(),
- );
-};
-
-export const buildCSS = async (
- minify: boolean,
- input: string,
- destination: string,
-): Promise => {
- const output =
- await $`bun x tailwindcss -i ${input} ${minify ? "--minify" : ""} -o ${destination}`;
- if (output.exitCode !== 0) {
- const err = new TextDecoder().decode(output.stderr);
- throw new Error(`Building tailwind failed: ${err}`);
- }
- console.log(`Successfully built CSS to ${destination}`);
- return;
-};
diff --git a/tailwind.config.js b/tailwind.config.js
index ebe918c..b8cd931 100644
--- a/tailwind.config.js
+++ b/tailwind.config.js
@@ -1,7 +1,7 @@
const plugin = require("tailwindcss/plugin");
/** @type {import('tailwindcss').Config} */
export default {
- content: ["./config/*.html"],
+ content: ["./src/**/*.astro"],
theme: {
extend: {
fontFamily: {
diff --git a/tsconfig.json b/tsconfig.json
index 565a04d..5378929 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,23 +1,3 @@
{
- "compilerOptions": {
- "module": "esnext",
- "forceConsistentCasingInFileNames": true,
- "removeComments": true,
- "strict": true,
- "importHelpers": true,
- "esModuleInterop": true,
- "allowSyntheticDefaultImports": true,
- "target": "ES2021",
- "noImplicitAny": true,
- "moduleResolution": "node",
- "sourceMap": false,
- "resolveJsonModule": true,
- "outDir": "dist",
- "baseUrl": ".",
- "typeRoots": ["src/@types"],
- "paths": {
- "*": ["node_modules/*", "src/@types"]
- }
- },
- "include": ["src/**/*"]
+ "extends": "astro/tsconfigs/base"
}