Initialize repository (#1)
* Convert to bun * Write "deploy to pages" action * Remove unneeded steps and files * Switch from eslint to biome for linting and formatting * Combine lint and format commands * Add feeds * Remove Atom feed * Add time to feed display * Work on the github pages workflow
This commit is contained in:
Carter McBride
GitHub
parent
434ada0298
commit
b4615da264
16 changed files with 369 additions and 3447 deletions
|
|
@ -1,50 +0,0 @@
|
||||||
{
|
|
||||||
"env": {
|
|
||||||
"browser": true,
|
|
||||||
"es2021": true
|
|
||||||
},
|
|
||||||
"extends": [
|
|
||||||
"eslint:recommended",
|
|
||||||
"plugin:@typescript-eslint/recommended"
|
|
||||||
],
|
|
||||||
"parser": "@typescript-eslint/parser",
|
|
||||||
"parserOptions": {
|
|
||||||
"ecmaVersion": 6,
|
|
||||||
"sourceType": "module"
|
|
||||||
},
|
|
||||||
"plugins": [
|
|
||||||
"@typescript-eslint"
|
|
||||||
],
|
|
||||||
"rules": {
|
|
||||||
"indent": [
|
|
||||||
"error",
|
|
||||||
2
|
|
||||||
],
|
|
||||||
"linebreak-style": [
|
|
||||||
"error",
|
|
||||||
"unix"
|
|
||||||
],
|
|
||||||
"quotes": [
|
|
||||||
"error",
|
|
||||||
"double"
|
|
||||||
],
|
|
||||||
"semi": [
|
|
||||||
"error",
|
|
||||||
"always"
|
|
||||||
],
|
|
||||||
"no-trailing-spaces": [
|
|
||||||
2,
|
|
||||||
{
|
|
||||||
"skipBlankLines": false
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"no-multiple-empty-lines": [
|
|
||||||
"error",
|
|
||||||
{
|
|
||||||
"max": 2,
|
|
||||||
"maxEOF": 1
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"@typescript-eslint/no-var-requires": 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
12
.github/FUNDING.yml
vendored
12
.github/FUNDING.yml
vendored
|
|
@ -1,12 +0,0 @@
|
||||||
# These are supported funding model platforms
|
|
||||||
|
|
||||||
github: georgemandis
|
|
||||||
patreon: #
|
|
||||||
open_collective: # Replace with a single Open Collective username
|
|
||||||
ko_fi: # Replace with a single Ko-fi username
|
|
||||||
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
|
||||||
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
|
||||||
liberapay: # Replace with a single Liberapay username
|
|
||||||
issuehunt: # Replace with a single IssueHunt username
|
|
||||||
otechie: # Replace with a single Otechie username
|
|
||||||
custom: #
|
|
||||||
38
.github/ISSUE_TEMPLATE/bug_report.md
vendored
38
.github/ISSUE_TEMPLATE/bug_report.md
vendored
|
|
@ -1,38 +0,0 @@
|
||||||
---
|
|
||||||
name: Bug report
|
|
||||||
about: Create a report to help us improve
|
|
||||||
title: ''
|
|
||||||
labels: ''
|
|
||||||
assignees: ''
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Describe the bug**
|
|
||||||
A clear and concise description of what the bug is.
|
|
||||||
|
|
||||||
**To Reproduce**
|
|
||||||
Steps to reproduce the behavior:
|
|
||||||
1. Go to '...'
|
|
||||||
2. Click on '....'
|
|
||||||
3. Scroll down to '....'
|
|
||||||
4. See error
|
|
||||||
|
|
||||||
**Expected behavior**
|
|
||||||
A clear and concise description of what you expected to happen.
|
|
||||||
|
|
||||||
**Screenshots**
|
|
||||||
If applicable, add screenshots to help explain your problem.
|
|
||||||
|
|
||||||
**Desktop (please complete the following information):**
|
|
||||||
- OS: [e.g. iOS]
|
|
||||||
- Browser [e.g. chrome, safari]
|
|
||||||
- Version [e.g. 22]
|
|
||||||
|
|
||||||
**Smartphone (please complete the following information):**
|
|
||||||
- Device: [e.g. iPhone6]
|
|
||||||
- OS: [e.g. iOS8.1]
|
|
||||||
- Browser [e.g. stock browser, safari]
|
|
||||||
- Version [e.g. 22]
|
|
||||||
|
|
||||||
**Additional context**
|
|
||||||
Add any other context about the problem here.
|
|
||||||
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
|
|
@ -1,20 +0,0 @@
|
||||||
---
|
|
||||||
name: Feature request
|
|
||||||
about: Suggest an idea for this project
|
|
||||||
title: ''
|
|
||||||
labels: ''
|
|
||||||
assignees: ''
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Is your feature request related to a problem? Please describe.**
|
|
||||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
|
||||||
|
|
||||||
**Describe the solution you'd like**
|
|
||||||
A clear and concise description of what you want to happen.
|
|
||||||
|
|
||||||
**Describe alternatives you've considered**
|
|
||||||
A clear and concise description of any alternative solutions or features you've considered.
|
|
||||||
|
|
||||||
**Additional context**
|
|
||||||
Add any other context or screenshots about the feature request here.
|
|
||||||
35
.github/workflows/build.yml
vendored
Normal file
35
.github/workflows/build.yml
vendored
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
|
||||||
|
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
|
||||||
|
|
||||||
|
name: Build feed
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
schedule:
|
||||||
|
- cron: "0 */6 * * *" # At minute 0 past every 6th hour.
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Setup Bun
|
||||||
|
uses: oven-sh/setup-bun@v1
|
||||||
|
with:
|
||||||
|
bun-version: latest
|
||||||
|
- run: bun install --frozen-lockfile
|
||||||
|
- run: bun run build
|
||||||
|
- name: Upload artifact
|
||||||
|
uses: actions/upload-pages-artifact@v3
|
||||||
|
with:
|
||||||
|
path: public/
|
||||||
|
deploy:
|
||||||
|
environment:
|
||||||
|
name: github-pages
|
||||||
|
url: ${{ steps.deployment.outputs.page_url }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: build
|
||||||
|
steps:
|
||||||
|
- name: Deploy to GitHub Pages
|
||||||
|
id: deployment
|
||||||
|
uses: actions/deploy-pages@v4
|
||||||
BIN
bun.lockb
Executable file
BIN
bun.lockb
Executable file
Binary file not shown.
|
|
@ -1,23 +1,87 @@
|
||||||
{
|
{
|
||||||
"Developer News": [
|
"Comics": [
|
||||||
"https://hacks.mozilla.org/feed/",
|
"http://www.catanacomics.com/rss",
|
||||||
"https://web.dev/feed.xml",
|
"http://feeds.feedburner.com/InvisibleBread",
|
||||||
"https://v8.dev/blog.atom",
|
"https://hejibits.com/rss#_=_",
|
||||||
"https://alistapart.com/main/feed/",
|
"http://www.hrwiki.org/wiki/Special:Updates",
|
||||||
"https://css-tricks.com/feed/",
|
"http://rockpapercynic.tumblr.com/rss"
|
||||||
"https://dev.to/feed",
|
],
|
||||||
"https://changelog.com/feed"
|
"Podcasts": [
|
||||||
],
|
"https://lilyandsam.show/feed",
|
||||||
"Blogs": [
|
"https://technosapiens.substack.com/feed"
|
||||||
"https://george.mand.is/feed.xml",
|
],
|
||||||
"https://joy.recurse.com/feed.atom"
|
"Friends and Family": [
|
||||||
],
|
"https://www.goodreads.com/review/list_rss/132710826?shelf=read",
|
||||||
"My GitHub Projects": [
|
"https://jgm23333.wixsite.com/my-site/blog-feed.xml",
|
||||||
"https://github.com/georgemandis.atom",
|
"https://joekhoury.blog/feed/",
|
||||||
"https://github.com/georgemandis/bubo-rss/releases.atom",
|
"https://social.warnick.me/users/sam.rss"
|
||||||
"https://github.com/georgemandis/konami-js/releases.atom",
|
],
|
||||||
"https://github.com/georgemandis/konami-js/commits/main.atom",
|
"Games": [
|
||||||
"https://github.com/javascriptforartists/cheer-me-up-and-sing-me-a-song/commits/master.atom",
|
"https://www.nomanssky.com/news/feed/",
|
||||||
"https://github.com/georgemandis/circuit-playground-midi-multi-tool/commits/master.atom"
|
"https://www.lexaloffle.com/bbs/feed.php?uid=1",
|
||||||
]
|
"https://www.factorio.com/blog/rss",
|
||||||
}
|
"https://kill-the-newsletter.com/feeds/pghhn8aaf264tukg.xml",
|
||||||
|
"http://www.suppermariobroth.com/rss",
|
||||||
|
"http://unknownworlds.com/subnautica/feed/",
|
||||||
|
"https://store.steampowered.com/feeds/news/app/1675200/?cc=US&l=english"
|
||||||
|
],
|
||||||
|
"Finance": [
|
||||||
|
"http://feeds.feedburner.com/MrMoneyMustache",
|
||||||
|
"http://feeds.feedburner.com/DoctorOfCredit",
|
||||||
|
"https://kill-the-newsletter.com/feeds/tgor1vrwcf3f0uyo.xml",
|
||||||
|
"https://kill-the-newsletter.com/feeds/5q4gs79dfh32yr3h.xml"
|
||||||
|
],
|
||||||
|
"News": [
|
||||||
|
"http://www.economist.com/rss/the_world_this_week_rss.xml",
|
||||||
|
"https://newsroom.churchofjesuschrist.org/rss",
|
||||||
|
"http://feeds.feedburner.com/LdsChurchGrowth",
|
||||||
|
"https://www.sltrib.com/arc/outboundfeeds/news/?outputType=xml",
|
||||||
|
"https://www.ksl.com/rss/news/news_utah",
|
||||||
|
"https://www.readtangle.com/posts/rss/"
|
||||||
|
],
|
||||||
|
"Parenting": [
|
||||||
|
"https://technosapiens.substack.com/feed",
|
||||||
|
"https://parentdata.org/feed/"
|
||||||
|
],
|
||||||
|
"Fashion": [
|
||||||
|
"http://www.heddels.com/feed/",
|
||||||
|
"http://putthison.com/rss",
|
||||||
|
"https://articlesofinterest.substack.com/feed",
|
||||||
|
"https://dappered.com/feed/",
|
||||||
|
"https://dieworkwear.com/feed/",
|
||||||
|
"http://fromsqualortoballer.com/rss",
|
||||||
|
"https://fabricateurialist.substack.com/feed"
|
||||||
|
],
|
||||||
|
"Products": [
|
||||||
|
"https://jellyfin.org/index.xml",
|
||||||
|
"https://gitlab.com/CalcProgrammer1/OpenRGB/-/tags?format=atom",
|
||||||
|
"http://feeds.feedburner.com/psblog",
|
||||||
|
"https://matrix.org/blog/feed/",
|
||||||
|
"https://brave.com/feed/",
|
||||||
|
"https://bitwarden.com/blog/feed.xml"
|
||||||
|
],
|
||||||
|
"Humor": [
|
||||||
|
"https://kill-the-newsletter.com/feeds/d90ng280lhh3p2nq.xml",
|
||||||
|
"http://www.altuniversebyu.com/feed/",
|
||||||
|
"https://aiweirdness.com/rss",
|
||||||
|
"http://mcmansionhell.tumblr.com/rss"
|
||||||
|
],
|
||||||
|
"Other": [
|
||||||
|
"https://kill-the-newsletter.com/feeds/400aiwxy5ox6ao0d.xml",
|
||||||
|
"http://brandonsanderson.com/feed/",
|
||||||
|
"https://kill-the-newsletter.com/feeds/j3c4wngg2hpjmdt8.xml",
|
||||||
|
"https://astralcodexten.substack.com/feed/"
|
||||||
|
],
|
||||||
|
"Tech News": [
|
||||||
|
"https://samwarnick.com/feed.rss",
|
||||||
|
"http://feeds.feedburner.com/CssTricks",
|
||||||
|
"http://chriscoyier.net/feed/",
|
||||||
|
"https://jenniferdaniel.substack.com/feed/",
|
||||||
|
"http://www.gamingonlinux.com/article_rss.php",
|
||||||
|
"http://www.theverge.com/rss/full.xml"
|
||||||
|
],
|
||||||
|
"Emulation": [
|
||||||
|
"http://melonds.kuribo64.net/rss.php",
|
||||||
|
"https://dolphin-emu.org/blog/feeds/"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,56 +1,53 @@
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
||||||
<meta http-equiv="X-UA-Compatible" content="ie=edge" />
|
|
||||||
<title>🦉 Bubo Reader</title>
|
|
||||||
<link rel="stylesheet" href="/style.css" />
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>🦉 Bubo Reader</h1>
|
|
||||||
|
|
||||||
{% for group, feeds in data %}
|
<head>
|
||||||
<h2>{{ group }}</h2>
|
<meta charset="UTF-8" />
|
||||||
{% for feed in feeds %}
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<details>
|
<meta http-equiv="X-UA-Compatible" content="ie=edge" />
|
||||||
<summary>
|
<title>🦉 Bubo Reader</title>
|
||||||
<span class="feed-title">{{ feed.title }}</span>
|
<link rel="stylesheet" href="/style.css" />
|
||||||
<span class="feed-url">({{ feed.feed }})</span>
|
</head>
|
||||||
</summary>
|
|
||||||
<ul>
|
<body>
|
||||||
{% for item in feed.items %}
|
<h1>🦉 Bubo Reader</h1>
|
||||||
<li>
|
|
||||||
{{ item.timestamp | formatDate }} -
|
{% for group, feeds in data %}
|
||||||
<a
|
<h2>{{ group }}</h2>
|
||||||
href="{{ item.link }}"
|
{% for feed in feeds %}
|
||||||
target="_blank"
|
<details>
|
||||||
rel="noopener norefferer nofollow"
|
<summary>
|
||||||
>{{ item.title }}</a
|
<span class="feed-title">{{ feed.title }}</span>
|
||||||
>
|
<span class="feed-url">({{ feed.feed }})</span>
|
||||||
</li>
|
</summary>
|
||||||
{% endfor %}
|
|
||||||
</ul>
|
|
||||||
</details>
|
|
||||||
{% endfor %} {% endfor %} {% if errors | length > 0 %}
|
|
||||||
<h2>Errors</h2>
|
|
||||||
<p>There were errors trying to parse these feeds:</p>
|
|
||||||
<ul>
|
<ul>
|
||||||
{% for error in errors %}
|
{% for item in feed.items %}
|
||||||
<li>{{ error }}</li>
|
<li>
|
||||||
|
{{ item.timestamp | formatDate }} @ {{ item.timestamp | formatTime }}—
|
||||||
|
<a href="{{ item.link }}" target="_blank" rel="noopener norefferer nofollow">{{ item.title }}</a>
|
||||||
|
</li>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</ul>
|
</ul>
|
||||||
{% endif %}
|
</details>
|
||||||
|
{% endfor %} {% endfor %} {% if errors | length > 0 %}
|
||||||
|
<h2>Errors</h2>
|
||||||
|
<p>There were errors trying to parse these feeds:</p>
|
||||||
|
<ul>
|
||||||
|
{% for error in errors %}
|
||||||
|
<li>{{ error }}</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<br />
|
||||||
|
<hr />
|
||||||
|
<p>Last updated {{ now }}.</p>
|
||||||
|
<p>
|
||||||
|
Powered by
|
||||||
|
<a href="https://github.com/georgemandis/bubo-rss">Bubo Reader (v{{ info.version }})</a>, a project by <a
|
||||||
|
href="https://george.mand.is">George Mandis</a>. ❤️
|
||||||
|
<a href="{{ info.funding.url }}">Sponsor on GitHub</a>
|
||||||
|
</p>
|
||||||
|
</body>
|
||||||
|
|
||||||
<br />
|
|
||||||
<hr />
|
|
||||||
<p>Last updated {{ now }}.</p>
|
|
||||||
<p>
|
|
||||||
Powered by
|
|
||||||
<a href="https://github.com/georgemandis/bubo-rss"
|
|
||||||
>Bubo Reader (v{{ info.version }})</a
|
|
||||||
>, a project by <a href="https://george.mand.is">George Mandis</a>. ❤️
|
|
||||||
<a href="{{ info.funding.url }}">Sponsor on GitHub</a>
|
|
||||||
</p>
|
|
||||||
</body>
|
|
||||||
</html>
|
</html>
|
||||||
|
|
|
||||||
9
eslint.config.js
Normal file
9
eslint.config.js
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
import pluginJs from "@eslint/js";
|
||||||
|
import globals from "globals";
|
||||||
|
import tseslint from "typescript-eslint";
|
||||||
|
|
||||||
|
export default [
|
||||||
|
{ languageOptions: { globals: { ...globals.browser, ...globals.node } } },
|
||||||
|
pluginJs.configs.recommended,
|
||||||
|
...tseslint.configs.recommended,
|
||||||
|
];
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
[build]
|
|
||||||
command = "npm run build:bubo"
|
|
||||||
publish = "./public/"
|
|
||||||
|
|
||||||
3067
package-lock.json
generated
3067
package-lock.json
generated
File diff suppressed because it is too large
Load diff
22
package.json
22
package.json
|
|
@ -3,14 +3,12 @@
|
||||||
"version": "2.0.2",
|
"version": "2.0.2",
|
||||||
"description": "A simple but effective feed reader (RSS, JSON)",
|
"description": "A simple but effective feed reader (RSS, JSON)",
|
||||||
"homepage": "https://github.com/georgemandis/bubo-rss",
|
"homepage": "https://github.com/georgemandis/bubo-rss",
|
||||||
"main": "src/index.js",
|
"main": "src/index.ts",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "tsc --watch",
|
|
||||||
"clean": "rm -rf dist",
|
"clean": "rm -rf dist",
|
||||||
"build": "tsc",
|
"build": "bun src/index.ts",
|
||||||
"bubo": "node dist/index.js",
|
"check": "biome check --write ./{src,config,public} ./eslint.config.js"
|
||||||
"build:bubo": "tsc && node dist/index.js"
|
|
||||||
},
|
},
|
||||||
"author": {
|
"author": {
|
||||||
"name": "George Mandis",
|
"name": "George Mandis",
|
||||||
|
|
@ -33,13 +31,15 @@
|
||||||
"rss-parser": "^3.13.0"
|
"rss-parser": "^3.13.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^20.2.5",
|
"@biomejs/biome": "^1.8.1",
|
||||||
|
"@types/bun": "latest",
|
||||||
"@types/nunjucks": "^3.2.2",
|
"@types/nunjucks": "^3.2.2",
|
||||||
"@types/xml2js": "^0.4.11",
|
"@types/xml2js": "^0.4.11",
|
||||||
"@typescript-eslint/eslint-plugin": "^5.59.8",
|
|
||||||
"@typescript-eslint/parser": "^5.59.8",
|
|
||||||
"eslint": "^8.42.0",
|
|
||||||
"tslib": "^2.5.3",
|
"tslib": "^2.5.3",
|
||||||
"typescript": "^5.1.3"
|
"typescript": "^5.1.3",
|
||||||
}
|
"typescript-eslint": "^7.13.1"
|
||||||
|
},
|
||||||
|
"trustedDependencies": [
|
||||||
|
"@biomejs/biome"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
16
src/@types/bubo.d.ts
vendored
16
src/@types/bubo.d.ts
vendored
|
|
@ -1,15 +1,15 @@
|
||||||
export interface Feeds {
|
export interface Feeds {
|
||||||
[key: string]: object[]
|
[key: string]: object[];
|
||||||
}
|
}
|
||||||
export interface FeedItem {
|
export interface FeedItem {
|
||||||
[key: string]: string | number | Date | FeedItem[];
|
[key: string]: string | number | Date | FeedItem[];
|
||||||
items: FeedItem[]
|
items: FeedItem[];
|
||||||
}
|
}
|
||||||
|
|
||||||
//NEW WAY
|
//NEW WAY
|
||||||
export type JSONValue =
|
export type JSONValue =
|
||||||
| string
|
| string
|
||||||
| number
|
| number
|
||||||
| boolean
|
| boolean
|
||||||
| { [x: string]: JSONValue }
|
| { [x: string]: JSONValue }
|
||||||
| Array<JSONValue>;
|
| Array<JSONValue>;
|
||||||
|
|
|
||||||
185
src/index.ts
185
src/index.ts
|
|
@ -10,27 +10,27 @@
|
||||||
* Licensed under the MIT License (http://opensource.org/licenses/MIT)
|
* Licensed under the MIT License (http://opensource.org/licenses/MIT)
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { writeFile } from "node:fs/promises";
|
||||||
|
import chalk from "chalk";
|
||||||
import fetch from "node-fetch";
|
import fetch from "node-fetch";
|
||||||
|
import type { Response } from "node-fetch";
|
||||||
import Parser from "rss-parser";
|
import Parser from "rss-parser";
|
||||||
import { Feeds, FeedItem } from "./@types/bubo";
|
import type { FeedItem, Feeds } from "./@types/bubo";
|
||||||
import { Response } from "node-fetch";
|
|
||||||
import { render } from "./renderer.js";
|
import { render } from "./renderer.js";
|
||||||
import {
|
import {
|
||||||
getLink,
|
getBuboInfo,
|
||||||
getTitle,
|
getFeedList,
|
||||||
getTimestamp,
|
getLink,
|
||||||
parseFeed,
|
getTimestamp,
|
||||||
getFeedList,
|
getTitle,
|
||||||
getBuboInfo
|
parseFeed,
|
||||||
} from "./utilities.js";
|
} from "./utilities.js";
|
||||||
import { writeFile } from "fs/promises";
|
|
||||||
import chalk from "chalk";
|
|
||||||
|
|
||||||
const buboInfo = await getBuboInfo();
|
const buboInfo = await getBuboInfo();
|
||||||
const parser = new Parser();
|
const parser = new Parser();
|
||||||
const feedList = await getFeedList();
|
const feedList = await getFeedList();
|
||||||
const feedListLength =
|
const feedListLength =
|
||||||
Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
|
Object.entries(feedList).flat(2).length - Object.keys(feedList).length;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end
|
* contentFromAllFeeds = Contains normalized, aggregated feed data and is passed to template renderer at the end
|
||||||
|
|
@ -42,14 +42,14 @@ const errors: unknown[] = [];
|
||||||
// benchmarking data + utility
|
// benchmarking data + utility
|
||||||
const initTime = Date.now();
|
const initTime = Date.now();
|
||||||
const benchmark = (startTime: number) =>
|
const benchmark = (startTime: number) =>
|
||||||
chalk.cyanBright.bold(`${(Date.now() - startTime) / 1000} seconds`);
|
chalk.cyanBright.bold(`${(Date.now() - startTime) / 1000} seconds`);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* These values are used to control throttling/batching the fetches:
|
* These values are used to control throttling/batching the fetches:
|
||||||
* - MAX_CONNECTION = max number of fetches to contain in a batch
|
* - MAX_CONNECTION = max number of fetches to contain in a batch
|
||||||
* - DELAY_MS = the delay in milliseconds between batches
|
* - DELAY_MS = the delay in milliseconds between batches
|
||||||
*/
|
*/
|
||||||
const MAX_CONNECTIONS = Infinity;
|
const MAX_CONNECTIONS = Number.POSITIVE_INFINITY;
|
||||||
const DELAY_MS = 850;
|
const DELAY_MS = 850;
|
||||||
|
|
||||||
const error = chalk.bold.red;
|
const error = chalk.bold.red;
|
||||||
|
|
@ -66,26 +66,26 @@ let completed = 0;
|
||||||
* and we want to build the static output.
|
* and we want to build the static output.
|
||||||
*/
|
*/
|
||||||
const finishBuild: () => void = async () => {
|
const finishBuild: () => void = async () => {
|
||||||
completed++;
|
completed++;
|
||||||
// if this isn't the last feed, just return early
|
// if this isn't the last feed, just return early
|
||||||
if (completed !== feedListLength) return;
|
if (completed !== feedListLength) return;
|
||||||
|
|
||||||
process.stdout.write("\nDone fetching everything!\n");
|
process.stdout.write("\nDone fetching everything!\n");
|
||||||
|
|
||||||
// generate the static HTML output from our template renderer
|
// generate the static HTML output from our template renderer
|
||||||
const output = render({
|
const output = render({
|
||||||
data: contentFromAllFeeds,
|
data: contentFromAllFeeds,
|
||||||
errors: errors,
|
errors: errors,
|
||||||
info: buboInfo
|
info: buboInfo,
|
||||||
});
|
});
|
||||||
|
|
||||||
// write the output to public/index.html
|
// write the output to public/index.html
|
||||||
await writeFile("./public/index.html", output);
|
await writeFile("./public/index.html", output);
|
||||||
process.stdout.write(
|
process.stdout.write(
|
||||||
`\nFinished writing to output:\n- ${feedListLength} feeds in ${benchmark(
|
`\nFinished writing to output:\n- ${feedListLength} feeds in ${benchmark(
|
||||||
initTime
|
initTime,
|
||||||
)}\n- ${errors.length} errors\n`
|
)}\n- ${errors.length} errors\n`,
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -96,77 +96,80 @@ const finishBuild: () => void = async () => {
|
||||||
* @returns Promise<void>
|
* @returns Promise<void>
|
||||||
*/
|
*/
|
||||||
const processFeed =
|
const processFeed =
|
||||||
({
|
({
|
||||||
group,
|
group,
|
||||||
feed,
|
feed,
|
||||||
startTime
|
startTime,
|
||||||
}: {
|
}: {
|
||||||
group: string;
|
group: string;
|
||||||
feed: string;
|
feed: string;
|
||||||
startTime: number;
|
startTime: number;
|
||||||
}) =>
|
}) =>
|
||||||
async (response: Response): Promise<void> => {
|
async (response: Response): Promise<void> => {
|
||||||
const body = await parseFeed(response);
|
const body = await parseFeed(response);
|
||||||
//skip to the next one if this didn't work out
|
//skip to the next one if this didn't work out
|
||||||
if (!body) return;
|
if (!body) return;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const contents: FeedItem = (
|
const contents: FeedItem = (
|
||||||
typeof body === "string" ? await parser.parseString(body) : body
|
typeof body === "string" ? await parser.parseString(body) : body
|
||||||
) as FeedItem;
|
) as FeedItem;
|
||||||
|
|
||||||
contents.feed = feed;
|
contents.feed = feed;
|
||||||
contents.title = getTitle(contents);
|
contents.title = getTitle(contents);
|
||||||
contents.link = getLink(contents);
|
contents.link = getLink(contents);
|
||||||
|
|
||||||
// try to normalize date attribute naming
|
// try to normalize date attribute naming
|
||||||
contents?.items?.forEach(item => {
|
for (const item of contents.items) {
|
||||||
item.timestamp = getTimestamp(item);
|
item.timestamp = getTimestamp(item);
|
||||||
item.title = getTitle(item);
|
item.title = getTitle(item);
|
||||||
item.link = getLink(item);
|
item.link = getLink(item);
|
||||||
});
|
}
|
||||||
|
|
||||||
contentFromAllFeeds[group].push(contents as object);
|
contentFromAllFeeds[group].push(contents as object);
|
||||||
process.stdout.write(
|
process.stdout.write(
|
||||||
`${success("Successfully fetched:")} ${feed} - ${benchmark(startTime)}\n`
|
`${success("Successfully fetched:")} ${feed} - ${benchmark(startTime)}\n`,
|
||||||
);
|
);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
process.stdout.write(
|
process.stdout.write(
|
||||||
`${error("Error processing:")} ${feed} - ${benchmark(
|
`${error("Error processing:")} ${feed} - ${benchmark(
|
||||||
startTime
|
startTime,
|
||||||
)}\n${err}\n`
|
)}\n${err}\n`,
|
||||||
);
|
);
|
||||||
errors.push(`Error processing: ${feed}\n\t${err}`);
|
errors.push(`Error processing: ${feed}\n\t${err}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
finishBuild();
|
finishBuild();
|
||||||
};
|
};
|
||||||
|
|
||||||
// go through each group of feeds and process
|
// go through each group of feeds and process
|
||||||
const processFeeds = () => {
|
const processFeeds = () => {
|
||||||
let idx = 0;
|
let idx = 0;
|
||||||
|
|
||||||
for (const [group, feeds] of Object.entries(feedList)) {
|
for (const [group, feeds] of Object.entries(feedList)) {
|
||||||
contentFromAllFeeds[group] = [];
|
contentFromAllFeeds[group] = [];
|
||||||
|
|
||||||
for (const feed of feeds) {
|
for (const feed of feeds) {
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
setTimeout(() => {
|
setTimeout(
|
||||||
process.stdout.write(`Fetching: ${feed}...\n`);
|
() => {
|
||||||
|
process.stdout.write(`Fetching: ${feed}...\n`);
|
||||||
|
|
||||||
fetch(feed)
|
fetch(feed)
|
||||||
.then(processFeed({ group, feed, startTime }))
|
.then(processFeed({ group, feed, startTime }))
|
||||||
.catch(err => {
|
.catch((err) => {
|
||||||
process.stdout.write(
|
process.stdout.write(
|
||||||
error(`Error fetching ${feed} ${benchmark(startTime)}\n`)
|
error(`Error fetching ${feed} ${benchmark(startTime)}\n`),
|
||||||
);
|
);
|
||||||
errors.push(`Error fetching ${feed} ${err.toString()}\n`);
|
errors.push(`Error fetching ${feed} ${err.toString()}\n`);
|
||||||
finishBuild();
|
finishBuild();
|
||||||
});
|
});
|
||||||
}, (idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS);
|
},
|
||||||
idx++;
|
(idx % (feedListLength / MAX_CONNECTIONS)) * DELAY_MS,
|
||||||
}
|
);
|
||||||
}
|
idx++;
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
processFeeds();
|
processFeeds();
|
||||||
|
|
|
||||||
|
|
@ -6,39 +6,44 @@
|
||||||
|
|
||||||
import nunjucks from "nunjucks";
|
import nunjucks from "nunjucks";
|
||||||
const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
|
const env: nunjucks.Environment = nunjucks.configure({ autoescape: true });
|
||||||
import { readFile } from "fs/promises";
|
import { readFile } from "node:fs/promises";
|
||||||
import { Feeds, JSONValue } from "./@types/bubo";
|
import type { Feeds, JSONValue } from "./@types/bubo";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Global filters for my Nunjucks templates
|
* Global filters for my Nunjucks templates
|
||||||
*/
|
*/
|
||||||
env.addFilter("formatDate", function (dateString): string {
|
env.addFilter("formatDate", (dateString): string => {
|
||||||
const date: Date = new Date(parseInt(dateString));
|
const date: Date = new Date(Number.parseInt(dateString));
|
||||||
return !isNaN(date.getTime()) ? date.toLocaleDateString() : dateString;
|
return !Number.isNaN(date.getTime()) ? date.toLocaleDateString() : dateString;
|
||||||
|
});
|
||||||
|
|
||||||
|
env.addFilter("formatTime", (dateString): string => {
|
||||||
|
const date: Date = new Date(Number.parseInt(dateString));
|
||||||
|
return !Number.isNaN(date.getTime()) ? date.toLocaleTimeString() : dateString;
|
||||||
});
|
});
|
||||||
|
|
||||||
env.addGlobal("now", new Date().toUTCString());
|
env.addGlobal("now", new Date().toUTCString());
|
||||||
|
|
||||||
// load the template
|
// load the template
|
||||||
const template: string = (
|
const template: string = (
|
||||||
await readFile(new URL("../config/template.html", import.meta.url))
|
await readFile(new URL("../config/template.html", import.meta.url))
|
||||||
).toString();
|
).toString();
|
||||||
|
|
||||||
// generate the static HTML output from our template renderer
|
// generate the static HTML output from our template renderer
|
||||||
const render = ({
|
const render = ({
|
||||||
data,
|
data,
|
||||||
errors,
|
errors,
|
||||||
info
|
info,
|
||||||
}: {
|
}: {
|
||||||
data: Feeds;
|
data: Feeds;
|
||||||
errors: unknown[];
|
errors: unknown[];
|
||||||
info?: JSONValue;
|
info?: JSONValue;
|
||||||
}) => {
|
}) => {
|
||||||
return env.renderString(template, {
|
return env.renderString(template, {
|
||||||
data,
|
data,
|
||||||
errors,
|
errors,
|
||||||
info
|
info,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
export { render };
|
export { render };
|
||||||
|
|
|
||||||
118
src/utilities.ts
118
src/utilities.ts
|
|
@ -1,90 +1,90 @@
|
||||||
/*
|
/*
|
||||||
There's a little inconsistency with how feeds report certain things like
|
There's a little inconsistency with how feeds report certain things like
|
||||||
title, links and timestamps. These helpers try to normalize that bit and
|
title, links and timestamps. These helpers try to normalize that bit and
|
||||||
provide an order-of-operations list of properties to look for.
|
provide an order-of-operations list of properties to look for.
|
||||||
|
|
||||||
Note: these are tightly-coupled to the template and a personal preference.
|
Note: these are tightly-coupled to the template and a personal preference.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Response } from "node-fetch";
|
import { readFile } from "node:fs/promises";
|
||||||
import { readFile } from "fs/promises";
|
import type { Response } from "node-fetch";
|
||||||
import { FeedItem, JSONValue } from "./@types/bubo";
|
import type { FeedItem, JSONValue } from "./@types/bubo";
|
||||||
|
|
||||||
export const getLink = (obj: FeedItem): string => {
|
export const getLink = (obj: FeedItem): string => {
|
||||||
const link_values: string[] = ["link", "url", "guid", "home_page_url"];
|
const link_values: string[] = ["link", "url", "guid", "home_page_url"];
|
||||||
const keys: string[] = Object.keys(obj);
|
const keys: string[] = Object.keys(obj);
|
||||||
const link_property: string | undefined = link_values.find(link_value =>
|
const link_property: string | undefined = link_values.find((link_value) =>
|
||||||
keys.includes(link_value)
|
keys.includes(link_value),
|
||||||
);
|
);
|
||||||
return link_property ? (obj[link_property] as string) : "";
|
return link_property ? (obj[link_property] as string) : "";
|
||||||
};
|
};
|
||||||
|
|
||||||
// fallback to URL for the title if not present
|
// fallback to URL for the title if not present
|
||||||
// (title -> url -> link)
|
// (title -> url -> link)
|
||||||
export const getTitle = (obj: FeedItem): string => {
|
export const getTitle = (obj: FeedItem): string => {
|
||||||
const title_values: string[] = ["title", "url", "link"];
|
const title_values: string[] = ["title", "url", "link"];
|
||||||
const keys: string[] = Object.keys(obj);
|
const keys: string[] = Object.keys(obj);
|
||||||
|
|
||||||
// if title is empty for some reason, fall back on url or link
|
// if title is empty for some reason, fall back on url or link
|
||||||
const title_property: string | undefined = title_values.find(
|
const title_property: string | undefined = title_values.find(
|
||||||
title_value => keys.includes(title_value) && obj[title_value]
|
(title_value) => keys.includes(title_value) && obj[title_value],
|
||||||
);
|
);
|
||||||
return title_property ? (obj[title_property] as string) : "";
|
return title_property ? (obj[title_property] as string) : "";
|
||||||
};
|
};
|
||||||
|
|
||||||
// More dependable way to get timestamps
|
// More dependable way to get timestamps
|
||||||
export const getTimestamp = (obj: FeedItem): string => {
|
export const getTimestamp = (obj: FeedItem): string => {
|
||||||
const dateString: string = (
|
const dateString: string = (
|
||||||
obj.pubDate ||
|
obj.pubDate ||
|
||||||
obj.isoDate ||
|
obj.isoDate ||
|
||||||
obj.date ||
|
obj.date ||
|
||||||
obj.date_published
|
obj.date_published
|
||||||
).toString();
|
).toString();
|
||||||
const timestamp: number = new Date(dateString).getTime();
|
const timestamp: number = new Date(dateString).getTime();
|
||||||
return isNaN(timestamp) ? dateString : timestamp.toString();
|
return Number.isNaN(timestamp) ? dateString : timestamp.toString();
|
||||||
};
|
};
|
||||||
|
|
||||||
// parse RSS/XML or JSON feeds
|
// parse RSS/XML or JSON feeds
|
||||||
export async function parseFeed(response: Response): Promise<JSONValue> {
|
export async function parseFeed(response: Response): Promise<JSONValue> {
|
||||||
const contentType = response.headers.get("content-type")?.split(";")[0];
|
const contentType = response.headers.get("content-type")?.split(";")[0];
|
||||||
|
|
||||||
if (!contentType) return {};
|
if (!contentType) return {};
|
||||||
|
|
||||||
const rssFeed = [contentType]
|
const rssFeed = [contentType]
|
||||||
.map(item =>
|
.map((item) =>
|
||||||
[
|
[
|
||||||
"application/atom+xml",
|
"application/atom+xml",
|
||||||
"application/rss+xml",
|
"application/rss+xml",
|
||||||
"application/xml",
|
"application/xml",
|
||||||
"text/xml",
|
"text/xml",
|
||||||
"text/html" // this is kind of a gamble
|
"text/html", // this is kind of a gamble
|
||||||
].includes(item)
|
].includes(item)
|
||||||
? response.text()
|
? response.text()
|
||||||
: false
|
: false,
|
||||||
)
|
)
|
||||||
.filter(_ => _)[0];
|
.filter((_) => _)[0];
|
||||||
|
|
||||||
const jsonFeed = [contentType]
|
const jsonFeed = [contentType]
|
||||||
.map(item =>
|
.map((item) =>
|
||||||
["application/json", "application/feed+json"].includes(item)
|
["application/json", "application/feed+json"].includes(item)
|
||||||
? (response.json() as Promise<JSONValue>)
|
? (response.json() as Promise<JSONValue>)
|
||||||
: false
|
: false,
|
||||||
)
|
)
|
||||||
.filter(_ => _)[0];
|
.filter((_) => _)[0];
|
||||||
|
|
||||||
return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
|
return (rssFeed && rssFeed) || (jsonFeed && jsonFeed) || {};
|
||||||
}
|
}
|
||||||
|
|
||||||
export const getFeedList = async (): Promise<JSONValue> => {
|
export const getFeedList = async (): Promise<JSONValue> => {
|
||||||
return JSON.parse(
|
return JSON.parse(
|
||||||
(
|
(
|
||||||
await readFile(new URL("../config/feeds.json", import.meta.url))
|
await readFile(new URL("../config/feeds.json", import.meta.url))
|
||||||
).toString()
|
).toString(),
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getBuboInfo = async (): Promise<JSONValue> => {
|
export const getBuboInfo = async (): Promise<JSONValue> => {
|
||||||
return JSON.parse(
|
return JSON.parse(
|
||||||
(await readFile(new URL("../package.json", import.meta.url))).toString()
|
(await readFile(new URL("../package.json", import.meta.url))).toString(),
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue