Build hn-reader: keyboard-driven Hacker News reader

Vite/React + shadcn/ui frontend with Superhuman-style keyboard nav (j/k/o/v/e/s/u, feed switching, localStorage caching for instant loads) backed by an Express API that polls the HN Firebase API into Postgres and serves cached feeds/comments. Instrumented with OpenTelemetry (traces/metrics/logs via OTLP/HTTP).
2026-07-25 12:16:08 +00:00
commit 95bef8bf7b
47 changed files with 12198 additions and 0 deletions
@@ -0,0 +1,11 @@
+const { Pool } = require('pg')
+
+const pool = new Pool({
+  host: process.env.POSTGRES_HOST || '127.0.0.1',
+  port: Number(process.env.POSTGRES_PORT || 5433),
+  database: process.env.POSTGRES_DB || 'hnreader',
+  user: process.env.POSTGRES_USER || 'hnreader',
+  password: process.env.POSTGRES_PASSWORD,
+})
+
+module.exports = { pool }
@@ -0,0 +1,160 @@
+const { pool } = require('./db')
+const { log } = require('./otel')
+
+const HN_BASE = 'https://hacker-news.firebaseio.com/v0'
+
+const FEEDS = {
+  top: 'topstories',
+  new: 'newstories',
+  best: 'beststories',
+  ask: 'askstories',
+  show: 'showstories',
+  job: 'jobstories',
+}
+
+const ITEMS_PER_FEED = 100
+const FETCH_CONCURRENCY = 20
+
+async function fetchJson(url) {
+  const res = await fetch(url)
+  if (!res.ok) throw new Error(`HN API ${res.status} for ${url}`)
+  return res.json()
+}
+
+function domainOf(url) {
+  if (!url) return null
+  try {
+    return new URL(url).hostname.replace(/^www\./, '')
+  } catch {
+    return null
+  }
+}
+
+async function mapWithConcurrency(items, limit, fn) {
+  const results = new Array(items.length)
+  let next = 0
+  async function worker() {
+    while (next < items.length) {
+      const i = next++
+      results[i] = await fn(items[i], i)
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker))
+  return results
+}
+
+async function fetchItem(id) {
+  return fetchJson(`${HN_BASE}/item/${id}.json`)
+}
+
+async function syncFeed(feedKey) {
+  const endpoint = FEEDS[feedKey]
+  const ids = (await fetchJson(`${HN_BASE}/${endpoint}.json`)).slice(0, ITEMS_PER_FEED)
+
+  const items = await mapWithConcurrency(ids, FETCH_CONCURRENCY, async (id) => {
+    try {
+      return await fetchItem(id)
+    } catch (err) {
+      log('ERROR', 'failed to fetch HN item', { id, error: String(err) })
+      return null
+    }
+  })
+
+  const client = await pool.connect()
+  try {
+    await client.query('BEGIN')
+    let rank = 0
+    for (const item of items) {
+      if (!item || item.deleted || item.dead) continue
+      await client.query(
+        `INSERT INTO stories (id, type, title, url, domain, by_user, score, descendants, story_text, hn_created_at, fetched_at)
+         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9, to_timestamp($10), now())
+         ON CONFLICT (id) DO UPDATE SET
+           title = EXCLUDED.title,
+           url = EXCLUDED.url,
+           domain = EXCLUDED.domain,
+           score = EXCLUDED.score,
+           descendants = EXCLUDED.descendants,
+           story_text = EXCLUDED.story_text,
+           fetched_at = now()`,
+        [
+          item.id,
+          item.type || 'story',
+          item.title || '(untitled)',
+          item.url || null,
+          domainOf(item.url),
+          item.by || null,
+          item.score || 0,
+          item.descendants || 0,
+          item.text || null,
+          item.time || Math.floor(Date.now() / 1000),
+        ]
+      )
+      await client.query(
+        `INSERT INTO story_state (story_id) VALUES ($1) ON CONFLICT (story_id) DO NOTHING`,
+        [item.id]
+      )
+      await client.query(
+        `INSERT INTO feed_items (feed, rank, story_id, updated_at) VALUES ($1,$2,$3, now())
+         ON CONFLICT (feed, rank) DO UPDATE SET story_id = EXCLUDED.story_id, updated_at = now()`,
+        [feedKey, rank, item.id]
+      )
+      rank++
+    }
+    // Drop stale ranks past what we just wrote (feed shrank).
+    await client.query('DELETE FROM feed_items WHERE feed = $1 AND rank >= $2', [feedKey, rank])
+    await client.query('COMMIT')
+    log('INFO', 'synced feed', { feed: feedKey, count: rank })
+  } catch (err) {
+    await client.query('ROLLBACK')
+    throw err
+  } finally {
+    client.release()
+  }
+}
+
+async function syncAll() {
+  for (const feedKey of Object.keys(FEEDS)) {
+    try {
+      await syncFeed(feedKey)
+    } catch (err) {
+      log('ERROR', 'feed sync failed', { feed: feedKey, error: String(err) })
+    }
+  }
+}
+
+const MAX_COMMENT_NODES = 300
+
+async function fetchCommentsTree(storyId, maxDepth = 6) {
+  let count = 0
+
+  async function walk(id, depth) {
+    if (count >= MAX_COMMENT_NODES) return null
+    const item = await fetchItem(id)
+    count++
+    if (!item || item.deleted || item.dead) return null
+
+    let kids = []
+    if (item.kids && depth < maxDepth && count < MAX_COMMENT_NODES) {
+      const childItems = await mapWithConcurrency(
+        item.kids,
+        FETCH_CONCURRENCY,
+        (kidId) => walk(kidId, depth + 1)
+      )
+      kids = childItems.filter(Boolean)
+    }
+
+    return {
+      id: item.id,
+      by: item.by || null,
+      text: item.text || null,
+      time: item.time || null,
+      kids,
+    }
+  }
+
+  return Promise.all((await fetchItem(storyId)).kids?.slice(0, 40).map((id) => walk(id, 1)) || [])
+    .then((list) => list.filter(Boolean))
+}
+
+module.exports = { FEEDS, syncAll, syncFeed, fetchItem, fetchCommentsTree }
@@ -0,0 +1,134 @@
+require('dotenv').config({ quiet: true })
+require('./otel') // must load before express/pg are used, for auto-instrumentation
+
+const path = require('path')
+const express = require('express')
+const cors = require('cors')
+const { pool } = require('./db')
+const { log } = require('./otel')
+const { FEEDS, syncAll, fetchCommentsTree } = require('./hn')
+
+const PORT = Number(process.env.PORT || 3012)
+const SYNC_INTERVAL_MS = 5 * 60 * 1000
+
+const app = express()
+app.use(cors())
+app.use(express.json())
+
+app.get('/api/health', async (_req, res) => {
+  try {
+    await pool.query('SELECT 1')
+    res.json({ ok: true })
+  } catch (err) {
+    res.status(503).json({ ok: false, error: String(err) })
+  }
+})
+
+app.get('/api/feed/:feed', async (req, res) => {
+  const { feed } = req.params
+  if (!FEEDS[feed]) return res.status(404).json({ error: 'unknown feed' })
+
+  const limit = Math.min(Number(req.query.limit) || 100, 100)
+  const includeArchived = req.query.includeArchived === 'true'
+
+  try {
+    const { rows } = await pool.query(
+      `SELECT s.id, s.type, s.title, s.url, s.domain, s.by_user, s.score, s.descendants,
+              s.story_text, s.hn_created_at, fi.rank,
+              COALESCE(st.read, false) AS read,
+              COALESCE(st.starred, false) AS starred,
+              COALESCE(st.archived, false) AS archived
+       FROM feed_items fi
+       JOIN stories s ON s.id = fi.story_id
+       LEFT JOIN story_state st ON st.story_id = s.id
+       WHERE fi.feed = $1 ${includeArchived ? '' : 'AND COALESCE(st.archived, false) = false'}
+       ORDER BY fi.rank ASC
+       LIMIT $2`,
+      [feed, limit]
+    )
+    res.json({ feed, items: rows })
+  } catch (err) {
+    log('ERROR', 'feed query failed', { feed, error: String(err) })
+    res.status(500).json({ error: 'internal error' })
+  }
+})
+
+app.get('/api/story/:id', async (req, res) => {
+  const id = Number(req.params.id)
+  try {
+    const { rows } = await pool.query(
+      `SELECT s.*, COALESCE(st.read, false) AS read,
+              COALESCE(st.starred, false) AS starred,
+              COALESCE(st.archived, false) AS archived
+       FROM stories s
+       LEFT JOIN story_state st ON st.story_id = s.id
+       WHERE s.id = $1`,
+      [id]
+    )
+    if (!rows.length) return res.status(404).json({ error: 'not found' })
+    res.json(rows[0])
+  } catch (err) {
+    log('ERROR', 'story query failed', { id, error: String(err) })
+    res.status(500).json({ error: 'internal error' })
+  }
+})
+
+app.get('/api/story/:id/comments', async (req, res) => {
+  const id = Number(req.params.id)
+  try {
+    const comments = await fetchCommentsTree(id)
+    res.json({ id, comments })
+  } catch (err) {
+    log('ERROR', 'comments fetch failed', { id, error: String(err) })
+    res.status(502).json({ error: 'failed to fetch comments' })
+  }
+})
+
+app.post('/api/story/:id/state', async (req, res) => {
+  const id = Number(req.params.id)
+  const { read, starred, archived } = req.body || {}
+  try {
+    const { rows } = await pool.query(
+      `INSERT INTO story_state (story_id, read, starred, archived, read_at, updated_at)
+       VALUES ($1, COALESCE($2, false), COALESCE($3, false), COALESCE($4, false),
+               CASE WHEN $2 THEN now() ELSE NULL END, now())
+       ON CONFLICT (story_id) DO UPDATE SET
+         read = COALESCE($2, story_state.read),
+         starred = COALESCE($3, story_state.starred),
+         archived = COALESCE($4, story_state.archived),
+         read_at = CASE WHEN COALESCE($2, story_state.read) AND NOT story_state.read
+                        THEN now() ELSE story_state.read_at END,
+         updated_at = now()
+       RETURNING *`,
+      [id, read, starred, archived]
+    )
+    res.json(rows[0])
+  } catch (err) {
+    log('ERROR', 'state update failed', { id, error: String(err) })
+    res.status(500).json({ error: 'internal error' })
+  }
+})
+
+app.post('/api/sync', async (_req, res) => {
+  syncAll().catch((err) => log('ERROR', 'manual sync failed', { error: String(err) }))
+  res.json({ ok: true, started: true })
+})
+
+// Serve the built frontend in production.
+const distDir = path.join(__dirname, '..', 'dist')
+app.use(express.static(distDir))
+app.get(/^(?!\/api).*/, (_req, res) => {
+  res.sendFile(path.join(distDir, 'index.html'))
+})
+
+app.listen(PORT, () => {
+  log('INFO', 'hn-reader-api listening', { port: PORT })
+})
+
+syncAll()
+  .then(() => log('INFO', 'initial sync complete'))
+  .catch((err) => log('ERROR', 'initial sync failed', { error: String(err) }))
+
+setInterval(() => {
+  syncAll().catch((err) => log('ERROR', 'periodic sync failed', { error: String(err) }))
+}, SYNC_INTERVAL_MS)
@@ -0,0 +1,59 @@
+// OpenTelemetry bootstrap: sends traces, metrics and logs to the LGTM
+// stack's OTLP/HTTP collector at http://localhost:4318. Must be required
+// before any other module so auto-instrumentation can patch http/express/pg.
+const { NodeSDK } = require('@opentelemetry/sdk-node')
+const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node')
+const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http')
+const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-http')
+const { OTLPLogExporter } = require('@opentelemetry/exporter-logs-otlp-http')
+const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics')
+const { BatchLogRecordProcessor } = require('@opentelemetry/sdk-logs')
+const { resourceFromAttributes } = require('@opentelemetry/resources')
+const { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } = require('@opentelemetry/semantic-conventions')
+const { logs } = require('@opentelemetry/api-logs')
+
+const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318'
+
+const resource = resourceFromAttributes({
+  [ATTR_SERVICE_NAME]: 'hn-reader-api',
+  [ATTR_SERVICE_VERSION]: '1.0.0',
+})
+
+const sdk = new NodeSDK({
+  resource,
+  traceExporter: new OTLPTraceExporter({ url: `${endpoint}/v1/traces` }),
+  metricReader: new PeriodicExportingMetricReader({
+    exporter: new OTLPMetricExporter({ url: `${endpoint}/v1/metrics` }),
+    exportIntervalMillis: 15000,
+  }),
+  logRecordProcessors: [
+    new BatchLogRecordProcessor(new OTLPLogExporter({ url: `${endpoint}/v1/logs` })),
+  ],
+  instrumentations: [
+    getNodeAutoInstrumentations({
+      '@opentelemetry/instrumentation-fs': { enabled: false },
+    }),
+  ],
+})
+
+sdk.start()
+
+process.on('SIGTERM', () => sdk.shutdown().finally(() => process.exit(0)))
+
+const otelLogger = logs.getLogger('hn-reader-api')
+
+// Small structured logger: prints to stdout (captured by journald) and
+// emits an OTLP log record so logs show up alongside traces in Grafana.
+function log(severity, message, attributes = {}) {
+  const line = `[${severity}] ${message} ${Object.keys(attributes).length ? JSON.stringify(attributes) : ''}`
+  if (severity === 'ERROR') console.error(line)
+  else console.log(line)
+
+  otelLogger.emit({
+    severityText: severity,
+    body: message,
+    attributes,
+  })
+}
+
+module.exports = { log }
@@ -0,0 +1,3 @@
+{
+  "type": "commonjs"
+}