hn-reader/server/hn.js

const { pool } = require('./db')
const { log } = require('./otel')

const HN_BASE = 'https://hacker-news.firebaseio.com/v0'

const FEEDS = {
  top: 'topstories',
  new: 'newstories',
  best: 'beststories',
  ask: 'askstories',
  show: 'showstories',
  job: 'jobstories',
}

const ITEMS_PER_FEED = 100
const FETCH_CONCURRENCY = 20

async function fetchJson(url) {
  const res = await fetch(url)
  if (!res.ok) throw new Error(`HN API ${res.status} for ${url}`)
  return res.json()
}

function domainOf(url) {
  if (!url) return null
  try {
    return new URL(url).hostname.replace(/^www\./, '')
  } catch {
    return null
  }
}

async function mapWithConcurrency(items, limit, fn) {
  const results = new Array(items.length)
  let next = 0
  async function worker() {
    while (next < items.length) {
      const i = next++
      results[i] = await fn(items[i], i)
    }
  }
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker))
  return results
}

async function fetchItem(id) {
  return fetchJson(`${HN_BASE}/item/${id}.json`)
}

async function syncFeed(feedKey) {
  const endpoint = FEEDS[feedKey]
  const ids = (await fetchJson(`${HN_BASE}/${endpoint}.json`)).slice(0, ITEMS_PER_FEED)

  const items = await mapWithConcurrency(ids, FETCH_CONCURRENCY, async (id) => {
    try {
      return await fetchItem(id)
    } catch (err) {
      log('ERROR', 'failed to fetch HN item', { id, error: String(err) })
      return null
    }
  })

  const client = await pool.connect()
  try {
    await client.query('BEGIN')
    let rank = 0
    for (const item of items) {
      if (!item || item.deleted || item.dead) continue
      await client.query(
        `INSERT INTO stories (id, type, title, url, domain, by_user, score, descendants, story_text, hn_created_at, fetched_at)
         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9, to_timestamp($10), now())
         ON CONFLICT (id) DO UPDATE SET
           title = EXCLUDED.title,
           url = EXCLUDED.url,
           domain = EXCLUDED.domain,
           score = EXCLUDED.score,
           descendants = EXCLUDED.descendants,
           story_text = EXCLUDED.story_text,
           fetched_at = now()`,
        [
          item.id,
          item.type || 'story',
          item.title || '(untitled)',
          item.url || null,
          domainOf(item.url),
          item.by || null,
          item.score || 0,
          item.descendants || 0,
          item.text || null,
          item.time || Math.floor(Date.now() / 1000),
        ]
      )
      await client.query(
        `INSERT INTO story_state (story_id) VALUES ($1) ON CONFLICT (story_id) DO NOTHING`,
        [item.id]
      )
      await client.query(
        `INSERT INTO feed_items (feed, rank, story_id, updated_at) VALUES ($1,$2,$3, now())
         ON CONFLICT (feed, rank) DO UPDATE SET story_id = EXCLUDED.story_id, updated_at = now()`,
        [feedKey, rank, item.id]
      )
      rank++
    }
    // Drop stale ranks past what we just wrote (feed shrank).
    await client.query('DELETE FROM feed_items WHERE feed = $1 AND rank >= $2', [feedKey, rank])
    await client.query('COMMIT')
    log('INFO', 'synced feed', { feed: feedKey, count: rank })
  } catch (err) {
    await client.query('ROLLBACK')
    throw err
  } finally {
    client.release()
  }
}

async function syncAll() {
  for (const feedKey of Object.keys(FEEDS)) {
    try {
      await syncFeed(feedKey)
    } catch (err) {
      log('ERROR', 'feed sync failed', { feed: feedKey, error: String(err) })
    }
  }
}

const MAX_COMMENT_NODES = 300

async function fetchCommentsTree(storyId, maxDepth = 6) {
  let count = 0

  async function walk(id, depth) {
    if (count >= MAX_COMMENT_NODES) return null
    const item = await fetchItem(id)
    count++
    if (!item || item.deleted || item.dead) return null

    let kids = []
    if (item.kids && depth < maxDepth && count < MAX_COMMENT_NODES) {
      const childItems = await mapWithConcurrency(
        item.kids,
        FETCH_CONCURRENCY,
        (kidId) => walk(kidId, depth + 1)
      )
      kids = childItems.filter(Boolean)
    }

    return {
      id: item.id,
      by: item.by || null,
      text: item.text || null,
      time: item.time || null,
      kids,
    }
  }

  return Promise.all((await fetchItem(storyId)).kids?.slice(0, 40).map((id) => walk(id, 1)) || [])
    .then((list) => list.filter(Boolean))
}

module.exports = { FEEDS, syncAll, syncFeed, fetchItem, fetchCommentsTree }