SERP API in Node.js: fetch, een worker pool en paginering

// serp-client.mjs — Node.js 18+, no dependency
const API_URL = "https://serpcrypto.com/api/v1/search";
const CREDITS_URL = "https://serpcrypto.com/api/v1/credits";
const headers = { Authorization: `Bearer ${process.env.SERPCRYPTO_API_KEY}` };

/**
 * @typedef {Object} OrganicResult
 * @property {number} position
 * @property {string} title
 * @property {string} link
 * @property {string} [snippet]
 * @property {string} [date]
 */

/**
 * @typedef {Object} SearchResponse
 * @property {Object} search_metadata
 * @property {string} search_metadata.id
 * @property {string} search_metadata.status
 * @property {number} search_metadata.credits_used
 * @property {OrganicResult[]} [organic_results]
 * @property {Object} [pagination]
 * @property {number} pagination.current
 * @property {number} [pagination.next_page]
 */

/**
 * Run one search (1 credit when it succeeds).
 * @param {string} q
 * @param {Record<string, string | number>} [params]
 * @returns {Promise<SearchResponse>}
 */
export async function search(q, params = {}) {
  const query = new URLSearchParams({ q, ...Object.fromEntries(Object.entries(params).map(([k, v]) => [k, String(v)])) });
  const res = await fetch(`${API_URL}?${query}`, { headers, signal: AbortSignal.timeout(90_000) });
  if (!res.ok) {
    const body = await res.json().catch(() => ({}));
    const err = new Error(`${res.status} ${body.error?.code ?? "error"}: ${body.error?.message ?? ""}`);
    err.status = res.status;
    throw err;
  }
  return res.json();
}

/** Free: credits left, used today and the concurrency limit. */
export async function credits() {
  const res = await fetch(CREDITS_URL, { headers });
  if (!res.ok) throw new Error(`credits: HTTP ${res.status}`);
  return res.json();
}

// batch.mjs — parallel searches, never more than the account's limit at once
import { search, credits } from "./serp-client.mjs";

const keywords = ["password manager", "note taking app", "time tracking software", "invoice software"];
const { concurrency_limit: limit } = await credits();

let next = 0;
const results = new Map();

async function worker() {
  while (next < keywords.length) {
    const kw = keywords[next++];
    try {
      const data = await search(kw, { gl: "us", hl: "en" });
      results.set(kw, data.organic_results?.[0]?.link ?? null);
    } catch (err) {
      results.set(kw, `failed: ${err.message}`);
    }
  }
}

await Promise.all(Array.from({ length: Math.min(limit, keywords.length) }, worker));
console.table(Object.fromEntries(results));

// pages.mjs — up to three pages, stop when results end
import { search } from "./serp-client.mjs";

async function topResults(q, pages = 3, params = {}) {
  const rows = [];
  for (let page = 1; page <= pages; page++) {
    const data = await search(q, { ...params, page });
    for (const r of data.organic_results ?? []) rows.push({ page, position: r.position, link: r.link });
    if (!data.pagination?.next_page) break; // nothing more: save the credits
  }
  return rows;
}

console.log(await topResults("headless cms", 3, { gl: "de", hl: "en" }));