scripts: add rescue/retry ops tooling

scripts/.gitignore

@@ -0,0 +1,26 @@
+# Local runtime artifacts (do not commit)
+.playwright-profile*
+
+# Local isolated deps for Playwright (created on demand)
+.rescue-deps/node_modules/
+node_modules/
+
+# Databases / caches / logs / reports produced by scripts
+*.sqlite
+*.sqlite-journal
+*.db
+*.db-journal
+cookie.db
+*.log
+*.json
+*.txt
+
+# Keep code/config files tracked (override broad patterns above)
+!package.json
+!pnpm-workspace.yaml
+!pnpm-lock.yaml
+!*.js
+!*.ts
+!*.sh
+!*.sql
+!.gitignore

scripts/.rescue-deps/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "omnivore-rescue-deps",
+  "private": true,
+  "type": "module",
+  "dependencies": {
+    "playwright-core": "^1.58.0"
+  }
+}

scripts/fetch-queue-status.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -euo pipefail
+
+POSTGRES_CONTAINER="${POSTGRES_CONTAINER:-omnivore-postgres}"
+PGUSER="${PGUSER:-postgres}"
+DBNAME="${DBNAME:-omnivore}"
+
+psql_in_container() {
+  docker exec -i "$POSTGRES_CONTAINER" psql -U "$PGUSER" "$DBNAME" -v ON_ERROR_STOP=1 "$@"
+}
+
+echo "Postgres container: $POSTGRES_CONTAINER"
+echo "Database: $DBNAME"
+echo ""
+
+echo "Library item state counts (deleted_at IS NULL):"
+psql_in_container -t -c "
+SELECT state, COUNT(*) AS count
+FROM omnivore.library_item
+WHERE deleted_at IS NULL
+GROUP BY state
+ORDER BY count DESC, state ASC;
+" | sed 's/^ *//'
+echo ""
+
+echo "PROCESSING age buckets:"
+psql_in_container -t -c "
+SELECT
+  SUM(CASE WHEN saved_at >= NOW() - INTERVAL '15 minutes' THEN 1 ELSE 0 END) AS last_15m,
+  SUM(CASE WHEN saved_at <  NOW() - INTERVAL '15 minutes' AND saved_at >= NOW() - INTERVAL '1 hour' THEN 1 ELSE 0 END) AS from_15m_to_1h,
+  SUM(CASE WHEN saved_at <  NOW() - INTERVAL '1 hour' AND saved_at >= NOW() - INTERVAL '24 hours' THEN 1 ELSE 0 END) AS from_1h_to_24h,
+  SUM(CASE WHEN saved_at <  NOW() - INTERVAL '24 hours' THEN 1 ELSE 0 END) AS over_24h
+FROM omnivore.library_item
+WHERE state = 'PROCESSING'
+  AND deleted_at IS NULL;
+" | sed 's/^ *//'
+echo ""
+
+echo "Oldest PROCESSING items:"
+psql_in_container -c "
+SELECT id, saved_at, updated_at, original_url
+FROM omnivore.library_item
+WHERE state = 'PROCESSING'
+  AND deleted_at IS NULL
+ORDER BY saved_at ASC
+LIMIT 20;
+"

scripts/fix-retry-duplicates.ts

@@ -0,0 +1,342 @@
+#!/usr/bin/env node
+/**
+ * Fixes duplicate LibraryItems created by earlier retry runs that didn't reuse
+ * the original LibraryItem id and/or didn't preserve saved_at.
+ *
+ * Typical symptom:
+ * - Old item is stuck in PROCESSING with URL containing tracking params (utm_*).
+ * - A newer "duplicate" item exists for the same user with the cleaned URL.
+ *
+ * This script:
+ * - Finds PROCESSING items where cleanUrl(original_url) differs from original_url
+ * - If another item exists for the same user with original_url == cleaned URL:
+ *   - Moves foreign key references (highlights, entity_labels, etc.) from duplicate -> original
+ *   - Optionally copies "content-ish" fields from duplicate -> original if duplicate looks more complete
+ *   - Updates original.original_url to the cleaned URL
+ *   - Hard-deletes the duplicate row (required due to unique index on user_id+md5(original_url))
+ *
+ * Usage:
+ *   direnv exec . npx tsx scripts/fix-retry-duplicates.ts            # dry-run
+ *   direnv exec . npx tsx scripts/fix-retry-duplicates.ts --apply   # apply changes
+ */
+
+import { execSync } from "node:child_process";
+
+type LibraryItemRow = {
+	id: string;
+	userId: string;
+	url: string;
+	state: string;
+	savedAt: string;
+	createdAt: string;
+	updatedAt: string;
+	folder: string;
+	hasRealContent: boolean;
+};
+
+const POSTGRES_CONTAINER = process.env.POSTGRES_CONTAINER || "omnivore-postgres";
+const APPLY = process.argv.includes("--apply");
+
+const SAVING_CONTENT = "Your link is being saved...";
+
+function sqlStringLiteral(value: string) {
+	return `'${value.replace(/'/g, "''")}'`;
+}
+
+function cleanUrl(input: string) {
+	let u: URL;
+	try {
+		u = new URL(input);
+	} catch {
+		return input;
+	}
+
+	u.hash = "";
+
+	const isTweet =
+		(u.hostname === "twitter.com" || u.hostname.endsWith(".twitter.com")) &&
+		u.pathname.includes("/status");
+
+	const filteredParams: Array<[string, string]> = [];
+	for (const [key, value] of u.searchParams) {
+		if (/^utm_\w+/i.test(key)) continue;
+		if (isTweet && (key === "s" || key === "t")) continue;
+		filteredParams.push([key, value]);
+	}
+
+	// normalize-url defaults to sorting query parameters; mimic that behavior.
+	filteredParams.sort((a, b) => {
+		if (a[0] < b[0]) return -1;
+		if (a[0] > b[0]) return 1;
+		if (a[1] < b[1]) return -1;
+		if (a[1] > b[1]) return 1;
+		return 0;
+	});
+
+	u.search = "";
+	for (const [key, value] of filteredParams) {
+		u.searchParams.append(key, value);
+	}
+
+	let out = u.toString();
+
+	// URL serialization adds a trailing slash for bare origins; preserve the original form if it had none.
+	if (/^https?:\/\/[^/]+$/i.test(input) && out.endsWith("/")) {
+		out = out.slice(0, -1);
+	}
+
+	return out;
+}
+
+function runPsqlJsonLines(sql: string): any[] {
+	const out = execSync(
+		`docker exec ${POSTGRES_CONTAINER} psql -U postgres omnivore -v ON_ERROR_STOP=1 -t -A -c "${sql.replace(/"/g, '\\"')}"`,
+		{ encoding: "utf-8" },
+	);
+
+	return out
+		.trim()
+		.split("\n")
+		.map((line) => line.trim())
+		.filter((line) => line.length > 0)
+		.map((line) => JSON.parse(line));
+}
+
+function chunk<T>(arr: T[], size: number): T[][] {
+	const chunks: T[][] = [];
+	for (let i = 0; i < arr.length; i += size) chunks.push(arr.slice(i, i + size));
+	return chunks;
+}
+
+function fetchProcessingItems(): LibraryItemRow[] {
+	const sql = `
+SELECT json_build_object(
+  'id', id,
+  'userId', user_id,
+  'url', original_url,
+  'state', state,
+  'savedAt', saved_at,
+  'createdAt', created_at,
+  'updatedAt', updated_at,
+  'folder', folder,
+  'hasRealContent', (readable_content IS NOT NULL AND readable_content <> ${sqlStringLiteral(SAVING_CONTENT)})
+)
+FROM omnivore.library_item
+WHERE state = 'PROCESSING' AND deleted_at IS NULL;
+`;
+	return runPsqlJsonLines(sql) as LibraryItemRow[];
+}
+
+type TargetKey = string;
+const targetKey = (userId: string, url: string): TargetKey => `${userId}::${url}`;
+
+function fetchItemsByUserAndUrl(targets: Array<{ userId: string; url: string }>) {
+	if (targets.length === 0) return new Map<TargetKey, LibraryItemRow>();
+
+	const result = new Map<TargetKey, LibraryItemRow>();
+	for (const group of chunk(targets, 200)) {
+		const values = group
+			.map(
+				(t) =>
+					`(${sqlStringLiteral(t.userId)}::uuid, ${sqlStringLiteral(t.url)}::text)`,
+			)
+			.join(",\n");
+
+		const sql = `
+WITH targets(user_id, original_url) AS (
+  VALUES
+  ${values}
+)
+SELECT json_build_object(
+  'id', li.id,
+  'userId', li.user_id,
+  'url', li.original_url,
+  'state', li.state,
+  'savedAt', li.saved_at,
+  'createdAt', li.created_at,
+  'updatedAt', li.updated_at,
+  'folder', li.folder,
+  'hasRealContent', (li.readable_content IS NOT NULL AND li.readable_content <> ${sqlStringLiteral(SAVING_CONTENT)})
+)
+FROM omnivore.library_item li
+INNER JOIN targets t
+  ON li.user_id = t.user_id AND li.original_url = t.original_url
+WHERE li.deleted_at IS NULL;
+`;
+
+		const rows = runPsqlJsonLines(sql) as LibraryItemRow[];
+		for (const row of rows) result.set(targetKey(row.userId, row.url), row);
+	}
+	return result;
+}
+
+type FixPair = {
+	original: LibraryItemRow;
+	duplicate: LibraryItemRow;
+	cleanedUrl: string;
+};
+
+function buildFixPairs(processing: LibraryItemRow[]): FixPair[] {
+	const candidates: Array<{ userId: string; url: string }> = [];
+	const cleanedByOriginalId = new Map<string, string>();
+
+	for (const item of processing) {
+		const cleaned = cleanUrl(item.url);
+		if (cleaned !== item.url) {
+			cleanedByOriginalId.set(item.id, cleaned);
+			candidates.push({ userId: item.userId, url: cleaned });
+		}
+	}
+
+	const candidateMap = fetchItemsByUserAndUrl(
+		Array.from(
+			new Map(candidates.map((t) => [targetKey(t.userId, t.url), t])).values(),
+		),
+	);
+
+	const pairs: FixPair[] = [];
+	for (const item of processing) {
+		const cleanedUrl = cleanedByOriginalId.get(item.id);
+		if (!cleanedUrl) continue;
+		const dup = candidateMap.get(targetKey(item.userId, cleanedUrl));
+		if (!dup) continue;
+		if (dup.id === item.id) continue;
+		pairs.push({ original: item, duplicate: dup, cleanedUrl });
+	}
+
+	// Stable output: oldest originals first
+	pairs.sort((a, b) => a.original.savedAt.localeCompare(b.original.savedAt));
+	return pairs;
+}
+
+function applyFix(pair: FixPair) {
+	const originalId = pair.original.id;
+	const duplicateId = pair.duplicate.id;
+	const cleanedUrl = pair.cleanedUrl;
+
+	const shouldCopyContent =
+		pair.duplicate.state !== "PROCESSING" || pair.duplicate.hasRealContent;
+
+	const sql = `
+BEGIN;
+
+-- Move FKs from duplicate -> original (avoid cascade deletes losing data)
+UPDATE omnivore.highlight SET library_item_id = ${sqlStringLiteral(originalId)}::uuid
+WHERE library_item_id = ${sqlStringLiteral(duplicateId)}::uuid;
+
+-- Avoid unique conflicts for labels, then move library-item labels
+DELETE FROM omnivore.entity_labels el_dup
+USING omnivore.entity_labels el_old
+WHERE el_dup.library_item_id = ${sqlStringLiteral(duplicateId)}::uuid
+  AND el_dup.highlight_id IS NULL
+  AND el_old.library_item_id = ${sqlStringLiteral(originalId)}::uuid
+  AND el_old.highlight_id IS NULL
+  AND el_dup.label_id = el_old.label_id;
+
+UPDATE omnivore.entity_labels
+SET library_item_id = ${sqlStringLiteral(originalId)}::uuid
+WHERE library_item_id = ${sqlStringLiteral(duplicateId)}::uuid
+  AND highlight_id IS NULL;
+
+UPDATE omnivore.ai_summaries
+SET library_item_id = ${sqlStringLiteral(originalId)}::uuid
+WHERE library_item_id = ${sqlStringLiteral(duplicateId)}::uuid;
+
+UPDATE omnivore.recommendation
+SET library_item_id = ${sqlStringLiteral(originalId)}::uuid
+WHERE library_item_id = ${sqlStringLiteral(duplicateId)}::uuid;
+
+UPDATE omnivore.discover_feed_save_link
+SET article_save_id = ${sqlStringLiteral(originalId)}::uuid
+WHERE article_save_id = ${sqlStringLiteral(duplicateId)}::uuid;
+
+-- If the duplicate has progressed further, copy content-ish fields back to the original.
+-- Keep original saved_at/folder/note/etc.
+${shouldCopyContent ? `UPDATE omnivore.library_item dst
+SET
+  state = src.state,
+  title = src.title,
+  author = src.author,
+  description = src.description,
+  metadata = src.metadata,
+  thumbnail = src.thumbnail,
+  item_type = src.item_type,
+  upload_file_id = src.upload_file_id,
+  content_reader = src.content_reader,
+  readable_content = src.readable_content,
+  text_content_hash = src.text_content_hash,
+  item_language = src.item_language,
+  word_count = src.word_count,
+  site_name = src.site_name,
+  site_icon = src.site_icon,
+  download_url = src.download_url,
+  preview_content_type = src.preview_content_type,
+  preview_content = src.preview_content,
+  links = src.links,
+  feed_content = src.feed_content
+FROM omnivore.library_item src
+WHERE dst.id = ${sqlStringLiteral(originalId)}::uuid
+  AND src.id = ${sqlStringLiteral(duplicateId)}::uuid;` : "-- skip content copy"}
+
+-- Make the original URL canonical (post-cleanUrl)
+UPDATE omnivore.library_item
+SET original_url = ${sqlStringLiteral(cleanedUrl)}::text
+WHERE id = ${sqlStringLiteral(originalId)}::uuid;
+
+-- Recompute label_names so retry script can preserve labels correctly going forward
+UPDATE omnivore.library_item li
+SET label_names = COALESCE((
+  SELECT ARRAY_AGG(l.name ORDER BY l.name)
+  FROM omnivore.entity_labels el
+  INNER JOIN omnivore.labels l ON l.id = el.label_id
+  WHERE el.library_item_id = li.id AND el.highlight_id IS NULL
+), ARRAY[]::text[])
+WHERE li.id = ${sqlStringLiteral(originalId)}::uuid;
+
+-- Hard delete is required due to unique index on (user_id, md5(original_url)) not considering deleted_at.
+DELETE FROM omnivore.library_item WHERE id = ${sqlStringLiteral(duplicateId)}::uuid;
+
+COMMIT;
+`;
+
+	execSync(
+		`docker exec ${POSTGRES_CONTAINER} psql -U postgres omnivore -v ON_ERROR_STOP=1 -c "${sql.replace(/"/g, '\\"')}"`,
+		{ stdio: "inherit" },
+	);
+}
+
+function main() {
+	const processing = fetchProcessingItems();
+	const pairs = buildFixPairs(processing);
+
+	console.log(
+		`Found ${pairs.length} duplicate pair(s) where a PROCESSING item has a cleaned-URL duplicate.`,
+	);
+	if (pairs.length === 0) return;
+
+	console.log("\nSample (up to 10):");
+	for (const p of pairs.slice(0, 10)) {
+		console.log(
+			`- user=${p.original.userId} keep=${p.original.id} drop=${p.duplicate.id}\n  old=${p.original.url}\n  new=${p.duplicate.url}`,
+		);
+	}
+
+	if (!APPLY) {
+		console.log(
+			"\nDry-run only. Re-run with `--apply` to perform the merge + delete.",
+		);
+		return;
+	}
+
+	console.log("\nApplying fixes...");
+	for (const p of pairs) {
+		console.log(
+			`Fixing keep=${p.original.id} drop=${p.duplicate.id} url=${p.cleanedUrl}`,
+		);
+		applyFix(p);
+	}
+
+	console.log("\nDone.");
+}
+
+main();

scripts/retry-failed-items.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Script to retry failed/stuck library items
+
+echo "Checking items stuck in PROCESSING state..."
+
+PROCESSING_COUNT=$(docker exec omnivore-postgres psql -U postgres omnivore -t -c "
+SELECT COUNT(*)
+FROM omnivore.library_item
+WHERE state = 'PROCESSING' AND deleted_at IS NULL;
+")
+
+echo "Found $PROCESSING_COUNT items in PROCESSING state"
+
+read -p "Do you want to reset these items to retry? (yes/no): " confirm
+
+if [ "$confirm" != "yes" ]; then
+    echo "Aborted"
+    exit 0
+fi
+
+echo "Resetting items to retry..."
+
+# Delete stuck items so they can be re-saved
+docker exec omnivore-postgres psql -U postgres omnivore -c "
+DELETE FROM omnivore.library_item
+WHERE state = 'PROCESSING'
+AND deleted_at IS NULL
+AND saved_at < NOW() - INTERVAL '1 hour';
+"
+
+echo "Items deleted. You can now re-save the URLs through the UI or API."
+echo ""
+echo "To get the list of URLs to re-save, run:"
+echo "  docker exec omnivore-postgres psql -U postgres omnivore -c \"SELECT original_url FROM omnivore.library_item WHERE state = 'PROCESSING' AND deleted_at IS NULL ORDER BY saved_at DESC;\""

scripts/retry-via-api.ts

@@ -0,0 +1,474 @@
+#!/usr/bin/env node
+/**
+ * Script to retry failed/stuck library items via GraphQL API
+ * This preserves existing metadata (labels, folders, notes, etc.)
+ */
+
+import { execSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import type { IncomingMessage, RequestOptions } from "node:http";
+import http from "node:http";
+import https from "node:https";
+
+const API_ENDPOINT =
+	process.env.API_ENDPOINT || "http://localhost:4000/api/graphql";
+const API_KEY = process.env.OMNIVORE_API_KEY;
+const STATES = (process.env.STATES || "PROCESSING")
+	.split(",")
+	.map((s) => s.trim().toUpperCase())
+	.filter(Boolean);
+const MAX_ITEMS = process.env.MAX_ITEMS
+	? Number.parseInt(process.env.MAX_ITEMS, 10)
+	: undefined;
+const SLEEP_MS = process.env.SLEEP_MS
+	? Number.parseInt(process.env.SLEEP_MS, 10)
+	: 100;
+const PROCESSING_OLDER_THAN_HOURS = process.env.PROCESSING_OLDER_THAN_HOURS
+	? Number.parseInt(process.env.PROCESSING_OLDER_THAN_HOURS, 10)
+	: undefined;
+const UPDATED_BEFORE_ISO = process.env.UPDATED_BEFORE_ISO || undefined;
+const DOMAIN_IN = process.env.DOMAIN_IN
+	? process.env.DOMAIN_IN.split(",").map((d) => d.trim().toLowerCase()).filter(Boolean)
+	: undefined;
+
+const ME_QUERY = `query Me { me { id } }`;
+
+const SAVE_URL_MUTATION = `
+  mutation SaveUrl($input: SaveUrlInput!) {
+    saveUrl(input: $input) {
+      ... on SaveSuccess {
+        url
+        clientRequestId
+      }
+      ... on SaveError {
+        errorCodes
+        message
+      }
+    }
+  }
+`;
+
+interface LibraryItem {
+	id: string;
+	url: string;
+	title: string;
+	folder: string;
+	savedAt: string;
+	publishedAt: string | null;
+	labelNames: string[];
+}
+
+function validateApiKey() {
+	if (!API_KEY) {
+		console.error("Error: OMNIVORE_API_KEY environment variable not set");
+		console.error("Run with: direnv exec . npx tsx scripts/retry-via-api.ts");
+		process.exit(1);
+	}
+	console.log(`Using API key: ${API_KEY.substring(0, 8)}...`);
+	console.log(`Using endpoint: ${API_ENDPOINT}`);
+}
+
+function validateConfig() {
+	const allowedStates = new Set(["PROCESSING", "FAILED"]);
+	const invalidStates = STATES.filter((s) => !allowedStates.has(s));
+	if (invalidStates.length) {
+		throw new Error(
+			`Invalid STATES: ${invalidStates.join(
+				", ",
+			)} (allowed: PROCESSING, FAILED)`,
+		);
+	}
+
+	if (UPDATED_BEFORE_ISO) {
+		const parsed = Date.parse(UPDATED_BEFORE_ISO);
+		if (Number.isNaN(parsed)) {
+			throw new Error(
+				`UPDATED_BEFORE_ISO must be an ISO timestamp, got: ${UPDATED_BEFORE_ISO}`,
+			);
+		}
+	}
+
+	if (DOMAIN_IN?.length) {
+		for (const domain of DOMAIN_IN) {
+			if (!/^[a-z0-9.-]+$/.test(domain)) {
+				throw new Error(
+					`DOMAIN_IN contains invalid domain "${domain}". Only [a-z0-9.-] allowed.`,
+				);
+			}
+		}
+	}
+}
+
+function getProcessingItems(userId: string): LibraryItem[] {
+	if (MAX_ITEMS != null && (!Number.isFinite(MAX_ITEMS) || MAX_ITEMS <= 0)) {
+		throw new Error(`MAX_ITEMS must be a positive integer, got: ${process.env.MAX_ITEMS}`);
+	}
+	if (
+		PROCESSING_OLDER_THAN_HOURS != null &&
+		(!Number.isFinite(PROCESSING_OLDER_THAN_HOURS) ||
+			PROCESSING_OLDER_THAN_HOURS <= 0)
+	) {
+		throw new Error(
+			`PROCESSING_OLDER_THAN_HOURS must be a positive integer, got: ${process.env.PROCESSING_OLDER_THAN_HOURS}`,
+		);
+	}
+
+	const statesClause = ` AND state IN (${STATES.map((s) => `'${s}'`).join(", ")})`;
+	const olderThanClause =
+		PROCESSING_OLDER_THAN_HOURS != null
+			? ` AND updated_at < (NOW() - INTERVAL '${PROCESSING_OLDER_THAN_HOURS} hours')`
+			: "";
+	const updatedBeforeClause = UPDATED_BEFORE_ISO
+		? ` AND updated_at <= '${UPDATED_BEFORE_ISO}'::timestamptz`
+		: "";
+	const domainClause =
+		DOMAIN_IN?.length
+			? ` AND lower(regexp_replace(original_url, '^https?://([^/]+).*$', '\\\\1')) IN (${DOMAIN_IN.map((d) => `'${d}'`).join(", ")})`
+			: "";
+
+	const query = `
+    SELECT json_build_object(
+			'id', id,
+			'url', original_url,
+			'title', title,
+			'folder', folder,
+			'savedAt', saved_at,
+			'publishedAt', published_at,
+			'labelNames', label_names
+		)
+    FROM omnivore.library_item
+    WHERE deleted_at IS NULL AND user_id = '${userId}'${statesClause}${olderThanClause}${updatedBeforeClause}${domainClause}
+    ORDER BY ${
+			PROCESSING_OLDER_THAN_HOURS != null ? "updated_at ASC" : "saved_at DESC"
+		}
+		${MAX_ITEMS ? `LIMIT ${MAX_ITEMS}` : ""};
+  `;
+
+	const result = execSync(
+		`docker exec omnivore-postgres psql -U postgres omnivore -t -A -c "${query}"`,
+		{ encoding: "utf-8" },
+	);
+
+	return result
+		.trim()
+		.split("\n")
+		.filter((line) => line.length > 0)
+		.map((line) => JSON.parse(line) as LibraryItem);
+}
+
+function createSaveUrlVariables(
+	url: string,
+	folder: string,
+	clientRequestId: string,
+	savedAt: string,
+	publishedAt: string | null,
+	labelNames: string[],
+) {
+	return {
+		input: {
+			url,
+			source: "api",
+			clientRequestId,
+			folder,
+			savedAt,
+			...(publishedAt ? { publishedAt } : {}),
+			labels: labelNames.map((name) => ({ name })),
+		},
+	};
+}
+
+function createRequestOptions(parsedUrl: URL, payloadLength: number) {
+	const port = parsedUrl.port ? Number(parsedUrl.port) : undefined;
+	const path = `${parsedUrl.pathname}${parsedUrl.search}`;
+	return {
+		hostname: parsedUrl.hostname,
+		port,
+		path,
+		method: "POST",
+		headers: {
+			Accept: "application/json",
+			"Content-Type": "application/json",
+			"Omnivore-Authorization": API_KEY,
+			"Content-Length": payloadLength,
+		},
+	};
+}
+
+interface GraphQLResponse {
+	data?: {
+		saveUrl?: {
+			url?: string;
+			clientRequestId?: string;
+			errorCodes?: string[];
+			message?: string;
+		};
+		[key: string]: unknown;
+	};
+	errors?: Array<{
+		message: string;
+		extensions?: Record<string, unknown>;
+	}>;
+}
+
+function previewBody(text: string, maxChars: number = 240) {
+	const trimmed = text.trim();
+	if (trimmed.length <= maxChars) return trimmed;
+	return `${trimmed.slice(0, maxChars)}…`;
+}
+
+function endpointHint(statusCode: number, bodyText: string) {
+	const trimmed = bodyText.trim();
+	const looksLikeNextError =
+		trimmed.includes("next-head-count") ||
+		trimmed.includes("405: Method Not Allowed") ||
+		trimmed.includes("__NEXT_DATA__");
+
+	if (statusCode === 405 || looksLikeNextError) {
+		return ` Hint: this usually means API_ENDPOINT is pointing at the web app (Next.js) instead of the Omnivore API service. For local docker-compose, try API_ENDPOINT=http://localhost:4000/api/graphql.`;
+	}
+
+	return "";
+}
+
+async function readResponseBody(res: IncomingMessage): Promise<Buffer> {
+	const chunks: Buffer[] = [];
+	return new Promise((resolve, reject) => {
+		res.on("data", (chunk: Buffer | string) => {
+			chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
+		});
+		res.on("end", () => resolve(Buffer.concat(chunks)));
+		res.on("error", reject);
+	});
+}
+
+function isLikelyJson(contentType: string | undefined, bodyText: string) {
+	const ct = contentType?.toLowerCase();
+	if (ct && (ct.includes("application/json") || ct.includes("+json"))) return true;
+	const trimmed = bodyText.trimStart();
+	return trimmed.startsWith("{") || trimmed.startsWith("[");
+}
+
+async function parseGraphQLResponse(res: IncomingMessage): Promise<GraphQLResponse> {
+	const statusCode = res.statusCode ?? 0;
+	const statusMessage = res.statusMessage ?? "";
+	const contentTypeHeader = res.headers["content-type"];
+	const contentType = Array.isArray(contentTypeHeader)
+		? contentTypeHeader.join(", ")
+		: contentTypeHeader;
+	const locationHeader = res.headers.location;
+	const location = Array.isArray(locationHeader)
+		? locationHeader.join(", ")
+		: locationHeader;
+
+	const bodyBuffer = await readResponseBody(res);
+	const bodyText = bodyBuffer.toString("utf-8");
+
+	if (!isLikelyJson(contentType, bodyText)) {
+		throw new Error(
+			`Non-JSON response from API (${statusCode} ${statusMessage}) content-type=${contentType ?? "unknown"}${location ? ` location=${location}` : ""} body="${previewBody(bodyText)}"${endpointHint(statusCode, bodyText)}`,
+		);
+	}
+
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(bodyText) as unknown;
+	} catch (e) {
+		throw new Error(
+			`Invalid JSON from API (${statusCode} ${statusMessage}) content-type=${contentType ?? "unknown"}${location ? ` location=${location}` : ""} body="${previewBody(bodyText)}"${endpointHint(statusCode, bodyText)}`,
+		);
+	}
+
+	if (statusCode < 200 || statusCode >= 300) {
+		throw new Error(
+			`HTTP ${statusCode} from API content-type=${contentType ?? "unknown"}${location ? ` location=${location}` : ""} body="${previewBody(bodyText)}"${endpointHint(statusCode, bodyText)}`,
+		);
+	}
+
+	return parsed as GraphQLResponse;
+}
+
+function makeHttpRequest(
+	options: RequestOptions,
+	payload: string,
+): Promise<GraphQLResponse> {
+	return new Promise((resolve, reject) => {
+		const parsedUrl = new URL(API_ENDPOINT);
+		const client = parsedUrl.protocol === "https:" ? https : http;
+
+		const req = client.request(options, async (res) => {
+			try {
+				resolve(await parseGraphQLResponse(res));
+			} catch (e) {
+				reject(e);
+			}
+		});
+
+		req.on("error", reject);
+		req.write(payload);
+		req.end();
+	});
+}
+
+async function saveUrl(
+	url: string,
+	folder: string = "inbox",
+	clientRequestId: string = randomUUID(),
+	savedAt: string = new Date().toISOString(),
+	publishedAt: string | null = null,
+	labelNames: string[] = [],
+) {
+	const variables = createSaveUrlVariables(
+		url,
+		folder,
+		clientRequestId,
+		savedAt,
+		publishedAt,
+		labelNames,
+	);
+	const payload = JSON.stringify({ query: SAVE_URL_MUTATION, variables });
+	const parsedUrl = new URL(API_ENDPOINT);
+	const options = createRequestOptions(parsedUrl, Buffer.byteLength(payload));
+
+	return makeHttpRequest(options, payload);
+}
+
+async function verifyApiConnection() {
+	const payload = JSON.stringify({
+		query: ME_QUERY,
+	});
+	const parsedUrl = new URL(API_ENDPOINT);
+	const options = createRequestOptions(parsedUrl, Buffer.byteLength(payload));
+	const result = await makeHttpRequest(options, payload);
+	if (result.errors?.length) {
+		throw new Error(`GraphQL errors: ${JSON.stringify(result.errors)}`);
+	}
+	const meId = (result.data as any)?.me?.id as string | undefined;
+	if (!meId) {
+		throw new Error("API did not return me.id");
+	}
+	return meId;
+}
+
+function deleteOldItem(id: string, userId: string) {
+	const query = `UPDATE omnivore.library_item SET deleted_at = NOW() WHERE id = '${id}' AND user_id = '${userId}';`;
+	execSync(
+		`docker exec omnivore-postgres psql -U postgres omnivore -c "${query}"`,
+	);
+}
+
+function displaySampleItems(items: LibraryItem[]) {
+	console.log("\nSample items:");
+	items.slice(0, 10).forEach((item, i) => {
+		console.log(`${i + 1}. ${item.url}`);
+	});
+
+	if (items.length > 10) {
+		console.log(`\n... and ${items.length - 10} more`);
+	}
+}
+
+async function retryItem(
+	item: LibraryItem,
+	index: number,
+	total: number,
+	userId: string,
+) {
+	console.log(
+		`[${index + 1}/${total}] Retrying: ${item.url.substring(0, 80)}...`,
+	);
+
+	const result = await saveUrl(
+		item.url,
+		item.folder || "inbox",
+		item.id,
+		item.savedAt,
+		item.publishedAt,
+		item.labelNames,
+	);
+
+	if (result.data?.saveUrl?.url) {
+		const newId = result.data.saveUrl.clientRequestId;
+		if (newId && newId !== item.id) {
+			deleteOldItem(item.id, userId);
+		}
+		console.log(`  ✓ Success`);
+		return { success: true };
+	}
+
+	if (result.errors?.length) {
+		console.log(`  ✗ Failed: ${JSON.stringify(result.errors)}`);
+		return { success: false };
+	}
+
+	const saveUrlResult = result.data?.saveUrl;
+	if (saveUrlResult?.errorCodes?.includes("UNAUTHORIZED")) {
+		console.error(
+			`  ✗ Failed: ${JSON.stringify(saveUrlResult)} (check OMNIVORE_API_KEY)`,
+		);
+		process.exitCode = 1;
+		throw new Error("Unauthorized");
+	}
+
+	if (
+		saveUrlResult?.errorCodes?.includes("UNKNOWN") &&
+		(saveUrlResult.message == null || saveUrlResult.message.length === 0)
+	) {
+		console.log(
+			"  ↳ SaveError UNKNOWN (no message). Check API logs for the underlying exception; common causes are queue/redis/content-fetch not running or misconfigured.",
+		);
+	}
+
+	console.log(`  ✗ Failed: ${JSON.stringify(result.data?.saveUrl || result)}`);
+	return { success: false };
+}
+
+async function processItems(items: LibraryItem[], userId: string) {
+	let successful = 0;
+	let failed = 0;
+
+	for (let i = 0; i < items.length; i++) {
+		try {
+			const result = await retryItem(items[i], i, items.length, userId);
+			if (result.success) successful++;
+			else failed++;
+
+			await new Promise((resolve) => setTimeout(resolve, SLEEP_MS));
+		} catch (error) {
+			failed++;
+			const message = error instanceof Error ? error.message : String(error);
+			console.log(`  ✗ Error: ${message}`);
+		}
+	}
+
+	return { successful, failed };
+}
+
+function displaySummary(total: number, successful: number, failed: number) {
+	console.log("\n=== Summary ===");
+	console.log(`Total: ${total}`);
+	console.log(`Successful: ${successful}`);
+	console.log(`Failed: ${failed}`);
+}
+
+async function main() {
+	validateApiKey();
+	validateConfig();
+	const userId = await verifyApiConnection();
+
+	console.log(`Fetching items in states: ${STATES.join(", ")}...`);
+	const items = getProcessingItems(userId);
+
+	console.log(`Found ${items.length} items to retry`);
+	if (items.length === 0) {
+		console.log("No items to retry!");
+		return;
+	}
+
+	displaySampleItems(items);
+	console.log("\nStarting retry process...");
+
+	const { successful, failed } = await processItems(items, userId);
+	displaySummary(items.length, successful, failed);
+}
+
+main().catch(console.error);

scripts/test-api.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Test API authentication
+
+echo "Testing API endpoint: $API_ENDPOINT"
+echo "Using API key: ${OMNIVORE_API_KEY:0:10}..."
+echo ""
+
+# Test with curl
+echo "Testing GraphQL endpoint..."
+curl -v "$API_ENDPOINT" \
+  -H "Content-Type: application/json" \
+  -H "Omnivore-Authorization: $OMNIVORE_API_KEY" \
+  -d '{"query":"query { me { id name email } }"}' \
+  2>&1 | head -50
+
+echo ""
+echo "---"
+echo "If you see HTML instead of JSON, check:"
+echo "1. API_ENDPOINT is correct (should end with /api/graphql)"
+echo "2. OMNIVORE_API_KEY is set correctly"
+echo "3. The API service is running"