From 6ff55c40435a392b9f063a5c27ebc4a6608344fd Mon Sep 17 00:00:00 2001 From: Rohit Amarnath <88762+ramarnat@users.noreply.github.com> Date: Wed, 28 Jan 2026 13:22:03 -0500 Subject: [PATCH] scripts: add migration and labeling utilities --- scripts/apply-labels-with-mapping.js | 371 ++++++++ scripts/apply-labels.js | 385 ++++++++ scripts/apply-single-label.js | 164 ++++ scripts/check-missing-labeled.sql | 33 + scripts/compare-urls.js | 337 +++++++ scripts/download-items-mapping.js | 224 +++++ scripts/find-redirected-urls.sql | 24 + scripts/import-pocket.js | 1268 ++++++++++++++++++++++++++ scripts/migrate-omnivore.js | 481 ++++++++++ scripts/package.json | 20 + scripts/pnpm-lock.yaml | 369 ++++++++ scripts/pnpm-workspace.yaml | 2 + scripts/test-auth.js | 53 ++ scripts/test-create-label.js | 67 ++ 14 files changed, 3798 insertions(+) create mode 100755 scripts/apply-labels-with-mapping.js create mode 100644 scripts/apply-labels.js create mode 100644 scripts/apply-single-label.js create mode 100644 scripts/check-missing-labeled.sql create mode 100644 scripts/compare-urls.js create mode 100755 scripts/download-items-mapping.js create mode 100644 scripts/find-redirected-urls.sql create mode 100644 scripts/import-pocket.js create mode 100644 scripts/migrate-omnivore.js create mode 100644 scripts/package.json create mode 100644 scripts/pnpm-lock.yaml create mode 100644 scripts/pnpm-workspace.yaml create mode 100644 scripts/test-auth.js create mode 100644 scripts/test-create-label.js diff --git a/scripts/apply-labels-with-mapping.js b/scripts/apply-labels-with-mapping.js new file mode 100755 index 000000000..c588fab02 --- /dev/null +++ b/scripts/apply-labels-with-mapping.js @@ -0,0 +1,371 @@ +#!/usr/bin/env node + +import Database from 'better-sqlite3'; +import fetch from 'node-fetch'; +import pLimit from 'p-limit'; +import chalk from 'chalk'; +import { writeFile } from 'fs/promises'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Configuration +const CONFIG = { + API_URL: + process.env.API_ENDPOINT || + process.env.API_URL || + 'http://localhost:4000/api/graphql', + API_KEY: process.env.OMNIVORE_API_KEY, + ARCHIVE_DB_PATH: + process.env.ARCHIVE_DB_PATH || + join(__dirname, '../self-hosting/archive-db/store.sqlite'), + MAPPING_DB_PATH: + process.env.MAPPING_DB_PATH || join(__dirname, 'url-id-mapping.sqlite'), + RATE_LIMIT: 3, // concurrent requests + BATCH_SIZE: 10, // items to process per batch +}; + +if (!CONFIG.API_KEY) { + console.error(chalk.red('❌ OMNIVORE_API_KEY environment variable is required')); + console.error( + chalk.gray( + 'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/apply-labels-with-mapping.js', + ), + ); + process.exit(1); +} + +class LabelApplicator { + constructor() { + this.archiveDb = null; + this.mappingDb = null; + this.stats = { + totalItemsWithLabels: 0, + processedItems: 0, + failedItems: [], + successfulItems: 0, + notFoundItems: [], + startTime: Date.now(), + }; + this.limit = pLimit(CONFIG.RATE_LIMIT); + this.labelNameToId = new Map(); + this.urlToPageId = new Map(); + } + + // Initialize databases + initDatabases() { + try { + // Archive database + this.archiveDb = new Database(CONFIG.ARCHIVE_DB_PATH, { readonly: true }); + console.log(chalk.green('✓ Connected to archive SQLite database')); + + // Mapping database + this.mappingDb = new Database(CONFIG.MAPPING_DB_PATH, { readonly: true }); + console.log(chalk.green('✓ Connected to mapping database')); + + // Load all URL to ID mappings into memory for fast lookup + const mappings = this.mappingDb.prepare('SELECT url, id FROM item_mapping').all(); + mappings.forEach(row => { + this.urlToPageId.set(row.url, row.id); + }); + console.log(chalk.green(`✓ Loaded ${mappings.length} URL-to-ID mappings`)); + + } catch (error) { + console.error(chalk.red('✗ Failed to connect to databases:'), error.message); + process.exit(1); + } + } + + // Make GraphQL request + async makeGraphQLRequest(query, variables = {}) { + const response = await fetch(CONFIG.API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': CONFIG.API_KEY, + }, + body: JSON.stringify({ + query, + variables, + }), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + + if (result.errors) { + throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`); + } + + return result.data; + } + + // Extract items with their labels from archive SQLite + extractItemLabelAssociations() { + const query = ` + SELECT + li.ZPAGEURLSTRING as url, + li.ZTITLE as title, + GROUP_CONCAT(label.ZNAME, '||') as labelNames + FROM ZLINKEDITEM li + JOIN Z_2LABELS l ON li.Z_PK = l.Z_2LINKEDITEMS + JOIN ZLINKEDITEMLABEL label ON l.Z_3LABELS1 = label.Z_PK + GROUP BY li.ZPAGEURLSTRING, li.ZTITLE + ORDER BY li.ZPAGEURLSTRING + `; + + const results = this.archiveDb.prepare(query).all(); + this.stats.totalItemsWithLabels = results.length; + + return results.map(row => ({ + url: row.url, + title: row.title || '', + labels: row.labelNames.split('||').filter(name => name && name.trim()), + })); + } + + // Get all labels from the new Omnivore instance + async fetchLabelsFromOmnivore() { + const query = ` + query { + labels { + ... on LabelsSuccess { + labels { + id + name + color + } + } + ... on LabelsError { + errorCodes + } + } + } + `; + + try { + const data = await this.makeGraphQLRequest(query); + + if (data.labels.errorCodes) { + throw new Error(`Failed to fetch labels: ${data.labels.errorCodes.join(', ')}`); + } + + // Build mapping of label name to ID + data.labels.labels.forEach(label => { + this.labelNameToId.set(label.name, label.id); + }); + + console.log(chalk.green(`✓ Fetched ${data.labels.labels.length} labels from Omnivore`)); + return data.labels.labels; + } catch (error) { + console.error(chalk.red('✗ Failed to fetch labels:'), error.message); + throw error; + } + } + + // Apply labels to an item using setLabels mutation + async applyLabelsToItem(pageId, labelNames, itemTitle = '', itemUrl = '') { + // Convert label names to IDs + const labelIds = labelNames + .map(name => this.labelNameToId.get(name)) + .filter(id => id); // Remove any undefined IDs + + if (labelIds.length === 0) { + throw new Error(`No valid labels found for item. Requested: ${labelNames.join(', ')}`); + } + + const mutation = ` + mutation SetLabels($input: SetLabelsInput!) { + setLabels(input: $input) { + ... on SetLabelsSuccess { + labels { + id + name + } + } + ... on SetLabelsError { + errorCodes + } + } + } + `; + + try { + const data = await this.makeGraphQLRequest(mutation, { + input: { + pageId: pageId, + labelIds: labelIds, + }, + }); + + if (data.setLabels.errorCodes) { + throw new Error(`Failed to set labels: ${data.setLabels.errorCodes.join(', ')}`); + } + + return data.setLabels.labels; + } catch (error) { + console.error(chalk.red(`✗ Failed to apply labels to "${itemTitle}" (${itemUrl}):`), error.message); + throw error; + } + } + + // Process items in batches + async processItems(itemsWithLabels) { + console.log(chalk.blue(`\n🏷️ Processing ${itemsWithLabels.length} items with labels...`)); + + const batches = []; + for (let i = 0; i < itemsWithLabels.length; i += CONFIG.BATCH_SIZE) { + batches.push(itemsWithLabels.slice(i, i + CONFIG.BATCH_SIZE)); + } + + for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { + const batch = batches[batchIndex]; + console.log(chalk.cyan(`\nProcessing batch ${batchIndex + 1}/${batches.length} (${batch.length} items)...`)); + + const promises = batch.map(item => + this.limit(async () => { + try { + // Look up the page ID from our mapping + const pageId = this.urlToPageId.get(item.url); + + if (!pageId) { + throw new Error(`Item not found in mapping for URL: ${item.url}`); + } + + // Apply labels + const appliedLabels = await this.applyLabelsToItem( + pageId, + item.labels, + item.title, + item.url + ); + + this.stats.successfulItems++; + console.log(chalk.green(`✓ Applied ${appliedLabels.length} labels to: ${item.title || item.url}`)); + console.log(chalk.gray(` Labels: ${appliedLabels.map(l => l.name).join(', ')}`)); + + return { success: true, item, appliedLabels, pageId }; + } catch (error) { + if (error.message.includes('not found in mapping')) { + this.stats.notFoundItems.push({ item, error: error.message }); + console.error(chalk.yellow(`⚠ Item not found in Omnivore: ${item.title || item.url}`)); + } else { + this.stats.failedItems.push({ item, error: error.message }); + console.error(chalk.red(`✗ Failed to process "${item.title}": ${error.message}`)); + } + return { success: false, item, error: error.message }; + } + }) + ); + + await Promise.all(promises); + + // Progress update + this.stats.processedItems += batch.length; + const progress = (this.stats.processedItems / this.stats.totalItemsWithLabels * 100).toFixed(1); + console.log(chalk.yellow(`Progress: ${progress}% (${this.stats.processedItems}/${this.stats.totalItemsWithLabels} items)`)); + } + } + + // Generate report + async generateReport() { + const duration = (Date.now() - this.stats.startTime) / 1000; + + const report = { + labelApplication: { + timestamp: new Date().toISOString(), + duration: `${duration.toFixed(2)} seconds`, + }, + summary: { + totalItemsWithLabels: this.stats.totalItemsWithLabels, + processedItems: this.stats.processedItems, + successfulItems: this.stats.successfulItems, + notFoundItems: this.stats.notFoundItems.length, + failedItems: this.stats.failedItems.length, + }, + notFoundItems: this.stats.notFoundItems.map(f => ({ + title: f.item.title, + url: f.item.url, + labels: f.item.labels, + })), + failures: this.stats.failedItems, + }; + + const reportPath = join(__dirname, 'label-application-report.json'); + await writeFile(reportPath, JSON.stringify(report, null, 2)); + + console.log(chalk.blue(`\n📊 Label Application Report:`)); + console.log(chalk.green(`✓ Successfully labeled: ${report.summary.successfulItems}/${report.summary.totalItemsWithLabels} items`)); + + if (report.summary.notFoundItems > 0) { + console.log(chalk.yellow(`⚠ Not found in Omnivore: ${report.summary.notFoundItems} items`)); + } + + if (report.summary.failedItems > 0) { + console.log(chalk.red(`✗ Failed items: ${report.summary.failedItems}`)); + } + + console.log(chalk.blue(`📄 Full report saved to: ${reportPath}`)); + return report; + } + + // Main process + async applyLabels() { + try { + console.log(chalk.bold.blue('🏷️ Starting Label Application Process (with Mapping)')); + console.log(chalk.gray(`API: ${CONFIG.API_URL}`)); + + this.initDatabases(); + + // Extract original label associations + console.log(chalk.blue('\n📤 Extracting label associations from archive SQLite...')); + const itemsWithLabels = this.extractItemLabelAssociations(); + console.log(chalk.green(`✓ Found ${itemsWithLabels.length} items with labels`)); + + if (itemsWithLabels.length === 0) { + console.log(chalk.yellow('No items with labels found. Nothing to do.')); + return; + } + + // Show sample of items to be processed + console.log(chalk.blue('\n📋 Sample items to process:')); + itemsWithLabels.slice(0, 3).forEach((item, index) => { + console.log(chalk.gray(`${index + 1}. ${item.title || 'No title'}`)); + console.log(chalk.gray(` URL: ${item.url}`)); + console.log(chalk.gray(` Labels: ${item.labels.join(', ')}`)); + }); + + // Fetch current labels from Omnivore + console.log(chalk.blue('\n📋 Fetching current labels from Omnivore...')); + await this.fetchLabelsFromOmnivore(); + + // Process items + await this.processItems(itemsWithLabels); + + // Generate report + await this.generateReport(); + + console.log(chalk.bold.green('\n🎉 Label application completed!')); + + } catch (error) { + console.error(chalk.red('\n💥 Label application failed:'), error.message); + process.exit(1); + } finally { + if (this.archiveDb) { + this.archiveDb.close(); + } + if (this.mappingDb) { + this.mappingDb.close(); + } + } + } +} + +// Run the label application +const applicator = new LabelApplicator(); +applicator.applyLabels(); diff --git a/scripts/apply-labels.js b/scripts/apply-labels.js new file mode 100644 index 000000000..177822a92 --- /dev/null +++ b/scripts/apply-labels.js @@ -0,0 +1,385 @@ +#!/usr/bin/env node + +import Database from 'better-sqlite3'; +import fetch from 'node-fetch'; +import pLimit from 'p-limit'; +import chalk from 'chalk'; +import { writeFile } from 'fs/promises'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Configuration +const CONFIG = { + API_URL: + process.env.API_ENDPOINT || + process.env.API_URL || + 'http://localhost:4000/api/graphql', + API_KEY: process.env.OMNIVORE_API_KEY, + ARCHIVE_DB_PATH: + process.env.ARCHIVE_DB_PATH || + join(__dirname, '../self-hosting/archive-db/store.sqlite'), + RATE_LIMIT: 3, // concurrent requests + BATCH_SIZE: 10, // items to process per batch +}; + +if (!CONFIG.API_KEY) { + console.error(chalk.red('❌ OMNIVORE_API_KEY environment variable is required')); + console.error( + chalk.gray( + 'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/apply-labels.js', + ), + ); + process.exit(1); +} + +class LabelApplicator { + constructor() { + this.db = null; + this.stats = { + totalItemsWithLabels: 0, + processedItems: 0, + failedItems: [], + successfulItems: 0, + startTime: Date.now(), + }; + this.limit = pLimit(CONFIG.RATE_LIMIT); + this.labelNameToId = new Map(); + this.urlToPageId = new Map(); + } + + // Initialize database connection + initDatabase() { + try { + this.db = new Database(CONFIG.ARCHIVE_DB_PATH, { readonly: true }); + console.log(chalk.green('✓ Connected to SQLite database')); + } catch (error) { + console.error(chalk.red('✗ Failed to connect to database:'), error.message); + process.exit(1); + } + } + + // Make GraphQL request + async makeGraphQLRequest(query, variables = {}) { + const response = await fetch(CONFIG.API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': CONFIG.API_KEY, + }, + body: JSON.stringify({ + query, + variables, + }), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + + if (result.errors) { + throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`); + } + + return result.data; + } + + // Extract items with their labels from SQLite + extractItemLabelAssociations() { + const query = ` + SELECT + li.ZPAGEURLSTRING as url, + li.ZTITLE as title, + GROUP_CONCAT(label.ZNAME, '||') as labelNames + FROM ZLINKEDITEM li + JOIN Z_2LABELS l ON li.Z_PK = l.Z_2LINKEDITEMS + JOIN ZLINKEDITEMLABEL label ON l.Z_3LABELS1 = label.Z_PK + GROUP BY li.ZPAGEURLSTRING, li.ZTITLE + ORDER BY li.ZPAGEURLSTRING + `; + + const results = this.db.prepare(query).all(); + this.stats.totalItemsWithLabels = results.length; + + return results.map(row => ({ + url: row.url, + title: row.title || '', + labels: row.labelNames.split('||').filter(name => name && name.trim()), + })); + } + + // Get all labels from the new Omnivore instance + async fetchLabelsFromOmnivore() { + const query = ` + query { + labels { + ... on LabelsSuccess { + labels { + id + name + color + } + } + ... on LabelsError { + errorCodes + } + } + } + `; + + try { + const data = await this.makeGraphQLRequest(query); + + if (data.labels.errorCodes) { + throw new Error(`Failed to fetch labels: ${data.labels.errorCodes.join(', ')}`); + } + + // Build mapping of label name to ID + data.labels.labels.forEach(label => { + this.labelNameToId.set(label.name, label.id); + }); + + console.log(chalk.green(`✓ Fetched ${data.labels.labels.length} labels from Omnivore`)); + return data.labels.labels; + } catch (error) { + console.error(chalk.red('✗ Failed to fetch labels:'), error.message); + throw error; + } + } + + // Search for an item by URL in the new Omnivore instance + async findItemByUrl(url) { + const query = ` + query SearchByUrl($query: String!) { + search(query: $query, first: 1) { + ... on SearchSuccess { + edges { + node { + id + url + title + labels { + id + name + } + } + } + } + ... on SearchError { + errorCodes + } + } + } + `; + + try { + // Create a search query that should match the exact URL + const searchQuery = `url:"${url}"`; + + const data = await this.makeGraphQLRequest(query, { query: searchQuery }); + + if (data.search.errorCodes) { + throw new Error(`Search failed: ${data.search.errorCodes.join(', ')}`); + } + + const edges = data.search.edges || []; + if (edges.length === 0) { + return null; + } + + const item = edges[0].node; + + // Verify URL match (sometimes search might return similar URLs) + if (item.url === url) { + return { + id: item.id, + url: item.url, + title: item.title, + currentLabels: item.labels || [], + }; + } + + return null; + } catch (error) { + console.error(chalk.yellow(`⚠ Failed to search for URL ${url}:`, error.message)); + return null; + } + } + + // Apply labels to an item using setLabels mutation + async applyLabelsToItem(pageId, labelNames, itemTitle = '') { + // Convert label names to IDs + const labelIds = labelNames + .map(name => this.labelNameToId.get(name)) + .filter(id => id); // Remove any undefined IDs + + if (labelIds.length === 0) { + throw new Error(`No valid labels found for item. Requested: ${labelNames.join(', ')}`); + } + + const mutation = ` + mutation SetLabels($input: SetLabelsInput!) { + setLabels(input: $input) { + ... on SetLabelsSuccess { + labels { + id + name + } + } + ... on SetLabelsError { + errorCodes + } + } + } + `; + + try { + const data = await this.makeGraphQLRequest(mutation, { + input: { + pageId: pageId, + labelIds: labelIds, + }, + }); + + if (data.setLabels.errorCodes) { + throw new Error(`Failed to set labels: ${data.setLabels.errorCodes.join(', ')}`); + } + + return data.setLabels.labels; + } catch (error) { + console.error(chalk.red(`✗ Failed to apply labels to "${itemTitle}":`, error.message)); + throw error; + } + } + + // Process items in batches + async processItems(itemsWithLabels) { + console.log(chalk.blue(`\n🏷️ Processing ${itemsWithLabels.length} items with labels...`)); + + const batches = []; + for (let i = 0; i < itemsWithLabels.length; i += CONFIG.BATCH_SIZE) { + batches.push(itemsWithLabels.slice(i, i + CONFIG.BATCH_SIZE)); + } + + for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { + const batch = batches[batchIndex]; + console.log(chalk.cyan(`\nProcessing batch ${batchIndex + 1}/${batches.length} (${batch.length} items)...`)); + + const promises = batch.map(item => + this.limit(async () => { + try { + // Find the item in the new system + const foundItem = await this.findItemByUrl(item.url); + + if (!foundItem) { + throw new Error(`Item not found in Omnivore for URL: ${item.url}`); + } + + // Apply labels + const appliedLabels = await this.applyLabelsToItem( + foundItem.id, + item.labels, + item.title + ); + + this.stats.successfulItems++; + console.log(chalk.green(`✓ Applied ${appliedLabels.length} labels to: ${item.title || foundItem.title}`)); + + return { success: true, item, appliedLabels }; + } catch (error) { + this.stats.failedItems.push({ item, error: error.message }); + console.error(chalk.red(`✗ Failed to process "${item.title}": ${error.message}`)); + return { success: false, item, error: error.message }; + } + }) + ); + + await Promise.all(promises); + + // Progress update + this.stats.processedItems += batch.length; + const progress = (this.stats.processedItems / this.stats.totalItemsWithLabels * 100).toFixed(1); + console.log(chalk.yellow(`Progress: ${progress}% (${this.stats.processedItems}/${this.stats.totalItemsWithLabels} items)`)); + } + } + + // Generate report + async generateReport() { + const duration = (Date.now() - this.stats.startTime) / 1000; + + const report = { + labelApplication: { + timestamp: new Date().toISOString(), + duration: `${duration.toFixed(2)} seconds`, + }, + summary: { + totalItemsWithLabels: this.stats.totalItemsWithLabels, + processedItems: this.stats.processedItems, + successfulItems: this.stats.successfulItems, + failedItems: this.stats.failedItems.length, + }, + failures: this.stats.failedItems, + }; + + const reportPath = join(__dirname, 'label-application-report.json'); + await writeFile(reportPath, JSON.stringify(report, null, 2)); + + console.log(chalk.blue(`\n📊 Label Application Report:`)); + console.log(chalk.green(`✓ Successfully labeled: ${report.summary.successfulItems}/${report.summary.totalItemsWithLabels} items`)); + + if (report.summary.failedItems > 0) { + console.log(chalk.red(`✗ Failed items: ${report.summary.failedItems}`)); + } + + console.log(chalk.blue(`📄 Full report saved to: ${reportPath}`)); + return report; + } + + // Main process + async applyLabels() { + try { + console.log(chalk.bold.blue('🏷️ Starting Label Application Process')); + console.log(chalk.gray(`API: ${CONFIG.API_URL}`)); + + this.initDatabase(); + + // Extract original label associations + console.log(chalk.blue('\n📤 Extracting label associations from SQLite...')); + const itemsWithLabels = this.extractItemLabelAssociations(); + console.log(chalk.green(`✓ Found ${itemsWithLabels.length} items with labels`)); + + if (itemsWithLabels.length === 0) { + console.log(chalk.yellow('No items with labels found. Nothing to do.')); + return; + } + + // Fetch current labels from Omnivore + console.log(chalk.blue('\n📋 Fetching current labels from Omnivore...')); + await this.fetchLabelsFromOmnivore(); + + // Process items + await this.processItems(itemsWithLabels); + + // Generate report + await this.generateReport(); + + console.log(chalk.bold.green('\n🎉 Label application completed!')); + + } catch (error) { + console.error(chalk.red('\n💥 Label application failed:'), error.message); + process.exit(1); + } finally { + if (this.db) { + this.db.close(); + } + } + } +} + +// Run the label application +const applicator = new LabelApplicator(); +applicator.applyLabels(); diff --git a/scripts/apply-single-label.js b/scripts/apply-single-label.js new file mode 100644 index 000000000..0276ab604 --- /dev/null +++ b/scripts/apply-single-label.js @@ -0,0 +1,164 @@ +#!/usr/bin/env node + +import fetch from 'node-fetch'; +import chalk from 'chalk'; + +const CONFIG = { + API_URL: + process.env.API_ENDPOINT || + process.env.API_URL || + 'http://localhost:4000/api/graphql', + API_KEY: process.env.OMNIVORE_API_KEY, +}; + +function parseArgs(argv) { + const args = {}; + for (let i = 2; i < argv.length; i++) { + const key = argv[i]; + if (!key.startsWith('--')) continue; + const name = key.slice(2); + const value = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true; + args[name] = value; + } + return args; +} + +const args = parseArgs(process.argv); +const pageId = args.pageId || process.env.PAGE_ID; +const labelName = args.label || process.env.LABEL_NAME; + +if (!CONFIG.API_KEY) { + console.error(chalk.red('❌ OMNIVORE_API_KEY environment variable is required')); + console.error( + chalk.gray( + 'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/apply-single-label.js --pageId --label ', + ), + ); + process.exit(1); +} + +if (!pageId || !labelName) { + console.error(chalk.red('❌ Missing required args')); + console.error( + chalk.gray( + 'Usage: node scripts/apply-single-label.js --pageId --label ', + ), + ); + process.exit(1); +} + +// Make GraphQL request +async function makeGraphQLRequest(query, variables = {}) { + const response = await fetch(CONFIG.API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': CONFIG.API_KEY, + }, + body: JSON.stringify({ + query, + variables, + }), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + + if (result.errors) { + throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`); + } + + return result.data; +} + +// Get labels to find artifact label ID +async function fetchLabels() { + const query = ` + query { + labels { + ... on LabelsSuccess { + labels { + id + name + color + } + } + ... on LabelsError { + errorCodes + } + } + } + `; + + const data = await makeGraphQLRequest(query); + + if (data.labels.errorCodes) { + throw new Error(`Failed to fetch labels: ${data.labels.errorCodes.join(', ')}`); + } + + return data.labels.labels; +} + +// Apply labels to an item using setLabels mutation +async function applyLabel(pageId, labelId, labelName) { + const mutation = ` + mutation SetLabels($input: SetLabelsInput!) { + setLabels(input: $input) { + ... on SetLabelsSuccess { + labels { + id + name + } + } + ... on SetLabelsError { + errorCodes + } + } + } + `; + + const data = await makeGraphQLRequest(mutation, { + input: { + pageId: pageId, + labelIds: [labelId], + }, + }); + + if (data.setLabels.errorCodes) { + throw new Error(`Failed to set labels: ${data.setLabels.errorCodes.join(', ')}`); + } + + console.log(chalk.green(`✓ Applied label "${labelName}"`)); + return data.setLabels.labels; +} + +async function main() { + try { + console.log( + chalk.blue(`🏷️ Applying label "${labelName}" to page ${pageId}`), + ); + + // Fetch labels to find artifact label ID + const labels = await fetchLabels(); + const targetLabel = labels.find(l => l.name === labelName); + + if (!targetLabel) { + throw new Error(`Label not found: ${labelName}`); + } + + console.log(chalk.gray(`Found label: ${targetLabel.id}`)); + + await applyLabel(pageId, targetLabel.id, labelName); + + console.log(chalk.bold.green('✅ Label applied successfully!')); + + } catch (error) { + console.error(chalk.red('❌ Failed to apply label:'), error.message); + process.exit(1); + } +} + +main(); diff --git a/scripts/check-missing-labeled.sql b/scripts/check-missing-labeled.sql new file mode 100644 index 000000000..5d56ce5c6 --- /dev/null +++ b/scripts/check-missing-labeled.sql @@ -0,0 +1,33 @@ +-- Check which items with labels from archive are missing in current mappings +.mode column +.headers on + +ATTACH DATABASE 'url-id-mapping.sqlite' AS mapping; +ATTACH DATABASE '../self-hosting/archive-db/store.sqlite' AS archive; + +-- Items with labels that couldn't be found +WITH labeled_items AS ( + SELECT DISTINCT + li.ZPAGEURLSTRING as url, + li.ZTITLE as title, + GROUP_CONCAT(label.ZNAME, ', ') as labels + FROM archive.ZLINKEDITEM li + JOIN archive.Z_2LABELS l ON li.Z_PK = l.Z_2LINKEDITEMS + JOIN archive.ZLINKEDITEMLABEL label ON l.Z_3LABELS1 = label.Z_PK + WHERE li.ZPAGEURLSTRING IN ( + 'https://github.com/danswer-ai/danswer', + 'https://segment.com/blog/rebuilding-our-infrastructure/', + 'https://svgl.vercel.app', + 'https://www.hyperledger.org' + ) + GROUP BY li.ZPAGEURLSTRING, li.ZTITLE +) +SELECT + li.url as original_url, + li.title, + li.labels, + m.url as mapped_url, + m.id as mapped_id +FROM labeled_items li +LEFT JOIN mapping.item_mapping m ON li.url = m.url +ORDER BY li.url; \ No newline at end of file diff --git a/scripts/compare-urls.js b/scripts/compare-urls.js new file mode 100644 index 000000000..e296b8739 --- /dev/null +++ b/scripts/compare-urls.js @@ -0,0 +1,337 @@ +#!/usr/bin/env node + +import { readFileSync, writeFileSync } from 'fs'; +import { parse } from 'csv-parse/sync'; +import fetch from 'node-fetch'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Configuration +const CONFIG = { + API_URL: + process.env.API_ENDPOINT || + process.env.API_URL || + 'http://localhost:4000/api/graphql', + API_KEY: process.env.OMNIVORE_API_KEY, + CSV_PATH: join(process.env.HOME, 'Downloads', 'part_000000.csv'), + BATCH_SIZE: 100, // GraphQL pagination size + OUTPUT_DIR: __dirname, +}; + +if (!CONFIG.API_KEY) { + throw new Error('OMNIVORE_API_KEY environment variable is not set'); +} + +// GraphQL query to fetch all URLs +const FETCH_URLS_QUERY = ` + query FetchUrls($after: String, $first: Int) { + search( + query: "in:all sort:saved-desc" + after: $after + first: $first + ) { + ... on SearchSuccess { + pageInfo { + totalCount + hasNextPage + endCursor + } + edges { + cursor + node { + id + title + url + originalArticleUrl + createdAt + updatedAt + } + } + } + ... on SearchError { + errorCodes + } + } + } +`; + +// Fetch all URLs from Omnivore with pagination +async function fetchAllOmnivoreUrls() { + const allUrls = []; + let hasNextPage = true; + let after = null; + let pageCount = 0; + + console.log('🔍 Fetching all URLs from Omnivore...'); + + while (hasNextPage) { + pageCount++; + console.log(`📄 Fetching page ${pageCount}...`); + + try { + const response = await fetch(CONFIG.API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': CONFIG.API_KEY, + }, + body: JSON.stringify({ + query: FETCH_URLS_QUERY, + variables: { + after, + first: CONFIG.BATCH_SIZE, + }, + }), + }); + + const result = await response.json(); + + if (result.errors) { + throw new Error(`GraphQL errors: ${JSON.stringify(result.errors)}`); + } + + const searchResult = result.data.search; + + if (searchResult.errorCodes) { + throw new Error(`Search error: ${searchResult.errorCodes.join(', ')}`); + } + + const items = searchResult.edges || []; + console.log(` Found ${items.length} items on page ${pageCount}`); + + // Extract URLs and metadata + for (const edge of items) { + const item = edge.node; + allUrls.push({ + id: item.id, + title: item.title, + url: item.url, + originalUrl: item.originalArticleUrl || item.url, + createdAt: item.createdAt, + updatedAt: item.updatedAt, + }); + } + + hasNextPage = searchResult.pageInfo.hasNextPage; + after = searchResult.pageInfo.endCursor; + + if (pageCount === 1) { + console.log(`📊 Total items in Omnivore: ${searchResult.pageInfo.totalCount}`); + } + + // Add delay to be nice to the API + await new Promise(resolve => setTimeout(resolve, 100)); + + } catch (error) { + console.error(`❌ Error fetching page ${pageCount}:`, error.message); + throw error; + } + } + + console.log(`✅ Fetched ${allUrls.length} URLs from Omnivore`); + return allUrls; +} + +// Parse CSV file and extract URLs +function parseCSVUrls() { + console.log('📋 Reading CSV file...'); + + try { + const csvContent = readFileSync(CONFIG.CSV_PATH, 'utf-8'); + const records = parse(csvContent, { + columns: true, + skip_empty_lines: true, + trim: true, + }); + + console.log(`✅ Found ${records.length} items in CSV`); + + return records.map(record => ({ + title: record.title || 'No Title', + url: record.resolved_url || record.given_url || record.url || '', + givenUrl: record.given_url || '', + resolvedUrl: record.resolved_url || '', + timeAdded: record.time_added ? new Date(parseInt(record.time_added) * 1000).toISOString() : null, + tags: record.tags || '', + })); + + } catch (error) { + console.error('❌ Error reading CSV file:', error.message); + throw error; + } +} + +// Normalize URL for comparison (remove protocol, www, trailing slash, fragments) +function normalizeUrl(url) { + if (!url) return ''; + + return url + .toLowerCase() + .replace(/^https?:\/\//, '') // Remove protocol + .replace(/^www\./, '') // Remove www + .replace(/\/$/, '') // Remove trailing slash + .replace(/#.*$/, '') // Remove fragments + .replace(/\?.*$/, ''); // Remove query parameters (optional - comment out to keep params) +} + +// Compare URLs +function compareUrls(omnivoreUrls, csvUrls) { + console.log('🔍 Comparing URLs...'); + + // Create normalized lookup sets + const omnivoreNormalized = new Map(); + const csvNormalized = new Map(); + + // Build Omnivore lookup + for (const item of omnivoreUrls) { + const normalized = normalizeUrl(item.url); + const originalNormalized = normalizeUrl(item.originalUrl); + + if (normalized) { + omnivoreNormalized.set(normalized, item); + } + if (originalNormalized && originalNormalized !== normalized) { + omnivoreNormalized.set(originalNormalized, item); + } + } + + // Build CSV lookup + for (const item of csvUrls) { + const normalizedUrl = normalizeUrl(item.url); + const normalizedGiven = normalizeUrl(item.givenUrl); + const normalizedResolved = normalizeUrl(item.resolvedUrl); + + if (normalizedUrl) { + csvNormalized.set(normalizedUrl, item); + } + if (normalizedGiven && normalizedGiven !== normalizedUrl) { + csvNormalized.set(normalizedGiven, item); + } + if (normalizedResolved && normalizedResolved !== normalizedUrl && normalizedResolved !== normalizedGiven) { + csvNormalized.set(normalizedResolved, item); + } + } + + // Find matches and differences + const inBoth = []; + const onlyInOmnivore = []; + const onlyInCSV = []; + + // Check Omnivore items + for (const [normalizedUrl, omnivoreItem] of omnivoreNormalized) { + if (csvNormalized.has(normalizedUrl)) { + inBoth.push({ + url: normalizedUrl, + omnivore: omnivoreItem, + csv: csvNormalized.get(normalizedUrl), + }); + } else { + onlyInOmnivore.push(omnivoreItem); + } + } + + // Check CSV items not in Omnivore + for (const [normalizedUrl, csvItem] of csvNormalized) { + if (!omnivoreNormalized.has(normalizedUrl)) { + onlyInCSV.push(csvItem); + } + } + + return { + inBoth, + onlyInOmnivore, + onlyInCSV, + stats: { + omnivoreTotal: omnivoreUrls.length, + csvTotal: csvUrls.length, + matches: inBoth.length, + omnivoreOnly: onlyInOmnivore.length, + csvOnly: onlyInCSV.length, + }, + }; +} + +// Generate comparison report +function generateReport(comparison) { + const { inBoth, onlyInOmnivore, onlyInCSV, stats } = comparison; + + console.log('\n📊 COMPARISON REPORT'); + console.log('=================='); + console.log(`📚 Total in Omnivore: ${stats.omnivoreTotal}`); + console.log(`📋 Total in CSV: ${stats.csvTotal}`); + console.log(`✅ URLs in both: ${stats.matches}`); + console.log(`🔵 Only in Omnivore: ${stats.omnivoreOnly}`); + console.log(`🔴 Only in CSV (missing from Omnivore): ${stats.csvOnly}`); + console.log(`📈 Import success rate: ${((stats.matches / stats.csvTotal) * 100).toFixed(1)}%`); + + // Save detailed reports + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + + // Save URLs only in CSV (missing from Omnivore) + if (onlyInCSV.length > 0) { + const missingFile = join(CONFIG.OUTPUT_DIR, `missing-from-omnivore-${timestamp}.json`); + writeFileSync(missingFile, JSON.stringify(onlyInCSV, null, 2)); + console.log(`\n💾 Saved ${onlyInCSV.length} missing URLs to: ${missingFile}`); + + // Save a simple list for easier processing + const missingList = join(CONFIG.OUTPUT_DIR, `missing-urls-list-${timestamp}.txt`); + const missingUrlsList = onlyInCSV.map(item => item.url).join('\n'); + writeFileSync(missingList, missingUrlsList); + console.log(`💾 Saved missing URLs list to: ${missingList}`); + } + + // Save full comparison report + const reportFile = join(CONFIG.OUTPUT_DIR, `url-comparison-report-${timestamp}.json`); + writeFileSync(reportFile, JSON.stringify({ + timestamp: new Date().toISOString(), + stats, + inBoth: inBoth.slice(0, 10), // First 10 matches as examples + onlyInOmnivore: onlyInOmnivore.slice(0, 10), // First 10 examples + onlyInCSV: onlyInCSV.slice(0, 100), // First 100 missing items + fullStats: { + totalMatches: inBoth.length, + totalOmnivoreOnly: onlyInOmnivore.length, + totalCSVOnly: onlyInCSV.length, + }, + }, null, 2)); + console.log(`💾 Saved full comparison report to: ${reportFile}`); + + return comparison; +} + +// Main function +async function main() { + try { + console.log('🚀 Starting URL comparison...'); + + if (!CONFIG.API_KEY) { + throw new Error('OMNIVORE_API_KEY environment variable is not set'); + } + + // Fetch all URLs from both sources + const [omnivoreUrls, csvUrls] = await Promise.all([ + fetchAllOmnivoreUrls(), + Promise.resolve(parseCSVUrls()), + ]); + + // Compare the URLs + const comparison = compareUrls(omnivoreUrls, csvUrls); + + // Generate and save report + generateReport(comparison); + + console.log('\n✅ Comparison complete!'); + + } catch (error) { + console.error('\n❌ Error during comparison:', error.message); + process.exit(1); + } +} + +// Run the script +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/scripts/download-items-mapping.js b/scripts/download-items-mapping.js new file mode 100755 index 000000000..96fdc584d --- /dev/null +++ b/scripts/download-items-mapping.js @@ -0,0 +1,224 @@ +#!/usr/bin/env node + +import Database from 'better-sqlite3'; +import fetch from 'node-fetch'; +import chalk from 'chalk'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Configuration +const CONFIG = { + API_URL: + process.env.API_ENDPOINT || + process.env.API_URL || + 'http://localhost:4000/api/graphql', + API_KEY: process.env.OMNIVORE_API_KEY, + MAPPING_DB_PATH: + process.env.MAPPING_DB_PATH || join(__dirname, 'url-id-mapping.sqlite'), + BATCH_SIZE: 100, // GraphQL allows up to 100 items per request +}; + +if (!CONFIG.API_KEY) { + console.error(chalk.red('❌ OMNIVORE_API_KEY environment variable is required')); + console.error( + chalk.gray( + 'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/download-items-mapping.js', + ), + ); + process.exit(1); +} + +class ItemsDownloader { + constructor() { + this.db = null; + this.stats = { + totalItems: 0, + downloadedItems: 0, + startTime: Date.now(), + }; + } + + // Initialize mapping database + initMappingDatabase() { + try { + this.db = new Database(CONFIG.MAPPING_DB_PATH); + console.log(chalk.green('✓ Created mapping database')); + + // Create mapping table + this.db.exec(` + CREATE TABLE IF NOT EXISTS item_mapping ( + id TEXT PRIMARY KEY, + url TEXT NOT NULL, + title TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_url ON item_mapping(url); + `); + + // Clear existing data for fresh download + this.db.exec('DELETE FROM item_mapping'); + console.log(chalk.blue('✓ Initialized item_mapping table')); + } catch (error) { + console.error(chalk.red('✗ Failed to initialize database:'), error.message); + process.exit(1); + } + } + + // Make GraphQL request + async makeGraphQLRequest(query, variables = {}) { + const response = await fetch(CONFIG.API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': CONFIG.API_KEY, + }, + body: JSON.stringify({ + query, + variables, + }), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + + if (result.errors) { + throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`); + } + + return result.data; + } + + // Download all items with pagination + async downloadAllItems() { + const query = ` + query GetItems($after: String, $first: Int) { + search(query: "", after: $after, first: $first) { + ... on SearchSuccess { + edges { + cursor + node { + id + url + title + } + } + pageInfo { + hasNextPage + endCursor + } + } + ... on SearchError { + errorCodes + } + } + } + `; + + let hasNextPage = true; + let cursor = null; + let pageCount = 0; + + while (hasNextPage) { + try { + const data = await this.makeGraphQLRequest(query, { + after: cursor, + first: CONFIG.BATCH_SIZE, + }); + + if (data.search.errorCodes) { + throw new Error(`Search failed: ${data.search.errorCodes.join(', ')}`); + } + + const { edges, pageInfo } = data.search; + + // Insert items into database + const insertStmt = this.db.prepare( + 'INSERT OR REPLACE INTO item_mapping (id, url, title) VALUES (?, ?, ?)' + ); + + const insertMany = this.db.transaction((items) => { + for (const item of items) { + insertStmt.run(item.node.id, item.node.url, item.node.title); + } + }); + + insertMany(edges); + + this.stats.downloadedItems += edges.length; + pageCount++; + + console.log(chalk.green( + `✓ Downloaded page ${pageCount}: ${edges.length} items (Total: ${this.stats.downloadedItems})` + )); + + hasNextPage = pageInfo.hasNextPage; + cursor = pageInfo.endCursor; + + // Small delay to avoid rate limiting + if (hasNextPage) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + } catch (error) { + console.error(chalk.red(`✗ Error downloading page ${pageCount + 1}:`), error.message); + throw error; + } + } + } + + // Generate report + async generateReport() { + const duration = (Date.now() - this.stats.startTime) / 1000; + + // Get stats from database + const totalCount = this.db.prepare('SELECT COUNT(*) as count FROM item_mapping').get().count; + const sampleItems = this.db.prepare('SELECT * FROM item_mapping LIMIT 5').all(); + + console.log(chalk.blue(`\n📊 Download Report:`)); + console.log(chalk.green(`✓ Total items downloaded: ${totalCount}`)); + console.log(chalk.gray(`⏱️ Duration: ${duration.toFixed(2)} seconds`)); + console.log(chalk.blue(`\n📄 Sample items:`)); + + sampleItems.forEach((item, index) => { + console.log(chalk.gray(`${index + 1}. ${item.title || 'No title'}`)); + console.log(chalk.gray(` ID: ${item.id}`)); + console.log(chalk.gray(` URL: ${item.url}\n`)); + }); + + console.log(chalk.green(`✓ Mapping database saved to: ${CONFIG.MAPPING_DB_PATH}`)); + } + + // Main process + async download() { + try { + console.log(chalk.bold.blue('📥 Starting Items Download Process')); + console.log(chalk.gray(`API: ${CONFIG.API_URL}`)); + + this.initMappingDatabase(); + + console.log(chalk.blue('\n📤 Downloading all items from Omnivore...')); + await this.downloadAllItems(); + + await this.generateReport(); + + console.log(chalk.bold.green('\n🎉 Download completed successfully!')); + + } catch (error) { + console.error(chalk.red('\n💥 Download failed:'), error.message); + process.exit(1); + } finally { + if (this.db) { + this.db.close(); + } + } + } +} + +// Run the download +const downloader = new ItemsDownloader(); +downloader.download(); diff --git a/scripts/find-redirected-urls.sql b/scripts/find-redirected-urls.sql new file mode 100644 index 000000000..6f0d53413 --- /dev/null +++ b/scripts/find-redirected-urls.sql @@ -0,0 +1,24 @@ +-- Search for partial matches to find redirected URLs +.mode column +.headers on + +ATTACH DATABASE 'url-id-mapping.sqlite' AS mapping; + +-- Search for potential matches +SELECT + url, + title, + id +FROM mapping.item_mapping +WHERE + url LIKE '%danswer%' + OR url LIKE '%segment.com%' + OR url LIKE '%svgl%' + OR url LIKE '%hyperledger%' + OR title LIKE '%danswer%' + OR title LIKE '%Segment%' + OR title LIKE '%Svgl%' + OR title LIKE '%SVG%' + OR title LIKE '%Hyperledger%' +ORDER BY url +LIMIT 20; \ No newline at end of file diff --git a/scripts/import-pocket.js b/scripts/import-pocket.js new file mode 100644 index 000000000..c65dc51bb --- /dev/null +++ b/scripts/import-pocket.js @@ -0,0 +1,1268 @@ +#!/usr/bin/env node + +import { readFileSync } from 'fs'; +import { parse } from 'csv-parse/sync'; +import Database from 'better-sqlite3'; +import fetch from 'node-fetch'; +import pLimit from 'p-limit'; +import chalk from 'chalk'; +import { writeFile } from 'fs/promises'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { randomUUID } from 'crypto'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Parse command line arguments +function getArgValue(argName) { + const argIndex = process.argv.indexOf(argName); + return argIndex !== -1 && argIndex + 1 < process.argv.length ? process.argv[argIndex + 1] : null; +} + +// Configuration +const CONFIG = { + API_URL: + process.env.API_ENDPOINT || + process.env.API_URL || + 'http://localhost:4000/api/graphql', + API_KEY: process.env.OMNIVORE_API_KEY, + CSV_PATH: getArgValue('--file') || join(dirname(__dirname), 'Downloads', 'part_000000.csv'), + TRACKING_DB_PATH: join(__dirname, 'pocket-import-progress.sqlite'), + BATCH_SIZE: 3, // Reduced to prevent server crashes + RATE_LIMIT: 2, // concurrent requests + MAX_PROCESSING_ITEMS: 3, // Never exceed this many items in PROCESSING state + STATUS_CHECK_INTERVAL: 10000, // Check status every 10 seconds + PROCESSING_TIMEOUT: 300000, // 5 minutes timeout for stuck items + TEST_MODE: process.argv.includes('--test'), + TEST_LIMIT: parseInt(getArgValue('--limit')) || 10, // items to process in test mode + DRY_RUN: process.argv.includes('--dry-run'), + COMPARE_MODE: process.argv.includes('--compare'), + MISSING_ONLY: process.argv.includes('--missing-only'), +}; + +if (!CONFIG.API_KEY) { + console.error(chalk.red('❌ OMNIVORE_API_KEY environment variable is required')); + console.error( + chalk.gray( + 'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/import-pocket.js --file ~/Downloads/part_000000.csv', + ), + ); + process.exit(1); +} + +// Processing Queue Manager +class ProcessingQueue { + constructor() { + this.items = new Map(); // url -> {item, startTime, checkCount} + this.maxSize = CONFIG.MAX_PROCESSING_ITEMS; + } + + add(item, url) { + if (this.items.size >= this.maxSize) { + throw new Error(`Processing queue full (${this.maxSize} items)`); + } + this.items.set(url, { + item, + startTime: Date.now(), + checkCount: 0 + }); + } + + remove(url) { + return this.items.delete(url); + } + + size() { + return this.items.size; + } + + isFull() { + return this.items.size >= this.maxSize; + } + + getTimeouts() { + const now = Date.now(); + const timeouts = []; + for (const [url, data] of this.items) { + if (now - data.startTime > CONFIG.PROCESSING_TIMEOUT) { + timeouts.push({ url, ...data }); + } + } + return timeouts; + } + + getAll() { + return Array.from(this.items.entries()).map(([url, data]) => ({ url, ...data })); + } + + async waitForSlot(checkInterval = 2000) { + while (this.isFull()) { + console.log(chalk.yellow(`⏳ Processing queue full (${this.size()}/${this.maxSize}), waiting...`)); + await new Promise(resolve => setTimeout(resolve, checkInterval)); + } + } +} + +class PocketImporter { + constructor() { + this.progressDb = null; + this.stats = { + totalItems: 0, + processedItems: 0, + successfulItems: 0, + failedItems: [], + skippedItems: 0, + timeoutItems: 0, + startTime: Date.now(), + }; + this.limit = pLimit(CONFIG.RATE_LIMIT); + this.processingQueue = new ProcessingQueue(); + this.statusMonitorActive = false; + } + + // Convert Unix timestamp to ISO string + convertPocketTimestamp(unixTimestamp) { + if (!unixTimestamp) return new Date().toISOString(); + return new Date(parseInt(unixTimestamp) * 1000).toISOString(); + } + + // Process tags string into array + processTags(tagString) { + if (!tagString || tagString.trim() === '') return []; + return tagString.split(',') + .map(tag => tag.trim()) + .filter(tag => tag.length > 0); + } + + // Map Pocket status to Omnivore folder + mapStatus(status) { + return status === 'archive' ? 'archive' : 'inbox'; + } + + // Initialize progress tracking database + initProgressDb() { + try { + this.progressDb = new Database(CONFIG.TRACKING_DB_PATH); + + // Create table to track processed URLs + this.progressDb.exec(` + CREATE TABLE IF NOT EXISTS processed_urls ( + url TEXT PRIMARY KEY, + pocket_title TEXT, + success INTEGER, + error_message TEXT, + processed_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + `); + + console.log(chalk.green('✓ Progress tracking database initialized')); + } catch (error) { + console.error(chalk.red('✗ Failed to initialize progress database:'), error.message); + process.exit(1); + } + } + + // Check if URL was already processed + isAlreadyProcessed(url) { + const stmt = this.progressDb.prepare('SELECT success FROM processed_urls WHERE url = ?'); + const result = stmt.get(url); + return result?.success === 1; + } + + // Mark URL as processed + markAsProcessed(url, title, success, errorMessage = null) { + const stmt = this.progressDb.prepare(` + INSERT OR REPLACE INTO processed_urls (url, pocket_title, success, error_message) + VALUES (?, ?, ?, ?) + `); + stmt.run(url, title, success ? 1 : 0, errorMessage); + } + + // Parse Pocket CSV export + parsePocketExport() { + try { + console.log(chalk.blue(`📋 Reading Pocket export from: ${CONFIG.CSV_PATH}`)); + + const csvContent = readFileSync(CONFIG.CSV_PATH, 'utf-8'); + const records = parse(csvContent, { + columns: true, + skip_empty_lines: true, + trim: true, + }); + + this.stats.totalItems = records.length; + console.log(chalk.green(`✓ Loaded ${records.length} items from Pocket export`)); + + // Transform Pocket data to our format + return records.map(record => ({ + url: record.url, + title: record.title || 'Untitled', + tags: this.processTags(record.tags), + savedAt: this.convertPocketTimestamp(record.time_added), + folder: this.mapStatus(record.status), + source: 'pocket-import', + })); + } catch (error) { + console.error(chalk.red('✗ Failed to parse Pocket export:'), error.message); + process.exit(1); + } + } + + // Make GraphQL request to Omnivore API + async makeGraphQLRequest(query, variables = {}) { + const response = await fetch(CONFIG.API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': CONFIG.API_KEY, + }, + body: JSON.stringify({ query, variables }), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + + // Log the full response for debugging + if (result.errors || !result.data) { + console.log(chalk.yellow('GraphQL Response:'), JSON.stringify(result, null, 2)); + } + + if (result.errors) { + throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`); + } + + return result.data; + } + + // Test authentication with the API key + async testAuthentication() { + try { + const query = ` + query { + me { + id + name + email + } + } + `; + + const data = await this.makeGraphQLRequest(query); + if (data.me) { + console.log(chalk.green(`✓ Authentication successful for user: ${data.me.name} (${data.me.email})`)); + return true; + } else { + console.log(chalk.red('✗ Authentication failed: No user returned')); + return false; + } + } catch (error) { + console.log(chalk.red('✗ Authentication failed:', error.message)); + return false; + } + } + + // Check status of a specific item by URL + async checkItemStatus(url) { + const query = ` + query SearchByUrl($query: String!) { + search(first: 1, query: $query) { + ... on SearchSuccess { + edges { + node { + id + title + url + state + savedAt + originalArticleUrl + } + } + } + ... on SearchError { + errorCodes + } + } + } + `; + + try { + // Search for exact URL match + const searchQuery = `url:"${url}"`; + const data = await this.makeGraphQLRequest(query, { query: searchQuery }); + + if (data.search.errorCodes) { + return { error: data.search.errorCodes.join(', ') }; + } + + if (data.search.edges && data.search.edges.length > 0) { + const item = data.search.edges[0].node; + return { + found: true, + state: item.state, + id: item.id, + title: item.title + }; + } + + return { found: false }; + } catch (error) { + return { error: error.message }; + } + } + + // Monitor processing items until they reach SUCCEEDED or timeout + async startStatusMonitoring() { + if (this.statusMonitorActive) return; + this.statusMonitorActive = true; + + const monitorLoop = async () => { + while (this.statusMonitorActive) { + try { + // Check for timeouts first + const timeouts = this.processingQueue.getTimeouts(); + for (const timeout of timeouts) { + console.log(chalk.red(`⏰ Timeout: ${timeout.item.title} (${((Date.now() - timeout.startTime) / 1000).toFixed(0)}s)`)); + this.processingQueue.remove(timeout.url); + this.stats.timeoutItems++; + this.markAsProcessed(timeout.url, timeout.item.title, false, 'Processing timeout'); + } + + // Check status of all processing items + const processingItems = this.processingQueue.getAll(); + + for (const processingItem of processingItems) { + const { url, item } = processingItem; + const statusResult = await this.checkItemStatus(url); + + processingItem.checkCount++; + + if (statusResult.found && statusResult.state === 'SUCCEEDED') { + console.log(chalk.green(`✅ Completed: ${item.title} (${processingItem.checkCount} checks)`)); + this.processingQueue.remove(url); + this.stats.successfulItems++; + this.markAsProcessed(url, item.title, true); + } else if (statusResult.found && statusResult.state === 'FAILED') { + console.log(chalk.red(`❌ Failed processing: ${item.title}`)); + this.processingQueue.remove(url); + this.stats.failedItems.push({ item, error: 'Processing failed in Omnivore' }); + this.markAsProcessed(url, item.title, false, 'Processing failed'); + } else if (statusResult.error) { + console.log(chalk.yellow(`⚠ Status check error for ${item.title}: ${statusResult.error}`)); + } + // If still PROCESSING, continue monitoring + } + + // Show queue status periodically + if (this.processingQueue.size() > 0) { + console.log(chalk.cyan(`🔄 Queue: ${this.processingQueue.size()}/${CONFIG.MAX_PROCESSING_ITEMS} processing`)); + } + + } catch (error) { + console.error(chalk.red('Status monitoring error:'), error.message); + } + + await new Promise(resolve => setTimeout(resolve, CONFIG.STATUS_CHECK_INTERVAL)); + } + }; + + // Start monitoring in background + monitorLoop().catch(error => { + console.error(chalk.red('Status monitoring crashed:'), error.message); + this.statusMonitorActive = false; + }); + } + + // Stop status monitoring + stopStatusMonitoring() { + this.statusMonitorActive = false; + } + + // Import a single item from Pocket with queue management + async importItem(item) { + if (this.isAlreadyProcessed(item.url)) { + this.stats.skippedItems++; + console.log(chalk.yellow(`⚠ Skipping already processed: ${item.title}`)); + return { skipped: true }; + } + + // Wait for queue slot if needed + if (this.processingQueue.isFull()) { + await this.processingQueue.waitForSlot(); + } + + const mutation = ` + mutation SaveUrl($input: SaveUrlInput!) { + saveUrl(input: $input) { + ... on SaveSuccess { + url + clientRequestId + } + ... on SaveError { + errorCodes + } + } + } + `; + + try { + // Convert tags to label inputs + const labelInputs = item.tags.map(tag => ({ name: tag })); + + const input = { + url: item.url, + source: item.source, + clientRequestId: randomUUID(), + savedAt: item.savedAt, + folder: item.folder, + }; + + // Only add labels if we have any + if (labelInputs.length > 0) { + input.labels = labelInputs; + } + + if (CONFIG.DRY_RUN) { + console.log(chalk.blue(`[DRY RUN] Would import: ${item.title}`)); + console.log(chalk.gray(` URL: ${item.url}`)); + console.log(chalk.gray(` Saved: ${item.savedAt}`)); + console.log(chalk.gray(` Tags: ${item.tags.join(', ')}`)); + console.log(chalk.gray(` Folder: ${item.folder}`)); + return { success: true, dryRun: true }; + } + + const data = await this.makeGraphQLRequest(mutation, { input }); + + // Check if the response has the expected structure + if (!data || !data.saveUrl) { + console.error(chalk.red('Unexpected GraphQL response:'), JSON.stringify(data, null, 2)); + throw new Error('Invalid GraphQL response structure'); + } + + if (data.saveUrl.errorCodes) { + throw new Error(`Failed to save item: ${data.saveUrl.errorCodes.join(', ')}`); + } + + // Check if we got a successful response + if (!data.saveUrl.url) { + console.error(chalk.red('GraphQL saveUrl response:'), JSON.stringify(data.saveUrl, null, 2)); + throw new Error('No URL returned from saveUrl mutation'); + } + + // Add to processing queue for monitoring (don't mark as processed yet) + try { + this.processingQueue.add(item, item.url); + console.log(chalk.blue(`📥 Queued for processing: ${item.title} [Queue: ${this.processingQueue.size()}/${CONFIG.MAX_PROCESSING_ITEMS}]`)); + } catch (queueError) { + console.error(chalk.red('Failed to add to processing queue:'), queueError.message); + // Fallback: mark as successful immediately + this.markAsProcessed(item.url, item.title, true); + this.stats.successfulItems++; + } + + return { success: true, url: data.saveUrl.url, queued: true }; + + } catch (error) { + console.error(chalk.red(`✗ Failed to import "${item.title}":`, error.message)); + this.markAsProcessed(item.url, item.title, false, error.message); + this.stats.failedItems.push({ item, error: error.message }); + return { success: false, error: error.message }; + } + } + + // Import items in batches with monitoring + async importItems(items) { + const totalItems = (CONFIG.TEST_MODE || CONFIG.DRY_RUN) ? Math.min(items.length, CONFIG.TEST_LIMIT) : items.length; + const itemsToProcess = items.slice(0, totalItems); + + console.log(chalk.blue(`\n📚 Importing ${totalItems} items from Pocket with queue monitoring...`)); + console.log(chalk.gray(`Max concurrent processing: ${CONFIG.MAX_PROCESSING_ITEMS}`)); + console.log(chalk.gray(`Processing timeout: ${CONFIG.PROCESSING_TIMEOUT / 1000}s`)); + + if (CONFIG.DRY_RUN) { + console.log(chalk.yellow('🏃 Running in DRY RUN mode - no items will be actually imported')); + return; // Skip monitoring for dry run + } + + // Start status monitoring + await this.startStatusMonitoring(); + + const batches = []; + for (let i = 0; i < itemsToProcess.length; i += CONFIG.BATCH_SIZE) { + batches.push(itemsToProcess.slice(i, i + CONFIG.BATCH_SIZE)); + } + + for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { + const batch = batches[batchIndex]; + console.log(chalk.blue(`\n📦 Processing batch ${batchIndex + 1}/${batches.length} (${batch.length} items)`)); + + // Process batch with concurrency limit + const promises = batch.map(item => + this.limit(async () => { + const result = await this.importItem(item); + this.stats.processedItems++; + + const progress = ((this.stats.processedItems / totalItems) * 100).toFixed(1); + + if (result.skipped) { + console.log(chalk.yellow(`[${progress}%] ⚠ Skipped: ${item.title}`)); + } else if (result.success) { + if (result.dryRun) { + console.log(chalk.green(`[${progress}%] ✓ Would import: ${item.title}`)); + } else if (result.queued) { + console.log(chalk.blue(`[${progress}%] 📤 Submitted: ${item.title}`)); + } else { + console.log(chalk.green(`[${progress}%] ✓ Imported: ${item.title}`)); + } + } else { + console.log(chalk.red(`[${progress}%] ✗ Failed: ${item.title}`)); + } + + return result; + }) + ); + + await Promise.all(promises); + + // Show current queue status + console.log(chalk.cyan(`Batch ${batchIndex + 1} complete. Processing queue: ${this.processingQueue.size()}/${CONFIG.MAX_PROCESSING_ITEMS}`)); + + // Small delay between batches to be respectful + if (batchIndex < batches.length - 1) { + await new Promise(resolve => setTimeout(resolve, 1000)); + } + } + + // Wait for all processing items to complete + console.log(chalk.blue('\n⏳ Waiting for all items to finish processing...')); + while (this.processingQueue.size() > 0) { + console.log(chalk.yellow(`Waiting for ${this.processingQueue.size()} items to complete processing...`)); + await new Promise(resolve => setTimeout(resolve, 5000)); + } + + // Stop monitoring + this.stopStatusMonitoring(); + console.log(chalk.green('✅ All items processed!')); + } + + // Generate and save import report + async generateReport() { + const duration = ((Date.now() - this.stats.startTime) / 1000).toFixed(2); + const successRate = ((this.stats.successfulItems / (this.stats.processedItems - this.stats.skippedItems)) * 100).toFixed(1); + + const report = { + timestamp: new Date().toISOString(), + duration: `${duration}s`, + totalItems: this.stats.totalItems, + processedItems: this.stats.processedItems, + successfulItems: this.stats.successfulItems, + skippedItems: this.stats.skippedItems, + failedItems: this.stats.failedItems.length, + timeoutItems: this.stats.timeoutItems, + successRate: `${successRate}%`, + errors: this.stats.failedItems, + queueConfig: { + maxProcessingItems: CONFIG.MAX_PROCESSING_ITEMS, + processingTimeout: `${CONFIG.PROCESSING_TIMEOUT / 1000}s`, + statusCheckInterval: `${CONFIG.STATUS_CHECK_INTERVAL / 1000}s`, + }, + config: { + testMode: CONFIG.TEST_MODE, + dryRun: CONFIG.DRY_RUN, + batchSize: CONFIG.BATCH_SIZE, + rateLimit: CONFIG.RATE_LIMIT, + }, + }; + + await writeFile( + join(__dirname, `pocket-import-report-${Date.now()}.json`), + JSON.stringify(report, null, 2) + ); + + console.log(chalk.blue('\n📊 Import Summary:')); + console.log(chalk.green(`✓ Successfully imported: ${this.stats.successfulItems}`)); + console.log(chalk.yellow(`⚠ Skipped (already processed): ${this.stats.skippedItems}`)); + console.log(chalk.red(`✗ Failed: ${this.stats.failedItems.length}`)); + console.log(chalk.magenta(`⏰ Timed out: ${this.stats.timeoutItems}`)); + console.log(chalk.blue(`📈 Success rate: ${successRate}%`)); + console.log(chalk.blue(`⏱ Duration: ${duration}s`)); + console.log(chalk.gray(`🔧 Queue management: Max ${CONFIG.MAX_PROCESSING_ITEMS} concurrent, ${CONFIG.PROCESSING_TIMEOUT / 1000}s timeout`)); + + if (this.stats.failedItems.length > 0) { + console.log(chalk.red('\n❌ Failed items:')); + this.stats.failedItems.forEach(failure => { + console.log(chalk.red(` • ${failure.item.title}: ${failure.error}`)); + }); + } + } + + // Check status of imported items via API + async checkImportedItemsStatus() { + console.log(chalk.blue('\n🔍 Checking status of imported items...')); + + const query = ` + query Search($after: String, $first: Int, $query: String) { + search(after: $after, first: $first, query: $query) { + ... on SearchSuccess { + edges { + node { + id + title + url + state + savedAt + readingProgressPercent + labels { + name + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + ... on SearchError { + errorCodes + } + } + } + `; + + try { + // Get items imported from pocket (using source identifier) + const data = await this.makeGraphQLRequest(query, { + first: 100, + query: 'source:pocket-import' + }); + + // Check if we got an error response + if (data.search.errorCodes) { + console.error(chalk.red('✗ Search API error:'), data.search.errorCodes.join(', ')); + return []; + } + + // Access edges from the SearchSuccess type + if (data.search.edges && data.search.edges.length > 0) { + console.log(chalk.green(`✓ Found ${data.search.edges.length} imported items via API`)); + + const statusCounts = {}; + data.search.edges.forEach(edge => { + const state = edge.node.state; + statusCounts[state] = (statusCounts[state] || 0) + 1; + }); + + console.log(chalk.blue('📊 Status breakdown via API:')); + Object.entries(statusCounts).forEach(([state, count]) => { + console.log(chalk.gray(` ${state}: ${count}`)); + }); + + return data.search.edges.map(edge => edge.node); + } else { + console.log(chalk.yellow('⚠ No items found via API search')); + return []; + } + } catch (error) { + console.error(chalk.red('✗ Failed to check status via API:'), error.message); + return []; + } + } + + // Check status via direct database query using Docker + async checkImportedItemsStatusViaDb() { + console.log(chalk.blue('\n🗄️ Checking status via database...')); + + try { + // Query the database directly via Docker + const { execSync } = await import('child_process'); + + // Get list of successfully processed URLs from local tracking database + const successfulUrls = this.progressDb.prepare('SELECT url FROM processed_urls WHERE success = 1').all(); + + if (successfulUrls.length === 0) { + console.log(chalk.yellow('⚠ No successfully processed URLs found in tracking database')); + return null; + } + + const urlList = successfulUrls.map(row => `'${row.url.replace(/'/g, "''")}'`).join(','); + + const dbQuery = ` + SELECT + li.id, + li.title, + li.original_url as url, + li.state, + li.saved_at, + li.reading_progress_top_percent, + li.folder, + array_agg(DISTINCT l.name) FILTER (WHERE l.name IS NOT NULL) as labels + FROM omnivore.library_item li + LEFT JOIN omnivore.entity_labels el ON li.id = el.library_item_id + LEFT JOIN omnivore.labels l ON el.label_id = l.id + WHERE li.original_url IN (${urlList}) + GROUP BY li.id, li.title, li.original_url, li.state, li.saved_at, li.reading_progress_top_percent, li.folder + ORDER BY li.saved_at DESC + LIMIT 100; + `; + + const dockerCommand = `docker exec omnivore-postgres psql -U postgres -d omnivore -c "${dbQuery.replace(/"/g, '\\"')}" -t -A -F','`; + + console.log(chalk.gray('Executing database query via Docker...')); + const result = execSync(dockerCommand, { encoding: 'utf-8' }); + + if (result.trim()) { + const lines = result.trim().split('\n').filter(line => line.trim()); + console.log(chalk.green(`✓ Found ${lines.length} imported items in database`)); + + const statusCounts = {}; + const folderCounts = {}; + + lines.forEach(line => { + const [id, title, url, state, savedAt, progress, folder, labels] = line.split(','); + statusCounts[state] = (statusCounts[state] || 0) + 1; + folderCounts[folder] = (folderCounts[folder] || 0) + 1; + }); + + console.log(chalk.blue('📊 Status breakdown via database:')); + Object.entries(statusCounts).forEach(([state, count]) => { + console.log(chalk.gray(` ${state}: ${count}`)); + }); + + console.log(chalk.blue('📁 Folder breakdown:')); + Object.entries(folderCounts).forEach(([folder, count]) => { + console.log(chalk.gray(` ${folder}: ${count}`)); + }); + + return lines; + } else { + console.log(chalk.yellow('⚠ No items found in database')); + return []; + } + } catch (error) { + console.error(chalk.red('✗ Failed to check database status:'), error.message); + console.log(chalk.yellow('💡 Make sure Docker is running and omnivore-postgres container is accessible')); + return []; + } + } + + // Get detailed import statistics + async getImportStatistics() { + console.log(chalk.blue('\n📈 Getting detailed import statistics...')); + + try { + const { execSync } = await import('child_process'); + + // Get statistics from our progress database + const progressStats = this.progressDb.prepare(` + SELECT + success, + COUNT(*) as count + FROM processed_urls + GROUP BY success + `).all(); + + console.log(chalk.blue('📊 Progress tracking statistics:')); + progressStats.forEach(stat => { + const status = stat.success ? 'Successful' : 'Failed'; + console.log(chalk.gray(` ${status}: ${stat.count}`)); + }); + + // Get recent failures + const recentFailures = this.progressDb.prepare(` + SELECT url, pocket_title, error_message, processed_at + FROM processed_urls + WHERE success = 0 + ORDER BY processed_at DESC + LIMIT 10 + `).all(); + + if (recentFailures.length > 0) { + console.log(chalk.red('\n❌ Recent failures:')); + recentFailures.forEach(failure => { + console.log(chalk.red(` • ${failure.pocket_title}: ${failure.error_message}`)); + }); + } + + // Get successful URLs from tracking database + const successfulUrls = this.progressDb.prepare('SELECT url FROM processed_urls WHERE success = 1').all(); + + // Check if imported items match expected dates (only if we have successful URLs) + if (successfulUrls.length > 0) { + const urlList = successfulUrls.map(row => `'${row.url.replace(/'/g, "''")}'`).join(','); + const dateQuery = ` + SELECT + DATE(li.saved_at) as saved_date, + COUNT(*) as count + FROM omnivore.library_item li + WHERE li.original_url IN (${urlList}) + GROUP BY DATE(li.saved_at) + ORDER BY saved_date DESC + LIMIT 10; + `; + + try { + const dockerCommand = `docker exec omnivore-postgres psql -U postgres -d omnivore -c "${dateQuery.replace(/"/g, '\\"')}" -t -A -F','`; + const dateResult = execSync(dockerCommand, { encoding: 'utf-8' }); + + if (dateResult.trim()) { + console.log(chalk.blue('\n📅 Import dates distribution:')); + const dateLines = dateResult.trim().split('\n').filter(line => line.trim()); + dateLines.forEach(line => { + const [date, count] = line.split(','); + console.log(chalk.gray(` ${date}: ${count} items`)); + }); + } + } catch (dbError) { + console.log(chalk.yellow('⚠ Could not fetch date distribution from database')); + } + } else { + console.log(chalk.yellow('⚠ Could not fetch date distribution from database')); + } + + } catch (error) { + console.error(chalk.red('✗ Failed to get statistics:'), error.message); + } + } + + // Load CSV items (wrapper around parsePocketExport for comparison) + loadCSVItems() { + try { + console.log(chalk.blue(`📋 Reading CSV file for comparison: ${CONFIG.CSV_PATH}`)); + + const csvContent = readFileSync(CONFIG.CSV_PATH, 'utf-8'); + const records = parse(csvContent, { + columns: true, + skip_empty_lines: true, + trim: true, + }); + + console.log(chalk.green(`✅ Loaded ${records.length} items from CSV`)); + + // Transform CSV data for comparison (simpler format than import) + return records.map(record => ({ + url: record.url || record.resolved_url || record.given_url || '', + title: record.title || 'No Title', + givenUrl: record.given_url || '', + resolvedUrl: record.resolved_url || '', + timeAdded: record.time_added ? new Date(parseInt(record.time_added) * 1000).toISOString() : null, + tags: record.tags || '', + })); + } catch (error) { + console.error(chalk.red('✗ Failed to load CSV file:'), error.message); + throw error; + } + } + + // Compare CSV URLs with Omnivore URLs + async compareUrls() { + console.log(chalk.blue('🔍 Starting URL comparison between CSV and Omnivore...')); + + // Initialize database and load CSV data + this.initProgressDb(); + const csvItems = this.loadCSVItems(); + + // Populate database with CSV data if needed + const csvInsertStmt = this.progressDb.prepare(` + INSERT OR REPLACE INTO processed_urls (url, pocket_title, success, processed_at) + VALUES (?, ?, 0, datetime('now')) + `); + + for (const item of csvItems) { + csvInsertStmt.run(item.url, item.title); + } + + console.log(chalk.green(`✅ Loaded ${csvItems.length} URLs from CSV into database`)); + + // Fetch all URLs from Omnivore using GraphQL + console.log(chalk.blue('🌐 Fetching all URLs from Omnivore...')); + const omnivoreUrls = await this.fetchAllOmnivoreUrls(); + + // Compare URLs + const comparison = this.performComparison(csvItems, omnivoreUrls); + + // Generate comparison report + await this.generateComparisonReport(comparison); + + return comparison; + } + + // Fetch all URLs from Omnivore with pagination + async fetchAllOmnivoreUrls() { + const allUrls = []; + let hasNextPage = true; + let after = null; + let pageCount = 0; + + const query = ` + query FetchUrls($after: String, $first: Int) { + search( + query: "in:all sort:saved-desc" + after: $after + first: $first + ) { + ... on SearchSuccess { + pageInfo { + totalCount + hasNextPage + endCursor + } + edges { + cursor + node { + id + title + url + originalArticleUrl + createdAt + updatedAt + } + } + } + ... on SearchError { + errorCodes + } + } + } + `; + + while (hasNextPage) { + pageCount++; + console.log(chalk.gray(` Fetching page ${pageCount}...`)); + + try { + const response = await fetch(CONFIG.API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': CONFIG.API_KEY, + }, + body: JSON.stringify({ + query, + variables: { after, first: 100 }, + }), + }); + + const result = await response.json(); + + if (result.errors) { + throw new Error(`GraphQL errors: ${JSON.stringify(result.errors)}`); + } + + const searchResult = result.data.search; + if (searchResult.errorCodes) { + throw new Error(`Search error: ${searchResult.errorCodes.join(', ')}`); + } + + const items = searchResult.edges || []; + console.log(chalk.gray(` Found ${items.length} items on page ${pageCount}`)); + + for (const edge of items) { + const item = edge.node; + allUrls.push({ + id: item.id, + title: item.title, + url: item.url, + originalUrl: item.originalArticleUrl || item.url, + createdAt: item.createdAt, + updatedAt: item.updatedAt, + }); + } + + hasNextPage = searchResult.pageInfo.hasNextPage; + after = searchResult.pageInfo.endCursor; + + if (pageCount === 1) { + console.log(chalk.blue(`📊 Total items in Omnivore: ${searchResult.pageInfo.totalCount}`)); + } + + await new Promise(resolve => setTimeout(resolve, 100)); + + } catch (error) { + console.error(chalk.red(`❌ Error fetching page ${pageCount}:`), error.message); + throw error; + } + } + + console.log(chalk.green(`✅ Fetched ${allUrls.length} URLs from Omnivore`)); + return allUrls; + } + + // Normalize URL for comparison + normalizeUrl(url) { + if (!url) return ''; + return url + .toLowerCase() + .replace(/^https?:\/\//, '') + .replace(/^www\./, '') + .replace(/\/$/, '') + .replace(/#.*$/, ''); + } + + // Perform URL comparison + performComparison(csvItems, omnivoreUrls) { + console.log(chalk.blue('🔍 Performing URL comparison...')); + + // Create normalized lookup maps + const omnivoreMap = new Map(); + const csvMap = new Map(); + + // Build Omnivore lookup + for (const item of omnivoreUrls) { + const normalizedUrl = this.normalizeUrl(item.url); + const normalizedOriginal = this.normalizeUrl(item.originalUrl); + + if (normalizedUrl) omnivoreMap.set(normalizedUrl, item); + if (normalizedOriginal && normalizedOriginal !== normalizedUrl) { + omnivoreMap.set(normalizedOriginal, item); + } + } + + // Build CSV lookup + for (const item of csvItems) { + const normalized = this.normalizeUrl(item.url); + if (normalized) csvMap.set(normalized, item); + } + + // Find matches and differences + const inBoth = []; + const onlyInOmnivore = []; + const onlyInCSV = []; + + // Check Omnivore items + for (const [normalizedUrl, omnivoreItem] of omnivoreMap) { + if (csvMap.has(normalizedUrl)) { + inBoth.push({ + url: normalizedUrl, + omnivore: omnivoreItem, + csv: csvMap.get(normalizedUrl), + }); + } else { + onlyInOmnivore.push(omnivoreItem); + } + } + + // Check CSV items not in Omnivore + for (const [normalizedUrl, csvItem] of csvMap) { + if (!omnivoreMap.has(normalizedUrl)) { + onlyInCSV.push(csvItem); + } + } + + return { + inBoth, + onlyInOmnivore, + onlyInCSV, + stats: { + omnivoreTotal: omnivoreUrls.length, + csvTotal: csvItems.length, + matches: inBoth.length, + omnivoreOnly: onlyInOmnivore.length, + csvOnly: onlyInCSV.length, + }, + }; + } + + // Generate comparison report + async generateComparisonReport(comparison) { + const { inBoth, onlyInOmnivore, onlyInCSV, stats } = comparison; + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + + console.log(chalk.blue('\n📊 COMPARISON REPORT')); + console.log(chalk.blue('==================')); + console.log(chalk.cyan(`📚 Total in Omnivore: ${stats.omnivoreTotal}`)); + console.log(chalk.cyan(`📋 Total in CSV: ${stats.csvTotal}`)); + console.log(chalk.green(`✅ URLs in both: ${stats.matches}`)); + console.log(chalk.blue(`🔵 Only in Omnivore: ${stats.omnivoreOnly}`)); + console.log(chalk.red(`🔴 Missing from Omnivore: ${stats.csvOnly}`)); + console.log(chalk.yellow(`📈 Import success rate: ${((stats.matches / stats.csvTotal) * 100).toFixed(1)}%`)); + + // Update database with comparison results + const updateStmt = this.progressDb.prepare(` + UPDATE processed_urls + SET status = 'FOUND_IN_OMNIVORE', success = 1 + WHERE url = ? + `); + + for (const match of inBoth) { + const csvItem = match.csv; + updateStmt.run(csvItem.url); + } + + // Save missing URLs + if (onlyInCSV.length > 0) { + const missingFile = join(__dirname, `missing-from-omnivore-${timestamp}.json`); + await writeFile(missingFile, JSON.stringify(onlyInCSV, null, 2)); + console.log(chalk.yellow(`\n💾 Saved ${onlyInCSV.length} missing URLs to: ${missingFile}`)); + + // Save simple list + const missingList = join(__dirname, `missing-urls-list-${timestamp}.txt`); + const missingUrlsList = onlyInCSV.map(item => item.url).join('\n'); + await writeFile(missingList, missingUrlsList); + console.log(chalk.yellow(`💾 Saved missing URLs list to: ${missingList}`)); + } + + // Save full report + const reportFile = join(__dirname, `url-comparison-report-${timestamp}.json`); + await writeFile(reportFile, JSON.stringify({ + timestamp: new Date().toISOString(), + stats, + summary: { + importSuccessRate: `${((stats.matches / stats.csvTotal) * 100).toFixed(1)}%`, + urlsInBoth: stats.matches, + urlsMissingFromOmnivore: stats.csvOnly, + urlsOnlyInOmnivore: stats.omnivoreOnly, + }, + missingUrls: onlyInCSV.slice(0, 100), // First 100 missing + }, null, 2)); + + console.log(chalk.green(`💾 Saved full comparison report to: ${reportFile}`)); + } + + // Filter items to only those missing from Omnivore + async filterMissingItems(items) { + console.log(chalk.blue('🔍 Filtering to only URLs missing from Omnivore...')); + + // Load comparison data from previous run (requires running --compare first) + const omnivoreUrls = await this.fetchAllOmnivoreUrls(); + + // Create normalized lookup map for Omnivore URLs + const omnivoreNormalized = new Set(); + + for (const item of omnivoreUrls) { + const normalized = this.normalizeUrl(item.url); + const originalNormalized = this.normalizeUrl(item.originalUrl); + + if (normalized) omnivoreNormalized.add(normalized); + if (originalNormalized && originalNormalized !== normalized) { + omnivoreNormalized.add(originalNormalized); + } + } + + // Filter items to only missing ones + const missingItems = items.filter(item => { + const normalized = this.normalizeUrl(item.url); + return normalized && !omnivoreNormalized.has(normalized); + }); + + console.log(chalk.green(`✅ Found ${missingItems.length} URLs missing from Omnivore (out of ${items.length} total)`)); + return missingItems; + } + + // Normalize URL for comparison (from comparison logic) + normalizeUrl(url) { + if (!url) return ''; + + return url + .toLowerCase() + .replace(/^https?:\/\//, '') // Remove protocol + .replace(/^www\./, '') // Remove www + .replace(/\/$/, '') // Remove trailing slash + .replace(/#.*$/, '') // Remove fragments + .replace(/\?.*$/, ''); // Remove query parameters + } + + // Main execution flow + async run() { + console.log(chalk.blue('🚀 Starting Pocket import to Omnivore...\n')); + + // Check if this is a comparison only + if (CONFIG.COMPARE_MODE) { + console.log(chalk.blue('🔍 Comparison mode: Comparing CSV URLs with Omnivore URLs...\n')); + await this.compareUrls(); + return; + } + + // Check if this is missing-only mode + if (CONFIG.MISSING_ONLY) { + console.log(chalk.blue('🔍 Missing-only mode: Importing only URLs missing from Omnivore...\n')); + } + + // Check if this is a status check only + if (process.argv.includes('--status-only')) { + this.initProgressDb(); + await this.checkImportedItemsStatus(); + await this.checkImportedItemsStatusViaDb(); + await this.getImportStatistics(); + + if (this.progressDb) { + this.progressDb.close(); + } + return; + } + + // Test authentication + const authSuccess = await this.testAuthentication(); + if (!authSuccess) { + console.log(chalk.red('✗ Authentication failed. Please check your OMNIVORE_API_KEY environment variable.')); + process.exit(1); + } + + // Initialize progress tracking + this.initProgressDb(); + + // Parse Pocket export + let items = this.parsePocketExport(); + + // Filter to missing-only if requested + if (CONFIG.MISSING_ONLY) { + items = await this.filterMissingItems(items); + } + + // Import items + await this.importItems(items); + + // Generate report + await this.generateReport(); + + // Check status of imported items + await this.checkImportedItemsStatus(); + await this.checkImportedItemsStatusViaDb(); + await this.getImportStatistics(); + + // Cleanup + if (this.progressDb) { + this.progressDb.close(); + } + + console.log(chalk.green('\n🎉 Pocket import completed!')); + } +} + +// Show help message +function showHelp() { + console.log(chalk.blue('📚 Pocket Import to Omnivore\n')); + console.log('Usage:'); + console.log(' node import-pocket.js [options]\n'); + console.log('Options:'); + console.log(' --help Show this help message'); + console.log(' --test Import only first 10 items for testing'); + console.log(' --dry-run Show what would be imported without actually importing'); + console.log(' --status-only Check status of previously imported items only'); + console.log(' --compare Compare CSV URLs with URLs in Omnivore (no import)'); + console.log(' --file Path to Pocket CSV export file'); + console.log(' --limit Number of items to process (default: 10 in test mode)'); + console.log('\nEnvironment Variables:'); + console.log(' OMNIVORE_API_KEY Required API key for Omnivore instance\n'); + console.log('Examples:'); + console.log(' node import-pocket.js --test --dry-run --limit 3'); + console.log(' node import-pocket.js --file ~/Downloads/pocket_export.csv'); + console.log(' node import-pocket.js --status-only'); + console.log(' node import-pocket.js --compare'); + console.log(' node import-pocket.js'); +} + +// Handle command line arguments +if (process.argv.includes('--help')) { + showHelp(); + process.exit(0); +} + +// Run the importer +const importer = new PocketImporter(); +importer.run().catch(error => { + console.error(chalk.red('💥 Import failed:'), error.message); + process.exit(1); +}); diff --git a/scripts/migrate-omnivore.js b/scripts/migrate-omnivore.js new file mode 100644 index 000000000..484a96829 --- /dev/null +++ b/scripts/migrate-omnivore.js @@ -0,0 +1,481 @@ +#!/usr/bin/env node + +import Database from 'better-sqlite3'; +import fetch from 'node-fetch'; +import pLimit from 'p-limit'; +import chalk from 'chalk'; +import { writeFile } from 'fs/promises'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Configuration +const CONFIG = { + API_URL: + process.env.API_ENDPOINT || + process.env.API_URL || + 'http://localhost:4000/api/graphql', + API_KEY: process.env.OMNIVORE_API_KEY, + DB_PATH: join(__dirname, '../self-hosting/archive-db/store.sqlite'), + BATCH_SIZE: 50, + RATE_LIMIT: 5, // concurrent requests + TEST_MODE: process.argv.includes('--test'), + TEST_LIMIT: 10, // items to process in test mode +}; + +if (!CONFIG.API_KEY) { + console.error('❌ OMNIVORE_API_KEY environment variable is required'); + console.error( + 'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/migrate-omnivore.js' + ); + process.exit(1); +} + +// Core Data timestamp offset (seconds between 1970-01-01 and 2001-01-01) +const CORE_DATA_EPOCH_OFFSET = 978307200; + +class OmnivoreMigrator { + constructor() { + this.db = null; + this.stats = { + totalItems: 0, + totalLabels: 0, + processedItems: 0, + processedLabels: 0, + failedItems: [], + failedLabels: [], + startTime: Date.now(), + }; + this.limit = pLimit(CONFIG.RATE_LIMIT); + } + + // Convert Core Data timestamp to ISO string + convertTimestamp(coreDataTimestamp) { + if (!coreDataTimestamp) return null; + const unixTimestamp = coreDataTimestamp + CORE_DATA_EPOCH_OFFSET; + return new Date(unixTimestamp * 1000).toISOString(); + } + + // Initialize database connection + initDatabase() { + try { + this.db = new Database(CONFIG.DB_PATH, { readonly: true }); + console.log(chalk.green('✓ Connected to SQLite database')); + } catch (error) { + console.error(chalk.red('✗ Failed to connect to database:'), error.message); + process.exit(1); + } + } + + // Extract labels from SQLite + extractLabels() { + const query = ` + SELECT + ZID as id, + ZNAME as name, + ZCOLOR as color, + ZLABELDESCRIPTION as description, + ZCREATEDAT as createdAt + FROM ZLINKEDITEMLABEL + ORDER BY ZNAME + `; + + const labels = this.db.prepare(query).all(); + this.stats.totalLabels = labels.length; + + return labels.map(label => ({ + id: label.id, + name: label.name, + color: label.color || '#3B82F6', // default blue if no color + description: label.description || '', + createdAt: this.convertTimestamp(label.createdAt), + })); + } + + // Extract items with their labels from SQLite + extractItems() { + // Check if we should only get missing items + const onlyMissing = process.argv.includes('--missing-only'); + // Check if we should get specific missing labeled items + const missingLabeled = process.argv.includes('--missing-labeled'); + + const itemsQuery = missingLabeled ? ` + SELECT + li.ZID as id, + li.ZPAGEURLSTRING as url, + li.ZTITLE as title, + li.ZDESCRIPTIONTEXT as description, + li.ZAUTHOR as author, + li.ZSITENAME as siteName, + li.ZCREATEDAT as createdAt, + li.ZSAVEDAT as savedAt, + li.ZUPDATEDAT as updatedAt, + li.ZPUBLISHDATE as publishedAt, + li.ZISARCHIVED as isArchived, + li.ZREADAT as readAt + FROM ZLINKEDITEM li + WHERE li.ZPAGEURLSTRING IN ( + 'https://github.com/danswer-ai/danswer', + 'https://segment.com/blog/rebuilding-our-infrastructure/', + 'https://www.hyperledger.org' + ) + ORDER BY li.ZCREATEDAT + ` : onlyMissing ? ` + SELECT + li.ZID as id, + li.ZPAGEURLSTRING as url, + li.ZTITLE as title, + li.ZDESCRIPTIONTEXT as description, + li.ZAUTHOR as author, + li.ZSITENAME as siteName, + li.ZCREATEDAT as createdAt, + li.ZSAVEDAT as savedAt, + li.ZUPDATEDAT as updatedAt, + li.ZPUBLISHDATE as publishedAt, + li.ZISARCHIVED as isArchived, + li.ZREADAT as readAt + FROM ZLINKEDITEM li + LEFT JOIN omnivore_mapping om ON li.ZPAGEURLSTRING = om.url + WHERE om.url IS NULL AND li.ZPAGEURLSTRING = 'https://svgl.vercel.app' + ORDER BY li.ZCREATEDAT + LIMIT 1 + ` : ` + SELECT + li.ZID as id, + li.ZPAGEURLSTRING as url, + li.ZTITLE as title, + li.ZDESCRIPTIONTEXT as description, + li.ZAUTHOR as author, + li.ZSITENAME as siteName, + li.ZCREATEDAT as createdAt, + li.ZSAVEDAT as savedAt, + li.ZUPDATEDAT as updatedAt, + li.ZPUBLISHDATE as publishedAt, + li.ZISARCHIVED as isArchived, + li.ZREADAT as readAt + FROM ZLINKEDITEM li + ORDER BY li.ZCREATEDAT + LIMIT 3 + `; + + const items = this.db.prepare(itemsQuery).all(); + this.stats.totalItems = items.length; + + // Get labels for each item + const labelsQuery = ` + SELECT l.ZNAME as labelName + FROM Z_2LABELS rel + JOIN ZLINKEDITEMLABEL l ON rel.Z_3LABELS1 = l.Z_PK + WHERE rel.Z_2LINKEDITEMS = ? + `; + const getLabels = this.db.prepare(labelsQuery); + + return items.map(item => { + const labels = getLabels.all(item.id).map(l => l.labelName); + + return { + id: item.id, + url: item.url, + title: item.title || '', + description: item.description || '', + author: item.author || '', + siteName: item.siteName || '', + labels: labels, + createdAt: this.convertTimestamp(item.createdAt), + savedAt: this.convertTimestamp(item.savedAt), + updatedAt: this.convertTimestamp(item.updatedAt), + publishedAt: this.convertTimestamp(item.publishedAt), + isArchived: Boolean(item.isArchived), + readAt: this.convertTimestamp(item.readAt), + }; + }); + } + + // Make GraphQL request + async makeGraphQLRequest(query, variables = {}) { + const response = await fetch(CONFIG.API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': CONFIG.API_KEY, + }, + body: JSON.stringify({ + query, + variables, + }), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + + // Log the full response for debugging + if (result.errors || !result.data) { + console.log(chalk.yellow('GraphQL Response:'), JSON.stringify(result, null, 2)); + } + + if (result.errors) { + throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`); + } + + return result.data; + } + + // Test authentication with the API key + async testAuthentication() { + try { + const query = ` + query { + me { + id + name + email + } + } + `; + + const data = await this.makeGraphQLRequest(query); + if (data.me) { + console.log(chalk.green(`✓ Authentication successful for user: ${data.me.name} (${data.me.email})`)); + return true; + } else { + console.log(chalk.red('✗ Authentication failed: No user returned')); + return false; + } + } catch (error) { + console.log(chalk.red('✗ Authentication failed:', error.message)); + return false; + } + } + + // Create a label in the new instance + async createLabel(label) { + const mutation = ` + mutation CreateLabel($input: CreateLabelInput!) { + createLabel(input: $input) { + ... on CreateLabelSuccess { + label { + id + name + color + } + } + ... on CreateLabelError { + errorCodes + } + } + } + `; + + try { + const data = await this.makeGraphQLRequest(mutation, { + input: { + name: label.name, + color: label.color, + description: label.description, + }, + }); + + if (data.createLabel.errorCodes) { + throw new Error(`Failed to create label: ${data.createLabel.errorCodes.join(', ')}`); + } + + return data.createLabel.label; + } catch (error) { + console.error(chalk.red(`✗ Failed to create label "${label.name}":`, error.message)); + this.stats.failedLabels.push({ label, error: error.message }); + return null; + } + } + + // Save an item to the new instance + async saveItem(item) { + const mutation = ` + mutation SaveUrl($input: SaveUrlInput!) { + saveUrl(input: $input) { + ... on SaveSuccess { + url + clientRequestId + } + ... on SaveError { + errorCodes + } + } + } + `; + + try { + const labelInputs = item.labels.map(labelName => ({ name: labelName })); + + const data = await this.makeGraphQLRequest(mutation, { + input: { + url: item.url, + source: 'migration', + clientRequestId: item.id, + labels: labelInputs.length > 0 ? labelInputs : undefined, + savedAt: item.savedAt, + publishedAt: item.publishedAt, + }, + }); + + if (data.saveUrl.errorCodes) { + throw new Error(`Failed to save item: ${data.saveUrl.errorCodes.join(', ')}`); + } + + return data.saveUrl; + } catch (error) { + console.error(chalk.red(`✗ Failed to save item "${item.title}":`, error.message)); + this.stats.failedItems.push({ item, error: error.message }); + return null; + } + } + + // Migrate labels + async migrateLabels(labels) { + console.log(chalk.blue(`\n📋 Migrating ${labels.length} labels...`)); + + for (const label of labels) { + await this.limit(async () => { + const result = await this.createLabel(label); + if (result) { + this.stats.processedLabels++; + console.log(chalk.green(`✓ Created label: ${label.name}`)); + } + }); + } + } + + // Migrate items in batches + async migrateItems(items) { + const totalItems = CONFIG.TEST_MODE ? Math.min(items.length, CONFIG.TEST_LIMIT) : items.length; + const itemsToProcess = items.slice(0, totalItems); + + console.log(chalk.blue(`\n📚 Migrating ${totalItems} items...`)); + + const batches = []; + for (let i = 0; i < itemsToProcess.length; i += CONFIG.BATCH_SIZE) { + batches.push(itemsToProcess.slice(i, i + CONFIG.BATCH_SIZE)); + } + + for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { + const batch = batches[batchIndex]; + console.log(chalk.cyan(`\nProcessing batch ${batchIndex + 1}/${batches.length} (${batch.length} items)...`)); + + const promises = batch.map(item => + this.limit(async () => { + const result = await this.saveItem(item); + if (result) { + this.stats.processedItems++; + console.log(chalk.green(`✓ Saved: ${item.title || item.url}`)); + } + return result; + }) + ); + + await Promise.all(promises); + + // Progress update + const progress = ((batchIndex + 1) / batches.length * 100).toFixed(1); + console.log(chalk.yellow(`Progress: ${progress}% (${this.stats.processedItems}/${totalItems} items)`)); + } + } + + // Generate migration report + async generateReport() { + const duration = (Date.now() - this.stats.startTime) / 1000; + + const report = { + migration: { + timestamp: new Date().toISOString(), + duration: `${duration.toFixed(2)} seconds`, + testMode: CONFIG.TEST_MODE, + }, + summary: { + totalLabels: this.stats.totalLabels, + processedLabels: this.stats.processedLabels, + failedLabels: this.stats.failedLabels.length, + totalItems: CONFIG.TEST_MODE ? Math.min(this.stats.totalItems, CONFIG.TEST_LIMIT) : this.stats.totalItems, + processedItems: this.stats.processedItems, + failedItems: this.stats.failedItems.length, + }, + failures: { + labels: this.stats.failedLabels, + items: this.stats.failedItems, + }, + }; + + const reportPath = join(__dirname, 'migration-report.json'); + await writeFile(reportPath, JSON.stringify(report, null, 2)); + + console.log(chalk.blue(`\n📊 Migration Report:`)); + console.log(chalk.green(`✓ Labels: ${report.summary.processedLabels}/${report.summary.totalLabels}`)); + console.log(chalk.green(`✓ Items: ${report.summary.processedItems}/${report.summary.totalItems}`)); + + if (report.summary.failedLabels > 0) { + console.log(chalk.red(`✗ Failed labels: ${report.summary.failedLabels}`)); + } + if (report.summary.failedItems > 0) { + console.log(chalk.red(`✗ Failed items: ${report.summary.failedItems}`)); + } + + console.log(chalk.blue(`📄 Full report saved to: ${reportPath}`)); + return report; + } + + // Main migration process + async migrate() { + try { + console.log(chalk.bold.blue('🚀 Starting Omnivore Migration')); + console.log(chalk.gray(`Mode: ${CONFIG.TEST_MODE ? 'TEST' : 'FULL'}`)); + console.log(chalk.gray(`API: ${CONFIG.API_URL}`)); + + this.initDatabase(); + + // Test authentication first + console.log(chalk.blue('\n🔑 Testing API authentication...')); + const authResult = await this.testAuthentication(); + if (!authResult) { + console.log(chalk.red('Migration aborted due to authentication failure.')); + return; + } + + // Extract data + console.log(chalk.blue('\n📤 Extracting data from SQLite database...')); + const labels = this.extractLabels(); + const items = this.extractItems(); + + console.log(chalk.green(`✓ Extracted ${labels.length} labels and ${items.length} items`)); + + // Migrate labels first + if (labels.length > 0) { + await this.migrateLabels(labels); + } + + // Migrate items + if (items.length > 0) { + await this.migrateItems(items); + } + + // Generate report + await this.generateReport(); + + console.log(chalk.bold.green('\n🎉 Migration completed!')); + + } catch (error) { + console.error(chalk.red('\n💥 Migration failed:'), error.message); + process.exit(1); + } finally { + if (this.db) { + this.db.close(); + } + } + } +} + +// Run migration +const migrator = new OmnivoreMigrator(); +migrator.migrate(); diff --git a/scripts/package.json b/scripts/package.json new file mode 100644 index 000000000..d905de1cd --- /dev/null +++ b/scripts/package.json @@ -0,0 +1,20 @@ +{ + "name": "omnivore-migration", + "version": "1.0.0", + "description": "Migrate data from old Omnivore SQLite database to new self-hosted instance", + "main": "migrate-omnivore.js", + "type": "module", + "scripts": { + "migrate": "node migrate-omnivore.js", + "test": "node migrate-omnivore.js --test" + }, + "dependencies": { + "better-sqlite3": "^12.4.1", + "chalk": "^5.3.0", + "csv-parse": "^6.1.0", + "node-fetch": "^3.3.2", + "p-limit": "^5.0.0" + }, + "author": "", + "license": "ISC" +} diff --git a/scripts/pnpm-lock.yaml b/scripts/pnpm-lock.yaml new file mode 100644 index 000000000..837df7ac1 --- /dev/null +++ b/scripts/pnpm-lock.yaml @@ -0,0 +1,369 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + better-sqlite3: + specifier: ^12.4.1 + version: 12.4.1 + chalk: + specifier: ^5.3.0 + version: 5.6.2 + csv-parse: + specifier: ^6.1.0 + version: 6.1.0 + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 + p-limit: + specifier: ^5.0.0 + version: 5.0.0 + +packages: + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + better-sqlite3@12.4.1: + resolution: {integrity: sha512-3yVdyZhklTiNrtg+4WqHpJpFDd+WHTg2oM7UcR80GqL05AOV0xEJzc6qNvFYoEtE+hRp1n9MpN6/+4yhlGkDXQ==} + engines: {node: 20.x || 22.x || 23.x || 24.x} + + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + + chalk@5.6.2: + resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + + csv-parse@6.1.0: + resolution: {integrity: sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==} + + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + + deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + + detect-libc@2.1.0: + resolution: {integrity: sha512-vEtk+OcP7VBRtQZ1EJ3bdgzSfBjgnEalLTp5zjJrS+2Z1w2KZly4SBdac/WDU3hhsNAZ9E8SC96ME4Ey8MZ7cg==} + engines: {node: '>=8'} + + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + + expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + + fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + + github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + + node-abi@3.77.0: + resolution: {integrity: sha512-DSmt0OEcLoK4i3NuscSbGjOf3bqiDEutejqENSplMSFA/gmB8mkED9G4pKWnPl7MDU4rSHebKPHeitpDfyH0cQ==} + engines: {node: '>=10'} + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + p-limit@5.0.0: + resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==} + engines: {node: '>=18'} + + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + hasBin: true + + pump@3.0.3: + resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==} + + rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + semver@7.7.2: + resolution: {integrity: sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==} + engines: {node: '>=10'} + hasBin: true + + simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + + simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + + tar-fs@2.1.4: + resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==} + + tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + + tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + yocto-queue@1.2.1: + resolution: {integrity: sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==} + engines: {node: '>=12.20'} + +snapshots: + + base64-js@1.5.1: {} + + better-sqlite3@12.4.1: + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.3 + + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + chalk@5.6.2: {} + + chownr@1.1.4: {} + + csv-parse@6.1.0: {} + + data-uri-to-buffer@4.0.1: {} + + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + + deep-extend@0.6.0: {} + + detect-libc@2.1.0: {} + + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + + expand-template@2.0.3: {} + + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + + file-uri-to-path@1.0.0: {} + + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + + fs-constants@1.0.0: {} + + github-from-package@0.0.0: {} + + ieee754@1.2.1: {} + + inherits@2.0.4: {} + + ini@1.3.8: {} + + mimic-response@3.1.0: {} + + minimist@1.2.8: {} + + mkdirp-classic@0.5.3: {} + + napi-build-utils@2.0.0: {} + + node-abi@3.77.0: + dependencies: + semver: 7.7.2 + + node-domexception@1.0.0: {} + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + p-limit@5.0.0: + dependencies: + yocto-queue: 1.2.1 + + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.1.0 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.77.0 + pump: 3.0.3 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.4 + tunnel-agent: 0.6.0 + + pump@3.0.3: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + + rc@1.2.8: + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + + safe-buffer@5.2.1: {} + + semver@7.7.2: {} + + simple-concat@1.0.1: {} + + simple-get@4.0.1: + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strip-json-comments@2.0.1: {} + + tar-fs@2.1.4: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.3 + tar-stream: 2.2.0 + + tar-stream@2.2.0: + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.5 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + + tunnel-agent@0.6.0: + dependencies: + safe-buffer: 5.2.1 + + util-deprecate@1.0.2: {} + + web-streams-polyfill@3.3.3: {} + + wrappy@1.0.2: {} + + yocto-queue@1.2.1: {} diff --git a/scripts/pnpm-workspace.yaml b/scripts/pnpm-workspace.yaml new file mode 100644 index 000000000..d0eb205b0 --- /dev/null +++ b/scripts/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +ignoredBuiltDependencies: + - better-sqlite3 diff --git a/scripts/test-auth.js b/scripts/test-auth.js new file mode 100644 index 000000000..49dc71cc5 --- /dev/null +++ b/scripts/test-auth.js @@ -0,0 +1,53 @@ +#!/usr/bin/env node + +import fetch from 'node-fetch'; + +const API_URL = + process.env.API_ENDPOINT || + process.env.API_URL || + 'http://localhost:4000/api/graphql'; +const API_KEY = process.env.OMNIVORE_API_KEY; + +if (!API_KEY) { + console.error('❌ OMNIVORE_API_KEY environment variable is required'); + console.error('Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/test-auth.js'); + process.exit(1); +} + +async function testAuth() { + console.log('Testing API authentication...'); + console.log('API URL:', API_URL); + console.log('API Key:', `${API_KEY.substring(0, 8)}...`); + + const query = ` + query { + me { + id + name + email + } + } + `; + + try { + const response = await fetch(API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': API_KEY, + }, + body: JSON.stringify({ query }), + }); + + console.log('Response status:', response.status); + console.log('Response headers:', Object.fromEntries(response.headers.entries())); + + const result = await response.json(); + console.log('Response body:', JSON.stringify(result, null, 2)); + + } catch (error) { + console.error('Error:', error.message); + } +} + +testAuth(); diff --git a/scripts/test-create-label.js b/scripts/test-create-label.js new file mode 100644 index 000000000..1a9e39b4e --- /dev/null +++ b/scripts/test-create-label.js @@ -0,0 +1,67 @@ +#!/usr/bin/env node + +import fetch from 'node-fetch'; + +const API_URL = + process.env.API_ENDPOINT || + process.env.API_URL || + 'http://localhost:4000/api/graphql'; +const API_KEY = process.env.OMNIVORE_API_KEY; + +if (!API_KEY) { + console.error('❌ OMNIVORE_API_KEY environment variable is required'); + console.error( + 'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/test-create-label.js', + ); + process.exit(1); +} + +async function testCreateLabel() { + console.log('Testing label creation with API key...'); + + const mutation = ` + mutation CreateLabel($input: CreateLabelInput!) { + createLabel(input: $input) { + ... on CreateLabelSuccess { + label { + id + name + color + } + } + ... on CreateLabelError { + errorCodes + } + } + } + `; + + const variables = { + input: { + name: 'test-migration-label', + color: '#FF0000', + description: 'Test label created by migration script' + } + }; + + try { + const response = await fetch(API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Omnivore-Authorization': API_KEY, + }, + body: JSON.stringify({ query: mutation, variables }), + }); + + console.log('Response status:', response.status); + + const result = await response.json(); + console.log('Response body:', JSON.stringify(result, null, 2)); + + } catch (error) { + console.error('Error:', error.message); + } +} + +testCreateLabel();