mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
scripts: add migration and labeling utilities
This commit is contained in:
Rohit Amarnath
parent
05e0550ca1
commit
6ff55c4043
14 changed files with 3798 additions and 0 deletions
371
scripts/apply-labels-with-mapping.js
Executable file
371
scripts/apply-labels-with-mapping.js
Executable file
|
|
@ -0,0 +1,371 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import fetch from 'node-fetch';
|
||||
import pLimit from 'p-limit';
|
||||
import chalk from 'chalk';
|
||||
import { writeFile } from 'fs/promises';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Configuration
|
||||
const CONFIG = {
|
||||
API_URL:
|
||||
process.env.API_ENDPOINT ||
|
||||
process.env.API_URL ||
|
||||
'http://localhost:4000/api/graphql',
|
||||
API_KEY: process.env.OMNIVORE_API_KEY,
|
||||
ARCHIVE_DB_PATH:
|
||||
process.env.ARCHIVE_DB_PATH ||
|
||||
join(__dirname, '../self-hosting/archive-db/store.sqlite'),
|
||||
MAPPING_DB_PATH:
|
||||
process.env.MAPPING_DB_PATH || join(__dirname, 'url-id-mapping.sqlite'),
|
||||
RATE_LIMIT: 3, // concurrent requests
|
||||
BATCH_SIZE: 10, // items to process per batch
|
||||
};
|
||||
|
||||
if (!CONFIG.API_KEY) {
|
||||
console.error(chalk.red('❌ OMNIVORE_API_KEY environment variable is required'));
|
||||
console.error(
|
||||
chalk.gray(
|
||||
'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/apply-labels-with-mapping.js',
|
||||
),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
class LabelApplicator {
|
||||
constructor() {
|
||||
this.archiveDb = null;
|
||||
this.mappingDb = null;
|
||||
this.stats = {
|
||||
totalItemsWithLabels: 0,
|
||||
processedItems: 0,
|
||||
failedItems: [],
|
||||
successfulItems: 0,
|
||||
notFoundItems: [],
|
||||
startTime: Date.now(),
|
||||
};
|
||||
this.limit = pLimit(CONFIG.RATE_LIMIT);
|
||||
this.labelNameToId = new Map();
|
||||
this.urlToPageId = new Map();
|
||||
}
|
||||
|
||||
// Initialize databases
|
||||
initDatabases() {
|
||||
try {
|
||||
// Archive database
|
||||
this.archiveDb = new Database(CONFIG.ARCHIVE_DB_PATH, { readonly: true });
|
||||
console.log(chalk.green('✓ Connected to archive SQLite database'));
|
||||
|
||||
// Mapping database
|
||||
this.mappingDb = new Database(CONFIG.MAPPING_DB_PATH, { readonly: true });
|
||||
console.log(chalk.green('✓ Connected to mapping database'));
|
||||
|
||||
// Load all URL to ID mappings into memory for fast lookup
|
||||
const mappings = this.mappingDb.prepare('SELECT url, id FROM item_mapping').all();
|
||||
mappings.forEach(row => {
|
||||
this.urlToPageId.set(row.url, row.id);
|
||||
});
|
||||
console.log(chalk.green(`✓ Loaded ${mappings.length} URL-to-ID mappings`));
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red('✗ Failed to connect to databases:'), error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Make GraphQL request
|
||||
async makeGraphQLRequest(query, variables = {}) {
|
||||
const response = await fetch(CONFIG.API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Omnivore-Authorization': CONFIG.API_KEY,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query,
|
||||
variables,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.errors) {
|
||||
throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`);
|
||||
}
|
||||
|
||||
return result.data;
|
||||
}
|
||||
|
||||
// Extract items with their labels from archive SQLite
|
||||
extractItemLabelAssociations() {
|
||||
const query = `
|
||||
SELECT
|
||||
li.ZPAGEURLSTRING as url,
|
||||
li.ZTITLE as title,
|
||||
GROUP_CONCAT(label.ZNAME, '||') as labelNames
|
||||
FROM ZLINKEDITEM li
|
||||
JOIN Z_2LABELS l ON li.Z_PK = l.Z_2LINKEDITEMS
|
||||
JOIN ZLINKEDITEMLABEL label ON l.Z_3LABELS1 = label.Z_PK
|
||||
GROUP BY li.ZPAGEURLSTRING, li.ZTITLE
|
||||
ORDER BY li.ZPAGEURLSTRING
|
||||
`;
|
||||
|
||||
const results = this.archiveDb.prepare(query).all();
|
||||
this.stats.totalItemsWithLabels = results.length;
|
||||
|
||||
return results.map(row => ({
|
||||
url: row.url,
|
||||
title: row.title || '',
|
||||
labels: row.labelNames.split('||').filter(name => name && name.trim()),
|
||||
}));
|
||||
}
|
||||
|
||||
// Get all labels from the new Omnivore instance
|
||||
async fetchLabelsFromOmnivore() {
|
||||
const query = `
|
||||
query {
|
||||
labels {
|
||||
... on LabelsSuccess {
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
}
|
||||
}
|
||||
... on LabelsError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
try {
|
||||
const data = await this.makeGraphQLRequest(query);
|
||||
|
||||
if (data.labels.errorCodes) {
|
||||
throw new Error(`Failed to fetch labels: ${data.labels.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
// Build mapping of label name to ID
|
||||
data.labels.labels.forEach(label => {
|
||||
this.labelNameToId.set(label.name, label.id);
|
||||
});
|
||||
|
||||
console.log(chalk.green(`✓ Fetched ${data.labels.labels.length} labels from Omnivore`));
|
||||
return data.labels.labels;
|
||||
} catch (error) {
|
||||
console.error(chalk.red('✗ Failed to fetch labels:'), error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply labels to an item using setLabels mutation
|
||||
async applyLabelsToItem(pageId, labelNames, itemTitle = '', itemUrl = '') {
|
||||
// Convert label names to IDs
|
||||
const labelIds = labelNames
|
||||
.map(name => this.labelNameToId.get(name))
|
||||
.filter(id => id); // Remove any undefined IDs
|
||||
|
||||
if (labelIds.length === 0) {
|
||||
throw new Error(`No valid labels found for item. Requested: ${labelNames.join(', ')}`);
|
||||
}
|
||||
|
||||
const mutation = `
|
||||
mutation SetLabels($input: SetLabelsInput!) {
|
||||
setLabels(input: $input) {
|
||||
... on SetLabelsSuccess {
|
||||
labels {
|
||||
id
|
||||
name
|
||||
}
|
||||
}
|
||||
... on SetLabelsError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
try {
|
||||
const data = await this.makeGraphQLRequest(mutation, {
|
||||
input: {
|
||||
pageId: pageId,
|
||||
labelIds: labelIds,
|
||||
},
|
||||
});
|
||||
|
||||
if (data.setLabels.errorCodes) {
|
||||
throw new Error(`Failed to set labels: ${data.setLabels.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
return data.setLabels.labels;
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`✗ Failed to apply labels to "${itemTitle}" (${itemUrl}):`), error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Process items in batches
|
||||
async processItems(itemsWithLabels) {
|
||||
console.log(chalk.blue(`\n🏷️ Processing ${itemsWithLabels.length} items with labels...`));
|
||||
|
||||
const batches = [];
|
||||
for (let i = 0; i < itemsWithLabels.length; i += CONFIG.BATCH_SIZE) {
|
||||
batches.push(itemsWithLabels.slice(i, i + CONFIG.BATCH_SIZE));
|
||||
}
|
||||
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
console.log(chalk.cyan(`\nProcessing batch ${batchIndex + 1}/${batches.length} (${batch.length} items)...`));
|
||||
|
||||
const promises = batch.map(item =>
|
||||
this.limit(async () => {
|
||||
try {
|
||||
// Look up the page ID from our mapping
|
||||
const pageId = this.urlToPageId.get(item.url);
|
||||
|
||||
if (!pageId) {
|
||||
throw new Error(`Item not found in mapping for URL: ${item.url}`);
|
||||
}
|
||||
|
||||
// Apply labels
|
||||
const appliedLabels = await this.applyLabelsToItem(
|
||||
pageId,
|
||||
item.labels,
|
||||
item.title,
|
||||
item.url
|
||||
);
|
||||
|
||||
this.stats.successfulItems++;
|
||||
console.log(chalk.green(`✓ Applied ${appliedLabels.length} labels to: ${item.title || item.url}`));
|
||||
console.log(chalk.gray(` Labels: ${appliedLabels.map(l => l.name).join(', ')}`));
|
||||
|
||||
return { success: true, item, appliedLabels, pageId };
|
||||
} catch (error) {
|
||||
if (error.message.includes('not found in mapping')) {
|
||||
this.stats.notFoundItems.push({ item, error: error.message });
|
||||
console.error(chalk.yellow(`⚠ Item not found in Omnivore: ${item.title || item.url}`));
|
||||
} else {
|
||||
this.stats.failedItems.push({ item, error: error.message });
|
||||
console.error(chalk.red(`✗ Failed to process "${item.title}": ${error.message}`));
|
||||
}
|
||||
return { success: false, item, error: error.message };
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
await Promise.all(promises);
|
||||
|
||||
// Progress update
|
||||
this.stats.processedItems += batch.length;
|
||||
const progress = (this.stats.processedItems / this.stats.totalItemsWithLabels * 100).toFixed(1);
|
||||
console.log(chalk.yellow(`Progress: ${progress}% (${this.stats.processedItems}/${this.stats.totalItemsWithLabels} items)`));
|
||||
}
|
||||
}
|
||||
|
||||
// Generate report
|
||||
async generateReport() {
|
||||
const duration = (Date.now() - this.stats.startTime) / 1000;
|
||||
|
||||
const report = {
|
||||
labelApplication: {
|
||||
timestamp: new Date().toISOString(),
|
||||
duration: `${duration.toFixed(2)} seconds`,
|
||||
},
|
||||
summary: {
|
||||
totalItemsWithLabels: this.stats.totalItemsWithLabels,
|
||||
processedItems: this.stats.processedItems,
|
||||
successfulItems: this.stats.successfulItems,
|
||||
notFoundItems: this.stats.notFoundItems.length,
|
||||
failedItems: this.stats.failedItems.length,
|
||||
},
|
||||
notFoundItems: this.stats.notFoundItems.map(f => ({
|
||||
title: f.item.title,
|
||||
url: f.item.url,
|
||||
labels: f.item.labels,
|
||||
})),
|
||||
failures: this.stats.failedItems,
|
||||
};
|
||||
|
||||
const reportPath = join(__dirname, 'label-application-report.json');
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2));
|
||||
|
||||
console.log(chalk.blue(`\n📊 Label Application Report:`));
|
||||
console.log(chalk.green(`✓ Successfully labeled: ${report.summary.successfulItems}/${report.summary.totalItemsWithLabels} items`));
|
||||
|
||||
if (report.summary.notFoundItems > 0) {
|
||||
console.log(chalk.yellow(`⚠ Not found in Omnivore: ${report.summary.notFoundItems} items`));
|
||||
}
|
||||
|
||||
if (report.summary.failedItems > 0) {
|
||||
console.log(chalk.red(`✗ Failed items: ${report.summary.failedItems}`));
|
||||
}
|
||||
|
||||
console.log(chalk.blue(`📄 Full report saved to: ${reportPath}`));
|
||||
return report;
|
||||
}
|
||||
|
||||
// Main process
|
||||
async applyLabels() {
|
||||
try {
|
||||
console.log(chalk.bold.blue('🏷️ Starting Label Application Process (with Mapping)'));
|
||||
console.log(chalk.gray(`API: ${CONFIG.API_URL}`));
|
||||
|
||||
this.initDatabases();
|
||||
|
||||
// Extract original label associations
|
||||
console.log(chalk.blue('\n📤 Extracting label associations from archive SQLite...'));
|
||||
const itemsWithLabels = this.extractItemLabelAssociations();
|
||||
console.log(chalk.green(`✓ Found ${itemsWithLabels.length} items with labels`));
|
||||
|
||||
if (itemsWithLabels.length === 0) {
|
||||
console.log(chalk.yellow('No items with labels found. Nothing to do.'));
|
||||
return;
|
||||
}
|
||||
|
||||
// Show sample of items to be processed
|
||||
console.log(chalk.blue('\n📋 Sample items to process:'));
|
||||
itemsWithLabels.slice(0, 3).forEach((item, index) => {
|
||||
console.log(chalk.gray(`${index + 1}. ${item.title || 'No title'}`));
|
||||
console.log(chalk.gray(` URL: ${item.url}`));
|
||||
console.log(chalk.gray(` Labels: ${item.labels.join(', ')}`));
|
||||
});
|
||||
|
||||
// Fetch current labels from Omnivore
|
||||
console.log(chalk.blue('\n📋 Fetching current labels from Omnivore...'));
|
||||
await this.fetchLabelsFromOmnivore();
|
||||
|
||||
// Process items
|
||||
await this.processItems(itemsWithLabels);
|
||||
|
||||
// Generate report
|
||||
await this.generateReport();
|
||||
|
||||
console.log(chalk.bold.green('\n🎉 Label application completed!'));
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n💥 Label application failed:'), error.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
if (this.archiveDb) {
|
||||
this.archiveDb.close();
|
||||
}
|
||||
if (this.mappingDb) {
|
||||
this.mappingDb.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run the label application
|
||||
const applicator = new LabelApplicator();
|
||||
applicator.applyLabels();
|
||||
385
scripts/apply-labels.js
Normal file
385
scripts/apply-labels.js
Normal file
|
|
@ -0,0 +1,385 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import fetch from 'node-fetch';
|
||||
import pLimit from 'p-limit';
|
||||
import chalk from 'chalk';
|
||||
import { writeFile } from 'fs/promises';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Configuration
|
||||
const CONFIG = {
|
||||
API_URL:
|
||||
process.env.API_ENDPOINT ||
|
||||
process.env.API_URL ||
|
||||
'http://localhost:4000/api/graphql',
|
||||
API_KEY: process.env.OMNIVORE_API_KEY,
|
||||
ARCHIVE_DB_PATH:
|
||||
process.env.ARCHIVE_DB_PATH ||
|
||||
join(__dirname, '../self-hosting/archive-db/store.sqlite'),
|
||||
RATE_LIMIT: 3, // concurrent requests
|
||||
BATCH_SIZE: 10, // items to process per batch
|
||||
};
|
||||
|
||||
if (!CONFIG.API_KEY) {
|
||||
console.error(chalk.red('❌ OMNIVORE_API_KEY environment variable is required'));
|
||||
console.error(
|
||||
chalk.gray(
|
||||
'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/apply-labels.js',
|
||||
),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
class LabelApplicator {
|
||||
constructor() {
|
||||
this.db = null;
|
||||
this.stats = {
|
||||
totalItemsWithLabels: 0,
|
||||
processedItems: 0,
|
||||
failedItems: [],
|
||||
successfulItems: 0,
|
||||
startTime: Date.now(),
|
||||
};
|
||||
this.limit = pLimit(CONFIG.RATE_LIMIT);
|
||||
this.labelNameToId = new Map();
|
||||
this.urlToPageId = new Map();
|
||||
}
|
||||
|
||||
// Initialize database connection
|
||||
initDatabase() {
|
||||
try {
|
||||
this.db = new Database(CONFIG.ARCHIVE_DB_PATH, { readonly: true });
|
||||
console.log(chalk.green('✓ Connected to SQLite database'));
|
||||
} catch (error) {
|
||||
console.error(chalk.red('✗ Failed to connect to database:'), error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Make GraphQL request
|
||||
async makeGraphQLRequest(query, variables = {}) {
|
||||
const response = await fetch(CONFIG.API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Omnivore-Authorization': CONFIG.API_KEY,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query,
|
||||
variables,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.errors) {
|
||||
throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`);
|
||||
}
|
||||
|
||||
return result.data;
|
||||
}
|
||||
|
||||
// Extract items with their labels from SQLite
|
||||
extractItemLabelAssociations() {
|
||||
const query = `
|
||||
SELECT
|
||||
li.ZPAGEURLSTRING as url,
|
||||
li.ZTITLE as title,
|
||||
GROUP_CONCAT(label.ZNAME, '||') as labelNames
|
||||
FROM ZLINKEDITEM li
|
||||
JOIN Z_2LABELS l ON li.Z_PK = l.Z_2LINKEDITEMS
|
||||
JOIN ZLINKEDITEMLABEL label ON l.Z_3LABELS1 = label.Z_PK
|
||||
GROUP BY li.ZPAGEURLSTRING, li.ZTITLE
|
||||
ORDER BY li.ZPAGEURLSTRING
|
||||
`;
|
||||
|
||||
const results = this.db.prepare(query).all();
|
||||
this.stats.totalItemsWithLabels = results.length;
|
||||
|
||||
return results.map(row => ({
|
||||
url: row.url,
|
||||
title: row.title || '',
|
||||
labels: row.labelNames.split('||').filter(name => name && name.trim()),
|
||||
}));
|
||||
}
|
||||
|
||||
// Get all labels from the new Omnivore instance
|
||||
async fetchLabelsFromOmnivore() {
|
||||
const query = `
|
||||
query {
|
||||
labels {
|
||||
... on LabelsSuccess {
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
}
|
||||
}
|
||||
... on LabelsError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
try {
|
||||
const data = await this.makeGraphQLRequest(query);
|
||||
|
||||
if (data.labels.errorCodes) {
|
||||
throw new Error(`Failed to fetch labels: ${data.labels.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
// Build mapping of label name to ID
|
||||
data.labels.labels.forEach(label => {
|
||||
this.labelNameToId.set(label.name, label.id);
|
||||
});
|
||||
|
||||
console.log(chalk.green(`✓ Fetched ${data.labels.labels.length} labels from Omnivore`));
|
||||
return data.labels.labels;
|
||||
} catch (error) {
|
||||
console.error(chalk.red('✗ Failed to fetch labels:'), error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Search for an item by URL in the new Omnivore instance
|
||||
async findItemByUrl(url) {
|
||||
const query = `
|
||||
query SearchByUrl($query: String!) {
|
||||
search(query: $query, first: 1) {
|
||||
... on SearchSuccess {
|
||||
edges {
|
||||
node {
|
||||
id
|
||||
url
|
||||
title
|
||||
labels {
|
||||
id
|
||||
name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
... on SearchError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
try {
|
||||
// Create a search query that should match the exact URL
|
||||
const searchQuery = `url:"${url}"`;
|
||||
|
||||
const data = await this.makeGraphQLRequest(query, { query: searchQuery });
|
||||
|
||||
if (data.search.errorCodes) {
|
||||
throw new Error(`Search failed: ${data.search.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
const edges = data.search.edges || [];
|
||||
if (edges.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const item = edges[0].node;
|
||||
|
||||
// Verify URL match (sometimes search might return similar URLs)
|
||||
if (item.url === url) {
|
||||
return {
|
||||
id: item.id,
|
||||
url: item.url,
|
||||
title: item.title,
|
||||
currentLabels: item.labels || [],
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.error(chalk.yellow(`⚠ Failed to search for URL ${url}:`, error.message));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply labels to an item using setLabels mutation
|
||||
async applyLabelsToItem(pageId, labelNames, itemTitle = '') {
|
||||
// Convert label names to IDs
|
||||
const labelIds = labelNames
|
||||
.map(name => this.labelNameToId.get(name))
|
||||
.filter(id => id); // Remove any undefined IDs
|
||||
|
||||
if (labelIds.length === 0) {
|
||||
throw new Error(`No valid labels found for item. Requested: ${labelNames.join(', ')}`);
|
||||
}
|
||||
|
||||
const mutation = `
|
||||
mutation SetLabels($input: SetLabelsInput!) {
|
||||
setLabels(input: $input) {
|
||||
... on SetLabelsSuccess {
|
||||
labels {
|
||||
id
|
||||
name
|
||||
}
|
||||
}
|
||||
... on SetLabelsError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
try {
|
||||
const data = await this.makeGraphQLRequest(mutation, {
|
||||
input: {
|
||||
pageId: pageId,
|
||||
labelIds: labelIds,
|
||||
},
|
||||
});
|
||||
|
||||
if (data.setLabels.errorCodes) {
|
||||
throw new Error(`Failed to set labels: ${data.setLabels.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
return data.setLabels.labels;
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`✗ Failed to apply labels to "${itemTitle}":`, error.message));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Process items in batches
|
||||
async processItems(itemsWithLabels) {
|
||||
console.log(chalk.blue(`\n🏷️ Processing ${itemsWithLabels.length} items with labels...`));
|
||||
|
||||
const batches = [];
|
||||
for (let i = 0; i < itemsWithLabels.length; i += CONFIG.BATCH_SIZE) {
|
||||
batches.push(itemsWithLabels.slice(i, i + CONFIG.BATCH_SIZE));
|
||||
}
|
||||
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
console.log(chalk.cyan(`\nProcessing batch ${batchIndex + 1}/${batches.length} (${batch.length} items)...`));
|
||||
|
||||
const promises = batch.map(item =>
|
||||
this.limit(async () => {
|
||||
try {
|
||||
// Find the item in the new system
|
||||
const foundItem = await this.findItemByUrl(item.url);
|
||||
|
||||
if (!foundItem) {
|
||||
throw new Error(`Item not found in Omnivore for URL: ${item.url}`);
|
||||
}
|
||||
|
||||
// Apply labels
|
||||
const appliedLabels = await this.applyLabelsToItem(
|
||||
foundItem.id,
|
||||
item.labels,
|
||||
item.title
|
||||
);
|
||||
|
||||
this.stats.successfulItems++;
|
||||
console.log(chalk.green(`✓ Applied ${appliedLabels.length} labels to: ${item.title || foundItem.title}`));
|
||||
|
||||
return { success: true, item, appliedLabels };
|
||||
} catch (error) {
|
||||
this.stats.failedItems.push({ item, error: error.message });
|
||||
console.error(chalk.red(`✗ Failed to process "${item.title}": ${error.message}`));
|
||||
return { success: false, item, error: error.message };
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
await Promise.all(promises);
|
||||
|
||||
// Progress update
|
||||
this.stats.processedItems += batch.length;
|
||||
const progress = (this.stats.processedItems / this.stats.totalItemsWithLabels * 100).toFixed(1);
|
||||
console.log(chalk.yellow(`Progress: ${progress}% (${this.stats.processedItems}/${this.stats.totalItemsWithLabels} items)`));
|
||||
}
|
||||
}
|
||||
|
||||
// Generate report
|
||||
async generateReport() {
|
||||
const duration = (Date.now() - this.stats.startTime) / 1000;
|
||||
|
||||
const report = {
|
||||
labelApplication: {
|
||||
timestamp: new Date().toISOString(),
|
||||
duration: `${duration.toFixed(2)} seconds`,
|
||||
},
|
||||
summary: {
|
||||
totalItemsWithLabels: this.stats.totalItemsWithLabels,
|
||||
processedItems: this.stats.processedItems,
|
||||
successfulItems: this.stats.successfulItems,
|
||||
failedItems: this.stats.failedItems.length,
|
||||
},
|
||||
failures: this.stats.failedItems,
|
||||
};
|
||||
|
||||
const reportPath = join(__dirname, 'label-application-report.json');
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2));
|
||||
|
||||
console.log(chalk.blue(`\n📊 Label Application Report:`));
|
||||
console.log(chalk.green(`✓ Successfully labeled: ${report.summary.successfulItems}/${report.summary.totalItemsWithLabels} items`));
|
||||
|
||||
if (report.summary.failedItems > 0) {
|
||||
console.log(chalk.red(`✗ Failed items: ${report.summary.failedItems}`));
|
||||
}
|
||||
|
||||
console.log(chalk.blue(`📄 Full report saved to: ${reportPath}`));
|
||||
return report;
|
||||
}
|
||||
|
||||
// Main process
|
||||
async applyLabels() {
|
||||
try {
|
||||
console.log(chalk.bold.blue('🏷️ Starting Label Application Process'));
|
||||
console.log(chalk.gray(`API: ${CONFIG.API_URL}`));
|
||||
|
||||
this.initDatabase();
|
||||
|
||||
// Extract original label associations
|
||||
console.log(chalk.blue('\n📤 Extracting label associations from SQLite...'));
|
||||
const itemsWithLabels = this.extractItemLabelAssociations();
|
||||
console.log(chalk.green(`✓ Found ${itemsWithLabels.length} items with labels`));
|
||||
|
||||
if (itemsWithLabels.length === 0) {
|
||||
console.log(chalk.yellow('No items with labels found. Nothing to do.'));
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch current labels from Omnivore
|
||||
console.log(chalk.blue('\n📋 Fetching current labels from Omnivore...'));
|
||||
await this.fetchLabelsFromOmnivore();
|
||||
|
||||
// Process items
|
||||
await this.processItems(itemsWithLabels);
|
||||
|
||||
// Generate report
|
||||
await this.generateReport();
|
||||
|
||||
console.log(chalk.bold.green('\n🎉 Label application completed!'));
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n💥 Label application failed:'), error.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run the label application
|
||||
const applicator = new LabelApplicator();
|
||||
applicator.applyLabels();
|
||||
164
scripts/apply-single-label.js
Normal file
164
scripts/apply-single-label.js
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
import chalk from 'chalk';
|
||||
|
||||
const CONFIG = {
|
||||
API_URL:
|
||||
process.env.API_ENDPOINT ||
|
||||
process.env.API_URL ||
|
||||
'http://localhost:4000/api/graphql',
|
||||
API_KEY: process.env.OMNIVORE_API_KEY,
|
||||
};
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {};
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
const key = argv[i];
|
||||
if (!key.startsWith('--')) continue;
|
||||
const name = key.slice(2);
|
||||
const value = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
|
||||
args[name] = value;
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
const args = parseArgs(process.argv);
|
||||
const pageId = args.pageId || process.env.PAGE_ID;
|
||||
const labelName = args.label || process.env.LABEL_NAME;
|
||||
|
||||
if (!CONFIG.API_KEY) {
|
||||
console.error(chalk.red('❌ OMNIVORE_API_KEY environment variable is required'));
|
||||
console.error(
|
||||
chalk.gray(
|
||||
'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/apply-single-label.js --pageId <uuid> --label <name>',
|
||||
),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!pageId || !labelName) {
|
||||
console.error(chalk.red('❌ Missing required args'));
|
||||
console.error(
|
||||
chalk.gray(
|
||||
'Usage: node scripts/apply-single-label.js --pageId <uuid> --label <name>',
|
||||
),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Make GraphQL request
|
||||
async function makeGraphQLRequest(query, variables = {}) {
|
||||
const response = await fetch(CONFIG.API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Omnivore-Authorization': CONFIG.API_KEY,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query,
|
||||
variables,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.errors) {
|
||||
throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`);
|
||||
}
|
||||
|
||||
return result.data;
|
||||
}
|
||||
|
||||
// Get labels to find artifact label ID
|
||||
async function fetchLabels() {
|
||||
const query = `
|
||||
query {
|
||||
labels {
|
||||
... on LabelsSuccess {
|
||||
labels {
|
||||
id
|
||||
name
|
||||
color
|
||||
}
|
||||
}
|
||||
... on LabelsError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await makeGraphQLRequest(query);
|
||||
|
||||
if (data.labels.errorCodes) {
|
||||
throw new Error(`Failed to fetch labels: ${data.labels.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
return data.labels.labels;
|
||||
}
|
||||
|
||||
// Apply labels to an item using setLabels mutation
|
||||
async function applyLabel(pageId, labelId, labelName) {
|
||||
const mutation = `
|
||||
mutation SetLabels($input: SetLabelsInput!) {
|
||||
setLabels(input: $input) {
|
||||
... on SetLabelsSuccess {
|
||||
labels {
|
||||
id
|
||||
name
|
||||
}
|
||||
}
|
||||
... on SetLabelsError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await makeGraphQLRequest(mutation, {
|
||||
input: {
|
||||
pageId: pageId,
|
||||
labelIds: [labelId],
|
||||
},
|
||||
});
|
||||
|
||||
if (data.setLabels.errorCodes) {
|
||||
throw new Error(`Failed to set labels: ${data.setLabels.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
console.log(chalk.green(`✓ Applied label "${labelName}"`));
|
||||
return data.setLabels.labels;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
console.log(
|
||||
chalk.blue(`🏷️ Applying label "${labelName}" to page ${pageId}`),
|
||||
);
|
||||
|
||||
// Fetch labels to find artifact label ID
|
||||
const labels = await fetchLabels();
|
||||
const targetLabel = labels.find(l => l.name === labelName);
|
||||
|
||||
if (!targetLabel) {
|
||||
throw new Error(`Label not found: ${labelName}`);
|
||||
}
|
||||
|
||||
console.log(chalk.gray(`Found label: ${targetLabel.id}`));
|
||||
|
||||
await applyLabel(pageId, targetLabel.id, labelName);
|
||||
|
||||
console.log(chalk.bold.green('✅ Label applied successfully!'));
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red('❌ Failed to apply label:'), error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
33
scripts/check-missing-labeled.sql
Normal file
33
scripts/check-missing-labeled.sql
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
-- Check which items with labels from archive are missing in current mappings
|
||||
.mode column
|
||||
.headers on
|
||||
|
||||
ATTACH DATABASE 'url-id-mapping.sqlite' AS mapping;
|
||||
ATTACH DATABASE '../self-hosting/archive-db/store.sqlite' AS archive;
|
||||
|
||||
-- Items with labels that couldn't be found
|
||||
WITH labeled_items AS (
|
||||
SELECT DISTINCT
|
||||
li.ZPAGEURLSTRING as url,
|
||||
li.ZTITLE as title,
|
||||
GROUP_CONCAT(label.ZNAME, ', ') as labels
|
||||
FROM archive.ZLINKEDITEM li
|
||||
JOIN archive.Z_2LABELS l ON li.Z_PK = l.Z_2LINKEDITEMS
|
||||
JOIN archive.ZLINKEDITEMLABEL label ON l.Z_3LABELS1 = label.Z_PK
|
||||
WHERE li.ZPAGEURLSTRING IN (
|
||||
'https://github.com/danswer-ai/danswer',
|
||||
'https://segment.com/blog/rebuilding-our-infrastructure/',
|
||||
'https://svgl.vercel.app',
|
||||
'https://www.hyperledger.org'
|
||||
)
|
||||
GROUP BY li.ZPAGEURLSTRING, li.ZTITLE
|
||||
)
|
||||
SELECT
|
||||
li.url as original_url,
|
||||
li.title,
|
||||
li.labels,
|
||||
m.url as mapped_url,
|
||||
m.id as mapped_id
|
||||
FROM labeled_items li
|
||||
LEFT JOIN mapping.item_mapping m ON li.url = m.url
|
||||
ORDER BY li.url;
|
||||
337
scripts/compare-urls.js
Normal file
337
scripts/compare-urls.js
Normal file
|
|
@ -0,0 +1,337 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import { readFileSync, writeFileSync } from 'fs';
|
||||
import { parse } from 'csv-parse/sync';
|
||||
import fetch from 'node-fetch';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Configuration
|
||||
const CONFIG = {
|
||||
API_URL:
|
||||
process.env.API_ENDPOINT ||
|
||||
process.env.API_URL ||
|
||||
'http://localhost:4000/api/graphql',
|
||||
API_KEY: process.env.OMNIVORE_API_KEY,
|
||||
CSV_PATH: join(process.env.HOME, 'Downloads', 'part_000000.csv'),
|
||||
BATCH_SIZE: 100, // GraphQL pagination size
|
||||
OUTPUT_DIR: __dirname,
|
||||
};
|
||||
|
||||
if (!CONFIG.API_KEY) {
|
||||
throw new Error('OMNIVORE_API_KEY environment variable is not set');
|
||||
}
|
||||
|
||||
// GraphQL query to fetch all URLs
|
||||
const FETCH_URLS_QUERY = `
|
||||
query FetchUrls($after: String, $first: Int) {
|
||||
search(
|
||||
query: "in:all sort:saved-desc"
|
||||
after: $after
|
||||
first: $first
|
||||
) {
|
||||
... on SearchSuccess {
|
||||
pageInfo {
|
||||
totalCount
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
edges {
|
||||
cursor
|
||||
node {
|
||||
id
|
||||
title
|
||||
url
|
||||
originalArticleUrl
|
||||
createdAt
|
||||
updatedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
... on SearchError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
// Fetch all URLs from Omnivore with pagination
|
||||
async function fetchAllOmnivoreUrls() {
|
||||
const allUrls = [];
|
||||
let hasNextPage = true;
|
||||
let after = null;
|
||||
let pageCount = 0;
|
||||
|
||||
console.log('🔍 Fetching all URLs from Omnivore...');
|
||||
|
||||
while (hasNextPage) {
|
||||
pageCount++;
|
||||
console.log(`📄 Fetching page ${pageCount}...`);
|
||||
|
||||
try {
|
||||
const response = await fetch(CONFIG.API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Omnivore-Authorization': CONFIG.API_KEY,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query: FETCH_URLS_QUERY,
|
||||
variables: {
|
||||
after,
|
||||
first: CONFIG.BATCH_SIZE,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.errors) {
|
||||
throw new Error(`GraphQL errors: ${JSON.stringify(result.errors)}`);
|
||||
}
|
||||
|
||||
const searchResult = result.data.search;
|
||||
|
||||
if (searchResult.errorCodes) {
|
||||
throw new Error(`Search error: ${searchResult.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
const items = searchResult.edges || [];
|
||||
console.log(` Found ${items.length} items on page ${pageCount}`);
|
||||
|
||||
// Extract URLs and metadata
|
||||
for (const edge of items) {
|
||||
const item = edge.node;
|
||||
allUrls.push({
|
||||
id: item.id,
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
originalUrl: item.originalArticleUrl || item.url,
|
||||
createdAt: item.createdAt,
|
||||
updatedAt: item.updatedAt,
|
||||
});
|
||||
}
|
||||
|
||||
hasNextPage = searchResult.pageInfo.hasNextPage;
|
||||
after = searchResult.pageInfo.endCursor;
|
||||
|
||||
if (pageCount === 1) {
|
||||
console.log(`📊 Total items in Omnivore: ${searchResult.pageInfo.totalCount}`);
|
||||
}
|
||||
|
||||
// Add delay to be nice to the API
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
} catch (error) {
|
||||
console.error(`❌ Error fetching page ${pageCount}:`, error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Fetched ${allUrls.length} URLs from Omnivore`);
|
||||
return allUrls;
|
||||
}
|
||||
|
||||
// Parse CSV file and extract URLs
|
||||
function parseCSVUrls() {
|
||||
console.log('📋 Reading CSV file...');
|
||||
|
||||
try {
|
||||
const csvContent = readFileSync(CONFIG.CSV_PATH, 'utf-8');
|
||||
const records = parse(csvContent, {
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
trim: true,
|
||||
});
|
||||
|
||||
console.log(`✅ Found ${records.length} items in CSV`);
|
||||
|
||||
return records.map(record => ({
|
||||
title: record.title || 'No Title',
|
||||
url: record.resolved_url || record.given_url || record.url || '',
|
||||
givenUrl: record.given_url || '',
|
||||
resolvedUrl: record.resolved_url || '',
|
||||
timeAdded: record.time_added ? new Date(parseInt(record.time_added) * 1000).toISOString() : null,
|
||||
tags: record.tags || '',
|
||||
}));
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Error reading CSV file:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize URL for comparison (remove protocol, www, trailing slash, fragments)
|
||||
function normalizeUrl(url) {
|
||||
if (!url) return '';
|
||||
|
||||
return url
|
||||
.toLowerCase()
|
||||
.replace(/^https?:\/\//, '') // Remove protocol
|
||||
.replace(/^www\./, '') // Remove www
|
||||
.replace(/\/$/, '') // Remove trailing slash
|
||||
.replace(/#.*$/, '') // Remove fragments
|
||||
.replace(/\?.*$/, ''); // Remove query parameters (optional - comment out to keep params)
|
||||
}
|
||||
|
||||
// Compare URLs
|
||||
function compareUrls(omnivoreUrls, csvUrls) {
|
||||
console.log('🔍 Comparing URLs...');
|
||||
|
||||
// Create normalized lookup sets
|
||||
const omnivoreNormalized = new Map();
|
||||
const csvNormalized = new Map();
|
||||
|
||||
// Build Omnivore lookup
|
||||
for (const item of omnivoreUrls) {
|
||||
const normalized = normalizeUrl(item.url);
|
||||
const originalNormalized = normalizeUrl(item.originalUrl);
|
||||
|
||||
if (normalized) {
|
||||
omnivoreNormalized.set(normalized, item);
|
||||
}
|
||||
if (originalNormalized && originalNormalized !== normalized) {
|
||||
omnivoreNormalized.set(originalNormalized, item);
|
||||
}
|
||||
}
|
||||
|
||||
// Build CSV lookup
|
||||
for (const item of csvUrls) {
|
||||
const normalizedUrl = normalizeUrl(item.url);
|
||||
const normalizedGiven = normalizeUrl(item.givenUrl);
|
||||
const normalizedResolved = normalizeUrl(item.resolvedUrl);
|
||||
|
||||
if (normalizedUrl) {
|
||||
csvNormalized.set(normalizedUrl, item);
|
||||
}
|
||||
if (normalizedGiven && normalizedGiven !== normalizedUrl) {
|
||||
csvNormalized.set(normalizedGiven, item);
|
||||
}
|
||||
if (normalizedResolved && normalizedResolved !== normalizedUrl && normalizedResolved !== normalizedGiven) {
|
||||
csvNormalized.set(normalizedResolved, item);
|
||||
}
|
||||
}
|
||||
|
||||
// Find matches and differences
|
||||
const inBoth = [];
|
||||
const onlyInOmnivore = [];
|
||||
const onlyInCSV = [];
|
||||
|
||||
// Check Omnivore items
|
||||
for (const [normalizedUrl, omnivoreItem] of omnivoreNormalized) {
|
||||
if (csvNormalized.has(normalizedUrl)) {
|
||||
inBoth.push({
|
||||
url: normalizedUrl,
|
||||
omnivore: omnivoreItem,
|
||||
csv: csvNormalized.get(normalizedUrl),
|
||||
});
|
||||
} else {
|
||||
onlyInOmnivore.push(omnivoreItem);
|
||||
}
|
||||
}
|
||||
|
||||
// Check CSV items not in Omnivore
|
||||
for (const [normalizedUrl, csvItem] of csvNormalized) {
|
||||
if (!omnivoreNormalized.has(normalizedUrl)) {
|
||||
onlyInCSV.push(csvItem);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
inBoth,
|
||||
onlyInOmnivore,
|
||||
onlyInCSV,
|
||||
stats: {
|
||||
omnivoreTotal: omnivoreUrls.length,
|
||||
csvTotal: csvUrls.length,
|
||||
matches: inBoth.length,
|
||||
omnivoreOnly: onlyInOmnivore.length,
|
||||
csvOnly: onlyInCSV.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Generate comparison report
|
||||
function generateReport(comparison) {
|
||||
const { inBoth, onlyInOmnivore, onlyInCSV, stats } = comparison;
|
||||
|
||||
console.log('\n📊 COMPARISON REPORT');
|
||||
console.log('==================');
|
||||
console.log(`📚 Total in Omnivore: ${stats.omnivoreTotal}`);
|
||||
console.log(`📋 Total in CSV: ${stats.csvTotal}`);
|
||||
console.log(`✅ URLs in both: ${stats.matches}`);
|
||||
console.log(`🔵 Only in Omnivore: ${stats.omnivoreOnly}`);
|
||||
console.log(`🔴 Only in CSV (missing from Omnivore): ${stats.csvOnly}`);
|
||||
console.log(`📈 Import success rate: ${((stats.matches / stats.csvTotal) * 100).toFixed(1)}%`);
|
||||
|
||||
// Save detailed reports
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
|
||||
// Save URLs only in CSV (missing from Omnivore)
|
||||
if (onlyInCSV.length > 0) {
|
||||
const missingFile = join(CONFIG.OUTPUT_DIR, `missing-from-omnivore-${timestamp}.json`);
|
||||
writeFileSync(missingFile, JSON.stringify(onlyInCSV, null, 2));
|
||||
console.log(`\n💾 Saved ${onlyInCSV.length} missing URLs to: ${missingFile}`);
|
||||
|
||||
// Save a simple list for easier processing
|
||||
const missingList = join(CONFIG.OUTPUT_DIR, `missing-urls-list-${timestamp}.txt`);
|
||||
const missingUrlsList = onlyInCSV.map(item => item.url).join('\n');
|
||||
writeFileSync(missingList, missingUrlsList);
|
||||
console.log(`💾 Saved missing URLs list to: ${missingList}`);
|
||||
}
|
||||
|
||||
// Save full comparison report
|
||||
const reportFile = join(CONFIG.OUTPUT_DIR, `url-comparison-report-${timestamp}.json`);
|
||||
writeFileSync(reportFile, JSON.stringify({
|
||||
timestamp: new Date().toISOString(),
|
||||
stats,
|
||||
inBoth: inBoth.slice(0, 10), // First 10 matches as examples
|
||||
onlyInOmnivore: onlyInOmnivore.slice(0, 10), // First 10 examples
|
||||
onlyInCSV: onlyInCSV.slice(0, 100), // First 100 missing items
|
||||
fullStats: {
|
||||
totalMatches: inBoth.length,
|
||||
totalOmnivoreOnly: onlyInOmnivore.length,
|
||||
totalCSVOnly: onlyInCSV.length,
|
||||
},
|
||||
}, null, 2));
|
||||
console.log(`💾 Saved full comparison report to: ${reportFile}`);
|
||||
|
||||
return comparison;
|
||||
}
|
||||
|
||||
// Main function
|
||||
async function main() {
|
||||
try {
|
||||
console.log('🚀 Starting URL comparison...');
|
||||
|
||||
if (!CONFIG.API_KEY) {
|
||||
throw new Error('OMNIVORE_API_KEY environment variable is not set');
|
||||
}
|
||||
|
||||
// Fetch all URLs from both sources
|
||||
const [omnivoreUrls, csvUrls] = await Promise.all([
|
||||
fetchAllOmnivoreUrls(),
|
||||
Promise.resolve(parseCSVUrls()),
|
||||
]);
|
||||
|
||||
// Compare the URLs
|
||||
const comparison = compareUrls(omnivoreUrls, csvUrls);
|
||||
|
||||
// Generate and save report
|
||||
generateReport(comparison);
|
||||
|
||||
console.log('\n✅ Comparison complete!');
|
||||
|
||||
} catch (error) {
|
||||
console.error('\n❌ Error during comparison:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the script
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
224
scripts/download-items-mapping.js
Executable file
224
scripts/download-items-mapping.js
Executable file
|
|
@ -0,0 +1,224 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import fetch from 'node-fetch';
|
||||
import chalk from 'chalk';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Configuration
|
||||
const CONFIG = {
|
||||
API_URL:
|
||||
process.env.API_ENDPOINT ||
|
||||
process.env.API_URL ||
|
||||
'http://localhost:4000/api/graphql',
|
||||
API_KEY: process.env.OMNIVORE_API_KEY,
|
||||
MAPPING_DB_PATH:
|
||||
process.env.MAPPING_DB_PATH || join(__dirname, 'url-id-mapping.sqlite'),
|
||||
BATCH_SIZE: 100, // GraphQL allows up to 100 items per request
|
||||
};
|
||||
|
||||
if (!CONFIG.API_KEY) {
|
||||
console.error(chalk.red('❌ OMNIVORE_API_KEY environment variable is required'));
|
||||
console.error(
|
||||
chalk.gray(
|
||||
'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/download-items-mapping.js',
|
||||
),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
class ItemsDownloader {
|
||||
constructor() {
|
||||
this.db = null;
|
||||
this.stats = {
|
||||
totalItems: 0,
|
||||
downloadedItems: 0,
|
||||
startTime: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// Initialize mapping database
|
||||
initMappingDatabase() {
|
||||
try {
|
||||
this.db = new Database(CONFIG.MAPPING_DB_PATH);
|
||||
console.log(chalk.green('✓ Created mapping database'));
|
||||
|
||||
// Create mapping table
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS item_mapping (
|
||||
id TEXT PRIMARY KEY,
|
||||
url TEXT NOT NULL,
|
||||
title TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_url ON item_mapping(url);
|
||||
`);
|
||||
|
||||
// Clear existing data for fresh download
|
||||
this.db.exec('DELETE FROM item_mapping');
|
||||
console.log(chalk.blue('✓ Initialized item_mapping table'));
|
||||
} catch (error) {
|
||||
console.error(chalk.red('✗ Failed to initialize database:'), error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Make GraphQL request
|
||||
async makeGraphQLRequest(query, variables = {}) {
|
||||
const response = await fetch(CONFIG.API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Omnivore-Authorization': CONFIG.API_KEY,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query,
|
||||
variables,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.errors) {
|
||||
throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`);
|
||||
}
|
||||
|
||||
return result.data;
|
||||
}
|
||||
|
||||
// Download all items with pagination
|
||||
async downloadAllItems() {
|
||||
const query = `
|
||||
query GetItems($after: String, $first: Int) {
|
||||
search(query: "", after: $after, first: $first) {
|
||||
... on SearchSuccess {
|
||||
edges {
|
||||
cursor
|
||||
node {
|
||||
id
|
||||
url
|
||||
title
|
||||
}
|
||||
}
|
||||
pageInfo {
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
}
|
||||
... on SearchError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
let hasNextPage = true;
|
||||
let cursor = null;
|
||||
let pageCount = 0;
|
||||
|
||||
while (hasNextPage) {
|
||||
try {
|
||||
const data = await this.makeGraphQLRequest(query, {
|
||||
after: cursor,
|
||||
first: CONFIG.BATCH_SIZE,
|
||||
});
|
||||
|
||||
if (data.search.errorCodes) {
|
||||
throw new Error(`Search failed: ${data.search.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
const { edges, pageInfo } = data.search;
|
||||
|
||||
// Insert items into database
|
||||
const insertStmt = this.db.prepare(
|
||||
'INSERT OR REPLACE INTO item_mapping (id, url, title) VALUES (?, ?, ?)'
|
||||
);
|
||||
|
||||
const insertMany = this.db.transaction((items) => {
|
||||
for (const item of items) {
|
||||
insertStmt.run(item.node.id, item.node.url, item.node.title);
|
||||
}
|
||||
});
|
||||
|
||||
insertMany(edges);
|
||||
|
||||
this.stats.downloadedItems += edges.length;
|
||||
pageCount++;
|
||||
|
||||
console.log(chalk.green(
|
||||
`✓ Downloaded page ${pageCount}: ${edges.length} items (Total: ${this.stats.downloadedItems})`
|
||||
));
|
||||
|
||||
hasNextPage = pageInfo.hasNextPage;
|
||||
cursor = pageInfo.endCursor;
|
||||
|
||||
// Small delay to avoid rate limiting
|
||||
if (hasNextPage) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`✗ Error downloading page ${pageCount + 1}:`), error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Generate report
|
||||
async generateReport() {
|
||||
const duration = (Date.now() - this.stats.startTime) / 1000;
|
||||
|
||||
// Get stats from database
|
||||
const totalCount = this.db.prepare('SELECT COUNT(*) as count FROM item_mapping').get().count;
|
||||
const sampleItems = this.db.prepare('SELECT * FROM item_mapping LIMIT 5').all();
|
||||
|
||||
console.log(chalk.blue(`\n📊 Download Report:`));
|
||||
console.log(chalk.green(`✓ Total items downloaded: ${totalCount}`));
|
||||
console.log(chalk.gray(`⏱️ Duration: ${duration.toFixed(2)} seconds`));
|
||||
console.log(chalk.blue(`\n📄 Sample items:`));
|
||||
|
||||
sampleItems.forEach((item, index) => {
|
||||
console.log(chalk.gray(`${index + 1}. ${item.title || 'No title'}`));
|
||||
console.log(chalk.gray(` ID: ${item.id}`));
|
||||
console.log(chalk.gray(` URL: ${item.url}\n`));
|
||||
});
|
||||
|
||||
console.log(chalk.green(`✓ Mapping database saved to: ${CONFIG.MAPPING_DB_PATH}`));
|
||||
}
|
||||
|
||||
// Main process
|
||||
async download() {
|
||||
try {
|
||||
console.log(chalk.bold.blue('📥 Starting Items Download Process'));
|
||||
console.log(chalk.gray(`API: ${CONFIG.API_URL}`));
|
||||
|
||||
this.initMappingDatabase();
|
||||
|
||||
console.log(chalk.blue('\n📤 Downloading all items from Omnivore...'));
|
||||
await this.downloadAllItems();
|
||||
|
||||
await this.generateReport();
|
||||
|
||||
console.log(chalk.bold.green('\n🎉 Download completed successfully!'));
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n💥 Download failed:'), error.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run the download
|
||||
const downloader = new ItemsDownloader();
|
||||
downloader.download();
|
||||
24
scripts/find-redirected-urls.sql
Normal file
24
scripts/find-redirected-urls.sql
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
-- Search for partial matches to find redirected URLs
|
||||
.mode column
|
||||
.headers on
|
||||
|
||||
ATTACH DATABASE 'url-id-mapping.sqlite' AS mapping;
|
||||
|
||||
-- Search for potential matches
|
||||
SELECT
|
||||
url,
|
||||
title,
|
||||
id
|
||||
FROM mapping.item_mapping
|
||||
WHERE
|
||||
url LIKE '%danswer%'
|
||||
OR url LIKE '%segment.com%'
|
||||
OR url LIKE '%svgl%'
|
||||
OR url LIKE '%hyperledger%'
|
||||
OR title LIKE '%danswer%'
|
||||
OR title LIKE '%Segment%'
|
||||
OR title LIKE '%Svgl%'
|
||||
OR title LIKE '%SVG%'
|
||||
OR title LIKE '%Hyperledger%'
|
||||
ORDER BY url
|
||||
LIMIT 20;
|
||||
1268
scripts/import-pocket.js
Normal file
1268
scripts/import-pocket.js
Normal file
File diff suppressed because it is too large
Load diff
481
scripts/migrate-omnivore.js
Normal file
481
scripts/migrate-omnivore.js
Normal file
|
|
@ -0,0 +1,481 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import fetch from 'node-fetch';
|
||||
import pLimit from 'p-limit';
|
||||
import chalk from 'chalk';
|
||||
import { writeFile } from 'fs/promises';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Configuration
|
||||
const CONFIG = {
|
||||
API_URL:
|
||||
process.env.API_ENDPOINT ||
|
||||
process.env.API_URL ||
|
||||
'http://localhost:4000/api/graphql',
|
||||
API_KEY: process.env.OMNIVORE_API_KEY,
|
||||
DB_PATH: join(__dirname, '../self-hosting/archive-db/store.sqlite'),
|
||||
BATCH_SIZE: 50,
|
||||
RATE_LIMIT: 5, // concurrent requests
|
||||
TEST_MODE: process.argv.includes('--test'),
|
||||
TEST_LIMIT: 10, // items to process in test mode
|
||||
};
|
||||
|
||||
if (!CONFIG.API_KEY) {
|
||||
console.error('❌ OMNIVORE_API_KEY environment variable is required');
|
||||
console.error(
|
||||
'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/migrate-omnivore.js'
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Core Data timestamp offset (seconds between 1970-01-01 and 2001-01-01)
|
||||
const CORE_DATA_EPOCH_OFFSET = 978307200;
|
||||
|
||||
class OmnivoreMigrator {
|
||||
constructor() {
|
||||
this.db = null;
|
||||
this.stats = {
|
||||
totalItems: 0,
|
||||
totalLabels: 0,
|
||||
processedItems: 0,
|
||||
processedLabels: 0,
|
||||
failedItems: [],
|
||||
failedLabels: [],
|
||||
startTime: Date.now(),
|
||||
};
|
||||
this.limit = pLimit(CONFIG.RATE_LIMIT);
|
||||
}
|
||||
|
||||
// Convert Core Data timestamp to ISO string
|
||||
convertTimestamp(coreDataTimestamp) {
|
||||
if (!coreDataTimestamp) return null;
|
||||
const unixTimestamp = coreDataTimestamp + CORE_DATA_EPOCH_OFFSET;
|
||||
return new Date(unixTimestamp * 1000).toISOString();
|
||||
}
|
||||
|
||||
// Initialize database connection
|
||||
initDatabase() {
|
||||
try {
|
||||
this.db = new Database(CONFIG.DB_PATH, { readonly: true });
|
||||
console.log(chalk.green('✓ Connected to SQLite database'));
|
||||
} catch (error) {
|
||||
console.error(chalk.red('✗ Failed to connect to database:'), error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract labels from SQLite
|
||||
extractLabels() {
|
||||
const query = `
|
||||
SELECT
|
||||
ZID as id,
|
||||
ZNAME as name,
|
||||
ZCOLOR as color,
|
||||
ZLABELDESCRIPTION as description,
|
||||
ZCREATEDAT as createdAt
|
||||
FROM ZLINKEDITEMLABEL
|
||||
ORDER BY ZNAME
|
||||
`;
|
||||
|
||||
const labels = this.db.prepare(query).all();
|
||||
this.stats.totalLabels = labels.length;
|
||||
|
||||
return labels.map(label => ({
|
||||
id: label.id,
|
||||
name: label.name,
|
||||
color: label.color || '#3B82F6', // default blue if no color
|
||||
description: label.description || '',
|
||||
createdAt: this.convertTimestamp(label.createdAt),
|
||||
}));
|
||||
}
|
||||
|
||||
// Extract items with their labels from SQLite
|
||||
extractItems() {
|
||||
// Check if we should only get missing items
|
||||
const onlyMissing = process.argv.includes('--missing-only');
|
||||
// Check if we should get specific missing labeled items
|
||||
const missingLabeled = process.argv.includes('--missing-labeled');
|
||||
|
||||
const itemsQuery = missingLabeled ? `
|
||||
SELECT
|
||||
li.ZID as id,
|
||||
li.ZPAGEURLSTRING as url,
|
||||
li.ZTITLE as title,
|
||||
li.ZDESCRIPTIONTEXT as description,
|
||||
li.ZAUTHOR as author,
|
||||
li.ZSITENAME as siteName,
|
||||
li.ZCREATEDAT as createdAt,
|
||||
li.ZSAVEDAT as savedAt,
|
||||
li.ZUPDATEDAT as updatedAt,
|
||||
li.ZPUBLISHDATE as publishedAt,
|
||||
li.ZISARCHIVED as isArchived,
|
||||
li.ZREADAT as readAt
|
||||
FROM ZLINKEDITEM li
|
||||
WHERE li.ZPAGEURLSTRING IN (
|
||||
'https://github.com/danswer-ai/danswer',
|
||||
'https://segment.com/blog/rebuilding-our-infrastructure/',
|
||||
'https://www.hyperledger.org'
|
||||
)
|
||||
ORDER BY li.ZCREATEDAT
|
||||
` : onlyMissing ? `
|
||||
SELECT
|
||||
li.ZID as id,
|
||||
li.ZPAGEURLSTRING as url,
|
||||
li.ZTITLE as title,
|
||||
li.ZDESCRIPTIONTEXT as description,
|
||||
li.ZAUTHOR as author,
|
||||
li.ZSITENAME as siteName,
|
||||
li.ZCREATEDAT as createdAt,
|
||||
li.ZSAVEDAT as savedAt,
|
||||
li.ZUPDATEDAT as updatedAt,
|
||||
li.ZPUBLISHDATE as publishedAt,
|
||||
li.ZISARCHIVED as isArchived,
|
||||
li.ZREADAT as readAt
|
||||
FROM ZLINKEDITEM li
|
||||
LEFT JOIN omnivore_mapping om ON li.ZPAGEURLSTRING = om.url
|
||||
WHERE om.url IS NULL AND li.ZPAGEURLSTRING = 'https://svgl.vercel.app'
|
||||
ORDER BY li.ZCREATEDAT
|
||||
LIMIT 1
|
||||
` : `
|
||||
SELECT
|
||||
li.ZID as id,
|
||||
li.ZPAGEURLSTRING as url,
|
||||
li.ZTITLE as title,
|
||||
li.ZDESCRIPTIONTEXT as description,
|
||||
li.ZAUTHOR as author,
|
||||
li.ZSITENAME as siteName,
|
||||
li.ZCREATEDAT as createdAt,
|
||||
li.ZSAVEDAT as savedAt,
|
||||
li.ZUPDATEDAT as updatedAt,
|
||||
li.ZPUBLISHDATE as publishedAt,
|
||||
li.ZISARCHIVED as isArchived,
|
||||
li.ZREADAT as readAt
|
||||
FROM ZLINKEDITEM li
|
||||
ORDER BY li.ZCREATEDAT
|
||||
LIMIT 3
|
||||
`;
|
||||
|
||||
const items = this.db.prepare(itemsQuery).all();
|
||||
this.stats.totalItems = items.length;
|
||||
|
||||
// Get labels for each item
|
||||
const labelsQuery = `
|
||||
SELECT l.ZNAME as labelName
|
||||
FROM Z_2LABELS rel
|
||||
JOIN ZLINKEDITEMLABEL l ON rel.Z_3LABELS1 = l.Z_PK
|
||||
WHERE rel.Z_2LINKEDITEMS = ?
|
||||
`;
|
||||
const getLabels = this.db.prepare(labelsQuery);
|
||||
|
||||
return items.map(item => {
|
||||
const labels = getLabels.all(item.id).map(l => l.labelName);
|
||||
|
||||
return {
|
||||
id: item.id,
|
||||
url: item.url,
|
||||
title: item.title || '',
|
||||
description: item.description || '',
|
||||
author: item.author || '',
|
||||
siteName: item.siteName || '',
|
||||
labels: labels,
|
||||
createdAt: this.convertTimestamp(item.createdAt),
|
||||
savedAt: this.convertTimestamp(item.savedAt),
|
||||
updatedAt: this.convertTimestamp(item.updatedAt),
|
||||
publishedAt: this.convertTimestamp(item.publishedAt),
|
||||
isArchived: Boolean(item.isArchived),
|
||||
readAt: this.convertTimestamp(item.readAt),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// Make GraphQL request
|
||||
async makeGraphQLRequest(query, variables = {}) {
|
||||
const response = await fetch(CONFIG.API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Omnivore-Authorization': CONFIG.API_KEY,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
query,
|
||||
variables,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
// Log the full response for debugging
|
||||
if (result.errors || !result.data) {
|
||||
console.log(chalk.yellow('GraphQL Response:'), JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
if (result.errors) {
|
||||
throw new Error(`GraphQL Error: ${JSON.stringify(result.errors)}`);
|
||||
}
|
||||
|
||||
return result.data;
|
||||
}
|
||||
|
||||
// Test authentication with the API key
|
||||
async testAuthentication() {
|
||||
try {
|
||||
const query = `
|
||||
query {
|
||||
me {
|
||||
id
|
||||
name
|
||||
email
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await this.makeGraphQLRequest(query);
|
||||
if (data.me) {
|
||||
console.log(chalk.green(`✓ Authentication successful for user: ${data.me.name} (${data.me.email})`));
|
||||
return true;
|
||||
} else {
|
||||
console.log(chalk.red('✗ Authentication failed: No user returned'));
|
||||
return false;
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(chalk.red('✗ Authentication failed:', error.message));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Create a label in the new instance
|
||||
async createLabel(label) {
|
||||
const mutation = `
|
||||
mutation CreateLabel($input: CreateLabelInput!) {
|
||||
createLabel(input: $input) {
|
||||
... on CreateLabelSuccess {
|
||||
label {
|
||||
id
|
||||
name
|
||||
color
|
||||
}
|
||||
}
|
||||
... on CreateLabelError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
try {
|
||||
const data = await this.makeGraphQLRequest(mutation, {
|
||||
input: {
|
||||
name: label.name,
|
||||
color: label.color,
|
||||
description: label.description,
|
||||
},
|
||||
});
|
||||
|
||||
if (data.createLabel.errorCodes) {
|
||||
throw new Error(`Failed to create label: ${data.createLabel.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
return data.createLabel.label;
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`✗ Failed to create label "${label.name}":`, error.message));
|
||||
this.stats.failedLabels.push({ label, error: error.message });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Save an item to the new instance
|
||||
async saveItem(item) {
|
||||
const mutation = `
|
||||
mutation SaveUrl($input: SaveUrlInput!) {
|
||||
saveUrl(input: $input) {
|
||||
... on SaveSuccess {
|
||||
url
|
||||
clientRequestId
|
||||
}
|
||||
... on SaveError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
try {
|
||||
const labelInputs = item.labels.map(labelName => ({ name: labelName }));
|
||||
|
||||
const data = await this.makeGraphQLRequest(mutation, {
|
||||
input: {
|
||||
url: item.url,
|
||||
source: 'migration',
|
||||
clientRequestId: item.id,
|
||||
labels: labelInputs.length > 0 ? labelInputs : undefined,
|
||||
savedAt: item.savedAt,
|
||||
publishedAt: item.publishedAt,
|
||||
},
|
||||
});
|
||||
|
||||
if (data.saveUrl.errorCodes) {
|
||||
throw new Error(`Failed to save item: ${data.saveUrl.errorCodes.join(', ')}`);
|
||||
}
|
||||
|
||||
return data.saveUrl;
|
||||
} catch (error) {
|
||||
console.error(chalk.red(`✗ Failed to save item "${item.title}":`, error.message));
|
||||
this.stats.failedItems.push({ item, error: error.message });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Migrate labels
|
||||
async migrateLabels(labels) {
|
||||
console.log(chalk.blue(`\n📋 Migrating ${labels.length} labels...`));
|
||||
|
||||
for (const label of labels) {
|
||||
await this.limit(async () => {
|
||||
const result = await this.createLabel(label);
|
||||
if (result) {
|
||||
this.stats.processedLabels++;
|
||||
console.log(chalk.green(`✓ Created label: ${label.name}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Migrate items in batches
|
||||
async migrateItems(items) {
|
||||
const totalItems = CONFIG.TEST_MODE ? Math.min(items.length, CONFIG.TEST_LIMIT) : items.length;
|
||||
const itemsToProcess = items.slice(0, totalItems);
|
||||
|
||||
console.log(chalk.blue(`\n📚 Migrating ${totalItems} items...`));
|
||||
|
||||
const batches = [];
|
||||
for (let i = 0; i < itemsToProcess.length; i += CONFIG.BATCH_SIZE) {
|
||||
batches.push(itemsToProcess.slice(i, i + CONFIG.BATCH_SIZE));
|
||||
}
|
||||
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex];
|
||||
console.log(chalk.cyan(`\nProcessing batch ${batchIndex + 1}/${batches.length} (${batch.length} items)...`));
|
||||
|
||||
const promises = batch.map(item =>
|
||||
this.limit(async () => {
|
||||
const result = await this.saveItem(item);
|
||||
if (result) {
|
||||
this.stats.processedItems++;
|
||||
console.log(chalk.green(`✓ Saved: ${item.title || item.url}`));
|
||||
}
|
||||
return result;
|
||||
})
|
||||
);
|
||||
|
||||
await Promise.all(promises);
|
||||
|
||||
// Progress update
|
||||
const progress = ((batchIndex + 1) / batches.length * 100).toFixed(1);
|
||||
console.log(chalk.yellow(`Progress: ${progress}% (${this.stats.processedItems}/${totalItems} items)`));
|
||||
}
|
||||
}
|
||||
|
||||
// Generate migration report
|
||||
async generateReport() {
|
||||
const duration = (Date.now() - this.stats.startTime) / 1000;
|
||||
|
||||
const report = {
|
||||
migration: {
|
||||
timestamp: new Date().toISOString(),
|
||||
duration: `${duration.toFixed(2)} seconds`,
|
||||
testMode: CONFIG.TEST_MODE,
|
||||
},
|
||||
summary: {
|
||||
totalLabels: this.stats.totalLabels,
|
||||
processedLabels: this.stats.processedLabels,
|
||||
failedLabels: this.stats.failedLabels.length,
|
||||
totalItems: CONFIG.TEST_MODE ? Math.min(this.stats.totalItems, CONFIG.TEST_LIMIT) : this.stats.totalItems,
|
||||
processedItems: this.stats.processedItems,
|
||||
failedItems: this.stats.failedItems.length,
|
||||
},
|
||||
failures: {
|
||||
labels: this.stats.failedLabels,
|
||||
items: this.stats.failedItems,
|
||||
},
|
||||
};
|
||||
|
||||
const reportPath = join(__dirname, 'migration-report.json');
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2));
|
||||
|
||||
console.log(chalk.blue(`\n📊 Migration Report:`));
|
||||
console.log(chalk.green(`✓ Labels: ${report.summary.processedLabels}/${report.summary.totalLabels}`));
|
||||
console.log(chalk.green(`✓ Items: ${report.summary.processedItems}/${report.summary.totalItems}`));
|
||||
|
||||
if (report.summary.failedLabels > 0) {
|
||||
console.log(chalk.red(`✗ Failed labels: ${report.summary.failedLabels}`));
|
||||
}
|
||||
if (report.summary.failedItems > 0) {
|
||||
console.log(chalk.red(`✗ Failed items: ${report.summary.failedItems}`));
|
||||
}
|
||||
|
||||
console.log(chalk.blue(`📄 Full report saved to: ${reportPath}`));
|
||||
return report;
|
||||
}
|
||||
|
||||
// Main migration process
|
||||
async migrate() {
|
||||
try {
|
||||
console.log(chalk.bold.blue('🚀 Starting Omnivore Migration'));
|
||||
console.log(chalk.gray(`Mode: ${CONFIG.TEST_MODE ? 'TEST' : 'FULL'}`));
|
||||
console.log(chalk.gray(`API: ${CONFIG.API_URL}`));
|
||||
|
||||
this.initDatabase();
|
||||
|
||||
// Test authentication first
|
||||
console.log(chalk.blue('\n🔑 Testing API authentication...'));
|
||||
const authResult = await this.testAuthentication();
|
||||
if (!authResult) {
|
||||
console.log(chalk.red('Migration aborted due to authentication failure.'));
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract data
|
||||
console.log(chalk.blue('\n📤 Extracting data from SQLite database...'));
|
||||
const labels = this.extractLabels();
|
||||
const items = this.extractItems();
|
||||
|
||||
console.log(chalk.green(`✓ Extracted ${labels.length} labels and ${items.length} items`));
|
||||
|
||||
// Migrate labels first
|
||||
if (labels.length > 0) {
|
||||
await this.migrateLabels(labels);
|
||||
}
|
||||
|
||||
// Migrate items
|
||||
if (items.length > 0) {
|
||||
await this.migrateItems(items);
|
||||
}
|
||||
|
||||
// Generate report
|
||||
await this.generateReport();
|
||||
|
||||
console.log(chalk.bold.green('\n🎉 Migration completed!'));
|
||||
|
||||
} catch (error) {
|
||||
console.error(chalk.red('\n💥 Migration failed:'), error.message);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run migration
|
||||
const migrator = new OmnivoreMigrator();
|
||||
migrator.migrate();
|
||||
20
scripts/package.json
Normal file
20
scripts/package.json
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"name": "omnivore-migration",
|
||||
"version": "1.0.0",
|
||||
"description": "Migrate data from old Omnivore SQLite database to new self-hosted instance",
|
||||
"main": "migrate-omnivore.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"migrate": "node migrate-omnivore.js",
|
||||
"test": "node migrate-omnivore.js --test"
|
||||
},
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^12.4.1",
|
||||
"chalk": "^5.3.0",
|
||||
"csv-parse": "^6.1.0",
|
||||
"node-fetch": "^3.3.2",
|
||||
"p-limit": "^5.0.0"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC"
|
||||
}
|
||||
369
scripts/pnpm-lock.yaml
Normal file
369
scripts/pnpm-lock.yaml
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
lockfileVersion: '9.0'
|
||||
|
||||
settings:
|
||||
autoInstallPeers: true
|
||||
excludeLinksFromLockfile: false
|
||||
|
||||
importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
better-sqlite3:
|
||||
specifier: ^12.4.1
|
||||
version: 12.4.1
|
||||
chalk:
|
||||
specifier: ^5.3.0
|
||||
version: 5.6.2
|
||||
csv-parse:
|
||||
specifier: ^6.1.0
|
||||
version: 6.1.0
|
||||
node-fetch:
|
||||
specifier: ^3.3.2
|
||||
version: 3.3.2
|
||||
p-limit:
|
||||
specifier: ^5.0.0
|
||||
version: 5.0.0
|
||||
|
||||
packages:
|
||||
|
||||
base64-js@1.5.1:
|
||||
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
|
||||
|
||||
better-sqlite3@12.4.1:
|
||||
resolution: {integrity: sha512-3yVdyZhklTiNrtg+4WqHpJpFDd+WHTg2oM7UcR80GqL05AOV0xEJzc6qNvFYoEtE+hRp1n9MpN6/+4yhlGkDXQ==}
|
||||
engines: {node: 20.x || 22.x || 23.x || 24.x}
|
||||
|
||||
bindings@1.5.0:
|
||||
resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
|
||||
|
||||
bl@4.1.0:
|
||||
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
|
||||
|
||||
buffer@5.7.1:
|
||||
resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
|
||||
|
||||
chalk@5.6.2:
|
||||
resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==}
|
||||
engines: {node: ^12.17.0 || ^14.13 || >=16.0.0}
|
||||
|
||||
chownr@1.1.4:
|
||||
resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
|
||||
|
||||
csv-parse@6.1.0:
|
||||
resolution: {integrity: sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==}
|
||||
|
||||
data-uri-to-buffer@4.0.1:
|
||||
resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
|
||||
engines: {node: '>= 12'}
|
||||
|
||||
decompress-response@6.0.0:
|
||||
resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
deep-extend@0.6.0:
|
||||
resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
|
||||
engines: {node: '>=4.0.0'}
|
||||
|
||||
detect-libc@2.1.0:
|
||||
resolution: {integrity: sha512-vEtk+OcP7VBRtQZ1EJ3bdgzSfBjgnEalLTp5zjJrS+2Z1w2KZly4SBdac/WDU3hhsNAZ9E8SC96ME4Ey8MZ7cg==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
end-of-stream@1.4.5:
|
||||
resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==}
|
||||
|
||||
expand-template@2.0.3:
|
||||
resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
fetch-blob@3.2.0:
|
||||
resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
|
||||
engines: {node: ^12.20 || >= 14.13}
|
||||
|
||||
file-uri-to-path@1.0.0:
|
||||
resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==}
|
||||
|
||||
formdata-polyfill@4.0.10:
|
||||
resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
|
||||
engines: {node: '>=12.20.0'}
|
||||
|
||||
fs-constants@1.0.0:
|
||||
resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
|
||||
|
||||
github-from-package@0.0.0:
|
||||
resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==}
|
||||
|
||||
ieee754@1.2.1:
|
||||
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
||||
|
||||
inherits@2.0.4:
|
||||
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
|
||||
|
||||
ini@1.3.8:
|
||||
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
|
||||
|
||||
mimic-response@3.1.0:
|
||||
resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
minimist@1.2.8:
|
||||
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
|
||||
|
||||
mkdirp-classic@0.5.3:
|
||||
resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
|
||||
|
||||
napi-build-utils@2.0.0:
|
||||
resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==}
|
||||
|
||||
node-abi@3.77.0:
|
||||
resolution: {integrity: sha512-DSmt0OEcLoK4i3NuscSbGjOf3bqiDEutejqENSplMSFA/gmB8mkED9G4pKWnPl7MDU4rSHebKPHeitpDfyH0cQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
node-domexception@1.0.0:
|
||||
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
|
||||
engines: {node: '>=10.5.0'}
|
||||
deprecated: Use your platform's native DOMException instead
|
||||
|
||||
node-fetch@3.3.2:
|
||||
resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
|
||||
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
|
||||
|
||||
once@1.4.0:
|
||||
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
|
||||
|
||||
p-limit@5.0.0:
|
||||
resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
prebuild-install@7.1.3:
|
||||
resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==}
|
||||
engines: {node: '>=10'}
|
||||
hasBin: true
|
||||
|
||||
pump@3.0.3:
|
||||
resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==}
|
||||
|
||||
rc@1.2.8:
|
||||
resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==}
|
||||
hasBin: true
|
||||
|
||||
readable-stream@3.6.2:
|
||||
resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
|
||||
engines: {node: '>= 6'}
|
||||
|
||||
safe-buffer@5.2.1:
|
||||
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
|
||||
|
||||
semver@7.7.2:
|
||||
resolution: {integrity: sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==}
|
||||
engines: {node: '>=10'}
|
||||
hasBin: true
|
||||
|
||||
simple-concat@1.0.1:
|
||||
resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
|
||||
|
||||
simple-get@4.0.1:
|
||||
resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
|
||||
|
||||
string_decoder@1.3.0:
|
||||
resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==}
|
||||
|
||||
strip-json-comments@2.0.1:
|
||||
resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
tar-fs@2.1.4:
|
||||
resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==}
|
||||
|
||||
tar-stream@2.2.0:
|
||||
resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
tunnel-agent@0.6.0:
|
||||
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
|
||||
|
||||
util-deprecate@1.0.2:
|
||||
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
|
||||
|
||||
web-streams-polyfill@3.3.3:
|
||||
resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}
|
||||
engines: {node: '>= 8'}
|
||||
|
||||
wrappy@1.0.2:
|
||||
resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
|
||||
|
||||
yocto-queue@1.2.1:
|
||||
resolution: {integrity: sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==}
|
||||
engines: {node: '>=12.20'}
|
||||
|
||||
snapshots:
|
||||
|
||||
base64-js@1.5.1: {}
|
||||
|
||||
better-sqlite3@12.4.1:
|
||||
dependencies:
|
||||
bindings: 1.5.0
|
||||
prebuild-install: 7.1.3
|
||||
|
||||
bindings@1.5.0:
|
||||
dependencies:
|
||||
file-uri-to-path: 1.0.0
|
||||
|
||||
bl@4.1.0:
|
||||
dependencies:
|
||||
buffer: 5.7.1
|
||||
inherits: 2.0.4
|
||||
readable-stream: 3.6.2
|
||||
|
||||
buffer@5.7.1:
|
||||
dependencies:
|
||||
base64-js: 1.5.1
|
||||
ieee754: 1.2.1
|
||||
|
||||
chalk@5.6.2: {}
|
||||
|
||||
chownr@1.1.4: {}
|
||||
|
||||
csv-parse@6.1.0: {}
|
||||
|
||||
data-uri-to-buffer@4.0.1: {}
|
||||
|
||||
decompress-response@6.0.0:
|
||||
dependencies:
|
||||
mimic-response: 3.1.0
|
||||
|
||||
deep-extend@0.6.0: {}
|
||||
|
||||
detect-libc@2.1.0: {}
|
||||
|
||||
end-of-stream@1.4.5:
|
||||
dependencies:
|
||||
once: 1.4.0
|
||||
|
||||
expand-template@2.0.3: {}
|
||||
|
||||
fetch-blob@3.2.0:
|
||||
dependencies:
|
||||
node-domexception: 1.0.0
|
||||
web-streams-polyfill: 3.3.3
|
||||
|
||||
file-uri-to-path@1.0.0: {}
|
||||
|
||||
formdata-polyfill@4.0.10:
|
||||
dependencies:
|
||||
fetch-blob: 3.2.0
|
||||
|
||||
fs-constants@1.0.0: {}
|
||||
|
||||
github-from-package@0.0.0: {}
|
||||
|
||||
ieee754@1.2.1: {}
|
||||
|
||||
inherits@2.0.4: {}
|
||||
|
||||
ini@1.3.8: {}
|
||||
|
||||
mimic-response@3.1.0: {}
|
||||
|
||||
minimist@1.2.8: {}
|
||||
|
||||
mkdirp-classic@0.5.3: {}
|
||||
|
||||
napi-build-utils@2.0.0: {}
|
||||
|
||||
node-abi@3.77.0:
|
||||
dependencies:
|
||||
semver: 7.7.2
|
||||
|
||||
node-domexception@1.0.0: {}
|
||||
|
||||
node-fetch@3.3.2:
|
||||
dependencies:
|
||||
data-uri-to-buffer: 4.0.1
|
||||
fetch-blob: 3.2.0
|
||||
formdata-polyfill: 4.0.10
|
||||
|
||||
once@1.4.0:
|
||||
dependencies:
|
||||
wrappy: 1.0.2
|
||||
|
||||
p-limit@5.0.0:
|
||||
dependencies:
|
||||
yocto-queue: 1.2.1
|
||||
|
||||
prebuild-install@7.1.3:
|
||||
dependencies:
|
||||
detect-libc: 2.1.0
|
||||
expand-template: 2.0.3
|
||||
github-from-package: 0.0.0
|
||||
minimist: 1.2.8
|
||||
mkdirp-classic: 0.5.3
|
||||
napi-build-utils: 2.0.0
|
||||
node-abi: 3.77.0
|
||||
pump: 3.0.3
|
||||
rc: 1.2.8
|
||||
simple-get: 4.0.1
|
||||
tar-fs: 2.1.4
|
||||
tunnel-agent: 0.6.0
|
||||
|
||||
pump@3.0.3:
|
||||
dependencies:
|
||||
end-of-stream: 1.4.5
|
||||
once: 1.4.0
|
||||
|
||||
rc@1.2.8:
|
||||
dependencies:
|
||||
deep-extend: 0.6.0
|
||||
ini: 1.3.8
|
||||
minimist: 1.2.8
|
||||
strip-json-comments: 2.0.1
|
||||
|
||||
readable-stream@3.6.2:
|
||||
dependencies:
|
||||
inherits: 2.0.4
|
||||
string_decoder: 1.3.0
|
||||
util-deprecate: 1.0.2
|
||||
|
||||
safe-buffer@5.2.1: {}
|
||||
|
||||
semver@7.7.2: {}
|
||||
|
||||
simple-concat@1.0.1: {}
|
||||
|
||||
simple-get@4.0.1:
|
||||
dependencies:
|
||||
decompress-response: 6.0.0
|
||||
once: 1.4.0
|
||||
simple-concat: 1.0.1
|
||||
|
||||
string_decoder@1.3.0:
|
||||
dependencies:
|
||||
safe-buffer: 5.2.1
|
||||
|
||||
strip-json-comments@2.0.1: {}
|
||||
|
||||
tar-fs@2.1.4:
|
||||
dependencies:
|
||||
chownr: 1.1.4
|
||||
mkdirp-classic: 0.5.3
|
||||
pump: 3.0.3
|
||||
tar-stream: 2.2.0
|
||||
|
||||
tar-stream@2.2.0:
|
||||
dependencies:
|
||||
bl: 4.1.0
|
||||
end-of-stream: 1.4.5
|
||||
fs-constants: 1.0.0
|
||||
inherits: 2.0.4
|
||||
readable-stream: 3.6.2
|
||||
|
||||
tunnel-agent@0.6.0:
|
||||
dependencies:
|
||||
safe-buffer: 5.2.1
|
||||
|
||||
util-deprecate@1.0.2: {}
|
||||
|
||||
web-streams-polyfill@3.3.3: {}
|
||||
|
||||
wrappy@1.0.2: {}
|
||||
|
||||
yocto-queue@1.2.1: {}
|
||||
2
scripts/pnpm-workspace.yaml
Normal file
2
scripts/pnpm-workspace.yaml
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
ignoredBuiltDependencies:
|
||||
- better-sqlite3
|
||||
53
scripts/test-auth.js
Normal file
53
scripts/test-auth.js
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
const API_URL =
|
||||
process.env.API_ENDPOINT ||
|
||||
process.env.API_URL ||
|
||||
'http://localhost:4000/api/graphql';
|
||||
const API_KEY = process.env.OMNIVORE_API_KEY;
|
||||
|
||||
if (!API_KEY) {
|
||||
console.error('❌ OMNIVORE_API_KEY environment variable is required');
|
||||
console.error('Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/test-auth.js');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
async function testAuth() {
|
||||
console.log('Testing API authentication...');
|
||||
console.log('API URL:', API_URL);
|
||||
console.log('API Key:', `${API_KEY.substring(0, 8)}...`);
|
||||
|
||||
const query = `
|
||||
query {
|
||||
me {
|
||||
id
|
||||
name
|
||||
email
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
try {
|
||||
const response = await fetch(API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Omnivore-Authorization': API_KEY,
|
||||
},
|
||||
body: JSON.stringify({ query }),
|
||||
});
|
||||
|
||||
console.log('Response status:', response.status);
|
||||
console.log('Response headers:', Object.fromEntries(response.headers.entries()));
|
||||
|
||||
const result = await response.json();
|
||||
console.log('Response body:', JSON.stringify(result, null, 2));
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
testAuth();
|
||||
67
scripts/test-create-label.js
Normal file
67
scripts/test-create-label.js
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
const API_URL =
|
||||
process.env.API_ENDPOINT ||
|
||||
process.env.API_URL ||
|
||||
'http://localhost:4000/api/graphql';
|
||||
const API_KEY = process.env.OMNIVORE_API_KEY;
|
||||
|
||||
if (!API_KEY) {
|
||||
console.error('❌ OMNIVORE_API_KEY environment variable is required');
|
||||
console.error(
|
||||
'Example: direnv exec . env OMNIVORE_API_KEY=... node scripts/test-create-label.js',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
async function testCreateLabel() {
|
||||
console.log('Testing label creation with API key...');
|
||||
|
||||
const mutation = `
|
||||
mutation CreateLabel($input: CreateLabelInput!) {
|
||||
createLabel(input: $input) {
|
||||
... on CreateLabelSuccess {
|
||||
label {
|
||||
id
|
||||
name
|
||||
color
|
||||
}
|
||||
}
|
||||
... on CreateLabelError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const variables = {
|
||||
input: {
|
||||
name: 'test-migration-label',
|
||||
color: '#FF0000',
|
||||
description: 'Test label created by migration script'
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch(API_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Omnivore-Authorization': API_KEY,
|
||||
},
|
||||
body: JSON.stringify({ query: mutation, variables }),
|
||||
});
|
||||
|
||||
console.log('Response status:', response.status);
|
||||
|
||||
const result = await response.json();
|
||||
console.log('Response body:', JSON.stringify(result, null, 2));
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
testCreateLabel();
|
||||
Loading…
Reference in a new issue