diff --git a/bin/cli.js b/bin/cli.js index f32f8ffa..3e7f8a8d 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -42,6 +42,7 @@ import setPodcast from '../commands/set-podcast.js'; import cleanSlugs from '../commands/clean-slugs.js'; import commentNotifications from '../commands/comment-notifications.js'; import memberNewsletterBackup from '../commands/member-newsletter-backup.js'; +import importJson from '../commands/import-json.js'; prettyCLI.command(addMemberCompSubscriptionCommands); prettyCLI.command(removeMemberCompSubscriptionCommands); @@ -82,6 +83,7 @@ prettyCLI.command(setPodcast); prettyCLI.command(cleanSlugs); prettyCLI.command(commentNotifications); prettyCLI.command(memberNewsletterBackup); +prettyCLI.command(importJson); prettyCLI.style({ usageCommandPlaceholder: () => '' diff --git a/commands/import-json.js b/commands/import-json.js new file mode 100644 index 00000000..745f40d5 --- /dev/null +++ b/commands/import-json.js @@ -0,0 +1,104 @@ +import {ui} from '@tryghost/pretty-cli'; +import importJson from '../tasks/import-json.js'; + +// Internal ID in case we need one. +const id = 'import-json'; + +const group = 'Content:'; + +// The command to run and any params +const flags = 'import-json '; + +// Description for the top level command +const desc = 'Import posts from a Ghost JSON export file'; + +// Descriptions for the individual params +const paramsDesc = [ + 'URL to your Ghost API', + 'Admin API key', + 'Path to the Ghost JSON export file' +]; + +// Configure all the options +const setup = (sywac) => { + sywac.boolean('-V --verbose', { + defaultValue: false, + desc: 'Show verbose output' + }); + sywac.boolean('-y --yes', { + defaultValue: false, + desc: 'Skip confirmation prompt' + }); + sywac.string('--importStatus', { + defaultValue: null, + desc: 'Override post status (null = retain original, "draft", "published")' + }); + sywac.enumeration('--contentType', { + defaultValue: 'all', + choices: ['all', 'posts', 'pages'], + desc: 'Type of content to import' + }); + sywac.number('--delayBetweenCalls', { + defaultValue: 50, + desc: 'The delay between API calls, in ms' + }); + sywac.boolean('--dryRun', { + defaultValue: false, + desc: 'Preview import without making changes' + }); + sywac.boolean('--skipCacheRefresh', { + defaultValue: false, + desc: 'Skip API fetch, use existing cache (faster for multiple files)' + }); + sywac.string('--fallback', { + defaultValue: null, + desc: 'User slug to use as fallback author when no matching author is found' + }); +}; + +// What to do when this command is executed +const run = async (argv) => { + let timer = Date.now(); + let context = {errors: []}; + + try { + // Fetch the tasks, configured correctly according to the options passed in + let runner = importJson.getTaskRunner(argv); + + // Run the migration + await runner.run(context); + + // Print any warnings about skipped posts + importJson.printWarnings(context); + + // Report success + if (argv.dryRun) { + ui.log.info(`Dry run complete. ${context.newPosts?.length || 0} posts would be imported.`); + } else { + ui.log.ok(`Successfully imported ${context.imported?.length || 0} posts in ${Date.now() - timer}ms.`); + } + + if (context.skipped?.length > 0) { + ui.log.warn(`${context.skipped.length} posts skipped due to missing authors.`); + } + + if (context.duplicatePosts?.length > 0) { + ui.log.info(`${context.duplicatePosts.length} duplicate posts were skipped.`); + } + } catch (error) { + ui.log.error('Done with errors', context.errors); + } finally { + // Clean up database connection + importJson.cleanup(context); + } +}; + +export default { + id, + group, + flags, + desc, + paramsDesc, + setup, + run +}; diff --git a/commands/interactive.js b/commands/interactive.js index 2f04e47d..db169347 100644 --- a/commands/interactive.js +++ b/commands/interactive.js @@ -104,6 +104,10 @@ const run = async () => { name: tasks.setTemplate.choice.name, value: tasks.setTemplate.choice.value }, + { + name: tasks.importJson.choice.name, + value: tasks.importJson.choice.value + }, new inquirer.Separator('--- Members API Utilities ---------'), { name: tasks.addMemberCompSubscription.choice.name, diff --git a/lib/import-db.js b/lib/import-db.js new file mode 100644 index 00000000..468fd694 --- /dev/null +++ b/lib/import-db.js @@ -0,0 +1,262 @@ +import {join, dirname} from 'node:path'; +import {createHash} from 'node:crypto'; +import fs from 'fs-extra'; +import Database from 'better-sqlite3'; + +/** + * Get a hash of the API URL for use in the database filename + * @param {string} apiURL - The Ghost API URL + * @returns {string} - A short hash of the URL + */ +const getUrlHash = (apiURL) => { + return createHash('md5').update(apiURL).digest('hex').slice(0, 12); +}; + +/** + * Open or create a persistent SQLite database for caching Ghost content + * @param {string} apiURL - The Ghost API URL (used to create unique db per site) + * @param {string} jsonFile - Path to the JSON import file (db stored in same folder) + * @returns {Database} - The SQLite database instance + */ +const openDatabase = (apiURL, jsonFile) => { + const jsonDir = dirname(jsonFile); + const urlHash = getUrlHash(apiURL); + const dbPath = join(jsonDir, `import-cache-${urlHash}.db`); + + const db = new Database(dbPath); + + // Enable WAL mode for better performance + db.pragma('journal_mode = WAL'); + + // Create tables if they don't exist + db.exec(` + CREATE TABLE IF NOT EXISTS posts ( + id TEXT PRIMARY KEY, + slug TEXT UNIQUE, + title TEXT, + type TEXT + ); + CREATE TABLE IF NOT EXISTS tags ( + id TEXT PRIMARY KEY, + slug TEXT UNIQUE, + name TEXT + ); + CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY, + slug TEXT UNIQUE, + email TEXT, + name TEXT + ); + CREATE INDEX IF NOT EXISTS idx_posts_slug ON posts(slug); + CREATE INDEX IF NOT EXISTS idx_tags_slug ON tags(slug); + CREATE INDEX IF NOT EXISTS idx_users_slug ON users(slug); + CREATE INDEX IF NOT EXISTS idx_users_email ON users(email); + + CREATE TABLE IF NOT EXISTS imported_slugs ( + slug TEXT PRIMARY KEY, + imported_at TEXT + ); + `); + + return db; +}; + +/** + * Refresh the cache by fetching all posts, tags, and users from the Ghost API + * @param {Database} db - The SQLite database instance + * @param {GhostAdminAPI} api - The Ghost Admin API client + * @param {Function} discover - The discover function for batch fetching + * @param {Object} options - Options for the refresh + * @returns {Object} - Counts of items cached + */ +const refreshCache = async (db, api, discover, options = {}) => { + const counts = {posts: 0, tags: 0, users: 0}; + + // Fetch and cache posts + const posts = await discover({ + api, + type: 'posts', + limit: 100, + fields: 'id,slug,title,type', + progress: options.verbose + }); + + const upsertPost = db.prepare(` + INSERT INTO posts (id, slug, title, type) VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET slug = excluded.slug, title = excluded.title, type = excluded.type + `); + + const insertPosts = db.transaction((postList) => { + for (const post of postList) { + upsertPost.run(post.id, post.slug, post.title, post.type || 'post'); + } + }); + insertPosts(posts); + counts.posts = posts.length; + + // Fetch and cache pages + const pages = await discover({ + api, + type: 'pages', + limit: 100, + fields: 'id,slug,title', + progress: options.verbose + }); + + const insertPages = db.transaction((pageList) => { + for (const page of pageList) { + upsertPost.run(page.id, page.slug, page.title, 'page'); + } + }); + insertPages(pages); + counts.posts += pages.length; + + // Fetch and cache tags + const tags = await discover({ + api, + type: 'tags', + limit: 100, + fields: 'id,slug,name', + progress: options.verbose + }); + + const upsertTag = db.prepare(` + INSERT INTO tags (id, slug, name) VALUES (?, ?, ?) + ON CONFLICT(id) DO UPDATE SET slug = excluded.slug, name = excluded.name + `); + + const insertTags = db.transaction((tagList) => { + for (const tag of tagList) { + upsertTag.run(tag.id, tag.slug, tag.name); + } + }); + insertTags(tags); + counts.tags = tags.length; + + // Fetch and cache users + const users = await discover({ + api, + type: 'users', + limit: 100, + fields: 'id,slug,email,name', + progress: options.verbose + }); + + const upsertUser = db.prepare(` + INSERT INTO users (id, slug, email, name) VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET slug = excluded.slug, email = excluded.email, name = excluded.name + `); + + const insertUsers = db.transaction((userList) => { + for (const user of userList) { + upsertUser.run(user.id, user.slug, user.email, user.name); + } + }); + insertUsers(users); + counts.users = users.length; + + return counts; +}; + +/** + * Check if a post exists by slug + * @param {Database} db - The SQLite database instance + * @param {string} slug - The post slug to check + * @returns {boolean} - True if the post exists + */ +const postExistsBySlug = (db, slug) => { + const stmt = db.prepare('SELECT 1 FROM posts WHERE slug = ?'); + return stmt.get(slug) !== undefined; +}; + +/** + * Find a tag by slug + * @param {Database} db - The SQLite database instance + * @param {string} slug - The tag slug to find + * @returns {Object|null} - The tag object or null if not found + */ +const findTagBySlug = (db, slug) => { + const stmt = db.prepare('SELECT id, slug, name FROM tags WHERE slug = ?'); + return stmt.get(slug) || null; +}; + +/** + * Find a user by slug + * @param {Database} db - The SQLite database instance + * @param {string} slug - The user slug to find + * @returns {Object|null} - The user object or null if not found + */ +const findUserBySlug = (db, slug) => { + const stmt = db.prepare('SELECT id, slug, email, name FROM users WHERE slug = ?'); + return stmt.get(slug) || null; +}; + +/** + * Find a user by email + * @param {Database} db - The SQLite database instance + * @param {string} email - The user email to find + * @returns {Object|null} - The user object or null if not found + */ +const findUserByEmail = (db, email) => { + const stmt = db.prepare('SELECT id, slug, email, name FROM users WHERE email = ?'); + return stmt.get(email) || null; +}; + +/** + * Add a newly imported post to the cache + * @param {Database} db - The SQLite database instance + * @param {Object} post - The post object to add + */ +const addPost = (db, post) => { + const stmt = db.prepare(` + INSERT INTO posts (id, slug, title, type) VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET slug = excluded.slug, title = excluded.title, type = excluded.type + `); + stmt.run(post.id, post.slug, post.title, post.type || 'post'); +}; + +/** + * Check if a slug has already been imported from a JSON file + * @param {Database} db - The SQLite database instance + * @param {string} slug - The original JSON slug to check + * @returns {boolean} - True if already imported + */ +const wasSlugImported = (db, slug) => { + const stmt = db.prepare('SELECT 1 FROM imported_slugs WHERE slug = ?'); + return stmt.get(slug) !== undefined; +}; + +/** + * Record that a slug was imported from a JSON file + * @param {Database} db - The SQLite database instance + * @param {string} slug - The original JSON slug that was imported + */ +const markSlugImported = (db, slug) => { + const stmt = db.prepare(` + INSERT OR IGNORE INTO imported_slugs (slug, imported_at) VALUES (?, ?) + `); + stmt.run(slug, new Date().toISOString()); +}; + +/** + * Close the database connection + * @param {Database} db - The SQLite database instance + */ +const closeDatabase = (db) => { + if (db) { + db.close(); + } +}; + +export { + openDatabase, + refreshCache, + postExistsBySlug, + findTagBySlug, + findUserBySlug, + findUserByEmail, + addPost, + wasSlugImported, + markSlugImported, + closeDatabase +}; diff --git a/package.json b/package.json index eff091b0..93132943 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ }, "dependencies": { "@inquirer/confirm": "3.2.0", + "better-sqlite3": "11.8.1", "@tryghost/admin-api": "1.14.4", "@tryghost/listr-smart-renderer": "0.5.22", "@tryghost/logging": "2.5.0", diff --git a/prompts/import-json.js b/prompts/import-json.js new file mode 100644 index 00000000..7e87061b --- /dev/null +++ b/prompts/import-json.js @@ -0,0 +1,97 @@ +import inquirer from 'inquirer'; +import {ui} from '@tryghost/pretty-cli'; +import importJson from '../tasks/import-json.js'; +import ghostAPICreds from '../lib/ghost-api-creds.js'; + +const choice = { + name: 'Import posts from Ghost JSON export', + value: 'importJson' +}; + +const options = [ + ...ghostAPICreds, + { + type: 'input', + name: 'jsonFile', + message: 'Path to the Ghost JSON export file:', + validate: (val) => { + if (!val || val.trim() === '') { + return 'Please enter a file path'; + } + return true; + } + }, + { + type: 'list', + name: 'contentType', + message: 'Type of content to import:', + choices: [ + { + name: 'All (posts and pages)', + value: 'all' + }, + { + name: 'Posts only', + value: 'posts' + }, + { + name: 'Pages only', + value: 'pages' + } + ] + }, + { + type: 'list', + name: 'dryRun', + message: 'Dry run mode (preview without making changes)?', + choices: [ + { + name: 'No - perform actual import', + value: false + }, + { + name: 'Yes - preview only', + value: true + } + ] + } +]; + +async function run() { + await inquirer.prompt(options).then(async (answers) => { + let timer = Date.now(); + let context = {errors: []}; + + try { + let runner = importJson.getTaskRunner(answers); + await runner.run(context); + + // Print any warnings about skipped posts + importJson.printWarnings(context); + + if (answers.dryRun) { + ui.log.info(`Dry run complete. ${context.newPosts?.length || 0} posts would be imported.`); + } else { + ui.log.ok(`Successfully imported ${context.imported?.length || 0} posts in ${Date.now() - timer}ms.`); + } + + if (context.skipped?.length > 0) { + ui.log.warn(`${context.skipped.length} posts skipped due to missing authors.`); + } + + if (context.duplicatePosts?.length > 0) { + ui.log.info(`${context.duplicatePosts.length} duplicate posts were skipped.`); + } + } catch (error) { + ui.log.error('Done with errors', context.errors); + } finally { + importJson.cleanup(context); + } + }); +} + +export default { + choice, + options, + run +}; diff --git a/prompts/index.js b/prompts/index.js index 2b96a654..d4974c98 100644 --- a/prompts/index.js +++ b/prompts/index.js @@ -31,6 +31,7 @@ import getPosts from './get-posts.js'; import setTemplate from './set-template.js'; import commentNotifications from './comment-notifications.js'; import memberNewsletterBackup from './member-newsletter-backup.js'; +import importJson from './import-json.js'; export default { zipSplit, @@ -63,5 +64,6 @@ export default { getPosts, setTemplate, commentNotifications, - memberNewsletterBackup + memberNewsletterBackup, + importJson }; diff --git a/tasks/import-json.js b/tasks/import-json.js new file mode 100644 index 00000000..a3ec973f --- /dev/null +++ b/tasks/import-json.js @@ -0,0 +1,527 @@ +import fs from 'fs-extra'; +import GhostAdminAPI from '@tryghost/admin-api'; +import {makeTaskRunner} from '@tryghost/listr-smart-renderer'; +import _ from 'lodash'; +import {ui} from '@tryghost/pretty-cli'; +import {discover} from '../lib/batch-ghost-discover.js'; +import {sleep} from '../lib/utils.js'; +import { + openDatabase, + refreshCache, + postExistsBySlug, + findTagBySlug, + findUserBySlug, + findUserByEmail, + addPost, + wasSlugImported, + markSlugImported, + closeDatabase +} from '../lib/import-db.js'; + +/** + * Normalize a date value to ISO 8601 format + * Handles: ISO strings, Unix timestamps (ms), Date objects + */ +const normalizeDate = (value) => { + if (!value) { + return null; + } + + // Already a valid ISO string + if (typeof value === 'string' && value.includes('T') && value.includes('Z')) { + return value; + } + + // Try to parse as a date + const date = new Date(value); + if (!isNaN(date.getTime())) { + return date.toISOString(); + } + + return null; +}; + +const initialise = (options) => { + return { + title: 'Initialising API connection', + task: (ctx, task) => { + let defaults = { + verbose: false, + delayBetweenCalls: 50, + dryRun: false, + contentType: 'all', + skipCacheRefresh: false + }; + + const url = options.apiURL.replace(/\/$/, ''); + const key = options.adminAPIKey; + const api = new GhostAdminAPI({ + url: url.replace('localhost', '127.0.0.1'), + key, + version: 'v5.0' + }); + + ctx.args = _.mergeWith(defaults, options); + ctx.api = api; + ctx.db = openDatabase(url, options.jsonFile); + ctx.imported = []; + ctx.skipped = []; + ctx.warnings = []; + ctx.newPosts = []; + ctx.duplicatePosts = []; + + task.output = `Initialised API connection for ${options.apiURL}`; + } + }; +}; + +const refreshCacheTask = (options) => { + return { + title: 'Refreshing content cache from Ghost API', + skip: (ctx) => { + if (ctx.args.skipCacheRefresh) { + return 'Skipping cache refresh (--skipCacheRefresh)'; + } + return false; + }, + task: async (ctx, task) => { + try { + const counts = await refreshCache(ctx.db, ctx.api, discover, { + verbose: ctx.args.verbose + }); + task.output = `Cached ${counts.posts} posts/pages, ${counts.tags} tags, ${counts.users} users`; + } catch (error) { + ctx.errors.push(error); + throw error; + } + } + }; +}; + +const readJsonFile = (options) => { + return { + title: 'Reading JSON import file', + task: async (ctx, task) => { + try { + const jsonPath = options.jsonFile; + + if (!await fs.pathExists(jsonPath)) { + throw new Error(`File not found: ${jsonPath}`); + } + + const fileContent = await fs.readJson(jsonPath); + ctx.importData = fileContent; + + // Extract the data from Ghost export format + if (fileContent.db && Array.isArray(fileContent.db) && fileContent.db[0]?.data) { + ctx.importData = fileContent.db[0].data; + } else if (fileContent.data) { + ctx.importData = fileContent.data; + } + + const postCount = ctx.importData.posts?.length || 0; + task.output = `Read ${postCount} posts from JSON file`; + } catch (error) { + ctx.errors.push(error); + throw error; + } + } + }; +}; + +const buildImportMaps = () => { + return { + title: 'Building import data maps', + task: async (ctx, task) => { + // Build tag ID to tag object map + ctx.tagsById = new Map(); + if (ctx.importData.tags) { + for (const tag of ctx.importData.tags) { + ctx.tagsById.set(tag.id, tag); + } + } + + // Build user ID to user object map + ctx.usersById = new Map(); + if (ctx.importData.users) { + for (const user of ctx.importData.users) { + ctx.usersById.set(user.id, user); + } + } + + // Build post ID to tag IDs map + ctx.postsTags = new Map(); + if (ctx.importData.posts_tags) { + for (const pt of ctx.importData.posts_tags) { + if (!ctx.postsTags.has(pt.post_id)) { + ctx.postsTags.set(pt.post_id, []); + } + ctx.postsTags.get(pt.post_id).push(pt.tag_id); + } + } + + // Build post ID to author IDs map + ctx.postsAuthors = new Map(); + if (ctx.importData.posts_authors) { + for (const pa of ctx.importData.posts_authors) { + if (!ctx.postsAuthors.has(pa.post_id)) { + ctx.postsAuthors.set(pa.post_id, []); + } + ctx.postsAuthors.get(pa.post_id).push(pa.author_id); + } + } + + // Build post ID to meta data map + ctx.postsMeta = new Map(); + if (ctx.importData.posts_meta) { + for (const meta of ctx.importData.posts_meta) { + ctx.postsMeta.set(meta.post_id, meta); + } + } + + task.output = `Built maps: ${ctx.tagsById.size} tags, ${ctx.usersById.size} users, ${ctx.postsTags.size} post-tag relations, ${ctx.postsAuthors.size} post-author relations, ${ctx.postsMeta.size} post meta`; + } + }; +}; + +const resolveFallbackAuthor = () => { + return { + title: 'Resolving fallback author', + skip: (ctx) => { + if (!ctx.args.fallback) { + return 'No --fallback provided'; + } + return false; + }, + task: async (ctx, task) => { + const fallbackUser = findUserBySlug(ctx.db, ctx.args.fallback); + if (!fallbackUser) { + throw new Error(`Fallback author not found in Ghost: ${ctx.args.fallback}`); + } + ctx.fallbackAuthor = {id: fallbackUser.id}; + task.output = `Fallback author: ${fallbackUser.name || fallbackUser.slug} (${fallbackUser.id})`; + } + }; +}; + +const analyzeImport = (options) => { + return { + title: 'Analyzing import data', + task: async (ctx, task) => { + const posts = ctx.importData.posts || []; + + for (const post of posts) { + // Filter by content type + const postType = post.type || 'post'; + if (options.contentType !== 'all') { + if (options.contentType === 'posts' && postType !== 'post') { + continue; + } + if (options.contentType === 'pages' && postType !== 'page') { + continue; + } + } + + // Check if post already exists by slug OR was already imported + if (postExistsBySlug(ctx.db, post.slug) || wasSlugImported(ctx.db, post.slug)) { + ctx.duplicatePosts.push(post); + } else { + ctx.newPosts.push(post); + } + } + + task.output = `Found ${ctx.newPosts.length} new posts, ${ctx.duplicatePosts.length} duplicates`; + } + }; +}; + +const showImportSummary = () => { + return { + title: 'Import Summary', + task: async (ctx, task) => { + const summary = []; + summary.push(`New posts to import: ${ctx.newPosts.length}`); + summary.push(`Duplicate posts (skipped): ${ctx.duplicatePosts.length}`); + + if (ctx.args.dryRun) { + summary.push('DRY RUN: No changes will be made'); + } + + task.output = summary.join(' | '); + } + }; +}; + +const importPosts = (options) => { + return { + title: 'Importing posts', + skip: (ctx) => { + if (ctx.newPosts.length === 0) { + return 'No new posts to import'; + } + if (ctx.args.dryRun) { + return 'Dry run mode - skipping import'; + } + return false; + }, + task: async (ctx) => { + let tasks = []; + + for (const post of ctx.newPosts) { + tasks.push({ + title: `${post.title}`, + task: async (innerCtx, task) => { + try { + // Resolve tags for this post + const tagIds = ctx.postsTags.get(post.id) || []; + const resolvedTags = []; + + for (const tagId of tagIds) { + const importTag = ctx.tagsById.get(tagId); + if (importTag) { + // Check if tag exists in Ghost + const existingTag = findTagBySlug(ctx.db, importTag.slug); + if (existingTag) { + resolvedTags.push({id: existingTag.id}); + } else { + // Tag will be created inline by Ghost + resolvedTags.push({name: importTag.name, slug: importTag.slug}); + } + } + } + + // Resolve authors for this post + const authorIds = ctx.postsAuthors.get(post.id) || []; + const resolvedAuthors = []; + + for (const authorId of authorIds) { + const importUser = ctx.usersById.get(authorId); + if (importUser) { + // Try to find by slug first, then by email + let existingUser = findUserBySlug(ctx.db, importUser.slug); + if (!existingUser && importUser.email) { + existingUser = findUserByEmail(ctx.db, importUser.email); + } + + if (existingUser) { + resolvedAuthors.push({id: existingUser.id}); + } + } + } + + // Skip post if no HTML content + if (!post.html || post.html.trim() === '') { + ctx.warnings.push({ + post: post.title, + reason: 'No HTML content' + }); + ctx.skipped.push(post); + task.skip('No HTML content'); + return; + } + + // Use fallback author if no match found + if (resolvedAuthors.length === 0 && ctx.fallbackAuthor) { + resolvedAuthors.push(ctx.fallbackAuthor); + } + + // Skip post if no author match + if (resolvedAuthors.length === 0) { + const authorInfo = authorIds.map(id => { + const u = ctx.usersById.get(id); + return u ? `${u.name || u.slug} (${u.email || 'no email'})` : id; + }).join(', '); + + ctx.warnings.push({ + post: post.title, + reason: `No matching author found: ${authorInfo}` + }); + ctx.skipped.push(post); + task.skip(`No matching author: ${authorInfo}`); + return; + } + + // Prepare post data for import + const postData = { + title: post.title, + slug: post.slug, + status: options.importStatus || post.status || 'draft', + feature_image: post.feature_image, + featured: post.featured, + type: post.type || 'post', + custom_excerpt: post.custom_excerpt, + codeinjection_head: post.codeinjection_head, + codeinjection_foot: post.codeinjection_foot, + custom_template: post.custom_template, + canonical_url: post.canonical_url, + tags: resolvedTags, + authors: resolvedAuthors + }; + + // Normalize date fields to ISO 8601 format + const createdAt = normalizeDate(post.created_at); + if (createdAt) { + postData.created_at = createdAt; + } + + const publishedAt = normalizeDate(post.published_at); + if (publishedAt) { + postData.published_at = publishedAt; + } + + const updatedAt = normalizeDate(post.updated_at); + if (updatedAt) { + postData.updated_at = updatedAt; + } + + // Add meta fields from posts_meta table + const meta = ctx.postsMeta.get(post.id); + if (meta) { + if (meta.og_image) { + postData.og_image = meta.og_image; + } + if (meta.og_title) { + postData.og_title = meta.og_title; + } + if (meta.og_description) { + postData.og_description = meta.og_description; + } + if (meta.twitter_image) { + postData.twitter_image = meta.twitter_image; + } + if (meta.twitter_title) { + postData.twitter_title = meta.twitter_title; + } + if (meta.twitter_description) { + postData.twitter_description = meta.twitter_description; + } + if (meta.meta_title) { + postData.meta_title = meta.meta_title; + } + if (meta.meta_description) { + postData.meta_description = meta.meta_description; + } + if (meta.email_subject) { + postData.email_subject = meta.email_subject; + } + if (meta.feature_image_alt) { + postData.feature_image_alt = meta.feature_image_alt; + } + if (meta.feature_image_caption) { + postData.feature_image_caption = meta.feature_image_caption; + } + } + + // Use HTML content with source: html + if (post.html) { + postData.html = post.html; + } + + // Determine which API to use based on type + const apiType = postData.type === 'page' ? 'pages' : 'posts'; + const result = await ctx.api[apiType].add(postData, {source: 'html'}); + + // Add to cache with actual slug from API + addPost(ctx.db, { + id: result.id, + slug: result.slug, + title: result.title, + type: result.type || postData.type + }); + + // Mark the ORIGINAL JSON slug as imported + // so we don't re-import if Ghost changed the slug (e.g., added -2) + markSlugImported(ctx.db, post.slug); + + ctx.imported.push(result); + task.output = `Imported: ${result.slug}`; + + await sleep(ctx.args.delayBetweenCalls); + } catch (error) { + error.resource = {title: post.title, slug: post.slug}; + // Log full error details for debugging + console.error('\n--- Import Error ---'); + console.error('Post:', post.title, `(${post.slug})`); + console.error('Error:', error.message); + if (error.context) { + console.error('Context:', error.context); + } + if (error.details) { + console.error('Details:', JSON.stringify(error.details, null, 2)); + } + if (error.response?.data) { + console.error('Response:', JSON.stringify(error.response.data, null, 2)); + } + console.error('--------------------\n'); + ctx.errors.push(error); + throw error; + } + } + }); + } + + let taskOptions = {concurrent: 1}; + return makeTaskRunner(tasks, taskOptions); + } + }; +}; + +const showResults = () => { + return { + title: 'Import complete', + task: async (ctx, task) => { + const results = []; + results.push(`Imported: ${ctx.imported.length}`); + results.push(`Skipped (no author): ${ctx.skipped.length}`); + results.push(`Duplicates: ${ctx.duplicatePosts.length}`); + + if (ctx.warnings.length > 0) { + results.push(`Warnings: ${ctx.warnings.length}`); + } + + task.output = results.join(' | '); + } + }; +}; + +const getFullTaskList = (options) => { + return [ + initialise(options), + refreshCacheTask(options), + readJsonFile(options), + buildImportMaps(), + resolveFallbackAuthor(), + analyzeImport(options), + showImportSummary(), + importPosts(options), + showResults() + ]; +}; + +const getTaskRunner = (options) => { + let tasks = getFullTaskList(options); + return makeTaskRunner(tasks, Object.assign({topLevel: true}, options)); +}; + +const cleanup = (ctx) => { + if (ctx && ctx.db) { + closeDatabase(ctx.db); + } +}; + +const printWarnings = (ctx) => { + if (ctx.warnings && ctx.warnings.length > 0) { + ui.log.warn('Posts skipped due to missing authors:'); + for (const warning of ctx.warnings) { + ui.log.warn(` - ${warning.post}: ${warning.reason}`); + } + } +}; + +export default { + initialise, + getFullTaskList, + getTaskRunner, + cleanup, + printWarnings +}; diff --git a/yarn.lock b/yarn.lock index 34a8317a..46d23203 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3233,6 +3233,21 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +better-sqlite3@11.8.1: + version "11.8.1" + resolved "https://registry.yarnpkg.com/better-sqlite3/-/better-sqlite3-11.8.1.tgz#bcb1c494984065a7ed76a5df5ecbcb0f068d47fa" + integrity sha512-9BxNaBkblMjhJW8sMRZxnxVTRgbRmssZW0Oxc1MPBTfiR+WW21e2Mk4qu8CzrcZb1LwPCnFsfDEzq+SNcBU8eg== + dependencies: + bindings "^1.5.0" + prebuild-install "^7.1.1" + +bindings@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" + integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== + dependencies: + file-uri-to-path "1.0.0" + bl@^4.0.3, bl@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" @@ -4704,6 +4719,11 @@ file-type@19.4.1: token-types "^6.0.0" uint8array-extras "^1.3.0" +file-uri-to-path@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" + integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== + fill-range@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"