Added custom taxonomy support and WP migration improvements (#1641)

- Added --customTaxonomies flag to wp-api and wp-xml commands to import
  custom taxonomy terms as Ghost tags
- Handled custom taxonomies differently per package: wp-api uses wp:term
  embedded data, wp-xml uses category domain attributes
- Batched processPosts in wp-api to avoid memory issues on large sites
- Replaced concat with spread in wp-api fetch for better performance
- Added Podigee podcast player embed support in wp-api processor
- Guarded against null/undefined YouTube URLs in mg-utils
- Added tests for custom taxonomy handling in both wp-api and wp-xml

Author: 55sketch

packages/mg-utils/src/lib/youtube-utils.ts

@@ -1,4 +1,7 @@
-const getYouTubeID = (videoUrl: string): string => {
+const getYouTubeID = (videoUrl: string | null | undefined): string => {
+    if (!videoUrl) {
+        return '';
+    }
     const arr = videoUrl.split(/(vi\/|v%3D|v=|\/v\/|youtu\.be\/|\/embed\/)/);
     return undefined !== arr[2] ? arr[2].split(/[^\w-]/i)[0] : arr[0];
 };

packages/mg-utils/src/test/youtube-utils.test.ts

@@ -34,4 +34,12 @@ describe('getYouTubeID', function () {
     it('returns the input string when no ID is found', function () {
         assert.equal(getYouTubeID('not-a-youtube-url'), 'not-a-youtube-url');
     });
+
+    it('returns empty string for null input', function () {
+        assert.equal(getYouTubeID(null), '');
+    });
+
+    it('returns empty string for undefined input', function () {
+        assert.equal(getYouTubeID(undefined), '');
+    });
 });

packages/mg-wp-api/lib/fetch.js

@@ -119,7 +119,7 @@ const buildTasks = (fileCache, tasks, api, type, limit, isAuthRequest, postsBefo
                     // Treat all types as posts, except users
                     let resultType = (type !== 'users') ? 'posts' : type;
 
-                    ctx.result[resultType] = ctx.result[resultType].concat(response);
+                    ctx.result[resultType].push(...response);
                 } catch (err) {
                     // eslint-disable-next-line no-console
                     console.error(`Failed to fetch ${type}, page ${page} of ${totalPages}`, err);

packages/mg-wp-api/lib/processor.js

@@ -125,9 +125,12 @@ const processTerm = (wpTerm) => {
     };
 };
 
-const processTerms = (wpTerms, fetchTags) => {
+const processTerms = (wpTerms, fetchTags, customTaxonomies) => {
     let categories = [];
     let tags = [];
+    let customTerms = [];
+
+    const allowedCustomTaxonomies = customTaxonomies || [];
 
     wpTerms.forEach((taxonomy) => {
         taxonomy.forEach((term) => {
@@ -138,10 +141,14 @@ const processTerms = (wpTerms, fetchTags) => {
             if (fetchTags && term.taxonomy === 'post_tag') {
                 tags.push(processTerm(term));
             }
+
+            if (allowedCustomTaxonomies.includes(term.taxonomy)) {
+                customTerms.push(processTerm(term));
+            }
         });
     });
 
-    return categories.concat(tags);
+    return categories.concat(tags).concat(customTerms);
 };
 
 // Extract co-authors from wp:term data (used by Co-Authors Plus and PublishPress Authors plugins)
@@ -693,6 +700,17 @@ const processContent = async ({html, excerptSelector, featureImageSrc = false, f
 
     await Promise.all(libsynPodcasts);
 
+    for (const el of parsed.$('script.podigee-podcast-player')) {
+        let configUrl = el.getAttribute('data-configuration');
+        if (!configUrl) {
+            continue;
+        }
+
+        let embedHTML = `<!--kg-card-begin: html--><iframe src="${configUrl}" style="width:100%;height:200px;" frameborder="0" scrolling="no"></iframe><!--kg-card-end: html-->`;
+
+        replaceWith(el, embedHTML);
+    }
+
     let wpEmbeds = parsed.$('.wp-block-embed.is-type-wp-embed').map(async (el) => {
         const blockquoteLink = el.querySelector('blockquote a');
         const bookmarkHref = blockquoteLink ? blockquoteLink.getAttribute('href') : null;
@@ -1109,7 +1127,7 @@ const processContent = async ({html, excerptSelector, featureImageSrc = false, f
  * }
  */
 const processPost = async (wpPost, users, options = {}, errors, fileCache) => { // eslint-disable-line no-shadow
-    let {tags: fetchTags, addTag, excerptSelector, excerpt, featureImageCaption} = options;
+    let {tags: fetchTags, addTag, excerptSelector, excerpt, featureImageCaption, customTaxonomies} = options;
 
     let slug = wpPost.slug;
     let titleText = parseFragment(wpPost.title.rendered).text();
@@ -1180,7 +1198,7 @@ const processPost = async (wpPost, users, options = {}, errors, fileCache) => {
 
     if (wpPost._embedded && wpPost._embedded['wp:term']) {
         const wpTerms = wpPost._embedded['wp:term'];
-        post.data.tags = processTerms(wpTerms, fetchTags);
+        post.data.tags = processTerms(wpTerms, fetchTags, customTaxonomies);
 
         post.data.tags.push({
             url: 'migrator-added-tag',
@@ -1263,7 +1281,16 @@ const processPosts = async (posts, users, options, errors, fileCache) => { // es
         posts = foundPosts;
     }
 
-    return Promise.all(posts.map(post => processPost(post, users, options, errors, fileCache)));
+    const BATCH_SIZE = 100;
+    const results = [];
+
+    for (let i = 0; i < posts.length; i += BATCH_SIZE) {
+        const batch = posts.slice(i, i + BATCH_SIZE);
+        const batchResults = await Promise.all(batch.map(post => processPost(post, users, options, errors, fileCache)));
+        results.push(...batchResults);
+    }
+
+    return results;
 };
 
 const processAuthors = (authors) => {

packages/mg-wp-api/test/process.test.js

@@ -194,6 +194,68 @@ describe('Process WordPress REST API JSON', function () {
         assert.equal(data.tags[5].data.name, '#wp');
     });
 
+    it('Can include custom taxonomy terms as tags', async function () {
+        const fixture = structuredClone(singlePostFixture);
+        fixture._embedded['wp:term'].push([
+            {id: 350, link: 'https://mysite.com/country/czech-republic', name: 'Czech Republic', slug: 'czech-republic', taxonomy: 'country'},
+            {id: 12, link: 'https://mysite.com/country/romania', name: 'Romania', slug: 'romania', taxonomy: 'country'}
+        ]);
+        fixture._embedded['wp:term'].push([
+            {id: 500, link: 'https://mysite.com/city/prague', name: 'Prague', slug: 'prague', taxonomy: 'city'}
+        ]);
+
+        const users = [];
+        const options = {tags: true, addTag: null, featureImage: 'featuredmedia', url: 'https://mysite.com', customTaxonomies: ['country']};
+        const post = await processor.processPost(fixture, users, options);
+
+        const tagSlugs = post.data.tags.map(t => t.data.slug);
+        assert.ok(tagSlugs.includes('czech-republic'), 'should include country term czech-republic');
+        assert.ok(tagSlugs.includes('romania'), 'should include country term romania');
+        assert.ok(!tagSlugs.includes('prague'), 'should not include city term when not in customTaxonomies');
+    });
+
+    it('Does not include custom taxonomy terms when customTaxonomies is not set', async function () {
+        const fixture = structuredClone(singlePostFixture);
+        fixture._embedded['wp:term'].push([
+            {id: 350, link: 'https://mysite.com/country/czech-republic', name: 'Czech Republic', slug: 'czech-republic', taxonomy: 'country'}
+        ]);
+
+        const users = [];
+        const options = {tags: true, addTag: null, featureImage: 'featuredmedia', url: 'https://mysite.com'};
+        const post = await processor.processPost(fixture, users, options);
+
+        const tagSlugs = post.data.tags.map(t => t.data.slug);
+        assert.ok(!tagSlugs.includes('czech-republic'), 'should not include country term when customTaxonomies is not set');
+    });
+
+    it('processTerms handles custom taxonomies directly', function () {
+        const wpTerms = [
+            [{id: 1, link: 'https://mysite.com/category/news', name: 'News', slug: 'news', taxonomy: 'category'}],
+            [{id: 2, link: 'https://mysite.com/tag/tech', name: 'Tech', slug: 'tech', taxonomy: 'post_tag'}],
+            [{id: 350, link: 'https://mysite.com/country/romania', name: 'Romania', slug: 'romania', taxonomy: 'country'}]
+        ];
+
+        const result = processor.processTerms(wpTerms, true, ['country']);
+        assert.equal(result.length, 3);
+        assert.equal(result[0].data.slug, 'news');
+        assert.equal(result[1].data.slug, 'tech');
+        assert.equal(result[2].data.slug, 'romania');
+    });
+
+    it('processTerms handles multiple custom taxonomies', function () {
+        const wpTerms = [
+            [{id: 1, link: 'https://mysite.com/category/news', name: 'News', slug: 'news', taxonomy: 'category'}],
+            [{id: 350, link: 'https://mysite.com/country/romania', name: 'Romania', slug: 'romania', taxonomy: 'country'}],
+            [{id: 500, link: 'https://mysite.com/city/bucharest', name: 'Bucharest', slug: 'bucharest', taxonomy: 'city'}]
+        ];
+
+        const result = processor.processTerms(wpTerms, false, ['country', 'city']);
+        assert.equal(result.length, 3);
+        assert.equal(result[0].data.slug, 'news');
+        assert.equal(result[1].data.slug, 'romania');
+        assert.equal(result[2].data.slug, 'bucharest');
+    });
+
     it('Can remove first image in post if same as feature image', async function () {
         const users = [];
         const options = {tags: true, addTag: null, featureImage: 'featuredmedia', url: 'https://mysite.com', cpt: 'mycpt'};

packages/mg-wp-xml/lib/process.js

@@ -129,6 +129,10 @@ const processTags = (categories, options = {}) => {
         'legal_regulatory'
     ];
 
+    if (options.customTaxonomies && Array.isArray(options.customTaxonomies)) {
+        allowedTerms = allowedTerms.concat(options.customTaxonomies);
+    }
+
     const categoriesArray = ensureArray(categories);
 
     for (const taxonomy of categoriesArray) {
@@ -146,7 +150,6 @@ const processTags = (categories, options = {}) => {
                 }
             });
         } else if (includeTags && allowedTerms.includes(domain)) {
-            // Only include tags if options.tags is not false
             tags.push({
                 url: `/tag/${nicename}`,
                 data: {

packages/mg-wp-xml/test/process.test.js

@@ -236,6 +236,52 @@ describe('Process', function () {
         assert.deepEqual(author.data.slug, 'hermione-example-com');
     });
 
+    it('Can include custom taxonomy terms as tags', async function () {
+        const categories = [
+            {'@_domain': 'category', '@_nicename': 'company-news', '#text': 'Company News'},
+            {'@_domain': 'post_tag', '@_nicename': 'programming', '#text': 'Programming'},
+            {'@_domain': 'country', '@_nicename': 'czech-republic', '#text': 'Czech Republic'},
+            {'@_domain': 'city', '@_nicename': 'prague', '#text': 'Prague'}
+        ];
+
+        const tags = process.processTags(categories, {tags: true, customTaxonomies: ['country']});
+        const slugs = tags.map(t => t.data.slug);
+
+        assert.ok(slugs.includes('company-news'), 'should include category');
+        assert.ok(slugs.includes('programming'), 'should include post_tag');
+        assert.ok(slugs.includes('czech-republic'), 'should include custom taxonomy term');
+        assert.ok(!slugs.includes('prague'), 'should not include taxonomy not in customTaxonomies');
+    });
+
+    it('Does not include custom taxonomy terms when customTaxonomies is not set', function () {
+        const categories = [
+            {'@_domain': 'category', '@_nicename': 'company-news', '#text': 'Company News'},
+            {'@_domain': 'country', '@_nicename': 'czech-republic', '#text': 'Czech Republic'}
+        ];
+
+        const tags = process.processTags(categories, {tags: true});
+        const slugs = tags.map(t => t.data.slug);
+
+        assert.ok(slugs.includes('company-news'), 'should include category');
+        assert.ok(!slugs.includes('czech-republic'), 'should not include custom taxonomy when not configured');
+    });
+
+    it('Can include multiple custom taxonomies', function () {
+        const categories = [
+            {'@_domain': 'category', '@_nicename': 'news', '#text': 'News'},
+            {'@_domain': 'country', '@_nicename': 'romania', '#text': 'Romania'},
+            {'@_domain': 'city', '@_nicename': 'bucharest', '#text': 'Bucharest'}
+        ];
+
+        const tags = process.processTags(categories, {tags: true, customTaxonomies: ['country', 'city']});
+        const slugs = tags.map(t => t.data.slug);
+
+        assert.equal(tags.length, 3);
+        assert.ok(slugs.includes('news'));
+        assert.ok(slugs.includes('romania'));
+        assert.ok(slugs.includes('bucharest'));
+    });
+
     it('Can extract featured image alt text and caption', async function () {
         let ctx = {
             options: {

packages/migrate/commands/wp-api.js

@@ -163,6 +163,12 @@ const options = [
         defaultValue: null,
         desc: 'The slug(s) of custom post type(s), e.g. `resources,newsletters`'
     },
+    {
+        type: 'array',
+        flags: '--customTaxonomies',
+        defaultValue: null,
+        desc: 'The slug(s) of custom taxonomy/taxonomies to import as Ghost tags, e.g. `country,city`'
+    },
     {
         type: 'boolean',
         flags: '--excerpt',

packages/migrate/commands/wp-xml.js

@@ -116,6 +116,12 @@ const options = [
         defaultValue: null,
         desc: 'The slug(s) of custom post type(s), e.g. `resources,newsletters`'
     },
+    {
+        type: 'array',
+        flags: '--customTaxonomies',
+        defaultValue: null,
+        desc: 'The slug(s) of custom taxonomy/taxonomies to import as Ghost tags, e.g. `country,city`'
+    },
     {
         type: 'boolean',
         flags: '--excerpt',