From f8f2c1077943070ace279e37d197d37a8da290fb Mon Sep 17 00:00:00 2001 From: Conrad Hoang Date: Fri, 12 Apr 2024 11:25:01 -0400 Subject: [PATCH 1/8] fix: imports nodes/tags only when needed --- packages/process/src/transformer.ts | 80 +++++++++++++++++++++++------ packages/process/src/utils.ts | 8 ++- 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/packages/process/src/transformer.ts b/packages/process/src/transformer.ts index 6217c96..14a28b2 100644 --- a/packages/process/src/transformer.ts +++ b/packages/process/src/transformer.ts @@ -8,6 +8,7 @@ import { ConfigType, validate, Tokenizer, + Node, } from '@markdoc/markdoc'; import { ScriptTarget, @@ -18,7 +19,7 @@ import { getNameOfDeclaration, isVariableStatement, } from 'typescript'; -import { dirname, join } from 'path'; +import { dirname, join, } from 'path'; import { load as loadYaml } from 'js-yaml'; import { parse as svelteParse, walk } from 'svelte/compiler'; import { render_html } from './renderer'; @@ -27,11 +28,12 @@ import { path_exists, read_file, relative_posix_path, + to_absolute_posix_path, write_to_file, } from './utils'; import * as default_schema from './default_schema'; import type { Config } from './config'; -import { LAYOUT_IMPORT, NODES_IMPORT, TAGS_IMPORT } from './constants'; +import { LAYOUT_IMPORT, } from './constants'; import { log_error, log_validation_error } from './log'; type Var = { @@ -74,6 +76,28 @@ export function transformer({ */ const ast = markdocParse(tokens); + const flatten_node = (node: Node): [NodeType[], string[]] => { + const aux_create_state = (node: Node): [NodeType[], string[]] => + node.tag ? [[], [node.tag]] : [[node.type], []]; + + return node.children.length + ? [ + aux_create_state(node), + ...node.children.map((n) => flatten_node(n)), + ].reduce( + (acc, node) => [ + [...acc[0], ...node[0]], + [...acc[1], ...node[1]], + ], + [[], []], + ) + : aux_create_state(node); + }; + + const [used_nodes, used_tags] = flatten_node(ast).map((nodes) => [ + ...new Set(nodes), + ]); + /** * load frontmatter */ @@ -94,9 +118,13 @@ export function transformer({ * add used svelte components to the script tag */ let dependencies = ''; - const tags = prepare_tags(tags_file); + const tag_comps_with_paths = tags_file ? each_exported_var(tags_file) : []; + const tags = prepare_tags(tags_file, tag_comps_with_paths); const has_tags = Object.keys(tags).length > 0; - const nodes = prepare_nodes(nodes_file); + const node_comps_with_paths = nodes_file + ? each_exported_var(nodes_file!) + : []; + const nodes = prepare_nodes(nodes_file, node_comps_with_paths); const has_nodes = Object.keys(nodes).length > 0; const partials = prepare_partials(partials_dir); @@ -104,20 +132,33 @@ export function transformer({ * add import for tags */ if (tags_file && has_tags) { - dependencies += `import * as ${TAGS_IMPORT} from '${relative_posix_path( - filename, - tags_file, - )}';`; + dependencies += tag_comps_with_paths + .filter(([comp, _]) => used_tags.includes(comp.toLowerCase())) + .map( + ([comp, path]) => + `import ${comp} from '${relative_posix_path( + filename, + to_absolute_posix_path(filename, tags_file, path), + )}';`, + ) + .join(''); } /** * add import for nodes */ + if (nodes_file && has_nodes) { - dependencies += `import * as ${NODES_IMPORT} from '${relative_posix_path( - filename, - nodes_file, - )}';`; + dependencies += node_comps_with_paths + .filter(([comp, _]) => used_nodes.includes(comp.toLowerCase())) + .map( + ([comp, path]) => + `import ${comp} from '${relative_posix_path( + filename, + to_absolute_posix_path(filename, nodes_file, path), + )}';`, + ) + .join(''); } /** @@ -130,6 +171,7 @@ export function transformer({ )}';`; } + /** * generate schema for markdoc extension */ @@ -397,10 +439,11 @@ function get_node_defaults(node_type: NodeType): Partial { function prepare_nodes( nodes_file: Config['nodes'], + comps_with_paths: Array<[string, string]>, ): Partial> { const nodes: Record = {}; if (nodes_file) { - for (const [name] of each_exported_var(nodes_file)) { + for (const [name] of comps_with_paths) { const type = name.toLowerCase() as NodeType; if (type === 'image') { } @@ -411,7 +454,7 @@ function prepare_nodes( node.attributes.src; } return new Tag( - `${NODES_IMPORT}.${name}`, + name, node.transformAttributes(config), node.transformChildren(config), ); @@ -423,16 +466,19 @@ function prepare_nodes( return nodes; } -function prepare_tags(tags_file: Config['tags']): Record { +function prepare_tags( + tags_file: Config['tags'], + comps_with_paths: Array<[string, string]>, +): Record { const tags: Record = {}; if (tags_file) { - for (const [name, value] of each_exported_var(tags_file)) { + for (const [name, value] of comps_with_paths) { /** * extract all exported variables from the components */ const attributes = get_component_vars(String(value), tags_file); tags[name.toLowerCase()] = { - render: `${TAGS_IMPORT}.${name}`, + render: name, attributes, }; } diff --git a/packages/process/src/utils.ts b/packages/process/src/utils.ts index ca4ff38..0a71be9 100644 --- a/packages/process/src/utils.ts +++ b/packages/process/src/utils.ts @@ -5,7 +5,8 @@ import { readdirSync, writeFileSync, } from 'fs'; -import { dirname, join, relative, sep } from 'path'; +import { dirname, join, relative, sep, resolve } from 'path'; +import path = require('path'); import { sep as posix_sep } from 'path/posix'; export function get_all_files(path: string): string[] { @@ -38,3 +39,8 @@ export function path_exists(path: string): boolean { export function relative_posix_path(from: string, to: string): string { return relative(dirname(from), to).split(sep).join(posix_sep); } +export function to_absolute_posix_path(...paths: string[]): string { + return resolve(...paths.slice(0, paths.length - 1).map(v => dirname(v)), paths[paths.length-1]) + .split(sep) + .join(posix_sep); +} From 94419fccc7dcf2d62a0ba005c8a5e793ee60faf2 Mon Sep 17 00:00:00 2001 From: Conrad Hoang Date: Sat, 13 Apr 2024 03:28:56 -0400 Subject: [PATCH 2/8] refactor: memoize tags and nodes meta data --- packages/process/src/transformer.ts | 147 ++++++++++++++++++---------- 1 file changed, 95 insertions(+), 52 deletions(-) diff --git a/packages/process/src/transformer.ts b/packages/process/src/transformer.ts index 14a28b2..8b47ab1 100644 --- a/packages/process/src/transformer.ts +++ b/packages/process/src/transformer.ts @@ -19,7 +19,7 @@ import { getNameOfDeclaration, isVariableStatement, } from 'typescript'; -import { dirname, join, } from 'path'; +import { dirname, join } from 'path'; import { load as loadYaml } from 'js-yaml'; import { parse as svelteParse, walk } from 'svelte/compiler'; import { render_html } from './renderer'; @@ -33,7 +33,7 @@ import { } from './utils'; import * as default_schema from './default_schema'; import type { Config } from './config'; -import { LAYOUT_IMPORT, } from './constants'; +import { LAYOUT_IMPORT } from './constants'; import { log_error, log_validation_error } from './log'; type Var = { @@ -41,6 +41,60 @@ type Var = { type: StringConstructor | NumberConstructor | BooleanConstructor; }; +let all_tags_comps_with_paths: [string, string][] | undefined; +let all_nodes_comps_with_paths: [string, string][] | undefined; + +let all_partials_with_node_instances: Record | undefined; +let all_patrials_with_nodes_and_tags: + | Map + | undefined; + +function init( + tags_file: string | null, + nodes_file: string | null, + partials_dir: string | null, +) { + if ( + all_nodes_comps_with_paths || + all_tags_comps_with_paths || + all_patrials_with_nodes_and_tags || + all_partials_with_node_instances + ) + return; + + all_tags_comps_with_paths = tags_file ? each_exported_var(tags_file) : []; + + all_nodes_comps_with_paths = nodes_file + ? each_exported_var(nodes_file) + : []; + all_partials_with_node_instances = prepare_partials(partials_dir); + all_patrials_with_nodes_and_tags = partials_dir + ? new Map( + Object.entries(all_partials_with_node_instances).map( + ([string, node]) => [string, flatten_node(node)], + ), + ) + : new Map(); +} + +function flatten_node(node: Node): [NodeType[], string[]] { + const aux_create_state = (node: Node): [NodeType[], string[]] => + node.tag ? [[], [node.tag]] : [[node.type], []]; + + return node.children.length + ? [ + aux_create_state(node), + ...node.children.map((n) => flatten_node(n)), + ].reduce( + (acc, node) => [ + [...acc[0], ...node[0]], + [...acc[1], ...node[1]], + ], + [[], []], + ) + : aux_create_state(node); +} + export function transformer({ content, filename, @@ -64,6 +118,8 @@ export function transformer({ validation_threshold: Config['validationThreshold']; allow_comments: Config['allowComments']; }): string { + init(tags_file, nodes_file, partials_dir); + /** * create tokenizer */ @@ -76,24 +132,6 @@ export function transformer({ */ const ast = markdocParse(tokens); - const flatten_node = (node: Node): [NodeType[], string[]] => { - const aux_create_state = (node: Node): [NodeType[], string[]] => - node.tag ? [[], [node.tag]] : [[node.type], []]; - - return node.children.length - ? [ - aux_create_state(node), - ...node.children.map((n) => flatten_node(n)), - ].reduce( - (acc, node) => [ - [...acc[0], ...node[0]], - [...acc[1], ...node[1]], - ], - [[], []], - ) - : aux_create_state(node); - }; - const [used_nodes, used_tags] = flatten_node(ast).map((nodes) => [ ...new Set(nodes), ]); @@ -118,47 +156,53 @@ export function transformer({ * add used svelte components to the script tag */ let dependencies = ''; - const tag_comps_with_paths = tags_file ? each_exported_var(tags_file) : []; - const tags = prepare_tags(tags_file, tag_comps_with_paths); - const has_tags = Object.keys(tags).length > 0; - const node_comps_with_paths = nodes_file - ? each_exported_var(nodes_file!) - : []; - const nodes = prepare_nodes(nodes_file, node_comps_with_paths); - const has_nodes = Object.keys(nodes).length > 0; - const partials = prepare_partials(partials_dir); + + const tags = prepare_tags(tags_file, all_tags_comps_with_paths ?? []); + const nodes = prepare_nodes(nodes_file, all_nodes_comps_with_paths ?? []); + + const partials = all_partials_with_node_instances; /** * add import for tags */ - if (tags_file && has_tags) { - dependencies += tag_comps_with_paths - .filter(([comp, _]) => used_tags.includes(comp.toLowerCase())) - .map( - ([comp, path]) => - `import ${comp} from '${relative_posix_path( - filename, - to_absolute_posix_path(filename, tags_file, path), - )}';`, - ) - .join(''); + if (used_tags.length) { + dependencies += + all_tags_comps_with_paths + ?.filter(([comp]) => used_tags.includes(comp.toLowerCase())) + .map( + ([comp, path]) => + `import ${comp} from '${relative_posix_path( + filename, + to_absolute_posix_path( + filename, + tags_file ?? '', + path, + ), + )}';`, + ) + .join('') ?? ''; } /** * add import for nodes */ - if (nodes_file && has_nodes) { - dependencies += node_comps_with_paths - .filter(([comp, _]) => used_nodes.includes(comp.toLowerCase())) - .map( - ([comp, path]) => - `import ${comp} from '${relative_posix_path( - filename, - to_absolute_posix_path(filename, nodes_file, path), - )}';`, - ) - .join(''); + if (used_nodes.length) { + dependencies += + all_nodes_comps_with_paths + ?.filter(([comp]) => used_nodes.includes(comp.toLowerCase())) + .map( + ([comp, path]) => + `import ${comp} from '${relative_posix_path( + filename, + to_absolute_posix_path( + filename, + nodes_file ?? '', + path, + ), + )}';`, + ) + .join('') ?? ''; } /** @@ -171,7 +215,6 @@ export function transformer({ )}';`; } - /** * generate schema for markdoc extension */ From c9240a5983aa221f308b40f02b0960b961346ce8 Mon Sep 17 00:00:00 2001 From: Conrad Hoang Date: Sat, 13 Apr 2024 04:45:57 -0400 Subject: [PATCH 3/8] fix: imports with partials --- packages/process/src/transformer.ts | 96 ++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 17 deletions(-) diff --git a/packages/process/src/transformer.ts b/packages/process/src/transformer.ts index 8b47ab1..2d52812 100644 --- a/packages/process/src/transformer.ts +++ b/packages/process/src/transformer.ts @@ -46,7 +46,7 @@ let all_nodes_comps_with_paths: [string, string][] | undefined; let all_partials_with_node_instances: Record | undefined; let all_patrials_with_nodes_and_tags: - | Map + | Map | undefined; function init( @@ -77,24 +77,68 @@ function init( : new Map(); } -function flatten_node(node: Node): [NodeType[], string[]] { - const aux_create_state = (node: Node): [NodeType[], string[]] => - node.tag ? [[], [node.tag]] : [[node.type], []]; +function flatten_node(node: Node): [NodeType[], string[], string[]] { + // returns nodes, tags and partials + const aux_create_state = (node: Node): [NodeType[], string[], string[]] => + is_partial_node(node) + ? [[], [], node.annotations[0].value] // first elem is the partial file's name others are the meta data, like passed-in vars + : node.tag + ? [[], [node.tag], []] + : [[node.type], [], []]; return node.children.length - ? [ + ? combines_nodes_tags_partials([ aux_create_state(node), ...node.children.map((n) => flatten_node(n)), - ].reduce( - (acc, node) => [ - [...acc[0], ...node[0]], - [...acc[1], ...node[1]], - ], - [[], []], - ) + ]) : aux_create_state(node); } +function combines_nodes_tags_partials( + data: [NodeType[], string[], string[]][], +): [NodeType[], string[], string[]] { + return data.reduce( + (acc, node) => acc.map((k, i) => k.concat(node[i])), + [[], [], []], + ) as [NodeType[], string[], string[]]; +} + +function is_partial_node(node: Node): boolean { + return ( + node.type == 'tag' && node.tag == 'partial' && !!node.annotations.length + ); +} + +function flatten_partials( + state: string[], + partialName: string, +): [NodeType[], string[], string[]] { + if (state.includes(partialName)) { + throw new Error( + `resolve failed: detected cyclic in these partials in ${[ + ...state, + ]}`, + ); + } + + if (!partialName.length) { + return [[], [], []]; + } + + state = [...state, partialName]; + const res = all_patrials_with_nodes_and_tags?.get(partialName) ?? [ + [], + [], + [], + ]; + const [, , remaining_partials] = res; + return remaining_partials.length + ? combines_nodes_tags_partials( + remaining_partials.map((v) => flatten_partials(state, v)), + ) + : res; +} + export function transformer({ content, filename, @@ -132,10 +176,26 @@ export function transformer({ */ const ast = markdocParse(tokens); - const [used_nodes, used_tags] = flatten_node(ast).map((nodes) => [ - ...new Set(nodes), + const [used_cur_nodes, used_cur_tags, used_partials] = flatten_node( + ast, + ).map((nodes) => [...new Set(nodes)]); + + const [used_partials_nodes, used_partials_tags, empty_partials] = + combines_nodes_tags_partials( + used_partials.map((p) => flatten_partials([], p)), + ); + + if (empty_partials.length) { + throw new Error('should never happend'); + } + + const [used_nodes, used_tags] = combines_nodes_tags_partials([ + [used_cur_nodes as NodeType[], used_cur_tags, []], + [used_partials_nodes, used_partials_tags, []], ]); + // + /** * load frontmatter */ @@ -165,7 +225,7 @@ export function transformer({ /** * add import for tags */ - if (used_tags.length) { + if (used_cur_tags.length) { dependencies += all_tags_comps_with_paths ?.filter(([comp]) => used_tags.includes(comp.toLowerCase())) @@ -187,10 +247,12 @@ export function transformer({ * add import for nodes */ - if (used_nodes.length) { + if (used_cur_nodes.length) { dependencies += all_nodes_comps_with_paths - ?.filter(([comp]) => used_nodes.includes(comp.toLowerCase())) + ?.filter(([comp]) => + used_nodes.includes(comp.toLowerCase() as NodeType), + ) .map( ([comp, path]) => `import ${comp} from '${relative_posix_path( From d28edce1024e3d7f6e79d2eac5380d1559f9e2be Mon Sep 17 00:00:00 2001 From: Conrad Hoang Date: Sat, 13 Apr 2024 05:00:41 -0400 Subject: [PATCH 4/8] fix: filter only used partial in the current doc --- packages/process/src/transformer.ts | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/packages/process/src/transformer.ts b/packages/process/src/transformer.ts index 2d52812..90f607f 100644 --- a/packages/process/src/transformer.ts +++ b/packages/process/src/transformer.ts @@ -115,7 +115,7 @@ function flatten_partials( ): [NodeType[], string[], string[]] { if (state.includes(partialName)) { throw new Error( - `resolve failed: detected cyclic in these partials in ${[ + `resolve failed: detected cyclic in these partials in the order ${[ ...state, ]}`, ); @@ -176,13 +176,13 @@ export function transformer({ */ const ast = markdocParse(tokens); - const [used_cur_nodes, used_cur_tags, used_partials] = flatten_node( + const [used_cur_nodes, used_cur_tags, used_cur_partials] = flatten_node( ast, ).map((nodes) => [...new Set(nodes)]); const [used_partials_nodes, used_partials_tags, empty_partials] = combines_nodes_tags_partials( - used_partials.map((p) => flatten_partials([], p)), + used_cur_partials.map((p) => flatten_partials([], p)), ); if (empty_partials.length) { @@ -217,10 +217,24 @@ export function transformer({ */ let dependencies = ''; - const tags = prepare_tags(tags_file, all_tags_comps_with_paths ?? []); - const nodes = prepare_nodes(nodes_file, all_nodes_comps_with_paths ?? []); + const tags = prepare_tags( + tags_file, + all_tags_comps_with_paths?.filter(([comp]) => + used_tags.includes(comp.toLowerCase()), + ) ?? [], + ); + const nodes = prepare_nodes( + nodes_file, + all_nodes_comps_with_paths?.filter( + ([comp]) => used_nodes?.includes(comp.toLowerCase() as NodeType), + ) ?? [], + ); - const partials = all_partials_with_node_instances; + const partials = Object.fromEntries( + Object.entries(all_partials_with_node_instances ?? {}).filter(([k]) => + used_cur_partials.includes(k), + ), + ); /** * add import for tags From 7f2e8d5ff394f4473a9da1559455ab7c1cafb207 Mon Sep 17 00:00:00 2001 From: Conrad Hoang Date: Tue, 30 Apr 2024 08:12:47 -0400 Subject: [PATCH 5/8] refactor: much nicer types fix: check partial nodes by name's annotations --- packages/process/src/transformer.ts | 241 ++++++++++++++++++---------- 1 file changed, 159 insertions(+), 82 deletions(-) diff --git a/packages/process/src/transformer.ts b/packages/process/src/transformer.ts index 90f607f..2878255 100644 --- a/packages/process/src/transformer.ts +++ b/packages/process/src/transformer.ts @@ -41,66 +41,109 @@ type Var = { type: StringConstructor | NumberConstructor | BooleanConstructor; }; -let all_tags_comps_with_paths: [string, string][] | undefined; -let all_nodes_comps_with_paths: [string, string][] | undefined; +type NodeName = NodeType; +type TagName = string; +type PartialName = string; + +type NodeTagPartialTriplet = [NodeName[], TagName[], PartialName[]]; +type TransformerState = { + nodes: Map; + tags: Map; + partials: Map; + normalized: { + nodes: NodeName[]; + tags: TagName[]; + }; +}; -let all_partials_with_node_instances: Record | undefined; -let all_patrials_with_nodes_and_tags: - | Map - | undefined; +let transformer_state: TransformerState | undefined; function init( tags_file: string | null, nodes_file: string | null, partials_dir: string | null, -) { - if ( - all_nodes_comps_with_paths || - all_tags_comps_with_paths || - all_patrials_with_nodes_and_tags || - all_partials_with_node_instances - ) - return; - - all_tags_comps_with_paths = tags_file ? each_exported_var(tags_file) : []; - - all_nodes_comps_with_paths = nodes_file - ? each_exported_var(nodes_file) +): TransformerState { + const node_with_paths = nodes_file ? each_exported_var(nodes_file) : []; + const node_with_schemas = [ + ...Object.entries(prepare_nodes(nodes_file, node_with_paths)), + ]; + + const node_state = node_with_paths.map<[NodeName, [path: string, Schema]]>( + ([node, path]) => [ + node as NodeName, + [ + path, + node_with_schemas.find( + ([snode]) => snode.toLowerCase() == node.toLowerCase(), + )![1], + ], + ], + ); + + const tag_with_paths = tags_file + ? each_exported_var(tags_file.toString()) : []; - all_partials_with_node_instances = prepare_partials(partials_dir); - all_patrials_with_nodes_and_tags = partials_dir - ? new Map( - Object.entries(all_partials_with_node_instances).map( - ([string, node]) => [string, flatten_node(node)], - ), - ) - : new Map(); + + const tag_with_schemas = [ + ...Object.entries(prepare_tags(tags_file, tag_with_paths)), + ]; + + const tag_state = tag_with_paths.map<[TagName, [path: string, Schema]]>( + ([tag, path]) => [ + tag, + [ + path, + tag_with_schemas.find( + ([stag]) => stag.toLowerCase() == tag.toLowerCase(), + )![1], + ], + ], + ); + + const partial_schemas = prepare_partials(partials_dir); + const partial_state = Object.entries(partial_schemas).map< + [PartialName, [NodeTagPartialTriplet, Node]] + >(([partial, schema]) => [partial, [flatten_node(schema), schema]]); + return { + tags: new Map(tag_state), + nodes: new Map(node_state), + partials: new Map(partial_state), + normalized: { + nodes: [...node_state.map(([k]) => k)], + tags: [...tag_state.map(([k]) => k)], + }, + }; } -function flatten_node(node: Node): [NodeType[], string[], string[]] { - // returns nodes, tags and partials - const aux_create_state = (node: Node): [NodeType[], string[], string[]] => +function flatten_node(node: Node): NodeTagPartialTriplet { + const aux_create_state = (node: Node): NodeTagPartialTriplet => is_partial_node(node) - ? [[], [], node.annotations[0].value] // first elem is the partial file's name others are the meta data, like passed-in vars + ? [ + [], + [], + node.annotations + .filter((a) => a.name == 'file') + .map((node) => node.value), + ] : node.tag ? [[], [node.tag], []] : [[node.type], [], []]; return node.children.length - ? combines_nodes_tags_partials([ + ? combine_nodes_tags_partials([ aux_create_state(node), - ...node.children.map((n) => flatten_node(n)), + ...node.children.map(flatten_node), ]) : aux_create_state(node); } -function combines_nodes_tags_partials( - data: [NodeType[], string[], string[]][], -): [NodeType[], string[], string[]] { +function combine_nodes_tags_partials( + data: NodeTagPartialTriplet[], +): NodeTagPartialTriplet { return data.reduce( (acc, node) => acc.map((k, i) => k.concat(node[i])), [[], [], []], - ) as [NodeType[], string[], string[]]; + ) as NodeTagPartialTriplet; } function is_partial_node(node: Node): boolean { @@ -110,31 +153,36 @@ function is_partial_node(node: Node): boolean { } function flatten_partials( - state: string[], - partialName: string, -): [NodeType[], string[], string[]] { - if (state.includes(partialName)) { + travel_state: PartialName[], + transformer_state: TransformerState, + partial_name: PartialName, +): NodeTagPartialTriplet { + if (travel_state.includes(partial_name)) { throw new Error( - `resolve failed: detected cyclic in these partials in the order ${[ - ...state, + `resolve deps failed: detected cyclic error in partial in the order ${[ + ...travel_state, + partial_name, ]}`, ); } - if (!partialName.length) { + if (!partial_name.length) { return [[], [], []]; } - state = [...state, partialName]; - const res = all_patrials_with_nodes_and_tags?.get(partialName) ?? [ + travel_state = [...travel_state, partial_name]; + const res = transformer_state.partials.get(partial_name)?.[0] ?? [ [], [], [], ]; const [, , remaining_partials] = res; + return remaining_partials.length - ? combines_nodes_tags_partials( - remaining_partials.map((v) => flatten_partials(state, v)), + ? combine_nodes_tags_partials( + remaining_partials.map((v) => + flatten_partials(travel_state, transformer_state, v), + ), ) : res; } @@ -162,7 +210,8 @@ export function transformer({ validation_threshold: Config['validationThreshold']; allow_comments: Config['allowComments']; }): string { - init(tags_file, nodes_file, partials_dir); + if (!transformer_state) + transformer_state = init(tags_file, nodes_file, partials_dir); /** * create tokenizer @@ -181,19 +230,44 @@ export function transformer({ ).map((nodes) => [...new Set(nodes)]); const [used_partials_nodes, used_partials_tags, empty_partials] = - combines_nodes_tags_partials( - used_cur_partials.map((p) => flatten_partials([], p)), + combine_nodes_tags_partials( + used_cur_partials.map((p) => + flatten_partials([], transformer_state!, p), + ), ); if (empty_partials.length) { throw new Error('should never happend'); } - const [used_nodes, used_tags] = combines_nodes_tags_partials([ - [used_cur_nodes as NodeType[], used_cur_tags, []], + const [used_nodes, used_tags] = combine_nodes_tags_partials([ + [used_cur_nodes as NodeName[], used_cur_tags, []], [used_partials_nodes, used_partials_tags, []], ]); + const used_normalized_nodes = [ + ...new Set( + used_nodes + .map((k) => + transformer_state!.normalized.nodes.find( + (n) => n.toLowerCase() == k.toLowerCase(), + ), + ) + .filter(Boolean), + ), + ]; + + const used_normalized_tags = [ + ...new Set( + used_tags.map( + (k) => + transformer_state!.normalized.tags.find( + (n) => n.toLowerCase() == k.toLowerCase(), + )!, + ), + ), + ]; + // /** @@ -217,23 +291,19 @@ export function transformer({ */ let dependencies = ''; - const tags = prepare_tags( - tags_file, - all_tags_comps_with_paths?.filter(([comp]) => - used_tags.includes(comp.toLowerCase()), - ) ?? [], - ); - const nodes = prepare_nodes( - nodes_file, - all_nodes_comps_with_paths?.filter( - ([comp]) => used_nodes?.includes(comp.toLowerCase() as NodeType), - ) ?? [], - ); + const tags = used_normalized_tags.map((name) => [ + name, + transformer_state!.tags.get(name)![0], + ]); // tags must be presented + + const nodes = used_normalized_nodes + .map((name) => [name, transformer_state!.nodes.get(name!)?.[0]]) + .filter(([, maybe_schema]) => maybe_schema) as [string, string][]; // node can be fall back to the default const partials = Object.fromEntries( - Object.entries(all_partials_with_node_instances ?? {}).filter(([k]) => - used_cur_partials.includes(k), - ), + [...transformer_state!.partials.entries()] + .filter(([k]) => used_cur_partials.includes(k)) + .map(([partial, [, node]]) => [partial, node]), ); /** @@ -241,8 +311,7 @@ export function transformer({ */ if (used_cur_tags.length) { dependencies += - all_tags_comps_with_paths - ?.filter(([comp]) => used_tags.includes(comp.toLowerCase())) + tags .map( ([comp, path]) => `import ${comp} from '${relative_posix_path( @@ -263,10 +332,7 @@ export function transformer({ if (used_cur_nodes.length) { dependencies += - all_nodes_comps_with_paths - ?.filter(([comp]) => - used_nodes.includes(comp.toLowerCase() as NodeType), - ) + nodes .map( ([comp, path]) => `import ${comp} from '${relative_posix_path( @@ -295,7 +361,13 @@ export function transformer({ * generate schema for markdoc extension */ if (generate_schema) { - create_schema(tags); + create_schema( + Object.fromEntries( + [...transformer_state.tags.entries()].map( + ([comp, [, schema]]) => [comp, schema], + ), + ), + ); } /** @@ -304,11 +376,20 @@ export function transformer({ const configuration: ConfigType = { tags: { ...config?.tags, - ...tags, + ...Object.fromEntries( + [...transformer_state.tags.entries()].map( + ([comp, [, schema]]) => [comp.toLowerCase(), schema], + ), + ), }, + nodes: { ...config?.nodes, - ...nodes, + ...Object.fromEntries( + [...transformer_state.nodes.entries()].map( + ([comp, [, schema]]) => [comp.toLowerCase(), schema], + ), + ), }, partials: { ...config?.partials, @@ -564,14 +645,10 @@ function prepare_nodes( if (nodes_file) { for (const [name] of comps_with_paths) { const type = name.toLowerCase() as NodeType; - if (type === 'image') { - } + nodes[name.toLowerCase()] = { ...get_node_defaults(type), transform(node, config) { - if (type === 'image') { - node.attributes.src; - } return new Tag( name, node.transformAttributes(config), From c06b977ffea43f449bb709c468c9b49314a30c27 Mon Sep 17 00:00:00 2001 From: Conrad Hoang Date: Wed, 1 May 2024 04:44:31 -0400 Subject: [PATCH 6/8] test: test suite should work now disable caching in the transformer for now, need to implement invalidate cache mechanism. --- packages/process/src/transformer.ts | 28 +++++++++---------- packages/process/tests/processor.test.mjs | 2 +- .../nodes, tags and partials/compiled.txt | 2 +- .../tests/processor/nodes/compiled.txt | 2 +- .../tests/processor/nodes/source.markdoc | 2 +- .../process/tests/processor/tags/compiled.txt | 2 +- 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/packages/process/src/transformer.ts b/packages/process/src/transformer.ts index 2878255..9c391ae 100644 --- a/packages/process/src/transformer.ts +++ b/packages/process/src/transformer.ts @@ -56,8 +56,6 @@ type TransformerState = { }; }; -let transformer_state: TransformerState | undefined; - function init( tags_file: string | null, nodes_file: string | null, @@ -210,8 +208,7 @@ export function transformer({ validation_threshold: Config['validationThreshold']; allow_comments: Config['allowComments']; }): string { - if (!transformer_state) - transformer_state = init(tags_file, nodes_file, partials_dir); + const transformer_state = init(tags_file, nodes_file, partials_dir); /** * create tokenizer @@ -232,7 +229,7 @@ export function transformer({ const [used_partials_nodes, used_partials_tags, empty_partials] = combine_nodes_tags_partials( used_cur_partials.map((p) => - flatten_partials([], transformer_state!, p), + flatten_partials([], transformer_state, p), ), ); @@ -249,7 +246,7 @@ export function transformer({ ...new Set( used_nodes .map((k) => - transformer_state!.normalized.nodes.find( + transformer_state.normalized.nodes.find( (n) => n.toLowerCase() == k.toLowerCase(), ), ) @@ -259,12 +256,13 @@ export function transformer({ const used_normalized_tags = [ ...new Set( - used_tags.map( - (k) => - transformer_state!.normalized.tags.find( - (n) => n.toLowerCase() == k.toLowerCase(), - )!, - ), + used_tags.map((k) => { + const maybe_tag = transformer_state.normalized.tags.find( + (n) => n.toLowerCase() == k.toLowerCase(), + ); + if (!maybe_tag) throw new Error(`Undefined tag: '${k}'`); + return maybe_tag!; + }), ), ]; @@ -293,15 +291,15 @@ export function transformer({ const tags = used_normalized_tags.map((name) => [ name, - transformer_state!.tags.get(name)![0], + transformer_state.tags.get(name)![0], ]); // tags must be presented const nodes = used_normalized_nodes - .map((name) => [name, transformer_state!.nodes.get(name!)?.[0]]) + .map((name) => [name, transformer_state.nodes.get(name!)?.[0]]) .filter(([, maybe_schema]) => maybe_schema) as [string, string][]; // node can be fall back to the default const partials = Object.fromEntries( - [...transformer_state!.partials.entries()] + [...transformer_state.partials.entries()] .filter(([k]) => used_cur_partials.includes(k)) .map(([partial, [, node]]) => [partial, node]), ); diff --git a/packages/process/tests/processor.test.mjs b/packages/process/tests/processor.test.mjs index 0f7aefa..e871356 100644 --- a/packages/process/tests/processor.test.mjs +++ b/packages/process/tests/processor.test.mjs @@ -78,7 +78,7 @@ test('preprocessor', async (context) => { content: before, filename: 'test.markdoc', }); - assert.equal(markup.code, after); + assert.equal(markup.code, after.trim()); } catch (error) { if (exception) { assert.equal(error.message, exception); diff --git a/packages/process/tests/processor/nodes, tags and partials/compiled.txt b/packages/process/tests/processor/nodes, tags and partials/compiled.txt index 94ac4ab..df07994 100644 --- a/packages/process/tests/processor/nodes, tags and partials/compiled.txt +++ b/packages/process/tests/processor/nodes, tags and partials/compiled.txt @@ -1 +1 @@ -
Heading 1Heading 2With ID With Class

slot content

I am a partialLorem IpsumI am nested
\ No newline at end of file +
Heading 1Heading 2With ID With Class

slot content

I am a partialLorem IpsumI am nested
diff --git a/packages/process/tests/processor/nodes/compiled.txt b/packages/process/tests/processor/nodes/compiled.txt index 5ace9da..2ff2e34 100644 --- a/packages/process/tests/processor/nodes/compiled.txt +++ b/packages/process/tests/processor/nodes/compiled.txt @@ -1 +1 @@ -
Heading 1Heading 2With ID With Class
\ No newline at end of file +
Heading 1Heading 2With ID With Class
diff --git a/packages/process/tests/processor/nodes/source.markdoc b/packages/process/tests/processor/nodes/source.markdoc index 7cc1d95..8103417 100644 --- a/packages/process/tests/processor/nodes/source.markdoc +++ b/packages/process/tests/processor/nodes/source.markdoc @@ -4,4 +4,4 @@ # With ID {% #my-id %} -# With Class{% .my-class %} \ No newline at end of file +# With Class{% .my-class %} diff --git a/packages/process/tests/processor/tags/compiled.txt b/packages/process/tests/processor/tags/compiled.txt index 38ac354..3713e8a 100644 --- a/packages/process/tests/processor/tags/compiled.txt +++ b/packages/process/tests/processor/tags/compiled.txt @@ -1 +1 @@ -

slot content

\ No newline at end of file +

slot content

From 7c22af5db02d8917affc0141b716024d65aaa47f Mon Sep 17 00:00:00 2001 From: Conrad Hoang Date: Wed, 1 May 2024 05:48:47 -0400 Subject: [PATCH 7/8] fix: rm import prefix when id svelte comps --- packages/process/src/renderer.ts | 2 +- packages/process/tests/processor.test.mjs | 1 + .../tests/processor/nodes, tags and partials/compiled.txt | 2 +- packages/process/tests/processor/nodes/compiled.txt | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/process/src/renderer.ts b/packages/process/src/renderer.ts index 895888e..73d4e54 100644 --- a/packages/process/src/renderer.ts +++ b/packages/process/src/renderer.ts @@ -92,7 +92,7 @@ function is_void_element(name: string): boolean { } function is_svelte_component(node: RenderableTreeNodes): boolean { - return Tag.isTag(node) && node.name.startsWith(IMPORT_PREFIX); + return Tag.isTag(node); } function generate_svelte_attribute_value(value: unknown): string { diff --git a/packages/process/tests/processor.test.mjs b/packages/process/tests/processor.test.mjs index e871356..f440f7b 100644 --- a/packages/process/tests/processor.test.mjs +++ b/packages/process/tests/processor.test.mjs @@ -78,6 +78,7 @@ test('preprocessor', async (context) => { content: before, filename: 'test.markdoc', }); + // Somehow when reading the compiled file, it adds a line line char to the output assert.equal(markup.code, after.trim()); } catch (error) { if (exception) { diff --git a/packages/process/tests/processor/nodes, tags and partials/compiled.txt b/packages/process/tests/processor/nodes, tags and partials/compiled.txt index df07994..ebbbd1c 100644 --- a/packages/process/tests/processor/nodes, tags and partials/compiled.txt +++ b/packages/process/tests/processor/nodes, tags and partials/compiled.txt @@ -1 +1 @@ -
Heading 1Heading 2With ID With Class

slot content

I am a partialLorem IpsumI am nested
+
Heading 1Heading 2With ID With Class

slot content

I am a partialLorem IpsumI am nested
diff --git a/packages/process/tests/processor/nodes/compiled.txt b/packages/process/tests/processor/nodes/compiled.txt index 2ff2e34..f4bd9ac 100644 --- a/packages/process/tests/processor/nodes/compiled.txt +++ b/packages/process/tests/processor/nodes/compiled.txt @@ -1 +1 @@ -
Heading 1Heading 2With ID With Class
+
Heading 1Heading 2With ID With Class
From 987c217ce3a454ba777207b063371f41d49f670a Mon Sep 17 00:00:00 2001 From: Conrad Hoang Date: Wed, 1 May 2024 06:15:30 -0400 Subject: [PATCH 8/8] fix: parallel exec in the playwight test file rm timeout to pass tests --- apps/demo/playwright.config.ts | 1 + apps/demo/tests/test.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/apps/demo/playwright.config.ts b/apps/demo/playwright.config.ts index 18ef86f..30d259a 100644 --- a/apps/demo/playwright.config.ts +++ b/apps/demo/playwright.config.ts @@ -5,6 +5,7 @@ const config: PlaywrightTestConfig = { command: 'npm run build && npm run preview', port: 4173, }, + testDir: 'tests', testMatch: /(.+\.)?(test|spec)\.[jt]s/, }; diff --git a/apps/demo/tests/test.ts b/apps/demo/tests/test.ts index bd01f59..5a95805 100644 --- a/apps/demo/tests/test.ts +++ b/apps/demo/tests/test.ts @@ -1,6 +1,9 @@ import { expect, test } from '@playwright/test'; +test.describe.configure({ mode: 'parallel' }); + test('tags work', async ({ page }) => { + test.setTimeout(0); await page.goto('http://localhost:4173/playground/tags'); expect(await page.content()).toContain('Addition'); @@ -9,6 +12,7 @@ test('tags work', async ({ page }) => { }); test('tags work with types', async ({ page }) => { + test.setTimeout(0); await page.goto('http://localhost:4173/playground/tags'); expect(await page.content()).toContain('Types'); @@ -18,6 +22,7 @@ test('tags work with types', async ({ page }) => { }); test('partials work', async ({ page }) => { + test.setTimeout(0); await page.goto('http://localhost:4173/playground/partials'); expect(await page.content()).toContain('I am a partial.'); @@ -26,6 +31,7 @@ test('partials work', async ({ page }) => { }); test('named layouts work', async ({ page }) => { + test.setTimeout(0); await page.goto('http://localhost:4173/playground/layout'); expect(await page.content()).toContain('I am on an alternative layout');