diff --git a/dev/import-tool/docs/huly/CARDS_INSTRUCTIONS.md b/dev/import-tool/docs/huly/CARDS_INSTRUCTIONS.md new file mode 100644 index 00000000000..a4a31058758 --- /dev/null +++ b/dev/import-tool/docs/huly/CARDS_INSTRUCTIONS.md @@ -0,0 +1,135 @@ +# Card Import Format Guide + +## Directory Structure + + +All files are organized in the following structure: + +``` +workspace/ +├── Recipes.yaml # Base type configuration +├── Recipes/ # Base type cards folder +│ ├── Classic Margherita Pizza.md +│ └── Chocolate Lava Cake.md +└── Recipes/Vegan/ # Child type folder + ├── Vegan Recipe.yaml # Child type configuration + └── Mushroom Risotto.md # Child type card +``` + +## Types (Master Tags) + +Types are described in YAML files and define the structure of cards. + +### Base Type +Create file `Recipes.yaml`: + +``` +class: card:class:MasterTag +title: Recipe +properties: + - label: cookingTime # Property name + type: TypeString # Data type + - label: servings + type: TypeNumber + # ... other properties +``` + +### Child Type +Create file `Recipes/Vegan/Vegan Recipe.yaml`: + +``` +class: card:class:MasterTag +title: Vegan Recipe +properties: + - label: proteinSource + type: TypeString + # ... additional properties +``` + +## Cards + +Cards are Markdown files with YAML header and content. + +### Base Type Card +Create file `Recipes/Classic Margherita Pizza.md`: + +``` +title: Classic Margherita Pizza +cookingTime: 30 minutes +servings: 4 +difficulty: Medium +category: Italian +calories: 850 +chef: Mario Rossi +``` + +# Content in Markdown format +## Sections +- Lists +- Instructions +- Notes + +### Child Type Card +Create file `Recipes/Vegan/Mushroom Risotto.md`: + +``` +title: Vegan Mushroom Risotto +cookingTime: 45 minutes +servings: 4 +difficulty: Medium +category: Italian +calories: 380 +chef: Maria Green +proteinSource: Mushrooms # Child type properties +isGlutenFree: true +allergens: None +``` + +# Content in Markdown format + +## Important Rules + +1. File Names: + - Type YAML files must end with `.yaml` + - Cards must have `.md` extension + - File names can contain spaces + +2. Directory Structure: + - Child types must be in a subfolder named after the type + - Child type cards must be in the same folder as its configuration + +3. Card YAML Header: + - Must start and end with `---` + - Must contain all properties defined in the type + - Values must match specified data types + +4. Card Content: + - After YAML header goes regular Markdown text + - Can use all Markdown features (headings, lists, tables, etc.) + +## Examples from Our System + +1. Base Recipe Type: + - File: `Recipes.yaml` + - Defines basic recipe properties (cooking time, servings, difficulty, etc.) + +2. Base Type Cards: + - `Recipes/Classic Margherita Pizza.md` - pizza recipe + - `Recipes/Chocolate Lava Cake.md` - dessert recipe + +3. Vegan Recipe Child Type: + - File: `Recipes/Vegan/Vegan Recipe.yaml` + - Adds specific properties (protein source, gluten-free, allergens) + +4. Child Type Card: + - `Recipes/Vegan/Mushroom Risotto.md` - vegan risotto + - Uses both base properties and vegan type properties + +## Supported Data Types (to be extended) + +- TypeString - text values +- TypeNumber - numeric values +- TypeBoolean - yes/no (true/false) +- TypeDate - dates +- TypeHyperlink - links +- TypeEnum - enumeration (list of possible values) (not supported yet) \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/AlphaBetaEnum.yaml b/dev/import-tool/docs/huly/example-workspace/AlphaBetaEnum.yaml new file mode 100644 index 00000000000..7f431c07054 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/AlphaBetaEnum.yaml @@ -0,0 +1,5 @@ +class: core:class:Enum +title: AlphaBeta +values: + - Alpha + - Beta diff --git a/dev/import-tool/docs/huly/example-workspace/Difficulty.yaml b/dev/import-tool/docs/huly/example-workspace/Difficulty.yaml new file mode 100644 index 00000000000..b025431660a --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/Difficulty.yaml @@ -0,0 +1,9 @@ +class: core:class:Enum +title: Difficulty +values: + - Easy + - Medium + - Hard + - Expert + - Impossible + \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/DocumentCard.md b/dev/import-tool/docs/huly/example-workspace/DocumentCard.md new file mode 100644 index 00000000000..3a573e3c900 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/DocumentCard.md @@ -0,0 +1,6 @@ +--- +class: card:types:Document +title: Document Example +--- +CONTENT +bla bla bla diff --git a/dev/import-tool/docs/huly/example-workspace/FamiliarHelpers.yaml b/dev/import-tool/docs/huly/example-workspace/FamiliarHelpers.yaml new file mode 100644 index 00000000000..c76fda52ee4 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/FamiliarHelpers.yaml @@ -0,0 +1,6 @@ +class: core:class:Association +typeA: ./SlaveCard/FamiliarTag.yaml +typeB: ./SlaveCard/MinionTag.yaml +nameA: familiar +nameB: helpers +type: 1:N diff --git a/dev/import-tool/docs/huly/example-workspace/FileCard.md b/dev/import-tool/docs/huly/example-workspace/FileCard.md new file mode 100644 index 00000000000..3dbadc01691 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/FileCard.md @@ -0,0 +1,10 @@ +--- +class: card:types:File +title: File Example +blobs: + - ./Recipes/files/cake.png +attachments: + - ./Recipes/files/cake.png +--- + +*DESCRiption* \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/RecipeAssociations.yaml b/dev/import-tool/docs/huly/example-workspace/RecipeAssociations.yaml new file mode 100644 index 00000000000..3b82b3e0e89 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/RecipeAssociations.yaml @@ -0,0 +1,7 @@ +# RecipeRelations.yaml +class: core:class:Association +typeA: "./Recipes.yaml" +typeB: "./Recipes.yaml" +nameA: recommendedDesserts +nameB: recommendedMainDishes +type: "N:N" # 1:1, 1:N, N:N \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/Recipes.yaml b/dev/import-tool/docs/huly/example-workspace/Recipes.yaml new file mode 100644 index 00000000000..c57b0ca48f3 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/Recipes.yaml @@ -0,0 +1,19 @@ +class: card:class:MasterTag +title: Recipe +properties: + - label: cookingTime + type: TypeString + - label: servings + type: TypeNumber + - label: difficulty + enumOf: "./Difficulty.yaml" + # isArray: true # for multiple values + - label: category + type: TypeString + - label: calories + type: TypeNumber + - label: chef + type: TypeString + - label: relatedRecipes + refTo: "./Recipes.yaml" + isArray: true \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/Recipes/Chocolate Lava Cake.md b/dev/import-tool/docs/huly/example-workspace/Recipes/Chocolate Lava Cake.md new file mode 100644 index 00000000000..1208904b0e0 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/Recipes/Chocolate Lava Cake.md @@ -0,0 +1,40 @@ +--- +title: Chocolate Lava Cake +cookingTime: 25 minutes +servings: 4 +difficulty: Medium +category: Dessert +calories: 450 +chef: Anna Smith +blobs: + - ./files/cake.png +recommendedMainDishes: + - ./Classic Margherita Pizza.md + - ./Vegan/Mushroom Risotto.md +--- + +# Chocolate Lava Cake + +## Ingredients +- 200g dark chocolate (70% cocoa) +- 200g butter +- 4 eggs +- 200g sugar +- 120g flour +- 1 tsp vanilla extract +- Pinch of salt +- Butter for ramekins +- Cocoa powder for dusting + +## Instructions +1. Melt chocolate and butter together +2. Whisk eggs and sugar until pale +3. Fold in chocolate mixture +4. Add flour and vanilla +5. Pour into buttered ramekins +6. Bake at 200°C (400°F) for 12 minutes + +## Notes +- Serve immediately while warm +- Can be prepared ahead and refrigerated +- Perfect with vanilla ice cream diff --git a/dev/import-tool/docs/huly/example-workspace/Recipes/Chocolate Lava Cake/Chocolate Sauce.md b/dev/import-tool/docs/huly/example-workspace/Recipes/Chocolate Lava Cake/Chocolate Sauce.md new file mode 100644 index 00000000000..7c42f5cbde7 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/Recipes/Chocolate Lava Cake/Chocolate Sauce.md @@ -0,0 +1,39 @@ +--- +title: Rich Chocolate Sauce +tags: + - ../DietaryType.yaml +cookingTime: 10 minutes +servings: 4 +difficulty: Easy +category: Dessert Components +calories: 200 +chef: Maria Green +restrictions: Vegetarian +allergens: Dairy +relatedRecipes: + - '../Chocolate Lava Cake.md' +--- + +# Rich Chocolate Sauce for Lava Cake + +## Ingredients +- 100g dark chocolate (70% cocoa) +- 100ml heavy cream +- 30g unsalted butter +- 1 tsp vanilla extract +- Pinch of sea salt + +## Instructions +1. Chop chocolate into small pieces +2. Heat cream until just simmering +3. Pour hot cream over chocolate +4. Let stand for 1 minute +5. Stir until smooth +6. Add butter and vanilla +7. Mix until glossy + +## Notes +- Use high-quality chocolate for best results +- Can be made ahead and reheated +- Store in refrigerator for up to 3 days +- Warm slightly before serving diff --git a/dev/import-tool/docs/huly/example-workspace/Recipes/Classic Margherita Pizza.md b/dev/import-tool/docs/huly/example-workspace/Recipes/Classic Margherita Pizza.md new file mode 100644 index 00000000000..81e72e7e093 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/Recipes/Classic Margherita Pizza.md @@ -0,0 +1,63 @@ +--- +title: Classic Margherita Pizza +tags: + - ./DietaryType.yaml +cookingTime: 30 minutes +servings: 4 +difficulty: Medium +category: Italian +calories: 850 +chef: Mario Rossi +restrictions: Vegetarian +allergens: Gluten, Dairy +recommendedDesserts: + - ./Chocolate Lava Cake.md + +--- + +# Classic Margherita Pizza + +## Ingredients +- 2 1/2 cups (300g) all-purpose flour +- 1 tsp salt +- 1 tsp active dry yeast +- 1 cup warm water +- 2 tbsp olive oil +- 1 cup tomato sauce +- 2 cups mozzarella cheese +- Fresh basil leaves +- Extra virgin olive oil + +## Instructions +1. Mix flour, salt, and yeast in a large bowl +2. Add warm water and olive oil, knead for 10 minutes +3. Let rise for 1 hour +4. Roll out dough and add toppings +5. Bake at 450°F (230°C) for 15-20 minutes + +## Notes +- For best results, use San Marzano tomatoes for the sauce +- Fresh mozzarella is preferred over pre-shredded +- Add basil leaves after baking + +# Classic Margherita Pizza + +## Ingredients +- Pizza dough +- San Marzano tomatoes +- Fresh mozzarella +- Fresh basil +- Extra virgin olive oil +- Salt + +## Instructions +1. Preheat oven to 450°F (230°C) +2. Roll out the pizza dough +3. Add tomato sauce +4. Add fresh mozzarella +5. Bake for 12-15 minutes +6. Add fresh basil and olive oil + +## Notes +- Best served immediately +- Use high-quality ingredients for authentic taste \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/Recipes/DietaryType.yaml b/dev/import-tool/docs/huly/example-workspace/Recipes/DietaryType.yaml new file mode 100644 index 00000000000..0c98fe40e82 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/Recipes/DietaryType.yaml @@ -0,0 +1,7 @@ +class: card:class:Tag +title: DietaryType +properties: + - label: restrictions + type: TypeString + - label: allergens + type: TypeString \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/Recipes/Vegan/Mushroom Risotto.md b/dev/import-tool/docs/huly/example-workspace/Recipes/Vegan/Mushroom Risotto.md new file mode 100644 index 00000000000..06e62f03100 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/Recipes/Vegan/Mushroom Risotto.md @@ -0,0 +1,43 @@ +--- +title: Vegan Mushroom Risotto +cookingTime: 45 minutes +servings: 4 +difficulty: Medium +category: Italian +calories: 380 +chef: Maria Green +proteinSource: Mushrooms +isGlutenFree: true +allergens: None +recommendedDesserts: + - ./Chocolate Lava Cake.md + +--- + +# Vegan Mushroom Risotto + +## Ingredients +- 300g Arborio rice +- 500g mixed mushrooms +- 1 onion, finely chopped +- 2 cloves garlic, minced +- 1 cup white wine +- 6 cups vegetable stock +- 2 tbsp nutritional yeast +- 2 tbsp olive oil +- Salt and pepper to taste +- Fresh parsley + +## Instructions +1. Sauté mushrooms until golden +2. Add onion and garlic, cook until soft +3. Add rice and toast for 2 minutes +4. Gradually add wine and stock +5. Cook until rice is creamy +6. Finish with nutritional yeast + +## Notes +- Use a variety of mushrooms for better flavor +- Keep stock warm while adding +- Stir constantly for creamy texture +- Nutritional yeast adds cheesy flavor \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/Recipes/Vegan/Vegan Recipe.yaml b/dev/import-tool/docs/huly/example-workspace/Recipes/Vegan/Vegan Recipe.yaml new file mode 100644 index 00000000000..a3e275d172b --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/Recipes/Vegan/Vegan Recipe.yaml @@ -0,0 +1,9 @@ +class: card:class:MasterTag +title: Vegan Recipe +properties: + - label: proteinSource + type: TypeString + - label: isGlutenFree + type: TypeBoolean + - label: allergens + type: TypeString \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/Recipes/files/cake.png b/dev/import-tool/docs/huly/example-workspace/Recipes/files/cake.png new file mode 100644 index 00000000000..589aa737e8c Binary files /dev/null and b/dev/import-tool/docs/huly/example-workspace/Recipes/files/cake.png differ diff --git a/dev/import-tool/docs/huly/example-workspace/SlaveCard.yaml b/dev/import-tool/docs/huly/example-workspace/SlaveCard.yaml new file mode 100644 index 00000000000..dda314ba429 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/SlaveCard.yaml @@ -0,0 +1,23 @@ +class: card:class:MasterTag +title: Slave Card 1 +properties: + - label: sex + type: TypeBoolean + # defaultValue: false + - label: color + type: TypeString + # defaultValue: "black" # todo: is default value supported? + # - label: ref + # refTo: ./SlaveCard.yaml + # - label: url + # type: TypeHyperlink + # - label: date + # type: TypeDate + - label: select-enum + enumOf: ./AlphaBetaEnum.yaml + - label: multy-enum + enumOf: ./AlphaBetaEnum.yaml + isArray: true + - label: multy-ref + refTo: ./SlaveCard.yaml + isArray: true diff --git a/dev/import-tool/docs/huly/example-workspace/SlaveCard/FamiliarTag.yaml b/dev/import-tool/docs/huly/example-workspace/SlaveCard/FamiliarTag.yaml new file mode 100644 index 00000000000..83945544978 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/SlaveCard/FamiliarTag.yaml @@ -0,0 +1,5 @@ +class: card:class:Tag +title: Familiar +properties: + - label: x + type: TypeString diff --git a/dev/import-tool/docs/huly/example-workspace/SlaveCard/Frodo.md b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Frodo.md new file mode 100644 index 00000000000..77694d0e36e --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Frodo.md @@ -0,0 +1,19 @@ +--- +title: "Frodo Baggins" +tags: + - ./FamiliarTag.yaml + - ./MinionTag.yaml +familiar: "./Gendalf.md" +sex: true +color: pink +x: from familiar +y: from minion +multy-enum: [Beta] +attachments: + - ../../CARDS_INSTRUCTIONS.md +blobs: + - ../Recipes/files/cake.png + - ../../CARDS_INSTRUCTIONS.md +--- + +A brave hobbit guided by Gandalf. diff --git a/dev/import-tool/docs/huly/example-workspace/SlaveCard/Gendalf.md b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Gendalf.md new file mode 100644 index 00000000000..0a311753af3 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Gendalf.md @@ -0,0 +1,16 @@ +--- +title: "Gandalf the Grey" +tags: ./FamiliarTag.yaml +helpers: "./Frodo.md" +sex: true +color: gray +x: from familiar +ref: ./Worker2.md +# select-enum: Alpha +# multy-enum: [Alpha, Beta] +multy-ref: + - ./Queen.md + - ./Worker1.md +--- + +A wise and powerful wizard who guides others. \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/SlaveCard/MinionTag.yaml b/dev/import-tool/docs/huly/example-workspace/SlaveCard/MinionTag.yaml new file mode 100644 index 00000000000..92ac7bf43ff --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/SlaveCard/MinionTag.yaml @@ -0,0 +1,5 @@ +class: card:class:Tag +title: Minion +properties: + - label: y + type: TypeString diff --git a/dev/import-tool/docs/huly/example-workspace/SlaveCard/Queen.md b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Queen.md new file mode 100644 index 00000000000..57ca9869b89 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Queen.md @@ -0,0 +1,6 @@ +--- +title: "Queen Ant" +children: "./Worker1.md" +--- + +The colony's queen responsible for reproduction. diff --git a/dev/import-tool/docs/huly/example-workspace/SlaveCard/Worker1.md b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Worker1.md new file mode 100644 index 00000000000..9fbec116697 --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Worker1.md @@ -0,0 +1,7 @@ +--- +title: "Worker Ant 1" +tags: ./MinionTag.yaml +mother: "./Queen.md" +--- + +A diligent worker ant. \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/SlaveCard/Worker2.md b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Worker2.md new file mode 100644 index 00000000000..70111bebfdc --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/SlaveCard/Worker2.md @@ -0,0 +1,7 @@ +--- +title: "Worker Ant 2" +tags: ./MinionTag.yaml +# mother: "./Queen.md" +--- + +Another hard-working ant. \ No newline at end of file diff --git a/dev/import-tool/docs/huly/example-workspace/SlaveMinion.yaml b/dev/import-tool/docs/huly/example-workspace/SlaveMinion.yaml new file mode 100644 index 00000000000..6af526758da --- /dev/null +++ b/dev/import-tool/docs/huly/example-workspace/SlaveMinion.yaml @@ -0,0 +1,6 @@ +class: core:class:Association +typeA: ./SlaveCard.yaml +typeB: ./SlaveCard/MinionTag.yaml +nameA: mother +nameB: children +type: 1:N diff --git a/packages/importer/package.json b/packages/importer/package.json index 3533bec328b..b0a674678d0 100644 --- a/packages/importer/package.json +++ b/packages/importer/package.json @@ -42,6 +42,7 @@ }, "dependencies": { "@hcengineering/attachment": "^0.6.14", + "@hcengineering/card": "^0.6.0", "@hcengineering/chunter": "^0.6.20", "@hcengineering/collaboration": "^0.6.0", "@hcengineering/contact": "^0.6.24", diff --git a/packages/importer/src/huly/huly.ts b/packages/importer/src/huly/huly.ts index 0d09a78cb65..7369ad0c3f6 100644 --- a/packages/importer/src/huly/huly.ts +++ b/packages/importer/src/huly/huly.ts @@ -14,15 +14,26 @@ // /* eslint-disable @typescript-eslint/no-unused-vars */ import { type Attachment } from '@hcengineering/attachment' -import contact, { Employee, SocialIdentity, type Person } from '@hcengineering/contact' -import { +import card, { Card, MasterTag, Tag } from '@hcengineering/card' +import contact, { Employee, type Person, SocialIdentity } from '@hcengineering/contact' +import documents, { + ControlledDocument, + DocumentCategory, + DocumentMeta, + DocumentState +} from '@hcengineering/controlled-documents' +import core, { AccountUuid, - buildSocialIdString, + Association, + Attribute, type Class, type Doc, + Enum, generateId, + isId, PersonId, type Ref, + Relation, SocialIdType, type Space, type TxOperations @@ -44,22 +55,20 @@ import { type ImportDocument, ImportDrawing, type ImportIssue, + ImportOrgSpace, type ImportProject, type ImportProjectType, type ImportTeamspace, type ImportWorkspace, - WorkspaceImporter, - ImportOrgSpace + WorkspaceImporter } from '../importer/importer' import { type Logger } from '../importer/logger' import { BaseMarkdownPreprocessor } from '../importer/preprocessor' import { type FileUploader } from '../importer/uploader' -import documents, { - DocumentState, - DocumentCategory, - ControlledDocument, - DocumentMeta -} from '@hcengineering/controlled-documents' +import { UnifiedDoc } from '../types' +import { readMarkdownContent, readYamlHeader } from './parsing' +import { UnifiedDocProcessor } from './unified' +import attachment from '@hcengineering/model-attachment' export interface HulyComment { author: string @@ -335,6 +344,7 @@ interface AttachmentMetadata { export class HulyFormatImporter { private readonly importerEmailPlaceholder = 'newuser@huly.io' private readonly importerNamePlaceholder = 'New User' + private readonly pathById = new Map, string>() private readonly refMetaByPath = new Map() private readonly fileMetaByPath = new Map() @@ -343,15 +353,19 @@ export class HulyFormatImporter { private personsByName = new Map>() private employeesByName = new Map>() private accountsByEmail = new Map() + private readonly personIdByEmail = new Map() + private readonly unifiedDocImporter = new UnifiedDocProcessor() + constructor ( private readonly client: TxOperations, private readonly fileUploader: FileUploader, private readonly logger: Logger, private readonly importerSocialId?: PersonId, private readonly importerPerson?: Ref - ) {} + ) { + } private async initCaches (): Promise { await this.cachePersonsByNames() @@ -476,6 +490,56 @@ export class HulyFormatImporter { } } + // Импортируем UnifiedDoc сущности + const { docs: unifiedDocs, mixins: unifiedMixins, files } = await this.unifiedDocImporter.importFromDirectory(folderPath) + + // Разбираем и добавляем в билдер по классу + for (const [path, docs] of unifiedDocs.entries()) { + for (const doc of docs) { + switch (doc._class) { + case card.class.MasterTag: + builder.addMasterTag(path, doc as UnifiedDoc) + break + case card.class.Tag: + builder.addTag(path, doc as UnifiedDoc) + break + case core.class.Attribute: + builder.addMasterTagAttributes(path, [doc as UnifiedDoc>]) + break + case core.class.Association: + builder.addAssociation(path, doc as UnifiedDoc) + break + case core.class.Relation: + builder.addRelation(path, doc as UnifiedDoc) + break + case core.class.Enum: + builder.addEnum(path, doc as UnifiedDoc) + break + case attachment.class.Attachment: + builder.addAttachment(path, doc as UnifiedDoc) + break + default: + if (isId(doc._class) || (doc._class as string).startsWith('card:types:')) { // todo: fix system cards validation + builder.addCard(path, doc as UnifiedDoc) + } else { + this.logger.error(`Unknown doc class ${String(doc._class)} for path ${path}`) + } + } + } + } + + // todo: attachments + + for (const [path, mixins] of unifiedMixins.entries()) { + for (const mixin of mixins) { + builder.addTagMixin(path, mixin) + } + } + + for (const [path, file] of files.entries()) { + builder.addFile(path, file) + } + // Process all yaml files first const yamlFiles = fs.readdirSync(folderPath).filter((f) => f.endsWith('.yaml') && f !== 'settings.yaml') @@ -521,6 +585,13 @@ export class HulyFormatImporter { break } + case core.class.Enum: + case core.class.Association: + case card.class.MasterTag: { + this.logger.log(`Skipping ${spaceName}: master tag already processed`) + break + } + default: { throw new Error(`Unknown space class ${spaceConfig.class} in ${spaceName}`) } @@ -545,7 +616,7 @@ export class HulyFormatImporter { for (const issueFile of issueFiles) { const issuePath = path.join(currentPath, issueFile) - const issueHeader = (await this.readYamlHeader(issuePath)) as HulyIssueHeader + const issueHeader = (await readYamlHeader(issuePath)) as HulyIssueHeader if (issueHeader.class === undefined) { this.logger.error(`Skipping ${issueFile}: not an issue`) @@ -569,7 +640,7 @@ export class HulyFormatImporter { class: tracker.class.Issue, title: issueHeader.title, number: parseInt(issueNumber ?? 'NaN'), - descrProvider: async () => await this.readMarkdownContent(issuePath), + descrProvider: async () => await readMarkdownContent(issuePath), status: { name: issueHeader.status }, priority: issueHeader.priority, estimation: issueHeader.estimation, @@ -657,7 +728,7 @@ export class HulyFormatImporter { for (const docFile of docFiles) { const docPath = path.join(currentPath, docFile) - const docHeader = (await this.readYamlHeader(docPath)) as HulyDocumentHeader + const docHeader = (await readYamlHeader(docPath)) as HulyDocumentHeader if (docHeader.class === undefined) { this.logger.error(`Skipping ${docFile}: not a document`) @@ -678,7 +749,7 @@ export class HulyFormatImporter { id: docMeta.id as Ref, class: document.class.Document, title: docHeader.title, - descrProvider: async () => await this.readMarkdownContent(docPath), + descrProvider: async () => await readMarkdownContent(docPath), subdocs: [] // Will be added via builder } @@ -705,7 +776,7 @@ export class HulyFormatImporter { for (const docFile of docFiles) { const docPath = path.join(currentPath, docFile) - const docHeader = (await this.readYamlHeader(docPath)) as + const docHeader = (await readYamlHeader(docPath)) as | HulyControlledDocumentHeader | HulyDocumentTemplateHeader @@ -906,7 +977,7 @@ export class HulyFormatImporter { reviewers: header.reviewers?.map((email) => this.findEmployeeByName(email)) ?? [], approvers: header.approvers?.map((email) => this.findEmployeeByName(email)) ?? [], coAuthors: header.coAuthors?.map((email) => this.findEmployeeByName(email)) ?? [], - descrProvider: async () => await this.readMarkdownContent(docPath), + descrProvider: async () => await readMarkdownContent(docPath), ccReason: header.changeControl?.reason, ccImpact: header.changeControl?.impact, ccDescription: header.changeControl?.description, @@ -944,7 +1015,7 @@ export class HulyFormatImporter { reviewers: header.reviewers?.map((email) => this.findEmployeeByName(email)) ?? [], approvers: header.approvers?.map((email) => this.findEmployeeByName(email)) ?? [], coAuthors: header.coAuthors?.map((email) => this.findEmployeeByName(email)) ?? [], - descrProvider: async () => await this.readMarkdownContent(docPath), + descrProvider: async () => await readMarkdownContent(docPath), ccReason: header.changeControl?.reason, ccImpact: header.changeControl?.impact, ccDescription: header.changeControl?.description, @@ -952,22 +1023,6 @@ export class HulyFormatImporter { } } - private async readYamlHeader (filePath: string): Promise { - this.logger.log('Read YAML header from: ' + filePath) - const content = fs.readFileSync(filePath, 'utf8') - const match = content.match(/^---\n([\s\S]*?)\n---/) - if (match != null) { - return yaml.load(match[1]) - } - return {} - } - - private async readMarkdownContent (filePath: string): Promise { - const content = fs.readFileSync(filePath, 'utf8') - const match = content.match(/^---\n[\s\S]*?\n---\n(.*)$/s) - return match != null ? match[1] : content - } - private async cacheAccountsByEmails (): Promise { const employees = await this.client.findAll( contact.mixin.Employee, diff --git a/packages/importer/src/huly/metadata.ts b/packages/importer/src/huly/metadata.ts new file mode 100644 index 00000000000..d125d2c24df --- /dev/null +++ b/packages/importer/src/huly/metadata.ts @@ -0,0 +1,73 @@ +import { Tag } from '@hcengineering/card' +import { Association, Attribute, Doc, generateId, generateUuid, Blob as PlatformBlob, Ref } from '@hcengineering/core' +import { UnifiedDoc } from '../types' + +export interface RelationMetadata { // todo: rename + association: Ref + field: 'docA' | 'docB' + type: '1:1' | '1:N' | 'N:N' +} +export type MapAttributeToUnifiedDoc = Map>> +export type MapNameToIsMany = Map // todo: rename + +export interface TagMetadata { + _id: string + attributes: MapAttributeToUnifiedDoc // title -> attribute id + associations: MapNameToIsMany // nameB -> isMany +} + +export class MetadataStorage { + private readonly pathToRef = new Map>() // todo: attachments to a separate map? + private readonly pathToMetadata = new Map() + private readonly pathToBlobUuid = new Map>() + + public getRefByPath (path: string): Ref { + let ref = this.pathToRef.get(path) + if (ref === undefined) { + ref = generateId() + this.pathToRef.set(path, ref) + } + return ref + } + + public getUuidByPath (path: string): Ref { + let uuid = this.pathToBlobUuid.get(path) + if (uuid === undefined) { + uuid = generateUuid() as Ref + this.pathToBlobUuid.set(path, uuid) + } + return uuid + } + + public hasMetadata (path: string): boolean { + return this.pathToMetadata.has(path) + } + + public getAttributes (path: string): MapAttributeToUnifiedDoc { + return this.pathToMetadata.get(path)?.attributes ?? new Map() + } + + public getAssociations (path: string): MapNameToIsMany { + return this.pathToMetadata.get(path)?.associations ?? new Map() + } + + public setAttributes (path: string, attributes: MapAttributeToUnifiedDoc): void { + const metadata = this.pathToMetadata.get(path) ?? { + _id: this.getRefByPath(path), + attributes: new Map(), + associations: new Map() + } + metadata.attributes = attributes + this.pathToMetadata.set(path, metadata) + } + + public addAssociation (tagPath: string, propName: string, relationMetadata: RelationMetadata): void { + const metadata = this.pathToMetadata.get(tagPath) ?? { + _id: this.getRefByPath(tagPath), + attributes: new Map(), + associations: new Map() + } + metadata.associations.set(propName, relationMetadata) + this.pathToMetadata.set(tagPath, metadata) + } +} diff --git a/packages/importer/src/huly/parsing.ts b/packages/importer/src/huly/parsing.ts new file mode 100644 index 00000000000..0e94b2afa87 --- /dev/null +++ b/packages/importer/src/huly/parsing.ts @@ -0,0 +1,17 @@ +import * as fs from 'fs' +import * as yaml from 'js-yaml' + +export async function readYamlHeader (filePath: string): Promise { + const content = fs.readFileSync(filePath, 'utf8') + const match = content.match(/^---\n([\s\S]*?)\n---/) + if (match != null) { + return yaml.load(match[1]) + } + return {} +} + +export async function readMarkdownContent (filePath: string): Promise { + const content = fs.readFileSync(filePath, 'utf8') + const match = content.match(/^---\n[\s\S]*?\n---\n(.*)$/s) + return match != null ? match[1] : content +} diff --git a/packages/importer/src/huly/unified.ts b/packages/importer/src/huly/unified.ts new file mode 100644 index 00000000000..785bd5521c0 --- /dev/null +++ b/packages/importer/src/huly/unified.ts @@ -0,0 +1,665 @@ +// unified.ts +import { Attachment } from '@hcengineering/attachment' +import card, { Card, MasterTag, Tag } from '@hcengineering/card' +import core, { + Association, + Attribute, + BlobType, + Class, + Doc, + Enum, + generateId, + Ref, + Relation +} from '@hcengineering/core' +import * as fs from 'fs' +import * as yaml from 'js-yaml' +import { contentType } from 'mime-types' +import * as path from 'path' +import { IntlString } from '../../../platform/types' +import { Props, UnifiedDoc, UnifiedFile, UnifiedMixin } from '../types' +import { MetadataStorage, RelationMetadata } from './metadata' +import { readMarkdownContent, readYamlHeader } from './parsing' + +export interface UnifiedDocProcessResult { + docs: Map>> + mixins: Map>> + files: Map +} + +export class UnifiedDocProcessor { + private readonly metadataStorage = new MetadataStorage() + async importFromDirectory (directoryPath: string): Promise { + const result: UnifiedDocProcessResult = { + docs: new Map(), + mixins: new Map(), + files: new Map() + } + // Первый проход - собираем метаданные + await this.processMetadata(directoryPath, result) + + await this.processSystemCards(directoryPath, result, new Map(), new Map()) // todo: get master tag relations + // Второй проход - обрабатываем карточки + await this.processCards(directoryPath, result, new Map(), new Map()) + + return result + } + + private async processMetadata ( + currentPath: string, + result: UnifiedDocProcessResult, + parentMasterTagId?: Ref + ): Promise { + const entries = fs.readdirSync(currentPath, { withFileTypes: true }) + + // Обрабатываем только YAML файлы + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.yaml')) continue + + const yamlPath = path.resolve(currentPath, entry.name) + console.log('Reading yaml file:', yamlPath) + const yamlConfig = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as Record + + switch (yamlConfig?.class) { + case card.class.MasterTag: { + const masterTagId = this.metadataStorage.getRefByPath(yamlPath) as Ref + const masterTag = await this.createMasterTag(yamlConfig, masterTagId, parentMasterTagId) + const masterTagAttrs = await this.createAttributes(yamlPath, yamlConfig, masterTagId) + + this.metadataStorage.setAttributes(yamlPath, masterTagAttrs) + result.docs.set(yamlPath, [masterTag, ...Array.from(masterTagAttrs.values())]) + + // Рекурсивно обрабатываем поддиректорию + const masterTagDir = path.join(currentPath, path.basename(yamlPath, '.yaml')) + if (fs.existsSync(masterTagDir) && fs.statSync(masterTagDir).isDirectory()) { + await this.processMetadata(masterTagDir, result, masterTagId) + } + break + } + case card.class.Tag: { + if (parentMasterTagId === undefined) { + throw new Error('Tag should be inside master tag folder: ' + currentPath) + } + await this.processTag(yamlPath, yamlConfig, result, parentMasterTagId) + break + } + case core.class.Association: { + const association = await this.createAssociation(yamlPath, yamlConfig) + result.docs.set(yamlPath, [association]) + break + } + case core.class.Enum: { + const enumDoc = await this.createEnum(yamlPath, yamlConfig) + result.docs.set(yamlPath, [enumDoc]) + break + } + default: + throw new Error('Unsupported class: ' + yamlConfig?.class) // todo: handle default case just convert to UnifiedDoc + } + } + } + + private async processCards ( + currentPath: string, + result: UnifiedDocProcessResult, + masterTagRelations: Map, + masterTagAttrs: Map>>, + masterTagId?: Ref + ): Promise { + const entries = fs.readdirSync(currentPath, { withFileTypes: true }) + + // Проверяем, есть ли YAML файл MasterTag'а для текущей директории + const yamlPath = currentPath + '.yaml' + if (fs.existsSync(yamlPath)) { + const yamlConfig = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as Record + if (yamlConfig?.class === card.class.MasterTag) { + masterTagId = this.metadataStorage.getRefByPath(yamlPath) as Ref + this.metadataStorage.getAssociations(yamlPath).forEach((relationMetadata, propName) => { + masterTagRelations.set(propName, relationMetadata) + }) + this.metadataStorage.getAttributes(yamlPath).forEach((attr, propName) => { + masterTagAttrs.set(propName, attr) + }) + } + } + + // Обрабатываем MD файлы с учетом текущего MasterTag'а + for (const entry of entries) { + if (entry.isFile() && entry.name.endsWith('.md')) { + const cardPath = path.join(currentPath, entry.name) + const { class: cardType, ...cardProps } = await readYamlHeader(cardPath) + + if (masterTagId !== undefined) { + await this.processCard(result, cardPath, cardProps, masterTagId, masterTagRelations, masterTagAttrs) + } + } + } + + // Рекурсивно обрабатываем поддиректории с передачей текущего MasterTag'а + for (const entry of entries) { + if (!entry.isDirectory()) continue + const dirPath = path.join(currentPath, entry.name) + await this.processCards(dirPath, result, masterTagRelations, masterTagAttrs, masterTagId) + } + } + + private async processSystemCards ( + currentDir: string, + result: UnifiedDocProcessResult, + masterTagRelations: Map, + masterTagAttrs: Map>> + ): Promise { + const entries = fs.readdirSync(currentDir, { withFileTypes: true }) + + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.md')) continue + const cardPath = path.join(currentDir, entry.name) + const { class: cardType, ...cardProps } = await readYamlHeader(cardPath) + + if (cardType.startsWith('card:types:') === false) { + throw new Error('Unsupported card type: ' + cardType + ' in ' + cardPath) + } + + await this.processCard(result, cardPath, cardProps, cardType, masterTagRelations, masterTagAttrs) // todo: get right master tag attributes + } + } + + private async processCard ( + result: UnifiedDocProcessResult, + cardPath: string, + cardProps: Record, + masterTagId: Ref, + masterTagRelations: Map, + masterTagAttrs: Map>>, + parentCardId?: Ref + ): Promise { + console.log('Processing card:', cardPath) + + if (cardProps.blobs !== undefined) { + await this.createBlobs(cardProps.blobs, cardPath, result) + } + + const cardWithRelations = await this.createCardWithRelations(cardProps, cardPath, masterTagId, masterTagRelations, masterTagAttrs, result.files, parentCardId) + + if (cardWithRelations.length > 0) { + const docs = result.docs.get(cardPath) ?? [] + docs.push(...cardWithRelations) + result.docs.set(cardPath, docs) + + const card = cardWithRelations[0] as UnifiedDoc + await this.applyTags(card, cardProps, cardPath, result) + + if (cardProps.attachments !== undefined) { + await this.createAttachments(cardProps.attachments, cardPath, card, result) + } + + const cardDir = path.join(path.dirname(cardPath), path.basename(cardPath, '.md')) + if (fs.existsSync(cardDir) && fs.statSync(cardDir).isDirectory()) { + await this.processCardDirectory(result, cardDir, masterTagId, masterTagRelations, masterTagAttrs, card.props._id as Ref) + } + } + } + + private async processCardDirectory ( + result: UnifiedDocProcessResult, + cardDir: string, + masterTagId: Ref, + masterTagRelations: Map, + masterTagAttrs: Map>>, + parentCardId?: Ref + ): Promise { + const entries = fs.readdirSync(cardDir, { withFileTypes: true }) + .filter(entry => entry.isFile() && entry.name.endsWith('.md')) + + for (const entry of entries) { + const childCardPath = path.join(cardDir, entry.name) + const { class: cardClass, ...cardProps } = await readYamlHeader(childCardPath) + await this.processCard(result, childCardPath, cardProps, masterTagId, masterTagRelations, masterTagAttrs, parentCardId) + } + } + + private async createMasterTag ( + data: Record, + masterTagId: Ref, + parentMasterTagId?: Ref + ): Promise> { + const { class: _class, title } = data + if (_class !== card.class.MasterTag) { + throw new Error('Invalid master tag data') + } + + return { + _class: card.class.MasterTag, + props: { + _id: masterTagId, + space: core.space.Model, + extends: parentMasterTagId ?? card.class.Card, + label: 'embedded:embedded:' + title as IntlString, // todo: check if it's correct + kind: 0, + icon: card.icon.MasterTag + } + } + } + + private async processTag ( + tagPath: string, + tagConfig: Record, + result: UnifiedDocProcessResult, + masterTagId: Ref, + parentTagId?: Ref + ): Promise { + const tagId = this.metadataStorage.getRefByPath(tagPath) as Ref + const tag = await this.createTag(tagConfig, tagId, masterTagId, parentTagId) + + const attributes = await this.createAttributes(tagPath, tagConfig, tagId) + this.metadataStorage.setAttributes(tagPath, attributes) + + const docs = result.docs.get(tagPath) ?? [] + docs.push(tag, ...Array.from(attributes.values())) + result.docs.set(tagPath, docs) + + // Обрабатываем дочерние теги + const tagDir = path.join(path.dirname(tagPath), path.basename(tagPath, '.yaml')) + if (fs.existsSync(tagDir) && fs.statSync(tagDir).isDirectory()) { + await this.processTagDirectory(tagDir, result, masterTagId, tagId) + } + } + + private async processTagDirectory ( + tagDir: string, + result: UnifiedDocProcessResult, + parentMasterTagId: Ref, + parentTagId: Ref + ): Promise { + const entries = fs.readdirSync(tagDir, { withFileTypes: true }) + + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.yaml')) continue + const childTagPath = path.join(tagDir, entry.name) + const childTagConfig = yaml.load(fs.readFileSync(childTagPath, 'utf8')) as Record + + if (childTagConfig?.class === card.class.Tag) { + await this.processTag(childTagPath, childTagConfig, result, parentMasterTagId, parentTagId) + } + } + } + + private async createTag ( + data: Record, + tagId: Ref, + masterTagId: Ref, + parentTagId?: Ref + ): Promise> { + const { class: _class, title } = data + if (_class !== card.class.Tag) { + throw new Error('Invalid tag data') + } + + return { + _class: card.class.Tag, + props: { + _id: tagId, + space: core.space.Model, + extends: parentTagId ?? masterTagId, + label: 'embedded:embedded:' + title as IntlString, + kind: 2, + icon: card.icon.Tag + } + } + } + + private async createAttributes ( + currentPath: string, + data: Record, + masterTagId: Ref + ): Promise>>> { + if (data.properties === undefined) { + return new Map() + } + + const attributesByLabel = new Map>>() + for (const property of data.properties) { + const type = await this.convertPropertyType(property, currentPath) + + const attr: UnifiedDoc> = { + _class: core.class.Attribute, + props: { + space: core.space.Model, + attributeOf: masterTagId, + name: generateId>(), + label: 'embedded:embedded:' + property.label as IntlString, + isCustom: true, + type, + defaultValue: property.defaultValue ?? null + } + } + attributesByLabel.set(property.label, attr) + } + return attributesByLabel + } + + private async convertPropertyType (property: Record, currentPath: string): Promise> { + let type: Record = {} + if (property.refTo !== undefined) { + const baseType: Record = {} + baseType._class = core.class.RefTo + const refPath = path.resolve(path.dirname(currentPath), property.refTo) + baseType.to = this.metadataStorage.getRefByPath(refPath) + baseType.label = core.string.Ref + type = property.isArray === true + ? { + _class: core.class.ArrOf, + label: core.string.Array, + of: baseType + } + : baseType + } else if (property.enumOf !== undefined) { + const baseType: Record = {} + baseType._class = core.class.EnumOf + const enumPath = path.resolve(path.dirname(currentPath), property.enumOf) + baseType.of = this.metadataStorage.getRefByPath(enumPath) + baseType.label = 'core:string:Enum' + type = property.isArray === true + ? { + _class: core.class.ArrOf, + label: core.string.Array, + of: baseType + } + : baseType + } else { + switch (property.type) { + case 'TypeString': + type._class = core.class.TypeString + type.label = core.string.String + break + case 'TypeNumber': + type._class = core.class.TypeNumber + type.label = core.string.Number + break + case 'TypeBoolean': + type._class = core.class.TypeBoolean + type.label = core.string.Boolean + break + default: + throw new Error('Unsupported type: ' + property.type + ' ' + currentPath) + } + } + return type + } + + private async createCardWithRelations ( + cardHeader: Record, + cardPath: string, + masterTagId: Ref, + masterTagRelations: Map, // todo: rename to masterTagsAssociations + masterTagAttrs: Map>>, + blobFiles: Map, + parentCardId?: Ref + ): Promise[]> { + const { _class, title, blobs: rawBlobs, tags: rawTags, ...customProperties } = cardHeader + const tags = rawTags !== undefined ? (Array.isArray(rawTags) ? rawTags : [rawTags]) : [] + const blobs = rawBlobs !== undefined ? (Array.isArray(rawBlobs) ? rawBlobs : [rawBlobs]) : [] + + const cardId = this.metadataStorage.getRefByPath(cardPath) as Ref + const cardProps: Record = { + _id: cardId, + space: core.space.Workspace, + title, + parent: parentCardId + } + + if (blobs.length > 0) { + const blobProps: Record = {} + for (const blob of blobs) { + const blobPath = path.resolve(path.dirname(cardPath), blob) + const blobFile = blobFiles.get(blobPath) + if (blobFile === undefined) { + throw new Error('Blob file not found: ' + blobPath + ' from:' + cardPath) + } + blobProps[blobFile._id] = { + file: blobFile._id, + type: blobFile.type, + name: blobFile.name, + metadata: {} // todo: blobFile.metadata + } + } + cardProps.blobs = blobProps + } + + const tagAssociations = new Map() + for (const tag of tags) { + const tagPath = path.resolve(path.dirname(cardPath), tag) + this.metadataStorage.getAssociations(tagPath).forEach((relationMetadata, propName) => { + tagAssociations.set(propName, relationMetadata) + }) + } + + const relations: UnifiedDoc[] = [] + for (const [key, value] of Object.entries(customProperties)) { + if (masterTagAttrs.has(key)) { + const attr = masterTagAttrs.get(key) + if (attr === undefined) { + throw new Error(`Attribute not found: ${key}, ${cardPath}`) // todo: keep the error till builder validation + } + + const attrProps = attr.props + console.log(key, attrProps.name, value) + + const attrType = attrProps.type + const attrBaseType = attrType._class === core.class.ArrOf ? attrType.of : attrType + const values = attrType._class === core.class.ArrOf ? value : [value] + const propValues = [] + for (const val of values) { + if (attrBaseType._class === core.class.RefTo) { + const refPath = path.resolve(path.dirname(cardPath), val) + const ref = this.metadataStorage.getRefByPath(refPath) as Ref + propValues.push(ref) + } else { + propValues.push(val) + } + } + cardProps[attrProps.name] = attrType._class === core.class.ArrOf ? propValues : propValues[0] + } else if (masterTagRelations.has(key) || tagAssociations.has(key)) { + const metadata = masterTagRelations.get(key) ?? tagAssociations.get(key) + if (metadata === undefined) { + throw new Error(`Association not found: ${key}, ${cardPath}`) // todo: keep the error till builder validation + } + const values = Array.isArray(value) ? value : [value] + for (const val of values) { + const otherCardPath = path.resolve(path.dirname(cardPath), val) + const otherCardId = this.metadataStorage.getRefByPath(otherCardPath) as Ref + const relation: UnifiedDoc = this.createRelation(metadata, cardId, otherCardId) + relations.push(relation) + } + } + } + + return [ + { + _class: masterTagId, + collabField: 'content', + contentProvider: () => readMarkdownContent(cardPath), + props: cardProps as Props // todo: what is the correct props type? + }, + ...relations + ] + } + + private createRelation (metadata: RelationMetadata, cardId: Ref, otherCardId: Ref): UnifiedDoc { + const otherCardField = metadata.field === 'docA' ? 'docB' : 'docA' + const relation: UnifiedDoc = { + _class: core.class.Relation, + props: { + _id: generateId(), + space: core.space.Model, + [metadata.field]: cardId, + [otherCardField]: otherCardId, + association: metadata.association + } as unknown as Props + } + return relation + } + + private async applyTags ( + card: UnifiedDoc, + cardHeader: Record, + cardPath: string, + result: UnifiedDocProcessResult + ): Promise { + const tags = cardHeader.tags !== undefined ? (Array.isArray(cardHeader.tags) ? cardHeader.tags : [cardHeader.tags]) : [] + if (tags.length === 0) return + + console.log(cardHeader.title, cardHeader.tags) + const mixins: UnifiedMixin[] = [] + for (const tagPath of tags) { + const cardDir = path.dirname(cardPath) + const tagAbsPath = path.resolve(cardDir, tagPath) + const tagId = this.metadataStorage.getRefByPath(tagAbsPath) as Ref + + const tagProps: Record = {} + this.metadataStorage.getAttributes(tagAbsPath).forEach((attr, label) => { + tagProps[attr.props.name] = cardHeader[label] + }) + + const mixin: UnifiedMixin = { + _class: card._class, + mixin: tagId, + props: { + _id: card.props._id as Ref, + space: core.space.Workspace, + __mixin: 'true', + ...tagProps + } as unknown as Props // todo: what is the correct props type? + } + mixins.push(mixin) + } + + if (mixins.length > 0) { + result.mixins.set(cardPath, mixins) + } + } + + private async createAttachments ( + attachments: string[], + cardPath: string, + card: UnifiedDoc, + result: UnifiedDocProcessResult + ): Promise { + for (const attachment of attachments) { + const attachmentPath = path.resolve(path.dirname(cardPath), attachment) + const file = await this.createFile(attachmentPath) + result.files.set(attachmentPath, file) + + const attachmentId = this.metadataStorage.getRefByPath(attachmentPath) as Ref + const attachmentDoc: UnifiedDoc = { + _class: 'attachment:class:Attachment' as Ref>, + props: { + _id: attachmentId, + space: core.space.Workspace, + attachedTo: card.props._id as Ref, + attachedToClass: card._class, + file: file._id, + name: file.name, + collection: 'attachments', + lastModified: Date.now(), + type: file.type, + size: file.size + } + } + result.docs.set(attachmentPath, [attachmentDoc]) + } + } + + private async createBlobs ( + blobs: string[], + cardPath: string, + result: UnifiedDocProcessResult + ): Promise { + for (const blob of blobs) { + const blobPath = path.resolve(path.dirname(cardPath), blob) + const file = await this.createFile(blobPath) + result.files.set(blobPath, file) + } + } + + private async createFile ( + fileAbsPath: string + ): Promise { + // const fileAbsPath = path.resolve(path.dirname(currentPath), filePath) + const fileName = path.basename(fileAbsPath) + const fileUuid = this.metadataStorage.getUuidByPath(fileAbsPath) + const type = contentType(fileName) + const size = fs.statSync(fileAbsPath).size + + const file: UnifiedFile = { + _id: fileUuid, // id for datastore + name: fileName, + type: type !== false ? type : 'application/octet-stream', + size, + blobProvider: async () => { + const data = fs.readFileSync(fileAbsPath) + const props = type !== false ? { type } : undefined + return new Blob([data], props) + } + } + return file + } + + private async createAssociation ( + yamlPath: string, + yamlConfig: Record + ): Promise> { + console.log('createAssociation', yamlPath) + const { class: _class, typeA, typeB, type, nameA, nameB } = yamlConfig + + const currentPath = path.dirname(yamlPath) + const associationId = this.metadataStorage.getRefByPath(yamlPath) as Ref + + const typeAPath = path.resolve(currentPath, typeA) + this.metadataStorage.addAssociation(typeAPath, nameB, { + association: associationId, + field: 'docA', + type + }) + + const typeBPath = path.resolve(currentPath, typeB) + this.metadataStorage.addAssociation(typeBPath, nameA, { + association: associationId, + field: 'docB', + type + }) + + const typeAId = this.metadataStorage.getRefByPath(typeAPath) as Ref + const typeBId = this.metadataStorage.getRefByPath(typeBPath) as Ref + + return { + _class, + props: { + _id: associationId, + space: core.space.Model, + classA: typeAId, + classB: typeBId, + nameA, + nameB, + type + } as unknown as Props + } + } + + private async createEnum ( + yamlPath: string, + yamlConfig: Record + ): Promise> { + const { title, values } = yamlConfig + const enumId = this.metadataStorage.getRefByPath(yamlPath) as Ref + return { + _class: core.class.Enum, + props: { + _id: enumId, + space: core.space.Model, + name: title, + enumValues: values + } + } + } +} diff --git a/packages/importer/src/importer/builder.ts b/packages/importer/src/importer/builder.ts index 3442468f51c..396608ae65a 100644 --- a/packages/importer/src/importer/builder.ts +++ b/packages/importer/src/importer/builder.ts @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. // +import card, { Card, MasterTag, Tag } from '@hcengineering/card' import documents, { ControlledDocument, DocumentState } from '@hcengineering/controlled-documents' -import { type DocumentQuery, type Ref, type Status, type TxOperations } from '@hcengineering/core' +import core, { Association, Attribute, Doc, Enum, Relation, type DocumentQuery, type Ref, type Status, type TxOperations } from '@hcengineering/core' import document from '@hcengineering/document' import tracker, { IssuePriority, type IssueStatus } from '@hcengineering/tracker' import { @@ -28,6 +29,8 @@ import { type ImportTeamspace, type ImportWorkspace } from './importer' +import { UnifiedDoc, UnifiedFile, UnifiedMixin } from '../types' +import { Attachment } from '@hcengineering/attachment' export interface ValidationError { path: string @@ -56,6 +59,17 @@ export class ImportWorkspaceBuilder { private readonly qmsDocsBySpace = new Map>() private readonly qmsDocsParents = new Map() + private readonly masterTags = new Map>() + private readonly masterTagAttributes = new Map>>() + private readonly tags = new Map>() + private readonly cards = new Map>() + private readonly associations = new Map>() + private readonly relations = new Map>() + private readonly enums = new Map>() + private readonly attachments = new Map>() + private readonly mixins = new Map>() + private readonly files = new Map() + private readonly projectTypes = new Map() private readonly issueStatusCache = new Map>() private readonly errors = new Map() @@ -221,10 +235,65 @@ export class ImportWorkspaceBuilder { return this } + addMasterTag (path: string, masterTag: UnifiedDoc): this { + this.validateAndAdd('masterTag', path, masterTag, (mt) => this.validateMasterTag(mt), this.masterTags, path) + return this + } + + addTag (path: string, tag: UnifiedDoc): this { + this.validateAndAdd('tag', path, tag, (t) => this.validateTag(t), this.tags, path) + return this + } + + addMasterTagAttributes (path: string, attributes: UnifiedDoc>[]): this { + for (const attribute of attributes) { + const key = path + '/' + attribute.props.name + this.validateAndAdd('masterTagAttribute', key, attribute, (a) => this.validateMasterTagAttribute(a), this.masterTagAttributes, key) + } + return this + } + + addCard (path: string, card: UnifiedDoc): this { + this.validateAndAdd('card', path, card, (c) => this.validateCard(c), this.cards, path) + return this + } + + addAssociation (path: string, association: UnifiedDoc): this { + this.validateAndAdd('association', path, association, (a) => this.validateAssociation(a), this.associations, path) + return this + } + + addRelation (path: string, relation: UnifiedDoc): this { + this.validateAndAdd('relation', path, relation, (r) => this.validateRelation(r), this.relations, path) + return this + } + + addEnum (path: string, enumDoc: UnifiedDoc): this { + this.validateAndAdd('enum', path, enumDoc, (e) => this.validateEnum(e), this.enums, path) + return this + } + + addAttachment (path: string, attachment: UnifiedDoc): this { + this.validateAndAdd('attachment', path, attachment, (a) => this.validateAttachment(a), this.attachments, path) + return this + } + + addTagMixin (path: string, mixin: UnifiedMixin): this { + this.validateAndAdd('tagMixin', path, mixin, (m) => this.validateTagMixin(m), this.mixins, path + '/' + mixin.mixin) // todo: fix mixin key + return this + } + + addFile (path: string, file: UnifiedFile): this { + this.validateAndAdd('file', path, file, (f) => this.validateFile(f), this.files, path) + return this + } + validate (): ValidationResult { // Perform cross-entity validation this.validateSpacesReferences() this.validateDocumentsReferences() + this.validateTagsReferences() + this.validateCardsReferences() return { isValid: this.errors.size === 0, @@ -289,6 +358,18 @@ export class ImportWorkspaceBuilder { ...Array.from(this.teamspaces.values()), ...Array.from(this.qmsSpaces.values()) ], + unifiedDocs: [ + ...Array.from(this.masterTags.values()), + ...Array.from(this.masterTagAttributes.values()), + ...Array.from(this.tags.values()), + ...Array.from(this.cards.values()), + ...Array.from(this.associations.values()), + ...Array.from(this.relations.values()), + ...Array.from(this.enums.values()), + ...Array.from(this.attachments.values()) + ], + mixins: Array.from(this.mixins.values()), + files: Array.from(this.files.values()), attachments: [] } } @@ -570,6 +651,75 @@ export class ImportWorkspaceBuilder { } } + private validateCardsReferences (): void { + // Проверка существования атрибутов + for (const [cardPath, card] of this.cards) { + if (card._class !== undefined) { + const masterTag = this.masterTags.get(card._class) + if (masterTag !== undefined) { + const attributes = Array.from(this.masterTagAttributes.values()) + .filter(a => a.props.attributeOf === card._class) + + // Проверяем, что все используемые атрибуты существуют + for (const [attrName] of Object.entries(card.props)) { + if (attrName !== 'title' && !attributes.some(a => a.props.name === attrName)) { + this.addError(cardPath, `Card uses non-existent attribute: ${attrName}`) + } + } + } + } + + // todo: check if tags are valid + // if (card.props.tags !== undefined) { + // for (const tagId of card.props.tags) { + // const tagExists = Array.from(this.tags.values()).some(t => t.props._id === tagId) + // if (!tagExists) { + // this.addError(cardPath, `Card references non-existent tag: ${tagId}`) + // } + // } + // } + } + } + + private validateTagsReferences (): void { + // Проверка ссылок MasterTag + // for (const [path, masterTag] of this.masterTags) { + // if (masterTag.props.extends !== undefined) { + // if (masterTag.props.extends !== card.class.Card && + // !this.masterTags.has(masterTag.props.extends)) { + // this.addError(path, `Invalid extends reference: ${masterTag.props.extends}`) + // } + // } + // } + + // // Проверка ссылок Tag + // for (const [path, tag] of this.tags) { + // if (tag.props.extends === undefined) { + // this.addError(path, 'extends (MasterTag reference) is required') + // } else if (!this.masterTags.has(tag.props.extends)) { + // this.addError(path, `Invalid MasterTag reference: ${tag.props.extends}`) + // } + // } + + // // Проверка ссылок атрибутов + // for (const [path, attribute] of this.masterTagAttributes) { + // if (attribute.props.attributeOf === undefined) { + // this.addError(path, 'attributeOf (MasterTag reference) is required') + // } else if (!this.masterTags.has(attribute.props.attributeOf)) { + // this.addError(path, `Invalid MasterTag reference: ${attribute.props.attributeOf}`) + // } + // } + + // // Проверка ссылок карточек + // for (const [path, card] of this.cards) { + // if (card._class === undefined) { + // this.addError(path, 'class (MasterTag reference) is required') + // } else if (!this.masterTags.has(card._class)) { + // this.addError(path, `Invalid MasterTag reference: ${card._class}`) + // } + // } + } + private addError (path: string, error: string): void { this.errors.set(path, { path, error }) } @@ -675,6 +825,213 @@ export class ImportWorkspaceBuilder { return errors } + private validateMasterTag (masterTag: UnifiedDoc): string[] { + const errors: string[] = [] + + // Проверка класса + if (masterTag._class !== card.class.MasterTag) { + errors.push('Invalid class: ' + masterTag._class) + } + + // Проверка обязательных полей + if (!this.validateStringDefined(masterTag.props.label)) { + errors.push('label is required') + } + + // Проверка уникальности имени + const existingTags = Array.from(this.masterTags.values()) + .filter(tag => tag.props.label === masterTag.props.label) + if (existingTags.length > 0) { + errors.push(`MasterTag with label "${masterTag.props.label}" already exists`) + } + + return errors + } + + private validateTag (tag: UnifiedDoc): string[] { + const errors: string[] = [] + + // Проверка класса + if (tag._class !== card.class.Tag) { + errors.push('Invalid class: ' + tag._class) + } + + // Проверка обязательных полей + if (!this.validateStringDefined(tag.props.label)) { + errors.push('label is required') + } + + // Проверка уникальности имени в рамках MasterTag + const existingTags = Array.from(this.tags.values()) + .filter(t => t.props.extends === tag.props.extends && + t.props.label === tag.props.label) + if (existingTags.length > 0) { + errors.push(`Tag with label "${tag.props.label}" already exists for this MasterTag`) + } + + return errors + } + + private validateMasterTagAttribute (attribute: UnifiedDoc>): string[] { + const errors: string[] = [] + + // Проверка класса + if (attribute._class !== core.class.Attribute) { + errors.push('Invalid class: ' + attribute._class) + } + + // Проверка обязательных полей + if (!this.validateStringDefined(attribute.props.name)) { + errors.push('name is required') + } + if (!this.validateStringDefined(attribute.props.label)) { + errors.push('label is required') + } + + // todo: fix Проверка связи с MasterTag + // if (attribute.props.attributeOf === undefined) { + // errors.push('attributeOf (MasterTag reference) is required') + // } else if (!this.masterTags.has(attribute.props.attributeOf)) { + // errors.push(`Invalid MasterTag reference: ${attribute.props.attributeOf}`) + // } + + // todo: fix Проверка типа атрибута + // if (attribute.props.type === undefined) { + // errors.push('type is required') + // } else { + // const validTypes = [ // todo: double check valid types + // 'TypeString', 'TypeNumber', 'TypeBoolean', 'TypeDate', + // 'TypeHyperlink', 'TypeEnum', 'TypeFileSize', 'TypeIntlString', + // 'TypeMarkup', 'TypeTimestamp', 'TypeRef', 'TypeCollection' + // ] + // if (!validTypes.includes(attribute.props.type)) { + // errors.push(`Invalid attribute type: ${attribute.props.type}`) + // } + // } + + // Проверка уникальности имени атрибута в рамках MasterTag + const existingAttributes = Array.from(this.masterTagAttributes.values()) + .filter(a => a.props.attributeOf === attribute.props.attributeOf && + a.props.name === attribute.props.name) + if (existingAttributes.length > 0) { + errors.push(`Attribute with name "${attribute.props.name}" already exists for this MasterTag`) + } + + return errors + } + + private validateAssociation (association: UnifiedDoc): string[] { + const errors: string[] = [] + + // todo: validate association + + return errors + } + + private validateRelation (relation: UnifiedDoc): string[] { + const errors: string[] = [] + + // todo: validate relation + + return errors + } + + private validateEnum (enumDoc: UnifiedDoc): string[] { + const errors: string[] = [] + + // todo: validate enum + + return errors + } + + private validateAttachment (attachment: UnifiedDoc): string[] { + const errors: string[] = [] + + // todo: validate attachment + + return errors + } + + private validateCard (card: UnifiedDoc): string[] { + const errors: string[] = [] + + // Проверка класса (должен быть ссылкой на MasterTag) + // if (card._class === undefined) { + // errors.push('class (MasterTag reference) is required') + // } else if (!this.masterTags.has(card._class)) { + // errors.push(`Invalid MasterTag reference: ${card._class}`) + // } + + // Проверка обязательных полей + if (!this.validateStringDefined(card.props.title)) { + errors.push('title is required') + } + + // Получаем MasterTag и его атрибуты + const masterTag = this.masterTags.get(card._class) + if (masterTag !== undefined) { + const attributes = Array.from(this.masterTagAttributes.values()) + .filter(a => a.props.attributeOf === card._class) + + // Проверяем значения атрибутов + for (const attribute of attributes) { + const value = (card.props as Record)[attribute.props.name] + + // Проверка обязательных атрибутов + // todo: check if required attribute is missing + // if (attribute.props.required && value === undefined) { + // errors.push(`Required attribute "${attribute.props.label}" is missing`) + // continue + // } + + // Проверка типов данных + if (value !== undefined) { + switch (attribute.props.type) { + case 'TypeString': + if (typeof value !== 'string') { + errors.push(`Attribute "${attribute.props.label}" must be a string`) + } + break + case 'TypeNumber': + if (typeof value !== 'number') { + errors.push(`Attribute "${attribute.props.label}" must be a number`) + } + break + case 'TypeBoolean': + if (typeof value !== 'boolean') { + errors.push(`Attribute "${attribute.props.label}" must be a boolean`) + } + break + case 'TypeDate': + if (!(value instanceof Date)) { + errors.push(`Attribute "${attribute.props.label}" must be a date`) + } + break + // todo: add other types as needed + } + } + } + } + + return errors + } + + private validateTagMixin (mixin: UnifiedMixin): string[] { + const errors: string[] = [] + + // todo: validate tag mixin + + return errors + } + + private validateFile (file: UnifiedFile): string[] { + const errors: string[] = [] + + // todo: validate file + + return errors + } + private validateOrgSpace (space: ImportOrgSpace): string[] { const errors: string[] = [] diff --git a/packages/importer/src/importer/importer.ts b/packages/importer/src/importer/importer.ts index 2ab4e1b103e..634539844bb 100644 --- a/packages/importer/src/importer/importer.ts +++ b/packages/importer/src/importer/importer.ts @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -import attachment, { Drawing, type Attachment } from '@hcengineering/attachment' +import attachment, { type Attachment, Drawing } from '@hcengineering/attachment' import chunter, { type ChatMessage } from '@hcengineering/chunter' import { Employee, type Person } from '@hcengineering/contact' import documents, { @@ -30,7 +30,9 @@ import documents, { useDocumentTemplate } from '@hcengineering/controlled-documents' import core, { + type AccountUuid, type AttachedData, + AttachedDoc, type Class, type CollaborativeDoc, type Data, @@ -39,6 +41,7 @@ import core, { generateId, makeCollabId, type Mixin, + type PersonId, type Blob as PlatformBlob, type Ref, RolesAssignment, @@ -46,9 +49,7 @@ import core, { type Space, type Status, type Timestamp, - type TxOperations, - type PersonId, - type AccountUuid + type TxOperations } from '@hcengineering/core' import document, { type Document, getFirstRank, type Teamspace } from '@hcengineering/document' import task, { @@ -69,14 +70,18 @@ import tracker, { TimeReportDayType } from '@hcengineering/tracker' import view from '@hcengineering/view' +import { Props, UnifiedDoc, UnifiedFile, UnifiedMixin } from '../types' +import { Logger } from './logger' import { type MarkdownPreprocessor, NoopMarkdownPreprocessor } from './preprocessor' import { type FileUploader } from './uploader' -import { Logger } from './logger' - export interface ImportWorkspace { projectTypes?: ImportProjectType[] spaces?: ImportSpace[] attachments?: ImportAttachment[] + + unifiedDocs?: UnifiedDoc>[] + mixins?: UnifiedMixin, Doc>[] + files?: UnifiedFile[] } export interface ImportProjectType { @@ -241,6 +246,10 @@ export class WorkspaceImporter { await this.importProjectTypes() await this.importSpaces() await this.importAttachments() + + await this.importUnifiedDocs() + await this.importUnifiedMixins() + await this.uploadFiles() } private async importProjectTypes (): Promise { @@ -1129,4 +1138,73 @@ export class WorkspaceImporter { return await this.client.createDoc(documents.class.ChangeControl, spaceId, changeControlData) } + + private async importUnifiedDocs (): Promise { + if (this.workspaceData.unifiedDocs === undefined) return + + for (const unifiedDoc of this.workspaceData.unifiedDocs) { + await this.createUnifiedDoc(unifiedDoc) + } + } + + private async createUnifiedDoc (unifiedDoc: UnifiedDoc>): Promise { + const { _class, props } = unifiedDoc + const _id = props._id ?? generateId>() + if (unifiedDoc.collabField !== undefined) { + const collabId = makeCollabId(_class, _id, unifiedDoc.collabField) + const collabContent = await unifiedDoc.contentProvider?.() ?? '' + const res = await this.createCollaborativeContent(_id, collabId, collabContent, props.space) + ;(props as any)[unifiedDoc.collabField] = res + } + + const hierarchy = this.client.getHierarchy() + if (hierarchy.isDerived(_class, core.class.AttachedDoc)) { + const { space, attachedTo, attachedToClass, collection, ...data } = props as unknown as Props + if ( + attachedTo === undefined || + space === undefined || + attachedToClass === undefined || + collection === undefined + ) { + throw new Error('Add collection step must have attachedTo, attachedToClass, collection and space') + } + await this.client.addCollection( + _class, + space, + attachedTo, + attachedToClass, + collection, + data, + _id as Ref | undefined + ) + } else { + await this.client.createDoc(_class, props.space, props as Data>, _id) + } + } + + private async importUnifiedMixins (): Promise { + if (this.workspaceData.mixins === undefined) return + + for (const mixin of this.workspaceData.mixins) { + await this.createUnifiedMixin(mixin) + } + } + + private async createUnifiedMixin (mixin: UnifiedMixin, Doc>): Promise { + const { _class, mixin: mixinClass, props } = mixin + const { _id, space, ...data } = props + await this.client.createMixin(_id ?? generateId>(), _class, space, mixinClass, data as Data>) + } + + private async uploadFiles (): Promise { + if (this.workspaceData.files === undefined) return + + for (const file of this.workspaceData.files) { + const id = file._id ?? generateId() + const uploadResult = await this.fileUploader.uploadFile(id, await file.blobProvider()) + if (!uploadResult.success) { + throw new Error('Failed to upload attachment file: ' + file.name) + } + } + } } diff --git a/packages/importer/src/types.ts b/packages/importer/src/types.ts new file mode 100644 index 00000000000..cc230793179 --- /dev/null +++ b/packages/importer/src/types.ts @@ -0,0 +1,26 @@ +import { Class, Data, Doc, Mixin, Ref, Space, Blob as PlatformBlob } from '@hcengineering/core' +export type Props = Data & Partial & { space: Ref } + +export interface UnifiedDoc { + _class: Ref> + props: Props + collabField?: string + contentProvider?: () => Promise +} + +export interface UnifiedMixin { // todo: extends T + _class: Ref> + mixin: Ref> + props: Props +} + +export interface UnifiedFile { + _id: Ref + name: string + type: string + size: number + blobProvider: blobProvider +} + +export type contentProvider = () => Promise +export type blobProvider = () => Promise