Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 65 additions & 26 deletions lib/lightning/projects/merge_projects.ex
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,13 @@ defmodule Lightning.Projects.MergeProjects do
A map with the merged project structure ready for import, containing
workflow mappings and project data.
"""
@spec merge_project(Project.t(), Project.t()) :: map()
@spec merge_project(Project.t(), Project.t(), map()) :: map()
def merge_project(source, target, new_uuid_map \\ %{})

def merge_project(
%Project{} = source_project,
%Project{} = target_project
%Project{} = target_project,
new_uuid_map
) do
source_project =
Repo.preload(source_project, workflows: [:jobs, :triggers, :edges])
Expand All @@ -39,19 +42,21 @@ defmodule Lightning.Projects.MergeProjects do

merge_project(
Map.from_struct(source_project),
Map.from_struct(target_project)
Map.from_struct(target_project),
new_uuid_map
)
end

def merge_project(source_project, target_project) do
def merge_project(source_project, target_project, new_uuid_map) do
workflow_mappings =
map_project_workflow_names(source_project, target_project)

# Build the merged project structure using the mappings
build_merged_project(
source_project,
target_project,
workflow_mappings
workflow_mappings,
new_uuid_map
)
end

Expand All @@ -75,24 +80,33 @@ defmodule Lightning.Projects.MergeProjects do
A map with the merged workflow structure ready for import, containing
UUID mappings and workflow data.
"""
@spec merge_workflow(Workflow.t(), Workflow.t()) :: map()
@spec merge_workflow(Workflow.t(), Workflow.t(), map()) :: map()
def merge_workflow(source, target, new_uuid_map \\ %{})

def merge_workflow(
%Workflow{} = source_workflow,
%Workflow{} = target_workflow
%Workflow{} = target_workflow,
new_uuid_map
) do
source_workflow = Repo.preload(source_workflow, [:jobs, :triggers, :edges])
target_workflow = Repo.preload(target_workflow, [:jobs, :triggers, :edges])

merge_workflow(
Map.from_struct(source_workflow),
Map.from_struct(target_workflow)
Map.from_struct(target_workflow),
new_uuid_map
)
end

def merge_workflow(source_workflow, target_workflow) do
def merge_workflow(source_workflow, target_workflow, new_uuid_map) do
node_mappings = map_workflow_node_ids(source_workflow, target_workflow)

build_merged_workflow(source_workflow, target_workflow, node_mappings)
build_merged_workflow(
source_workflow,
target_workflow,
node_mappings,
new_uuid_map
)
end

defp map_workflow_node_ids(source_workflow, target_workflow) do
Expand Down Expand Up @@ -425,7 +439,12 @@ defmodule Lightning.Projects.MergeProjects do
end)
end

defp build_merged_workflow(source_workflow, target_workflow, node_mappings) do
defp build_merged_workflow(
source_workflow,
target_workflow,
node_mappings,
new_uuid_map
) do
source_trigger_ids = Enum.map(source_workflow.triggers, & &1.id)

{trigger_mappings, job_mappings} =
Expand All @@ -435,7 +454,8 @@ defmodule Lightning.Projects.MergeProjects do
build_merged_jobs(
source_workflow.jobs,
target_workflow.jobs,
job_mappings
job_mappings,
new_uuid_map
)

{trigger_mappings, merged_triggers} =
Expand All @@ -452,7 +472,8 @@ defmodule Lightning.Projects.MergeProjects do
build_merged_edges(
source_workflow.edges,
target_workflow.edges,
node_mappings
node_mappings,
new_uuid_map
)

initial_positions = Map.get(source_workflow, :positions) || %{}
Expand All @@ -476,7 +497,7 @@ defmodule Lightning.Projects.MergeProjects do
})
end

defp build_merged_jobs(source_jobs, target_jobs, job_mappings) do
defp build_merged_jobs(source_jobs, target_jobs, job_mappings, new_uuid_map) do
target_jobs_by_id = Map.new(target_jobs, &{&1.id, &1})

# Process source jobs (matched and new)
Expand All @@ -486,7 +507,9 @@ defmodule Lightning.Projects.MergeProjects do
{%{}, []},
fn source_job, {new_mapping, merged_jobs} ->
mapped_id =
Map.get(job_mappings, source_job.id) || Ecto.UUID.generate()
Map.get(job_mappings, source_job.id) ||
Map.get(new_uuid_map, source_job.id) ||
Ecto.UUID.generate()

target_job = Map.get(target_jobs_by_id, mapped_id)

Expand Down Expand Up @@ -575,7 +598,12 @@ defmodule Lightning.Projects.MergeProjects do
{new_mapping, merged_from_source ++ deleted_targets}
end

defp build_merged_edges(source_edges, target_edges, node_mappings) do
defp build_merged_edges(
source_edges,
target_edges,
node_mappings,
new_uuid_map
) do
merged_from_source =
Enum.map(source_edges, fn source_edge ->
from_id =
Expand All @@ -589,7 +617,11 @@ defmodule Lightning.Projects.MergeProjects do
target_edge = find_edge(target_edges, mapped_from_id, mapped_to_id)

mapped_id =
if target_edge, do: target_edge.id, else: Ecto.UUID.generate()
if target_edge do
target_edge.id
else
Map.get(new_uuid_map, source_edge.id) || Ecto.UUID.generate()
end

source_edge
|> Map.take([
Expand Down Expand Up @@ -642,12 +674,18 @@ defmodule Lightning.Projects.MergeProjects do
Enum.find(workflows, &(&1.name == name))
end

defp build_merged_project(source_project, target_project, workflow_mappings) do
defp build_merged_project(
source_project,
target_project,
workflow_mappings,
new_uuid_map
) do
merged_workflows =
build_merged_workflows(
source_project.workflows,
target_project.workflows,
workflow_mappings
workflow_mappings,
new_uuid_map
)

target_project
Expand All @@ -662,19 +700,20 @@ defmodule Lightning.Projects.MergeProjects do
defp build_merged_workflows(
source_workflows,
target_workflows,
workflow_mappings
workflow_mappings,
new_uuid_map
) do
# Process source workflows (matched and new)
merged_from_source =
Enum.map(source_workflows, fn source_workflow ->
case Map.get(workflow_mappings, source_workflow.id) do
nil ->
build_new_workflow(source_workflow)
build_new_workflow(source_workflow, new_uuid_map)

target_id ->
# Matched workflow - merge using existing merge_workflow logic
target_workflow = Enum.find(target_workflows, &(&1.id == target_id))
merge_workflow(source_workflow, target_workflow)
merge_workflow(source_workflow, target_workflow, new_uuid_map)
end
end)

Expand All @@ -691,10 +730,10 @@ defmodule Lightning.Projects.MergeProjects do
merged_from_source ++ deleted_targets
end

defp build_new_workflow(source_workflow) do
defp build_new_workflow(source_workflow, new_uuid_map) do
job_id_map =
Map.new(source_workflow.jobs, fn job ->
{job.id, Ecto.UUID.generate()}
{job.id, Map.get(new_uuid_map, job.id) || Ecto.UUID.generate()}
end)

trigger_id_map =
Expand Down Expand Up @@ -746,7 +785,7 @@ defmodule Lightning.Projects.MergeProjects do
:enabled
])
|> Map.merge(%{
id: Ecto.UUID.generate(),
id: Map.get(new_uuid_map, edge.id) || Ecto.UUID.generate(),
source_job_id: edge.source_job_id && mapped_from_id,
source_trigger_id: edge.source_trigger_id && mapped_from_id,
target_job_id: mapped_to_id
Expand All @@ -757,7 +796,7 @@ defmodule Lightning.Projects.MergeProjects do
source_workflow
|> Map.take([:name, :concurrency, :enable_job_logs])
|> Map.merge(%{
"id" => Ecto.UUID.generate(),
"id" => Map.get(new_uuid_map, source_workflow.id) || Ecto.UUID.generate(),
"jobs" => jobs,
"triggers" => triggers,
"edges" => edges
Expand Down
54 changes: 47 additions & 7 deletions lib/mix/tasks/merge_projects.ex
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ defmodule Mix.Tasks.Lightning.MergeProjects do
## Options

* `-o, --output PATH` - Write output to file instead of stdout
* `--uuid SOURCE_UUID:TARGET_UUID` - Map source UUID to target UUID for any entity (workflow, job, or edge) (repeatable)

## Examples

Expand All @@ -29,6 +30,13 @@ defmodule Mix.Tasks.Lightning.MergeProjects do

# Merge with explicit output flag
mix lightning.merge_projects staging.state.json main.state.json --output result.json

# Merge with UUID mappings for workflows, jobs, and edges
mix lightning.merge_projects staging.state.json main.state.json \\
--uuid 550e8400-e29b-41d4-a716-446655440000:650e8400-e29b-41d4-a716-446655440001 \\
--uuid a1b2c3d4-e5f6-4a5b-8c7d-1e2f3a4b5c6d:b2c3d4e5-f6a7-4b5c-8d7e-2f3a4b5c6d7e \\
--uuid f6a7b8c9-d0e1-4f5a-9b0c-5d6e7f8a9b0c:a7b8c9d0-e1f2-4a5b-9c0d-6e7f8a9b0c1d \\
-o merged.json
"""
use Mix.Task

Expand All @@ -38,7 +46,7 @@ defmodule Mix.Tasks.Lightning.MergeProjects do
def run(args) do
{opts, positional, invalid} =
OptionParser.parse(args,
strict: [output: :string],
strict: [output: :string, uuid: :keep],
aliases: [o: :output]
)

Expand All @@ -50,7 +58,8 @@ defmodule Mix.Tasks.Lightning.MergeProjects do
Unknown option(s): #{invalid_opts}

Valid options:
-o, --output PATH Write output to file instead of stdout
-o, --output PATH Write output to file instead of stdout
--uuid SOURCE_UUID:TARGET_UUID Map source UUID to target UUID (repeatable)

Run `mix help lightning.merge_projects` for more information.
""")
Expand All @@ -67,11 +76,12 @@ defmodule Mix.Tasks.Lightning.MergeProjects do

true ->
[source_file, target_file] = positional
merge_and_output(source_file, target_file, opts)
uuid_map = parse_uuid_mappings(opts)
merge_and_output(source_file, target_file, opts, uuid_map)
end
end

defp merge_and_output(source_file, target_file, opts) do
defp merge_and_output(source_file, target_file, opts, uuid_map) do
output_path = Keyword.get(opts, :output)

if output_path do
Expand All @@ -83,14 +93,14 @@ defmodule Mix.Tasks.Lightning.MergeProjects do

target_project = read_state_file(target_file, "target")

merged_project = perform_merge(source_project, target_project)
merged_project = perform_merge(source_project, target_project, uuid_map)

output = encode_json(merged_project)
write_output(output, output_path)
end

defp perform_merge(source_project, target_project) do
MergeProjects.merge_project(source_project, target_project)
defp perform_merge(source_project, target_project, uuid_map) do
MergeProjects.merge_project(source_project, target_project, uuid_map)
rescue
e in KeyError ->
Mix.raise("""
Expand All @@ -113,6 +123,36 @@ defmodule Mix.Tasks.Lightning.MergeProjects do
""")
end

defp parse_uuid_mappings(opts) do
opts
|> Keyword.get_values(:uuid)
|> Enum.reduce(%{}, fn mapping_str, acc ->
case String.split(mapping_str, ":") do
[source_id, target_id] ->
source_id = String.trim(source_id)
target_id = String.trim(target_id)

# Support both string and integer keys by adding both forms
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@josephjclark by the way why do we need to add both keys in here? And why are we only parsing the source and not the target?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, it's actually this part that's failing in the CI. I'll have a look tomorrow

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is probably AI silliness that has not been properly validated by this silly human

I do need string and integer types, although maybe there's a better way I can handle this (like maybe my test code should be casting numbers to strings)

acc
|> Map.put(source_id, target_id)
|> then(fn map ->
case Integer.parse(source_id) do
{int_id, ""} -> Map.put(map, int_id, target_id)
_ -> map
end
end)

_other ->
Mix.raise("""
Invalid UUID mapping format: #{mapping_str}

Expected format: SOURCE_UUID:TARGET_UUID
Example: --uuid 550e8400-e29b-41d4-a716-446655440000:650e8400-e29b-41d4-a716-446655440001
""")
end
end)
end

defp encode_json(project) do
Jason.encode!(project, pretty: true)
rescue
Expand Down
Loading