Skip to content

System diagram #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

System diagram #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1ef9912
Added a system diagram.
DanGraur Mar 10, 2024
939a08a
Added png version of diagram.
DanGraur Mar 10, 2024
bf26b89
Updated README to use the png system diagram.
DanGraur Mar 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Mixtera

A lightweight, user-friendly data-plane for LLM training.

![](docs/source/_static/system_diagram/mixtera_system_diagram.drawio.png)
193 changes: 193 additions & 0 deletions docs/source/_static/system_diagram/mixtera_system_diagram.drawio
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
<mxfile host="app.diagrams.net" modified="2024-03-10T16:37:38.933Z" agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" etag="3KsFqLSJIImLk-mXuLIZ" version="22.1.21" type="device">
<diagram name="Page-1" id="bFX4RP2RZVkXfkcDbsq_">
<mxGraphModel dx="1136" dy="621" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="Zkzzl6DhGFWdUMVNODvd-27" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" vertex="1" parent="1">
<mxGeometry x="240" y="130" width="200" height="90" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0.5;entryY=0;entryDx=0;entryDy=0;dashed=1;" edge="1" parent="1" source="Zkzzl6DhGFWdUMVNODvd-1" target="Zkzzl6DhGFWdUMVNODvd-3">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="228" y="45" />
</Array>
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-25" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;dashed=1;" edge="1" parent="1" source="Zkzzl6DhGFWdUMVNODvd-1" target="Zkzzl6DhGFWdUMVNODvd-11">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="453" y="50" />
<mxPoint x="453" y="180" />
<mxPoint x="453" y="180" />
</Array>
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-26" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" source="Zkzzl6DhGFWdUMVNODvd-1" target="Zkzzl6DhGFWdUMVNODvd-2">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-1" value="Storage Layer&lt;br&gt;[Object Storage/NFS/DFS]" style="shape=cylinder3;whiteSpace=wrap;html=1;boundedLbl=1;backgroundOutline=1;size=15;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="1">
<mxGeometry x="250" y="10" width="180" height="70" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-2" value="Mixtera Server" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1">
<mxGeometry x="280" y="150" width="120" height="60" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-3" value="Client Node" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;verticalAlign=middle;" vertex="1" parent="1">
<mxGeometry x="140" y="290" width="175" height="110" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-6" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;startArrow=classic;startFill=1;" edge="1" parent="1" source="Zkzzl6DhGFWdUMVNODvd-3" target="Zkzzl6DhGFWdUMVNODvd-2">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="330" y="280" as="sourcePoint" />
<mxPoint x="380" y="230" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-7" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;startArrow=classic;startFill=1;" edge="1" parent="1" source="Zkzzl6DhGFWdUMVNODvd-11" target="Zkzzl6DhGFWdUMVNODvd-2">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="435" y="290" as="sourcePoint" />
<mxPoint x="340" y="210" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-8" value="Mixtera Client" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
<mxGeometry x="145" y="300" width="70" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-9" value="Mixtera Client" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
<mxGeometry x="240" y="300" width="70" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-10" value="..." style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="197.5" y="300" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-11" value="Client Node" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;verticalAlign=middle;" vertex="1" parent="1">
<mxGeometry x="365" y="290" width="175" height="110" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-12" value="Mixtera Client" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
<mxGeometry x="370" y="300" width="70" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-13" value="Mixtera Client" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
<mxGeometry x="465" y="300" width="70" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-14" value="..." style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="422.5" y="300" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-15" value="PyTorch / TensorFlow / Jax" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1">
<mxGeometry x="145" y="360" width="165" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-16" value="PyTorch / TensorFlow / Jax" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="1">
<mxGeometry x="370" y="360" width="165" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-18" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.229;entryY=0.636;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;" edge="1" parent="1" source="Zkzzl6DhGFWdUMVNODvd-8" target="Zkzzl6DhGFWdUMVNODvd-3">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="210" y="350" as="sourcePoint" />
<mxPoint x="260" y="300" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-19" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.229;entryY=0.636;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;" edge="1" parent="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="274.89" y="330" as="sourcePoint" />
<mxPoint x="274.89" y="360" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-20" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.229;entryY=0.636;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;" edge="1" parent="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="404.8" y="330" as="sourcePoint" />
<mxPoint x="404.8" y="360" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-21" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0.229;entryY=0.636;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;" edge="1" parent="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="499.8" y="330" as="sourcePoint" />
<mxPoint x="499.8" y="360" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-22" value="..." style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
<mxGeometry x="310" y="330" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-28" value="&lt;font style=&quot;font-size: 10px;&quot;&gt;Mixtera&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="240" y="130" width="50" height="15" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-29" value="" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
<mxGeometry x="491" y="20" width="80" height="140" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-30" value="&lt;font style=&quot;font-size: 10px;&quot;&gt;Legend&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" vertex="1" parent="1">
<mxGeometry x="491" y="20" width="80" height="15" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-31" value="&amp;lt;request &amp;amp; get mixture chunks&amp;gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;rotation=35;fontSize=7;" vertex="1" parent="1">
<mxGeometry x="321" y="236" width="170" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-33" value="&amp;lt;retrieve instances from storage layer based on chunk&amp;gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;rotation=90;fontSize=7;" vertex="1" parent="1">
<mxGeometry x="363" y="133" width="196" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-34" value="&amp;lt;highly privileged data access&amp;gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=7;rotation=90;" vertex="1" parent="1">
<mxGeometry x="320" y="95" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-36" value="" style="endArrow=classic;html=1;rounded=0;entryX=0.75;entryY=0;entryDx=0;entryDy=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" target="Zkzzl6DhGFWdUMVNODvd-3">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="340" y="210" as="sourcePoint" />
<mxPoint x="410" y="190" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-37" value="" style="endArrow=classic;html=1;rounded=0;entryX=0.25;entryY=0;entryDx=0;entryDy=0;dashed=1;dashPattern=8 8;" edge="1" parent="1" target="Zkzzl6DhGFWdUMVNODvd-11">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="340" y="210" as="sourcePoint" />
<mxPoint x="281" y="300" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-38" value="&amp;lt;Highly privileged instances read through server&amp;gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=7;rotation=-48.7;" vertex="1" parent="1">
<mxGeometry x="285" y="244" width="70" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-42" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;endArrow=none;endFill=0;dashed=1;dashPattern=1 4;" edge="1" parent="1" source="Zkzzl6DhGFWdUMVNODvd-41" target="Zkzzl6DhGFWdUMVNODvd-2">
<mxGeometry relative="1" as="geometry">
<Array as="points">
<mxPoint x="270" y="180" />
<mxPoint x="270" y="180" />
</Array>
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-41" value="Generates static/dynamic data mixtures based on job query and needs;&lt;br style=&quot;font-size: 7px;&quot;&gt;&lt;br style=&quot;font-size: 7px;&quot;&gt;Mixtures are sharded into chunks and are yielded on request to the clients&lt;br&gt;&lt;br&gt;In special cases, if the data is highly sensitive, the Server can also yield the instances directly" style="shape=note;whiteSpace=wrap;html=1;backgroundOutline=1;darkOpacity=0.05;size=11;fillColor=#ffe6cc;strokeColor=#d79b00;fontSize=7;" vertex="1" parent="1">
<mxGeometry x="100" y="110" width="80" height="130" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-43" value="Requests mixture chunks. Uses the chunks to orchestrate reads from the source data layer(s). Handles handover to the training/preprocessing layer." style="shape=note;whiteSpace=wrap;html=1;backgroundOutline=1;darkOpacity=0.05;size=11;fillColor=#ffe6cc;strokeColor=#d79b00;fontSize=7;" vertex="1" parent="1">
<mxGeometry x="559" y="270" width="80" height="90" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-44" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;endArrow=none;endFill=0;dashed=1;dashPattern=1 4;" edge="1" parent="1" source="Zkzzl6DhGFWdUMVNODvd-13" target="Zkzzl6DhGFWdUMVNODvd-43">
<mxGeometry relative="1" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-46" value="&lt;font style=&quot;font-size: 7px;&quot;&gt;Control Path&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=7;" vertex="1" parent="1">
<mxGeometry x="501" y="30" width="60" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-47" value="" style="endArrow=classic;html=1;rounded=0;entryX=1;entryY=1;entryDx=0;entryDy=0;exitX=0;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="Zkzzl6DhGFWdUMVNODvd-46" target="Zkzzl6DhGFWdUMVNODvd-46">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="430" y="150" as="sourcePoint" />
<mxPoint x="480" y="100" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-48" value="&lt;font style=&quot;font-size: 7px;&quot;&gt;Regular Data Path&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=7;" vertex="1" parent="1">
<mxGeometry x="501" y="60" width="59" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-49" value="" style="endArrow=classic;html=1;rounded=0;entryX=1;entryY=1;entryDx=0;entryDy=0;exitX=0;exitY=1;exitDx=0;exitDy=0;dashed=1;" edge="1" parent="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="501" y="89.91" as="sourcePoint" />
<mxPoint x="561" y="89.91" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-50" value="&lt;font style=&quot;font-size: 7px;&quot;&gt;Privileged Data Path&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=7;" vertex="1" parent="1">
<mxGeometry x="501" y="90" width="59" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-51" value="" style="endArrow=classic;html=1;rounded=0;entryX=1;entryY=1;entryDx=0;entryDy=0;exitX=0;exitY=1;exitDx=0;exitDy=0;dashed=1;dashPattern=8 8;" edge="1" parent="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="501" y="119.91" as="sourcePoint" />
<mxPoint x="561" y="119.91" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-52" value="&lt;font style=&quot;font-size: 7px;&quot;&gt;Comment&lt;/font&gt;" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=7;" vertex="1" parent="1">
<mxGeometry x="501" y="119.81" width="59" height="30" as="geometry" />
</mxCell>
<mxCell id="Zkzzl6DhGFWdUMVNODvd-53" value="" style="endArrow=classic;html=1;rounded=0;entryX=1;entryY=1;entryDx=0;entryDy=0;exitX=0;exitY=1;exitDx=0;exitDy=0;dashed=1;dashPattern=1 4;" edge="1" parent="1">
<mxGeometry width="50" height="50" relative="1" as="geometry">
<mxPoint x="501" y="149.72" as="sourcePoint" />
<mxPoint x="561" y="149.72" as="targetPoint" />
</mxGeometry>
</mxCell>
</root>
</mxGraphModel>
</diagram>
</mxfile>
Binary file added docs/source/_static/system_diagram/mixtera_system_diagram.drawio.pdf
View file
Open in desktop
Binary file not shown.
Binary file added docs/source/_static/system_diagram/mixtera_system_diagram.drawio.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.