{ "type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [ { "id": "el1", "x": 150, "y": 8, "width": 501.59999999999997, "height": 27.5, "angle": 0, "strokeColor": "#1f2933", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1001, "version": 1, "versionNonce": 5001, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "Tensor Parallel (TP)：把单个算子切开（Megatron）", "originalText": "Tensor Parallel (TP)：把单个算子切开（Megatron）", "fontSize": 22, "fontFamily": 2, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 22 }, { "id": "el2", "x": 120, "y": 40, "width": 280.8, "height": 16.25, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1002, "version": 1, "versionNonce": 5002, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "一个大矩阵乘 Y = X·W，按列把 W 切到多卡，各算一片，再拼/规约", "originalText": "一个大矩阵乘 Y = X·W，按列把 W 切到多卡，各算一片，再拼/规约", "fontSize": 13, "fontFamily": 2, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 13 }, { "id": "el3", "x": 40, "y": 150, "width": 90, "height": 70, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1003, "version": 1, "versionNonce": 5003, "isDeleted": false, "boundElements": [ { "type": "text", "id": "el4" } ], "updated": 1700000000000, "link": null, "locked": false, "type": "rectangle" }, { "id": "el4", "x": 45, "y": 167.5, "width": 80, "height": 35.0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1004, "version": 1, "versionNonce": 5004, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "X\n(输入)", "originalText": "X\n(输入)", "fontSize": 14, "fontFamily": 2, "textAlign": "center", "verticalAlign": "middle", "containerId": "el3", "lineHeight": 1.25, "baseline": 14 }, { "id": "el5", "x": 180, "y": 96, "width": 620, "height": 30, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1005, "version": 1, "versionNonce": 5005, "isDeleted": false, "boundElements": [ { "type": "text", "id": "el6" } ], "updated": 1700000000000, "link": null, "locked": false, "type": "rectangle" }, { "id": "el6", "x": 185, "y": 103.5, "width": 610, "height": 15.0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1006, "version": 1, "versionNonce": 5006, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "X 每卡一份（相同输入，广播）", "originalText": "X 每卡一份（相同输入，广播）", "fontSize": 12, "fontFamily": 2, "textAlign": "center", "verticalAlign": "middle", "containerId": "el5", "lineHeight": 1.25, "baseline": 12 }, { "id": "el7", "x": 130, "y": 172, "width": 48, "height": 51, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1007, "version": 1, "versionNonce": 5007, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "arrow", "points": [ [ 0, 0 ], [ 48, -51 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow" }, { "id": "el8", "x": 180, "y": 150, "width": 140, "height": 90, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1008, "version": 1, "versionNonce": 5008, "isDeleted": false, "boundElements": [ { "type": "text", "id": "el9" } ], "updated": 1700000000000, "link": null, "locked": false, "type": "rectangle" }, { "id": "el9", "x": 185, "y": 172.5, "width": 130, "height": 45.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1009, "version": 1, "versionNonce": 5009, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "GPU0\nW_0 列切片\nY_0=X·W_0", "originalText": "GPU0\nW_0 列切片\nY_0=X·W_0", "fontSize": 12, "fontFamily": 2, "textAlign": "center", "verticalAlign": "middle", "containerId": "el8", "lineHeight": 1.25, "baseline": 12 }, { "id": "el10", "x": 250, "y": 126, "width": 0, "height": 24, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1010, "version": 1, "versionNonce": 5010, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "arrow", "points": [ [ 0, 0 ], [ 0, 24 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow" }, { "id": "el11", "x": 340, "y": 150, "width": 140, "height": 90, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1011, "version": 1, "versionNonce": 5011, "isDeleted": false, "boundElements": [ { "type": "text", "id": "el12" } ], "updated": 1700000000000, "link": null, "locked": false, "type": "rectangle" }, { "id": "el12", "x": 345, "y": 172.5, "width": 130, "height": 45.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1012, "version": 1, "versionNonce": 5012, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "GPU1\nW_1 列切片\nY_1=X·W_1", "originalText": "GPU1\nW_1 列切片\nY_1=X·W_1", "fontSize": 12, "fontFamily": 2, "textAlign": "center", "verticalAlign": "middle", "containerId": "el11", "lineHeight": 1.25, "baseline": 12 }, { "id": "el13", "x": 410, "y": 126, "width": 0, "height": 24, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1013, "version": 1, "versionNonce": 5013, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "arrow", "points": [ [ 0, 0 ], [ 0, 24 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow" }, { "id": "el14", "x": 500, "y": 150, "width": 140, "height": 90, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1014, "version": 1, "versionNonce": 5014, "isDeleted": false, "boundElements": [ { "type": "text", "id": "el15" } ], "updated": 1700000000000, "link": null, "locked": false, "type": "rectangle" }, { "id": "el15", "x": 505, "y": 172.5, "width": 130, "height": 45.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1015, "version": 1, "versionNonce": 5015, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "GPU2\nW_2 列切片\nY_2=X·W_2", "originalText": "GPU2\nW_2 列切片\nY_2=X·W_2", "fontSize": 12, "fontFamily": 2, "textAlign": "center", "verticalAlign": "middle", "containerId": "el14", "lineHeight": 1.25, "baseline": 12 }, { "id": "el16", "x": 570, "y": 126, "width": 0, "height": 24, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1016, "version": 1, "versionNonce": 5016, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "arrow", "points": [ [ 0, 0 ], [ 0, 24 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow" }, { "id": "el17", "x": 660, "y": 150, "width": 140, "height": 90, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1017, "version": 1, "versionNonce": 5017, "isDeleted": false, "boundElements": [ { "type": "text", "id": "el18" } ], "updated": 1700000000000, "link": null, "locked": false, "type": "rectangle" }, { "id": "el18", "x": 665, "y": 172.5, "width": 130, "height": 45.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1018, "version": 1, "versionNonce": 5018, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "GPU3\nW_3 列切片\nY_3=X·W_3", "originalText": "GPU3\nW_3 列切片\nY_3=X·W_3", "fontSize": 12, "fontFamily": 2, "textAlign": "center", "verticalAlign": "middle", "containerId": "el17", "lineHeight": 1.25, "baseline": 12 }, { "id": "el19", "x": 730, "y": 126, "width": 0, "height": 24, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1019, "version": 1, "versionNonce": 5019, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "arrow", "points": [ [ 0, 0 ], [ 0, 24 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow" }, { "id": "el20", "x": 180, "y": 280, "width": 620, "height": 46, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1020, "version": 1, "versionNonce": 5020, "isDeleted": false, "boundElements": [ { "type": "text", "id": "el21" } ], "updated": 1700000000000, "link": null, "locked": false, "type": "rectangle" }, { "id": "el21", "x": 185, "y": 294.875, "width": 610, "height": 16.25, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1021, "version": 1, "versionNonce": 5021, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "All-Gather → 拼出完整 Y （行并行变体则用 All-Reduce）", "originalText": "All-Gather → 拼出完整 Y （行并行变体则用 All-Reduce）", "fontSize": 13, "fontFamily": 2, "textAlign": "center", "verticalAlign": "middle", "containerId": "el20", "lineHeight": 1.25, "baseline": 13 }, { "id": "el22", "x": 250, "y": 240, "width": 0, "height": 40, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1022, "version": 1, "versionNonce": 5022, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "arrow", "points": [ [ 0, 0 ], [ 0, 40 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow" }, { "id": "el23", "x": 410, "y": 240, "width": 0, "height": 40, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1023, "version": 1, "versionNonce": 5023, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "arrow", "points": [ [ 0, 0 ], [ 0, 40 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow" }, { "id": "el24", "x": 570, "y": 240, "width": 0, "height": 40, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1024, "version": 1, "versionNonce": 5024, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "arrow", "points": [ [ 0, 0 ], [ 0, 40 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow" }, { "id": "el25", "x": 730, "y": 240, "width": 0, "height": 40, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1025, "version": 1, "versionNonce": 5025, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "arrow", "points": [ [ 0, 0 ], [ 0, 40 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow" }, { "id": "el26", "x": 40, "y": 344, "width": 403.2, "height": 15.0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1026, "version": 1, "versionNonce": 5026, "isDeleted": false, "boundElements": [], "updated": 1700000000000, "link": null, "locked": false, "type": "text", "text": "前向 1 次、反向 1 次集合通信，很密集 → 一般只在节点内 NVLink；column→row 串联可省一次", "originalText": "前向 1 次、反向 1 次集合通信，很密集 → 一般只在节点内 NVLink；column→row 串联可省一次", "fontSize": 12, "fontFamily": 2, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12 } ], "appState": { "gridSize": null, "viewBackgroundColor": "#ffffff" }, "files": {} }