{
  "type": "excalidraw",
  "version": 2,
  "source": "https://excalidraw.com",
  "elements": [
    {
      "id": "el1",
      "x": 150,
      "y": 8,
      "width": 514.8,
      "height": 27.5,
      "angle": 0,
      "strokeColor": "#1f2933",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1001,
      "version": 1,
      "versionNonce": 5001,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "Sequence Parallel (SP)：在非 TP 区按序列切（省激活）",
      "originalText": "Sequence Parallel (SP)：在非 TP 区按序列切（省激活）",
      "fontSize": 22,
      "fontFamily": 2,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "lineHeight": 1.25,
      "baseline": 22
    },
    {
      "id": "el2",
      "x": 110,
      "y": 40,
      "width": 397.8,
      "height": 16.25,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1002,
      "version": 1,
      "versionNonce": 5002,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "与 TP 配套：LayerNorm/Dropout 区按序列切，Attention/MLP 区用 TP",
      "originalText": "与 TP 配套：LayerNorm/Dropout 区按序列切，Attention/MLP 区用 TP",
      "fontSize": 13,
      "fontFamily": 2,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "lineHeight": 1.25,
      "baseline": 13
    },
    {
      "id": "el3",
      "x": 44,
      "y": 72,
      "width": 265.2,
      "height": 16.25,
      "angle": 0,
      "strokeColor": "#b8860b",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1003,
      "version": 1,
      "versionNonce": 5003,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "LayerNorm / Dropout 区：按序列切（每卡一段激活）",
      "originalText": "LayerNorm / Dropout 区：按序列切（每卡一段激活）",
      "fontSize": 13,
      "fontFamily": 2,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "lineHeight": 1.25,
      "baseline": 13
    },
    {
      "id": "el4",
      "x": 40,
      "y": 92,
      "width": 910,
      "height": 52,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "#fff9db",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": {
        "type": 3
      },
      "seed": 1004,
      "version": 1,
      "versionNonce": 5004,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "rectangle"
    },
    {
      "id": "el5",
      "x": 40,
      "y": 100,
      "width": 170,
      "height": 36,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "#ffec99",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": {
        "type": 3
      },
      "seed": 1005,
      "version": 1,
      "versionNonce": 5005,
      "isDeleted": false,
      "boundElements": [
        {
          "type": "text",
          "id": "el6"
        }
      ],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "rectangle"
    },
    {
      "id": "el6",
      "x": 45,
      "y": 110.5,
      "width": 160,
      "height": 15.0,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1006,
      "version": 1,
      "versionNonce": 5006,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "GPU0: seq 片 0",
      "originalText": "GPU0: seq 片 0",
      "fontSize": 12,
      "fontFamily": 2,
      "textAlign": "center",
      "verticalAlign": "middle",
      "containerId": "el5",
      "lineHeight": 1.25,
      "baseline": 12
    },
    {
      "id": "el7",
      "x": 235,
      "y": 100,
      "width": 170,
      "height": 36,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "#ffec99",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": {
        "type": 3
      },
      "seed": 1007,
      "version": 1,
      "versionNonce": 5007,
      "isDeleted": false,
      "boundElements": [
        {
          "type": "text",
          "id": "el8"
        }
      ],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "rectangle"
    },
    {
      "id": "el8",
      "x": 240,
      "y": 110.5,
      "width": 160,
      "height": 15.0,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1008,
      "version": 1,
      "versionNonce": 5008,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "GPU1: seq 片 1",
      "originalText": "GPU1: seq 片 1",
      "fontSize": 12,
      "fontFamily": 2,
      "textAlign": "center",
      "verticalAlign": "middle",
      "containerId": "el7",
      "lineHeight": 1.25,
      "baseline": 12
    },
    {
      "id": "el9",
      "x": 430,
      "y": 100,
      "width": 170,
      "height": 36,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "#ffec99",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": {
        "type": 3
      },
      "seed": 1009,
      "version": 1,
      "versionNonce": 5009,
      "isDeleted": false,
      "boundElements": [
        {
          "type": "text",
          "id": "el10"
        }
      ],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "rectangle"
    },
    {
      "id": "el10",
      "x": 435,
      "y": 110.5,
      "width": 160,
      "height": 15.0,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1010,
      "version": 1,
      "versionNonce": 5010,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "GPU2: seq 片 2",
      "originalText": "GPU2: seq 片 2",
      "fontSize": 12,
      "fontFamily": 2,
      "textAlign": "center",
      "verticalAlign": "middle",
      "containerId": "el9",
      "lineHeight": 1.25,
      "baseline": 12
    },
    {
      "id": "el11",
      "x": 625,
      "y": 100,
      "width": 170,
      "height": 36,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "#ffec99",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": {
        "type": 3
      },
      "seed": 1011,
      "version": 1,
      "versionNonce": 5011,
      "isDeleted": false,
      "boundElements": [
        {
          "type": "text",
          "id": "el12"
        }
      ],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "rectangle"
    },
    {
      "id": "el12",
      "x": 630,
      "y": 110.5,
      "width": 160,
      "height": 15.0,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1012,
      "version": 1,
      "versionNonce": 5012,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "GPU3: seq 片 3",
      "originalText": "GPU3: seq 片 3",
      "fontSize": 12,
      "fontFamily": 2,
      "textAlign": "center",
      "verticalAlign": "middle",
      "containerId": "el11",
      "lineHeight": 1.25,
      "baseline": 12
    },
    {
      "id": "el13",
      "x": 495.0,
      "y": 150,
      "width": 0.0,
      "height": 30,
      "angle": 0,
      "strokeColor": "#e8590c",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": {
        "type": 2
      },
      "seed": 1013,
      "version": 1,
      "versionNonce": 5013,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "arrow",
      "points": [
        [
          0,
          0
        ],
        [
          0.0,
          30
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow"
    },
    {
      "id": "el14",
      "x": 505.0,
      "y": 156,
      "width": 138.6,
      "height": 13.75,
      "angle": 0,
      "strokeColor": "#e8590c",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1014,
      "version": 1,
      "versionNonce": 5014,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "All-Gather（进 TP，序列拼回）",
      "originalText": "All-Gather（进 TP，序列拼回）",
      "fontSize": 11,
      "fontFamily": 2,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "lineHeight": 1.25,
      "baseline": 11
    },
    {
      "id": "el15",
      "x": 40,
      "y": 190,
      "width": 910,
      "height": 56,
      "angle": 0,
      "strokeColor": "#1971c2",
      "backgroundColor": "#a5d8ff",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": {
        "type": 3
      },
      "seed": 1015,
      "version": 1,
      "versionNonce": 5015,
      "isDeleted": false,
      "boundElements": [
        {
          "type": "text",
          "id": "el16"
        }
      ],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "rectangle"
    },
    {
      "id": "el16",
      "x": 45,
      "y": 209.875,
      "width": 900,
      "height": 16.25,
      "angle": 0,
      "strokeColor": "#1971c2",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1016,
      "version": 1,
      "versionNonce": 5016,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "Attention / MLP 区：按 TP 切（各卡持权重分片）",
      "originalText": "Attention / MLP 区：按 TP 切（各卡持权重分片）",
      "fontSize": 13,
      "fontFamily": 2,
      "textAlign": "center",
      "verticalAlign": "middle",
      "containerId": "el15",
      "lineHeight": 1.25,
      "baseline": 13
    },
    {
      "id": "el17",
      "x": 495.0,
      "y": 250,
      "width": 0.0,
      "height": 30,
      "angle": 0,
      "strokeColor": "#e8590c",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": {
        "type": 2
      },
      "seed": 1017,
      "version": 1,
      "versionNonce": 5017,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "arrow",
      "points": [
        [
          0,
          0
        ],
        [
          0.0,
          30
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow"
    },
    {
      "id": "el18",
      "x": 505.0,
      "y": 256,
      "width": 178.2,
      "height": 13.75,
      "angle": 0,
      "strokeColor": "#e8590c",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1018,
      "version": 1,
      "versionNonce": 5018,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "Reduce-Scatter（出 TP，回到按序列切）",
      "originalText": "Reduce-Scatter（出 TP，回到按序列切）",
      "fontSize": 11,
      "fontFamily": 2,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "lineHeight": 1.25,
      "baseline": 11
    },
    {
      "id": "el19",
      "x": 40,
      "y": 290,
      "width": 910,
      "height": 40,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "#ffec99",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": {
        "type": 3
      },
      "seed": 1019,
      "version": 1,
      "versionNonce": 5019,
      "isDeleted": false,
      "boundElements": [
        {
          "type": "text",
          "id": "el20"
        }
      ],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "rectangle"
    },
    {
      "id": "el20",
      "x": 45,
      "y": 301.875,
      "width": 900,
      "height": 16.25,
      "angle": 0,
      "strokeColor": "#f08c00",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1020,
      "version": 1,
      "versionNonce": 5020,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "下一段 LayerNorm / Dropout：又按序列切",
      "originalText": "下一段 LayerNorm / Dropout：又按序列切",
      "fontSize": 13,
      "fontFamily": 2,
      "textAlign": "center",
      "verticalAlign": "middle",
      "containerId": "el19",
      "lineHeight": 1.25,
      "baseline": 13
    },
    {
      "id": "el21",
      "x": 40,
      "y": 345,
      "width": 460.79999999999995,
      "height": 15.0,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "roundness": null,
      "seed": 1021,
      "version": 1,
      "versionNonce": 5021,
      "isDeleted": false,
      "boundElements": [],
      "updated": 1700000000000,
      "link": null,
      "locked": false,
      "type": "text",
      "text": "通信总量与纯 TP 的 all-reduce 等价（拆成 all-gather+reduce-scatter），但显著降激活显存",
      "originalText": "通信总量与纯 TP 的 all-reduce 等价（拆成 all-gather+reduce-scatter），但显著降激活显存",
      "fontSize": 12,
      "fontFamily": 2,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "lineHeight": 1.25,
      "baseline": 12
    }
  ],
  "appState": {
    "gridSize": null,
    "viewBackgroundColor": "#ffffff"
  },
  "files": {}
}