Qwen3-Coder-480B

# InferenceClass for the H200 shape, validated serving Qwen3-Coder-480B
# multi-node on EKS. 8x NVIDIA H200 on an EKS p5en.48xlarge, with EFA.
#
# Both the GPU and the EFA fabric are claim: DRA devices. A multi-node gang's
# nodeSelector requests both, so the scheduler co-schedules the whole gang on a
# pool that has them and DRA binds 8 GPUs + 16 EFA interfaces per pod. The EFA
# device is installed by the EFA DRA driver (DRANET) in the serving stack.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceClass
metadata:
  name: eks-h200-8x-p5en
spec:
  description: "EKS p5en.48xlarge, 8x NVIDIA H200, EFA"
  provisioning:
    provider: EKS
    eks:
      instanceType: p5en.48xlarge
      diskSizeGb: 1024
      accelerator:
        type: nvidia-h200
        count: 8
  devices:
  - name: gpu
    claim: DRA
    driver: gpu.nvidia.com
    deviceClassName: gpu.nvidia.com
    count: 8
    attributes:
      architecture: { string: Hopper }
    capacity:
      memory: { value: "140Gi" }   # advertised below the ~141 GiB the driver reports
  - name: efa
    claim: DRA
    driver: dra.net
    deviceClassName: efa.networking.k8s.aws
    count: 16

# An EKS InferenceCluster with a two-node H200 pool over EFA, validated serving
# Qwen3-Coder-480B as a multi-node gang. The H200 nodes come from an EC2
# Capacity Block reserved for ML.
#
# fabric: EFA turns on Elastic Fabric Adapter for the gang's cross-node traffic;
# without it multi-node NCCL falls back to TCP, which is slow and unstable.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceCluster
metadata:
  name: eks-coder
  labels:
    modelplane.ai/region: us
spec:
  cluster:
    source: EKS
    eks:
      region: us-east-2
  nodePools:
  - name: gpu-h200
    className: eks-h200-8x-p5en
    nodeCount: 2
    minNodeCount: 2
    maxNodeCount: 2
    zones:
    - us-east-2b
    fabric: EFA
    capacityBlock:
      capacityReservationId: cr-0123456789abcdef0  # replace with your reservation ID

# The shared, read-write-many cache the multi-node gang serves from. Hydrated
# once per matched cluster from the gated Hugging Face repo; every gang pod
# mounts it at /mnt/models. ~960 GB of BF16 weights, so sizeGiB leaves headroom.
#
# The repo is gated, so it needs a Hugging Face token. Create the authSecret once
# in the ModelCache's namespace on the control plane; Modelplane propagates it to
# each matched cluster.
apiVersion: modelplane.ai/v1alpha1
kind: ModelCache
metadata:
  name: qwen3-coder
  namespace: ml-team
spec:
  source: HuggingFace
  huggingFace:
    repo: Qwen/Qwen3-Coder-480B-A35B-Instruct
    authSecret:
      name: hf-token
      key: HF_TOKEN
    sizeGiB: 1100

# Qwen3-Coder-480B served BF16 across two H200 nodes, validated end to end on
# EKS over EFA. A 480B MoE doesn't fit one node, so the engine is a Leader +
# Worker gang spanning two nodes via LeaderWorkerSet, both pods mounting the
# shared ModelCache at /mnt/models.
#
# Each member requests 8 GPUs + 16 EFA interfaces per node; the scheduler
# co-schedules the gang on the H200 pool. The worker joins the leader through
# $(MODELPLANE_LEADER_ADDRESS), which Modelplane injects.
#
# Notes on the engine flags:
#   --distributed-executor-backend=mp with --nnodes/--node-rank/--master-addr/
#     --headless is vLLM's native multiprocessing multi-node path.
#     vllm/vllm-openai:v0.23.0 no longer ships Ray, so the Ray-based
#     multi-node-serving.sh helper doesn't work on this image; the MP backend
#     needs nothing extra.
#   TP8 x PP2: tensor-parallel within a node over NVLink, pipeline-parallel
#     across the two nodes. tensor-parallel-size = GPUs per node,
#     pipeline-parallel-size = nodes.
#   --tool-call-parser=qwen3_xml is the parser for Qwen3-Coder specifically
#     (the dense Qwen3 models use hermes). The model is non-thinking, so there's
#     no reasoning parser.
#   --max-model-len=32768 caps context to fit; the native 256K isn't needed.
#   FI_PROVIDER=efa / NCCL_DEBUG=INFO point NCCL at the EFA fabric.
apiVersion: modelplane.ai/v1alpha1
kind: ModelDeployment
metadata:
  name: qwen3-coder
  namespace: ml-team
spec:
  replicas: 1
  clusterSelector:
    matchLabels:
      modelplane.ai/region: us
  modelCacheRef:
    name: qwen3-coder
  engines:
  - name: qwen3-coder
    members:
    - role: Leader
      nodeSelector:
        devices:
        - name: gpu
          count: 8
          selectors:
          - cel: |
              device.driver == "gpu.nvidia.com" && device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("120Gi")) >= 0
        - name: efa
          count: 16
          selectors:
          - cel: |
              device.driver == "dra.net"
      template:
        spec:
          containers:
          - name: engine
            image: vllm/vllm-openai:v0.23.0
            env:
            - name: NCCL_DEBUG
              value: "INFO"
            - name: FI_PROVIDER
              value: "efa"
            command:
            - /bin/sh
            - -c
            - >-
              exec vllm serve /mnt/models
              --served-model-name=qwen3-coder
              --tensor-parallel-size=8
              --pipeline-parallel-size=2
              --distributed-executor-backend=mp
              --nnodes=2 --node-rank=0
              --master-addr=$(MODELPLANE_LEADER_ADDRESS)
              --max-model-len=32768
              --gpu-memory-utilization=0.92
              --enable-auto-tool-choice
              --tool-call-parser=qwen3_xml
              --port=8000
    - role: Worker
      worker:
        nodes: 1
      nodeSelector:
        devices:
        - name: gpu
          count: 8
          selectors:
          - cel: |
              device.driver == "gpu.nvidia.com" && device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("120Gi")) >= 0
        - name: efa
          count: 16
          selectors:
          - cel: |
              device.driver == "dra.net"
      template:
        spec:
          containers:
          - name: engine
            image: vllm/vllm-openai:v0.23.0
            env:
            - name: NCCL_DEBUG
              value: "INFO"
            - name: FI_PROVIDER
              value: "efa"
            command:
            - /bin/sh
            - -c
            - >-
              exec vllm serve /mnt/models
              --served-model-name=qwen3-coder
              --tensor-parallel-size=8
              --pipeline-parallel-size=2
              --distributed-executor-backend=mp
              --nnodes=2 --node-rank=1
              --master-addr=$(MODELPLANE_LEADER_ADDRESS)
              --headless
              --max-model-len=32768
              --gpu-memory-utilization=0.92

# Exposes the multi-node BF16 qwen3-coder deployment as a single
# OpenAI-compatible URL. Read the public address from status.address:
#   kubectl get ms qwen3-coder -n ml-team -o jsonpath='{.status.address}'
apiVersion: modelplane.ai/v1alpha1
kind: ModelService
metadata:
  name: qwen3-coder
  namespace: ml-team
spec:
  endpoints:
  - selector:
      matchLabels:
        modelplane.ai/deployment: qwen3-coder