Poolside/Laguna-S-2.1

# An InferenceClass describing a Nebius gpu-h100-sxm node with 8x NVIDIA
# H100 80GB. Nebius sizes nodes by platform + preset rather than an
# instance type; gpu-h100-sxm + 8gpu-128vcpu-1600gb is one 8x H100 SXM
# node. The devices block describes the hardware DRA-style; it is what
# the Laguna ModelDeployment's nodeSelector matches on.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceClass
metadata:
  name: nebius-h100-8x
spec:
  description: "Nebius gpu-h100-sxm, 8x NVIDIA H100 80GB"
  provisioning:
    provider: Nebius
    nebius:
      platform: gpu-h100-sxm
      preset: 8gpu-128vcpu-1600gb
      diskSizeGb: 200
      driversPreset: cuda13.0
      accelerator:
        type: nvidia-h100
        count: 8
  devices:
  - name: gpu
    claim: DRA
    driver: gpu.nvidia.com
    deviceClassName: gpu.nvidia.com
    count: 8
    attributes:
      architecture: { string: Hopper }
      cudaComputeCapability: { version: "9.0.0" }
    capacity:
      # H100 80GB real usable VRAM (NVIDIA DRA driver), not nominal 80GB.
      memory: { value: "81559Mi" }

# An InferenceCluster backed by a Nebius mk8s cluster with a single 8x
# H100 GPU node group. Modelplane provisions the full mk8s cluster (VPC,
# control plane, system + GPU node groups) and installs the inference
# stack. ModelCache RWX storage is auto-provisioned via a Nebius shared
# filesystem, so model-cache.yaml works unchanged.
#
# Auth is the Nebius ClusterProviderConfig named default (service-account
# key + projectID); Modelplane reuses that identity to reach the cluster.
#
# eu-north1 (Finland) keeps inference in the EU for data residency.
#
# Clean teardown - delete the ML resources first (they hold a usage on the
# cluster), then the cluster:
#   kubectl delete modeldeployment,modelservice,modelcache laguna -n ml-team
#   kubectl delete inferencecluster nebius-eu-north --cascade=foreground
apiVersion: modelplane.ai/v1alpha1
kind: InferenceCluster
metadata:
  name: nebius-eu-north
  labels:
    modelplane.ai/region: eu-north
spec:
  cluster:
    source: Nebius
    nebius: {}

  nodePools:
  - name: gpu-h100
    className: nebius-h100-8x
    nodeCount: 1
    minNodeCount: 1
    maxNodeCount: 1

# The model weights, staged once per cluster on a Nebius shared filesystem
# (RWX) and mounted at /mnt/models in the serving pod, so the engine reads
# FP8 weights (~121 GiB) locally instead of pulling them from Hugging Face
# on every start.
#
# poolside/Laguna-S-2.1-FP8 is a public repo (OpenMDW-1.1), so no authSecret /
# HF token is needed. Add one only if you point this at a gated repo.
apiVersion: modelplane.ai/v1alpha1
kind: ModelCache
metadata:
  name: laguna
  namespace: ml-team
spec:
  source: HuggingFace
  huggingFace:
    repo: poolside/Laguna-S-2.1-FP8
    sizeGiB: 200

# Laguna-S-2.1 (118B total / 8B active MoE) served FP8 as a single
# Standalone vLLM engine on one 8x H100 node, weights streamed from the
# shared ModelCache at /mnt/models. FP8 weights (~121 GiB) fit one node
# with headroom for KV cache, so no gang / multi-node and no
# prefill/decode disaggregation are needed - just tensor parallelism
# across the 8 GPUs over NVLink.
#
# Notes on the engine flags:
#   --tensor-parallel-size=8 shards the model across the 8 H100s in the
#     node. FP8 is selected by the checkpoint (Laguna-S-2.1-FP8), not a flag.
#   --tool-call-parser=poolside_v1 and --reasoning-parser=poolside_v1 are
#     Poolside-specific and must exist in the engine build; --trust-remote-code
#     loads Laguna's custom modeling code from the cached repo.
#   --enable-auto-tool-choice turns on server-side tool selection.
#   --max-model-len can be lowered to shrink the KV cache footprint; 262144
#     matches Poolside's reference config.
#   The engine container must be named `engine` and listen on :8000 (Modelplane
#     scrapes 8000).
#
# H100-specific FP8 settings, carried over from a community 4x H100 build:
#   VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER=0 selects an FP8 GEMM path that is
#     correct on Hopper; --enforce-eager avoids a CUDA-graph capture OOM on
#     H100 80GB. Validate and relax these during the run if profiling allows.
apiVersion: modelplane.ai/v1alpha1
kind: ModelDeployment
metadata:
  name: laguna
  namespace: ml-team
spec:
  replicas: 1
  template:
    spec:
      modelCacheRef:
        name: laguna
      engines:
      - name: laguna
        members:
        - role: Standalone
          nodeSelector:
            devices:
            - name: gpu
              count: 8
              selectors:
              - cel: |
                  device.driver == "gpu.nvidia.com" && device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("79Gi")) >= 0
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.25.1
                command: ["vllm", "serve", "poolside/Laguna-S-2.1-FP8"]
                args:
                - --served-model-name=laguna
                - --tensor-parallel-size=8
                - --max-model-len=262144
                - --gpu-memory-utilization=0.9
                - --trust-remote-code
                - --enable-auto-tool-choice
                - --tool-call-parser=poolside_v1
                - --reasoning-parser=poolside_v1
                - --enforce-eager
                - --port=8000
                env:
                - name: VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER
                  value: "0"

# One OpenAI-compatible endpoint for the deployment. Read its public address:
#   kubectl get ms laguna -n ml-team -o jsonpath='{.status.address}'
# then call it, e.g.:
#   curl "$ADDR/v1/chat/completions" -H 'Content-Type: application/json' \
#     -d '{"model":"laguna","messages":[{"role":"user","content":"hello"}]}'
apiVersion: modelplane.ai/v1alpha1
kind: ModelService
metadata:
  name: laguna
  namespace: ml-team
spec:
  endpoints:
  - selector:
      matchLabels:
        modelplane.ai/deployment: laguna

# Laguna-S-2.1 FP8 served by SGLang instead of vLLM. SGLang has native
# LagunaForCausalLM support and its own poolside_v1 reasoning and tool-call
# parsers; the reasoning parser extracts the thinking block into a separate
# reasoning field. Same single-node 8x H100 shape as the vLLM variant
# (model-deployment.yaml): tensor-parallel across the 8 GPUs, weights streamed
# from the shared ModelCache at /mnt/models.
#
# FP8 is auto-detected from the checkpoint's quantization_config, so no quant
# flag or env is needed. Reasoning extraction is opt-in: the client sends
# enable_thinking=true (chat_template_kwargs), otherwise there is no thinking
# block to split.
#
# Image lmsysorg/sglang:v0.5.12.post1-cu129 - v0.5.12 is the first release with
# Poolside Laguna support (PR #24204, which also added the poolside_v1 parsers),
# .post1 is its point bugfix, cu129 for Hopper. The image does not auto-launch a
# server, so the launch command is set explicitly; SGLang defaults to port
# 30000, so --port=8000 is required for the Modelplane engine port (Modelplane
# scrapes 8000). Poolside's trtllm_mha and DFLASH cookbook optimizations are
# left off (not validated on H100).
apiVersion: modelplane.ai/v1alpha1
kind: ModelDeployment
metadata:
  name: laguna
  namespace: ml-team
spec:
  replicas: 1
  template:
    spec:
      modelCacheRef:
        name: laguna
      engines:
      - name: laguna
        members:
        - role: Standalone
          nodeSelector:
            devices:
            - name: gpu
              count: 8
              selectors:
              - cel: |
                  device.driver == "gpu.nvidia.com" && device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("79Gi")) >= 0
          template:
            spec:
              containers:
              - name: engine
                image: lmsysorg/sglang:v0.5.12.post1-cu129
                command: ["python3", "-m", "sglang.launch_server"]
                args:
                - --model-path=poolside/Laguna-S-2.1-FP8
                - --served-model-name=laguna
                - --tp-size=8
                - --context-length=262144
                - --trust-remote-code
                - --tool-call-parser=poolside_v1
                - --reasoning-parser=poolside_v1
                - --host=0.0.0.0
                - --port=8000