coding-stack-qwen3.8-27b-huihui/docker-compose.yaml

services:
  # vLLM server - serves the Huihui Qwen3.8-27B AWQ W4A16 checkpoint as "foundation"
  vllm:
    # Moving target: re-pull deliberately. 0.27.2rc1 (nightly) is the tested build;
    # v0.18.0 (also cached locally) predates qwen3_5 support.
    image: vllm/vllm-openai:nightly
    container_name: coding-stack-vllm
    restart: unless-stopped
    volumes:
      # HF cache root: /storage-models/vllm/hub/models--shawnw3i--…
      - /storage-models/vllm:/models
    environment:
      - HF_HOME=/models
      - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1
      - VLLM_NO_USAGE_STATS=1
      - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:512
      - VLLM_FLOAT32_MATMUL_PRECISION=high
      - VLLM_USE_FLASHINFER_SAMPLER=1
      - OMP_NUM_THREADS=1
      - CUDA_DEVICE_MAX_CONNECTIONS=8
      - VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
      - TRITON_CACHE_DIR=/root/.triton/cache
      - VLLM_WORKER_MULTIPROC_METHOD=spawn
      - VLLM_MARLIN_USE_ATOMIC_ADD=1
      - VLLM_LOGGING_LEVEL=INFO
    shm_size: 8g
    ports:
      - "18080:8000"
    entrypoint: ["python3", "-m", "vllm.entrypoints.openai.api_server"]
    command:
      - --model
      - shawnw3i/Huihui-Qwen3.8-27B-abliterated-AWQ-MTP
      - --port
      - "8000"
      - --served-model-name
      - foundation
      # ~248k tokens: 20 GB AWQ weights + ~15.5 GiB f16 KV + overhead on 48 GB.
      - --max-model-len
      - "253952"
      - --reasoning-parser
      - qwen3
      - --tool-call-parser
      - qwen3_coder
      - --dtype
      - float16
      # Vision tower skipped. Drop this and lower --max-model-len to ~200000
      # if you ever want images through vLLM.
      - --language-model-only
      - --gpu-memory-utilization
      - "0.93"
      # One 250k-token sequence needs the whole KV pool; more seqs would starve it.
      - --max-num-seqs
      - "1"
      - --max-num-batched-tokens
      - "16384"
      - --enable-chunked-prefill
      - --enable-prefix-caching
      - --enable-auto-tool-choice
      - --trust-remote-code
      - --host
      - 0.0.0.0
      - --disable-log-stats
    networks:
      - coding-stack-network
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 900s
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

  # OpenAI-compatible facade on :11433 that injects a SearXNG web_search tool
  proxy:
    image: python:3.11-slim
    container_name: coding-stack-proxy
    restart: unless-stopped
    ports:
      - "11433:11536"
    volumes:
      - ./proxy-passthrough.py:/app/proxy.py:ro
    environment:
      - BACKEND_URL=http://vllm:8000
      - SEARXNG_URL=http://host.docker.internal:8889
      - SEARXNG_RESULTS=5
      # vLLM serves without auth unless --api-key is added above
      # - BACKEND_API_KEY=
    extra_hosts:
      - "host.docker.internal:host-gateway"
    command: sh -c "pip install -q flask requests && python /app/proxy.py"
    networks:
      - coding-stack-network
    depends_on:
      - vllm

networks:
  coding-stack-network:
    driver: bridge