diff --git a/.agents/adding-backends.md b/.agents/adding-backends.md index a7880a17b400..f0f44b815e12 100644 --- a/.agents/adding-backends.md +++ b/.agents/adding-backends.md @@ -92,7 +92,7 @@ If you have a `prepare.sh` doing the clone, delete it — the recipe belongs in - CUDA 13 builds: Add after other CUDA 13 builds (e.g., after `gpu-nvidia-cuda-13-chatterbox`) **Additional build types you may need:** -- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.2.1"` +- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.14.0-full"` - Intel/SYCL: Use `build-type: 'intel'` or `build-type: 'sycl_f16'`/`sycl_f32` with `base-image: "intel/oneapi-basekit:2025.3.2-0-devel-ubuntu24.04"` - L4T (ARM): Use `build-type: 'l4t'` with `platforms: 'linux/arm64'` and `runs-on: 'ubuntu-24.04-arm'` diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 021d555ec993..a98eeeb7a906 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -11,7 +11,7 @@ Let's say the user wants to build a particular backend for a given platform. For - Use `.github/backend-matrix.yml` as a reference — it's the data-only YAML that lists every backend variant's `build-type`, `base-image`, `platforms`, etc. (`backend.yml` and `backend_pr.yml` consume it via `scripts/changed-backends.js`). - l4t and cublas also require the CUDA major and minor version. - For llama-cpp / ik-llama-cpp / turboquant the matrix also sets `builder-base-image` pointing at a prebuilt `quay.io/go-skynet/ci-cache:base-grpc-*` tag. Local `make backends/` defaults to `BUILDER_TARGET=builder-fromsource` and doesn't need it — the Dockerfile's from-source stage installs everything itself. -- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.2.1 make docker-build-coqui` +- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.14.0-full make docker-build-coqui` - Unless the user specifies that they want you to run the command, then just print it because not all agent frontends handle long running jobs well and the output may overflow your context - The user may say they want to build AMD or ROCM instead of hipblas, or Intel instead of SYCL or NVIDIA insted of l4t or cublas. Ask for confirmation if there is ambiguity. - Sometimes the user may need extra parameters to be added to `docker build` (e.g. `--platform` for cross-platform builds or `--progress` to view the full logs), in which case you can generate the `docker build` command directly. diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 6742049e68ff..4cbf3ea73d21 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -51,7 +51,7 @@ The C++ backend Dockerfiles (`Dockerfile.{llama-cpp,ik-llama-cpp,turboquant}`) c | `base-grpc-cuda-13-amd64` | the above + CUDA 13.0 toolkit (Ubuntu 22.04 base) | | `base-grpc-cuda-13-arm64` | the above + CUDA 13.0 sbsa toolkit (Ubuntu 24.04 base) | | `base-grpc-l4t-cuda-12-arm64` | JetPack r36.4.0 base (CUDA preinstalled, `SKIP_DRIVERS=true`) + gRPC | -| `base-grpc-rocm-amd64` | rocm/dev-ubuntu-24.04:7.2.1 base + hipblas/hipblaslt/rocblas + gRPC | +| `base-grpc-rocm-amd64` | rocm/dev-ubuntu-24.04:7.14.0-full base + hipblas/hipblaslt/rocblas + gRPC | | `base-grpc-vulkan-amd64` / `base-grpc-vulkan-arm64` | Ubuntu 24.04 + Vulkan SDK 1.4.335 + gRPC | | `base-grpc-intel-amd64` | intel/oneapi-basekit:2025.3.2 base + gRPC | diff --git a/.docker/install-base-deps.sh b/.docker/install-base-deps.sh index 4331921ca6e3..430085e90536 100755 --- a/.docker/install-base-deps.sh +++ b/.docker/install-base-deps.sh @@ -226,16 +226,18 @@ fi # --- 6. ROCm / HIP build deps (BUILD_TYPE=hipblas) --- if [ "${BUILD_TYPE:-}" = "hipblas" ] && [ "${SKIP_DRIVERS:-false}" = "false" ]; then - apt-get update - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev - apt-get clean - rm -rf /var/lib/apt/lists/* - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, - # which results in local-ai and others not being able to locate the libraries. - # We run ldconfig ourselves to work around this packaging deficiency. + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed — BLAS is consolidated and split per + # GPU arch (amdrocm-blas-gfx*, amdrocm-blas-dev, amdrocm-blas-host). The + # rocm/dev-ubuntu-*:*-full base image already ships those BLAS dev libs and + # headers, so no apt install is needed here. + # TheRock ships the ROCm libs under /opt/rocm/core*/lib (reached via the + # /opt/rocm/lib alternatives symlink) but registers no ld.so.conf.d entry, so + # the dynamic linker can't resolve them. Without this the ldd-based GPU-lib + # packaging (scripts/build/package-gpu-libs.sh) silently skips librocm_kpack, + # rocBLAS, hipBLASLt, the bundled rocm_sysdeps (zlib/zstd/elf) and the LLVM + # runtime, and the built backend fails to load at runtime. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf ldconfig # Log which GPU architectures have rocBLAS kernel support echo "rocBLAS library data architectures:" diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 53cb07d8dd90..efbf1d570e5e 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -2288,7 +2288,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-rerankers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "rerankers" dockerfile: "./backend/Dockerfile.python" @@ -2302,7 +2302,7 @@ include: tag-suffix: '-gpu-rocm-hipblas-llama-cpp' builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "llama-cpp" dockerfile: "./backend/Dockerfile.llama-cpp" @@ -2316,7 +2316,7 @@ include: tag-suffix: '-gpu-rocm-hipblas-bonsai' builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "bonsai" dockerfile: "./backend/Dockerfile.bonsai" @@ -2329,9 +2329,32 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vllm' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" + skip-drivers: 'false' + backend: "vllm" + # CDNA data-center arches -- the only ones wheels.vllm.ai/rocm publishes a + # wheel for. Pinned rather than inherited so this image keeps installing that + # prebuilt wheel; the repo-wide default list contains consumer arches, which + # would silently turn this into a multi-hour source build. + amdgpu-targets: 'gfx942,gfx950' + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' + # Consumer/RDNA AMD GPUs. No prebuilt vllm wheel exists for these arches, so + # install.sh builds vllm from source -- a separate image because the two install + # paths cannot share one venv. One arch per image: each AMD torch device package + # is ~1.6 GB installed, so bundling arches is not free. + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-gfx1151-vllm' + runs-on: 'ubuntu-latest' + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vllm" + amdgpu-targets: 'gfx1151' dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' @@ -2342,7 +2365,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vllm-omni' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vllm-omni" dockerfile: "./backend/Dockerfile.python" @@ -2355,7 +2378,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-sglang' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "sglang" dockerfile: "./backend/Dockerfile.python" @@ -2368,9 +2391,14 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-transformers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "transformers" + # AMD torch device packages are per-GPU-arch and ~1.6 GB installed each, so + # this image carries a deliberate set instead of the repo-wide 11-arch default + # (which would add ~17 GB): both CDNA data-center arches plus Strix Halo. + # Adding an arch is one entry here at ~1.6 GB. + amdgpu-targets: 'gfx942,gfx950,gfx1151' dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' @@ -2381,9 +2409,14 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-diffusers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "diffusers" + # AMD torch device packages are per-GPU-arch and ~1.6 GB installed each, so + # this image carries a deliberate set instead of the repo-wide 11-arch default + # (which would add ~17 GB): both CDNA data-center arches plus Strix Halo. + # Adding an arch is one entry here at ~1.6 GB. + amdgpu-targets: 'gfx942,gfx950,gfx1151' dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' @@ -2394,7 +2427,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-ace-step' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "ace-step" dockerfile: "./backend/Dockerfile.python" @@ -2408,9 +2441,14 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-kokoro' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "kokoro" + # AMD torch device packages are per-GPU-arch and ~1.6 GB installed each, so + # this image carries a deliberate set instead of the repo-wide 11-arch default + # (which would add ~17 GB): both CDNA data-center arches plus Strix Halo. + # Adding an arch is one entry here at ~1.6 GB. + amdgpu-targets: 'gfx942,gfx950,gfx1151' dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' @@ -2421,7 +2459,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vibevoice' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vibevoice" dockerfile: "./backend/Dockerfile.python" @@ -2434,7 +2472,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-liquid-audio' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "liquid-audio" dockerfile: "./backend/Dockerfile.python" @@ -2447,7 +2485,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-qwen-asr' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "qwen-asr" dockerfile: "./backend/Dockerfile.python" @@ -2473,7 +2511,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-nemo' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "nemo" dockerfile: "./backend/Dockerfile.python" @@ -2486,7 +2524,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-qwen-tts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "qwen-tts" dockerfile: "./backend/Dockerfile.python" @@ -2499,7 +2537,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-fish-speech' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "fish-speech" dockerfile: "./backend/Dockerfile.python" @@ -2512,7 +2550,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-voxcpm' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "voxcpm" dockerfile: "./backend/Dockerfile.python" @@ -2525,7 +2563,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-pocket-tts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "pocket-tts" dockerfile: "./backend/Dockerfile.python" @@ -2538,7 +2576,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-faster-whisper' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "faster-whisper" dockerfile: "./backend/Dockerfile.python" @@ -2551,7 +2589,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-coqui' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "coqui" dockerfile: "./backend/Dockerfile.python" @@ -4229,7 +4267,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-whisper' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "whisper" @@ -4242,7 +4280,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-crispasr' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "crispasr" @@ -4351,7 +4389,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-parakeet-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "parakeet-cpp" @@ -4540,7 +4578,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-moss-transcribe-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "moss-transcribe-cpp" @@ -4688,7 +4726,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-ced' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "ced" @@ -4836,7 +4874,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-voice-detect' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "voice-detect" @@ -4984,7 +5022,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-face-detect' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "face-detect" @@ -5093,7 +5131,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-acestep-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "acestep-cpp" @@ -6055,7 +6093,7 @@ include: # platforms: 'linux/amd64' # tag-latest: 'auto' # tag-suffix: '-gpu-hipblas-rfdetr' - # base-image: "rocm/dev-ubuntu-24.04:7.2.1" + # base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" # runs-on: 'ubuntu-latest' # skip-drivers: 'false' # backend: "rfdetr" @@ -6127,7 +6165,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-neutts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "neutts" dockerfile: "./backend/Dockerfile.python" diff --git a/.github/workflows/base-images.yml b/.github/workflows/base-images.yml index 637b603a56fb..87ce9c3fded7 100644 --- a/.github/workflows/base-images.yml +++ b/.github/workflows/base-images.yml @@ -83,7 +83,7 @@ jobs: ubuntu-version: '2404' - tag: 'base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: 'rocm/dev-ubuntu-24.04:7.2.1' + base-image: 'rocm/dev-ubuntu-24.04:7.14.0-full' build-type: 'hipblas' cuda-major-version: '' cuda-minor-version: '' diff --git a/.github/workflows/image-pr.yml b/.github/workflows/image-pr.yml index 2605c2541252..76b720c905f1 100644 --- a/.github/workflows/image-pr.yml +++ b/.github/workflows/image-pr.yml @@ -80,7 +80,7 @@ platforms: 'linux/amd64' tag-latest: 'false' tag-suffix: '-hipblas' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' makeflags: "--jobs=3 --output-sync=target" ubuntu-version: '2404' diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index cd90e849b9c2..28a5ff8a6cf5 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -85,7 +85,7 @@ platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-hipblas' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' makeflags: "--jobs=3 --output-sync=target" ubuntu-version: '2404' diff --git a/Dockerfile b/Dockerfile index 4ca9b32791ab..cf9c48503e88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -157,17 +157,22 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ rm -rf /var/lib/apt/lists/* \ ; fi -RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed — BLAS is consolidated and split + # per GPU arch (amdrocm-blas-gfx*, amdrocm-blas-dev, amdrocm-blas-host). + # The rocm/dev-ubuntu-*:*-full base image already ships those dev libraries + # and headers, so no apt install is needed here anymore. We still set the + # AMD capability marker. echo "amd" > /run/localai/capability && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # TheRock ships the ROCm libs under /opt/rocm/core*/lib (reached via the + # /opt/rocm/lib alternatives symlink) but registers no ld.so.conf.d entry, + # so the dynamic linker can't find them. Register the path so ldconfig + # (and the ldd-based backend GPU-lib packaging) resolves librocm_kpack, + # rocBLAS, hipBLASLt, the bundled rocm_sysdeps (zlib/zstd/elf) and the + # LLVM runtime — otherwise the built backend fails at runtime with e.g. + # "librocm_kpack.so.0 / librocm_sysdeps_z.so.1: cannot open shared object". + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/Dockerfile.base-grpc-builder b/backend/Dockerfile.base-grpc-builder index 8843a0a72c29..c7c681dde82b 100644 --- a/backend/Dockerfile.base-grpc-builder +++ b/backend/Dockerfile.base-grpc-builder @@ -26,7 +26,7 @@ # base-grpc-cuda-13-amd64 ubuntu:22.04 + CUDA 13.0 # base-grpc-cuda-13-arm64 ubuntu:24.04 + CUDA 13.0 (sbsa) # base-grpc-l4t-cuda-12-arm64 ubuntu:22.04 + CUDA 12.x (legacy JetPack) -# base-grpc-rocm-amd64 rocm/dev-ubuntu-24.04:7.2.1 + hipblas +# base-grpc-rocm-amd64 rocm/dev-ubuntu-24.04:7.14.0-full + hipblas # base-grpc-vulkan-amd64 ubuntu:24.04 + Vulkan SDK 1.4.335 # base-grpc-vulkan-arm64 ubuntu:24.04 + Vulkan SDK ARM 1.4.335 # base-grpc-intel-amd64 intel/oneapi-basekit:2025.3.2 (sycl) diff --git a/backend/Dockerfile.golang b/backend/Dockerfile.golang index e5ab48957883..58fc6a403aba 100644 --- a/backend/Dockerfile.golang +++ b/backend/Dockerfile.golang @@ -168,16 +168,19 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ rm -rf /var/lib/apt/lists/* \ ; fi -RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ + # ROCm 7.x ("TheRock" packaging) consolidates the legacy BLAS development + # packages into amdrocm-blas-dev. Some Go/C++ matrix entries still use ROCm + # 6.x, so select the package set exposed by the base image's repository. apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ + if apt-cache show amdrocm-blas-dev >/dev/null 2>&1; then \ + apt-get install -y --no-install-recommends amdrocm-blas-dev; \ + else \ + apt-get install -y --no-install-recommends hipblas-dev hipblaslt-dev rocblas-dev; \ + fi && \ rm -rf /var/lib/apt/lists/* && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # Register TheRock's library paths so cgo backends resolve the ROCm libs. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 2522a6f56be0..5380bd7777e6 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -6,6 +6,11 @@ FROM ${BASE_IMAGE} AS builder ARG BACKEND=rerankers ARG BUILD_TYPE ENV BUILD_TYPE=${BUILD_TYPE} +# AMD GPU target arch(es) for ROCm builds (e.g. gfx1151, or a ';'-list). Exposed +# to install.sh so ROCm backends can select AMD's per-GPU torch and target the +# right arch when building from source, instead of hardcoding one. +ARG AMDGPU_TARGETS +ENV AMDGPU_TARGETS=${AMDGPU_TARGETS} ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION ARG SKIP_DRIVERS=false @@ -170,16 +175,15 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ rm -rf /var/lib/apt/lists/* \ ; fi -RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency +RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed (consolidated + arch-split). The + # rocm/dev-ubuntu-*:*-full base image already ships the BLAS dev libs and + # headers, so no apt install is needed. TheRock also scatters the ROCm libs + # across /opt/rocm/lib, /opt/rocm/lib/rocm_sysdeps/lib and /opt/rocm/llvm/lib + # with no ld.so.conf.d entry — register them so ldconfig (and Python + # backends that dlopen ROCm libs at runtime) can resolve them. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/index.yaml b/backend/index.yaml index 5c6468f32f8a..db52309854a5 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -4199,6 +4199,16 @@ uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-vllm" mirrors: - localai/localai-backends:latest-gpu-rocm-hipblas-vllm +# Consumer/RDNA AMD GPUs (gfx1151, Strix Halo). Separate image because vllm has no +# prebuilt wheel for these arches and has to be built from source -- see +# backend/python/vllm/install.sh. Deliberately NOT in the `amd:` capability map: +# that map cannot know the user's gfx arch, so `rocm-vllm` stays the automatic +# choice and this one is installed by name. +- !!merge <<: *vllm + name: "rocm-gfx1151-vllm" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-gfx1151-vllm" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-gfx1151-vllm - !!merge <<: *vllm name: "intel-vllm" uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-vllm" @@ -4229,6 +4239,11 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-vllm" mirrors: - localai/localai-backends:master-gpu-rocm-hipblas-vllm +- !!merge <<: *vllm + name: "rocm-gfx1151-vllm-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-gfx1151-vllm" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-gfx1151-vllm - !!merge <<: *vllm name: "intel-vllm-development" uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-vllm" diff --git a/backend/python/diffusers/install.sh b/backend/python/diffusers/install.sh index 83703b1b2853..710f4007b903 100755 --- a/backend/python/diffusers/install.sh +++ b/backend/python/diffusers/install.sh @@ -27,4 +27,37 @@ if [ "x${BUILD_PROFILE}" == "xl4t13" ]; then PY_STANDALONE_TAG="20251120" fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheels do not enumerate Strix +# Halo (device_count == 0). AMD ships stable ROCm 7.14 builds via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install torch + +# torchvision (the same pinned versions, from that index) in an isolated step: +# uv aborts on that index's 403-for-missing-package responses, so we must not let +# it resolve PyPI-only packages (diffusers, transformers, ...) there. Everything +# else then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # GPU arch(es) from the build. AMDGPU_TARGETS is the repo-wide comma-separated + # gfx list (see backend/cpp/llama-cpp/Makefile); each entry maps to one + # `device-gfx` extra and they compose: torch[device-gfx942,device-gfx1151] + # installs both device packages side by side. + # + # Each device package is ~1.6 GB installed, so an image carries only the arches + # its matrix entry names -- the repo-wide 11-arch default would add ~17 GB. + # That is why the hipblas matrix entries for this backend set `amdgpu-targets` + # explicitly instead of inheriting that default. + _gpu_extras="" + for _a in ${AMDGPU_TARGETS//,/ }; do + _a="${_a%% *}" + [ -n "${_a}" ] || continue + _gpu_extras="${_gpu_extras:+${_gpu_extras},}device-${_a}" + done + # Local `make` builds outside CI pass no targets; gfx1151 is the arch this was + # hardware-validated on. + _gpu_extras="${_gpu_extras:-device-gfx1151}" + echo "${0##*/}: AMD torch device extras: ${_gpu_extras}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[${_gpu_extras}]==2.10.0+rocm7.14.0" \ + "torchvision==0.25.0+rocm7.14.0" +fi + installRequirements diff --git a/backend/python/diffusers/requirements-hipblas.txt b/backend/python/diffusers/requirements-hipblas.txt index f3666d5f504c..47846fc5c91c 100644 --- a/backend/python/diffusers/requirements-hipblas.txt +++ b/backend/python/diffusers/requirements-hipblas.txt @@ -1,6 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 -torchvision==0.25.0+rocm7.0 +# torch + torchvision are installed separately from AMD's multi-arch ROCm 7.14 +# index in install.sh (gfx1151 needs torch built against ROCm 7.13+; the +# community whl/rocm7.0 wheel does not enumerate Strix Halo). uv aborts on that +# index's 403-for-missing-pkg, so torch is pulled there; everything below resolves +# from PyPI. diffusers==0.38.0 opencv-python transformers==4.57.6 diff --git a/backend/python/kokoro/install.sh b/backend/python/kokoro/install.sh index 3e1ab8606480..13d5f4d33b50 100755 --- a/backend/python/kokoro/install.sh +++ b/backend/python/kokoro/install.sh @@ -20,6 +20,39 @@ if [ "x${BUILD_PROFILE}" == "xl4t12" ]; then USE_PIP=true fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheels do not enumerate Strix +# Halo (device_count == 0). AMD ships stable ROCm 7.14 builds via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install torch + +# torchaudio (the same pinned versions, from that index) in an isolated step: uv +# aborts on that index's 403-for-missing-package responses, so we must not let it +# resolve PyPI-only packages (kokoro, transformers, ...) there. Everything else +# then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # GPU arch(es) from the build. AMDGPU_TARGETS is the repo-wide comma-separated + # gfx list (see backend/cpp/llama-cpp/Makefile); each entry maps to one + # `device-gfx` extra and they compose: torch[device-gfx942,device-gfx1151] + # installs both device packages side by side. + # + # Each device package is ~1.6 GB installed, so an image carries only the arches + # its matrix entry names -- the repo-wide 11-arch default would add ~17 GB. + # That is why the hipblas matrix entries for this backend set `amdgpu-targets` + # explicitly instead of inheriting that default. + _gpu_extras="" + for _a in ${AMDGPU_TARGETS//,/ }; do + _a="${_a%% *}" + [ -n "${_a}" ] || continue + _gpu_extras="${_gpu_extras:+${_gpu_extras},}device-${_a}" + done + # Local `make` builds outside CI pass no targets; gfx1151 is the arch this was + # hardware-validated on. + _gpu_extras="${_gpu_extras:-device-gfx1151}" + echo "${0##*/}: AMD torch device extras: ${_gpu_extras}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[${_gpu_extras}]==2.10.0+rocm7.14.0" \ + "torchaudio==2.10.0+rocm7.14.0" +fi + installRequirements # spaCy is a dependency of misaki (used by kokoro for English phonemization). diff --git a/backend/python/kokoro/requirements-hipblas.txt b/backend/python/kokoro/requirements-hipblas.txt index 0ca1352282c9..9edeff371eeb 100644 --- a/backend/python/kokoro/requirements-hipblas.txt +++ b/backend/python/kokoro/requirements-hipblas.txt @@ -1,6 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 -torchaudio==2.10.0+rocm7.0 +# torch + torchaudio are installed separately from AMD's multi-arch ROCm 7.14 +# index in install.sh (gfx1151 needs torch built against ROCm 7.13+; the +# community whl/rocm7.0 wheel does not enumerate Strix Halo). uv aborts on that +# index's 403-for-missing-pkg, so torch is pulled there; everything below resolves +# from PyPI. transformers accelerate kokoro diff --git a/backend/python/transformers/install.sh b/backend/python/transformers/install.sh index f95d391e9f88..e1963da21a86 100755 --- a/backend/python/transformers/install.sh +++ b/backend/python/transformers/install.sh @@ -19,4 +19,36 @@ if [ "x${BUILD_PROFILE}" == "xintel" ]; then EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheel does not enumerate Strix +# Halo (device_count == 0). AMD ships a stable ROCm 7.14 build via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install it (plus +# its rocm/triton deps, which all live on that same index) in an isolated step: +# uv aborts on that index's 403-for-missing-package responses, so we must not let +# it look up PyPI-only packages (accelerate, transformers, ...) there. Everything +# else then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # GPU arch(es) from the build. AMDGPU_TARGETS is the repo-wide comma-separated + # gfx list (see backend/cpp/llama-cpp/Makefile); each entry maps to one + # `device-gfx` extra and they compose: torch[device-gfx942,device-gfx1151] + # installs both device packages side by side. + # + # Each device package is ~1.6 GB installed, so an image carries only the arches + # its matrix entry names -- the repo-wide 11-arch default would add ~17 GB. + # That is why the hipblas matrix entries for this backend set `amdgpu-targets` + # explicitly instead of inheriting that default. + _gpu_extras="" + for _a in ${AMDGPU_TARGETS//,/ }; do + _a="${_a%% *}" + [ -n "${_a}" ] || continue + _gpu_extras="${_gpu_extras:+${_gpu_extras},}device-${_a}" + done + # Local `make` builds outside CI pass no targets; gfx1151 is the arch this was + # hardware-validated on. + _gpu_extras="${_gpu_extras:-device-gfx1151}" + echo "${0##*/}: AMD torch device extras: ${_gpu_extras}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[${_gpu_extras}]==2.12.0+rocm7.14.0" +fi + installRequirements diff --git a/backend/python/transformers/requirements-hipblas.txt b/backend/python/transformers/requirements-hipblas.txt index 0632addc2473..fc5350d64eec 100644 --- a/backend/python/transformers/requirements-hipblas.txt +++ b/backend/python/transformers/requirements-hipblas.txt @@ -1,5 +1,7 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 +# torch is installed separately from AMD's multi-arch ROCm index in install.sh +# (gfx1151 needs torch built against ROCm 7.13+; the community whl/rocm7.0 wheel +# does not enumerate Strix Halo). uv aborts on that index's 403-for-missing-pkg, +# so torch is pulled with pip there; everything below resolves from PyPI. accelerate transformers>=5.15.1 llvmlite==0.49.0 diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index a4124977c282..338d8fd6c003 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -48,6 +48,33 @@ if [ "x${BUILD_TYPE}" == "xhipblas" ]; then PYTHON_PATCH="12" PY_STANDALONE_TAG="20251120" EXTRA_PIP_INSTALL_FLAGS+=" --index-strategy=unsafe-best-match" + + # GPU arch(es) to build for. AMDGPU_TARGETS is the repo-wide comma-separated + # gfx list (see backend/cpp/llama-cpp/Makefile); local `make` builds outside CI + # pass none, and gfx1151 is the arch this was hardware-validated on. + _gpu_targets="${AMDGPU_TARGETS:-gfx1151}" + + # AMD publishes a per-GPU torch on its multi-arch index, selected with the + # torch[device-gfx] extra. The extras compose, so a multi-arch target + # list installs one device package per arch -- each ~1.6 GB installed. + _gpu_extras="" + for _a in ${_gpu_targets//,/ }; do + _a="${_a%% *}" + [ -n "${_a}" ] || continue + _gpu_extras="${_gpu_extras:+${_gpu_extras},}device-${_a}" + done + + # Does the prebuilt ROCm wheel cover EVERY requested arch? It is CDNA-only. + # One uncovered arch is enough to need the source build, because a single venv + # holds a single vllm. + _vllm_prebuilt=true + for _a in ${_gpu_targets//,/ }; do + case "${_a%% *}" in + ""|gfx942|gfx950) ;; + *) _vllm_prebuilt=false ;; + esac + done + echo "${0##*/}: AMD targets ${_gpu_targets} -> prebuilt vllm wheel: ${_vllm_prebuilt}" fi # cublas13 pulls the vLLM wheel from a per-tag cu130 index (PyPI's vllm wheel @@ -226,22 +253,98 @@ elif [ "x${BUILD_TYPE}" == "xintel" ]; then export CMAKE_PREFIX_PATH="$(python -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH:-}" VLLM_TARGET_DEVICE=xpu uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps . popd -# AMD ROCm: install vllm from its dedicated ROCm wheel index instead of the -# CUDA-only PyPI wheel. installRequirements brings the base ROCm -# torch/transformers (requirements-hipblas.txt), then we pull vllm (plus the -# matching ROCm torch, via --upgrade) from wheels.vllm.ai/rocm. This is the -# method upstream prescribes for AMD; the Python-3.12 pin is set above. -# There is intentionally no requirements-hipblas-after.txt: a bare `vllm` -# there would resolve to the CUDA wheel, and installRequirements never loads -# a ${BUILD_TYPE}-after file for hipblas anyway (BUILD_TYPE == BUILD_PROFILE). -# https://docs.vllm.ai/en/latest/getting_started/installation/gpu.html?device=rocm -elif [ "x${BUILD_TYPE}" == "xhipblas" ]; then +# AMD ROCm. Two paths, picked by the target arch, because the prebuilt wheel exists +# for only one of them: +# +# * CDNA data-center (gfx942 / gfx950) -> the prebuilt wheel from +# wheels.vllm.ai/rocm, exactly as before. Unchanged, and still what the +# published ROCm image builds. +# * anything else, e.g. gfx1151 (RDNA 3.5, Strix Halo) -> build from source. +# There is NO prebuilt vllm wheel for these GPUs anywhere: the wheels.vllm.ai +# build is CDNA-only and pinned to a rocm7.2.3 torch that cannot even enumerate +# Strix Halo (device_count == 0), and AMD's multi-arch index carries torch and +# triton but no vllm. That is why AMD ships a separate rocm/vllm:*_rdna_* +# container; the branch below reproduces its recipe inside LocalAI's normal +# Dockerfile.python build (torch 2.11.0+rocm7.14.0, vllm 0.23.1.dev1 @ 9ddef7117), +# mirroring the intel/cpu source-build branches above. +# +# Deciding on the arch instead of replacing the prebuilt path keeps the published +# CDNA image what it is today, build time included. _gpu_targets / _vllm_prebuilt +# are computed in the pre-venv hipblas block above. +elif [ "x${BUILD_TYPE}" == "xhipblas" ] && [ "${_vllm_prebuilt}" = "true" ]; then installRequirements # --upgrade reconciles the base ROCm torch to whatever the vllm ROCm wheel # pins; --extra-index-url adds the ROCm wheel repository on top of PyPI. + # There is intentionally no requirements-hipblas-after.txt: a bare `vllm` + # there would resolve to the CUDA wheel, and installRequirements never loads + # a ${BUILD_TYPE}-after file for hipblas anyway (BUILD_TYPE == BUILD_PROFILE). uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} \ --extra-index-url https://wheels.vllm.ai/rocm/ --upgrade vllm +elif [ "x${BUILD_TYPE}" == "xhipblas" ]; then + # vllm's HIP/C++ compile is large; MAX_JOBS=1 (set above for CUDA hosts) would + # take hours. 8 jobs balances throughput vs the per-job RAM of the ROCm kernels. + export MAX_JOBS=8 + + # rocm_smi's CMake package (pulled by vllm's ROCm build) probes system libdrm + # via pkg-config; these are OS-level build tools, not ROCm. No-op outside the + # Docker builder stage. The ROCm toolchain itself comes entirely from pip below. + if command -v apt-get >/dev/null 2>&1 && [ "$(id -u)" = "0" ]; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + pkg-config libdrm-dev + fi + + ensureVenv + # Self-contained ROCm 7.14 build -- reproduces AMD's own rocm/vllm rdna recipe. + # From AMD's multi-arch index: the per-GPU torch (torch[device-gfx]) AND + # the full devel SDK (rocm-sdk-devel: hipcc + the HIP CMake packages that the + # runtime _rocm_sdk_core lacks). Everything lives in the venv, so the build + # needs NO system ROCm at /opt/rocm and reproduces on any base image with pip. + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[${_gpu_extras}]==2.11.0+rocm7.14.0" \ + "torchvision==0.26.0+rocm7.14.0" \ + "rocm-sdk-devel==7.14.0" + installRequirements + + # Expand the devel tree and link the installed rocm-sdk-device-* wheels into + # it (required before the SDK's hipcc/CMake are usable), then resolve every + # path from the SDK's own CLI — no hardcoded location, no host dependency. + rocm-sdk init + ROCM_PATH="$(rocm-sdk path --root)" + export ROCM_PATH ROCM_HOME="$ROCM_PATH" + export CMAKE_PREFIX_PATH="$(rocm-sdk path --cmake)${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH}" + export PATH="$(rocm-sdk path --bin):$PATH" + + # vLLM detects ROCm at runtime via `import amdsmi` (platforms/__init__.py: + # amdsmi_init() + get_processor_handles()); without it vLLM falls back to + # UnspecifiedPlatform and dies with "Device string must not be empty". + # rocm-sdk-core ships amd_smi in-place under _rocm_sdk_core/share/amd_smi, + # where it resolves its native libamd_smi.so RELATIVE to that directory. + # A pip copy into site-packages breaks that relative lookup, and forcing the + # lib via LD_LIBRARY_PATH shadows torch's own bundled ROCm runtime and drops + # device_count to 0. So register the in-place package with a .pth entry -- + # `import amdsmi` then resolves where its libs do, no LD override needed. + # (AMD's rocm/vllm image likewise imports amdsmi in-place from this path.) + _sp="$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" + printf '_rocm_sdk_core/share/amd_smi\n' > "${_sp}/amdsmi.pth" + + # Build vllm from source against this ROCm 7.14 torch. Pin to the release tag + # closest to what AMD's rocm/vllm rdna image ships (0.23.1.dev1+g9ddef7117 + # sits between v0.23.0 and v0.23.1rc0). + VLLM_REF="${VLLM_REF:-v0.23.0}" + _vllm_src=$(mktemp -d) + trap 'rm -rf "${_vllm_src}"' EXIT + git clone --depth 1 --branch "${VLLM_REF}" \ + https://github.com/vllm-project/vllm "${_vllm_src}/vllm" + pushd "${_vllm_src}/vllm" + # Strip vllm's own torch/triton pins so it builds against our ROCm wheels. + python use_existing_torch.py || true + uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} -r requirements/rocm.txt + # No GPU in the build container -> pin the target arch(es) explicitly. + export PYTORCH_ROCM_ARCH="${_gpu_targets}" GPU_TARGETS="${_gpu_targets}" GPU_ARCHS="${_gpu_targets}" + VLLM_TARGET_DEVICE=rocm uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps . + popd # FROM_SOURCE=true on a CPU build skips the prebuilt vllm wheel in # requirements-cpu-after.txt and compiles vllm locally against the host's # actual CPU. Not used by default because it takes ~30-40 minutes, but diff --git a/backend/python/vllm/requirements-hipblas.txt b/backend/python/vllm/requirements-hipblas.txt index 1163f1b647c6..c26500ea1ed3 100644 --- a/backend/python/vllm/requirements-hipblas.txt +++ b/backend/python/vllm/requirements-hipblas.txt @@ -1,5 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/nightly/rocm7.0 +# torch is installed separately from AMD's multi-arch ROCm 7.14 index in +# install.sh (gfx1151 needs torch built against ROCm 7.13+; the community +# whl/rocm7.0 wheel does not enumerate Strix Halo). vllm itself is built from +# source there against that torch. uv aborts on the AMD index's 403-for-missing +# -pkg, so torch is pulled there; everything below resolves from PyPI. accelerate -torch transformers bitsandbytes \ No newline at end of file diff --git a/backend/python/vllm/run.sh b/backend/python/vllm/run.sh index 83607fb8d659..7c3668a5d096 100755 --- a/backend/python/vllm/run.sh +++ b/backend/python/vllm/run.sh @@ -47,6 +47,20 @@ if [ -d "${EDIR}/toolchain/usr/bin" ]; then export LD_LIBRARY_PATH="${_libpath}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" fi +# ROCm (hipblas) profile: Triton JIT-compiles its launcher/util C modules at first +# inference (e.g. the top-k/top-p sampling kernel), and torch._inductor's ISA probe +# shells out to a C compiler too. The LocalAI runtime image ships hipcc but not +# cc/gcc, and the FROM-scratch backend omits them as well -- so Triton dies with +# "Failed to find C compiler". rocm-sdk-devel (installed for the gfx1151 build) +# already bundles amdclang, so just put it on PATH and expose it as CC/CXX. Mirrors +# the CPU-toolchain block above; no-op for other profiles (the dir doesn't exist). +_rocm_sdk_bin="$(ls -d ${EDIR}/venv/lib/python*/site-packages/_rocm_sdk_devel/bin 2>/dev/null | head -1)" +if [ -n "${_rocm_sdk_bin}" ] && [ -x "${_rocm_sdk_bin}/amdclang" ]; then + export PATH="${_rocm_sdk_bin}:${PATH}" + export CC="${CC:-${_rocm_sdk_bin}/amdclang}" + export CXX="${CXX:-${_rocm_sdk_bin}/amdclang++}" +fi + # Multi-node DP follower mode: when the first arg is `serve`, exec into # vllm's own CLI instead of LocalAI's backend.py gRPC server. The # follower speaks ZMQ directly to the head node's vllm ranks — there