From 8304e1ed59648fe2493d20d47f54d5a17f6b1b86 Mon Sep 17 00:00:00 2001 From: Tai An Date: Fri, 4 Sep 2026 15:26:06 -0700 Subject: [PATCH 1/3] fix(quantization): pin the producing backend on imported quantized models (#11875) ImportModel hands the copied GGUF to importers.ImportLocalPath, which detects the file format and defaults every GGUF to `backend: llama-cpp`. For a model this service just produced with a backend stock llama.cpp cannot read, the generated config names an engine that cannot load the file, and the import silently registers an unloadable model. Correcting `backend:` by hand makes the same file work. The job record already carries the backend that served StartQuantization, so carry it into the config instead of keeping the detected default. The gallery publishes a quantizer as a release channel of the engine that runs its output ("llama-cpp-quantization" is llama.cpp's quantizer, whose GGUF is served by "llama-cpp"), so the channel suffix is stripped to get the serving backend. A backend that both quantizes and serves ("rocmfp4") carries no suffix and passes through unchanged, as do pinned hardware variants ("rocm-rocmfp4"), which are valid values for a config's backend field. An empty job backend leaves the detected default in place. Also replace the importer's generic "Fine-tuned model (GGUF)" description for this path: the model was quantized, not fine-tuned, and the job knows the type. Signed-off-by: Tai An --- core/services/quantization/service.go | 28 +++++++++++++++++++++- core/services/quantization/service_test.go | 22 +++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/core/services/quantization/service.go b/core/services/quantization/service.go index a6a6eefcbc09..b2767f0a66be 100644 --- a/core/services/quantization/service.go +++ b/core/services/quantization/service.go @@ -621,6 +621,21 @@ func sanitizeQuantModelName(s string) string { return strings.ToLower(s) } +// inferenceBackendFor returns the backend that can load what a quantization +// backend produced. +// +// The gallery publishes a quantizer as a release channel of the engine that +// runs its output: "llama-cpp-quantization" is llama.cpp's quantizer, and the +// GGUF it writes is served by "llama-cpp". The suffix is a channel marker and +// carries no engine information, so stripping it yields the backend to pin in +// the imported model's config. Names that carry no channel suffix (a backend +// that both quantizes and serves, such as "rocmfp4") are already the engine +// name and pass through unchanged, as do pinned hardware variants +// ("rocm-rocmfp4"), which are valid values for a config's `backend:`. +func inferenceBackendFor(quantBackend string) string { + return strings.TrimSuffix(config.NormalizeBackendName(quantBackend), "-quantization") +} + // ImportModel imports a quantized model into LocalAI asynchronously. func (s *QuantizationService) ImportModel(ctx context.Context, userID, jobID string, req schema.QuantizationImportRequest) (string, error) { s.mu.Lock() @@ -719,6 +734,17 @@ func (s *QuantizationService) ImportModel(ctx context.Context, userID, jobID str cfg.Name = modelName + // The importer detects the file format and defaults to llama-cpp for any + // GGUF. That is wrong for a model this service just quantized with a + // backend stock llama.cpp cannot read: the job knows which backend + // produced the file, so pin that one instead of the detected default. + if backend := inferenceBackendFor(job.Backend); backend != "" { + cfg.Backend = backend + } + if job.QuantizationType != "" { + cfg.Description = "Quantized model (" + job.QuantizationType + ", GGUF)" + } + // Write YAML config yamlData, err := yaml.Marshal(cfg) if err != nil { @@ -812,4 +838,4 @@ func (s *QuantizationService) GetOutputPath(userID, jobID string) (string, strin downloadName := filepath.Base(outputFile) return outputFile, downloadName, nil -} +} \ No newline at end of file diff --git a/core/services/quantization/service_test.go b/core/services/quantization/service_test.go index ae862ffcaa85..8af5e392c61e 100644 --- a/core/services/quantization/service_test.go +++ b/core/services/quantization/service_test.go @@ -329,6 +329,28 @@ var _ = Describe("QuantizationService", func() { }) }) + Describe("imported model backend", func() { + It("strips the quantization channel suffix so the config pins the serving engine", func() { + Expect(inferenceBackendFor("llama-cpp-quantization")).To(Equal("llama-cpp")) + }) + + It("leaves a backend that both quantizes and serves unchanged", func() { + Expect(inferenceBackendFor("rocmfp4")).To(Equal("rocmfp4")) + }) + + It("keeps a pinned hardware variant, which is a valid backend value", func() { + Expect(inferenceBackendFor("rocm-rocmfp4-quantization")).To(Equal("rocm-rocmfp4")) + }) + + It("normalizes dots the way gallery names are written", func() { + Expect(inferenceBackendFor("llama.cpp-quantization")).To(Equal("llama-cpp")) + }) + + It("returns empty for an unset backend so the detected default is kept", func() { + Expect(inferenceBackendFor("")).To(BeEmpty()) + }) + }) + Describe("compile-time adapter contract", func() { It("satisfies syncstate.Store for *distributed.QuantStore", func() { // Guards against drift between the adapter and the component interface; From 2c770a9b0079f00ab4e77970dda195d3a0a67957 Mon Sep 17 00:00:00 2001 From: Tai An Date: Mon, 7 Sep 2026 06:02:38 -0700 Subject: [PATCH 2/3] style: restore trailing newline in service.go for gofmt Signed-off-by: Anai Guo From 9282ec47e5b6ae7a31a3da025d88186dc1ef2226 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 27 Sep 2026 19:05:49 +0000 Subject: [PATCH 3/3] style(quantization): restore trailing newline in service.go gofmt requires the file to end with a newline; the previous style commit did not actually add it. Signed-off-by: Ettore Di Giacinto Assisted-by: Claude:claude-opus-5-5 [Claude Code] --- core/services/quantization/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/services/quantization/service.go b/core/services/quantization/service.go index b2767f0a66be..01154320533b 100644 --- a/core/services/quantization/service.go +++ b/core/services/quantization/service.go @@ -838,4 +838,4 @@ func (s *QuantizationService) GetOutputPath(userID, jobID string) (string, strin downloadName := filepath.Base(outputFile) return outputFile, downloadName, nil -} \ No newline at end of file +}