Skip to content

Commit 5bb315a

Browse files
committed
feat(#3996): resolve image output capability from models.dev
Resolve an omitted output_capabilities.image flag from the model's models.dev output modalities, matching modality names case-insensitively. Explicit true or false overrides remain authoritative, while a missing store or catalogue record conservatively disables image output. Wire the resolver through provider base configuration, update schema and model documentation, and cover catalogue hits, misses, nil stores, and override precedence.
1 parent fe76e0a commit 5bb315a

3 files changed

Lines changed: 51 additions & 4 deletions

File tree

agent-schema.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1742,7 +1742,7 @@
17421742
},
17431743
"output_capabilities": {
17441744
"$ref": "#/definitions/OutputCapabilitiesConfig",
1745-
"description": "Explicit, owner-declared generative output capabilities for this model. Never inferred from the model name or any catalogue — omit it, or leave a flag unset/false, to preserve existing behavior. Consumed by capability-gated request contracts (e.g. the Gemini gateway image-output route) that must not activate for a model unless its owner has affirmatively opted it in. Cannot be combined with first_available (set output_capabilities.image on the candidate models instead)."
1745+
"description": "Optional generative output capability override for this model. When omitted, Docker Agent detects output modalities from the models.dev catalogue. An explicit flag is authoritative, including false. Cannot be combined with first_available (set output_capabilities.image on the candidate models instead)."
17461746
},
17471747
"cost": {
17481748
"$ref": "#/definitions/CostConfig",
@@ -1776,11 +1776,11 @@
17761776
},
17771777
"OutputCapabilitiesConfig": {
17781778
"type": "object",
1779-
"description": "Explicit, owner-declared generative output capabilities for a model. There is no automatic detection — no catalogue of output-capable models exists, and matching on the model name string is deliberately avoided as unreliable — so this is nil/omitted by default. Omitted or false preserves existing behavior; only an explicit true opts a model into output-capability-gated behavior.",
1779+
"description": "Generative output capability overrides for a model. When omitted, Docker Agent detects output modalities from the models.dev catalogue. Explicit flags are authoritative, including false, and are useful for custom models or correcting catalogue metadata.",
17801780
"properties": {
17811781
"image": {
17821782
"type": "boolean",
1783-
"description": "Whether the model is declared able to generate image output. Never inferred; must be set explicitly by the owner."
1783+
"description": "Whether the model can generate image output. When omitted, Docker Agent uses models.dev output modalities; an explicit value overrides the catalogue."
17841784
}
17851785
},
17861786
"additionalProperties": false

pkg/model/provider/base/base.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package base
22

33
import (
4+
"context"
5+
46
"github.com/docker/docker-agent/pkg/config/latest"
57
"github.com/docker/docker-agent/pkg/environment"
68
"github.com/docker/docker-agent/pkg/model/provider/options"
@@ -77,6 +79,16 @@ func (c *Config) CapsOverride() *modelinfo.CapsOverride {
7779
return &modelinfo.CapsOverride{Image: caps.Image, PDF: caps.PDF, Audio: caps.Audio, Video: caps.Video}
7880
}
7981

82+
// ImageOutputEnabled resolves the model's image-output capability from its
83+
// explicit tri-state configuration and, when unset, the models.dev catalogue.
84+
func (c *Config) ImageOutputEnabled(ctx context.Context) bool {
85+
var override *bool
86+
if caps := c.ModelConfig.OutputCapabilities; caps != nil {
87+
override = caps.Image
88+
}
89+
return modelinfo.ResolveOutputImage(ctx, c.ModelOptions.ModelsDevStore(), c.ID(), override)
90+
}
91+
8092
// EmbeddingResult contains the embedding and usage information
8193
type EmbeddingResult struct {
8294
Embedding []float64

pkg/modelinfo/modelinfo.go

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -639,7 +639,42 @@ func LoadCaps(ctx context.Context, store *modelsdev.Store, id modelsdev.ID) Mode
639639
return capsFromModalities(model.Modalities.Input)
640640
}
641641

642-
// capsFromModalities maps a models.dev input-modality list to the capability
642+
// ResolveOutputImage applies an explicit image-output override when present;
643+
// otherwise it derives support from the models.dev output modalities. Missing
644+
// catalogue data conservatively disables image output.
645+
func ResolveOutputImage(ctx context.Context, store *modelsdev.Store, id modelsdev.ID, override *bool) bool {
646+
if override != nil {
647+
return *override
648+
}
649+
if store == nil {
650+
return false
651+
}
652+
653+
ctx, cancel := context.WithTimeout(ctx, loadCapsTimeout)
654+
defer cancel()
655+
656+
model, err := store.GetModel(ctx, id)
657+
if err != nil {
658+
if ctx.Err() != nil {
659+
slog.WarnContext(ctx, "modelinfo: models.dev output lookup timed out, disabling image output",
660+
"model", id.String(), "timeout", loadCapsTimeout)
661+
} else {
662+
warnCapsLookupMiss(ctx, id, err)
663+
}
664+
return false
665+
}
666+
return hasOutputModality(model.Modalities.Output, "image")
667+
}
668+
669+
func hasOutputModality(modalities []string, expected string) bool {
670+
for _, modality := range modalities {
671+
if strings.EqualFold(modality, expected) {
672+
return true
673+
}
674+
}
675+
return false
676+
}
677+
643678
// booleans it grants. Unknown modality names are ignored.
644679
func capsFromModalities(input []string) ModelCapabilities {
645680
var mc ModelCapabilities

0 commit comments

Comments
 (0)