diff options
| author | phantomreactor <[email protected]> | 2025-05-03 01:53:58 +0530 |
|---|---|---|
| committer | adamdottv <[email protected]> | 2025-05-02 15:29:46 -0500 |
| commit | ff0ef3bb432f1cedb6e5b8a0168bfa7c9e9e15f0 (patch) | |
| tree | e027f8eee09fafe33b98c6316d84b0f5e6a8edc0 /internal/llm | |
| parent | 0095832be3b6c9ae9c45dfed70ecd22302e08dc9 (diff) | |
| download | opencode-ff0ef3bb432f1cedb6e5b8a0168bfa7c9e9e15f0.tar.gz opencode-ff0ef3bb432f1cedb6e5b8a0168bfa7c9e9e15f0.zip | |
feat: add support for images
Diffstat (limited to 'internal/llm')
| -rw-r--r-- | internal/llm/agent/agent.go | 48 | ||||
| -rw-r--r-- | internal/llm/models/anthropic.go | 107 | ||||
| -rw-r--r-- | internal/llm/models/azure.go | 241 | ||||
| -rw-r--r-- | internal/llm/models/gemini.go | 84 | ||||
| -rw-r--r-- | internal/llm/models/groq.go | 81 | ||||
| -rw-r--r-- | internal/llm/models/models.go | 23 | ||||
| -rw-r--r-- | internal/llm/models/openai.go | 260 | ||||
| -rw-r--r-- | internal/llm/prompt/title.go | 3 | ||||
| -rw-r--r-- | internal/llm/provider/anthropic.go | 12 | ||||
| -rw-r--r-- | internal/llm/provider/bedrock.go | 9 | ||||
| -rw-r--r-- | internal/llm/provider/gemini.go | 9 | ||||
| -rw-r--r-- | internal/llm/provider/openai.go | 13 |
12 files changed, 481 insertions, 409 deletions
diff --git a/internal/llm/agent/agent.go b/internal/llm/agent/agent.go index ba65f594e..cbc30fa96 100644 --- a/internal/llm/agent/agent.go +++ b/internal/llm/agent/agent.go @@ -39,7 +39,7 @@ func (e *AgentEvent) Response() message.Message { } type Service interface { - Run(ctx context.Context, sessionID string, content string) (<-chan AgentEvent, error) + Run(ctx context.Context, sessionID string, content string, attachments ...message.Attachment) (<-chan AgentEvent, error) Cancel(sessionID string) IsSessionBusy(sessionID string) bool IsBusy() bool @@ -59,7 +59,7 @@ type agent struct { titleProvider provider.Provider activeRequests sync.Map - pauseLock sync.RWMutex // Lock for pausing message processing + pauseLock sync.RWMutex // Lock for pausing message processing } func NewAgent( @@ -122,6 +122,9 @@ func (a *agent) IsSessionBusy(sessionID string) bool { } func (a *agent) generateTitle(ctx context.Context, sessionID string, content string) error { + if content == "" { + return nil + } if a.titleProvider == nil { return nil } @@ -129,16 +132,13 @@ func (a *agent) generateTitle(ctx context.Context, sessionID string, content str if err != nil { return err } + parts := []message.ContentPart{message.TextContent{Text: content}} response, err := a.titleProvider.SendMessages( ctx, []message.Message{ { - Role: message.User, - Parts: []message.ContentPart{ - message.TextContent{ - Text: content, - }, - }, + Role: message.User, + Parts: parts, }, }, make([]tools.BaseTool, 0), @@ -163,7 +163,10 @@ func (a *agent) err(err error) AgentEvent { } } -func (a *agent) Run(ctx context.Context, sessionID string, content string) (<-chan AgentEvent, error) { +func (a *agent) Run(ctx context.Context, sessionID string, content string, attachments ...message.Attachment) (<-chan AgentEvent, error) { + if !a.provider.Model().SupportsAttachments && attachments != nil { + attachments = nil + } events := make(chan AgentEvent) if a.IsSessionBusy(sessionID) { return nil, ErrSessionBusy @@ -177,10 +180,13 @@ func (a *agent) Run(ctx context.Context, sessionID string, content string) (<-ch defer logging.RecoverPanic("agent.Run", func() { events <- a.err(fmt.Errorf("panic while running the agent")) }) - - result := a.processGeneration(genCtx, sessionID, content) + var attachmentParts []message.ContentPart + for _, attachment := range attachments { + attachmentParts = append(attachmentParts, message.BinaryContent{Path: attachment.FilePath, MIMEType: attachment.MimeType, Data: attachment.Content}) + } + result := a.processGeneration(genCtx, sessionID, content, attachmentParts) if result.Err() != nil && !errors.Is(result.Err(), ErrRequestCancelled) && !errors.Is(result.Err(), context.Canceled) { - logging.ErrorPersist(fmt.Sprintf("Generation error for session %s: %v", sessionID, result)) + logging.ErrorPersist(result.Err().Error()) } logging.Debug("Request completed", "sessionID", sessionID) a.activeRequests.Delete(sessionID) @@ -191,7 +197,7 @@ func (a *agent) Run(ctx context.Context, sessionID string, content string) (<-ch return events, nil } -func (a *agent) processGeneration(ctx context.Context, sessionID, content string) AgentEvent { +func (a *agent) processGeneration(ctx context.Context, sessionID, content string, attachmentParts []message.ContentPart) AgentEvent { // Get the current session to check for summary currentSession, err := a.sessions.Get(ctx, sessionID) if err != nil { @@ -227,7 +233,7 @@ func (a *agent) processGeneration(ctx context.Context, sessionID, content string }() } - userMsg, err := a.createUserMessage(ctx, sessionID, content) + userMsg, err := a.createUserMessage(ctx, sessionID, content, attachmentParts) if err != nil { return a.err(fmt.Errorf("failed to create user message: %w", err)) } @@ -251,6 +257,7 @@ func (a *agent) processGeneration(ctx context.Context, sessionID, content string // Append the new user message to the conversation history messages = append(messages, userMsg) + for { // Check for cancellation before each iteration select { @@ -280,12 +287,12 @@ func (a *agent) processGeneration(ctx context.Context, sessionID, content string } } -func (a *agent) createUserMessage(ctx context.Context, sessionID, content string) (message.Message, error) { +func (a *agent) createUserMessage(ctx context.Context, sessionID, content string, attachmentParts []message.ContentPart) (message.Message, error) { + parts := []message.ContentPart{message.TextContent{Text: content}} + parts = append(parts, attachmentParts...) return a.messages.Create(ctx, sessionID, message.CreateMessageParams{ - Role: message.User, - Parts: []message.ContentPart{ - message.TextContent{Text: content}, - }, + Role: message.User, + Parts: parts, }) } @@ -419,7 +426,6 @@ func (a *agent) streamAndHandleEvents(ctx context.Context, sessionID string, msg } continue } - toolResult, toolErr := tool.Run(ctx, tools.ToolCall{ ID: toolCall.ID, Name: toolCall.Name, @@ -575,7 +581,7 @@ func (a *agent) PauseSession(sessionID string) error { if !a.IsSessionBusy(sessionID) { return nil // Session is not active, no need to pause } - + logging.InfoPersist(fmt.Sprintf("Pausing session: %s", sessionID)) a.pauseLock.Lock() // Acquire write lock to block new operations return nil diff --git a/internal/llm/models/anthropic.go b/internal/llm/models/anthropic.go index d5f410e90..156924a47 100644 --- a/internal/llm/models/anthropic.go +++ b/internal/llm/models/anthropic.go @@ -14,64 +14,69 @@ const ( // https://docs.anthropic.com/en/docs/about-claude/models/all-models var AnthropicModels = map[ModelID]Model{ Claude35Sonnet: { - ID: Claude35Sonnet, - Name: "Claude 3.5 Sonnet", - Provider: ProviderAnthropic, - APIModel: "claude-3-5-sonnet-latest", - CostPer1MIn: 3.0, - CostPer1MInCached: 3.75, - CostPer1MOutCached: 0.30, - CostPer1MOut: 15.0, - ContextWindow: 200000, - DefaultMaxTokens: 5000, + ID: Claude35Sonnet, + Name: "Claude 3.5 Sonnet", + Provider: ProviderAnthropic, + APIModel: "claude-3-5-sonnet-latest", + CostPer1MIn: 3.0, + CostPer1MInCached: 3.75, + CostPer1MOutCached: 0.30, + CostPer1MOut: 15.0, + ContextWindow: 200000, + DefaultMaxTokens: 5000, + SupportsAttachments: true, }, Claude3Haiku: { - ID: Claude3Haiku, - Name: "Claude 3 Haiku", - Provider: ProviderAnthropic, - APIModel: "claude-3-haiku-20240307", // doesn't support "-latest" - CostPer1MIn: 0.25, - CostPer1MInCached: 0.30, - CostPer1MOutCached: 0.03, - CostPer1MOut: 1.25, - ContextWindow: 200000, - DefaultMaxTokens: 4096, + ID: Claude3Haiku, + Name: "Claude 3 Haiku", + Provider: ProviderAnthropic, + APIModel: "claude-3-haiku-20240307", // doesn't support "-latest" + CostPer1MIn: 0.25, + CostPer1MInCached: 0.30, + CostPer1MOutCached: 0.03, + CostPer1MOut: 1.25, + ContextWindow: 200000, + DefaultMaxTokens: 4096, + SupportsAttachments: true, }, Claude37Sonnet: { - ID: Claude37Sonnet, - Name: "Claude 3.7 Sonnet", - Provider: ProviderAnthropic, - APIModel: "claude-3-7-sonnet-latest", - CostPer1MIn: 3.0, - CostPer1MInCached: 3.75, - CostPer1MOutCached: 0.30, - CostPer1MOut: 15.0, - ContextWindow: 200000, - DefaultMaxTokens: 50000, - CanReason: true, + ID: Claude37Sonnet, + Name: "Claude 3.7 Sonnet", + Provider: ProviderAnthropic, + APIModel: "claude-3-7-sonnet-latest", + CostPer1MIn: 3.0, + CostPer1MInCached: 3.75, + CostPer1MOutCached: 0.30, + CostPer1MOut: 15.0, + ContextWindow: 200000, + DefaultMaxTokens: 50000, + CanReason: true, + SupportsAttachments: true, }, Claude35Haiku: { - ID: Claude35Haiku, - Name: "Claude 3.5 Haiku", - Provider: ProviderAnthropic, - APIModel: "claude-3-5-haiku-latest", - CostPer1MIn: 0.80, - CostPer1MInCached: 1.0, - CostPer1MOutCached: 0.08, - CostPer1MOut: 4.0, - ContextWindow: 200000, - DefaultMaxTokens: 4096, + ID: Claude35Haiku, + Name: "Claude 3.5 Haiku", + Provider: ProviderAnthropic, + APIModel: "claude-3-5-haiku-latest", + CostPer1MIn: 0.80, + CostPer1MInCached: 1.0, + CostPer1MOutCached: 0.08, + CostPer1MOut: 4.0, + ContextWindow: 200000, + DefaultMaxTokens: 4096, + SupportsAttachments: true, }, Claude3Opus: { - ID: Claude3Opus, - Name: "Claude 3 Opus", - Provider: ProviderAnthropic, - APIModel: "claude-3-opus-latest", - CostPer1MIn: 15.0, - CostPer1MInCached: 18.75, - CostPer1MOutCached: 1.50, - CostPer1MOut: 75.0, - ContextWindow: 200000, - DefaultMaxTokens: 4096, + ID: Claude3Opus, + Name: "Claude 3 Opus", + Provider: ProviderAnthropic, + APIModel: "claude-3-opus-latest", + CostPer1MIn: 15.0, + CostPer1MInCached: 18.75, + CostPer1MOutCached: 1.50, + CostPer1MOut: 75.0, + ContextWindow: 200000, + DefaultMaxTokens: 4096, + SupportsAttachments: true, }, } diff --git a/internal/llm/models/azure.go b/internal/llm/models/azure.go index 6b7bac3a0..416597302 100644 --- a/internal/llm/models/azure.go +++ b/internal/llm/models/azure.go @@ -18,140 +18,151 @@ const ( var AzureModels = map[ModelID]Model{ AzureGPT41: { - ID: AzureGPT41, - Name: "Azure OpenAI – GPT 4.1", - Provider: ProviderAzure, - APIModel: "gpt-4.1", - CostPer1MIn: OpenAIModels[GPT41].CostPer1MIn, - CostPer1MInCached: OpenAIModels[GPT41].CostPer1MInCached, - CostPer1MOut: OpenAIModels[GPT41].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[GPT41].CostPer1MOutCached, - ContextWindow: OpenAIModels[GPT41].ContextWindow, - DefaultMaxTokens: OpenAIModels[GPT41].DefaultMaxTokens, + ID: AzureGPT41, + Name: "Azure OpenAI – GPT 4.1", + Provider: ProviderAzure, + APIModel: "gpt-4.1", + CostPer1MIn: OpenAIModels[GPT41].CostPer1MIn, + CostPer1MInCached: OpenAIModels[GPT41].CostPer1MInCached, + CostPer1MOut: OpenAIModels[GPT41].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[GPT41].CostPer1MOutCached, + ContextWindow: OpenAIModels[GPT41].ContextWindow, + DefaultMaxTokens: OpenAIModels[GPT41].DefaultMaxTokens, + SupportsAttachments: true, }, AzureGPT41Mini: { - ID: AzureGPT41Mini, - Name: "Azure OpenAI – GPT 4.1 mini", - Provider: ProviderAzure, - APIModel: "gpt-4.1-mini", - CostPer1MIn: OpenAIModels[GPT41Mini].CostPer1MIn, - CostPer1MInCached: OpenAIModels[GPT41Mini].CostPer1MInCached, - CostPer1MOut: OpenAIModels[GPT41Mini].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[GPT41Mini].CostPer1MOutCached, - ContextWindow: OpenAIModels[GPT41Mini].ContextWindow, - DefaultMaxTokens: OpenAIModels[GPT41Mini].DefaultMaxTokens, + ID: AzureGPT41Mini, + Name: "Azure OpenAI – GPT 4.1 mini", + Provider: ProviderAzure, + APIModel: "gpt-4.1-mini", + CostPer1MIn: OpenAIModels[GPT41Mini].CostPer1MIn, + CostPer1MInCached: OpenAIModels[GPT41Mini].CostPer1MInCached, + CostPer1MOut: OpenAIModels[GPT41Mini].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[GPT41Mini].CostPer1MOutCached, + ContextWindow: OpenAIModels[GPT41Mini].ContextWindow, + DefaultMaxTokens: OpenAIModels[GPT41Mini].DefaultMaxTokens, + SupportsAttachments: true, }, AzureGPT41Nano: { - ID: AzureGPT41Nano, - Name: "Azure OpenAI – GPT 4.1 nano", - Provider: ProviderAzure, - APIModel: "gpt-4.1-nano", - CostPer1MIn: OpenAIModels[GPT41Nano].CostPer1MIn, - CostPer1MInCached: OpenAIModels[GPT41Nano].CostPer1MInCached, - CostPer1MOut: OpenAIModels[GPT41Nano].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[GPT41Nano].CostPer1MOutCached, - ContextWindow: OpenAIModels[GPT41Nano].ContextWindow, - DefaultMaxTokens: OpenAIModels[GPT41Nano].DefaultMaxTokens, + ID: AzureGPT41Nano, + Name: "Azure OpenAI – GPT 4.1 nano", + Provider: ProviderAzure, + APIModel: "gpt-4.1-nano", + CostPer1MIn: OpenAIModels[GPT41Nano].CostPer1MIn, + CostPer1MInCached: OpenAIModels[GPT41Nano].CostPer1MInCached, + CostPer1MOut: OpenAIModels[GPT41Nano].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[GPT41Nano].CostPer1MOutCached, + ContextWindow: OpenAIModels[GPT41Nano].ContextWindow, + DefaultMaxTokens: OpenAIModels[GPT41Nano].DefaultMaxTokens, + SupportsAttachments: true, }, AzureGPT45Preview: { - ID: AzureGPT45Preview, - Name: "Azure OpenAI – GPT 4.5 preview", - Provider: ProviderAzure, - APIModel: "gpt-4.5-preview", - CostPer1MIn: OpenAIModels[GPT45Preview].CostPer1MIn, - CostPer1MInCached: OpenAIModels[GPT45Preview].CostPer1MInCached, - CostPer1MOut: OpenAIModels[GPT45Preview].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[GPT45Preview].CostPer1MOutCached, - ContextWindow: OpenAIModels[GPT45Preview].ContextWindow, - DefaultMaxTokens: OpenAIModels[GPT45Preview].DefaultMaxTokens, + ID: AzureGPT45Preview, + Name: "Azure OpenAI – GPT 4.5 preview", + Provider: ProviderAzure, + APIModel: "gpt-4.5-preview", + CostPer1MIn: OpenAIModels[GPT45Preview].CostPer1MIn, + CostPer1MInCached: OpenAIModels[GPT45Preview].CostPer1MInCached, + CostPer1MOut: OpenAIModels[GPT45Preview].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[GPT45Preview].CostPer1MOutCached, + ContextWindow: OpenAIModels[GPT45Preview].ContextWindow, + DefaultMaxTokens: OpenAIModels[GPT45Preview].DefaultMaxTokens, + SupportsAttachments: true, }, AzureGPT4o: { - ID: AzureGPT4o, - Name: "Azure OpenAI – GPT-4o", - Provider: ProviderAzure, - APIModel: "gpt-4o", - CostPer1MIn: OpenAIModels[GPT4o].CostPer1MIn, - CostPer1MInCached: OpenAIModels[GPT4o].CostPer1MInCached, - CostPer1MOut: OpenAIModels[GPT4o].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[GPT4o].CostPer1MOutCached, - ContextWindow: OpenAIModels[GPT4o].ContextWindow, - DefaultMaxTokens: OpenAIModels[GPT4o].DefaultMaxTokens, + ID: AzureGPT4o, + Name: "Azure OpenAI – GPT-4o", + Provider: ProviderAzure, + APIModel: "gpt-4o", + CostPer1MIn: OpenAIModels[GPT4o].CostPer1MIn, + CostPer1MInCached: OpenAIModels[GPT4o].CostPer1MInCached, + CostPer1MOut: OpenAIModels[GPT4o].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[GPT4o].CostPer1MOutCached, + ContextWindow: OpenAIModels[GPT4o].ContextWindow, + DefaultMaxTokens: OpenAIModels[GPT4o].DefaultMaxTokens, + SupportsAttachments: true, }, AzureGPT4oMini: { - ID: AzureGPT4oMini, - Name: "Azure OpenAI – GPT-4o mini", - Provider: ProviderAzure, - APIModel: "gpt-4o-mini", - CostPer1MIn: OpenAIModels[GPT4oMini].CostPer1MIn, - CostPer1MInCached: OpenAIModels[GPT4oMini].CostPer1MInCached, - CostPer1MOut: OpenAIModels[GPT4oMini].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[GPT4oMini].CostPer1MOutCached, - ContextWindow: OpenAIModels[GPT4oMini].ContextWindow, - DefaultMaxTokens: OpenAIModels[GPT4oMini].DefaultMaxTokens, + ID: AzureGPT4oMini, + Name: "Azure OpenAI – GPT-4o mini", + Provider: ProviderAzure, + APIModel: "gpt-4o-mini", + CostPer1MIn: OpenAIModels[GPT4oMini].CostPer1MIn, + CostPer1MInCached: OpenAIModels[GPT4oMini].CostPer1MInCached, + CostPer1MOut: OpenAIModels[GPT4oMini].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[GPT4oMini].CostPer1MOutCached, + ContextWindow: OpenAIModels[GPT4oMini].ContextWindow, + DefaultMaxTokens: OpenAIModels[GPT4oMini].DefaultMaxTokens, + SupportsAttachments: true, }, AzureO1: { - ID: AzureO1, - Name: "Azure OpenAI – O1", - Provider: ProviderAzure, - APIModel: "o1", - CostPer1MIn: OpenAIModels[O1].CostPer1MIn, - CostPer1MInCached: OpenAIModels[O1].CostPer1MInCached, - CostPer1MOut: OpenAIModels[O1].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[O1].CostPer1MOutCached, - ContextWindow: OpenAIModels[O1].ContextWindow, - DefaultMaxTokens: OpenAIModels[O1].DefaultMaxTokens, - CanReason: OpenAIModels[O1].CanReason, + ID: AzureO1, + Name: "Azure OpenAI – O1", + Provider: ProviderAzure, + APIModel: "o1", + CostPer1MIn: OpenAIModels[O1].CostPer1MIn, + CostPer1MInCached: OpenAIModels[O1].CostPer1MInCached, + CostPer1MOut: OpenAIModels[O1].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[O1].CostPer1MOutCached, + ContextWindow: OpenAIModels[O1].ContextWindow, + DefaultMaxTokens: OpenAIModels[O1].DefaultMaxTokens, + CanReason: OpenAIModels[O1].CanReason, + SupportsAttachments: true, }, AzureO1Mini: { - ID: AzureO1Mini, - Name: "Azure OpenAI – O1 mini", - Provider: ProviderAzure, - APIModel: "o1-mini", - CostPer1MIn: OpenAIModels[O1Mini].CostPer1MIn, - CostPer1MInCached: OpenAIModels[O1Mini].CostPer1MInCached, - CostPer1MOut: OpenAIModels[O1Mini].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[O1Mini].CostPer1MOutCached, - ContextWindow: OpenAIModels[O1Mini].ContextWindow, - DefaultMaxTokens: OpenAIModels[O1Mini].DefaultMaxTokens, - CanReason: OpenAIModels[O1Mini].CanReason, + ID: AzureO1Mini, + Name: "Azure OpenAI – O1 mini", + Provider: ProviderAzure, + APIModel: "o1-mini", + CostPer1MIn: OpenAIModels[O1Mini].CostPer1MIn, + CostPer1MInCached: OpenAIModels[O1Mini].CostPer1MInCached, + CostPer1MOut: OpenAIModels[O1Mini].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[O1Mini].CostPer1MOutCached, + ContextWindow: OpenAIModels[O1Mini].ContextWindow, + DefaultMaxTokens: OpenAIModels[O1Mini].DefaultMaxTokens, + CanReason: OpenAIModels[O1Mini].CanReason, + SupportsAttachments: true, }, AzureO3: { - ID: AzureO3, - Name: "Azure OpenAI – O3", - Provider: ProviderAzure, - APIModel: "o3", - CostPer1MIn: OpenAIModels[O3].CostPer1MIn, - CostPer1MInCached: OpenAIModels[O3].CostPer1MInCached, - CostPer1MOut: OpenAIModels[O3].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[O3].CostPer1MOutCached, - ContextWindow: OpenAIModels[O3].ContextWindow, - DefaultMaxTokens: OpenAIModels[O3].DefaultMaxTokens, - CanReason: OpenAIModels[O3].CanReason, + ID: AzureO3, + Name: "Azure OpenAI – O3", + Provider: ProviderAzure, + APIModel: "o3", + CostPer1MIn: OpenAIModels[O3].CostPer1MIn, + CostPer1MInCached: OpenAIModels[O3].CostPer1MInCached, + CostPer1MOut: OpenAIModels[O3].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[O3].CostPer1MOutCached, + ContextWindow: OpenAIModels[O3].ContextWindow, + DefaultMaxTokens: OpenAIModels[O3].DefaultMaxTokens, + CanReason: OpenAIModels[O3].CanReason, + SupportsAttachments: true, }, AzureO3Mini: { - ID: AzureO3Mini, - Name: "Azure OpenAI – O3 mini", - Provider: ProviderAzure, - APIModel: "o3-mini", - CostPer1MIn: OpenAIModels[O3Mini].CostPer1MIn, - CostPer1MInCached: OpenAIModels[O3Mini].CostPer1MInCached, - CostPer1MOut: OpenAIModels[O3Mini].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[O3Mini].CostPer1MOutCached, - ContextWindow: OpenAIModels[O3Mini].ContextWindow, - DefaultMaxTokens: OpenAIModels[O3Mini].DefaultMaxTokens, - CanReason: OpenAIModels[O3Mini].CanReason, + ID: AzureO3Mini, + Name: "Azure OpenAI – O3 mini", + Provider: ProviderAzure, + APIModel: "o3-mini", + CostPer1MIn: OpenAIModels[O3Mini].CostPer1MIn, + CostPer1MInCached: OpenAIModels[O3Mini].CostPer1MInCached, + CostPer1MOut: OpenAIModels[O3Mini].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[O3Mini].CostPer1MOutCached, + ContextWindow: OpenAIModels[O3Mini].ContextWindow, + DefaultMaxTokens: OpenAIModels[O3Mini].DefaultMaxTokens, + CanReason: OpenAIModels[O3Mini].CanReason, + SupportsAttachments: false, }, AzureO4Mini: { - ID: AzureO4Mini, - Name: "Azure OpenAI – O4 mini", - Provider: ProviderAzure, - APIModel: "o4-mini", - CostPer1MIn: OpenAIModels[O4Mini].CostPer1MIn, - CostPer1MInCached: OpenAIModels[O4Mini].CostPer1MInCached, - CostPer1MOut: OpenAIModels[O4Mini].CostPer1MOut, - CostPer1MOutCached: OpenAIModels[O4Mini].CostPer1MOutCached, - ContextWindow: OpenAIModels[O4Mini].ContextWindow, - DefaultMaxTokens: OpenAIModels[O4Mini].DefaultMaxTokens, - CanReason: OpenAIModels[O4Mini].CanReason, + ID: AzureO4Mini, + Name: "Azure OpenAI – O4 mini", + Provider: ProviderAzure, + APIModel: "o4-mini", + CostPer1MIn: OpenAIModels[O4Mini].CostPer1MIn, + CostPer1MInCached: OpenAIModels[O4Mini].CostPer1MInCached, + CostPer1MOut: OpenAIModels[O4Mini].CostPer1MOut, + CostPer1MOutCached: OpenAIModels[O4Mini].CostPer1MOutCached, + ContextWindow: OpenAIModels[O4Mini].ContextWindow, + DefaultMaxTokens: OpenAIModels[O4Mini].DefaultMaxTokens, + CanReason: OpenAIModels[O4Mini].CanReason, + SupportsAttachments: true, }, } diff --git a/internal/llm/models/gemini.go b/internal/llm/models/gemini.go index 00bf7387f..f73910166 100644 --- a/internal/llm/models/gemini.go +++ b/internal/llm/models/gemini.go @@ -12,52 +12,56 @@ const ( var GeminiModels = map[ModelID]Model{ Gemini25Flash: { - ID: Gemini25Flash, - Name: "Gemini 2.5 Flash", - Provider: ProviderGemini, - APIModel: "gemini-2.5-flash-preview-04-17", - CostPer1MIn: 0.15, - CostPer1MInCached: 0, - CostPer1MOutCached: 0, - CostPer1MOut: 0.60, - ContextWindow: 1000000, - DefaultMaxTokens: 50000, + ID: Gemini25Flash, + Name: "Gemini 2.5 Flash", + Provider: ProviderGemini, + APIModel: "gemini-2.5-flash-preview-04-17", + CostPer1MIn: 0.15, + CostPer1MInCached: 0, + CostPer1MOutCached: 0, + CostPer1MOut: 0.60, + ContextWindow: 1000000, + DefaultMaxTokens: 50000, + SupportsAttachments: true, }, Gemini25: { - ID: Gemini25, - Name: "Gemini 2.5 Pro", - Provider: ProviderGemini, - APIModel: "gemini-2.5-pro-preview-03-25", - CostPer1MIn: 1.25, - CostPer1MInCached: 0, - CostPer1MOutCached: 0, - CostPer1MOut: 10, - ContextWindow: 1000000, - DefaultMaxTokens: 50000, + ID: Gemini25, + Name: "Gemini 2.5 Pro", + Provider: ProviderGemini, + APIModel: "gemini-2.5-pro-preview-03-25", + CostPer1MIn: 1.25, + CostPer1MInCached: 0, + CostPer1MOutCached: 0, + CostPer1MOut: 10, + ContextWindow: 1000000, + DefaultMaxTokens: 50000, + SupportsAttachments: true, }, Gemini20Flash: { - ID: Gemini20Flash, - Name: "Gemini 2.0 Flash", - Provider: ProviderGemini, - APIModel: "gemini-2.0-flash", - CostPer1MIn: 0.10, - CostPer1MInCached: 0, - CostPer1MOutCached: 0, - CostPer1MOut: 0.40, - ContextWindow: 1000000, - DefaultMaxTokens: 6000, + ID: Gemini20Flash, + Name: "Gemini 2.0 Flash", + Provider: ProviderGemini, + APIModel: "gemini-2.0-flash", + CostPer1MIn: 0.10, + CostPer1MInCached: 0, + CostPer1MOutCached: 0, + CostPer1MOut: 0.40, + ContextWindow: 1000000, + DefaultMaxTokens: 6000, + SupportsAttachments: true, }, Gemini20FlashLite: { - ID: Gemini20FlashLite, - Name: "Gemini 2.0 Flash Lite", - Provider: ProviderGemini, - APIModel: "gemini-2.0-flash-lite", - CostPer1MIn: 0.05, - CostPer1MInCached: 0, - CostPer1MOutCached: 0, - CostPer1MOut: 0.30, - ContextWindow: 1000000, - DefaultMaxTokens: 6000, + ID: Gemini20FlashLite, + Name: "Gemini 2.0 Flash Lite", + Provider: ProviderGemini, + APIModel: "gemini-2.0-flash-lite", + CostPer1MIn: 0.05, + CostPer1MInCached: 0, + CostPer1MOutCached: 0, + CostPer1MOut: 0.30, + ContextWindow: 1000000, + DefaultMaxTokens: 6000, + SupportsAttachments: true, }, } diff --git a/internal/llm/models/groq.go b/internal/llm/models/groq.go index 749895b40..19917f20b 100644 --- a/internal/llm/models/groq.go +++ b/internal/llm/models/groq.go @@ -28,55 +28,60 @@ var GroqModels = map[ModelID]Model{ ContextWindow: 128_000, DefaultMaxTokens: 50000, // for some reason, the groq api doesn't like the reasoningEffort parameter - CanReason: false, + CanReason: false, + SupportsAttachments: false, }, Llama4Scout: { - ID: Llama4Scout, - Name: "Llama4Scout", - Provider: ProviderGROQ, - APIModel: "meta-llama/llama-4-scout-17b-16e-instruct", - CostPer1MIn: 0.11, - CostPer1MInCached: 0, - CostPer1MOutCached: 0, - CostPer1MOut: 0.34, - ContextWindow: 128_000, // 10M when? + ID: Llama4Scout, + Name: "Llama4Scout", + Provider: ProviderGROQ, + APIModel: "meta-llama/llama-4-scout-17b-16e-instruct", + CostPer1MIn: 0.11, + CostPer1MInCached: 0, + CostPer1MOutCached: 0, + CostPer1MOut: 0.34, + ContextWindow: 128_000, // 10M when? + SupportsAttachments: true, }, Llama4Maverick: { - ID: Llama4Maverick, - Name: "Llama4Maverick", - Provider: ProviderGROQ, - APIModel: "meta-llama/llama-4-maverick-17b-128e-instruct", - CostPer1MIn: 0.20, - CostPer1MInCached: 0, - CostPer1MOutCached: 0, - CostPer1MOut: 0.20, - ContextWindow: 128_000, + ID: Llama4Maverick, + Name: "Llama4Maverick", + Provider: ProviderGROQ, + APIModel: "meta-llama/llama-4-maverick-17b-128e-instruct", + CostPer1MIn: 0.20, + CostPer1MInCached: 0, + CostPer1MOutCached: 0, + CostPer1MOut: 0.20, + ContextWindow: 128_000, + SupportsAttachments: true, }, Llama3_3_70BVersatile: { - ID: Llama3_3_70BVersatile, - Name: "Llama3_3_70BVersatile", - Provider: ProviderGROQ, - APIModel: "llama-3.3-70b-versatile", - CostPer1MIn: 0.59, - CostPer1MInCached: 0, - CostPer1MOutCached: 0, - CostPer1MOut: 0.79, - ContextWindow: 128_000, + ID: Llama3_3_70BVersatile, + Name: "Llama3_3_70BVersatile", + Provider: ProviderGROQ, + APIModel: "llama-3.3-70b-versatile", + CostPer1MIn: 0.59, + CostPer1MInCached: 0, + CostPer1MOutCached: 0, + CostPer1MOut: 0.79, + ContextWindow: 128_000, + SupportsAttachments: false, }, DeepseekR1DistillLlama70b: { - ID: DeepseekR1DistillLlama70b, - Name: "DeepseekR1DistillLlama70b", - Provider: ProviderGROQ, - APIModel: "deepseek-r1-distill-llama-70b", - CostPer1MIn: 0.75, - CostPer1MInCached: 0, - CostPer1MOutCached: 0, - CostPer1MOut: 0.99, - ContextWindow: 128_000, - CanReason: true, + ID: DeepseekR1DistillLlama70b, + Name: "DeepseekR1DistillLlama70b", + Provider: ProviderGROQ, + APIModel: "deepseek-r1-distill-llama-70b", + CostPer1MIn: 0.75, + CostPer1MInCached: 0, + CostPer1MOutCached: 0, + CostPer1MOut: 0.99, + ContextWindow: 128_000, + CanReason: true, + SupportsAttachments: false, }, } diff --git a/internal/llm/models/models.go b/internal/llm/models/models.go index 3403bec2d..7fc90e7b8 100644 --- a/internal/llm/models/models.go +++ b/internal/llm/models/models.go @@ -8,17 +8,18 @@ type ( ) type Model struct { - ID ModelID `json:"id"` - Name string `json:"name"` - Provider ModelProvider `json:"provider"` - APIModel string `json:"api_model"` - CostPer1MIn float64 `json:"cost_per_1m_in"` - CostPer1MOut float64 `json:"cost_per_1m_out"` - CostPer1MInCached float64 `json:"cost_per_1m_in_cached"` - CostPer1MOutCached float64 `json:"cost_per_1m_out_cached"` - ContextWindow int64 `json:"context_window"` - DefaultMaxTokens int64 `json:"default_max_tokens"` - CanReason bool `json:"can_reason"` + ID ModelID `json:"id"` + Name string `json:"name"` + Provider ModelProvider `json:"provider"` + APIModel string `json:"api_model"` + CostPer1MIn float64 `json:"cost_per_1m_in"` + CostPer1MOut float64 `json:"cost_per_1m_out"` + CostPer1MInCached float64 `json:"cost_per_1m_in_cached"` + CostPer1MOutCached float64 `json:"cost_per_1m_out_cached"` + ContextWindow int64 `json:"context_window"` + DefaultMaxTokens int64 `json:"default_max_tokens"` + CanReason bool `json:"can_reason"` + SupportsAttachments bool `json:"supports_attachments"` } // Model IDs diff --git a/internal/llm/models/openai.go b/internal/llm/models/openai.go index f0cbb298c..abe0e30c5 100644 --- a/internal/llm/models/openai.go +++ b/internal/llm/models/openai.go @@ -19,151 +19,163 @@ const ( var OpenAIModels = map[ModelID]Model{ GPT41: { - ID: GPT41, - Name: "GPT 4.1", - Provider: ProviderOpenAI, - APIModel: "gpt-4.1", - CostPer1MIn: 2.00, - CostPer1MInCached: 0.50, - CostPer1MOutCached: 0.0, - CostPer1MOut: 8.00, - ContextWindow: 1_047_576, - DefaultMaxTokens: 20000, + ID: GPT41, + Name: "GPT 4.1", + Provider: ProviderOpenAI, + APIModel: "gpt-4.1", + CostPer1MIn: 2.00, + CostPer1MInCached: 0.50, + CostPer1MOutCached: 0.0, + CostPer1MOut: 8.00, + ContextWindow: 1_047_576, + DefaultMaxTokens: 20000, + SupportsAttachments: true, }, GPT41Mini: { - ID: GPT41Mini, - Name: "GPT 4.1 mini", - Provider: ProviderOpenAI, - APIModel: "gpt-4.1", - CostPer1MIn: 0.40, - CostPer1MInCached: 0.10, - CostPer1MOutCached: 0.0, - CostPer1MOut: 1.60, - ContextWindow: 200_000, - DefaultMaxTokens: 20000, + ID: GPT41Mini, + Name: "GPT 4.1 mini", + Provider: ProviderOpenAI, + APIModel: "gpt-4.1", + CostPer1MIn: 0.40, + CostPer1MInCached: 0.10, + CostPer1MOutCached: 0.0, + CostPer1MOut: 1.60, + ContextWindow: 200_000, + DefaultMaxTokens: 20000, + SupportsAttachments: true, }, GPT41Nano: { - ID: GPT41Nano, - Name: "GPT 4.1 nano", - Provider: ProviderOpenAI, - APIModel: "gpt-4.1-nano", - CostPer1MIn: 0.10, - CostPer1MInCached: 0.025, - CostPer1MOutCached: 0.0, - CostPer1MOut: 0.40, - ContextWindow: 1_047_576, - DefaultMaxTokens: 20000, + ID: GPT41Nano, + Name: "GPT 4.1 nano", + Provider: ProviderOpenAI, + APIModel: "gpt-4.1-nano", + CostPer1MIn: 0.10, + CostPer1MInCached: 0.025, + CostPer1MOutCached: 0.0, + CostPer1MOut: 0.40, + ContextWindow: 1_047_576, + DefaultMaxTokens: 20000, + SupportsAttachments: true, }, GPT45Preview: { - ID: GPT45Preview, - Name: "GPT 4.5 preview", - Provider: ProviderOpenAI, - APIModel: "gpt-4.5-preview", - CostPer1MIn: 75.00, - CostPer1MInCached: 37.50, - CostPer1MOutCached: 0.0, - CostPer1MOut: 150.00, - ContextWindow: 128_000, - DefaultMaxTokens: 15000, + ID: GPT45Preview, + Name: "GPT 4.5 preview", + Provider: ProviderOpenAI, + APIModel: "gpt-4.5-preview", + CostPer1MIn: 75.00, + CostPer1MInCached: 37.50, + CostPer1MOutCached: 0.0, + CostPer1MOut: 150.00, + ContextWindow: 128_000, + DefaultMaxTokens: 15000, + SupportsAttachments: true, }, GPT4o: { - ID: GPT4o, - Name: "GPT 4o", - Provider: ProviderOpenAI, - APIModel: "gpt-4o", - CostPer1MIn: 2.50, - CostPer1MInCached: 1.25, - CostPer1MOutCached: 0.0, - CostPer1MOut: 10.00, - ContextWindow: 128_000, - DefaultMaxTokens: 4096, + ID: GPT4o, + Name: "GPT 4o", + Provider: ProviderOpenAI, + APIModel: "gpt-4o", + CostPer1MIn: 2.50, + CostPer1MInCached: 1.25, + CostPer1MOutCached: 0.0, + CostPer1MOut: 10.00, + ContextWindow: 128_000, + DefaultMaxTokens: 4096, + SupportsAttachments: true, }, GPT4oMini: { - ID: GPT4oMini, - Name: "GPT 4o mini", - Provider: ProviderOpenAI, - APIModel: "gpt-4o-mini", - CostPer1MIn: 0.15, - CostPer1MInCached: 0.075, - CostPer1MOutCached: 0.0, - CostPer1MOut: 0.60, - ContextWindow: 128_000, + ID: GPT4oMini, + Name: "GPT 4o mini", + Provider: ProviderOpenAI, + APIModel: "gpt-4o-mini", + CostPer1MIn: 0.15, + CostPer1MInCached: 0.075, + CostPer1MOutCached: 0.0, + CostPer1MOut: 0.60, + ContextWindow: 128_000, + SupportsAttachments: true, }, O1: { - ID: O1, - Name: "O1", - Provider: ProviderOpenAI, - APIModel: "o1", - CostPer1MIn: 15.00, - CostPer1MInCached: 7.50, - CostPer1MOutCached: 0.0, - CostPer1MOut: 60.00, - ContextWindow: 200_000, - DefaultMaxTokens: 50000, - CanReason: true, + ID: O1, + Name: "O1", + Provider: ProviderOpenAI, + APIModel: "o1", + CostPer1MIn: 15.00, + CostPer1MInCached: 7.50, + CostPer1MOutCached: 0.0, + CostPer1MOut: 60.00, + ContextWindow: 200_000, + DefaultMaxTokens: 50000, + CanReason: true, + SupportsAttachments: true, }, O1Pro: { - ID: O1Pro, - Name: "o1 pro", - Provider: ProviderOpenAI, - APIModel: "o1-pro", - CostPer1MIn: 150.00, - CostPer1MInCached: 0.0, - CostPer1MOutCached: 0.0, - CostPer1MOut: 600.00, - ContextWindow: 200_000, - DefaultMaxTokens: 50000, - CanReason: true, + ID: O1Pro, + Name: "o1 pro", + Provider: ProviderOpenAI, + APIModel: "o1-pro", + CostPer1MIn: 150.00, + CostPer1MInCached: 0.0, + CostPer1MOutCached: 0.0, + CostPer1MOut: 600.00, + ContextWindow: 200_000, + DefaultMaxTokens: 50000, + CanReason: true, + SupportsAttachments: true, }, O1Mini: { - ID: O1Mini, - Name: "o1 mini", - Provider: ProviderOpenAI, - APIModel: "o1-mini", - CostPer1MIn: 1.10, - CostPer1MInCached: 0.55, - CostPer1MOutCached: 0.0, - CostPer1MOut: 4.40, - ContextWindow: 128_000, - DefaultMaxTokens: 50000, - CanReason: true, + ID: O1Mini, + Name: "o1 mini", + Provider: ProviderOpenAI, + APIModel: "o1-mini", + CostPer1MIn: 1.10, + CostPer1MInCached: 0.55, + CostPer1MOutCached: 0.0, + CostPer1MOut: 4.40, + ContextWindow: 128_000, + DefaultMaxTokens: 50000, + CanReason: true, + SupportsAttachments: true, }, O3: { - ID: O3, - Name: "o3", - Provider: ProviderOpenAI, - APIModel: "o3", - CostPer1MIn: 10.00, - CostPer1MInCached: 2.50, - CostPer1MOutCached: 0.0, - CostPer1MOut: 40.00, - ContextWindow: 200_000, - CanReason: true, + ID: O3, + Name: "o3", + Provider: ProviderOpenAI, + APIModel: "o3", + CostPer1MIn: 10.00, + CostPer1MInCached: 2.50, + CostPer1MOutCached: 0.0, + CostPer1MOut: 40.00, + ContextWindow: 200_000, + CanReason: true, + SupportsAttachments: true, }, O3Mini: { - ID: O3Mini, - Name: "o3 mini", - Provider: ProviderOpenAI, - APIModel: "o3-mini", - CostPer1MIn: 1.10, - CostPer1MInCached: 0.55, - CostPer1MOutCached: 0.0, - CostPer1MOut: 4.40, - ContextWindow: 200_000, - DefaultMaxTokens: 50000, - CanReason: true, + ID: O3Mini, + Name: "o3 mini", + Provider: ProviderOpenAI, + APIModel: "o3-mini", + CostPer1MIn: 1.10, + CostPer1MInCached: 0.55, + CostPer1MOutCached: 0.0, + CostPer1MOut: 4.40, + ContextWindow: 200_000, + DefaultMaxTokens: 50000, + CanReason: true, + SupportsAttachments: false, }, O4Mini: { - ID: O4Mini, - Name: "o4 mini", - Provider: ProviderOpenAI, - APIModel: "o4-mini", - CostPer1MIn: 1.10, - CostPer1MInCached: 0.275, - CostPer1MOutCached: 0.0, - CostPer1MOut: 4.40, - ContextWindow: 128_000, - DefaultMaxTokens: 50000, - CanReason: true, + ID: O4Mini, + Name: "o4 mini", + Provider: ProviderOpenAI, + APIModel: "o4-mini", + CostPer1MIn: 1.10, + CostPer1MInCached: 0.275, + CostPer1MOutCached: 0.0, + CostPer1MOut: 4.40, + ContextWindow: 128_000, + DefaultMaxTokens: 50000, + CanReason: true, + SupportsAttachments: true, }, } diff --git a/internal/llm/prompt/title.go b/internal/llm/prompt/title.go index 98570d3fa..956481520 100644 --- a/internal/llm/prompt/title.go +++ b/internal/llm/prompt/title.go @@ -8,5 +8,6 @@ func TitlePrompt(_ models.ModelProvider) string { - the title should be a summary of the user's message - it should be one line long - do not use quotes or colons -- the entire text you return will be used as the title` +- the entire text you return will be used as the title +- never return anything that is more than one sentence (one line) long` } diff --git a/internal/llm/provider/anthropic.go b/internal/llm/provider/anthropic.go index a96fe83ee..345904cf3 100644 --- a/internal/llm/provider/anthropic.go +++ b/internal/llm/provider/anthropic.go @@ -13,6 +13,7 @@ import ( "github.com/anthropics/anthropic-sdk-go/bedrock" "github.com/anthropics/anthropic-sdk-go/option" "github.com/opencode-ai/opencode/internal/config" + "github.com/opencode-ai/opencode/internal/llm/models" "github.com/opencode-ai/opencode/internal/llm/tools" "github.com/opencode-ai/opencode/internal/logging" "github.com/opencode-ai/opencode/internal/message" @@ -70,7 +71,14 @@ func (a *anthropicClient) convertMessages(messages []message.Message) (anthropic Type: "ephemeral", } } - anthropicMessages = append(anthropicMessages, anthropic.NewUserMessage(content)) + var contentBlocks []anthropic.ContentBlockParamUnion + contentBlocks = append(contentBlocks, content) + for _, binaryContent := range msg.BinaryContent() { + base64Image := binaryContent.String(models.ProviderAnthropic) + imageBlock := anthropic.NewImageBlockBase64(binaryContent.MIMEType, base64Image) + contentBlocks = append(contentBlocks, imageBlock) + } + anthropicMessages = append(anthropicMessages, anthropic.NewUserMessage(contentBlocks...)) case message.Assistant: blocks := []anthropic.ContentBlockParamUnion{} @@ -204,6 +212,7 @@ func (a *anthropicClient) send(ctx context.Context, messages []message.Message, jsonData, _ := json.Marshal(preparedMessages) logging.Debug("Prepared messages", "messages", string(jsonData)) } + attempts := 0 for { attempts++ @@ -213,6 +222,7 @@ func (a *anthropicClient) send(ctx context.Context, messages []message.Message, ) // If there is an error we are going to see if we can retry the call if err != nil { + logging.Error("Error in Anthropic API call", "error", err) retry, after, retryErr := a.shouldRetry(attempts, err) if retryErr != nil { return nil, retryErr diff --git a/internal/llm/provider/bedrock.go b/internal/llm/provider/bedrock.go index ca0d508c3..9f42e5b18 100644 --- a/internal/llm/provider/bedrock.go +++ b/internal/llm/provider/bedrock.go @@ -55,7 +55,7 @@ func newBedrockClient(opts providerClientOptions) BedrockClient { if strings.Contains(string(opts.model.APIModel), "anthropic") { // Create Anthropic client with Bedrock configuration anthropicOpts := opts - anthropicOpts.anthropicOptions = append(anthropicOpts.anthropicOptions, + anthropicOpts.anthropicOptions = append(anthropicOpts.anthropicOptions, WithAnthropicBedrock(true), WithAnthropicDisableCache(), ) @@ -84,7 +84,7 @@ func (b *bedrockClient) send(ctx context.Context, messages []message.Message, to func (b *bedrockClient) stream(ctx context.Context, messages []message.Message, tools []tools.BaseTool) <-chan ProviderEvent { eventChan := make(chan ProviderEvent) - + if b.childProvider == nil { go func() { eventChan <- ProviderEvent{ @@ -95,6 +95,7 @@ func (b *bedrockClient) stream(ctx context.Context, messages []message.Message, }() return eventChan } - + return b.childProvider.stream(ctx, messages, tools) -}
\ No newline at end of file +} + diff --git a/internal/llm/provider/gemini.go b/internal/llm/provider/gemini.go index d8fd6619f..9aee8e53a 100644 --- a/internal/llm/provider/gemini.go +++ b/internal/llm/provider/gemini.go @@ -57,11 +57,16 @@ func (g *geminiClient) convertMessages(messages []message.Message) []*genai.Cont for _, msg := range messages { switch msg.Role { case message.User: + var parts []genai.Part + parts = append(parts, genai.Text(msg.Content().String())) + for _, binaryContent := range msg.BinaryContent() { + imageFormat := strings.Split(binaryContent.MIMEType, "/") + parts = append(parts, genai.ImageData(imageFormat[1], binaryContent.Data)) + } history = append(history, &genai.Content{ - Parts: []genai.Part{genai.Text(msg.Content().String())}, + Parts: parts, Role: "user", }) - case message.Assistant: content := &genai.Content{ Role: "model", diff --git a/internal/llm/provider/openai.go b/internal/llm/provider/openai.go index d68cfbc2d..8a561c77b 100644 --- a/internal/llm/provider/openai.go +++ b/internal/llm/provider/openai.go @@ -12,6 +12,7 @@ import ( "github.com/openai/openai-go/option" "github.com/openai/openai-go/shared" "github.com/opencode-ai/opencode/internal/config" + "github.com/opencode-ai/opencode/internal/llm/models" "github.com/opencode-ai/opencode/internal/llm/tools" "github.com/opencode-ai/opencode/internal/logging" "github.com/opencode-ai/opencode/internal/message" @@ -71,7 +72,17 @@ func (o *openaiClient) convertMessages(messages []message.Message) (openaiMessag for _, msg := range messages { switch msg.Role { case message.User: - openaiMessages = append(openaiMessages, openai.UserMessage(msg.Content().String())) + var content []openai.ChatCompletionContentPartUnionParam + textBlock := openai.ChatCompletionContentPartTextParam{Text: msg.Content().String()} + content = append(content, openai.ChatCompletionContentPartUnionParam{OfText: &textBlock}) + for _, binaryContent := range msg.BinaryContent() { + imageURL := openai.ChatCompletionContentPartImageImageURLParam{URL: binaryContent.String(models.ProviderOpenAI)} + imageBlock := openai.ChatCompletionContentPartImageParam{ImageURL: imageURL} + + content = append(content, openai.ChatCompletionContentPartUnionParam{OfImageURL: &imageBlock}) + } + + openaiMessages = append(openaiMessages, openai.UserMessage(content)) case message.Assistant: assistantMsg := openai.ChatCompletionAssistantMessageParam{ |
