Skip to content

Commit 3ac7949

Browse files
jhayniejhaynie
authored
Normalize Anthropic system messages (#19)
* Normalize Anthropic system messages * Fix Anthropic system message edge cases
1 parent 96b78f3 commit 3ac7949

2 files changed

Lines changed: 306 additions & 0 deletions

File tree

‎autorouter.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1001,12 +1001,88 @@ func normalizeProviderRequest(raw map[string]any, providerName string) {
10011001
const defaultAnthropicMaxTokens = 1024
10021002

10031003
func normalizeAnthropicRequest(raw map[string]any) {
1004+
normalizeAnthropicSystemMessages(raw)
1005+
10041006
if hasPositiveNumber(raw["max_tokens"]) {
10051007
return
10061008
}
10071009
raw["max_tokens"] = defaultAnthropicMaxTokens
10081010
}
10091011

1012+
func normalizeAnthropicSystemMessages(raw map[string]any) {
1013+
messages, ok := raw["messages"].([]any)
1014+
if !ok || len(messages) == 0 {
1015+
return
1016+
}
1017+
1018+
filtered := make([]any, 0, len(messages))
1019+
systemParts := make([]any, 0, 1)
1020+
for _, item := range messages {
1021+
message, ok := item.(map[string]any)
1022+
if !ok {
1023+
filtered = append(filtered, item)
1024+
continue
1025+
}
1026+
role, _ := message["role"].(string)
1027+
if role != "system" {
1028+
filtered = append(filtered, item)
1029+
continue
1030+
}
1031+
if content, exists := message["content"]; exists {
1032+
systemParts = append(systemParts, content)
1033+
}
1034+
}
1035+
1036+
raw["messages"] = filtered
1037+
if len(systemParts) > 0 {
1038+
raw["system"] = mergeAnthropicSystem(raw["system"], systemParts)
1039+
}
1040+
}
1041+
1042+
func mergeAnthropicSystem(existing any, systemParts []any) any {
1043+
systemText := joinTextParts(systemParts)
1044+
if existing == nil {
1045+
if systemText != "" {
1046+
return systemText
1047+
}
1048+
return systemParts[0]
1049+
}
1050+
1051+
existingText := joinTextParts([]any{existing})
1052+
if existingText != "" && systemText != "" {
1053+
return existingText + "\n\n" + systemText
1054+
}
1055+
if systemText != "" {
1056+
return systemText
1057+
}
1058+
return existing
1059+
}
1060+
1061+
func joinTextParts(parts []any) string {
1062+
values := make([]string, 0, len(parts))
1063+
for _, part := range parts {
1064+
switch v := part.(type) {
1065+
case string:
1066+
if strings.TrimSpace(v) != "" {
1067+
values = append(values, v)
1068+
}
1069+
case []any:
1070+
for _, item := range v {
1071+
block, ok := item.(map[string]any)
1072+
if !ok {
1073+
continue
1074+
}
1075+
blockType, _ := block["type"].(string)
1076+
text, _ := block["text"].(string)
1077+
if blockType == "text" && strings.TrimSpace(text) != "" {
1078+
values = append(values, text)
1079+
}
1080+
}
1081+
}
1082+
}
1083+
return strings.Join(values, "\n\n")
1084+
}
1085+
10101086
func hasPositiveNumber(value any) bool {
10111087
switch v := value.(type) {
10121088
case int:

‎autorouter_test.go‎

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1506,6 +1506,236 @@ func TestAutoRouter_AnthropicPreservesMaxTokens(t *testing.T) {
15061506
}
15071507
}
15081508

1509+
func TestAutoRouter_AnthropicMovesSystemMessageToTopLevel(t *testing.T) {
1510+
var receivedBody map[string]any
1511+
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1512+
body, _ := io.ReadAll(r.Body)
1513+
json.Unmarshal(body, &receivedBody)
1514+
w.Header().Set("Content-Type", "application/json")
1515+
w.WriteHeader(http.StatusOK)
1516+
w.Write([]byte(`{"id":"msg_test","type":"message","model":"claude-3-opus","content":[{"type":"text","text":"Hello"}],"usage":{"input_tokens":8,"output_tokens":1}}`))
1517+
}))
1518+
defer upstream.Close()
1519+
1520+
provider := &mockProvider{
1521+
name: "anthropic",
1522+
parseFn: func(body io.ReadCloser) (BodyMetadata, []byte, error) {
1523+
data, _ := io.ReadAll(body)
1524+
return BodyMetadata{Model: "claude-3-opus"}, data, nil
1525+
},
1526+
enrichFn: func(req *http.Request, meta BodyMetadata, body []byte) error { return nil },
1527+
resolveFn: func(meta BodyMetadata) (*url.URL, error) {
1528+
return url.Parse(upstream.URL)
1529+
},
1530+
extractFn: func(resp *http.Response) (ResponseMetadata, []byte, error) {
1531+
body, _ := io.ReadAll(resp.Body)
1532+
return ResponseMetadata{ID: "msg_test"}, body, nil
1533+
},
1534+
}
1535+
1536+
router := NewAutoRouter(
1537+
WithAutoRouterDetector(ProviderDetectorFunc(func(hint ProviderHint) string { return "anthropic" })),
1538+
)
1539+
router.RegisterProvider(provider)
1540+
1541+
req := httptest.NewRequestWithContext(context.Background(), "POST", "/", bytes.NewReader([]byte(`{"model":"claude-3-opus","messages":[{"role":"system","content":"You are terse."},{"role":"user","content":"Hello"}]}`)))
1542+
req.Header.Set("Content-Type", "application/json")
1543+
w := httptest.NewRecorder()
1544+
1545+
router.ServeHTTP(w, req)
1546+
1547+
if w.Code != http.StatusOK {
1548+
t.Fatalf("StatusCode = %d, want 200", w.Code)
1549+
}
1550+
if got := receivedBody["system"]; got != "You are terse." {
1551+
t.Fatalf("system = %v, want %q", got, "You are terse.")
1552+
}
1553+
messages, ok := receivedBody["messages"].([]any)
1554+
if !ok {
1555+
t.Fatalf("messages = %T, want []any", receivedBody["messages"])
1556+
}
1557+
if len(messages) != 1 {
1558+
t.Fatalf("len(messages) = %d, want 1", len(messages))
1559+
}
1560+
message, ok := messages[0].(map[string]any)
1561+
if !ok {
1562+
t.Fatalf("messages[0] = %T, want map[string]any", messages[0])
1563+
}
1564+
if got := message["role"]; got != "user" {
1565+
t.Fatalf("messages[0].role = %v, want user", got)
1566+
}
1567+
if got := receivedBody["max_tokens"]; got != float64(defaultAnthropicMaxTokens) {
1568+
t.Fatalf("max_tokens = %v, want %d", got, defaultAnthropicMaxTokens)
1569+
}
1570+
}
1571+
1572+
func TestAutoRouter_AnthropicMergesSystemMessageWithExistingSystem(t *testing.T) {
1573+
var receivedBody map[string]any
1574+
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1575+
body, _ := io.ReadAll(r.Body)
1576+
json.Unmarshal(body, &receivedBody)
1577+
w.Header().Set("Content-Type", "application/json")
1578+
w.WriteHeader(http.StatusOK)
1579+
w.Write([]byte(`{"id":"msg_test","type":"message","model":"claude-3-opus","content":[{"type":"text","text":"Hello"}],"usage":{"input_tokens":8,"output_tokens":1}}`))
1580+
}))
1581+
defer upstream.Close()
1582+
1583+
provider := &mockProvider{
1584+
name: "anthropic",
1585+
parseFn: func(body io.ReadCloser) (BodyMetadata, []byte, error) {
1586+
data, _ := io.ReadAll(body)
1587+
return BodyMetadata{Model: "claude-3-opus"}, data, nil
1588+
},
1589+
enrichFn: func(req *http.Request, meta BodyMetadata, body []byte) error { return nil },
1590+
resolveFn: func(meta BodyMetadata) (*url.URL, error) {
1591+
return url.Parse(upstream.URL)
1592+
},
1593+
extractFn: func(resp *http.Response) (ResponseMetadata, []byte, error) {
1594+
body, _ := io.ReadAll(resp.Body)
1595+
return ResponseMetadata{ID: "msg_test"}, body, nil
1596+
},
1597+
}
1598+
1599+
router := NewAutoRouter(
1600+
WithAutoRouterDetector(ProviderDetectorFunc(func(hint ProviderHint) string { return "anthropic" })),
1601+
)
1602+
router.RegisterProvider(provider)
1603+
1604+
req := httptest.NewRequestWithContext(context.Background(), "POST", "/", bytes.NewReader([]byte(`{"model":"claude-3-opus","system":"Existing system.","messages":[{"role":"system","content":"Additional system."},{"role":"user","content":"Hello"}]}`)))
1605+
req.Header.Set("Content-Type", "application/json")
1606+
w := httptest.NewRecorder()
1607+
1608+
router.ServeHTTP(w, req)
1609+
1610+
if w.Code != http.StatusOK {
1611+
t.Fatalf("StatusCode = %d, want 200", w.Code)
1612+
}
1613+
if got := receivedBody["system"]; got != "Existing system.\n\nAdditional system." {
1614+
t.Fatalf("system = %v, want merged system", got)
1615+
}
1616+
messages, ok := receivedBody["messages"].([]any)
1617+
if !ok || len(messages) != 1 {
1618+
t.Fatalf("messages = %#v, want one non-system message", receivedBody["messages"])
1619+
}
1620+
}
1621+
1622+
func TestAutoRouter_AnthropicUsesSystemMessageWhenExistingSystemEmpty(t *testing.T) {
1623+
var receivedBody map[string]any
1624+
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1625+
body, _ := io.ReadAll(r.Body)
1626+
json.Unmarshal(body, &receivedBody)
1627+
w.Header().Set("Content-Type", "application/json")
1628+
w.WriteHeader(http.StatusOK)
1629+
w.Write([]byte(`{"id":"msg_test","type":"message","model":"claude-3-opus","content":[{"type":"text","text":"Hello"}],"usage":{"input_tokens":8,"output_tokens":1}}`))
1630+
}))
1631+
defer upstream.Close()
1632+
1633+
provider := &mockProvider{
1634+
name: "anthropic",
1635+
parseFn: func(body io.ReadCloser) (BodyMetadata, []byte, error) {
1636+
data, _ := io.ReadAll(body)
1637+
return BodyMetadata{Model: "claude-3-opus"}, data, nil
1638+
},
1639+
enrichFn: func(req *http.Request, meta BodyMetadata, body []byte) error { return nil },
1640+
resolveFn: func(meta BodyMetadata) (*url.URL, error) {
1641+
return url.Parse(upstream.URL)
1642+
},
1643+
extractFn: func(resp *http.Response) (ResponseMetadata, []byte, error) {
1644+
body, _ := io.ReadAll(resp.Body)
1645+
return ResponseMetadata{ID: "msg_test"}, body, nil
1646+
},
1647+
}
1648+
1649+
router := NewAutoRouter(
1650+
WithAutoRouterDetector(ProviderDetectorFunc(func(hint ProviderHint) string { return "anthropic" })),
1651+
)
1652+
router.RegisterProvider(provider)
1653+
1654+
req := httptest.NewRequestWithContext(context.Background(), "POST", "/", bytes.NewReader([]byte(`{"model":"claude-3-opus","system":"","messages":[{"role":"system","content":"Use terse answers."},{"role":"user","content":"Hello"}]}`)))
1655+
req.Header.Set("Content-Type", "application/json")
1656+
w := httptest.NewRecorder()
1657+
1658+
router.ServeHTTP(w, req)
1659+
1660+
if w.Code != http.StatusOK {
1661+
t.Fatalf("StatusCode = %d, want 200", w.Code)
1662+
}
1663+
if got := receivedBody["system"]; got != "Use terse answers." {
1664+
t.Fatalf("system = %v, want %q", got, "Use terse answers.")
1665+
}
1666+
messages, ok := receivedBody["messages"].([]any)
1667+
if !ok || len(messages) != 1 {
1668+
t.Fatalf("messages = %#v, want one non-system message", receivedBody["messages"])
1669+
}
1670+
message, ok := messages[0].(map[string]any)
1671+
if !ok {
1672+
t.Fatalf("messages[0] = %T, want map[string]any", messages[0])
1673+
}
1674+
if got := message["role"]; got != "user" {
1675+
t.Fatalf("messages[0].role = %v, want user", got)
1676+
}
1677+
if got := receivedBody["max_tokens"]; got != float64(defaultAnthropicMaxTokens) {
1678+
t.Fatalf("max_tokens = %v, want %d", got, defaultAnthropicMaxTokens)
1679+
}
1680+
}
1681+
1682+
func TestAutoRouter_AnthropicRemovesSystemMessageWithMissingContent(t *testing.T) {
1683+
var receivedBody map[string]any
1684+
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1685+
body, _ := io.ReadAll(r.Body)
1686+
json.Unmarshal(body, &receivedBody)
1687+
w.Header().Set("Content-Type", "application/json")
1688+
w.WriteHeader(http.StatusOK)
1689+
w.Write([]byte(`{"id":"msg_test","type":"message","model":"claude-3-opus","content":[{"type":"text","text":"Hello"}],"usage":{"input_tokens":8,"output_tokens":1}}`))
1690+
}))
1691+
defer upstream.Close()
1692+
1693+
provider := &mockProvider{
1694+
name: "anthropic",
1695+
parseFn: func(body io.ReadCloser) (BodyMetadata, []byte, error) {
1696+
data, _ := io.ReadAll(body)
1697+
return BodyMetadata{Model: "claude-3-opus"}, data, nil
1698+
},
1699+
enrichFn: func(req *http.Request, meta BodyMetadata, body []byte) error { return nil },
1700+
resolveFn: func(meta BodyMetadata) (*url.URL, error) {
1701+
return url.Parse(upstream.URL)
1702+
},
1703+
extractFn: func(resp *http.Response) (ResponseMetadata, []byte, error) {
1704+
body, _ := io.ReadAll(resp.Body)
1705+
return ResponseMetadata{ID: "msg_test"}, body, nil
1706+
},
1707+
}
1708+
1709+
router := NewAutoRouter(
1710+
WithAutoRouterDetector(ProviderDetectorFunc(func(hint ProviderHint) string { return "anthropic" })),
1711+
)
1712+
router.RegisterProvider(provider)
1713+
1714+
req := httptest.NewRequestWithContext(context.Background(), "POST", "/", bytes.NewReader([]byte(`{"model":"claude-3-opus","system":"Existing system.","messages":[{"role":"system"},{"role":"system","content":null},{"role":"user","content":"Hello"}]}`)))
1715+
req.Header.Set("Content-Type", "application/json")
1716+
w := httptest.NewRecorder()
1717+
1718+
router.ServeHTTP(w, req)
1719+
1720+
if w.Code != http.StatusOK {
1721+
t.Fatalf("StatusCode = %d, want 200", w.Code)
1722+
}
1723+
if got := receivedBody["system"]; got != "Existing system." {
1724+
t.Fatalf("system = %v, want existing system unchanged", got)
1725+
}
1726+
messages, ok := receivedBody["messages"].([]any)
1727+
if !ok || len(messages) != 1 {
1728+
t.Fatalf("messages = %#v, want one non-system message", receivedBody["messages"])
1729+
}
1730+
message, ok := messages[0].(map[string]any)
1731+
if !ok {
1732+
t.Fatalf("messages[0] = %T, want map[string]any", messages[0])
1733+
}
1734+
if got := message["role"]; got != "user" {
1735+
t.Fatalf("messages[0].role = %v, want user", got)
1736+
}
1737+
}
1738+
15091739
func TestAutoRouter_StreamingWritesGatewayMetadataEvent(t *testing.T) {
15101740
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
15111741
w.Header().Set("Content-Type", "text/event-stream")

0 commit comments

Comments
 (0)