refactor: add context prefix cache for response api

Author: zhuangjie1125

components/model/ark/chatmodel.go

@@ -246,7 +246,9 @@ func buildResponsesAPIChatModel(config *ChatModelConfig) (*responsesAPIChatModel
 	}
 	var opts []arkruntime.ConfigOption
 
-	if config.Region != "" {
+	if config.Region == "" {
+		opts = append(opts, arkruntime.WithRegion(defaultRegion))
+	} else {
 		opts = append(opts, arkruntime.WithRegion(config.Region))
 	}
 
@@ -503,12 +505,10 @@ func (cm *ChatModel) IsCallbacksEnabled() bool {
 //
 // Note:
 //   - It is unavailable for doubao models of version 1.6 and above.
-//   - Currently, only supports calling by ContextAPI.
 func (cm *ChatModel) CreatePrefixCache(ctx context.Context, prefix []*schema.Message, ttl int) (info *CacheInfo, err error) {
 	if cm.respChatModel.cache != nil && ptrFromOrZero(cm.respChatModel.cache.APIType) == ResponsesAPI {
-		return nil, fmt.Errorf("CreatePrefixCache is not supported by ResponsesAPI")
+		return cm.respChatModel.createContextByResponseAPI(ctx, prefix, ttl)
 	}
-
 	return cm.createContextByContextAPI(ctx, prefix, ttl, model.ContextModeCommonPrefix, nil)
 }
 

components/model/ark/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/openai/openai-go v1.10.1
 	github.com/smartystreets/goconvey v1.8.1
 	github.com/stretchr/testify v1.11.1
-	github.com/volcengine/volcengine-go-sdk v1.1.47
+	github.com/volcengine/volcengine-go-sdk v1.1.48
 
 )
 

components/model/ark/go.sum

@@ -157,6 +157,8 @@ github.com/volcengine/volcengine-go-sdk v1.1.44 h1:WLoLlzt67ZlJeow55PPx65/Mh52De
 github.com/volcengine/volcengine-go-sdk v1.1.44/go.mod h1:oxoVo+A17kvkwPkIeIHPVLjSw7EQAm+l/Vau1YGHN+A=
 github.com/volcengine/volcengine-go-sdk v1.1.47 h1:aSHWIfo0fKhG+V6IO76W9Cf+1fzp3pykNoT5ulBVMsw=
 github.com/volcengine/volcengine-go-sdk v1.1.47/go.mod h1:oxoVo+A17kvkwPkIeIHPVLjSw7EQAm+l/Vau1YGHN+A=
+github.com/volcengine/volcengine-go-sdk v1.1.48 h1:eHffKYHLUMKdZDmUQgmD1+dEzoVTIsiSlNNdnmEBs8A=
+github.com/volcengine/volcengine-go-sdk v1.1.48/go.mod h1:oxoVo+A17kvkwPkIeIHPVLjSw7EQAm+l/Vau1YGHN+A=
 github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
 github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=

components/model/ark/responses_api.go

@@ -98,15 +98,18 @@ func (cm *responsesAPIChatModel) Generate(ctx context.Context, input []*schema.M
 		cacheCfg.Enabled = responseReq.Caching.Type.String() == responses.CacheType_enabled.String()
 		cacheCfg.ExpireAt = responseReq.ExpireAt
 	}
-	msg, err := cm.toOutputMessage(responseObject, cacheCfg)
 
+	outMsg, err = cm.toOutputMessage(responseObject, cacheCfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert output to schema.Message: %w", err)
+	}
 	callbacks.OnEnd(ctx, &model.CallbackOutput{
 		Message:    outMsg,
 		Config:     config,
 		TokenUsage: cm.toModelTokenUsage(responseObject.Usage),
 		Extra:      map[string]any{callbackExtraKeyThinking: specOptions.thinking},
 	})
-	return msg, nil
+	return outMsg, nil
 
 }
 
@@ -1014,3 +1017,30 @@ func ensureDataURL(dataOfBase64, mimeType string) (string, error) {
 	}
 	return fmt.Sprintf("data:%s;base64,%s", mimeType, dataOfBase64), nil
 }
+
+func (cm *responsesAPIChatModel) createContextByResponseAPI(ctx context.Context, prefix []*schema.Message, ttl int) (info *CacheInfo, err error) {
+	responseReq := &responses.ResponsesRequest{
+		Model:    cm.model,
+		ExpireAt: ptrOf(time.Now().Unix() + int64(ttl)),
+		Store:    ptrOf(true),
+		Caching: &responses.ResponsesCaching{
+			Type:   responses.CacheType_enabled.Enum(),
+			Prefix: ptrOf(true),
+		},
+	}
+	err = cm.populateInput(prefix, responseReq)
+	if err != nil {
+		return nil, err
+	}
+	responseObject, err := cm.client.CreateResponses(ctx, responseReq)
+	if err != nil {
+		return nil, err
+	}
+
+	info = &CacheInfo{
+		ContextID: responseObject.Id,
+		Usage:     *cm.toEinoTokenUsage(responseObject.Usage),
+	}
+
+	return info, nil
+}