From 5207bb8653fde1b7aa7553ea3d44c3bcf1b0f51d Mon Sep 17 00:00:00 2001 From: Sean Goedecke Date: Thu, 17 Jul 2025 00:11:06 +0000 Subject: [PATCH 1/3] Support responseFormat in prompt.yml files --- cmd/eval/eval_test.go | 93 ++++++++++++++-- cmd/run/run.go | 14 ++- cmd/run/run_test.go | 96 +++++++++++++++++ examples/json_response_prompt.yml | 19 ++++ examples/json_schema_prompt.yml | 61 +++++++++++ internal/azuremodels/types.go | 27 +++-- pkg/prompt/prompt.go | 82 ++++++++++++++ pkg/prompt/prompt_test.go | 172 ++++++++++++++++++++++++++++++ 8 files changed, 543 insertions(+), 21 deletions(-) create mode 100644 examples/json_response_prompt.yml create mode 100644 examples/json_schema_prompt.yml diff --git a/cmd/eval/eval_test.go b/cmd/eval/eval_test.go index 78b67439..2909959f 100644 --- a/cmd/eval/eval_test.go +++ b/cmd/eval/eval_test.go @@ -511,6 +511,7 @@ description: Testing JSON with failing evaluators model: openai/gpt-4o testData: - input: "hello" + expected: "hello world" messages: - role: user content: "{{input}}" @@ -553,18 +554,94 @@ evaluators: output := out.String() + // Verify JSON structure var result EvaluationSummary err = json.Unmarshal([]byte(output), &result) require.NoError(t, err) - // Verify failing test is properly represented - require.Equal(t, 1, result.Summary.TotalTests) - require.Equal(t, 0, result.Summary.PassedTests) - require.Equal(t, 1, result.Summary.FailedTests) - require.Equal(t, 0.0, result.Summary.PassRate) + // Verify JSON doesn't contain human-readable text + require.NotContains(t, output, "Running evaluation:") + }) + + t.Run("eval with responseFormat and jsonSchema", func(t *testing.T) { + const yamlBody = ` +name: JSON Schema Evaluation +description: Testing responseFormat and jsonSchema in eval +model: openai/gpt-4o +responseFormat: json_schema +jsonSchema: + name: response_schema + strict: true + schema: + type: object + properties: + message: + type: string + description: The response message + confidence: + type: number + description: Confidence score + required: + - message + additionalProperties: false +testData: + - input: "hello" + expected: "hello world" +messages: + - role: user + content: "Respond to: {{input}}" +evaluators: + - name: contains-message + string: + contains: "message" +` - require.Len(t, result.TestResults, 1) - require.False(t, result.TestResults[0].EvaluationResults[0].Passed) - require.Equal(t, 0.0, result.TestResults[0].EvaluationResults[0].Score) + tmpDir := t.TempDir() + promptFile := filepath.Join(tmpDir, "test.prompt.yml") + err := os.WriteFile(promptFile, []byte(yamlBody), 0644) + require.NoError(t, err) + + client := azuremodels.NewMockClient() + var capturedRequest azuremodels.ChatCompletionOptions + client.MockGetChatCompletionStream = func(ctx context.Context, req azuremodels.ChatCompletionOptions, org string) (*azuremodels.ChatCompletionResponse, error) { + capturedRequest = req + response := `{"message": "hello world", "confidence": 0.95}` + reader := sse.NewMockEventReader([]azuremodels.ChatCompletion{ + { + Choices: []azuremodels.ChatChoice{ + { + Message: &azuremodels.ChatChoiceMessage{ + Content: &response, + }, + }, + }, + }, + }) + return &azuremodels.ChatCompletionResponse{Reader: reader}, nil + } + + out := new(bytes.Buffer) + cfg := command.NewConfig(out, out, client, true, 100) + + cmd := NewEvalCommand(cfg) + cmd.SetArgs([]string{promptFile}) + + err = cmd.Execute() + require.NoError(t, err) + + // Verify that responseFormat and jsonSchema were included in the request + require.NotNil(t, capturedRequest.ResponseFormat) + require.Equal(t, "json_schema", capturedRequest.ResponseFormat.Type) + require.NotNil(t, capturedRequest.ResponseFormat.JsonSchema) + + schema := *capturedRequest.ResponseFormat.JsonSchema + require.Equal(t, "response_schema", schema["name"]) + require.Equal(t, true, schema["strict"]) + require.Contains(t, schema, "schema") + + // Verify the test passed + output := out.String() + require.Contains(t, output, "✓ PASSED") + require.Contains(t, output, "🎉 All tests passed!") }) } diff --git a/cmd/run/run.go b/cmd/run/run.go index 1fe574b2..134b9140 100644 --- a/cmd/run/run.go +++ b/cmd/run/run.go @@ -351,9 +351,17 @@ func NewRunCommand(cfg *command.Config) *cobra.Command { } } - req := azuremodels.ChatCompletionOptions{ - Messages: conversation.GetMessages(), - Model: modelName, + var req azuremodels.ChatCompletionOptions + if pf != nil { + // Use the prompt file's BuildChatCompletionOptions method to include responseFormat and jsonSchema + req = pf.BuildChatCompletionOptions(conversation.GetMessages()) + // Override the model name if provided via CLI + req.Model = modelName + } else { + req = azuremodels.ChatCompletionOptions{ + Messages: conversation.GetMessages(), + Model: modelName, + } } mp.UpdateRequest(&req) diff --git a/cmd/run/run_test.go b/cmd/run/run_test.go index eb10649c..a1802a16 100644 --- a/cmd/run/run_test.go +++ b/cmd/run/run_test.go @@ -331,6 +331,102 @@ messages: require.Equal(t, "System message", *capturedReq.Messages[0].Content) require.Equal(t, "User message", *capturedReq.Messages[1].Content) }) + + t.Run("--file with responseFormat and jsonSchema", func(t *testing.T) { + const yamlBody = ` +name: JSON Schema Test +description: Test responseFormat and jsonSchema +model: openai/test-model +responseFormat: json_schema +jsonSchema: + name: person_schema + strict: true + schema: + type: object + properties: + name: + type: string + description: The name + age: + type: integer + description: The age + required: + - name + - age + additionalProperties: false +messages: + - role: system + content: You are a helpful assistant. + - role: user + content: "Generate a person" +` + + tmp, err := os.CreateTemp(t.TempDir(), "*.prompt.yml") + require.NoError(t, err) + _, err = tmp.WriteString(yamlBody) + require.NoError(t, err) + require.NoError(t, tmp.Close()) + + client := azuremodels.NewMockClient() + modelSummary := &azuremodels.ModelSummary{ + Name: "test-model", + Publisher: "openai", + Task: "chat-completion", + } + client.MockListModels = func(ctx context.Context) ([]*azuremodels.ModelSummary, error) { + return []*azuremodels.ModelSummary{modelSummary}, nil + } + + var capturedRequest azuremodels.ChatCompletionOptions + client.MockGetChatCompletionStream = func(ctx context.Context, req azuremodels.ChatCompletionOptions, org string) (*azuremodels.ChatCompletionResponse, error) { + capturedRequest = req + reply := "hello this is a test response" + reader := sse.NewMockEventReader([]azuremodels.ChatCompletion{ + { + Choices: []azuremodels.ChatChoice{ + { + Message: &azuremodels.ChatChoiceMessage{ + Content: &reply, + }, + }, + }, + }, + }) + return &azuremodels.ChatCompletionResponse{Reader: reader}, nil + } + + out := new(bytes.Buffer) + cfg := command.NewConfig(out, out, client, true, 100) + + cmd := NewRunCommand(cfg) + cmd.SetArgs([]string{"--file", tmp.Name()}) + + err = cmd.Execute() + require.NoError(t, err) + + // Verify that responseFormat and jsonSchema were included in the request + require.NotNil(t, capturedRequest.ResponseFormat) + require.Equal(t, "json_schema", capturedRequest.ResponseFormat.Type) + require.NotNil(t, capturedRequest.ResponseFormat.JsonSchema) + + schema := *capturedRequest.ResponseFormat.JsonSchema + require.Contains(t, schema, "name") + require.Contains(t, schema, "schema") + require.Equal(t, "person_schema", schema["name"]) + + schemaContent := schema["schema"].(map[string]interface{}) + require.Equal(t, "object", schemaContent["type"]) + require.Contains(t, schemaContent, "properties") + require.Contains(t, schemaContent, "required") + + properties := schemaContent["properties"].(map[string]interface{}) + require.Contains(t, properties, "name") + require.Contains(t, properties, "age") + + required := schemaContent["required"].([]interface{}) + require.Contains(t, required, "name") + require.Contains(t, required, "age") + }) } func TestParseTemplateVariables(t *testing.T) { diff --git a/examples/json_response_prompt.yml b/examples/json_response_prompt.yml new file mode 100644 index 00000000..134f8dee --- /dev/null +++ b/examples/json_response_prompt.yml @@ -0,0 +1,19 @@ +name: JSON Response Example +description: Example prompt demonstrating responseFormat with json +model: openai/gpt-4o +responseFormat: json +messages: + - role: system + content: You are a helpful assistant that responds in JSON format. + - role: user + content: "Provide a summary of {{topic}} in JSON format with title, description, and key_points array." +testData: + - topic: "artificial intelligence" + - topic: "climate change" +evaluators: + - name: contains-json-structure + string: + contains: "{" + - name: has-title + string: + contains: "title" diff --git a/examples/json_schema_prompt.yml b/examples/json_schema_prompt.yml new file mode 100644 index 00000000..c8a33fdb --- /dev/null +++ b/examples/json_schema_prompt.yml @@ -0,0 +1,61 @@ +name: JSON Schema Response Example +description: Example prompt demonstrating responseFormat and jsonSchema usage +model: openai/gpt-4o +responseFormat: json_schema +jsonSchema: + type: object + description: A structured response containing person information + properties: + name: + type: string + description: The full name of the person + age: + type: integer + description: The age of the person in years + minimum: 0 + maximum: 150 + email: + type: string + description: The email address of the person + format: email + skills: + type: array + description: A list of skills the person has + items: + type: string + address: + type: object + description: The person's address + properties: + street: + type: string + description: Street address + city: + type: string + description: City name + country: + type: string + description: Country name + required: + - city + - country + required: + - name + - age +messages: + - role: system + content: You are a helpful assistant that provides structured information about people. + - role: user + content: "Generate information for a person named {{name}} who is {{age}} years old." +testData: + - name: "Alice Johnson" + age: "30" + - name: "Bob Smith" + age: "25" +evaluators: + - name: has-required-fields + string: + contains: "name" + - name: valid-json-structure + string: + contains: "age" diff --git a/internal/azuremodels/types.go b/internal/azuremodels/types.go index 29d4a7d1..32a93096 100644 --- a/internal/azuremodels/types.go +++ b/internal/azuremodels/types.go @@ -6,6 +6,23 @@ import ( "github.com/github/gh-models/internal/sse" ) +// ChatCompletionOptions represents available options for a chat completion request. +type ChatCompletionOptions struct { + MaxTokens *int `json:"max_tokens,omitempty"` + Messages []ChatMessage `json:"messages"` + Model string `json:"model"` + Stream bool `json:"stream,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + ResponseFormat *ResponseFormat `json:"response_format,omitempty"` +} + +// ResponseFormat represents the response format specification +type ResponseFormat struct { + Type string `json:"type"` + JsonSchema *map[string]interface{} `json:"json_schema,omitempty"` +} + // ChatMessageRole represents the role of a chat message. type ChatMessageRole string @@ -24,16 +41,6 @@ type ChatMessage struct { Role ChatMessageRole `json:"role"` } -// ChatCompletionOptions represents available options for a chat completion request. -type ChatCompletionOptions struct { - MaxTokens *int `json:"max_tokens,omitempty"` - Messages []ChatMessage `json:"messages"` - Model string `json:"model"` - Stream bool `json:"stream,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` -} - // ChatChoiceMessage is a message from a choice in a chat conversation. type ChatChoiceMessage struct { Content *string `json:"content,omitempty"` diff --git a/pkg/prompt/prompt.go b/pkg/prompt/prompt.go index 75a805c7..7e21a939 100644 --- a/pkg/prompt/prompt.go +++ b/pkg/prompt/prompt.go @@ -16,6 +16,8 @@ type File struct { Description string `yaml:"description"` Model string `yaml:"model"` ModelParameters ModelParameters `yaml:"modelParameters"` + ResponseFormat *string `yaml:"responseFormat,omitempty"` + JsonSchema *JsonSchema `yaml:"jsonSchema,omitempty"` Messages []Message `yaml:"messages"` // TestData and Evaluators are only used by eval command TestData []map[string]interface{} `yaml:"testData,omitempty"` @@ -65,6 +67,19 @@ type Choice struct { Score float64 `yaml:"score"` } +// JsonSchema represents a JSON schema for structured responses +type JsonSchema struct { + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Strict *bool `yaml:"strict,omitempty" json:"strict,omitempty"` + Schema map[string]interface{} `yaml:"schema,omitempty" json:"schema,omitempty"` + // Legacy fields for backward compatibility + Type string `yaml:"type,omitempty" json:"type,omitempty"` + Properties map[string]interface{} `yaml:"properties,omitempty" json:"properties,omitempty"` + Required []string `yaml:"required,omitempty" json:"required,omitempty"` + Items interface{} `yaml:"items,omitempty" json:"items,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` +} + // LoadFromFile loads and parses a prompt file from the given path func LoadFromFile(filePath string) (*File, error) { data, err := os.ReadFile(filePath) @@ -77,9 +92,35 @@ func LoadFromFile(filePath string) (*File, error) { return nil, err } + // Validate responseFormat if provided + if err := promptFile.validateResponseFormat(); err != nil { + return nil, err + } + return &promptFile, nil } +// validateResponseFormat validates the responseFormat field +func (f *File) validateResponseFormat() error { + if f.ResponseFormat == nil { + return nil + } + + switch *f.ResponseFormat { + case "text", "json_object", "json_schema", "guidance": + // Valid values + default: + return fmt.Errorf("invalid responseFormat: %s. Must be 'text', 'json_object', 'json_schema', or 'guidance'", *f.ResponseFormat) + } + + // If responseFormat is "json_schema", jsonSchema must be provided + if *f.ResponseFormat == "json_schema" && f.JsonSchema == nil { + return fmt.Errorf("jsonSchema is required when responseFormat is 'json_schema'") + } + + return nil +} + // TemplateString templates a string with the given data using simple {{variable}} replacement func TemplateString(templateStr string, data interface{}) (string, error) { result := templateStr @@ -146,5 +187,46 @@ func (f *File) BuildChatCompletionOptions(messages []azuremodels.ChatMessage) az req.TopP = f.ModelParameters.TopP } + // Apply response format + if f.ResponseFormat != nil { + responseFormat := &azuremodels.ResponseFormat{ + Type: *f.ResponseFormat, + } + if f.JsonSchema != nil { + // Convert JsonSchema to map[string]interface{} + schemaMap := make(map[string]interface{}) + + // Use new format if available (name + schema) + if f.JsonSchema.Name != "" { + schemaMap["name"] = f.JsonSchema.Name + if f.JsonSchema.Strict != nil { + schemaMap["strict"] = *f.JsonSchema.Strict + } + if f.JsonSchema.Schema != nil { + schemaMap["schema"] = f.JsonSchema.Schema + } + } else { + // Fall back to legacy format + if f.JsonSchema.Type != "" { + schemaMap["type"] = f.JsonSchema.Type + } + if f.JsonSchema.Properties != nil { + schemaMap["properties"] = f.JsonSchema.Properties + } + if f.JsonSchema.Required != nil { + schemaMap["required"] = f.JsonSchema.Required + } + if f.JsonSchema.Items != nil { + schemaMap["items"] = f.JsonSchema.Items + } + if f.JsonSchema.Description != "" { + schemaMap["description"] = f.JsonSchema.Description + } + } + responseFormat.JsonSchema = &schemaMap + } + req.ResponseFormat = responseFormat + } + return req } diff --git a/pkg/prompt/prompt_test.go b/pkg/prompt/prompt_test.go index a6ef1264..31066b3b 100644 --- a/pkg/prompt/prompt_test.go +++ b/pkg/prompt/prompt_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/github/gh-models/internal/azuremodels" "github.com/stretchr/testify/require" ) @@ -91,4 +92,175 @@ evaluators: _, err = LoadFromFile(promptFilePath) require.Error(t, err) }) + + t.Run("loads prompt file with responseFormat text", func(t *testing.T) { + const yamlBody = ` +name: Text Response Format Test +description: Test with text response format +model: openai/gpt-4o +responseFormat: text +messages: + - role: user + content: "Hello" +` + + tmpDir := t.TempDir() + promptFilePath := filepath.Join(tmpDir, "test.prompt.yml") + err := os.WriteFile(promptFilePath, []byte(yamlBody), 0644) + require.NoError(t, err) + + promptFile, err := LoadFromFile(promptFilePath) + require.NoError(t, err) + require.NotNil(t, promptFile.ResponseFormat) + require.Equal(t, "text", *promptFile.ResponseFormat) + require.Nil(t, promptFile.JsonSchema) + }) + + t.Run("loads prompt file with responseFormat json_object", func(t *testing.T) { + const yamlBody = ` +name: JSON Object Response Format Test +description: Test with JSON object response format +model: openai/gpt-4o +responseFormat: json_object +messages: + - role: user + content: "Hello" +` + + tmpDir := t.TempDir() + promptFilePath := filepath.Join(tmpDir, "test.prompt.yml") + err := os.WriteFile(promptFilePath, []byte(yamlBody), 0644) + require.NoError(t, err) + + promptFile, err := LoadFromFile(promptFilePath) + require.NoError(t, err) + require.NotNil(t, promptFile.ResponseFormat) + require.Equal(t, "json_object", *promptFile.ResponseFormat) + require.Nil(t, promptFile.JsonSchema) + }) + + t.Run("loads prompt file with responseFormat json_schema and jsonSchema", func(t *testing.T) { + const yamlBody = ` +name: JSON Schema Response Format Test +description: Test with JSON schema response format +model: openai/gpt-4o +responseFormat: json_schema +jsonSchema: + name: person_info + strict: true + schema: + type: object + properties: + name: + type: string + description: The name of the person + age: + type: integer + description: The age of the person + required: + - name + additionalProperties: false +messages: + - role: user + content: "Hello" +` + + tmpDir := t.TempDir() + promptFilePath := filepath.Join(tmpDir, "test.prompt.yml") + err := os.WriteFile(promptFilePath, []byte(yamlBody), 0644) + require.NoError(t, err) + + promptFile, err := LoadFromFile(promptFilePath) + require.NoError(t, err) + require.NotNil(t, promptFile.ResponseFormat) + require.Equal(t, "json_schema", *promptFile.ResponseFormat) + require.NotNil(t, promptFile.JsonSchema) + require.Equal(t, "person_info", promptFile.JsonSchema.Name) + require.True(t, *promptFile.JsonSchema.Strict) + require.Contains(t, promptFile.JsonSchema.Schema, "type") + require.Contains(t, promptFile.JsonSchema.Schema, "properties") + }) + + t.Run("validates invalid responseFormat", func(t *testing.T) { + const yamlBody = ` +name: Invalid Response Format Test +description: Test with invalid response format +model: openai/gpt-4o +responseFormat: invalid_format +messages: + - role: user + content: "Hello" +` + + tmpDir := t.TempDir() + promptFilePath := filepath.Join(tmpDir, "test.prompt.yml") + err := os.WriteFile(promptFilePath, []byte(yamlBody), 0644) + require.NoError(t, err) + + _, err = LoadFromFile(promptFilePath) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid responseFormat: invalid_format") + }) + + t.Run("validates json_schema requires jsonSchema", func(t *testing.T) { + const yamlBody = ` +name: JSON Schema Missing Test +description: Test json_schema without jsonSchema +model: openai/gpt-4o +responseFormat: json_schema +messages: + - role: user + content: "Hello" +` + + tmpDir := t.TempDir() + promptFilePath := filepath.Join(tmpDir, "test.prompt.yml") + err := os.WriteFile(promptFilePath, []byte(yamlBody), 0644) + require.NoError(t, err) + + _, err = LoadFromFile(promptFilePath) + require.Error(t, err) + require.Contains(t, err.Error(), "jsonSchema is required when responseFormat is 'json_schema'") + }) + + t.Run("BuildChatCompletionOptions includes responseFormat and jsonSchema", func(t *testing.T) { + promptFile := &File{ + Model: "openai/gpt-4o", + ResponseFormat: func() *string { s := "json_schema"; return &s }(), + JsonSchema: &JsonSchema{ + Name: "test_schema", + Strict: func() *bool { b := true; return &b }(), + Schema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "name": map[string]interface{}{ + "type": "string", + "description": "The name", + }, + }, + "required": []string{"name"}, + }, + }, + } + + messages := []azuremodels.ChatMessage{ + { + Role: azuremodels.ChatMessageRoleUser, + Content: func() *string { s := "Hello"; return &s }(), + }, + } + options := promptFile.BuildChatCompletionOptions(messages) + require.NotNil(t, options.ResponseFormat) + require.Equal(t, "json_schema", options.ResponseFormat.Type) + require.NotNil(t, options.ResponseFormat.JsonSchema) + + schema := *options.ResponseFormat.JsonSchema + require.Equal(t, "test_schema", schema["name"]) + require.Equal(t, true, schema["strict"]) + require.Contains(t, schema, "schema") + + schemaContent := schema["schema"].(map[string]interface{}) + require.Equal(t, "object", schemaContent["type"]) + require.Contains(t, schemaContent, "properties") + }) } From 4527131f63b9b5e9d1982c51d05ad875cce7f58b Mon Sep 17 00:00:00 2001 From: Sean Goedecke Date: Thu, 17 Jul 2025 00:22:12 +0000 Subject: [PATCH 2/3] Fixup format parsing --- examples/json_response_prompt.yml | 2 +- pkg/prompt/prompt.go | 61 ++++++++++--------------------- 2 files changed, 20 insertions(+), 43 deletions(-) diff --git a/examples/json_response_prompt.yml b/examples/json_response_prompt.yml index 134f8dee..e6cd206b 100644 --- a/examples/json_response_prompt.yml +++ b/examples/json_response_prompt.yml @@ -1,7 +1,7 @@ name: JSON Response Example description: Example prompt demonstrating responseFormat with json model: openai/gpt-4o -responseFormat: json +responseFormat: json_object messages: - role: system content: You are a helpful assistant that responds in JSON format. diff --git a/pkg/prompt/prompt.go b/pkg/prompt/prompt.go index 7e21a939..de60c4c3 100644 --- a/pkg/prompt/prompt.go +++ b/pkg/prompt/prompt.go @@ -69,15 +69,9 @@ type Choice struct { // JsonSchema represents a JSON schema for structured responses type JsonSchema struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` + Name string `yaml:"name" json:"name"` Strict *bool `yaml:"strict,omitempty" json:"strict,omitempty"` - Schema map[string]interface{} `yaml:"schema,omitempty" json:"schema,omitempty"` - // Legacy fields for backward compatibility - Type string `yaml:"type,omitempty" json:"type,omitempty"` - Properties map[string]interface{} `yaml:"properties,omitempty" json:"properties,omitempty"` - Required []string `yaml:"required,omitempty" json:"required,omitempty"` - Items interface{} `yaml:"items,omitempty" json:"items,omitempty"` - Description string `yaml:"description,omitempty" json:"description,omitempty"` + Schema map[string]interface{} `yaml:"schema" json:"schema"` } // LoadFromFile loads and parses a prompt file from the given path @@ -92,7 +86,6 @@ func LoadFromFile(filePath string) (*File, error) { return nil, err } - // Validate responseFormat if provided if err := promptFile.validateResponseFormat(); err != nil { return nil, err } @@ -107,15 +100,22 @@ func (f *File) validateResponseFormat() error { } switch *f.ResponseFormat { - case "text", "json_object", "json_schema", "guidance": - // Valid values + case "text", "json_object", "json_schema": default: - return fmt.Errorf("invalid responseFormat: %s. Must be 'text', 'json_object', 'json_schema', or 'guidance'", *f.ResponseFormat) + return fmt.Errorf("invalid responseFormat: %s. Must be 'text', 'json_object', or 'json_schema'", *f.ResponseFormat) } - // If responseFormat is "json_schema", jsonSchema must be provided - if *f.ResponseFormat == "json_schema" && f.JsonSchema == nil { - return fmt.Errorf("jsonSchema is required when responseFormat is 'json_schema'") + // If responseFormat is "json_schema", jsonSchema must be provided with required fields + if *f.ResponseFormat == "json_schema" { + if f.JsonSchema == nil { + return fmt.Errorf("jsonSchema is required when responseFormat is 'json_schema'") + } + if f.JsonSchema.Name == "" { + return fmt.Errorf("jsonSchema.name is required when responseFormat is 'json_schema'") + } + if f.JsonSchema.Schema == nil { + return fmt.Errorf("jsonSchema.schema is required when responseFormat is 'json_schema'") + } } return nil @@ -195,34 +195,11 @@ func (f *File) BuildChatCompletionOptions(messages []azuremodels.ChatMessage) az if f.JsonSchema != nil { // Convert JsonSchema to map[string]interface{} schemaMap := make(map[string]interface{}) - - // Use new format if available (name + schema) - if f.JsonSchema.Name != "" { - schemaMap["name"] = f.JsonSchema.Name - if f.JsonSchema.Strict != nil { - schemaMap["strict"] = *f.JsonSchema.Strict - } - if f.JsonSchema.Schema != nil { - schemaMap["schema"] = f.JsonSchema.Schema - } - } else { - // Fall back to legacy format - if f.JsonSchema.Type != "" { - schemaMap["type"] = f.JsonSchema.Type - } - if f.JsonSchema.Properties != nil { - schemaMap["properties"] = f.JsonSchema.Properties - } - if f.JsonSchema.Required != nil { - schemaMap["required"] = f.JsonSchema.Required - } - if f.JsonSchema.Items != nil { - schemaMap["items"] = f.JsonSchema.Items - } - if f.JsonSchema.Description != "" { - schemaMap["description"] = f.JsonSchema.Description - } + schemaMap["name"] = f.JsonSchema.Name + if f.JsonSchema.Strict != nil { + schemaMap["strict"] = *f.JsonSchema.Strict } + schemaMap["schema"] = f.JsonSchema.Schema responseFormat.JsonSchema = &schemaMap } req.ResponseFormat = responseFormat From e96d38b15a2c691fecb88a39887a08dda0a7b2d1 Mon Sep 17 00:00:00 2001 From: Sean Goedecke Date: Thu, 17 Jul 2025 14:41:31 +1000 Subject: [PATCH 3/3] Update examples/json_schema_prompt.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- examples/json_schema_prompt.yml | 77 +++++++++++++++++---------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/examples/json_schema_prompt.yml b/examples/json_schema_prompt.yml index c8a33fdb..a10484dc 100644 --- a/examples/json_schema_prompt.yml +++ b/examples/json_schema_prompt.yml @@ -3,45 +3,48 @@ description: Example prompt demonstrating responseFormat and jsonSchema usage model: openai/gpt-4o responseFormat: json_schema jsonSchema: - type: object - description: A structured response containing person information - properties: - name: - type: string - description: The full name of the person - age: - type: integer - description: The age of the person in years - minimum: 0 - maximum: 150 - email: - type: string - description: The email address of the person - format: email - skills: - type: array - description: A list of skills the person has - items: + name: Person Information Schema + strict: true + schema: + type: object + description: A structured response containing person information + properties: + name: type: string - address: - type: object - description: The person's address - properties: - street: - type: string - description: Street address - city: - type: string - description: City name - country: + description: The full name of the person + age: + type: integer + description: The age of the person in years + minimum: 0 + maximum: 150 + email: + type: string + description: The email address of the person + format: email + skills: + type: array + description: A list of skills the person has + items: type: string - description: Country name - required: - - city - - country - required: - - name - - age + address: + type: object + description: The person's address + properties: + street: + type: string + description: Street address + city: + type: string + description: City name + country: + type: string + description: Country name + required: + - city + - country + required: + - name + - age messages: - role: system content: You are a helpful assistant that provides structured information about people.