Skip to content

Commit f1be7d0

Browse files
sgoedeckesgoedecke
committed
Use string format for jsonSchema
1 parent 3bc7b92 commit f1be7d0

File tree

5 files changed

+162
-138
lines changed

5 files changed

+162
-138
lines changed

‎cmd/eval/eval_test.go

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -569,21 +569,7 @@ name: JSON Schema Evaluation
569569
description: Testing responseFormat and jsonSchema in eval
570570
model: openai/gpt-4o
571571
responseFormat: json_schema
572-
jsonSchema:
573-
name: response_schema
574-
strict: true
575-
schema:
576-
type: object
577-
properties:
578-
message:
579-
type: string
580-
description: The response message
581-
confidence:
582-
type: number
583-
description: Confidence score
584-
required:
585-
- message
586-
additionalProperties: false
572+
jsonSchema: '{"name": "response_schema", "strict": true, "schema": {"type": "object", "properties": {"message": {"type": "string", "description": "The response message"}, "confidence": {"type": "number", "description": "Confidence score"}}, "required": ["message"], "additionalProperties": false}}'
587573
testData:
588574
- input: "hello"
589575
expected: "hello world"

‎cmd/run/run_test.go

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -341,22 +341,7 @@ name: JSON Schema Test
341341
description: Test responseFormat and jsonSchema
342342
model: openai/test-model
343343
responseFormat: json_schema
344-
jsonSchema:
345-
name: person_schema
346-
strict: true
347-
schema:
348-
type: object
349-
properties:
350-
name:
351-
type: string
352-
description: The name
353-
age:
354-
type: integer
355-
description: The age
356-
required:
357-
- name
358-
- age
359-
additionalProperties: false
344+
jsonSchema: '{"name": "person_schema", "strict": true, "schema": {"type": "object", "properties": {"name": {"type": "string", "description": "The name"}, "age": {"type": "integer", "description": "The age"}}, "required": ["name", "age"], "additionalProperties": false}}'
360345
messages:
361346
- role: system
362347
content: You are a helpful assistant.

‎examples/json_schema_prompt.yml

Lines changed: 43 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,52 @@
1-
name: JSON Schema Response Example
2-
description: Example prompt demonstrating responseFormat and jsonSchema usage
3-
model: openai/gpt-4o
1+
name: JSON Schema String Format Example
2+
description: Example using JSON string format for jsonSchema
3+
model: openai/gpt-4o-mini
44
responseFormat: json_schema
5-
jsonSchema:
6-
name: Person Information Schema
7-
strict: true
8-
schema:
9-
type: object
10-
description: A structured response containing person information
11-
properties:
12-
name:
13-
type: string
14-
description: The full name of the person
15-
age:
16-
type: integer
17-
description: The age of the person in years
18-
minimum: 0
19-
maximum: 150
20-
email:
21-
type: string
22-
description: The email address of the person
23-
format: email
24-
skills:
25-
type: array
26-
description: A list of skills the person has
27-
items:
28-
type: string
29-
address:
30-
type: object
31-
description: The person's address
32-
properties:
33-
street:
34-
type: string
35-
description: Street address
36-
city:
37-
type: string
38-
description: City name
39-
country:
40-
type: string
41-
description: Country name
42-
required:
43-
- city
44-
- country
45-
required:
46-
- name
47-
- age
5+
jsonSchema: |-
6+
{
7+
"name": "animal_description",
8+
"strict": true,
9+
"schema": {
10+
"type": "object",
11+
"properties": {
12+
"name": {
13+
"type": "string",
14+
"description": "The name of the animal"
15+
},
16+
"habitat": {
17+
"type": "string",
18+
"description": "The habitat where the animal lives"
19+
},
20+
"diet": {
21+
"type": "string",
22+
"description": "What the animal eats",
23+
"enum": ["carnivore", "herbivore", "omnivore"]
24+
},
25+
"characteristics": {
26+
"type": "array",
27+
"description": "Key characteristics of the animal",
28+
"items": {
29+
"type": "string"
30+
}
31+
}
32+
},
33+
"required": ["name", "habitat", "diet"],
34+
"additionalProperties": false
35+
}
36+
}
4837
messages:
4938
- role: system
50-
content: You are a helpful assistant that provides structured information about people.
39+
content: You are a helpful assistant that provides detailed information about animals.
5140
- role: user
52-
content: "Generate information for a person named {{name}} who is {{age}} years old."
41+
content: "Describe a {{animal}} in detail."
5342
testData:
54-
- name: "Alice Johnson"
55-
age: "30"
56-
- name: "Bob Smith"
57-
age: "25"
43+
- animal: "dog"
44+
- animal: "cat"
45+
- animal: "elephant"
5846
evaluators:
59-
- name: has-required-fields
47+
- name: has-name
6048
string:
6149
contains: "name"
62-
- name: valid-json-structure
50+
- name: has-habitat
6351
string:
64-
contains: "age"
52+
contains: "habitat"

‎pkg/prompt/prompt.go

Lines changed: 51 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
package prompt
33

44
import (
5+
"encoding/json"
56
"fmt"
67
"os"
78
"strings"
@@ -67,11 +68,30 @@ type Choice struct {
6768
Score float64 `yaml:"score"`
6869
}
6970

70-
// JsonSchema represents a JSON schema for structured responses
71-
type JsonSchema struct {
72-
Name string `yaml:"name" json:"name"`
73-
Strict *bool `yaml:"strict,omitempty" json:"strict,omitempty"`
74-
Schema map[string]interface{} `yaml:"schema" json:"schema"`
71+
// JsonSchema represents a JSON schema for structured responses as a JSON string
72+
type JsonSchema string
73+
74+
// UnmarshalYAML implements custom YAML unmarshaling for JsonSchema
75+
// Only supports JSON string format
76+
func (js *JsonSchema) UnmarshalYAML(node *yaml.Node) error {
77+
// Only support string nodes (JSON format)
78+
if node.Kind != yaml.ScalarNode {
79+
return fmt.Errorf("jsonSchema must be a JSON string")
80+
}
81+
82+
var jsonStr string
83+
if err := node.Decode(&jsonStr); err != nil {
84+
return err
85+
}
86+
87+
// Validate that it's valid JSON
88+
var temp interface{}
89+
if err := json.Unmarshal([]byte(jsonStr), &temp); err != nil {
90+
return fmt.Errorf("invalid JSON in jsonSchema: %w", err)
91+
}
92+
93+
*js = JsonSchema(jsonStr)
94+
return nil
7595
}
7696

7797
// LoadFromFile loads and parses a prompt file from the given path
@@ -105,16 +125,24 @@ func (f *File) validateResponseFormat() error {
105125
return fmt.Errorf("invalid responseFormat: %s. Must be 'text', 'json_object', or 'json_schema'", *f.ResponseFormat)
106126
}
107127

108-
// If responseFormat is "json_schema", jsonSchema must be provided with required fields
128+
// If responseFormat is "json_schema", jsonSchema must be provided
109129
if *f.ResponseFormat == "json_schema" {
110130
if f.JsonSchema == nil {
111131
return fmt.Errorf("jsonSchema is required when responseFormat is 'json_schema'")
112132
}
113-
if f.JsonSchema.Name == "" {
114-
return fmt.Errorf("jsonSchema.name is required when responseFormat is 'json_schema'")
133+
134+
// Parse and validate the JSON schema
135+
var schema map[string]interface{}
136+
if err := json.Unmarshal([]byte(*f.JsonSchema), &schema); err != nil {
137+
return fmt.Errorf("invalid JSON in jsonSchema: %w", err)
138+
}
139+
140+
// Check for required fields
141+
if _, ok := schema["name"]; !ok {
142+
return fmt.Errorf("jsonSchema must contain 'name' field")
115143
}
116-
if f.JsonSchema.Schema == nil {
117-
return fmt.Errorf("jsonSchema.schema is required when responseFormat is 'json_schema'")
144+
if _, ok := schema["schema"]; !ok {
145+
return fmt.Errorf("jsonSchema must contain 'schema' field")
118146
}
119147
}
120148

@@ -193,13 +221,20 @@ func (f *File) BuildChatCompletionOptions(messages []azuremodels.ChatMessage) az
193221
Type: *f.ResponseFormat,
194222
}
195223
if f.JsonSchema != nil {
196-
// Convert JsonSchema to map[string]interface{}
197-
schemaMap := make(map[string]interface{})
198-
schemaMap["name"] = f.JsonSchema.Name
199-
if f.JsonSchema.Strict != nil {
200-
schemaMap["strict"] = *f.JsonSchema.Strict
224+
// Parse the JSON schema string into a map
225+
var schemaMap map[string]interface{}
226+
if err := json.Unmarshal([]byte(*f.JsonSchema), &schemaMap); err != nil {
227+
// This should not happen as we validate during unmarshaling
228+
// but we'll handle it gracefully
229+
schemaMap = map[string]interface{}{
230+
"name": "default_schema",
231+
"strict": true,
232+
"schema": map[string]interface{}{
233+
"type": "object",
234+
"properties": map[string]interface{}{},
235+
},
236+
}
201237
}
202-
schemaMap["schema"] = f.JsonSchema.Schema
203238
responseFormat.JsonSchema = &schemaMap
204239
}
205240
req.ResponseFormat = responseFormat

‎pkg/prompt/prompt_test.go

Lines changed: 66 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package prompt
22

33
import (
4+
"encoding/json"
45
"os"
56
"path/filepath"
67
"testing"
@@ -139,27 +140,35 @@ messages:
139140
require.Nil(t, promptFile.JsonSchema)
140141
})
141142

142-
t.Run("loads prompt file with responseFormat json_schema and jsonSchema", func(t *testing.T) {
143+
t.Run("loads prompt file with responseFormat json_schema and jsonSchema as JSON string", func(t *testing.T) {
143144
const yamlBody = `
144-
name: JSON Schema Response Format Test
145-
description: Test with JSON schema response format
145+
name: JSON Schema String Format Test
146+
description: Test with JSON schema as JSON string
146147
model: openai/gpt-4o
147148
responseFormat: json_schema
148-
jsonSchema:
149-
name: person_info
150-
strict: true
151-
schema:
152-
type: object
153-
properties:
154-
name:
155-
type: string
156-
description: The name of the person
157-
age:
158-
type: integer
159-
description: The age of the person
160-
required:
161-
- name
162-
additionalProperties: false
149+
jsonSchema: |-
150+
{
151+
"name": "describe_animal",
152+
"strict": true,
153+
"schema": {
154+
"type": "object",
155+
"properties": {
156+
"name": {
157+
"type": "string",
158+
"description": "The name of the animal"
159+
},
160+
"habitat": {
161+
"type": "string",
162+
"description": "The habitat the animal lives in"
163+
}
164+
},
165+
"additionalProperties": false,
166+
"required": [
167+
"name",
168+
"habitat"
169+
]
170+
}
171+
}
163172
messages:
164173
- role: user
165174
content: "Hello"
@@ -175,10 +184,29 @@ messages:
175184
require.NotNil(t, promptFile.ResponseFormat)
176185
require.Equal(t, "json_schema", *promptFile.ResponseFormat)
177186
require.NotNil(t, promptFile.JsonSchema)
178-
require.Equal(t, "person_info", promptFile.JsonSchema.Name)
179-
require.True(t, *promptFile.JsonSchema.Strict)
180-
require.Contains(t, promptFile.JsonSchema.Schema, "type")
181-
require.Contains(t, promptFile.JsonSchema.Schema, "properties")
187+
188+
// Parse the JSON schema string to verify its contents
189+
var schema map[string]interface{}
190+
err = json.Unmarshal([]byte(*promptFile.JsonSchema), &schema)
191+
require.NoError(t, err)
192+
193+
require.Equal(t, "describe_animal", schema["name"])
194+
require.Equal(t, true, schema["strict"])
195+
require.Contains(t, schema, "schema")
196+
197+
// Verify the nested schema structure
198+
nestedSchema := schema["schema"].(map[string]interface{})
199+
require.Equal(t, "object", nestedSchema["type"])
200+
require.Contains(t, nestedSchema, "properties")
201+
require.Contains(t, nestedSchema, "required")
202+
203+
properties := nestedSchema["properties"].(map[string]interface{})
204+
require.Contains(t, properties, "name")
205+
require.Contains(t, properties, "habitat")
206+
207+
required := nestedSchema["required"].([]interface{})
208+
require.Contains(t, required, "name")
209+
require.Contains(t, required, "habitat")
182210
})
183211

184212
t.Run("validates invalid responseFormat", func(t *testing.T) {
@@ -224,23 +252,25 @@ messages:
224252
})
225253

226254
t.Run("BuildChatCompletionOptions includes responseFormat and jsonSchema", func(t *testing.T) {
255+
jsonSchemaStr := `{
256+
"name": "test_schema",
257+
"strict": true,
258+
"schema": {
259+
"type": "object",
260+
"properties": {
261+
"name": {
262+
"type": "string",
263+
"description": "The name"
264+
}
265+
},
266+
"required": ["name"]
267+
}
268+
}`
269+
227270
promptFile := &File{
228271
Model: "openai/gpt-4o",
229272
ResponseFormat: func() *string { s := "json_schema"; return &s }(),
230-
JsonSchema: &JsonSchema{
231-
Name: "test_schema",
232-
Strict: func() *bool { b := true; return &b }(),
233-
Schema: map[string]interface{}{
234-
"type": "object",
235-
"properties": map[string]interface{}{
236-
"name": map[string]interface{}{
237-
"type": "string",
238-
"description": "The name",
239-
},
240-
},
241-
"required": []string{"name"},
242-
},
243-
},
273+
JsonSchema: func() *JsonSchema { js := JsonSchema(jsonSchemaStr); return &js }(),
244274
}
245275

246276
messages := []azuremodels.ChatMessage{

0 commit comments

Comments
 (0)