AI Math Reasoning Learns Approaches Not Topics

We present AXIOM, a trust-first neuro-symbolic execution architecture for natural-language mathematical reasoning. In AXIOM, the language model functions strictly as a canonicalizer: it rewrites infor...

Hi, I’ve been working on this (using structured model outputs on the batch API) and I think I’ve succeeded. I would like to share my findings in case could help someone else. The following code is based on the official example. from openai import OpenAI from pydantic import BaseModel client = OpenAI() class Step(BaseModel): explanation: str output: str class MathReasoning(BaseModel): steps: list[Step] final_answer: str So, the trick to replicate the “parse” behavior depends on the API primitive you are using, I leave examples for both: In the first place you can use the following snippet to manually imitate a call to the parse method. # https://api.openai.com/v1/chat/completions from openai.lib._parsing import type_to_response_format_param, parse_chat_completion response = client.chat.completions.create( model="gpt-5-nano-2025-08-07", messages=[ {"role": "system", "content": "You are a helpful math tutor. Guide the user through the solution step by step."}, {"role": "user", "content": "how can I solve 8x + 7 = -23"} ], response_format=type_to_response_format_param(MathReasoning), ) parsed_response = parse_chat_completion( chat_completion=response, response_format=MathReasoning, input_tools=[] ) parsed_response.choices[0].message.parsed # https://api.openai.com/v1/responses from openai.lib._parsing._responses import type_to_text_format_param, parse_response response = client.responses.create( model="gpt-5-nano-2025-08-07", input=[ {"role": "system", "content": "You are a helpful math tutor. Guide the user through the solution step by step."}, {"role": "user", "content": "how can I solve 8x + 7 = -23"} ], text={"format": type_to_text_format_param(MathReasoning)}, ) parsed_respose = parse_response( response=response, text_format=MathReasoning, input_tools=[] ) parsed_respose.output_parsed Now that we know this, making the call to the batch API is easy: # https://api.openai.com/v1/chat/completions from openai.lib._parsing import type_to_response_format_param, records = [ { "custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": { "model": "gpt-5-nano-2025-08-07", "messages": [ {"role": "system", "content": "You are a helpful math tutor. Guide the user through the solution step by step."}, {"role": "user", "content": "how can I solve 8x + 7 = -23"} ], "response_format": type_to_response_format_param(MathReasoning) } }, { "custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": { "model": "gpt-5-nano-2025-08-07", "messages": [ {"role": "system", "content": "You are a helpful math tutor. Guide the user through the solution step by step."}, {"role": "user", "content": "how can I solve 8x = 8"} ], "response_format": type_to_response_format_param(MathReasoning) } } ] with open("/tmp/completions_batch_input.jsonl", "w") as f: for record in records: f.write(json.dumps(record) + "\n") .... client.batches.create( input_file_id="file-example-input-file-id", endpoint="/v1/chat/completions", completion_window="24h", metadata={ "description": "example structured over completions" } ) .... from openai.types.chat.chat_completion import ChatCompletion from openai.lib._parsing import parse_chat_completion import json file_response = client.files.content("file-output-example") completions = [ parse_chat_completion( chat_completion=ChatCompletion\ .model_validate(json.loads(line)['response']['body']), response_format=MathReasoning, input_tools=[] )\ .choices[0]\ .message\ .parsed for line in file_response.read().splitlines() if line.strip() ] completions # https://api.openai.com/v1/responses from openai.lib._parsing._responses import type_to_text_format_param records = [ { "custom_id": "request-1", "method": "POST", "url": "/v1/responses", "body": { "model": "gpt-5-nano-2025-08-07", "input": [ {"role": "system", "content": "You are a helpful math tutor. Guide the user through the solution step by step."}, {"role": "user", "content": "how can I solve 8x + 7 = -23"} ], "text": {"format": type_to_text_format_param(MathReasoning)} } }, { "custom_id": "request-2", "method": "POST", "url": "/v1/responses", "body": { "model": "gpt-5-nano-2025-08-07", "input": [ {"role": "system", "content": "You are a helpful math tutor. Guide the user through the solution step by step."}, {"role": "user", "content": "how can I solve 8x = 8"} ], "text": {"format": type_to_text_format_param(MathReasoning)} } } ] with open("/tmp/responses_batch_input.jsonl", "w") as f: for record in records: f.write(json.dumps(record) + "\n") .... client.batches.create( input_file_id="file-example-input-file-id", endpoint="/v1/responses", completion_window="24h", metadata={ "description": "example structured over responses" } ) .... from openai.types.responses import Response from openai.lib._parsing._responses import parse_response import json file_response = client.files.content("file-output-example") completions = [ parse_response( response=Response\ .model_validate(json.loads(line)['response']['body']), text_format=MathReasoning, input_tools=[] )\ .output_parsed\ for line in file_response.read().splitlines() if line.strip() ] completions

Apple researchers introduced GSM-Symbolic, a new benchmark to reveal the weaknesses in large language models' mathematical reasoning, showing that they rely heavily on pattern-matching rather than gen...