Vision & Audio

from definable.models import OpenAIChat
from definable.media import Image

model = OpenAIChat(id="gpt-4o")

# From a URL
response = model.invoke(messages=[{
    "role": "user",
    "content": "Describe what you see.",
    "images": [Image(url="https://example.com/photo.jpg")],
}])
print(response.content)

image = Image(url="https://example.com/chart.png", detail="high")
# Options: "low", "high", "auto" (default)

from definable.media import Audio

response = model.invoke(messages=[{
    "role": "user",
    "content": "Transcribe this audio.",
    "audio": [Audio(filepath="/path/to/recording.mp3")],
}])

model = OpenAIChat(
    id="gpt-4o-audio-preview",
    modalities=["text", "audio"],
    audio={"voice": "alloy", "format": "wav"},
)

response = model.invoke(
    messages=[{"role": "user", "content": "Read this sentence aloud: Hello world!"}]
)

# Access the audio output
print(response.audio.transcript)
audio_bytes = response.audio.get_content_bytes()

from definable.media import Video

response = model.invoke(messages=[{
    "role": "user",
    "content": "Summarize what happens in this video.",
    "videos": [Video(filepath="/path/to/clip.mp4")],
}])

from definable.media import File

response = model.invoke(messages=[{
    "role": "user",
    "content": "Summarize this document.",
    "files": [File(filepath="/path/to/report.pdf", mime_type="application/pdf")],
}])

from definable.agents import Agent
from definable.media import Image

agent = Agent(
    model=OpenAIChat(id="gpt-4o"),
    instructions="You are a helpful visual assistant.",
)

output = agent.run(
    "What's in this image?",
    images=[Image(url="https://example.com/photo.jpg")],
)
print(output.content)