feat(api): add message image content

.stats.yml

@@ -1,2 +1,2 @@
 configured_endpoints: 64
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-2e14236d4015bf3b956290ea8b656224a0c7b206a356c6af2a7ae43fdbceb04c.yml
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-084b8f68408c6b689a55200a78bcf233769bfcd8e999d9fadaeb399152b05bcd.yml

api.md

@@ -349,14 +349,20 @@ Types:
 - <code><a href="./src/resources/beta/threads/messages.ts">ImageFileContentBlock</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">ImageFileDelta</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">ImageFileDeltaBlock</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">ImageURL</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">ImageURLContentBlock</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">ImageURLDelta</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">ImageURLDeltaBlock</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">Message</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageContent</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageContentDelta</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">MessageContentPartParam</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageDeleted</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageDelta</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">MessageDeltaEvent</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">Text</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">TextContentBlock</a></code>
+- <code><a href="./src/resources/beta/threads/messages.ts">TextContentBlockParam</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">TextDelta</a></code>
 - <code><a href="./src/resources/beta/threads/messages.ts">TextDeltaBlock</a></code>
 

src/lib/AssistantStream.ts

@@ -7,6 +7,7 @@ import {
  ImageFile,
  TextDelta,
  Messages,
+ MessageContent,
 } from 'openai/resources/beta/threads/messages';
 import * as Core from 'openai/core';
 import { RequestOptions } from 'openai/core';
@@ -87,7 +88,7 @@ export class AssistantStream
  #messageSnapshot: Message | undefined;
  #finalRun: Run | undefined;
  #currentContentIndex: number | undefined;
- #currentContent: TextContentBlock | ImageFileContentBlock | undefined;
+ #currentContent: MessageContent | undefined;
  #currentToolCallIndex: number | undefined;
  #currentToolCall: ToolCall | undefined;
 
@@ -624,10 +625,8 @@ export class AssistantStream
  currentContent,
  );
  } else {
- snapshot.content[contentElement.index] = contentElement as
- | TextContentBlock
- | ImageFileContentBlock;
- //This is a new element
+ snapshot.content[contentElement.index] = contentElement as MessageContent;
+ // This is a new element
  newContent.push(contentElement);
  }
  }
@@ -650,7 +649,7 @@ export class AssistantStream
 
  #accumulateContent(
  contentElement: MessageContentDelta,
- currentContent: TextContentBlock | ImageFileContentBlock | undefined,
+ currentContent: MessageContent | undefined,
  ): TextContentBlock | ImageFileContentBlock {
  return AssistantStream.accumulateDelta(currentContent as unknown as Record<any, any>, contentElement) as
  | TextContentBlock

src/resources/beta/threads/index.ts

@@ -11,14 +11,20 @@ export {
  ImageFileContentBlock,
  ImageFileDelta,
  ImageFileDeltaBlock,
+ ImageURL,
+ ImageURLContentBlock,
+ ImageURLDelta,
+ ImageURLDeltaBlock,
  Message,
  MessageContent,
  MessageContentDelta,
+ MessageContentPartParam,
  MessageDeleted,
  MessageDelta,
  MessageDeltaEvent,
  Text,
  TextContentBlock,
+ TextContentBlockParam,
  TextDelta,
  TextDeltaBlock,
  MessageCreateParams,

src/resources/beta/threads/messages.ts

@@ -249,9 +249,16 @@ export namespace FilePathDeltaAnnotation {
 export interface ImageFile {
  /**
  * The [File](https://platform.openai.com/docs/api-reference/files) ID of the image
- * in the message content.
+ * in the message content. Set `purpose="vision"` when uploading the File if you
+ * need to later display the file content.
  */
  file_id: string;
+
+ /**
+ * Specifies the detail level of the image if specified by the user. `low` uses
+ * fewer tokens, you can opt in to high resolution using `high`.
+ */
+ detail?: 'auto' | 'low' | 'high';
 }
 
 /**
@@ -268,9 +275,16 @@ export interface ImageFileContentBlock {
 }
 
 export interface ImageFileDelta {
+ /**
+ * Specifies the detail level of the image if specified by the user. `low` uses
+ * fewer tokens, you can opt in to high resolution using `high`.
+ */
+ detail?: 'auto' | 'low' | 'high';
+
  /**
  * The [File](https://platform.openai.com/docs/api-reference/files) ID of the image
- * in the message content.
+ * in the message content. Set `purpose="vision"` when uploading the File if you
+ * need to later display the file content.
  */
  file_id?: string;
 }
@@ -293,6 +307,63 @@ export interface ImageFileDeltaBlock {
  image_file?: ImageFileDelta;
 }
 
+export interface ImageURL {
+ /**
+ * The external URL of the image, must be a supported image types: jpeg, jpg, png,
+ * gif, webp.
+ */
+ url: string;
+
+ /**
+ * Specifies the detail level of the image. `low` uses fewer tokens, you can opt in
+ * to high resolution using `high`. Default value is `auto`
+ */
+ detail?: 'auto' | 'low' | 'high';
+}
+
+/**
+ * References an image URL in the content of a message.
+ */
+export interface ImageURLContentBlock {
+ image_url: ImageURL;
+
+ /**
+ * The type of the content part.
+ */
+ type: 'image_url';
+}
+
+export interface ImageURLDelta {
+ /**
+ * Specifies the detail level of the image. `low` uses fewer tokens, you can opt in
+ * to high resolution using `high`.
+ */
+ detail?: 'auto' | 'low' | 'high';
+
+ /**
+ * The URL of the image, must be a supported image types: jpeg, jpg, png, gif,
+ * webp.
+ */
+ url?: string;
+}
+
+/**
+ * References an image URL in the content of a message.
+ */
+export interface ImageURLDeltaBlock {
+ /**
+ * The index of the content part in the message.
+ */
+ index: number;
+
+ /**
+ * Always `image_url`.
+ */
+ type: 'image_url';
+
+ image_url?: ImageURLDelta;
+}
+
 /**
  * Represents a message within a
  * [thread](https://platform.openai.com/docs/api-reference/threads).
@@ -406,13 +477,19 @@ export namespace Message {
  * References an image [File](https://platform.openai.com/docs/api-reference/files)
  * in the content of a message.
  */
-export type MessageContent = ImageFileContentBlock | TextContentBlock;
+export type MessageContent = ImageFileContentBlock | ImageURLContentBlock | TextContentBlock;
+
+/**
+ * References an image [File](https://platform.openai.com/docs/api-reference/files)
+ * in the content of a message.
+ */
+export type MessageContentDelta = ImageFileDeltaBlock | TextDeltaBlock | ImageURLDeltaBlock;
 
 /**
  * References an image [File](https://platform.openai.com/docs/api-reference/files)
  * in the content of a message.
  */
-export type MessageContentDelta = ImageFileDeltaBlock | TextDeltaBlock;
+export type MessageContentPartParam = ImageFileContentBlock | ImageURLContentBlock | TextContentBlockParam;
 
 export interface MessageDeleted {
  id: string;
@@ -479,6 +556,21 @@ export interface TextContentBlock {
  type: 'text';
 }
 
+/**
+ * The text content that is part of a message.
+ */
+export interface TextContentBlockParam {
+ /**
+ * Text content to be sent to the model
+ */
+ text: string;
+
+ /**
+ * Always `text`.
+ */
+ type: 'text';
+}
+
 export interface TextDelta {
  annotations?: Array<AnnotationDelta>;
 
@@ -507,9 +599,9 @@ export interface TextDeltaBlock {
 
 export interface MessageCreateParams {
  /**
- * The content of the message.
+ * The text contents of the message.
  */
- content: string;
+ content: string | Array<MessageContentPartParam>;
 
  /**
  * The role of the entity that is creating the message. Allowed values include:
@@ -591,14 +683,20 @@ export namespace Messages {
  export import ImageFileContentBlock = MessagesAPI.ImageFileContentBlock;
  export import ImageFileDelta = MessagesAPI.ImageFileDelta;
  export import ImageFileDeltaBlock = MessagesAPI.ImageFileDeltaBlock;
+ export import ImageURL = MessagesAPI.ImageURL;
+ export import ImageURLContentBlock = MessagesAPI.ImageURLContentBlock;
+ export import ImageURLDelta = MessagesAPI.ImageURLDelta;
+ export import ImageURLDeltaBlock = MessagesAPI.ImageURLDeltaBlock;
  export import Message = MessagesAPI.Message;
  export import MessageContent = MessagesAPI.MessageContent;
  export import MessageContentDelta = MessagesAPI.MessageContentDelta;
+ export import MessageContentPartParam = MessagesAPI.MessageContentPartParam;
  export import MessageDeleted = MessagesAPI.MessageDeleted;
  export import MessageDelta = MessagesAPI.MessageDelta;
  export import MessageDeltaEvent = MessagesAPI.MessageDeltaEvent;
  export import Text = MessagesAPI.Text;
  export import TextContentBlock = MessagesAPI.TextContentBlock;
+ export import TextContentBlockParam = MessagesAPI.TextContentBlockParam;
  export import TextDelta = MessagesAPI.TextDelta;
  export import TextDeltaBlock = MessagesAPI.TextDeltaBlock;
  export import MessagesPage = MessagesAPI.MessagesPage;

src/resources/beta/threads/runs/runs.ts

@@ -9,6 +9,7 @@ import { sleep } from 'openai/core';
 import { RunSubmitToolOutputsParamsStream } from 'openai/lib/AssistantStream';
 import * as RunsAPI from 'openai/resources/beta/threads/runs/runs';
 import * as AssistantsAPI from 'openai/resources/beta/assistants';
+import * as MessagesAPI from 'openai/resources/beta/threads/messages';
 import * as ThreadsAPI from 'openai/resources/beta/threads/threads';
 import * as StepsAPI from 'openai/resources/beta/threads/runs/steps';
 import { CursorPage, type CursorPageParams } from 'openai/pagination';
@@ -747,9 +748,9 @@ export interface RunCreateParamsBase {
 export namespace RunCreateParams {
  export interface AdditionalMessage {
  /**
- * The content of the message.
+ * The text contents of the message.
  */
- content: string;
+ content: string | Array<MessagesAPI.MessageContentPartParam>;
 
  /**
  * The role of the entity that is creating the message. Allowed values include:
@@ -999,9 +1000,9 @@ export interface RunCreateAndPollParams {
 export namespace RunCreateAndPollParams {
  export interface AdditionalMessage {
  /**
- * The content of the message.
+ * The text contents of the message.
  */
- content: string;
+ content: string | Array<MessagesAPI.MessageContentPartParam>;
 
  /**
  * The role of the entity that is creating the message. Allowed values include:
@@ -1204,9 +1205,9 @@ export interface RunCreateAndStreamParams {
 export namespace RunCreateAndStreamParams {
  export interface AdditionalMessage {
  /**
- * The content of the message.
+ * The text contents of the message.
  */
- content: string;
+ content: string | Array<MessagesAPI.MessageContentPartParam>;
 
  /**
  * The role of the entity that is creating the message. Allowed values include:
@@ -1409,9 +1410,9 @@ export interface RunStreamParams {
 export namespace RunStreamParams {
  export interface AdditionalMessage {
  /**
- * The content of the message.
+ * The text contents of the message.
  */
- content: string;
+ content: string | Array<MessagesAPI.MessageContentPartParam>;
 
  /**
  * The role of the entity that is creating the message. Allowed values include:

src/resources/beta/threads/threads.ts

@@ -284,9 +284,9 @@ export interface ThreadCreateParams {
 export namespace ThreadCreateParams {
  export interface Message {
  /**
- * The content of the message.
+ * The text contents of the message.
  */
- content: string;
+ content: string | Array<MessagesAPI.MessageContentPartParam>;
 
  /**
  * The role of the entity that is creating the message. Allowed values include:
@@ -623,9 +623,9 @@ export namespace ThreadCreateAndRunParams {
  export namespace Thread {
  export interface Message {
  /**
- * The content of the message.
+ * The text contents of the message.
  */
- content: string;
+ content: string | Array<MessagesAPI.MessageContentPartParam>;
 
  /**
  * The role of the entity that is creating the message. Allowed values include:
@@ -973,9 +973,9 @@ export namespace ThreadCreateAndRunPollParams {
  export namespace Thread {
  export interface Message {
  /**
- * The content of the message.
+ * The text contents of the message.
  */
- content: string;
+ content: string | Array<MessagesAPI.MessageContentPartParam>;
 
  /**
  * The role of the entity that is creating the message. Allowed values include:
@@ -1302,9 +1302,9 @@ export namespace ThreadCreateAndRunStreamParams {
  export namespace Thread {
  export interface Message {
  /**
- * The content of the message.
+ * The text contents of the message.
  */
- content: string;
+ content: string | Array<MessagesAPI.MessageContentPartParam>;
 
  /**
  * The role of the entity that is creating the message. Allowed values include:
@@ -1503,14 +1503,20 @@ export namespace Threads {
  export import ImageFileContentBlock = MessagesAPI.ImageFileContentBlock;
  export import ImageFileDelta = MessagesAPI.ImageFileDelta;
  export import ImageFileDeltaBlock = MessagesAPI.ImageFileDeltaBlock;
+ export import ImageURL = MessagesAPI.ImageURL;
+ export import ImageURLContentBlock = MessagesAPI.ImageURLContentBlock;
+ export import ImageURLDelta = MessagesAPI.ImageURLDelta;
+ export import ImageURLDeltaBlock = MessagesAPI.ImageURLDeltaBlock;
  export import Message = MessagesAPI.Message;
  export import MessageContent = MessagesAPI.MessageContent;
  export import MessageContentDelta = MessagesAPI.MessageContentDelta;
+ export import MessageContentPartParam = MessagesAPI.MessageContentPartParam;
  export import MessageDeleted = MessagesAPI.MessageDeleted;
  export import MessageDelta = MessagesAPI.MessageDelta;
  export import MessageDeltaEvent = MessagesAPI.MessageDeltaEvent;
  export import Text = MessagesAPI.Text;
  export import TextContentBlock = MessagesAPI.TextContentBlock;
+ export import TextContentBlockParam = MessagesAPI.TextContentBlockParam;
  export import TextDelta = MessagesAPI.TextDelta;
  export import TextDeltaBlock = MessagesAPI.TextDeltaBlock;
  export import MessagesPage = MessagesAPI.MessagesPage;