fixes filename / base64 conversions in openai completions converters (#9512) (#9570)

Co-authored-by: Davy De Waele <[email protected]>
Co-authored-by: Christian Bromann <[email protected]>
Co-authored-by: Hunter Lovell <[email protected]>
Co-authored-by: Hunter Lovell <[email protected]>

Author: 4 people

.changeset/red-boats-doubt.md

@@ -0,0 +1,6 @@
+---
+"@langchain/openai": patch
+---
+
+fixes filename / base64 conversions in openai completions converters (#9512)
+

libs/providers/langchain-openai/src/converters/completions.ts

@@ -34,7 +34,11 @@ import type {
 } from "openai/resources/chat/completions";
 import { OpenAI as OpenAIClient } from "openai";
 import { handleMultiModalOutput } from "../utils/output.js";
-import { isReasoningModel, messageToOpenAIRole } from "../utils/misc.js";
+import {
+  getRequiredFilenameFromMetadata,
+  isReasoningModel,
+  messageToOpenAIRole,
+} from "../utils/misc.js";
 
 /**
  * @deprecated This converter is an internal detail of the OpenAI provider. Do not use it directly. This will be revisited in a future release.
@@ -157,6 +161,9 @@ export const completionsApiContentBlockConverter: StandardContentBlockConverter<
   fromStandardFileBlock(block): ChatCompletionContentPart.File {
     if (block.source_type === "url") {
       const data = parseBase64DataUrl({ dataUrl: block.url });
+
+      const filename = getRequiredFilenameFromMetadata(block);
+
       if (!data) {
         throw new Error(
           `URL file blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`
@@ -169,15 +176,16 @@ export const completionsApiContentBlockConverter: StandardContentBlockConverter<
           file_data: block.url, // formatted as base64 data URL
           ...(block.metadata?.filename || block.metadata?.name
             ? {
-                filename: (block.metadata?.filename ||
-                  block.metadata?.name) as string,
+                filename,
               }
             : {}),
         },
       };
     }
 
     if (block.source_type === "base64") {
+      const filename = getRequiredFilenameFromMetadata(block);
+
       return {
         type: "file",
         file: {
@@ -186,9 +194,7 @@ export const completionsApiContentBlockConverter: StandardContentBlockConverter<
           block.metadata?.name ||
           block.metadata?.title
             ? {
-                filename: (block.metadata?.filename ||
-                  block.metadata?.name ||
-                  block.metadata?.title) as string,
+                filename,
               }
             : {}),
         },
@@ -550,10 +556,13 @@ export const convertStandardContentBlockToCompletionsContentPart: Converter<
   }
   if (block.type === "file") {
     if (block.data) {
+      const filename = getRequiredFilenameFromMetadata(block);
+
       return {
         type: "file",
         file: {
-          file_data: block.data.toString(),
+          file_data: `data:${block.mimeType};base64,${block.data}`,
+          filename: filename,
         },
       };
     }

libs/providers/langchain-openai/src/converters/responses.ts

@@ -25,7 +25,12 @@ import type {
 import { ResponseInputMessageContentList } from "openai/resources/responses/responses.js";
 import { ChatOpenAIReasoningSummary } from "../types.js";
 import { isCustomToolCall, parseCustomToolCall } from "../utils/tools.js";
-import { iife, isReasoningModel, messageToOpenAIRole } from "../utils/misc.js";
+import {
+  getRequiredFilenameFromMetadata,
+  iife,
+  isReasoningModel,
+  messageToOpenAIRole,
+} from "../utils/misc.js";
 import { Converter } from "@langchain/core/utils/format";
 import { completionsApiContentBlockConverter } from "./completions.js";
 
@@ -755,10 +760,8 @@ export const convertStandardContentMessageToResponsesInput: Converter<
     const resolveFileItem = (
       block: ContentBlock.Multimodal.File | ContentBlock.Multimodal.Video
     ): OpenAIClient.Responses.ResponseInputFile | undefined => {
-      const filename =
-        block.metadata?.filename ??
-        block.metadata?.name ??
-        block.metadata?.title;
+      const filename = getRequiredFilenameFromMetadata(block);
+
       if (block.fileId && typeof filename === "string") {
         return {
           type: "input_file",

libs/providers/langchain-openai/src/converters/tests/completions.test.ts

@@ -1,7 +1,11 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { describe, it, expect } from "vitest";
 import { ChatCompletionMessage } from "openai/resources";
-import { convertCompletionsMessageToBaseMessage } from "../completions.js";
+import {
+  completionsApiContentBlockConverter,
+  convertCompletionsMessageToBaseMessage,
+  convertStandardContentBlockToCompletionsContentPart,
+} from "../completions.js";
 
 describe("convertCompletionsMessageToBaseMessage", () => {
   describe("OpenRouter image response handling", () => {
@@ -124,4 +128,154 @@ describe("convertCompletionsMessageToBaseMessage", () => {
       ]);
     });
   });
+
+  describe("convertStandardContentBlockToCompletionsContentPart", () => {
+    it("can convert image block with base64 data to image_url data URL", () => {
+      const block = {
+        type: "image",
+        data: "iVBORw0KGgoAAAANSUhEUgAAAAE",
+        mimeType: "image/png",
+      } as any;
+
+      const result = convertStandardContentBlockToCompletionsContentPart(block);
+      expect(result).toEqual({
+        type: "image_url",
+        image_url: {
+          url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAE",
+        },
+      });
+    });
+
+    it("can convert image block with url to image_url", () => {
+      const block = {
+        type: "image",
+        url: "https://example.com/cat.png",
+      } as any;
+
+      const result = convertStandardContentBlockToCompletionsContentPart(block);
+      expect(result).toEqual({
+        type: "image_url",
+        image_url: {
+          url: "https://example.com/cat.png",
+        },
+      });
+    });
+
+    it("will throw an error when when no filename is providing to a base64 file block", () => {
+      const block = {
+        type: "file",
+        data: "iVBORw0KGgoAAAANSUhEUgAAAAE",
+        mimeType: "application/pdf",
+      } as any;
+
+      expect(() =>
+        convertStandardContentBlockToCompletionsContentPart(block)
+      ).toThrowError(
+        "a filename or name or title is needed via meta-data for OpenAI when working with multimodal blocks"
+      );
+    });
+
+    it("will convert a file block to an openai file payload when a filename is provided", () => {
+      const block = {
+        type: "file",
+        data: "iVBORw0KGgoAAAANSUhEUgAAAAE",
+        mimeType: "application/pdf",
+        metadata: { filename: "cat.pdf" },
+      } as any;
+
+      const result = convertStandardContentBlockToCompletionsContentPart(block);
+      expect(result).toEqual({
+        type: "file",
+        file: {
+          file_data: "data:application/pdf;base64,iVBORw0KGgoAAAANSUhEUgAAAAE",
+          filename: "cat.pdf",
+        },
+      });
+    });
+  });
+
+  describe("completionsApiContentBlockConverter.fromStandardFileBlock", () => {
+    it("throws when base64 file block is missing filename/name/title metadata", () => {
+      const block = {
+        source_type: "base64",
+        mime_type: "application/pdf",
+        data: "AAABBB",
+        // metadata intentionally missing
+      } as any;
+
+      expect(() =>
+        completionsApiContentBlockConverter.fromStandardFileBlock!(block)
+      ).toThrowError(
+        "a filename or name or title is needed via meta-data for OpenAI when working with multimodal blocks"
+      );
+    });
+
+    it("converts base64 file block to file with data URL and filename from metadata.filename", () => {
+      const block = {
+        source_type: "base64",
+        mime_type: "application/pdf",
+        data: "AAABBB",
+        metadata: { filename: "doc.pdf" },
+      } as any;
+
+      const result =
+        completionsApiContentBlockConverter.fromStandardFileBlock!(block);
+      expect(result).toEqual({
+        type: "file",
+        file: {
+          file_data: "data:application/pdf;base64,AAABBB",
+          filename: "doc.pdf",
+        },
+      });
+    });
+
+    it("converts url data-url file block to file with file_data equal to url and includes filename from metadata.name", () => {
+      const dataUrl = "data:application/pdf;base64,AAABBB";
+      const block = {
+        source_type: "url",
+        url: dataUrl,
+        metadata: { name: "report.pdf" },
+      } as any;
+
+      const result =
+        completionsApiContentBlockConverter.fromStandardFileBlock!(block);
+      expect(result).toEqual({
+        type: "file",
+        file: {
+          file_data: dataUrl,
+          filename: "report.pdf",
+        },
+      });
+    });
+
+    it("returns file_id for id source_type", () => {
+      const block = {
+        source_type: "id",
+        id: "file_123",
+      } as any;
+
+      const result =
+        completionsApiContentBlockConverter.fromStandardFileBlock!(block);
+      expect(result).toEqual({
+        type: "file",
+        file: {
+          file_id: "file_123",
+        },
+      });
+    });
+
+    it("throws when url is not a data URL", () => {
+      const block = {
+        source_type: "url",
+        url: "https://example.com/file.pdf",
+        metadata: { filename: "file.pdf" },
+      } as any;
+
+      expect(() =>
+        completionsApiContentBlockConverter.fromStandardFileBlock!(block)
+      ).toThrowError(
+        `URL file blocks with source_type url must be formatted as a data URL for ChatOpenAI`
+      );
+    });
+  });
 });

libs/providers/langchain-openai/src/converters/tests/responses.test.ts

@@ -310,4 +310,52 @@ describe("convertStandardContentMessageToResponsesInput", () => {
 
     expect(result).toEqual([{ type: "custom", payload: "data" }]);
   });
+
+  it("converts file payloads when filename is provided", () => {
+    const message = new HumanMessage({
+      contentBlocks: [
+        {
+          type: "file",
+          mimeType: "application/pdf",
+          data: "iVBORw0KGgoAAAANSUhEUgAAAAE",
+          metadata: { filename: "sample.pdf" },
+        },
+      ],
+    });
+
+    const result = convertStandardContentMessageToResponsesInput(message);
+
+    expect(result).toEqual([
+      {
+        role: "user",
+        type: "message",
+        content: [
+          {
+            type: "input_file",
+            file_data:
+              "data:application/pdf;base64,iVBORw0KGgoAAAANSUhEUgAAAAE",
+            filename: "sample.pdf",
+          },
+        ],
+      },
+    ]);
+  });
+
+  it("throws error when file payload does not contain filename", () => {
+    const message = new HumanMessage({
+      contentBlocks: [
+        {
+          type: "file",
+          mimeType: "application/pdf",
+          data: "iVBORw0KGgoAAAANSUhEUgAAAAE",
+        },
+      ],
+    });
+
+    expect(() =>
+      convertStandardContentMessageToResponsesInput(message)
+    ).toThrowError(
+      `a filename or name or title is needed via meta-data for OpenAI when working with multimodal blocks`
+    );
+  });
 });

libs/providers/langchain-openai/src/utils/misc.ts

@@ -1,5 +1,10 @@
 import type { OpenAI as OpenAIClient } from "openai";
-import { BaseMessage, ChatMessage } from "@langchain/core/messages";
+import {
+  BaseMessage,
+  ChatMessage,
+  ContentBlock,
+  Data,
+} from "@langchain/core/messages";
 
 export const iife = <T>(fn: () => T) => fn();
 
@@ -25,6 +30,24 @@ export function extractGenericMessageCustomRole(message: ChatMessage) {
   return message.role as OpenAIClient.ChatCompletionRole;
 }
 
+export function getRequiredFilenameFromMetadata(
+  block:
+    | ContentBlock.Multimodal.File
+    | ContentBlock.Multimodal.Video
+    | Data.StandardFileBlock
+): string {
+  const filename = (block.metadata?.filename ??
+    block.metadata?.name ??
+    block.metadata?.title) as string;
+
+  if (!filename) {
+    throw new Error(
+      "a filename or name or title is needed via meta-data for OpenAI when working with multimodal blocks"
+    );
+  }
+
+  return filename;
+}
 export function messageToOpenAIRole(
   message: BaseMessage
 ): OpenAIClient.ChatCompletionRole {