Merge pull request #325 from H2Shami/helicone-models

add helicone models + helicone model generation script

Author: rekram1-node

package.json

@@ -15,7 +15,8 @@
     }
   },
   "scripts": {
-    "validate": "bun ./packages/core/script/validate.ts"
+    "validate": "bun ./packages/core/script/validate.ts",
+    "helicone:generate": "bun ./packages/core/script/generate-helicone.ts"
   },
   "dependencies": {
     "@cloudflare/workers-types": "^4.20250801.0",

packages/core/script/generate-helicone.ts

@@ -0,0 +1,208 @@
+#!/usr/bin/env bun
+
+import { z } from "zod";
+import path from "node:path";
+import { mkdir, rm, readdir, stat } from "node:fs/promises";
+
+// Helicone public model registry endpoint
+const DEFAULT_ENDPOINT =
+  "https://jawn.helicone.ai/v1/public/model-registry/models";
+
+// Zod schemas to validate the Helicone response
+const Pricing = z
+  .object({
+    prompt: z.number().optional(),
+    completion: z.number().optional(),
+    cacheRead: z.number().optional(),
+    cacheWrite: z.number().optional(),
+    reasoning: z.number().optional(),
+  })
+  .passthrough();
+
+const Endpoint = z
+  .object({
+    provider: z.string(),
+    providerSlug: z.string().optional(),
+    supportsPtb: z.boolean().optional(),
+    pricing: Pricing.optional(),
+  })
+  .passthrough();
+
+const ModelItem = z
+  .object({
+    id: z.string(),
+    name: z.string(),
+    author: z.string().optional(),
+    contextLength: z.number().optional(),
+    maxOutput: z.number().optional(),
+    trainingDate: z.string().optional(),
+    description: z.string().optional(),
+    inputModalities: z.array(z.string()).optional(),
+    outputModalities: z.array(z.string()).optional(),
+    supportedParameters: z.array(z.string()).optional(),
+    endpoints: z.array(Endpoint).optional(),
+  })
+  .passthrough();
+
+const HeliconeResponse = z
+  .object({
+    data: z.object({
+      models: z.array(ModelItem),
+      total: z.number().optional(),
+      filters: z.any().optional(),
+    }),
+  })
+  .passthrough();
+
+function pickEndpoint(m: z.infer<typeof ModelItem>) {
+  if (!m.endpoints || m.endpoints.length === 0) return undefined;
+  // Prefer endpoint that matches author if available
+  if (m.author) {
+    const match = m.endpoints.find((e) => e.provider === m.author);
+    if (match) return match;
+  }
+  return m.endpoints[0];
+}
+
+function boolFromParams(params: string[] | undefined, keys: string[]): boolean {
+  if (!params) return false;
+  const set = new Set(params.map((p) => p.toLowerCase()));
+  return keys.some((k) => set.has(k.toLowerCase()));
+}
+
+function sanitizeModalities(values: string[] | undefined): string[] {
+  if (!values) return ["text"]; // default to text
+  const allowed = new Set(["text", "audio", "image", "video", "pdf"]);
+  const out = values.map((v) => v.toLowerCase()).filter((v) => allowed.has(v));
+  return out.length > 0 ? out : ["text"];
+}
+
+function formatToml(model: z.infer<typeof ModelItem>) {
+  const ep = pickEndpoint(model);
+  const pricing = ep?.pricing;
+
+  const supported = model.supportedParameters ?? [];
+
+  const nowISO = new Date().toISOString().slice(0, 10);
+  const rdRaw = model.trainingDate ? String(model.trainingDate) : nowISO;
+  const releaseDate = rdRaw.slice(0, 10);
+  const lastUpdated = releaseDate;
+  const knowledge = model.trainingDate
+    ? String(model.trainingDate).slice(0, 7)
+    : undefined;
+
+  const attachment = false; // Not exposed by Helicone registry
+  const temperature = boolFromParams(supported, ["temperature"]);
+  const toolCall = boolFromParams(supported, ["tools", "tool_choice"]);
+  const reasoning = boolFromParams(supported, [
+    "reasoning",
+    "include_reasoning",
+  ]);
+
+  const inputMods = sanitizeModalities(model.inputModalities);
+  const outputMods = sanitizeModalities(model.outputModalities);
+
+  const lines: string[] = [];
+  lines.push(`name = "${model.name.replaceAll('"', '\\"')}"`);
+  lines.push(`release_date = "${releaseDate}"`);
+  lines.push(`last_updated = "${lastUpdated}"`);
+  lines.push(`attachment = ${attachment}`);
+  lines.push(`reasoning = ${reasoning}`);
+  lines.push(`temperature = ${temperature}`);
+  lines.push(`tool_call = ${toolCall}`);
+  if (knowledge) lines.push(`knowledge = "${knowledge}"`);
+  lines.push(`open_weights = false`);
+  lines.push("");
+
+  if (
+    pricing &&
+    (pricing.prompt ??
+      pricing.completion ??
+      pricing.cacheRead ??
+      pricing.cacheWrite ??
+      (reasoning && pricing.reasoning)) !== undefined
+  ) {
+    lines.push(`[cost]`);
+    if (pricing.prompt !== undefined) lines.push(`input = ${pricing.prompt}`);
+    if (pricing.completion !== undefined)
+      lines.push(`output = ${pricing.completion}`);
+    if (reasoning && pricing.reasoning !== undefined)
+      lines.push(`reasoning = ${pricing.reasoning}`);
+    if (pricing.cacheRead !== undefined)
+      lines.push(`cache_read = ${pricing.cacheRead}`);
+    if (pricing.cacheWrite !== undefined)
+      lines.push(`cache_write = ${pricing.cacheWrite}`);
+    lines.push("");
+  }
+
+  const context = model.contextLength ?? 0;
+  const output = model.maxOutput ?? 4096;
+  lines.push(`[limit]`);
+  lines.push(`context = ${context}`);
+  lines.push(`output = ${output}`);
+  lines.push("");
+
+  lines.push(`[modalities]`);
+  lines.push(`input = [${inputMods.map((m) => `"${m}"`).join(", ")}]`);
+  lines.push(`output = [${outputMods.map((m) => `"${m}"`).join(", ")}]`);
+
+  return lines.join("\n") + "\n";
+}
+
+async function main() {
+  const endpoint = DEFAULT_ENDPOINT;
+
+  const outDir = path.join(
+    import.meta.dirname,
+    "..",
+    "..",
+    "..",
+    "providers",
+    "helicone",
+    "models",
+  );
+
+  const res = await fetch(endpoint);
+  if (!res.ok) {
+    console.error(`Failed to fetch registry: ${res.status} ${res.statusText}`);
+    process.exit(1);
+  }
+  const json = await res.json();
+
+  const parsed = HeliconeResponse.safeParse(json);
+  if (!parsed.success) {
+    parsed.error.cause = json;
+    console.error("Invalid Helicone response:", parsed.error.errors);
+    console.error("When parsing:", parsed.error.cause);
+    process.exit(1);
+  }
+
+  const models = parsed.data.data.models;
+
+  // Clean output directory: remove subfolders and existing TOML files
+  await mkdir(outDir, { recursive: true });
+  for (const entry of await readdir(outDir)) {
+    const p = path.join(outDir, entry);
+    const st = await stat(p);
+    if (st.isDirectory()) {
+      await rm(p, { recursive: true, force: true });
+    } else if (st.isFile() && entry.endsWith(".toml")) {
+      await rm(p, { force: true });
+    }
+  }
+  let created = 0;
+
+  for (const m of models) {
+    const fileSafeId = m.id.replaceAll("/", "-");
+    const filePath = path.join(outDir, `${fileSafeId}.toml`);
+    const toml = formatToml(m);
+    await Bun.write(filePath, toml);
+    created++;
+  }
+
+  console.log(
+    `Generated ${created} model file(s) under providers/helicone/models/*.toml`,
+  );
+}
+
+await main();

providers/helicone/README.md

@@ -0,0 +1,20 @@
+Helicone Models
+
+Generate model TOMLs from Helicone’s public registry.
+
+Prerequisites
+- Install Bun: https://bun.sh
+
+Commands
+- Generate files: `bun run helicone:generate`
+- Validate configs: `bun validate`
+
+Details
+- Source endpoint: `https://jawn.helicone.ai/v1/public/model-registry/models`
+- Output path: `providers/helicone/models/<model-id>.toml` (flat, no provider folders)
+- Dates: `release_date`/`last_updated` use `YYYY-MM-DD`; `knowledge` uses `YYYY-MM`.
+- Pricing: writes `cost.reasoning` only when `reasoning = true`.
+- Modalities: sanitized to `["text", "audio", "image", "video", "pdf"]`.
+
+Notes
+- The generator cleans the output folder before writing: removes any nested provider folders and existing TOML files to keep Model IDs flat (e.g., `claude-3.5-haiku`).

providers/helicone/logo.svg

@@ -0,0 +1,22 @@
+name = "OpenAI ChatGPT-4o"
+release_date = "2024-08-14"
+last_updated = "2024-08-14"
+attachment = false
+reasoning = false
+temperature = true
+tool_call = true
+knowledge = "2024-08"
+open_weights = false
+
+[cost]
+input = 5
+output = 20
+cache_read = 2.5
+
+[limit]
+context = 128000
+output = 16384
+
+[modalities]
+input = ["text", "image"]
+output = ["text"]

providers/helicone/models/chatgpt-4o-latest.toml

@@ -0,0 +1,23 @@
+name = "Anthropic: Claude 3 Haiku"
+release_date = "2024-03-07"
+last_updated = "2024-03-07"
+attachment = false
+reasoning = false
+temperature = true
+tool_call = true
+knowledge = "2024-03"
+open_weights = false
+
+[cost]
+input = 0.25
+output = 1.25
+cache_read = 0.03
+cache_write = 0.3
+
+[limit]
+context = 200000
+output = 4096
+
+[modalities]
+input = ["text", "image"]
+output = ["text"]

providers/helicone/models/claude-3-haiku-20240307.toml

@@ -0,0 +1,23 @@
+name = "Anthropic: Claude 3.5 Haiku"
+release_date = "2024-10-22"
+last_updated = "2024-10-22"
+attachment = false
+reasoning = false
+temperature = true
+tool_call = true
+knowledge = "2024-10"
+open_weights = false
+
+[cost]
+input = 0.7999999999999999
+output = 4
+cache_read = 0.08
+cache_write = 1
+
+[limit]
+context = 200000
+output = 8192
+
+[modalities]
+input = ["text", "image"]
+output = ["text"]

providers/helicone/models/claude-3.5-haiku.toml

@@ -0,0 +1,23 @@
+name = "Anthropic: Claude 3.5 Sonnet v2"
+release_date = "2024-10-22"
+last_updated = "2024-10-22"
+attachment = false
+reasoning = false
+temperature = true
+tool_call = true
+knowledge = "2024-10"
+open_weights = false
+
+[cost]
+input = 3
+output = 15
+cache_read = 0.30000000000000004
+cache_write = 3.75
+
+[limit]
+context = 200000
+output = 8192
+
+[modalities]
+input = ["text", "image"]
+output = ["text"]

providers/helicone/models/claude-3.5-sonnet-v2.toml

@@ -0,0 +1,23 @@
+name = "Anthropic: Claude 3.7 Sonnet"
+release_date = "2025-02-19"
+last_updated = "2025-02-19"
+attachment = false
+reasoning = false
+temperature = true
+tool_call = true
+knowledge = "2025-02"
+open_weights = false
+
+[cost]
+input = 3
+output = 15
+cache_read = 0.30000000000000004
+cache_write = 3.75
+
+[limit]
+context = 200000
+output = 64000
+
+[modalities]
+input = ["text", "image"]
+output = ["text"]

providers/helicone/models/claude-3.7-sonnet.toml

@@ -0,0 +1,23 @@
+name = "Anthropic: Claude 4.5 Haiku"
+release_date = "2025-10-01"
+last_updated = "2025-10-01"
+attachment = false
+reasoning = false
+temperature = true
+tool_call = true
+knowledge = "2025-10"
+open_weights = false
+
+[cost]
+input = 1
+output = 5
+cache_read = 0.09999999999999999
+cache_write = 1.25
+
+[limit]
+context = 200000
+output = 8192
+
+[modalities]
+input = ["text", "image"]
+output = ["text"]