[clickhouse] Convert OTel traces model to native format (#6935)

## Which problem is this PR solving?
- Part of #5058

## Description of the changes
- Based on the `ch-go` wire protocol, convert the OTel traces model to
the ClickHouse native format for batch insertion.

## How was this change tested?
-  unit tests

## Checklist
- [x] I have read
https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md
- [x] I have signed all commits
- [x] I have added unit tests for the new functionality
- [x] I have run lint and test steps successfully
  - for `jaeger`: `make lint test`
  - for `jaeger-ui`: `npm run lint` and `npm run test`

---------

Signed-off-by: zhengkezhou1 <[email protected]>

Author: zhengkezhou1

go.mod

@@ -5,6 +5,7 @@ go 1.23.7
 toolchain go1.24.2
 
 require (
+	github.com/ClickHouse/ch-go v0.65.1
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2
 	github.com/Shopify/sarama v1.37.2
 	github.com/apache/thrift v0.21.0
@@ -107,6 +108,9 @@ require (
 require (
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.2 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/twmb/murmur3 v1.1.8 // indirect
 	go.opentelemetry.io/collector/extension/extensionauth v1.30.0 // indirect
 	go.opentelemetry.io/collector/extension/extensiontest v0.124.0 // indirect

go.sum

@@ -20,6 +20,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mx
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/ClickHouse/ch-go v0.65.1 h1:SLuxmLl5Mjj44/XbINsK2HFvzqup0s6rwKLFH347ZhU=
+github.com/ClickHouse/ch-go v0.65.1/go.mod h1:bsodgURwmrkvkBe5jw1qnGDgyITsYErfONKAHn05nv4=
 github.com/Code-Hex/go-generics-cache v1.5.1 h1:6vhZGc5M7Y/YD8cIUcY8kcuQLB4cHR7U+0KMqAA0KcU=
 github.com/Code-Hex/go-generics-cache v1.5.1/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4=
 github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
@@ -179,6 +181,10 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
 github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
@@ -568,6 +574,8 @@ github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsF
 github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770=
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30/go.mod h1:sH0u6fq6x4R5M7WxkoQFY/o7UaiItec0o1LinLCJNq8=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/shirou/gopsutil/v4 v4.25.3 h1:SeA68lsu8gLggyMbmCn8cmp97V1TI9ld9sVzAUcKcKE=
 github.com/shirou/gopsutil/v4 v4.25.3/go.mod h1:xbuxyoZj+UsgnZrENu3lQivsngRR5BdjbJwf2fv4szA=
 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c h1:aqg5Vm5dwtvL+YgDpBcK1ITf3o96N/K7/wsRXQnUTEs=
@@ -889,8 +897,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
-golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
+golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -991,8 +999,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ=
-golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0=
+golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
+golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/storage/v2/clickhouse/README.md

@@ -0,0 +1,37 @@
+## Clickhouse
+
+### Differences from the implementation in [otel collector contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/clickhouseexporter)
+
+#### Trace Storage Format
+
+The most significant difference lies in the handling of **Attributes**. In the OTel-contrib implementation, everything within the Attributes is converted to [strings](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/80b3df26b7028a4bbe1eb606a6142cd4df9c3c74/exporter/clickhouseexporter/internal/metrics_model.go#L171-L177):
+
+```golang
+func AttributesToMap(attributes pcommon.Map) column.IterableOrderedMap {
+	return orderedmap.CollectN(func(yield func(string, string) bool) {
+		for k, v := range attributes.All() {
+			yield(k, v.AsString())
+		}
+	}, attributes.Len())
+}
+```
+
+The primary reason for this is that it leads to the loss of the original data types and \~\~cannot be used directly as query parameters\~\~ (Clickhouse provides casting functions). For example, if an attribute has an `int64` value, we might want to perform the following operation:
+
+```sql
+SELECT * FROM test WHERE resource.attributes['container.restart.count'] > 10
+```
+
+To address the above issues, the following improvements have been implemented:
+
+  * Instead of directly using a Map for storage, the key and value are split into two separate arrays.
+  * More Columns are used to store values of different types:
+      * For basic types like bool, double, int, and string, corresponding type array columns are used for storage: `Array(Int64)`, `Array(Bool)`, etc.
+      * For complex types like slice and map, they are serialized into JSON format strings before storage: `Array(String)`.
+
+The `Value` type here actually refers to the `pdata` data types from the `otel-collector` pipeline. In our architecture, the `value_warpper` is responsible for wrapping the Protobuf-generated Go structures (which are the concrete implementation of `pdata`) into the `Value` type. Although `pdata` itself is based on the OTLP specification, encapsulating it into `Value` via the `value_warpper` creates a higher-level abstraction, which presents some challenges for directly storing `Value` in ClickHouse. Specifically, when deserializing `Slice` and `Map` data contained within the `Value`, the fact that JSON cannot natively distinguish whether a `Number` is an integer (`int`) or a floating-point number (`double`) leads to a loss of type information. Furthermore, directly handling the potentially dynamically nested `pdata` structures within the `Value` can also be quite complex. Therefore, to ensure the accuracy and completeness of data types in ClickHouse, and to effectively handle these nested telemetry data, we need to convert the `pdata` data inside `Value` into the standard `OTLP/JSON` format for storage.
+
+#### Data Read and Write Methods
+
+The OTel-contrib implementation uses `database/sql` for writing data. Using the provided generic interface is unnecessary; using the client provided by Clickhouse is a better choice.
+For write operations, `ch-go`'s `chpool` is used in batch mode. For read operations, `clickhouse-go` is used.

internal/storage/v2/clickhouse/tracestore/dbmodel/dbmodel.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2025 The Jaeger Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package dbmodel
+
+import "time"
+
+// Trace Domain model in Clickhouse.
+// This struct represents the schema for storing OTel pipeline Traces in Clickhouse.
+type Trace struct {
+	Resource Resource
+	Scope    Scope
+	Span     Span
+	Links    []Link
+	Events   []Event
+}
+
+type Resource struct {
+	Attributes AttributesGroup
+}
+
+type Scope struct {
+	Name       string
+	Version    string
+	Attributes AttributesGroup
+}
+
+type Span struct {
+	Timestamp     time.Time
+	TraceId       string
+	SpanId        string
+	ParentSpanId  string
+	TraceState    string
+	Name          string
+	Kind          string
+	Duration      time.Time
+	StatusCode    string
+	StatusMessage string
+	Attributes    AttributesGroup
+}
+
+type Link struct {
+	TraceId    string
+	SpanId     string
+	TraceState string
+	Attributes AttributesGroup
+}
+
+type Event struct {
+	Name       string
+	Timestamp  time.Time
+	Attributes AttributesGroup
+}

internal/storage/v2/clickhouse/tracestore/dbmodel/fixtures/dbmodel.json

@@ -0,0 +1,213 @@
+{
+    "Resource": {
+        "Attributes": {
+            "BoolKeys": [
+                "browser.mobile"
+            ],
+            "BoolValues": [
+                true
+            ],
+            "DoubleKeys": [
+                "host.memory.swap"
+            ],
+            "DoubleValues": [
+                2048
+            ],
+            "IntKeys": [
+                "process.parent.pid",
+                "process.pid",
+                "faas.max_memory"
+            ],
+            "IntValues": [
+                111,
+                1234,
+                134217728
+            ],
+            "StrKeys": [
+                "service.names",
+                "service.instance.id"
+            ],
+            "StrValues": [
+                "clickhouse",
+                "627cc493-f310-47de-96bd-71410b7dec09"
+            ],
+            "BytesKeys": [
+                "oci.manifest.digest"
+            ],
+            "BytesValues": [
+                "c2hhMjU2OmU0Y2E2MmMwZDYyZjNlODg2ZTY4NDgwNmRmZTlkNGUwY2RhNjBkNTQ5ODY4OTgxNzNjMTA4Mzg1NmNmZGEwZjQ="
+            ]
+        }
+    },
+    "Scope": {
+        "Name": "io.opentelemetry.contrib.clickhouse",
+        "Version": "1.0.0",
+        "Attributes": {
+            "BoolKeys": [
+                "library.feature.async_processing_enabled",
+                "library.security.data_masking_active"
+            ],
+            "BoolValues": [
+                true,
+                false
+            ],
+            "DoubleKeys": [
+                "component.config.sampling.ratio"
+            ],
+            "DoubleValues": [
+                0.75
+            ],
+            "IntKeys": [
+                "component.max_workers",
+                "component.min_workers"
+            ],
+            "IntValues": [
+                10,
+                2
+            ],
+            "StrKeys": [
+                "library.language",
+                "library.version"
+            ],
+            "StrValues": [
+                "go",
+                "v2.2.2"
+            ],
+            "BytesKeys": [
+                "scope.test.bytes.value"
+            ],
+            "BytesValues": [
+                "AQIDBA=="
+            ]
+        }
+    },
+    "Span": {
+        "Timestamp": "2023-12-25T09:53:49Z",
+        "TraceId": "01020300000000000000000000000000",
+        "SpanId": "0102030000000000",
+        "ParentSpanId": "0102040000000000",
+        "TraceState": "trace state",
+        "Name": "call db",
+        "Kind": "Internal",
+        "Duration": "2023-12-25T09:54:49Z",
+        "StatusCode": "Error",
+        "StatusMessage": "error",
+        "Attributes": {
+            "BoolKeys": [
+                "app.payment.card_valid",
+                "app.payment.charged"
+            ],
+            "BoolValues": [
+                true,
+                true
+            ],
+            "DoubleKeys": [
+                "app.payment.amount"
+            ],
+            "DoubleValues": [
+                99.99
+            ],
+            "IntKeys": [
+                "app.payment.count"
+            ],
+            "IntValues": [
+                5
+            ],
+            "StrKeys": [
+                "app.payment.id"
+            ],
+            "StrValues": [
+                "123456789"
+            ],
+            "BytesKeys": [
+                "span.test.bytes.value"
+            ],
+            "BytesValues": [
+                "AQIDBAUG"
+            ]
+        }
+    },
+    "Links": [
+        {
+            "TraceId": "01020500000000000000000000000000",
+            "SpanId": "0102050000000000",
+            "TraceState": "test",
+            "Attributes": {
+                "BoolKeys": [
+                    "is.retry"
+                ],
+                "BoolValues": [
+                    true
+                ],
+                "DoubleKeys": [
+                    "similarity.score"
+                ],
+                "DoubleValues": [
+                    0.85
+                ],
+                "IntKeys": [
+                    "correlation.id"
+                ],
+                "IntValues": [
+                    1324141
+                ],
+                "StrKeys": [
+                    "related.resource.id"
+                ],
+                "StrValues": [
+                    "resource-123"
+                ],
+                "BytesKeys": [
+                    "link.test.bytes.value"
+                ],
+                "BytesValues": [
+                    "AQIDBAUG"
+                ]
+            }
+        }
+    ],
+    "Events": [
+        {
+            "Name": "event1",
+            "Timestamp": "2023-12-25T09:53:49Z",
+            "Attributes": {
+                "BoolKeys": [
+                    "inventory.available",
+                    "payment.successful"
+                ],
+                "BoolValues": [
+                    true,
+                    true
+                ],
+                "DoubleKeys": [
+                    "product.price",
+                    "order.discount.rate"
+                ],
+                "DoubleValues": [
+                    6.04,
+                    0.04
+                ],
+                "IntKeys": [
+                    "order.quantity"
+                ],
+                "IntValues": [
+                    2
+                ],
+                "StrKeys": [
+                    "order.id",
+                    "product.id"
+                ],
+                "StrValues": [
+                    "123456789",
+                    "987654321"
+                ],
+                "BytesKeys": [
+                    "event.test.bytes.value"
+                ],
+                "BytesValues": [
+                    "AQIDBAUG"
+                ]
+            }
+        }
+    ]
+}