deepflowio · lzf575 · Jan 3, 2025
diff --git a/server/ingester/ckissu/ckissu.go b/server/ingester/ckissu/ckissu.go
@@ -381,7 +381,7 @@ func (i *Issu) addColumnDatasource(index int, d *DatasourceInfo, isMapTable, isA
 		return nil, fmt.Errorf("invalid table name %s", d.name)
 	}
 
-	rawTable := flow_metrics.GetMetricsTables(ckdb.MergeTree, common.CK_VERSION, ckdb.DF_CLUSTER, ckdb.DF_STORAGE_POLICY, i.ckdbType, 7, 1, 7, 1, i.cfg.GetCKDBColdStorages())[flow_metrics.MetricsTableNameToID(d.name[:lastDotIndex+1]+d.baseTable)]
+	rawTable := flow_metrics.GetMetricsTables(ckdb.MergeTree, common.CK_VERSION, ckdb.DF_CLUSTER, ckdb.DF_STORAGE_POLICY, i.ckdbType, 7, 1, 7, 1, i.cfg.GetCKDBColdStorages())[flow_metrics.MetricsTableNameToID(d.name[:lastDotIndex+1]+"1m")]
 	// create table mv
 	aggrInterval := ckdb.AggregationHour
 	if d.interval == ckdb.TimeFuncDay {

diff --git a/server/ingester/ckissu/updates.go b/server/ingester/ckissu/updates.go
@@ -20,7 +20,7 @@ import (
 	"github.com/deepflowio/deepflow/server/libs/ckdb"
 )
 
-var AllColumnAdds = [][]*ColumnAdds{ColumnAdd64, ColumnAdd65, ColumnAdd66}
+var AllColumnAdds = [][]*ColumnAdds{ColumnAdd64, ColumnAdd65, ColumnAdd66, ColumnAdd70}
 var AllIndexAdds = [][]*IndexAdd{getIndexAdds(IndexAdd64), getIndexAdds(IndexAdd65)}
 var AllColumnMods = [][]*ColumnMod{}
 var AllColumnRenames = [][]*ColumnRename{getColumnRenames(ColumnRename65)}
@@ -376,3 +376,24 @@ var ColumnAdd66 = []*ColumnAdds{
 		ColumnType:  ckdb.UInt8,
 	},
 }
+
+var ColumnAdd70 = []*ColumnAdds{
+	{
+		Dbs:         []string{"deepflow_tenant"},
+		Tables:      []string{"deepflow_collector", "deepflow_collector_local"},
+		ColumnNames: []string{"host"},
+		ColumnType:  ckdb.LowCardinalityString,
+	},
+	{
+		Dbs:         []string{"deepflow_admin"},
+		Tables:      []string{"deepflow_server", "deepflow_server_local"},
+		ColumnNames: []string{"host"},
+		ColumnType:  ckdb.LowCardinalityString,
+	},
+	{
+		Dbs:         []string{"ext_metrics"},
+		Tables:      []string{"metrics", "metrics_local"},
+		ColumnNames: []string{"host"},
+		ColumnType:  ckdb.LowCardinalityString,
+	},
+}
diff --git a/server/ingester/common/const.go b/server/ingester/common/const.go
@@ -17,5 +17,5 @@
 package common
 
 const (
-	CK_VERSION = "v6.6.3.0" // 用于表示clickhouse的表版本号
+	CK_VERSION = "v7.0.0.0" // 用于表示clickhouse的表版本号
 )
diff --git a/server/ingester/config/watcher.go b/server/ingester/config/watcher.go
@@ -29,6 +29,7 @@ import (
 	"k8s.io/client-go/rest"
 
 	libs "github.com/deepflowio/deepflow/server/libs/kubernetes"
+	"github.com/deepflowio/deepflow/server/libs/utils"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -170,15 +171,6 @@ func (w *Watcher) Run() {
 	}
 }
 
-func indexOf(ss []string, s string) int {
-	for i, v := range ss {
-		if v == s {
-			return i
-		}
-	}
-	return -1
-}
-
 func (w *Watcher) getMyClickhouseEndpointsExternal() ([]Endpoint, error) {
 	podNames, err := w.getPodNames()
 	if err != nil {
@@ -197,7 +189,7 @@ func (w *Watcher) getMyClickhouseEndpointsExternal() ([]Endpoint, error) {
 // 2. Input the list of all clickhouse endpoints, and sort by IP
 // 3, my corresponding 'clickhouse endpoint' is on position 'index%len'  in the 'clickhouse endpoints list'
 func getMyClickhouseEndpoints(podNames []string, myName string, endpoints []Endpoint) ([]Endpoint, error) {
-	myIndex := indexOf(podNames, myName)
+	myIndex := utils.IndexOf(podNames, myName)
 	if myIndex < 0 {
 		return nil, fmt.Errorf("can't find my pod name(%s) in pods(%v)", myName, podNames)
 	}

diff --git a/server/ingester/ext_metrics/dbwriter/ext_metrics.go b/server/ingester/ext_metrics/dbwriter/ext_metrics.go
@@ -36,6 +36,7 @@ type ExtMetrics struct {
 	UniversalTag flow_metrics.UniversalTag
 
 	VTableName string
+	Host       string
 
 	AgentID uint16
 
@@ -107,6 +108,7 @@ func (m *ExtMetrics) Columns() []*ckdb.Column {
 		ckdb.NewColumn("tag_values", ckdb.ArrayLowCardinalityString).SetComment("额外的tag对应的值"),
 		ckdb.NewColumn("metrics_float_names", ckdb.ArrayLowCardinalityString).SetComment("额外的float类型metrics"),
 		ckdb.NewColumn("metrics_float_values", ckdb.ArrayFloat64).SetComment("额外的float metrics值"),
+		ckdb.NewColumn("host", ckdb.LowCardinalityString),
 	)
 
 	return columns

diff --git a/server/ingester/ext_metrics/dbwriter/ext_metrics_column_block.go b/server/ingester/ext_metrics/dbwriter/ext_metrics_column_block.go
@@ -21,6 +21,7 @@ import (
 	"github.com/deepflowio/deepflow/server/libs/ckdb"
 	"github.com/deepflowio/deepflow/server/libs/datatype"
 	flow_metrics "github.com/deepflowio/deepflow/server/libs/flow-metrics"
+	"github.com/deepflowio/deepflow/server/libs/utils"
 )
 
 type ExtMetricsBlock struct {
@@ -33,6 +34,7 @@ type ExtMetricsBlock struct {
 	ColTagValues          *proto.ColArr[string]
 	ColMetricsFloatNames  *proto.ColArr[string]
 	ColMetricsFloatValues *proto.ColArr[float64]
+	ColHost               *proto.ColLowCardinality[string]
 }
 
 func (b *ExtMetricsBlock) Reset() {
@@ -46,6 +48,7 @@ func (b *ExtMetricsBlock) Reset() {
 	b.ColTagValues.Reset()
 	b.ColMetricsFloatNames.Reset()
 	b.ColMetricsFloatValues.Reset()
+	b.ColHost.Reset()
 }
 
 func (b *ExtMetricsBlock) ToInput(input proto.Input) proto.Input {
@@ -57,6 +60,7 @@ func (b *ExtMetricsBlock) ToInput(input proto.Input) proto.Input {
 		proto.InputColumn{Name: ckdb.COLUMN_TAG_VALUES, Data: b.ColTagValues},
 		proto.InputColumn{Name: ckdb.COLUMN_METRICS_FLOAT_NAMES, Data: b.ColMetricsFloatNames},
 		proto.InputColumn{Name: ckdb.COLUMN_METRICS_FLOAT_VALUES, Data: b.ColMetricsFloatValues},
+		proto.InputColumn{Name: ckdb.COLUMN_HOST, Data: b.ColHost},
 	)
 	if b.MsgType != datatype.MESSAGE_TYPE_DFSTATS && b.MsgType != datatype.MESSAGE_TYPE_SERVER_DFSTATS {
 		input = b.UniversalTagBlock.ToInput(input)
@@ -73,6 +77,7 @@ func (n *ExtMetrics) NewColumnBlock() ckdb.CKColumnBlock {
 		ColTagValues:          new(proto.ColStr).Array(),
 		ColMetricsFloatNames:  new(proto.ColStr).LowCardinality().Array(),
 		ColMetricsFloatValues: new(proto.ColFloat64).Array(),
+		ColHost:               new(proto.ColStr).LowCardinality(),
 	}
 }
 
@@ -89,4 +94,9 @@ func (n *ExtMetrics) AppendToColumnBlock(b ckdb.CKColumnBlock) {
 	block.ColTagValues.Append(n.TagValues)
 	block.ColMetricsFloatNames.Append(n.MetricsFloatNames)
 	block.ColMetricsFloatValues.Append(n.MetricsFloatValues)
+	if i := utils.IndexOf(n.TagNames, ckdb.COLUMN_HOST); i >= 0 {
+		block.ColHost.Append(n.TagValues[i])
+	} else {
+		block.ColHost.Append("")
+	}
 }
diff --git a/server/libs/ckdb/column.go b/server/libs/ckdb/column.go
@@ -106,6 +106,7 @@ const (
 	COLUMN_GPROCESS_ID                = "gprocess_id"
 	COLUMN_GPROCESS_ID_0              = "gprocess_id_0"
 	COLUMN_GPROCESS_ID_1              = "gprocess_id_1"
+	COLUMN_HOST                       = "host"
 	COLUMN_HOST_ID                    = "host_id"
 	COLUMN_HOST_ID_0                  = "host_id_0"
 	COLUMN_HOST_ID_1                  = "host_id_1"

diff --git a/server/libs/utils/utils.go b/server/libs/utils/utils.go
@@ -524,3 +524,12 @@ func EscapeJsonString(str string) string {
 func CloneStringSlice(strs []string) []string {
 	return append([]string{}, strs...)
 }
+
+func IndexOf(strs []string, str string) int {
+	for i, v := range strs {
+		if v == str {
+			return i
+		}
+	}
+	return -1
+}
diff --git a/server/querier/db_descriptions/clickhouse/tag/deepflow_admin/deepflow_server_common b/server/querier/db_descriptions/clickhouse/tag/deepflow_admin/deepflow_server_common
@@ -1,2 +1,3 @@
-# Name                     , ClientName                , ServerName                , Type          , EnumFile           , Category    , Permission   , Deprecated
-time                       , time                      , time                      , time          ,                    , Timestamp   , 111          , 0
+# Name                     , ClientName                , ServerName                , Type          , EnumFile           , Category        , Permission   , Deprecated
+time                       , time                      , time                      , time          ,                    , Timestamp       , 111           , 0
+host                       , host                      , host                      , string        ,                    , Universal Tag   , 111           , 0
diff --git a/server/querier/db_descriptions/clickhouse/tag/deepflow_admin/deepflow_server_common.ch b/server/querier/db_descriptions/clickhouse/tag/deepflow_admin/deepflow_server_common.ch
@@ -1,2 +1,3 @@
 # Name                     , DisplayName                , Description
 time                       , 时间                       ,
+host                       , 数据节点                   ,
diff --git a/server/querier/db_descriptions/clickhouse/tag/deepflow_admin/deepflow_server_common.en b/server/querier/db_descriptions/clickhouse/tag/deepflow_admin/deepflow_server_common.en
@@ -1,2 +1,3 @@
 # Name                     , DisplayName                , Description
 time                       , Time                       ,
+host                       , Data Node                  ,
diff --git a/server/querier/db_descriptions/clickhouse/tag/deepflow_tenant/deepflow_collector_common b/server/querier/db_descriptions/clickhouse/tag/deepflow_tenant/deepflow_collector_common
@@ -1,2 +1,3 @@
-# Name                     , ClientName                , ServerName                , Type          , EnumFile           , Category    , Permission   , Deprecated
-time                       , time                      , time                      , time          ,                    , Timestamp   , 111          , 0
+# Name                     , ClientName                , ServerName                , Type          , EnumFile           , Category        , Permission   , Deprecated
+time                       , time                      , time                      , time          ,                    , Timestamp       , 111          , 0
+host                       , host                      , host                      , string        ,                    , Universal Tag   , 111          , 0
diff --git a/server/querier/db_descriptions/clickhouse/tag/deepflow_tenant/deepflow_collector_common.ch b/server/querier/db_descriptions/clickhouse/tag/deepflow_tenant/deepflow_collector_common.ch
@@ -1,2 +1,3 @@
 # Name                     , DisplayName                , Description
 time                       , 时间                       ,
+host                       , 采集节点                   ,
diff --git a/server/querier/db_descriptions/clickhouse/tag/deepflow_tenant/deepflow_collector_common.en b/server/querier/db_descriptions/clickhouse/tag/deepflow_tenant/deepflow_collector_common.en
@@ -1,2 +1,3 @@
 # Name                     , DisplayName                , Description
 time                       , Time                       ,
+host                       , Collection Node            ,