feat(connector): add iggy-pinot external connector

.markdownlint.yml

@@ -19,6 +19,11 @@ default: true
 MD013:
   line_length: 120
   tables: false
+MD022: false # Headings should be surrounded by blank lines
+MD031: false # Fenced code blocks should be surrounded by blank lines
+MD032: false # Lists should be surrounded by blank lines
 MD033:
   allowed_elements: [details, summary, img]
+MD040: false # Fenced code blocks should have a language specified
 MD041: false # First line in file should be a top level heading
+MD060: false # Table column style

foreign/java/external-processors/iggy-connector-pinot/.gitignore

@@ -0,0 +1,3 @@
+# Documentation files (kept locally, not in git)
+*.md
+*.md.tmp

foreign/java/external-processors/iggy-connector-pinot/build.gradle.kts

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+    id("iggy.java-library-conventions")
+}
+
+dependencies {
+    // Iggy SDK - use local project when building within Iggy repository
+    api(project(":iggy"))
+
+    // Apache Pinot dependencies (provided - not bundled with connector)
+    compileOnly(libs.pinot.spi)
+
+    // Serialization support - use Jackson 2.x for Pinot compatibility
+    implementation(libs.jackson2.databind) {
+        exclude(group = "tools.jackson.core")
+    }
+
+    // Apache Commons
+    implementation(libs.commons.lang3)
+
+    // Logging
+    compileOnly(libs.slf4j.api)
+
+    // Testing
+    testImplementation(platform(libs.junit.bom))
+    testImplementation(libs.bundles.testing)
+    testImplementation(libs.pinot.spi) // Need Pinot SPI for tests
+    testRuntimeOnly(libs.slf4j.simple)
+}
+
+// Task to copy runtime dependencies for Docker deployment (flattened into libs directory)
+tasks.register<Copy>("copyDependencies") {
+    from(configurations.runtimeClasspath)
+    into(layout.buildDirectory.dir("libs"))
+}
+
+// Make jar task depend on copyDependencies
+tasks.named("jar") {
+    finalizedBy("copyDependencies")
+}
+
+publishing {
+    publications {
+        named<MavenPublication>("maven") {
+            artifactId = "pinot-connector"
+
+            pom {
+                name = "Apache Iggy - Pinot Connector"
+                description = "Apache Iggy connector plugin for Apache Pinot stream ingestion"
+            }
+        }
+    }
+}

foreign/java/external-processors/iggy-connector-pinot/deployment/schema.json

@@ -0,0 +1,31 @@
+{
+  "schemaName": "test_events",
+  "dimensionFieldSpecs": [
+    {
+      "name": "userId",
+      "dataType": "STRING"
+    },
+    {
+      "name": "eventType",
+      "dataType": "STRING"
+    },
+    {
+      "name": "deviceType",
+      "dataType": "STRING"
+    }
+  ],
+  "metricFieldSpecs": [
+    {
+      "name": "duration",
+      "dataType": "LONG"
+    }
+  ],
+  "dateTimeFieldSpecs": [
+    {
+      "name": "timestamp",
+      "dataType": "LONG",
+      "format": "1:MILLISECONDS:EPOCH",
+      "granularity": "1:MILLISECONDS"
+    }
+  ]
+}

foreign/java/external-processors/iggy-connector-pinot/deployment/table.json

@@ -0,0 +1,42 @@
+{
+  "tableName": "test_events",
+  "tableType": "REALTIME",
+  "segmentsConfig": {
+    "timeColumnName": "timestamp",
+    "timeType": "MILLISECONDS",
+    "replication": "1",
+    "schemaName": "test_events"
+  },
+  "tenants": {
+    "broker": "DefaultTenant",
+    "server": "DefaultTenant"
+  },
+  "tableIndexConfig": {
+    "loadMode": "MMAP",
+    "streamConfigs": {
+      "streamType": "iggy",
+      "stream.iggy.topic.name": "test-events",
+      "stream.iggy.consumer.type": "lowlevel",
+      "stream.iggy.consumer.factory.class.name": "org.apache.iggy.connector.pinot.consumer.IggyConsumerFactory",
+      "realtime.segment.consumer.factory.class.name": "org.apache.iggy.connector.pinot.consumer.IggyConsumerFactory",
+      "stream.iggy.decoder.class.name": "org.apache.iggy.connector.pinot.decoder.IggyJsonMessageDecoder",
+
+      "stream.iggy.host": "iggy",
+      "stream.iggy.port": "8090",
+      "stream.iggy.username": "iggy",
+      "stream.iggy.password": "iggy",
+
+      "stream.iggy.stream.id": "test-stream",
+      "stream.iggy.topic.id": "test-events",
+      "stream.iggy.consumer.group": "pinot-integration-test",
+
+      "stream.iggy.poll.batch.size": "100",
+      "stream.iggy.connection.pool.size": "4",
+      "stream.iggy.consumer.prop.auto.offset.reset": "smallest",
+
+      "realtime.segment.flush.threshold.rows": "1000",
+      "realtime.segment.flush.threshold.time": "600000"
+    }
+  },
+  "metadata": {}
+}

foreign/java/external-processors/iggy-connector-pinot/docker-compose.yml

@@ -0,0 +1,135 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+services:
+  # Apache Iggy Server (from official Apache repo)
+  iggy:
+    image: apache/iggy:latest
+    container_name: iggy-server
+    ports:
+      - "8090:8090"  # TCP
+      - "3000:3000"  # HTTP
+      - "8080:8080"  # QUIC
+    environment:
+      - IGGY_SYSTEM_LOGGING_LEVEL=info
+      - IGGY_TCP_ADDRESS=0.0.0.0:8090
+      - IGGY_HTTP_ENABLED=true
+      - IGGY_HTTP_ADDRESS=0.0.0.0:3000
+      - IGGY_QUIC_ADDRESS=0.0.0.0:8080
+      - IGGY_ROOT_USERNAME=iggy
+      - IGGY_ROOT_PASSWORD=iggy
+    cap_add:
+      - SYS_NICE
+    security_opt:
+      - seccomp:unconfined
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+    networks:
+      - iggy-pinot-network
+
+  # Zookeeper (required by Pinot)
+  zookeeper:
+    image: zookeeper:3.9
+    container_name: pinot-zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    networks:
+      - iggy-pinot-network
+
+  # Apache Pinot Controller
+  pinot-controller:
+    image: apachepinot/pinot:latest
+    container_name: pinot-controller
+    command: "StartController -zkAddress zookeeper:2181"
+    ports:
+      - "9000:9000"
+    environment:
+      JAVA_OPTS: "-Xms1G -Xmx2G -XX:+UseG1GC"
+    depends_on:
+      - zookeeper
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    volumes:
+      - ./build/libs:/opt/pinot/plugins/pinot-stream-ingestion/iggy-connector
+      - ../../java-sdk/build/libs/iggy-0.6.0.jar:/opt/pinot/plugins/pinot-stream-ingestion/iggy-connector/iggy-0.6.0.jar
+      - ./deployment:/opt/pinot/deployment
+    networks:
+      - iggy-pinot-network
+
+  # Apache Pinot Broker
+  pinot-broker:
+    image: apachepinot/pinot:latest
+    container_name: pinot-broker
+    command: "StartBroker -zkAddress zookeeper:2181"
+    ports:
+      - "8099:8099"
+    environment:
+      JAVA_OPTS: "-Xms512M -Xmx1G -XX:+UseG1GC"
+    depends_on:
+      - pinot-controller
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8099/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    networks:
+      - iggy-pinot-network
+
+  # Apache Pinot Server
+  pinot-server:
+    image: apachepinot/pinot:latest
+    container_name: pinot-server
+    command: "StartServer -zkAddress zookeeper:2181"
+    ports:
+      - "8098:8098"
+      - "8097:8097"
+    environment:
+      JAVA_OPTS: "-Xms1G -Xmx2G -XX:+UseG1GC -Dplugins.include=iggy-connector"
+    depends_on:
+      - pinot-broker
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8097/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    volumes:
+      - ./build/libs:/opt/pinot/plugins/pinot-stream-ingestion/iggy-connector
+      - ../../java-sdk/build/libs/iggy-0.6.0.jar:/opt/pinot/plugins/pinot-stream-ingestion/iggy-connector/iggy-0.6.0.jar
+      - ./deployment:/opt/pinot/deployment
+    networks:
+      - iggy-pinot-network
+
+networks:
+  iggy-pinot-network:
+    driver: bridge

foreign/java/external-processors/iggy-connector-pinot/examples/sample-messages.json

@@ -0,0 +1,47 @@
+{
+  "description": "Sample messages to send to Iggy for testing Pinot ingestion",
+  "messages": [
+    {
+      "userId": "user_12345",
+      "sessionId": "session_abc123",
+      "eventType": "page_view",
+      "pageUrl": "/products/laptop",
+      "deviceType": "desktop",
+      "browser": "Chrome",
+      "country": "USA",
+      "city": "San Francisco",
+      "duration": 45000,
+      "pageLoadTime": 1200,
+      "scrollDepth": 75,
+      "eventTimestamp": 1701234567890
+    },
+    {
+      "userId": "user_67890",
+      "sessionId": "session_def456",
+      "eventType": "click",
+      "pageUrl": "/checkout",
+      "deviceType": "mobile",
+      "browser": "Safari",
+      "country": "UK",
+      "city": "London",
+      "duration": 2000,
+      "pageLoadTime": 800,
+      "scrollDepth": 100,
+      "eventTimestamp": 1701234570000
+    },
+    {
+      "userId": "user_12345",
+      "sessionId": "session_abc123",
+      "eventType": "purchase",
+      "pageUrl": "/confirmation",
+      "deviceType": "desktop",
+      "browser": "Chrome",
+      "country": "USA",
+      "city": "San Francisco",
+      "duration": 10000,
+      "pageLoadTime": 950,
+      "scrollDepth": 50,
+      "eventTimestamp": 1701234580000
+    }
+  ]
+}