clickhouse-webhooks-events

# ClickHouse Data Ingestion ## Overview Build data ingestion pipelines into ClickHouse from HTTP webhooks, Kafka, and streaming sources with proper batching, deduplication, and error handling. ## Prerequisites - ClickHouse table with appropriate engine (see `clickhouse-core-workflow-a`) - `@clickhouse/client` connected ## Instructions ### Step 1: Webhook Receiver with Batched Inserts ```typescript import express from 'express'; import { createClient } from '@clickhouse/client'; const client = createClient({ url: process.env.CLICKHOUSE_HOST! }); const app = express(); app.use(express.json()); // Buffer for batching — ClickHouse hates one-row-at-a-time inserts const buffer: Record<string, unknown>[] = []; const BATCH_SIZE = 5_000; const FLUSH_INTERVAL_MS = 5_000; async function flushBuffer() { if (buffer.length === 0) return; const batch = buffer.splice(0, buffer.length); try { await client.insert({ table: 'analytics.events', values: batch, format: 'JSONEachRow', }); console.log(`Flushed ${batch.length} events to ClickHouse`); } catch (err) { console.error('Insert failed, re-queuing:', (err as Error).message); buffer.unshift(...batch); // Put back at front for retry } } // Flush periodically setInterval(flushBuffer, FLUSH_INTERVAL_MS); // Webhook endpoint app.post('/ingest', async (req, res) => { const events = Array.isArray(req.body) ? req.body : [req.body]; for (const event of events) { buffer.push({ ...