┌────────────────────────────────────────────────┐
│            OpenTelemetry SDK                    │
│  ┌──────────┐  ┌──────────┐  ┌──────────────┐ │
│  │  Traces   │  │ Metrics  │  │    Logs      │ │
│  └─────┬────┘  └─────┬────┘  └──────┬───────┘ │
│  ┌─────▼─────────────▼──────────────▼───────┐  │
│  │         OTLP Exporter                    │  │
│  └─────────────────┬────────────────────────┘  │
└────────────────────┼───────────────────────────┘
        ┌────────────┼────────────┐
   ┌────▼───┐  ┌────▼────┐  ┌───▼────┐
   │ Jaeger │  │Prometheus│  │  Loki  │
   └────┬───┘  └────┬────┘  └───┬────┘
   ┌────▼────────────▼───────────▼───┐
   │        Grafana Dashboard         │
   └─────────────────────────────────┘

class OTelGrpcInterceptor(grpc.aio.UnaryUnaryClientInterceptor):
    """OpenTelemetry gRPC tracing interceptor"""

    async def intercept_unary_unary(self, continuation, client_call_details, request):
        method = client_call_details.method
        service, method_name = self._parse_method(method)

        with self._tracer.start_as_current_span(
            f"grpc.{service}/{method_name}",
            kind=trace.SpanKind.CLIENT,
            attributes={
                "rpc.system": "grpc",
                "rpc.service": service,
                "rpc.method": method_name,
                "onto.object_type": self._extract_object_type(request),
            },
        ) as span:
            metadata = list(client_call_details.metadata or [])
            inject(metadata, setter=GrpcMetadataSetter())
            new_details = client_call_details._replace(metadata=metadata)

            try:
                response = await continuation(new_details, request)
                span.set_status(StatusCode.OK)
                return response
            except grpc.aio.AioRpcError as e:
                span.set_status(StatusCode.ERROR, str(e))
                raise

Client SDK → API Gateway → Ontology Service → Data Service → Iceberg
    │            │               │                │            │
    ├── span ────┤               │                │            │
    │            ├── span ───────┤                │            │
    │            │               ├── span ────────┤            │
    │            │               │                ├── span ────┤
    └────────────┴───────────────┴────────────────┴────────────┘
                        TraceID: abc-123

class OntoPlatformMetrics:
    def __init__(self):
        self._meter = metrics.get_meter("coomia-dip")

        self.request_counter = self._meter.create_counter(
            "onto.requests.total", description="Total requests", unit="1",
        )
        self.request_duration = self._meter.create_histogram(
            "onto.request.duration", description="Request duration", unit="ms",
        )
        self.object_operations = self._meter.create_counter(
            "onto.objects.operations.total", description="Object operations",
        )
        self.auth_evaluations = self._meter.create_counter(
            "onto.auth.evaluations.total", description="Auth evaluations",
        )
        self.masking_operations = self._meter.create_counter(
            "onto.masking.operations.total", description="Masking operations",
        )

    def record_request(self, method, object_type, status, duration_ms):
        attrs = {"method": method, "object_type": object_type, "status": status}
        self.request_counter.add(1, attrs)
        self.request_duration.record(duration_ms, attrs)

class OTelLogHandler(logging.Handler):
    """Auto-inject trace context into logs"""

    def emit(self, record: logging.LogRecord) -> None:
        span = trace.get_current_span()
        if span.is_recording():
            ctx = span.get_span_context()
            record.trace_id = format(ctx.trace_id, "032x")
            record.span_id = format(ctx.span_id, "016x")
        else:
            record.trace_id = "0" * 32
            record.span_id = "0" * 16

        log_entry = {
            "timestamp": record.created,
            "level": record.levelname,
            "message": record.getMessage(),
            "trace_id": record.trace_id,
            "span_id": record.span_id,
            "service": "coomia-dip",
            "attributes": getattr(record, "attributes", {}),
        }
        self._export(log_entry)

groups:
  - name: coomia-dip-alerts
    rules:
      - alert: HighRequestLatency
        expr: histogram_quantile(0.99, onto_request_duration_bucket) > 1000
        for: 5m
        labels: { severity: warning }

      - alert: HighErrorRate
        expr: rate(onto_requests_total{status="error"}[5m]) / rate(onto_requests_total[5m]) > 0.05
        for: 2m
        labels: { severity: critical }

      - alert: HighAuthDenialRate
        expr: rate(onto_auth_evaluations_total{result="denied"}[5m]) > 100
        for: 5m
        labels: { severity: warning }

      - alert: ServiceDown
        expr: up{job=~"onto-.*"} == 0
        for: 1m
        labels: { severity: critical }

class TestObservability:
    def test_trace_propagation(self):
        with tracer.start_as_current_span("test-root"):
            response = await client.objects.get("Employee", "emp-001")
            spans = exporter.get_finished_spans()
            trace_ids = set(s.context.trace_id for s in spans)
            assert len(trace_ids) == 1

    def test_metrics_recorded(self):
        await client.objects.get("Employee", "emp-001")
        metric_data = reader.get_metrics_data()
        request_metric = find_metric(metric_data, "onto.requests.total")
        assert request_metric.data_points[0].value >= 1

    def test_log_trace_correlation(self):
        with tracer.start_as_current_span("test") as span:
            trace_id = format(span.get_span_context().trace_id, "032x")
            logger.info("test message")
            log_entry = log_exporter.get_last_entry()
            assert log_entry["trace_id"] == trace_id