返回博客

自动供给模式:资源按需创建

传统平台中,定义一个业务对象后,还需要大量运维工作:

Coomia发布于 2025年12月28日9 分钟阅读
分享本文Twitter / X

自动供给模式:资源按需创建

系列:S10 设计模式 · 第 13 篇 | 难度:高级 | 阅读时间:18 分钟

#TL;DR

  • 自动供给模式(Auto-Provisioning Pattern)让 coomia-dip 在用户定义 Ontology Schema 时自动创建底层基础设施资源——存储表、索引、推理引擎实例、监控仪表盘等,无需手动运维操作。
  • coomia-dip 通过"Schema 变更 → 资源需求推导 → 资源编排 → 健康检查"四阶段管道实现全自动化供给。
  • 支持按需扩缩容(Scale-to-Zero)、资源模板(Resource Template)和多环境供给(Dev/Staging/Prod),实现真正的"定义即部署"体验。

#引言:Ontology 即基础设施

传统平台中,定义一个业务对象后,还需要大量运维工作:

Code
1. 创建数据库表和索引
2. 配置数据同步管道
3. 设置权限和访问控制
4. 部署推理引擎实例
5. 配置监控和告警
6. 生成 API 文档和 SDK

在 coomia-dip 中,这些全部自动完成。用户只需定义 ObjectType Schema,平台自动推导并创建所有必要的基础设施资源。

#一、自动供给管道

#1.1 Schema 变更检测

Python
from dataclasses import dataclass, field
from typing import Any
from enum import Enum

class ChangeType(Enum):
    CREATE = "create"
    UPDATE = "update"
    DELETE = "delete"

@dataclass
class SchemaChange:
    """Detected change in Ontology Schema."""
    change_type: ChangeType
    object_type: str
    old_schema: dict | None = None
    new_schema: dict | None = None
    diff: dict = field(default_factory=dict)

class SchemaChangeDetector:
    """Detect and classify schema changes."""

    async def detect_changes(
        self, old_schemas: dict[str, dict], new_schemas: dict[str, dict]
    ) -> list[SchemaChange]:
        """Detect all changes between old and new schemas."""
        changes = []

        # 新增的 ObjectType
        for name in set(new_schemas) - set(old_schemas):
            changes.append(SchemaChange(
                change_type=ChangeType.CREATE,
                object_type=name,
                new_schema=new_schemas[name],
            ))

        # 删除的 ObjectType
        for name in set(old_schemas) - set(new_schemas):
            changes.append(SchemaChange(
                change_type=ChangeType.DELETE,
                object_type=name,
                old_schema=old_schemas[name],
            ))

        # 修改的 ObjectType
        for name in set(old_schemas) & set(new_schemas):
            if old_schemas[name] != new_schemas[name]:
                changes.append(SchemaChange(
                    change_type=ChangeType.UPDATE,
                    object_type=name,
                    old_schema=old_schemas[name],
                    new_schema=new_schemas[name],
                    diff=self._compute_diff(old_schemas[name], new_schemas[name]),
                ))

        return changes

#1.2 资源需求推导

Python
@dataclass
class ResourceRequirement:
    """A resource that needs to be provisioned."""
    resource_type: str           # iceberg_table | index | grpc_service | dashboard
    Layer: str                   # target Layer
    config: dict[str, Any]       # resource configuration
    dependencies: list[str] = field(default_factory=list)  # 依赖的其他资源
    priority: int = 0            # 优先级(0=最高)

class ResourceDeriver:
    """Derive required resources from schema changes."""

    async def derive(self, change: SchemaChange) -> list[ResourceRequirement]:
        """Derive all resource requirements from a schema change."""
        requirements = []

        if change.change_type == ChangeType.CREATE:
            schema = change.new_schema

            # 1. Iceberg 表
            requirements.append(ResourceRequirement(
                resource_type="iceberg_table",
                Layer="data-Layer",
                config={
                    "table_name": self._to_table_name(change.object_type),
                    "schema": self._to_iceberg_schema(schema),
                    "partition_spec": self._derive_partition(schema),
                    "sort_order": self._derive_sort_order(schema),
                },
                priority=0,
            ))

            # 2. 索引
            indexed_fields = self._find_indexed_fields(schema)
            if indexed_fields:
                requirements.append(ResourceRequirement(
                    resource_type="search_index",
                    Layer="data-Layer",
                    config={
                        "index_name": f"idx_{change.object_type.lower()}",
                        "fields": indexed_fields,
                    },
                    dependencies=[f"iceberg_table:{change.object_type}"],
                    priority=1,
                ))

            # 3. gRPC 服务端点注册
            requirements.append(ResourceRequirement(
                resource_type="grpc_endpoint",
                Layer="control-Layer",
                config={
                    "service": "ObjectQueryService",
                    "object_type": change.object_type,
                    "methods": ["Query", "Get", "Create", "Update", "Delete"],
                },
                priority=1,
            ))

            # 4. 推理引擎配置(如果有状态机或规则)
            if "state_machine" in schema or "rules" in schema:
                requirements.append(ResourceRequirement(
                    resource_type="reasoning_instance",
                    Layer="intelligence-Layer",
                    config={
                        "object_type": change.object_type,
                        "rules": schema.get("rules", []),
                        "state_machine": schema.get("state_machine"),
                    },
                    priority=2,
                ))

            # 5. 监控仪表盘
            requirements.append(ResourceRequirement(
                resource_type="monitoring_dashboard",
                Layer="deployment-Layer",
                config={
                    "object_type": change.object_type,
                    "metrics": ["object_count", "query_latency", "write_throughput"],
                },
                priority=3,
            ))

            # 6. SDK 类型生成
            requirements.append(ResourceRequirement(
                resource_type="sdk_type_generation",
                Layer="sdk-Layer",
                config={
                    "object_type": change.object_type,
                    "schema": schema,
                    "languages": ["python", "typescript"],
                },
                priority=4,
            ))

        return requirements

    def _to_table_name(self, object_type: str) -> str:
        """Convert ObjectType name to Iceberg table name."""
        import re
        return re.sub(r'(?<!^)(?=[A-Z])', '_', object_type).lower()

    def _to_iceberg_schema(self, schema: dict) -> dict:
        """Convert Ontology schema to Iceberg schema."""
        type_mapping = {
            "string": "string",
            "integer": "long",
            "float": "double",
            "boolean": "boolean",
            "datetime": "timestamptz",
            "array": "list",
            "object": "struct",
        }
        iceberg_fields = {}
        for name, prop in schema.get("properties", {}).items():
            iceberg_type = type_mapping.get(prop["type"], "string")
            iceberg_fields[name] = {
                "type": iceberg_type,
                "required": prop.get("required", False),
            }
        return iceberg_fields

    def _derive_partition(self, schema: dict) -> list:
        """Derive Iceberg partition spec from schema hints."""
        partitions = []
        for name, prop in schema.get("properties", {}).items():
            if prop.get("partition"):
                if prop["type"] == "datetime":
                    partitions.append({"field": name, "transform": "month"})
                else:
                    partitions.append({"field": name, "transform": "identity"})
        return partitions

#1.3 资源编排执行

Python
class ResourceOrchestrator:
    """Orchestrate resource provisioning across Layers."""

    async def provision(
        self,
        requirements: list[ResourceRequirement],
        context: dict,
    ) -> dict:
        """Provision all required resources in dependency order."""
        # 按依赖关系拓扑排序
        ordered = self._topological_sort(requirements)

        results = {}
        for req in ordered:
            try:
                provisioner = self._provisioners[req.resource_type]
                result = await provisioner.provision(req.config, context)
                results[f"{req.resource_type}:{req.config.get('object_type', '')}"] = {
                    "status": "provisioned",
                    "details": result,
                }
            except Exception as e:
                results[f"{req.resource_type}:{req.config.get('object_type', '')}"] = {
                    "status": "failed",
                    "error": str(e),
                }
                # 根据策略决定是否继续
                if req.priority == 0:  # 关键资源失败则停止
                    raise ProvisioningError(
                        f"Critical resource failed: {req.resource_type}: {e}"
                    )

        return results

    def _topological_sort(
        self, requirements: list[ResourceRequirement]
    ) -> list[ResourceRequirement]:
        """Sort requirements by dependencies."""
        # 先按 priority 排序,再按依赖关系
        return sorted(requirements, key=lambda r: (r.priority, len(r.dependencies)))

#二、资源模板

#2.1 预定义资源模板

Python
class ResourceTemplate:
    """Predefined resource templates for common patterns."""

    TEMPLATES = {
        "standard_entity": {
            "description": "Standard business entity with CRUD + search",
            "resources": [
                {"type": "iceberg_table", "Layer": "data-Layer"},
                {"type": "search_index", "Layer": "data-Layer"},
                {"type": "grpc_endpoint", "Layer": "control-Layer"},
                {"type": "monitoring_dashboard", "Layer": "deployment-Layer"},
                {"type": "sdk_type_generation", "Layer": "sdk-Layer"},
            ],
        },
        "rule_entity": {
            "description": "Entity with reasoning rules and state machine",
            "resources": [
                {"type": "iceberg_table", "Layer": "data-Layer"},
                {"type": "search_index", "Layer": "data-Layer"},
                {"type": "grpc_endpoint", "Layer": "control-Layer"},
                {"type": "reasoning_instance", "Layer": "intelligence-Layer"},
                {"type": "monitoring_dashboard", "Layer": "deployment-Layer"},
                {"type": "sdk_type_generation", "Layer": "sdk-Layer"},
                {"type": "agent_monitor", "Layer": "agent-runtime"},
            ],
        },
        "event_entity": {
            "description": "Append-only event/log entity",
            "resources": [
                {"type": "iceberg_table", "Layer": "data-Layer",
                 "config_override": {"partition": "day", "sort": "timestamp"}},
                {"type": "monitoring_dashboard", "Layer": "deployment-Layer"},
            ],
        },
    }

    @classmethod
    def get_template(cls, template_name: str) -> dict | None:
        """Get a resource template by name."""
        return cls.TEMPLATES.get(template_name)

#2.2 自定义资源模板

Python
class CustomTemplateRegistry:
    """Registry for custom resource templates."""

    async def register_template(
        self,
        name: str,
        template: dict,
        tenant_id: str,
    ) -> None:
        """Register a custom resource template."""
        # 验证模板格式
        await self._validate_template(template)
        await self._store.save({
            "name": name,
            "template": template,
            "tenant_id": tenant_id,
            "created_at": datetime.utcnow().isoformat(),
        })

    async def apply_template(
        self,
        template_name: str,
        object_type: str,
        schema: dict,
        context: dict,
    ) -> dict:
        """Apply a template to create resources for an ObjectType."""
        template = await self._store.get(template_name)
        requirements = self._template_to_requirements(
            template, object_type, schema
        )
        return await self._orchestrator.provision(requirements, context)

#三、按需扩缩容

#3.1 Scale-to-Zero

不活跃的资源自动缩容到零,降低成本:

Python
class ScaleToZeroManager:
    """Manage scale-to-zero for inactive resources."""

    async def check_and_scale(self) -> list[dict]:
        """Check resource activity and scale inactive ones to zero."""
        actions = []

        for resource in await self._resource_store.list_active():
            last_activity = await self._activity_tracker.get_last_activity(
                resource.resource_id
            )

            idle_hours = (datetime.utcnow() - last_activity).total_seconds() / 3600

            if idle_hours > resource.idle_threshold_hours:
                await self._scale_to_zero(resource)
                actions.append({
                    "resource_id": resource.resource_id,
                    "action": "scaled_to_zero",
                    "idle_hours": idle_hours,
                })

        return actions

    async def _scale_to_zero(self, resource) -> None:
        """Scale a resource to zero."""
        if resource.resource_type == "reasoning_instance":
            await self._intelligence_plane.stop_instance(resource.instance_id)
        elif resource.resource_type == "agent_monitor":
            await self._agent_runtime.pause_agent(resource.agent_id)

    async def scale_up_on_demand(self, resource_id: str) -> None:
        """Scale up a resource when it's needed again."""
        resource = await self._resource_store.get(resource_id)
        if resource.status == "scaled_to_zero":
            await self._provisioners[resource.resource_type].resume(resource)
            resource.status = "active"
            await self._resource_store.save(resource)

#3.2 自动水平扩展

Python
class AutoScaler:
    """Automatically scale resources based on load."""

    async def evaluate(self, resource_id: str) -> dict:
        """Evaluate if a resource needs scaling."""
        metrics = await self._metrics_service.get_resource_metrics(resource_id)

        action = None
        if metrics["cpu_utilization"] > 80 or metrics["queue_depth"] > 100:
            action = "scale_up"
        elif metrics["cpu_utilization"] < 20 and metrics["queue_depth"] < 5:
            action = "scale_down"

        if action:
            await self._execute_scaling(resource_id, action)

        return {
            "resource_id": resource_id,
            "metrics": metrics,
            "action": action,
        }

#四、多环境供给

#4.1 环境感知供给

Python
class EnvironmentAwareProvisioner:
    """Provision resources differently per environment."""

    ENV_CONFIGS = {
        "dev": {
            "iceberg_table": {"replicas": 1, "storage_class": "standard"},
            "reasoning_instance": {"cpu": "0.5", "memory": "512Mi"},
            "monitoring_dashboard": {"enabled": False},
        },
        "staging": {
            "iceberg_table": {"replicas": 2, "storage_class": "standard"},
            "reasoning_instance": {"cpu": "1", "memory": "1Gi"},
            "monitoring_dashboard": {"enabled": True},
        },
        "production": {
            "iceberg_table": {"replicas": 3, "storage_class": "ssd"},
            "reasoning_instance": {"cpu": "4", "memory": "8Gi"},
            "monitoring_dashboard": {"enabled": True},
        },
    }

    async def provision(
        self,
        requirement: ResourceRequirement,
        environment: str,
        context: dict,
    ) -> dict:
        """Provision with environment-specific configuration."""
        env_config = self.ENV_CONFIGS.get(environment, self.ENV_CONFIGS["dev"])
        resource_config = env_config.get(requirement.resource_type, {})

        # 合并环境配置和需求配置
        merged_config = {**requirement.config, **resource_config}
        requirement.config = merged_config

        return await self._provisioners[requirement.resource_type].provision(
            merged_config, context
        )

#五、资源健康检查与自愈

#5.1 定期健康检查

Python
class ResourceHealthChecker:
    """Check health of provisioned resources."""

    async def check_all(self, tenant_id: str) -> dict:
        """Run health checks on all provisioned resources."""
        resources = await self._resource_store.list_by_tenant(tenant_id)
        results = {"healthy": 0, "unhealthy": 0, "details": []}

        for resource in resources:
            healthy = await self._check_resource(resource)
            if healthy:
                results["healthy"] += 1
            else:
                results["unhealthy"] += 1
                # 尝试自愈
                await self._attempt_self_heal(resource)

            results["details"].append({
                "resource_id": resource.resource_id,
                "type": resource.resource_type,
                "healthy": healthy,
            })

        return results

    async def _attempt_self_heal(self, resource) -> bool:
        """Attempt to self-heal an unhealthy resource."""
        try:
            await self._provisioners[resource.resource_type].recreate(
                resource.config
            )
            return True
        except Exception:
            await self._alert_service.send(
                severity="critical",
                title=f"Resource self-heal failed: {resource.resource_id}",
            )
            return False

#六、资源清理与回收

Python
class ResourceCleanup:
    """Clean up resources when ObjectTypes are deleted."""

    async def cleanup(
        self, object_type: str, context: dict
    ) -> dict:
        """Clean up all resources associated with an ObjectType."""
        resources = await self._resource_store.find_by_object_type(object_type)

        cleaned = []
        for resource in reversed(resources):  # 反序清理(先清子资源)
            try:
                await self._provisioners[resource.resource_type].deprovision(
                    resource.config
                )
                cleaned.append(resource.resource_id)
            except Exception as e:
                await self._alert_service.send(
                    severity="warning",
                    title=f"Resource cleanup failed: {resource.resource_id}",
                    message=str(e),
                )

        return {"cleaned": len(cleaned), "total": len(resources)}

#Key Takeaways

  1. 定义即部署:用户只需定义 Ontology Schema,所有底层资源自动创建
  2. 四阶段管道:检测变更 → 推导需求 → 编排供给 → 健康检查
  3. 资源模板:预定义和自定义模板加速常见模式的资源供给
  4. Scale-to-Zero:不活跃资源自动缩容,按需恢复
  5. 环境感知:Dev/Staging/Prod 不同环境自动适配资源规格
  6. 自愈能力:健康检查 + 自动重建确保资源持续可用

#Next Article

S10-14: 层级多租户:组织结构感知的资源隔离

#Tags

#设计模式 #自动供给 #AutoProvisioning #Schema驱动 #ScaleToZero #资源模板 #多环境 #自愈