自动供给模式:资源按需创建
传统平台中,定义一个业务对象后,还需要大量运维工作:
Coomia发布于 2025年12月28日9 分钟阅读
分享本文Twitter / X
自动供给模式:资源按需创建
“系列:S10 设计模式 · 第 13 篇 | 难度:高级 | 阅读时间:18 分钟
#TL;DR
- 自动供给模式(Auto-Provisioning Pattern)让 coomia-dip 在用户定义 Ontology Schema 时自动创建底层基础设施资源——存储表、索引、推理引擎实例、监控仪表盘等,无需手动运维操作。
- coomia-dip 通过"Schema 变更 → 资源需求推导 → 资源编排 → 健康检查"四阶段管道实现全自动化供给。
- 支持按需扩缩容(Scale-to-Zero)、资源模板(Resource Template)和多环境供给(Dev/Staging/Prod),实现真正的"定义即部署"体验。
#引言:Ontology 即基础设施
传统平台中,定义一个业务对象后,还需要大量运维工作:
Code
1. 创建数据库表和索引
2. 配置数据同步管道
3. 设置权限和访问控制
4. 部署推理引擎实例
5. 配置监控和告警
6. 生成 API 文档和 SDK
在 coomia-dip 中,这些全部自动完成。用户只需定义 ObjectType Schema,平台自动推导并创建所有必要的基础设施资源。
#一、自动供给管道
#1.1 Schema 变更检测
Python
from dataclasses import dataclass, field
from typing import Any
from enum import Enum
class ChangeType(Enum):
CREATE = "create"
UPDATE = "update"
DELETE = "delete"
@dataclass
class SchemaChange:
"""Detected change in Ontology Schema."""
change_type: ChangeType
object_type: str
old_schema: dict | None = None
new_schema: dict | None = None
diff: dict = field(default_factory=dict)
class SchemaChangeDetector:
"""Detect and classify schema changes."""
async def detect_changes(
self, old_schemas: dict[str, dict], new_schemas: dict[str, dict]
) -> list[SchemaChange]:
"""Detect all changes between old and new schemas."""
changes = []
# 新增的 ObjectType
for name in set(new_schemas) - set(old_schemas):
changes.append(SchemaChange(
change_type=ChangeType.CREATE,
object_type=name,
new_schema=new_schemas[name],
))
# 删除的 ObjectType
for name in set(old_schemas) - set(new_schemas):
changes.append(SchemaChange(
change_type=ChangeType.DELETE,
object_type=name,
old_schema=old_schemas[name],
))
# 修改的 ObjectType
for name in set(old_schemas) & set(new_schemas):
if old_schemas[name] != new_schemas[name]:
changes.append(SchemaChange(
change_type=ChangeType.UPDATE,
object_type=name,
old_schema=old_schemas[name],
new_schema=new_schemas[name],
diff=self._compute_diff(old_schemas[name], new_schemas[name]),
))
return changes
#1.2 资源需求推导
Python
@dataclass
class ResourceRequirement:
"""A resource that needs to be provisioned."""
resource_type: str # iceberg_table | index | grpc_service | dashboard
Layer: str # target Layer
config: dict[str, Any] # resource configuration
dependencies: list[str] = field(default_factory=list) # 依赖的其他资源
priority: int = 0 # 优先级(0=最高)
class ResourceDeriver:
"""Derive required resources from schema changes."""
async def derive(self, change: SchemaChange) -> list[ResourceRequirement]:
"""Derive all resource requirements from a schema change."""
requirements = []
if change.change_type == ChangeType.CREATE:
schema = change.new_schema
# 1. Iceberg 表
requirements.append(ResourceRequirement(
resource_type="iceberg_table",
Layer="data-Layer",
config={
"table_name": self._to_table_name(change.object_type),
"schema": self._to_iceberg_schema(schema),
"partition_spec": self._derive_partition(schema),
"sort_order": self._derive_sort_order(schema),
},
priority=0,
))
# 2. 索引
indexed_fields = self._find_indexed_fields(schema)
if indexed_fields:
requirements.append(ResourceRequirement(
resource_type="search_index",
Layer="data-Layer",
config={
"index_name": f"idx_{change.object_type.lower()}",
"fields": indexed_fields,
},
dependencies=[f"iceberg_table:{change.object_type}"],
priority=1,
))
# 3. gRPC 服务端点注册
requirements.append(ResourceRequirement(
resource_type="grpc_endpoint",
Layer="control-Layer",
config={
"service": "ObjectQueryService",
"object_type": change.object_type,
"methods": ["Query", "Get", "Create", "Update", "Delete"],
},
priority=1,
))
# 4. 推理引擎配置(如果有状态机或规则)
if "state_machine" in schema or "rules" in schema:
requirements.append(ResourceRequirement(
resource_type="reasoning_instance",
Layer="intelligence-Layer",
config={
"object_type": change.object_type,
"rules": schema.get("rules", []),
"state_machine": schema.get("state_machine"),
},
priority=2,
))
# 5. 监控仪表盘
requirements.append(ResourceRequirement(
resource_type="monitoring_dashboard",
Layer="deployment-Layer",
config={
"object_type": change.object_type,
"metrics": ["object_count", "query_latency", "write_throughput"],
},
priority=3,
))
# 6. SDK 类型生成
requirements.append(ResourceRequirement(
resource_type="sdk_type_generation",
Layer="sdk-Layer",
config={
"object_type": change.object_type,
"schema": schema,
"languages": ["python", "typescript"],
},
priority=4,
))
return requirements
def _to_table_name(self, object_type: str) -> str:
"""Convert ObjectType name to Iceberg table name."""
import re
return re.sub(r'(?<!^)(?=[A-Z])', '_', object_type).lower()
def _to_iceberg_schema(self, schema: dict) -> dict:
"""Convert Ontology schema to Iceberg schema."""
type_mapping = {
"string": "string",
"integer": "long",
"float": "double",
"boolean": "boolean",
"datetime": "timestamptz",
"array": "list",
"object": "struct",
}
iceberg_fields = {}
for name, prop in schema.get("properties", {}).items():
iceberg_type = type_mapping.get(prop["type"], "string")
iceberg_fields[name] = {
"type": iceberg_type,
"required": prop.get("required", False),
}
return iceberg_fields
def _derive_partition(self, schema: dict) -> list:
"""Derive Iceberg partition spec from schema hints."""
partitions = []
for name, prop in schema.get("properties", {}).items():
if prop.get("partition"):
if prop["type"] == "datetime":
partitions.append({"field": name, "transform": "month"})
else:
partitions.append({"field": name, "transform": "identity"})
return partitions
#1.3 资源编排执行
Python
class ResourceOrchestrator:
"""Orchestrate resource provisioning across Layers."""
async def provision(
self,
requirements: list[ResourceRequirement],
context: dict,
) -> dict:
"""Provision all required resources in dependency order."""
# 按依赖关系拓扑排序
ordered = self._topological_sort(requirements)
results = {}
for req in ordered:
try:
provisioner = self._provisioners[req.resource_type]
result = await provisioner.provision(req.config, context)
results[f"{req.resource_type}:{req.config.get('object_type', '')}"] = {
"status": "provisioned",
"details": result,
}
except Exception as e:
results[f"{req.resource_type}:{req.config.get('object_type', '')}"] = {
"status": "failed",
"error": str(e),
}
# 根据策略决定是否继续
if req.priority == 0: # 关键资源失败则停止
raise ProvisioningError(
f"Critical resource failed: {req.resource_type}: {e}"
)
return results
def _topological_sort(
self, requirements: list[ResourceRequirement]
) -> list[ResourceRequirement]:
"""Sort requirements by dependencies."""
# 先按 priority 排序,再按依赖关系
return sorted(requirements, key=lambda r: (r.priority, len(r.dependencies)))
#二、资源模板
#2.1 预定义资源模板
Python
class ResourceTemplate:
"""Predefined resource templates for common patterns."""
TEMPLATES = {
"standard_entity": {
"description": "Standard business entity with CRUD + search",
"resources": [
{"type": "iceberg_table", "Layer": "data-Layer"},
{"type": "search_index", "Layer": "data-Layer"},
{"type": "grpc_endpoint", "Layer": "control-Layer"},
{"type": "monitoring_dashboard", "Layer": "deployment-Layer"},
{"type": "sdk_type_generation", "Layer": "sdk-Layer"},
],
},
"rule_entity": {
"description": "Entity with reasoning rules and state machine",
"resources": [
{"type": "iceberg_table", "Layer": "data-Layer"},
{"type": "search_index", "Layer": "data-Layer"},
{"type": "grpc_endpoint", "Layer": "control-Layer"},
{"type": "reasoning_instance", "Layer": "intelligence-Layer"},
{"type": "monitoring_dashboard", "Layer": "deployment-Layer"},
{"type": "sdk_type_generation", "Layer": "sdk-Layer"},
{"type": "agent_monitor", "Layer": "agent-runtime"},
],
},
"event_entity": {
"description": "Append-only event/log entity",
"resources": [
{"type": "iceberg_table", "Layer": "data-Layer",
"config_override": {"partition": "day", "sort": "timestamp"}},
{"type": "monitoring_dashboard", "Layer": "deployment-Layer"},
],
},
}
@classmethod
def get_template(cls, template_name: str) -> dict | None:
"""Get a resource template by name."""
return cls.TEMPLATES.get(template_name)
#2.2 自定义资源模板
Python
class CustomTemplateRegistry:
"""Registry for custom resource templates."""
async def register_template(
self,
name: str,
template: dict,
tenant_id: str,
) -> None:
"""Register a custom resource template."""
# 验证模板格式
await self._validate_template(template)
await self._store.save({
"name": name,
"template": template,
"tenant_id": tenant_id,
"created_at": datetime.utcnow().isoformat(),
})
async def apply_template(
self,
template_name: str,
object_type: str,
schema: dict,
context: dict,
) -> dict:
"""Apply a template to create resources for an ObjectType."""
template = await self._store.get(template_name)
requirements = self._template_to_requirements(
template, object_type, schema
)
return await self._orchestrator.provision(requirements, context)
#三、按需扩缩容
#3.1 Scale-to-Zero
不活跃的资源自动缩容到零,降低成本:
Python
class ScaleToZeroManager:
"""Manage scale-to-zero for inactive resources."""
async def check_and_scale(self) -> list[dict]:
"""Check resource activity and scale inactive ones to zero."""
actions = []
for resource in await self._resource_store.list_active():
last_activity = await self._activity_tracker.get_last_activity(
resource.resource_id
)
idle_hours = (datetime.utcnow() - last_activity).total_seconds() / 3600
if idle_hours > resource.idle_threshold_hours:
await self._scale_to_zero(resource)
actions.append({
"resource_id": resource.resource_id,
"action": "scaled_to_zero",
"idle_hours": idle_hours,
})
return actions
async def _scale_to_zero(self, resource) -> None:
"""Scale a resource to zero."""
if resource.resource_type == "reasoning_instance":
await self._intelligence_plane.stop_instance(resource.instance_id)
elif resource.resource_type == "agent_monitor":
await self._agent_runtime.pause_agent(resource.agent_id)
async def scale_up_on_demand(self, resource_id: str) -> None:
"""Scale up a resource when it's needed again."""
resource = await self._resource_store.get(resource_id)
if resource.status == "scaled_to_zero":
await self._provisioners[resource.resource_type].resume(resource)
resource.status = "active"
await self._resource_store.save(resource)
#3.2 自动水平扩展
Python
class AutoScaler:
"""Automatically scale resources based on load."""
async def evaluate(self, resource_id: str) -> dict:
"""Evaluate if a resource needs scaling."""
metrics = await self._metrics_service.get_resource_metrics(resource_id)
action = None
if metrics["cpu_utilization"] > 80 or metrics["queue_depth"] > 100:
action = "scale_up"
elif metrics["cpu_utilization"] < 20 and metrics["queue_depth"] < 5:
action = "scale_down"
if action:
await self._execute_scaling(resource_id, action)
return {
"resource_id": resource_id,
"metrics": metrics,
"action": action,
}
#四、多环境供给
#4.1 环境感知供给
Python
class EnvironmentAwareProvisioner:
"""Provision resources differently per environment."""
ENV_CONFIGS = {
"dev": {
"iceberg_table": {"replicas": 1, "storage_class": "standard"},
"reasoning_instance": {"cpu": "0.5", "memory": "512Mi"},
"monitoring_dashboard": {"enabled": False},
},
"staging": {
"iceberg_table": {"replicas": 2, "storage_class": "standard"},
"reasoning_instance": {"cpu": "1", "memory": "1Gi"},
"monitoring_dashboard": {"enabled": True},
},
"production": {
"iceberg_table": {"replicas": 3, "storage_class": "ssd"},
"reasoning_instance": {"cpu": "4", "memory": "8Gi"},
"monitoring_dashboard": {"enabled": True},
},
}
async def provision(
self,
requirement: ResourceRequirement,
environment: str,
context: dict,
) -> dict:
"""Provision with environment-specific configuration."""
env_config = self.ENV_CONFIGS.get(environment, self.ENV_CONFIGS["dev"])
resource_config = env_config.get(requirement.resource_type, {})
# 合并环境配置和需求配置
merged_config = {**requirement.config, **resource_config}
requirement.config = merged_config
return await self._provisioners[requirement.resource_type].provision(
merged_config, context
)
#五、资源健康检查与自愈
#5.1 定期健康检查
Python
class ResourceHealthChecker:
"""Check health of provisioned resources."""
async def check_all(self, tenant_id: str) -> dict:
"""Run health checks on all provisioned resources."""
resources = await self._resource_store.list_by_tenant(tenant_id)
results = {"healthy": 0, "unhealthy": 0, "details": []}
for resource in resources:
healthy = await self._check_resource(resource)
if healthy:
results["healthy"] += 1
else:
results["unhealthy"] += 1
# 尝试自愈
await self._attempt_self_heal(resource)
results["details"].append({
"resource_id": resource.resource_id,
"type": resource.resource_type,
"healthy": healthy,
})
return results
async def _attempt_self_heal(self, resource) -> bool:
"""Attempt to self-heal an unhealthy resource."""
try:
await self._provisioners[resource.resource_type].recreate(
resource.config
)
return True
except Exception:
await self._alert_service.send(
severity="critical",
title=f"Resource self-heal failed: {resource.resource_id}",
)
return False
#六、资源清理与回收
Python
class ResourceCleanup:
"""Clean up resources when ObjectTypes are deleted."""
async def cleanup(
self, object_type: str, context: dict
) -> dict:
"""Clean up all resources associated with an ObjectType."""
resources = await self._resource_store.find_by_object_type(object_type)
cleaned = []
for resource in reversed(resources): # 反序清理(先清子资源)
try:
await self._provisioners[resource.resource_type].deprovision(
resource.config
)
cleaned.append(resource.resource_id)
except Exception as e:
await self._alert_service.send(
severity="warning",
title=f"Resource cleanup failed: {resource.resource_id}",
message=str(e),
)
return {"cleaned": len(cleaned), "total": len(resources)}
#Key Takeaways
- 定义即部署:用户只需定义 Ontology Schema,所有底层资源自动创建
- 四阶段管道:检测变更 → 推导需求 → 编排供给 → 健康检查
- 资源模板:预定义和自定义模板加速常见模式的资源供给
- Scale-to-Zero:不活跃资源自动缩容,按需恢复
- 环境感知:Dev/Staging/Prod 不同环境自动适配资源规格
- 自愈能力:健康检查 + 自动重建确保资源持续可用
#Next Article
S10-14: 层级多租户:组织结构感知的资源隔离
#Tags
#设计模式 #自动供给 #AutoProvisioning #Schema驱动 #ScaleToZero #资源模板 #多环境 #自愈