华为云 MaaS DeepSeek 推理服务高并发架构实战：从单一请求到万级 QPS 的演进之路

┌─────────────────────────────────────────────────────┐
│                   用户请求/API网关                      │
└────────────────────────┬────────────────────────────┘
                         │
┌────────────────────────▼────────────────────────────┐
│              华为云 ELB（负载均衡）                      │
└────┬──────────┬──────────┬──────────┬───────────────┘
     │          │          │          │
┌────▼───┐ ┌───▼────┐ ┌───▼────┐ ┌───▼────┐
│GPU实例1│ │GPU实例2│ │GPU实例3│ │GPU实例N│ ← 弹性伸缩
│DeepSeek│ │DeepSeek│ │DeepSeek│ │DeepSeek│
└────────┘ └────────┘ └────────┘ └────────┘
     │          │          │          │
┌────▼──────────▼──────────▼──────────▼──────────────┐
│              共享存储 (Model Files / KV Cache)        │
└─────────────────────────────────────────────────────┘

部署方式：     Flexus 独占实例 / Serverless 弹性推理
实例规格：     GPU 加速型（推荐 p2vs.2xlarge，2*Ascend 910B）
推理引擎：     MindSpore Lite / vLLM（支持）
并行策略：     单实例 / Tensor Parallel=2
最大并发数：   推荐起始 8-16
上下文窗口：   32K（默认）/ 128K（需扩容显存）

# benchmark_single.py
import time
import asyncio
import aiohttp
import statistics

API_URL = "https://maas-infer.cn-north-4.myhuaweicloud.com/v1/chat/completions"
API_KEY = "your-api-key-here"

async def send_request(session, prompt: str, idx: int) -> dict:
    """发送单个推理请求并记录耗时"""
    payload = {
        "model": "deepseek-r1",
        "messages": [{"role": "user", "content": prompt}],
        "max_tokens": 512,
        "temperature": 0.7
    }
    headers = {
        "Authorization": f"Bearer {API_KEY}",
        "Content-Type": "application/json"
    }

    start = time.time()
    ttft = None

    # 流式响应，记录首 Token 时间
    async with session.post(API_URL, json=payload, headers=headers) as resp:
        async for chunk in resp.content:
            if ttft is None:
                ttft = time.time() - start
            # 处理流式数据...（简化）

    total_time = time.time() - start
    return {
        "idx": idx,
        "ttft": ttft,
        "total_time": total_time
    }

async def run_benchmark(concurrency: int, num_requests: int):
    """运行并发基准测试"""
    prompts = [f"请详细解释什么是transformer架构的第{i}个关键技术点" 
               for i in range(num_requests)]

    async with aiohttp.ClientSession() as session:
        sem = asyncio.Semaphore(concurrency)

        async def bounded_request(prompt, idx):
            async with sem:
                return await send_request(session, prompt, idx)

        tasks = [bounded_request(prompts[i], i) for i in range(num_requests)]
        results = await asyncio.gather(*tasks)

    # 统计
    ttfts = [r["ttft"] for r in results if r["ttft"]]
    totals = [r["total_time"] for r in results]

    print(f"并发数: {concurrency}")
    print(f"总请求数: {num_requests}")
    print(f"平均 TTFT: {statistics.mean(ttfts):.3f}s")
    print(f"P95 TTFT: {sorted(ttfts)[int(len(ttfts)*0.95)]:.3f}s")
    print(f"平均总耗时: {statistics.mean(totals):.3f}s")
    print(f"总耗时: {sum(totals):.1f}s")
    print(f"QPS: {num_requests / sum(totals):.2f}")

    return results

# 测试不同并发级别
for concurrency in [1, 4, 8, 16]:
    print(f"\n{'='*50}")
    print(f"测试并发数: {concurrency}")
    print(f"{'='*50}")
    asyncio.run(run_benchmark(concurrency, concurrency * 5))

推理时间 = 模型计算时间 + 显存带宽等待时间 + 调度开销

QPS
│  
│                  ● ─── 显存瓶颈区
│               ●
│            ●        ← 计算瓶颈区
│         ●
│      ●
│   ●
│● ─── 带宽瓶颈区
└─────────────────────────→ 并发数

模型参数（FP16）：  约 670GB（全部 Expert）→ 需要多卡拆分
激活参数（FP16）：  约 37GB（每个 Token）
KV Cache（32K）：   约 2-4GB（MLA，每个序列）
KV Cache（128K）：  约 8-16GB
推理引擎开销：      约 2-4GB

# 自动伸缩配置示例（通过华为云 API）
autoscaling_config = {
    "service_name": "deepseek-r1-production",
    "min_replicas": 2,        # 最小实例数，应对突发流量
    "max_replicas": 50,       # 最大实例数，控制成本
    "scaling_policies": [
        {
            "metric_type": "gpu_utilization",
            "target_value": 70,  # GPU 利用率 > 70% 时扩容
            "scale_up_cooldown": 60,    # 扩容冷却时间（秒）
            "scale_down_cooldown": 300  # 缩容冷却时间（防止抖动）
        },
        {
            "metric_type": "request_queue_depth",
            "target_value": 100,  # 排队请求 > 100 时扩容
            "scale_up_cooldown": 30
        },
        {
            "metric_type": "avg_request_latency_p95",
            "target_value": 5000,  # P95 延迟 > 5s 时扩容（毫秒）
            "scale_up_cooldown": 30
        }
    ]
}

# 定时伸缩策略
schedule_config = {
    "service_name": "deepseek-r1-production",
    "schedules": [
        {
            "name": "workday-daytime",
            "cron": "0 8 * * 1-5",     # 工作日 8:00
            "action": "scale_to",
            "replicas": 20
        },
        {
            "name": "workday-night",
            "cron": "0 22 * * 1-5",    # 工作日 22:00
            "action": "scale_to",
            "replicas": 5
        },
        {
            "name": "weekend",
            "cron": "0 8 * * 6,7",     # 周末
            "action": "scale_to",
            "replicas": 8
        }
    ]
}

# 智能请求分发器
class SmartRouter:
    def __init__(self, instance_registry):
        """
        instance_registry: {
            "instance-1": {"load": 0.7, "queue_depth": 3, "status": "healthy"},
            "instance-2": {"load": 0.3, "queue_depth": 1, "status": "healthy"},
            ...
        }
        """
        self.registry = instance_registry

    def select_instance(self, request: dict) -> str:
        """
        选择最优实例处理请求
        综合考虑负载、队列深度、实例健康状况
        """
        candidates = [
            inst for inst, info in self.registry.items()
            if info["status"] == "healthy"
        ]

        if not candidates:
            raise RuntimeError("No healthy instances available")

        # 评分：负载越轻、队列越短，得分越高
        def score(inst_id: str) -> float:
            info = self.registry[inst_id]
            load_score = 1.0 - info["load"]
            queue_score = 1.0 / (1.0 + info["queue_depth"])
            return load_score * 0.6 + queue_score * 0.4

        best = max(candidates, key=score)

        # 更新选中实例的负载估计
        self.registry[best]["queue_depth"] += 1

        return best

    def complete_request(self, instance_id: str):
        """请求完成后回调，更新队列深度"""
        if instance_id in self.registry:
            self.registry[instance_id]["queue_depth"] = max(
                0, self.registry[instance_id]["queue_depth"] - 1
            )

请求入口
    │
    ├── 高优队列（实时交互） ──→ 优先级权重 10 ──→ 专用实例池 P1
    │       │                                     (低延迟保障)
    │       └── 降级条件：P95 TTFT > 2s → 触发扩容
    │
    ├── 普通队列（业务API）  ──→ 优先级权重 5  ──→ 共享实例池 P2
    │       │                                     (均衡调度)
    │       └── 降级条件：P95 TTFT > 5s → 借用P3资源
    │
    └── 批量队列（离线任务） ──→ 优先级权重 1  ──→ 弹性实例池 P3
                                                (成本优先，容忍延迟)

# 持续批处理的简化实现
class ContinuousBatchingEngine:
    """
    持续批处理推理引擎的简化实现
    核心思想：不等整个 Batch 完成，动态插入/移除请求
    """
    def __init__(self, model, max_batch_size=32, max_total_tokens=4096):
        self.model = model
        self.max_batch_size = max_batch_size
        self.max_total_tokens = max_total_tokens
        self.active_requests = {}     # req_id → RequestState
        self.running_batch = []        # 当前批处理中的请求 ID
        self.waiting_queue = []        # 等待进入批处理的请求

        # 统计
        self.total_tokens_generated = 0
        self.batch_count = 0

    class RequestState:
        def __init__(self, prompt_tokens, max_tokens):
            self.prompt_tokens = prompt_tokens
            self.max_tokens = max_tokens
            self.generated_tokens = []
            self.kv_cache = None      # 预分配的 KV Cache
            self.finished = False
            self.prefill_done = False

    def add_request(self, req_id, prompt_tokens, max_tokens):
        """新增推理请求"""
        self.waiting_queue.append((req_id, prompt_tokens, max_tokens))
        self.active_requests[req_id] = self.RequestState(prompt_tokens, max_tokens)

    def scheduling_step(self):
        """
        调度步骤：在每次推理迭代前执行
        决定哪些请求进入本次 batch
        """
        # 1. 移除已完成的请求
        self.running_batch = [
            rid for rid in self.running_batch
            if not self.active_requests[rid].finished
        ]

        # 2. 计算当前 batch 的 token 总量
        current_tokens = sum(
            len(self.active_requests[rid].generated_tokens) + 1
            for rid in self.running_batch
        )

        # 3. 从等待队列中取新请求，填满 batch
        while self.waiting_queue and len(self.running_batch) < self.max_batch_size:
            req_id, prompt_len, max_tokens = self.waiting_queue.pop(0)

            # 检查加入后是否超过总 token 限制
            if current_tokens + prompt_len + max_tokens <= self.max_total_tokens:
                self.running_batch.append(req_id)
                current_tokens += prompt_len
            else:
                # 放回队列等待下一轮
                self.waiting_queue.insert(0, (req_id, prompt_len, max_tokens))
                break

        return self.running_batch

    def prefill_and_decode(self):
        """
        执行一次推理迭代：
        - Prefill: 处理新请求的 Prompt
        - Decode: 为所有活跃请求生成下一个 Token
        """
        batch = self.scheduling_step()
        if not batch:
            return

        # 构建 Batch 输入
        input_data = []
        for rid in batch:
            state = self.active_requests[rid]
            if not state.prefill_done:
                # Prefill 阶段：处理完整 Prompt
                input_data.append(state.prompt_tokens)
            else:
                # Decode 阶段：只输入上一个 Token
                input_data.append([state.generated_tokens[-1]])

        # 执行推理（调用模型 forward）
        next_tokens, new_kv = self.model.forward(
            input_ids=input_data,
            kv_cache={rid: self.active_requests[rid].kv_cache 
                      for rid in batch},
        )

        # 更新状态
        for i, rid in enumerate(batch):
            state = self.active_requests[rid]
            state.kv_cache = new_kv[rid]
            state.prefill_done = True
            state.generated_tokens.append(next_tokens[i])

            if len(state.generated_tokens) >= state.max_tokens:
                state.finished = True  # 达到最大生成长度
                # 或者检测到 EOS Token

        self.total_tokens_generated += len(batch)
        self.batch_count += 1

# 前缀缓存管理器
class PrefixCacheManager:
    """
    缓存已计算过的 Prompt 前缀的 KV Cache
    对多轮对话、系统提示词等场景特别有效
    """
    def __init__(self, cache_capacity=1000, min_cache_len=128):
        self.cache = {}           # prompt_hash → KV Cache
        self.cache_capacity = cache_capacity
        self.min_cache_len = min_cache_len
        self.hit_count = 0
        self.miss_count = 0

    def _hash_prompt(self, prompt_tokens: list) -> str:
        """对 Prompt Token 序列做哈希（可用滚动哈希支持部分匹配）"""
        # 生产环境使用更高效的滚动哈希（如 Robin Hood Hashing）
        return hashlib.sha256(bytes(prompt_tokens)).hexdigest()

    def lookup(self, prompt_tokens: list) -> tuple:
        """
        查找缓存
        返回: (cached_prefix_len, cached_kv_cache)
        如果缓存命中，返回匹配的最长前缀
        """
        # 简化实现：精确匹配整个 Prompt
        h = self._hash_prompt(prompt_tokens)

        if h in self.cache:
            self.hit_count += 1
            return (len(prompt_tokens), self.cache[h])

        # 部分匹配（高效实现需要 Trie 或基数树）
        # ... 这里简化处理
        self.miss_count += 1
        return (0, None)

    def store(self, prompt_tokens: list, kv_cache):
        """存入缓存，LRU 淘汰"""
        if len(self.cache) >= self.cache_capacity:
            # LRU 淘汰
            lru_key = min(self.cache, key=lambda k: self.cache[k].get("access_time", 0))
            del self.cache[lru_key]

        h = self._hash_prompt(prompt_tokens)
        self.cache[h] = {
            "kv_cache": kv_cache,
            "access_time": time.time(),
            "length": len(prompt_tokens)
        }

    @property
    def hit_rate(self) -> float:
        total = self.hit_count + self.miss_count
        return self.hit_count / total if total > 0 else 0

MaaS 推理配置 → 高级设置
├── Prefix Caching: 启用
├── 缓存容量: 5000 条（默认）
└── 最小匹配长度: 64 Tokens

# KV Cache 量化器
class KVCacheQuantizer:
    """
    KV Cache 从 FP16 量化到 INT8/INT4
    在显存受限时，牺牲微小精度换取更大 Batch 容量
    """
    def __init__(self, quant_bits=8):
        self.quant_bits = quant_bits

    def quantize(self, kv_cache: torch.Tensor) -> tuple:
        """
        量化 KV Cache：FP16 → INT{8,4}
        返回: (quantized_tensor, scale, zero_point)
        """
        if self.quant_bits == 8:
            # INT8 对称量化
            scale = kv_cache.abs().max() / 127.0
            quantized = torch.round(kv_cache / scale).clamp(-128, 127).to(torch.int8)
            return quantized, scale, 0
        elif self.quant_bits == 4:
            # INT4 非对称量化（分组量化）
            group_size = 32
            orig_shape = kv_cache.shape
            flattened = kv_cache.flatten()

            # 按 group_size 分组
            num_groups = (flattened.shape[0] + group_size - 1) // group_size
            padded = torch.zeros(num_groups * group_size, dtype=flattened.dtype)
            padded[:flattened.shape[0]] = flattened
            grouped = padded.reshape(num_groups, group_size)

            min_vals = grouped.min(dim=1).values
            max_vals = grouped.max(dim=1).values
            scale = (max_vals - min_vals) / 15.0
            zero_point = torch.round(-min_vals / scale).clamp(0, 15).to(torch.uint8)

            quantized = torch.round((grouped - min_vals.unsqueeze(1)) 
                                   / scale.unsqueeze(1)).clamp(0, 15).to(torch.uint8)

            # 打包为 2 个 INT4 到一个字节
            packed = (quantized[:, ::2] << 4) | quantized[:, 1::2]

            return packed, scale, zero_point

# 发送指标到华为云 CES（Cloud Eye）
metrics_payload = {
    "namespace": "MaaS_DeepSeek",
    "metrics": [
        # 请求量指标
        {"metric_name": "request_count", "value": 15234, "unit": "Count"},
        {"metric_name": "qps", "value": 42.5, "unit": "Count/Second"},
        {"metric_name": "active_connections", "value": 128, "unit": "Count"},

        # 延迟指标
        {"metric_name": "ttft_avg", "value": 0.85, "unit": "Seconds"},
        {"metric_name": "ttft_p95", "value": 1.92, "unit": "Seconds"},
        {"metric_name": "ttft_p99", "value": 3.45, "unit": "Seconds"},
        {"metric_name": "tpot_avg", "value": 0.038, "unit": "Seconds/Token"},
        {"metric_name": "total_latency_avg", "value": 3.2, "unit": "Seconds"},

        # 吞吐指标
        {"metric_name": "output_tokens_per_second", "value": 1520, "unit": "Tokens/S"},
        {"metric_name": "batch_size_avg", "value": 12.5, "unit": "Count"},

        # 资源指标
        {"metric_name": "gpu_utilization", "value": 78.3, "unit": "Percent"},
        {"metric_name": "gpu_memory_usage", "value": 62.5, "unit": "Percent"},
        {"metric_name": "kv_cache_usage", "value": 45.2, "unit": "Percent"},
        {"metric_name": "queue_depth", "value": 23, "unit": "Count"},

        # 错误指标
        {"metric_name": "error_rate", "value": 0.12, "unit": "Percent"},
        {"metric_name": "timeout_rate", "value": 0.05, "unit": "Percent"},
    ]
}

# 关键告警规则
alarm_rules = [
    # 业务连续性
    {"metric": "error_rate", "threshold": 1.0, "duration": 300,
     "action": "page_oncall", "severity": "critical"},

    # 服务质量退化
    {"metric": "ttft_p95", "threshold": 5000, "duration": 120,
     "action": "auto_scale_up", "severity": "warning"},

    # 资源瓶颈
    {"metric": "gpu_memory_usage", "threshold": 95, "duration": 300,
     "action": "scale_up", "severity": "critical"},
    {"metric": "queue_depth", "threshold": 500, "duration": 60,
     "action": "scale_up", "severity": "warning"},

    # 实例异常
    {"metric": "instance_health", "threshold": 0, "duration": 30,
     "action": "restart_instance", "severity": "critical"},
]

# 请求层面的链路追踪（OpenTelemetry 集成）
import opentracing
import opentracing.tags as tags

class TracingMiddleware:
    """
    推理请求的全链路追踪
    支持追踪每个环节的耗时分布
    """
    def __init__(self, tracer):
        self.tracer = tracer

    async def process_request(self, request):
        """为请求创建追踪 Span"""
        span = self.tracer.start_span("inference_request")
        span.set_tag(tags.HTTP_METHOD, "POST")
        span.set_tag(tags.HTTP_URL, request.url)
        span.set_tag("request.model", request.model)
        span.set_tag("request.max_tokens", request.max_tokens)
        return span

    async def trace_prefill(self, span, prefill_time_ms, prompt_tokens):
        """记录 Prefill 阶段"""
        prefill_span = self.tracer.start_span(
            "prefill", child_of=span
        )
        prefill_span.set_tag("tokens", prompt_tokens)
        prefill_span.set_tag("duration_ms", prefill_time_ms)
        prefill_span.finish()

    async def trace_decode(self, span, num_tokens, total_decode_time_ms):
        """记录 Decode 阶段"""
        decode_span = self.tracer.start_span(
            "decode", child_of=span
        )
        decode_span.set_tag("tokens_generated", num_tokens)
        decode_span.set_tag("total_duration_ms", total_decode_time_ms)
        avg_tpot = total_decode_time_ms / num_tokens if num_tokens > 0 else 0
        decode_span.set_tag("avg_tpot_ms", avg_tpot)
        decode_span.finish()

    async def trace_queue_wait(self, span, wait_time_ms):
        """记录排队等待时间"""
        wait_span = self.tracer.start_span(
            "queue_wait", child_of=span
        )
        wait_span.set_tag("duration_ms", wait_time_ms)
        wait_span.finish()

# Spot 实例配置
spot_config = {
    "instance_type": "p2vs.2xlarge",
    "spot_price": "15.00",     # 竞价上限（元/小时）
    "spot_strategy": "spot_as_price",  # 低于上限即购买
    "lifecycle": "auto_recycle",  # 被回收时自动拉起
    "fallback_to_ondemand": True   # 无 Spot 时回退按需
}

原始 Prompt（使用 850 Tokens）：
  "你是一个专业的AI助手，你的名字叫小华...（啰嗦的系统提示词）..."

优化后 Prompt（使用 120 Tokens）：
  "你是AI助手小华，响应简洁、准确、友好。"

# 语义缓存层（Sematic Cache）
class SemanticCache:
    """
    对相似语义的请求缓存推理结果
    使用 Embedding 相似度匹配
    """
    def __init__(self, threshold=0.92, max_size=5000):
        self.threshold = threshold
        self.max_size = max_size
        self.cache = {}          # text_hash → cached_result
        self.embeddings = {}     # text_hash → embedding_vector

    def search(self, query: str, query_embedding: list) -> str or None:
        """在缓存中查找语义相似的请求"""
        if not self.embeddings:
            return None

        # 计算余弦相似度
        best_match = None
        best_score = 0

        for key, emb in self.embeddings.items():
            similarity = self._cosine_similarity(query_embedding, emb)
            if similarity > best_score:
                best_score = similarity
                best_match = key

        if best_score >= self.threshold:
            return self.cache[best_match]
        return None

    def store(self, query: str, query_embedding: list, result: str):
        """存入缓存"""
        key = hashlib.md5(query.encode()).hexdigest()

        if len(self.cache) >= self.max_size:
            # LRU 淘汰
            oldest = next(iter(self.cache))
            del self.cache[oldest]
            del self.embeddings[oldest]

        self.cache[key] = result
        self.embeddings[key] = query_embedding