INFO 03-20 19:40:50 worker.py:267] Memory profiling takes 16.56 secondsINFO 03-20 19:40:50 worker.py:267] the current vLLM instance can use total_gpu_memory (9x.00 GiB) x gpu_memory_utilization (0.95) = 9x.00 GiBINFO 03-20 19:40:50 worker.py:267] model weights take 42.59GiB; non_torch_memory takes 2.30GiB; PyTorch activation peak memory takes 1.42GiB; the rest of the memory reserved for KV Cache is 4x.00 GiB.

  time curl http://172.17.98.24:8080/v1/chat/completions   -H "Content-Type: application/json"   -d '{    "model": "ds",    "messages": [      {        "role": "system",         "content": "你是个友善的AI助手。"      },      {        "role": "user",        "content": "介绍一下深度学习好吗。"      }    ],    "max_tokens": 2048  }'

sum(DCGM_FI_DEV_FB_USED{PodName=~"", NamespaceName=~""})by(NamespaceName, PodName)

#抓取vllm nsightnohup nsys launch  -t cuda,nvtx,osrt,cudnn,cublas,opengl,cudla-verbose,cusolver-verbose   --cudabacktrace=all --cuda-memory-usage=true --cuda-um-cpu-page-faults=true --cuda-um-gpu-page-faults=true --session-new vllm-capture1 vllm serve /model/DeepSeek-R1/ --port 8080 --trust-remote-code --served-model-name ds --enable-reasoning  --reasoning-parser deepseek_r1 --max-num-batched-tokens 8192  --max-model-len  8192  --enable-prefix-caching  --gpu-memory-utilization 0.90 --tensor-parallel-size 16 --enforce-eager > /tmp/vllm-capture.log 2>&1 &

#阻塞同步export CUDA_LAUNCH_BLOCKING=1
#PDBfrom vllm import LLM,sampling_params
prompt=[        "Hello, my name is",    "The president of the United States is",    "The capital of France is",    "The future of AI is"，]model_name = "Qwen/Qwen2.5-1.5B"
llm=LLM(model= model_name,trust_remote_code=True,gpu_memory_utilization=0.95,max_model_len=8192)ans=llm.generate(prompts=prompt)for output in ans:    prompt = output.prompt    generated_text = output.outputs[0].text    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")

  /usr/local/lib/python3.10/bdb.py(598)run()-> exec(cmd, globals, locals)  <string>(1)<module>()  /mnt/workspace/gx/main.py(11)<module>()-> ans=llm.generate(prompts=prompt)  /usr/local/lib/python3.10/site-packages/vllm/utils.py(838)inner()-> return fn(*args, **kwargs)  /usr/local/lib/python3.10/site-packages/vllm/entrypoints/llm.py(316)generate()-> outputs = self._run_engine(use_tqdm=use_tqdm)  /usr/local/lib/python3.10/site-packages/vllm/entrypoints/llm.py(569)_run_engine()-> step_outputs = self.llm_engine.step()  /usr/local/lib/python3.10/site-packages/vllm/engine/llm_engine.py(911)step()-> output = self.model_executor.execute_model(  /usr/local/lib/python3.10/site-packages/vllm/executor/gpu_executor.py(110)execute_model()-> output = self.driver_worker.execute_model(execute_model_req)  /usr/local/lib/python3.10/site-packages/vllm/worker/worker_base.py(272)execute_model()-> output = self.model_runner.execute_model(> /usr/local/lib/python3.10/site-packages/torch/utils/_contextlib.py(114)decorate_context()->[SamplerOutput..._metrics=None)]-> with ctx_factory():(Pdb) 

  for device_id in range(8): #8表示几张卡            print(torch.cuda.memory_summary(device_id))

  Block* malloc(      c10::DeviceIndex device,      size_t orig_size,      cudaStream_t stream) {    // done outside the lock because we don't know what locks the recorder needs    // to have...    auto context = maybeGatherContext(RecordContext::STATE);
    std::unique_lock<std::recursive_mutex> lock(mutex);
    if (C10_LIKELY(captures_underway.empty())) {      // Processes end-of-life events for outstanding allocations used on      // multiple streams (checks if their GPU-side uses are complete and      // recycles their memory if so)      //      // Q. Why skip process_events if a capture might be underway?      // A. process_events involves cudaEventQueries, illegal during CUDA graph      //    capture.      //    Dumb simple solution: defer reclaiming these allocations until after      //    capture. Cross-stream memory use is uncommon, so the deferral's      //    effect on memory use during capture should be small.      process_events(context);    }    size_t size = round_size(orig_size);    auto& pool = get_pool(size, stream);    const size_t alloc_size = get_allocation_size(size);    AllocParams params(device, size, stream, &pool, alloc_size, stats);    params.stat_types = get_stat_types_for_pool(pool);
    // First, try to get a block from the existing pool.    bool block_found =        // Search pool        get_free_block(params)        // Trigger callbacks and retry search        || (trigger_free_memory_callbacks(params) && get_free_block(params));
    // Can't reuse an existing block; try to get a new one.    if (!block_found) {      // Do garbage collection if the flag is set.      if (C10_UNLIKELY(              set_fraction &&              CUDAAllocatorConfig::garbage_collection_threshold() > 0.0)) {        garbage_collect_cached_blocks(context);      }      // Attempt allocate      // WARNING: alloc_block may release the allocator lock when calling      // cudaMalloc. So far this function has not modified allocator state, but      // keep in mind that any observed allocator state may change across calls      // to alloc_block since it may release the lock.      block_found = alloc_block(params, false, context, lock)          // Free enough available cached blocks to satisfy alloc and retry          // alloc.          || (release_available_cached_blocks(params, context) &&              alloc_block(params, false, context, lock))          // Free all non-split cached blocks and retry alloc.          || (C10_LIKELY(captures_underway.empty()) &&              release_cached_blocks(context) &&              alloc_block(params, true, context, lock));    }
    if (!block_found) {      //OOM流程    }
    bool split_remainder = should_split(params.block, params.size());    return alloc_found_block(        params, orig_size, std::move(context), split_remainder);  }

void free(Block* block) {    std::shared_ptr<GatheredContext> context =    maybeGatherContext(RecordContext::ALL);    std::lock_guard<std::recursive_mutex> lock(mutex);
    block->allocated = false;
    // following logic might modifying underlaying Block, causing the size    // changed. We store ahead for reporting    auto orig_block_ptr = block->ptr;    auto orig_block_size = block->size;
    StatTypes stat_types = get_stat_types_for_pool(*block->pool);    for_each_selected_stat_type(stat_types, [&](size_t stat_type) {        stats.allocation[stat_type].decrease(1);        stats.allocated_bytes[stat_type].decrease(block->size);    });    auto allocated_bytes_gauge =    STATIC_GAUGE(pytorch.CUDACachingAllocator.allocated_bytes);    allocated_bytes_gauge.record(        stats.allocated_bytes[static_cast<int64_t>(StatType::AGGREGATE)]        .current);
    record_trace(        TraceEntry::FREE_REQUESTED,        int64_t(block->ptr),        block->requested_size,        block->stream,        block->device,        context ? context : block->context_when_allocated);
    if (block->size >= CUDAAllocatorConfig::max_split_size())        stats.oversize_allocations.decrease(1);
    if (!block->stream_uses.empty()) {        if (C10_UNLIKELY(!captures_underway.empty())) {            // It's forbidden to cudaEventQuery an event recorded during CUDA graph            // capture. We conservatively defer recording end-of-life events until            // the next call to process_events() (which won't happen until no            // captures are underway)            needs_events_deferred_until_no_capture.push_back(block);        } else {            insert_events(block);        }    } else {        free_block(block, context);    }
    c10::reportMemoryUsageToProfiler(        orig_block_ptr,        -static_cast<int64_t>(orig_block_size),        stats.allocated_bytes[static_cast<size_t>(StatType::AGGREGATE)].current,        stats.reserved_bytes[static_cast<size_t>(StatType::AGGREGATE)].current,        c10::Device(c10::DeviceType::CUDA, block->device));}

  size_t try_merge_blocks(Block* dst, Block* src, BlockPool& pool) {    if (!src || src->allocated || src->event_count > 0 ||        !src->stream_uses.empty() || dst->mapped != src->mapped) {      return 0;    }
    AT_ASSERT(dst->is_split() && src->is_split());
    if (dst->prev == src) { // [src dst]      dst->ptr = src->ptr;      dst->prev = src->prev;      if (dst->prev) {        dst->prev->next = dst;      }      dst->context_when_segment_allocated =          std::move(src->context_when_segment_allocated);    } else { // [dest src]      dst->next = src->next;      if (dst->next) {        dst->next->prev = dst;      }    }    const size_t subsumed_size = src->size;    dst->size += subsumed_size;    // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores)    auto erased =        src->mapped ? pool.blocks.erase(src) : pool.unmapped.erase(src);    TORCH_INTERNAL_ASSERT_DEBUG_ONLY(erased == 1);    delete src;
    return subsumed_size;  }