从Language Model到Chat Application：对话接口的设计与实现

{% for message in messages %}  {{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}  {{ '<|im_start|>assistant\n' }}{% endif %}

[    {"role": "user", "content": "Hi there!"},    {"role": "assistant", "content": "Nice to meet you!"},    {"role": "user", "content": "Can I ask a question?"}]

<|im_start|>userHi there!<|im_end|><|im_start|>assistantNice to meet you!<|im_end|><|im_start|>userCan I ask a question?<|im_end|>

@app.post("/v1/chat/completions")async def create_chat_completion(request: ChatCompletionRequest,                                 raw_request: Request):
    ...
    try:        prompt = tokenizer.apply_chat_template(            conversation=request.messages,            tokenize=False,            add_generation_prompt=request.add_generation_prompt)  ...    result_generator = engine.generate(prompt, sampling_params, request_id,                                       token_ids)

python3 -m vllm.entrypoints.openai.api_server --model Qwen/Qwen-7B-Chat --trust-remote-code

@register_chat_format("qwen")def format_qwen(    messages: List[llama_types.ChatCompletionRequestMessage],    **kwargs: Any,) -> ChatFormatterResponse:    _roles = dict(user="<|im_start|>user", assistant="<|im_start|>assistant")    system_message="You are a helpful assistant."    system_template="<|im_start|>system\n{system_message}"    system_message=system_template.format(system_message=system_message)    _messages = _map_roles(messages, _roles)    _messages.append((_roles["assistant"], None))    _sep = "<|im_end|>"    _prompt = _format_chatml(system_message, _messages, _sep)    _sep2 = "<|endoftext|>"    return ChatFormatterResponse(prompt=_prompt,stop=_sep2)

/opt/conda310/bin/python -m llama_cpp.server --model /mnt/nas1/gguf/qwen-14b-chat-f16.gguf  --n_gpu_layers 128 --host 0.0.0.0 --chat_format qwen

"<|im_start|>systemYou are a helpful assistant.<|im_end|><|im_start|>userAnswer the following questions as best you can. You have access to the following APIs:
get_current_weather: Call this tool to interact with the get_current_weather API. What is the get_current_weather API useful for? Get the current weather in a given location. Parameters: {"type": "object", "properties": {"location": {"type": "string", "description": "The city and state, e.g. San Francisco, CA"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, "required": ["location"]}
Use the following format:
Question: the input question you must answerThought: you should always think about what to doAction: the action to take, should be one of [get_current_weather]Action Input: the input to the actionObservation: the result of the action... (this Thought/Action/Action Input/Observation can be repeated zero or more times)Thought: I now know the final answerFinal Answer: the final answer to the original input question
Begin!
Question: 杭州市余杭区天气如何？<|im_end|><|im_start|>assistant

if request.stream:    if request.functions:        raise HTTPException(            status_code=400,            detail="Invalid request: Function calling is not yet implemented for stream mode.",        )

def contains_custom_function(value: str) -> bool:    return value and 'get_' in value

export MODEL_TYPE=qwenexport CHECKPOINT_PATH=/path/to/modelexport START_PORT=50233
python3 -m maga_transformer.start_server

# 配置qwen服务域名为openai endpointimport osos.environ["OPENAI_API_KEY"] = "xxxx" # you can use any string for keyos.environ["OPENAI_API_BASE"] = "http://localhost:50233"
# langchain打印每一步的完整信息from langchain.globals import set_debug, set_verboseset_debug(True)set_verbose(True)
from langchain_core.pydantic_v1 import BaseModel, Fieldfrom langchain.chains.openai_functions import create_structured_output_chainfrom langchain_community.chat_models import ChatOpenAIfrom langchain_core.prompts import ChatPromptTemplate
# 定义一个Dog对象class Dog(BaseModel):    """Identifying information about a dog."""    name: str = Field(..., description="The dog's name")    color: str = Field(..., description="The dog's color")
# 定义prompt模板：用function call提取{input}中的对象。llm = ChatOpenAI(model="anything-you-like", temperature=0.2)prompt = ChatPromptTemplate.from_messages(    [        ("system", "You are an algorithm for extracting information into structured formats, and respond with function call."),        ("user", "extract information from the following input: {input}"),    ])
# 构建chain并调用，输出解析的结果。output是一个`Dog`对象。chain = create_structured_output_chain(Dog, llm, prompt)output = chain.run("John had a dog named Harry, who was a brown beagle that loved chicken")print(str(output))

name='Harry' color='brown'

import osos.environ["OPENAI_API_KEY"] = "xxxx" # you can use any string for keyos.environ["OPENAI_API_BASE"] = "http://localhost:50233"
from llama_index.readers import SimpleDirectoryReader, JSONReader, PDFReaderfrom llama_index.embeddings import HuggingFaceEmbeddingfrom llama_index import VectorStoreIndex, ServiceContextfrom llama_index.llms import OpenAI
# 先从huggingface上拉一个embedding模型，给文本召回用embed_model = HuggingFaceEmbedding(model_name="sentence-transformers/all-MiniLM-L6-v2")service_context = ServiceContext.from_defaults(    llm=llm, embed_model=embed_model)llm = OpenAI()
# 从本地目录加载所有文档，并建立向量索引documents = SimpleDirectoryReader("/home/wangyin.yx/workspace/ChatGLM3").load_data()index = VectorStoreIndex.from_documents(documents, service_context=service_context)
# 进行对话查询query_engine = index.as_query_engine()response = query_engine.query("如何在mac上部署chatglm？")print(response)