LangChain Streaming Chat Integration

Frontend → POST /stream-tool-chat-with-openai → ToolController → LangChain Pipeline → StreamingResponse

@router.post("/stream-tool-chat-with-openai")
async def stream_chat(chat_input: ToolChatBase):
    controller = ToolController()
    controller.initialization_service_code(code=chat_input.code)
    
    response_generator = await controller.service_hub_handler(chat_input)
    return StreamingResponseWithStatusCode(response_generator, media_type="text/event-stream")

from langchain.chat_models import ChatOpenAI

self.llm = ChatOpenAI(
    model_name="gpt-4",
    temperature=0.7,
    streaming=False,
    openai_api_key="your-api-key"
)

from langchain.memory import ConversationSummaryBufferMemory

self.memory = ConversationSummaryBufferMemory(
    llm=self.llm,
    max_token_limit=1000,
    return_messages=True
)

from langchain.agents import tool

@tool
def web_search_preview(query: str) -> str:
    return f"Search results for {query}"

tools = [web_search_preview]
llm_with_tools = self.llm.bind_tools(tools=tools, tool_choice='auto')

from langchain.prompts import ChatPromptTemplate

prompt = ChatPromptTemplate.from_messages([
    ("system", "You're a helpful assistant."),
    ("human", "{input}")
])

async def tool_calls_run(self, query: str) -> AsyncGenerator[str, None]:
    chain = LLMChain(
        llm=self.llm_with_tools,
        memory=self.memory,
        prompt=prompt,
        verbose=True
    )
    
    async for chunk in chain.astream(input={"input": query}):
        yield f"data: {chunk.content}\n\n"
    yield "data: [DONE]\n\n"