Check out the Goofballs Gaming Group community on Discord - hang out with 3 other members and enjoy free voice and text chat.

For those unable to use HF, go to Mistral and get a free account to get an API key. Then use this class, it will simuate the results getting back from InferenceClient when you use chat_stream(). import os import config from mistralai import Mistral class TextPacket: def __init__(self): self.choices = [] class TextMessage: def __init__(self): self.role:str = None self.content:str = None class TextGroup: def __init__(self): self.index = 0 self.finish_reason:str = None self.delta:TextMessage = TextMessage() self.message:TextMessage = TextMessage() class MistralGenerator(): def __init__(self): self.api_key = config.MISTRALAI_APIKEY self.model = "mistral-small-latest" self.client = Mistral(api_key=self.api_key) def chat_complete(self, query, max_tokens=512, temperature=0.7, top_p=0.9): chat_response = self.client.chat.complete( model= self.model, messages = [ { "role": "user", "content": query, }, ], max_tokens=max_tokens, temperature=temperature, top_p=top_p, ) print(chat_response.choices[0].message.content) return chat_response.choices[0].message.content def chat_stream(self, messages, max_tokens=512, temperature=0.7, top_p=0.9): stream_response = self.client.chat.stream( model=self.model, messages=messages, max_tokens=max_tokens, temperature=temperature, top_p=top_p, stream=True ) for chunk in stream_response: #print(chunk.data.choices[0].delta.content) message = TextPacket() group = TextGroup() group.index = 0 group.delta.role = "assistant" group.delta.content = chunk.data.choices[0].delta.content message.choices.append(group) yield message # Final stop message for stream message = TextPacket() group = TextGroup() group.index = 0 group.delta.role = "assistant" group.delta.content = "" group.delta.finish_reason = "stop" message.choices.append(group) yield message

For those stuck with HF, get a free Mistral account to get the API key. Then you can use this class to do text generation. The chat_stream() will simulate the packets HF’s InferenceClient() would return, so it can be a plug and play… import os import config from mistralai import Mistral class TextPacket: def **init**(self): self.choices = class TextMessage: def **init**(self): self.role:str = None self.content:str = None class TextGroup: def **init**(self): self.index = 0 self.finish_reason:str = None self.delta:TextMessage = TextMessage() self.message:TextMessage = TextMessage() class MistralGenerator(): def **init**(self): self.api_key = config.MISTRALAI_APIKEY self.model = “mistral-small-latest” self.client = Mistral(api_key=self.api_key) def chat_complete(self, query, max_tokens=512, temperature=0.7, top_p=0.9): chat_response = self.client.chat.complete( model= self.model, messages = [ { "role": "user", "content": query, }, ], max_tokens=max_tokens, temperature=temperature, top_p=top_p, ) print(chat_response.choices[0].message.content) return chat_response.choices[0].message.content def chat_stream(self, messages, max_tokens=512, temperature=0.7, top_p=0.9): stream_response = self.client.chat.stream( model=self.model, messages=messages, max_tokens=max_tokens, temperature=temperature, top_p=top_p, stream=True ) for chunk in stream_response: message = TextPacket() group = TextGroup() group.index = 0 group.delta.role = "assistant" group.delta.content = chunk.data.choices[0].delta.content message.choices.append(group) yield message # Final stop message for stream message = TextPacket() group = TextGroup() group.index = 0 group.delta.role = "assistant" group.delta.content = "" group.delta.finish_reason = "stop" message.choices.append(group) yield message