Update

app-org.py

@@ -12,7 +12,10 @@ torch
 """
 # pylint: disable=line-too-long, missing-module-docstring, missing-function-docstring
 # import torch
+from time import time
+
 import gradio as gr
+from about_time import about_time
 from examples_list import examples_list
 from transformers import AutoModel, AutoTokenizer  # AutoModelForCausalLM,
 
@@ -35,10 +38,21 @@ def chat(message, history):
     # inputs = tokenizer(prompt, return_tensors="pt").to(device=device)
     # output = model.generate(**inputs, do_sample=True, top_p=0.95, top_k=0, max_new_tokens=256)
     # return tokenizer.decode(output[0], skip_special_tokens=True)
-    for response, _ in chat_model.stream_chat(
-        tokenizer, message, history, max_length=2048, top_p=0.7, temperature=0.95
-    ):
-        yield response
+    flag = 1
+    then = time()
+    prefix = ""
+    prelude = 0.0
+    with about_time() as dur:
+        for response, _ in chat_model.stream_chat(
+            tokenizer, message, history, max_length=2048, top_p=0.7, temperature=0.95
+        ):
+            if flag:
+                flag = 0
+                prelude = time() - then
+                prefix = f"{prelude:.2f}s"
+            yield f"{prefix} {response}"
+    suffix = f"\n(time elapsed: {dur.duration_human}, {(time() - prelude)/len(response):.2f}s/char)"
+    yield f"{response}{suffix}"
 
 chatbot = gr.Chatbot([], label="Bot", height=450)
 textbox = gr.Textbox('', scale=10, label='', lines=2, placeholder="Ask me anything")

app.py

@@ -29,8 +29,10 @@ demo.launch()
 # import torch
 import random
 import time
+
 import gradio as gr
 
+
 def respond2(message, chat_history):
     if chat_history is None:
         chat_history = []
@@ -51,7 +53,8 @@ def respond2(message, chat_history):
     yield "", chat_history
 
 def stream_chat():
-    """samples:
+    """
+    List samples.
 
     Sure [('test me', 'Sure')]
     Sure, [('test me', 'Sure,')]
@@ -154,4 +157,4 @@ with gr.Blocks() as demo:
 
     msg.submit(respond2, [msg, chatbot], [msg, chatbot])
 
-# demo.queue(max_size=2).launch()
+# demo.queue(max_size=2).launch()