fix: better answer it

Signed-off-by: yihong0618 <zouzou0208@gmail.com>

handlers/chatgpt.py

@@ -96,12 +96,12 @@ def chatgpt_pro_handler(message: Message, bot: TeleBot) -> None:
 
     player_message = []
     # restart will lose all TODO
-    if str(message.from_user.id) not in chatgpt_player_dict:
+    if str(message.from_user.id) not in chatgpt_pro_player_dict:
-        chatgpt_player_dict[str(message.from_user.id)] = (
+        chatgpt_pro_player_dict[str(message.from_user.id)] = (
             player_message  # for the imuutable list
         )
     else:
-        player_message = chatgpt_player_dict[str(message.from_user.id)]
+        player_message = chatgpt_pro_player_dict[str(message.from_user.id)]
     if m.strip() == "clear":
         bot.reply_to(
             message,

handlers/useful.py

@@ -6,6 +6,8 @@ from expiringdict import ExpiringDict
 from os import environ
 import time
 import datetime
+import threading
+import queue
 
 from openai import OpenAI
 import google.generativeai as genai
@@ -26,6 +28,7 @@ COHERE_API_KEY = environ.get("COHERE_API_KEY")
 COHERE_MODEL = "command-r-plus"
 # if you want to use cohere for answer it, set it to True
 USE_CHHERE = False
+USE_CLAUDE = True
 if COHERE_API_KEY:
     co = cohere.Client(api_key=COHERE_API_KEY)
 
@@ -63,7 +66,6 @@ model = genai.GenerativeModel(
     generation_config=generation_config,
     safety_settings=safety_settings,
 )
-convo = model.start_chat()
 
 #### ChatGPT init ####
 CHATGPT_API_KEY = environ.get("OPENAI_API_KEY")
@@ -72,11 +74,39 @@ QWEN_API_KEY = environ.get("TOGETHER_API_KEY")
 QWEN_MODEL = "Qwen/Qwen2-72B-Instruct"
 CHATGPT_PRO_MODEL = "gpt-4o-2024-05-13"
 
+#### CLAUDE ####
+ANTHROPIC_API_KEY = environ.get("ANTHROPIC_API_KEY")
+ANTHROPIC_BASE_URL = environ.get("ANTHROPIC_BASE_URL")
+ANTHROPIC_MODEL = "claude-3-5-sonnet-20240620"
+# use openai for claude
+claude_client = OpenAI(
+    api_key=ANTHROPIC_API_KEY, base_url=ANTHROPIC_BASE_URL, timeout=20
+)
 
 client = OpenAI(api_key=CHATGPT_API_KEY, base_url=CHATGPT_BASE_URL, timeout=300)
 qwen_client = Together(api_key=QWEN_API_KEY, timeout=300)
 
 
+def _run_in_thread(func, *args, **kwargs):
+    result_queue = queue.Queue()
+
+    def wrapper():
+        try:
+            result = func(*args, **kwargs)
+            result_queue.put(result)
+        except Exception as e:
+            result_queue.put(e)
+
+    thread = threading.Thread(target=wrapper)
+    thread.start()
+    thread.join()
+
+    result = result_queue.get()
+    if isinstance(result, Exception):
+        raise result
+    return result
+
+
 def md_handler(message: Message, bot: TeleBot):
     """pretty md: /md <address>"""
     who = ""
@@ -89,7 +119,7 @@ def latest_handle_messages(message: Message, bot: TeleBot):
     chat_id = message.chat.id
     chat_user_id = message.from_user.id
     # if is bot command, ignore
-    if message.text.startswith("/"):
+    if message.text and message.text.startswith("/"):
         return
     # start command ignore
     elif message.text.startswith(
@@ -127,6 +157,34 @@ def latest_handle_messages(message: Message, bot: TeleBot):
         print(chat_message_dict[chat_id].text)
 
 
+def get_gpt_answer(message):
+    chatgpt_reply_text = ""
+    player_message = [{"role": "user", "content": message}]
+    try:
+        r = client.chat.completions.create(
+            messages=player_message, max_tokens=4096, model=CHATGPT_PRO_MODEL
+        )
+        chatgpt_reply_text = r.choices[0].message.content.encode("utf8").decode()
+    except Exception as e:
+        print(e)
+        chatgpt_reply_text = "answer wrong"
+    return chatgpt_reply_text
+
+
+def get_claude_answer(message):
+    chatgpt_reply_text = ""
+    player_message = [{"role": "user", "content": message}]
+    try:
+        r = claude_client.chat.completions.create(
+            messages=player_message, max_tokens=4096, model=ANTHROPIC_MODEL
+        )
+        chatgpt_reply_text = r.choices[0].message.content.encode("utf8").decode()
+    except Exception as e:
+        print(e)
+        chatgpt_reply_text = "answer wrong"
+    return chatgpt_reply_text
+
+
 def answer_it_handler(message: Message, bot: TeleBot):
     """answer_it: /answer_it"""
     # answer the last message in the chat group
@@ -137,14 +195,19 @@ def answer_it_handler(message: Message, bot: TeleBot):
     latest_message = chat_message_dict.get(chat_id)
     m = latest_message.text.strip()
     m = enrich_text_with_urls(m)
-    full = ""
+    full = "Question:\n" + m + "\n---\n"
     ##### Gemini #####
     who = "Gemini Pro"
     # show something, make it more responsible
     reply_id = bot_reply_first(latest_message, who, bot)
+    chatgpt_answer = _run_in_thread(get_gpt_answer, m)
+
+    claude_answer = ""
+    if ANTHROPIC_API_KEY:
+        claude_answer = _run_in_thread(get_claude_answer, m)
 
     try:
-        r = convo.send_message(m, stream=True)
+        r = model.generate_content(m, stream=True)
         s = ""
         start = time.time()
         for e in r:
@@ -153,49 +216,32 @@ def answer_it_handler(message: Message, bot: TeleBot):
                 start = time.time()
                 bot_reply_markdown(reply_id, who, s, bot, split_text=False)
         bot_reply_markdown(reply_id, who, s, bot)
-        convo.history.clear()
     except Exception as e:
         print(e)
-        convo.history.clear()
         bot_reply_markdown(reply_id, who, "Error", bot)
 
     full += f"{who}:\n{s}"
     chat_id_list = [reply_id.message_id]
+
     ##### ChatGPT #####
     who = "ChatGPT Pro"
     reply_id = bot_reply_first(latest_message, who, bot)
+    # get gpt answer using thread
 
-    player_message = [{"role": "user", "content": m}]
+    bot_reply_markdown(reply_id, who, chatgpt_answer, bot)
 
-    try:
+    full += f"\n---\n{who}:\n{chatgpt_answer}"
-        r = client.chat.completions.create(
-            messages=player_message,
-            max_tokens=4096,
-            model=CHATGPT_PRO_MODEL,
-            stream=True,
-        )
-        s = ""
-        start = time.time()
-        for chunk in r:
-            if chunk.choices[0].delta.content is None:
-                break
-            s += chunk.choices[0].delta.content
-            if time.time() - start > 1.5:
-                start = time.time()
-                bot_reply_markdown(reply_id, who, s, bot, split_text=False)
-        # maybe not complete
-        try:
-            bot_reply_markdown(reply_id, who, s, bot)
-        except:
-            pass
-
-    except Exception as e:
-        print(e)
-        bot_reply_markdown(reply_id, who, "answer wrong", bot)
-
-    full += f"\n---\n{who}:\n{s}"
     chat_id_list.append(reply_id.message_id)
 
+    ##### Claude #####
+    if USE_CLAUDE and ANTHROPIC_API_KEY:
+        who = "Claude Pro"
+        reply_id = bot_reply_first(latest_message, who, bot)
+        bot_reply_markdown(reply_id, who, claude_answer, bot)
+
+        full += f"\n---\n{who}:\n{claude_answer}"
+        chat_id_list.append(reply_id.message_id)
+
     ##### Cohere #####
     if USE_CHHERE and COHERE_API_KEY:
         full, chat_id = cohere_answer(latest_message, bot, full, m)

pdm.lock

@@ -5,7 +5,7 @@
 groups = ["default"]
 strategy = ["cross_platform", "inherit_metadata"]
 lock_version = "4.4.1"
-content_hash = "sha256:92559499879e6b4ab7bb1fd673848820016f25b06e5b87f38dc95cdb15215dcb"
+content_hash = "sha256:d025c0ed6e8de18327d2d51e25f8888abf6941e38a46058c3cae5cfb10a192f9"
 
 [[package]]
 name = "aiohttp"
@@ -164,6 +164,52 @@ files = [
     {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"},
 ]
 
+[[package]]
+name = "beautifulsoup4"
+version = "4.12.3"
+requires_python = ">=3.6.0"
+summary = "Screen-scraping library"
+groups = ["default"]
+dependencies = [
+    "soupsieve>1.2",
+]
+files = [
+    {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"},
+    {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"},
+]
+
+[[package]]
+name = "boto3"
+version = "1.34.135"
+requires_python = ">=3.8"
+summary = "The AWS SDK for Python"
+groups = ["default"]
+dependencies = [
+    "botocore<1.35.0,>=1.34.135",
+    "jmespath<2.0.0,>=0.7.1",
+    "s3transfer<0.11.0,>=0.10.0",
+]
+files = [
+    {file = "boto3-1.34.135-py3-none-any.whl", hash = "sha256:6f5d7a20afbe45e3f7c6b5e96071752d36c3942535b1f7924964f1fdf25376a7"},
+    {file = "boto3-1.34.135.tar.gz", hash = "sha256:344f635233c85dbb509b87638232ff9132739f90bb5e6bf01fa0e0a521a9107e"},
+]
+
+[[package]]
+name = "botocore"
+version = "1.34.135"
+requires_python = ">=3.8"
+summary = "Low-level, data-driven core of boto 3."
+groups = ["default"]
+dependencies = [
+    "jmespath<2.0.0,>=0.7.1",
+    "python-dateutil<3.0.0,>=2.1",
+    "urllib3!=2.2.0,<3,>=1.25.4; python_version >= \"3.10\"",
+]
+files = [
+    {file = "botocore-1.34.135-py3-none-any.whl", hash = "sha256:3aa9e85e7c479babefb5a590e844435449df418085f3c74d604277bc52dc3109"},
+    {file = "botocore-1.34.135.tar.gz", hash = "sha256:2e72f37072f75cb1391fca9d7a4c32cecb52a3557d62431d0f59d5311dc7d0cf"},
+]
+
 [[package]]
 name = "cachetools"
 version = "5.3.3"
@@ -380,6 +426,29 @@ files = [
     {file = "cligj-0.7.2.tar.gz", hash = "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27"},
 ]
 
+[[package]]
+name = "cohere"
+version = "5.5.8"
+requires_python = "<4.0,>=3.8"
+summary = ""
+groups = ["default"]
+dependencies = [
+    "boto3<2.0.0,>=1.34.0",
+    "fastavro<2.0.0,>=1.9.4",
+    "httpx-sse<0.5.0,>=0.4.0",
+    "httpx>=0.21.2",
+    "parameterized<0.10.0,>=0.9.0",
+    "pydantic>=1.9.2",
+    "requests<3.0.0,>=2.0.0",
+    "tokenizers<1,>=0.15",
+    "types-requests<3.0.0,>=2.0.0",
+    "typing-extensions>=4.0.0",
+]
+files = [
+    {file = "cohere-5.5.8-py3-none-any.whl", hash = "sha256:e1ed84b90eadd13c6a68ee28e378a0bb955f8945eadc6eb7ee126b3399cafd54"},
+    {file = "cohere-5.5.8.tar.gz", hash = "sha256:84ce7666ff8fbdf4f41fb5f6ca452ab2639a514bc88967a2854a9b1b820d6ea0"},
+]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -597,6 +666,34 @@ files = [
     {file = "expiringdict-1.2.2.tar.gz", hash = "sha256:300fb92a7e98f15b05cf9a856c1415b3bc4f2e132be07daa326da6414c23ee09"},
 ]
 
+[[package]]
+name = "fastavro"
+version = "1.9.4"
+requires_python = ">=3.8"
+summary = "Fast read/write of AVRO files"
+groups = ["default"]
+files = [
+    {file = "fastavro-1.9.4-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:60cb38f07462a7fb4e4440ed0de67d3d400ae6b3d780f81327bebde9aa55faef"},
+    {file = "fastavro-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:063d01d197fc929c20adc09ca9f0ca86d33ac25ee0963ce0b438244eee8315ae"},
+    {file = "fastavro-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87a9053fcfbc895f2a16a4303af22077e3a8fdcf1cd5d6ed47ff2ef22cbba2f0"},
+    {file = "fastavro-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:02bf1276b7326397314adf41b34a4890f6ffa59cf7e0eb20b9e4ab0a143a1598"},
+    {file = "fastavro-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56bed9eca435389a8861e6e2d631ec7f8f5dda5b23f93517ac710665bd34ca29"},
+    {file = "fastavro-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:0cd2099c8c672b853e0b20c13e9b62a69d3fbf67ee7c59c7271ba5df1680310d"},
+    {file = "fastavro-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:af8c6d8c43a02b5569c093fc5467469541ac408c79c36a5b0900d3dd0b3ba838"},
+    {file = "fastavro-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4a138710bd61580324d23bc5e3df01f0b82aee0a76404d5dddae73d9e4c723f"},
+    {file = "fastavro-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:903d97418120ca6b6a7f38a731166c1ccc2c4344ee5e0470d09eb1dc3687540a"},
+    {file = "fastavro-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c443eeb99899d062dbf78c525e4614dd77e041a7688fa2710c224f4033f193ae"},
+    {file = "fastavro-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ac26ab0774d1b2b7af6d8f4300ad20bbc4b5469e658a02931ad13ce23635152f"},
+    {file = "fastavro-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:cf7247874c22be856ba7d1f46a0f6e0379a6025f1a48a7da640444cbac6f570b"},
+    {file = "fastavro-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:68912f2020e1b3d70557260b27dd85fb49a4fc6bfab18d384926127452c1da4c"},
+    {file = "fastavro-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6925ce137cdd78e109abdb0bc33aad55de6c9f2d2d3036b65453128f2f5f5b92"},
+    {file = "fastavro-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b928cd294e36e35516d0deb9e104b45be922ba06940794260a4e5dbed6c192a"},
+    {file = "fastavro-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:90c9838bc4c991ffff5dd9d88a0cc0030f938b3fdf038cdf6babde144b920246"},
+    {file = "fastavro-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:eca6e54da571b06a3c5a72dbb7212073f56c92a6fbfbf847b91c347510f8a426"},
+    {file = "fastavro-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a4b02839ac261100cefca2e2ad04cdfedc556cb66b5ec735e0db428e74b399de"},
+    {file = "fastavro-1.9.4.tar.gz", hash = "sha256:56b8363e360a1256c94562393dc7f8611f3baf2b3159f64fb2b9c6b87b14e876"},
+]
+
 [[package]]
 name = "filelock"
 version = "3.14.0"
@@ -1060,6 +1157,17 @@ files = [
     {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"},
 ]
 
+[[package]]
+name = "httpx-sse"
+version = "0.4.0"
+requires_python = ">=3.8"
+summary = "Consume Server-Sent Event (SSE) messages with HTTPX."
+groups = ["default"]
+files = [
+    {file = "httpx-sse-0.4.0.tar.gz", hash = "sha256:1e81a3a3070ce322add1d3529ed42eb5f70817f45ed6ec915ab753f961139721"},
+    {file = "httpx_sse-0.4.0-py3-none-any.whl", hash = "sha256:f329af6eae57eaa2bdfd962b42524764af68075ea87370a2de920af5341e318f"},
+]
+
 [[package]]
 name = "huggingface-hub"
 version = "0.23.0"
@@ -1119,6 +1227,17 @@ files = [
     {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"},
 ]
 
+[[package]]
+name = "jmespath"
+version = "1.0.1"
+requires_python = ">=3.7"
+summary = "JSON Matching Expressions"
+groups = ["default"]
+files = [
+    {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
+    {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
+]
+
 [[package]]
 name = "kiwisolver"
 version = "1.4.5"
@@ -1189,6 +1308,17 @@ files = [
     {file = "kiwisolver-1.4.5.tar.gz", hash = "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec"},
 ]
 
+[[package]]
+name = "markdown"
+version = "3.6"
+requires_python = ">=3.8"
+summary = "Python implementation of John Gruber's Markdown."
+groups = ["default"]
+files = [
+    {file = "Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f"},
+    {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"},
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "3.0.0"
@@ -1697,6 +1827,17 @@ files = [
     {file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"},
 ]
 
+[[package]]
+name = "parameterized"
+version = "0.9.0"
+requires_python = ">=3.7"
+summary = "Parameterized testing with any Python test framework"
+groups = ["default"]
+files = [
+    {file = "parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b"},
+    {file = "parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1"},
+]
+
 [[package]]
 name = "pendulum"
 version = "3.0.0"
@@ -2292,6 +2433,20 @@ files = [
     {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"},
 ]
 
+[[package]]
+name = "s3transfer"
+version = "0.10.2"
+requires_python = ">=3.8"
+summary = "An Amazon S3 Transfer Manager"
+groups = ["default"]
+dependencies = [
+    "botocore<2.0a.0,>=1.33.2",
+]
+files = [
+    {file = "s3transfer-0.10.2-py3-none-any.whl", hash = "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69"},
+    {file = "s3transfer-0.10.2.tar.gz", hash = "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6"},
+]
+
 [[package]]
 name = "safetensors"
 version = "0.4.3"
@@ -2463,6 +2618,17 @@ files = [
     {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"},
 ]
 
+[[package]]
+name = "soupsieve"
+version = "2.5"
+requires_python = ">=3.8"
+summary = "A modern CSS selector implementation for Beautiful Soup."
+groups = ["default"]
+files = [
+    {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"},
+    {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"},
+]
+
 [[package]]
 name = "svgwrite"
 version = "1.4.3"
@@ -2826,6 +2992,20 @@ files = [
     {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"},
 ]
 
+[[package]]
+name = "types-requests"
+version = "2.32.0.20240622"
+requires_python = ">=3.8"
+summary = "Typing stubs for requests"
+groups = ["default"]
+dependencies = [
+    "urllib3>=2",
+]
+files = [
+    {file = "types-requests-2.32.0.20240622.tar.gz", hash = "sha256:ed5e8a412fcc39159d6319385c009d642845f250c63902718f605cd90faade31"},
+    {file = "types_requests-2.32.0.20240622-py3-none-any.whl", hash = "sha256:97bac6b54b5bd4cf91d407e62f0932a74821bc2211f22116d9ee1dd643826caf"},
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.11.0"

pyproject.toml

@@ -19,5 +19,8 @@ dependencies = [
     "dify-client>=0.1.10",
     "chattts-fork>=0.0.1",
     "expiringdict>=1.2.2",
+    "beautifulsoup4>=4.12.3",
+    "Markdown>=3.6",
+    "cohere>=5.5.8",
 ]
 requires-python = ">=3.10"

requirements.txt

@@ -9,6 +9,9 @@ antlr4-python3-runtime==4.9.3
 anyio==4.3.0
 async-timeout==4.0.3; python_version < "3.11"
 attrs==23.2.0
+beautifulsoup4==4.12.3
+boto3==1.34.135
+botocore==1.34.135
 cachetools==5.3.3
 cairocffi==1.7.0
 cairosvg==2.7.1
@@ -19,6 +22,7 @@ chattts-fork==0.0.1
 click==8.1.7
 click-plugins==1.1.1
 cligj==0.7.2
+cohere==5.5.8
 colorama==0.4.6; platform_system == "Windows"
 colour==0.1.5
 contourpy==1.2.1
@@ -34,6 +38,7 @@ encodec==0.1.1
 eval-type-backport==0.2.0
 exceptiongroup==1.2.1; python_version < "3.11"
 expiringdict==1.2.2
+fastavro==1.9.4
 filelock==3.14.0
 fiona==1.9.6
 fonttools==4.51.0
@@ -56,11 +61,14 @@ h11==0.14.0
 httpcore==1.0.5
 httplib2==0.22.0
 httpx==0.27.0
+httpx-sse==0.4.0
 huggingface-hub==0.23.0
 idna==3.7
 intel-openmp==2021.4.0; platform_system == "Windows"
 jinja2==3.1.4
+jmespath==1.0.1
 kiwisolver==1.4.5
+markdown==3.6
 markdown-it-py==3.0.0
 markupsafe==2.1.5
 matplotlib==3.8.4
@@ -88,6 +96,7 @@ openai==1.30.1
 osmnx==1.9.2
 packaging==24.0
 pandas==2.2.2
+parameterized==0.9.0
 pendulum==3.0.0
 pillow==10.3.0
 platformdirs==4.2.1
@@ -112,12 +121,14 @@ regex==2024.5.15
 requests==2.31.0
 rich==13.7.1
 rsa==4.9
+s3transfer==0.10.2
 safetensors==0.4.3
 scipy==1.13.1
 shapely==2.0.4
 shellingham==1.5.4
 six==1.16.0
 sniffio==1.3.1
+soupsieve==2.5
 svgwrite==1.4.3
 sympy==1.12
 tabulate==0.9.0
@@ -133,6 +144,7 @@ tqdm==4.66.4
 transformers==4.41.1
 triton==2.3.0; platform_system == "Linux" and platform_machine == "x86_64" and python_version < "3.12"
 typer==0.12.3
+types-requests==2.32.0.20240622
 typing-extensions==4.11.0
 tzdata==2024.1
 uritemplate==4.1.1