Merge pull request #726 from RockChinQ/feat/qq-botpy-cache

Feat: qq-botpy 适配器对 member 和 group 的 openid 进行静态缓存

pkg/config/impls/json.py

@@ -45,3 +45,7 @@ class JSONConfigFile(file_model.ConfigFile):
     async def save(self, cfg: dict):
         with open(self.config_file_name, 'w', encoding='utf-8') as f:
             json.dump(cfg, f, indent=4, ensure_ascii=False)
+
+    def save_sync(self, cfg: dict):
+        with open(self.config_file_name, 'w', encoding='utf-8') as f:
+            json.dump(cfg, f, indent=4, ensure_ascii=False)

pkg/config/impls/pymodule.py

@@ -60,3 +60,6 @@ class PythonModuleConfigFile(file_model.ConfigFile):
 
     async def save(self, data: dict):
         logging.warning('Python模块配置文件不支持保存')
+
+    def save_sync(self, data: dict):
+        logging.warning('Python模块配置文件不支持保存')

pkg/config/manager.py

@@ -26,6 +26,9 @@ class ConfigManager:
     async def dump_config(self):
         await self.file.save(self.data)
 
+    def dump_config_sync(self):
+        self.file.save_sync(self.data)
+
 
 async def load_python_module_config(config_name: str, template_name: str) -> ConfigManager:
     """加载Python模块配置文件"""

pkg/config/model.py

@@ -25,3 +25,7 @@ class ConfigFile(metaclass=abc.ABCMeta):
     @abc.abstractmethod
     async def save(self, data: dict):
         pass
+
+    @abc.abstractmethod
+    def save_sync(self, data: dict):
+        pass

pkg/core/bootutils/files.py

@@ -20,6 +20,7 @@ required_files = {
 required_paths = [
     "temp",
     "data",
+    "data/metadata",
     "data/prompts",
     "data/scenario",
     "data/logs",

pkg/platform/sources/qqbotpy.py

@@ -17,6 +17,7 @@ import botpy.types.message as botpy_message_type
 from .. import adapter as adapter_model
 from ...pipeline.longtext.strategies import forward
 from ...core import app
+from ...config import manager as cfg_mgr
 
 
 class OfficialGroupMessage(mirai.GroupMessage):
@@ -34,6 +35,7 @@ cached_message_ids = {}
 
 id_index = 0
 
+
 def save_msg_id(message_id: str) -> int:
     """保存消息id"""
     global id_index, cached_message_ids
@@ -43,43 +45,82 @@ def save_msg_id(message_id: str) -> int:
     cached_message_ids[str(crt_index)] = message_id
     return crt_index
 
-cached_member_openids = {}
-"""QQ官方 用户的id是字符串，而YiriMirai的用户id是整数，所以需要一个索引来进行转换"""
 
-member_openid_index = 100
+def char_to_value(char):
+    """将单个字符转换为相应的数值。"""
+    if '0' <= char <= '9':
+        return ord(char) - ord('0')
+    elif 'A' <= char <= 'Z':
+        return ord(char) - ord('A') + 10
     
-def save_member_openid(member_openid: str) -> int:
-    """保存用户id"""
-    global member_openid_index, cached_member_openids
+    return ord(char) - ord('a') + 36
 
-    if member_openid in cached_member_openids.values():
-        return list(cached_member_openids.keys())[list(cached_member_openids.values()).index(member_openid)]
+def digest(s: str) -> int:
+    """计算字符串的hash值。"""
+    # 取末尾的8位
+    sub_s = s[-10:]
 
-    crt_index = member_openid_index
-    member_openid_index += 1
-    cached_member_openids[str(crt_index)] = member_openid
-    return crt_index
+    number = 0
+    base = 36
 
-cached_group_openids = {}
-"""QQ官方 群组的id是字符串，而YiriMirai的群组id是整数，所以需要一个索引来进行转换"""
+    for i in range(len(sub_s)):
+        number = number * base + char_to_value(sub_s[i])
 
-group_openid_index = 1000
+    return number
 
-def save_group_openid(group_openid: str) -> int:
-    """保存群组id"""
-    global group_openid_index, cached_group_openids
+K = typing.TypeVar("K")
+V = typing.TypeVar("V")
 
-    if group_openid in cached_group_openids.values():
-        return list(cached_group_openids.keys())[list(cached_group_openids.values()).index(group_openid)]
 
-    crt_index = group_openid_index
-    group_openid_index += 1
-    cached_group_openids[str(crt_index)] = group_openid
-    return crt_index
+class OpenIDMapping(typing.Generic[K, V]):
+
+    map: dict[K, V]
+
+    dump_func: typing.Callable
+
+    digest_func: typing.Callable[[K], V]
+
+    def __init__(self, map: dict[K, V], dump_func: typing.Callable, digest_func: typing.Callable[[K], V] = digest):
+        self.map = map
+
+        self.dump_func = dump_func
+
+        self.digest_func = digest_func
+
+    def __getitem__(self, key: K) -> V:
+        return self.map[key]
+
+    def __setitem__(self, key: K, value: V):
+        self.map[key] = value
+        self.dump_func()
+
+    def __contains__(self, key: K) -> bool:
+        return key in self.map
+
+    def __delitem__(self, key: K):
+        del self.map[key]
+        self.dump_func()
+
+    def getkey(self, value: V) -> K:
+        return list(self.map.keys())[list(self.map.values()).index(value)]
+    
+    def save_openid(self, key: K) -> V:
+        
+        if key in self.map:
+            return self.map[key]
+        
+        value = self.digest_func(key)
+
+        self.map[key] = value
+
+        self.dump_func()
+
+        return value
 
 
 class OfficialMessageConverter(adapter_model.MessageConverter):
     """QQ 官方消息转换器"""
+
     @staticmethod
     def yiri2target(message_chain: mirai.MessageChain):
         """将 YiriMirai 的消息链转换为 QQ 官方消息"""
@@ -92,7 +133,9 @@ class OfficialMessageConverter(adapter_model.MessageConverter):
         elif type(message_chain) is str:
             msg_list = [mirai.Plain(text=message_chain)]
         else:
-            raise Exception("Unknown message type: " + str(message_chain) + str(type(message_chain)))
+            raise Exception(
+                "Unknown message type: " + str(message_chain) + str(type(message_chain))
+            )
 
         offcial_messages: list[dict] = []
         """
@@ -110,36 +153,24 @@ class OfficialMessageConverter(adapter_model.MessageConverter):
         # 遍历并转换
         for component in msg_list:
             if type(component) is mirai.Plain:
-                offcial_messages.append({
-                    "type": "text",
-                    "content": component.text
-                })
+                offcial_messages.append({"type": "text", "content": component.text})
             elif type(component) is mirai.Image:
                 if component.url is not None:
-                    offcial_messages.append(
-                        {
-                            "type": "image",
-                            "content": component.url
-                        }
-                    )
+                    offcial_messages.append({"type": "image", "content": component.url})
                 elif component.path is not None:
                     offcial_messages.append(
-                        {
-                            "type": "file_image",
-                            "content": component.path
-                        }
+                        {"type": "file_image", "content": component.path}
                     )
             elif type(component) is mirai.At:
-                offcial_messages.append(
-                    {
-                        "type": "at",
-                        "content": ""
-                    }
-                )
+                offcial_messages.append({"type": "at", "content": ""})
             elif type(component) is mirai.AtAll:
-                print("上层组件要求发送 AtAll 消息，但 QQ 官方 API 不支持此消息类型，忽略此消息。")
+                print(
+                    "上层组件要求发送 AtAll 消息，但 QQ 官方 API 不支持此消息类型，忽略此消息。"
+                )
             elif type(component) is mirai.Voice:
-                print("上层组件要求发送 Voice 消息，但 QQ 官方 API 不支持此消息类型，忽略此消息。")
+                print(
+                    "上层组件要求发送 Voice 消息，但 QQ 官方 API 不支持此消息类型，忽略此消息。"
+                )
             elif type(component) is forward.Forward:
                 # 转发消息
                 yiri_forward_node_list = component.node_list
@@ -150,20 +181,31 @@ class OfficialMessageConverter(adapter_model.MessageConverter):
                         message_chain = yiri_forward_node.message_chain
 
                         # 平铺
-                        offcial_messages.extend(OfficialMessageConverter.yiri2target(message_chain))
+                        offcial_messages.extend(
+                            OfficialMessageConverter.yiri2target(message_chain)
+                        )
                     except Exception as e:
                         import traceback
+
                         traceback.print_exc()
 
         return offcial_messages
 
     @staticmethod
-    def extract_message_chain_from_obj(message: typing.Union[botpy_message.Message, botpy_message.DirectMessage], message_id: str = None, bot_account_id: int = 0) -> mirai.MessageChain:
+    def extract_message_chain_from_obj(
+        message: typing.Union[botpy_message.Message, botpy_message.DirectMessage],
+        message_id: str = None,
+        bot_account_id: int = 0,
+    ) -> mirai.MessageChain:
         yiri_msg_list = []
 
         # 存id
 
-        yiri_msg_list.append(mirai.models.message.Source(id=save_msg_id(message_id), time=datetime.datetime.now()))
+        yiri_msg_list.append(
+            mirai.models.message.Source(
+                id=save_msg_id(message_id), time=datetime.datetime.now()
+            )
+        )
 
         if type(message) is not botpy_message.DirectMessage:
             yiri_msg_list.append(mirai.At(target=bot_account_id))
@@ -179,7 +221,9 @@ class OfficialMessageConverter(adapter_model.MessageConverter):
             if attachment.content_type == "image":
                 yiri_msg_list.append(mirai.Image(url=attachment.url))
             else:
-                logging.warning("不支持的附件类型：" + attachment.content_type + "，忽略此附件。")
+                logging.warning(
+                    "不支持的附件类型：" + attachment.content_type + "，忽略此附件。"
+                )
 
         content = re.sub(r"<@!\d+>", "", str(message.content))
         if content.strip() != "":
@@ -192,25 +236,36 @@ class OfficialMessageConverter(adapter_model.MessageConverter):
 
 class OfficialEventConverter(adapter_model.EventConverter):
     """事件转换器"""
-    @staticmethod
-    def yiri2target(event: typing.Type[mirai.Event]):
+
+    member_openid_mapping: OpenIDMapping[str, int]
+    group_openid_mapping: OpenIDMapping[str, int]
+
+    def __init__(self, member_openid_mapping: OpenIDMapping[str, int], group_openid_mapping: OpenIDMapping[str, int]):
+        self.member_openid_mapping = member_openid_mapping
+        self.group_openid_mapping = group_openid_mapping
+
+    def yiri2target(self, event: typing.Type[mirai.Event]):
         if event == mirai.GroupMessage:
             return botpy_message.Message
         elif event == mirai.FriendMessage:
             return botpy_message.DirectMessage
         else:
-            raise Exception("未支持转换的事件类型(YiriMirai -> Official): " + str(event))
+            raise Exception(
+                "未支持转换的事件类型(YiriMirai -> Official): " + str(event)
+            )
 
-    @staticmethod
-    def target2yiri(event: typing.Union[botpy_message.Message, botpy_message.DirectMessage]) -> mirai.Event:
+    def target2yiri(
+        self,
+        event: typing.Union[botpy_message.Message, botpy_message.DirectMessage]
+    ) -> mirai.Event:
         import mirai.models.entities as mirai_entities
 
         if type(event) == botpy_message.Message:  # 频道内，转群聊事件
             permission = "MEMBER"
 
-            if '2' in event.member.roles:
+            if "2" in event.member.roles:
                 permission = "ADMINISTRATOR"
-            elif '4' in event.member.roles:
+            elif "4" in event.member.roles:
                 permission = "OWNER"
 
             return mirai.GroupMessage(
@@ -221,15 +276,25 @@ class OfficialEventConverter(adapter_model.EventConverter):
                     group=mirai_entities.Group(
                         id=event.channel_id,
                         name=event.author.username,
-                        permission=mirai_entities.Permission.Member
+                        permission=mirai_entities.Permission.Member,
+                    ),
+                    special_title="",
+                    join_timestamp=int(
+                        datetime.datetime.strptime(
+                            event.member.joined_at, "%Y-%m-%dT%H:%M:%S%z"
+                        ).timestamp()
                     ),
-                    special_title='',
-                    join_timestamp=int(datetime.datetime.strptime(event.member.joined_at, "%Y-%m-%dT%H:%M:%S%z").timestamp()),
                     last_speak_timestamp=datetime.datetime.now().timestamp(),
                     mute_time_remaining=0,
                 ),
-                message_chain=OfficialMessageConverter.extract_message_chain_from_obj(event, event.id),
-                time=int(datetime.datetime.strptime(event.timestamp, "%Y-%m-%dT%H:%M:%S%z").timestamp()),
+                message_chain=OfficialMessageConverter.extract_message_chain_from_obj(
+                    event, event.id
+                ),
+                time=int(
+                    datetime.datetime.strptime(
+                        event.timestamp, "%Y-%m-%dT%H:%M:%S%z"
+                    ).timestamp()
+                ),
             )
         elif type(event) == botpy_message.DirectMessage:  # 私聊，转私聊事件
             return mirai.FriendMessage(
@@ -238,12 +303,18 @@ class OfficialEventConverter(adapter_model.EventConverter):
                     nickname=event.author.username,
                     remark=event.author.username,
                 ),
-                message_chain=OfficialMessageConverter.extract_message_chain_from_obj(event, event.id),
-                time=int(datetime.datetime.strptime(event.timestamp, "%Y-%m-%dT%H:%M:%S%z").timestamp()),
+                message_chain=OfficialMessageConverter.extract_message_chain_from_obj(
+                    event, event.id
+                ),
+                time=int(
+                    datetime.datetime.strptime(
+                        event.timestamp, "%Y-%m-%dT%H:%M:%S%z"
+                    ).timestamp()
+                ),
             )
         elif type(event) == botpy_message.GroupMessage:
 
-            replacing_member_id = save_member_openid(event.author.member_openid)
+            replacing_member_id = self.member_openid_mapping.save_openid(event.author.member_openid)
 
             return OfficialGroupMessage(
                 sender=mirai_entities.GroupMember(
@@ -251,29 +322,36 @@ class OfficialEventConverter(adapter_model.EventConverter):
                     member_name=replacing_member_id,
                     permission="MEMBER",
                     group=mirai_entities.Group(
-                        id=save_group_openid(event.group_openid),
+                        id=self.group_openid_mapping.save_openid(event.group_openid),
                         name=replacing_member_id,
-                        permission=mirai_entities.Permission.Member
+                        permission=mirai_entities.Permission.Member,
                     ),
-                    special_title='',
+                    special_title="",
                     join_timestamp=int(0),
                     last_speak_timestamp=datetime.datetime.now().timestamp(),
                     mute_time_remaining=0,
                 ),
-                message_chain=OfficialMessageConverter.extract_message_chain_from_obj(event, event.id),
-                time=int(datetime.datetime.strptime(event.timestamp, "%Y-%m-%dT%H:%M:%S%z").timestamp()),
+                message_chain=OfficialMessageConverter.extract_message_chain_from_obj(
+                    event, event.id
+                ),
+                time=int(
+                    datetime.datetime.strptime(
+                        event.timestamp, "%Y-%m-%dT%H:%M:%S%z"
+                    ).timestamp()
+                ),
             )
 
 
 @adapter_model.adapter_class("qq-botpy")
 class OfficialAdapter(adapter_model.MessageSourceAdapter):
     """QQ 官方消息适配器"""
+
     bot: botpy.Client = None
 
     bot_account_id: int = 0
 
-    message_converter: OfficialMessageConverter = OfficialMessageConverter()
-    # event_handler: adapter_model.EventHandler = adapter_model.EventHandler()
+    message_converter: OfficialMessageConverter
+    event_converter: OfficialEventConverter
 
     cfg: dict = None
 
@@ -285,6 +363,11 @@ class OfficialAdapter(adapter_model.MessageSourceAdapter):
 
     ap: app.Application
 
+    metadata: cfg_mgr.ConfigManager = None
+
+    member_openid_mapping: OpenIDMapping[str, int] = None
+    group_openid_mapping: OpenIDMapping[str, int] = None
+
     def __init__(self, cfg: dict, ap: app.Application):
         """初始化适配器"""
         self.cfg = cfg
@@ -292,20 +375,17 @@ class OfficialAdapter(adapter_model.MessageSourceAdapter):
 
         switchs = {}
 
-        for intent in cfg['intents']:
+        for intent in cfg["intents"]:
             switchs[intent] = True
 
-        del cfg['intents']
+        del cfg["intents"]
 
         intents = botpy.Intents(**switchs)
 
         self.bot = botpy.Client(intents=intents)
 
     async def send_message(
-        self,
-        target_type: str,
-        target_id: str,
-        message: mirai.MessageChain
+        self, target_type: str, target_id: str, message: mirai.MessageChain
     ):
         pass
 
@@ -313,7 +393,7 @@ class OfficialAdapter(adapter_model.MessageSourceAdapter):
         self,
         message_source: mirai.MessageEvent,
         message: mirai.MessageChain,
-        quote_origin: bool = False
+        quote_origin: bool = False,
     ):
         message_list = self.message_converter.yiri2target(message)
         tasks = []
@@ -323,40 +403,50 @@ class OfficialAdapter(adapter_model.MessageSourceAdapter):
         for msg in message_list:
             args = {}
 
-            if msg['type'] == 'text':
-                args['content'] = msg['content']
-            elif msg['type'] == 'image':
-                args['image'] = msg['content']
-            elif msg['type'] == 'file_image':
-                args['file_image'] = msg["content"]
+            if msg["type"] == "text":
+                args["content"] = msg["content"]
+            elif msg["type"] == "image":
+                args["image"] = msg["content"]
+            elif msg["type"] == "file_image":
+                args["file_image"] = msg["content"]
             else:
                 continue
 
             if quote_origin:
-                args['message_reference'] = botpy_message_type.Reference(message_id=cached_message_ids[str(message_source.message_chain.message_id)])
+                args["message_reference"] = botpy_message_type.Reference(
+                    message_id=cached_message_ids[
+                        str(message_source.message_chain.message_id)
+                    ]
+                )
 
             if type(message_source) == mirai.GroupMessage:
-                args['channel_id'] = str(message_source.sender.group.id)
-                args['msg_id'] = cached_message_ids[str(message_source.message_chain.message_id)]
+                args["channel_id"] = str(message_source.sender.group.id)
+                args["msg_id"] = cached_message_ids[
+                    str(message_source.message_chain.message_id)
+                ]
                 await self.bot.api.post_message(**args)
             elif type(message_source) == mirai.FriendMessage:
-                args['guild_id'] = str(message_source.sender.id)
-                args['msg_id'] = cached_message_ids[str(message_source.message_chain.message_id)]
+                args["guild_id"] = str(message_source.sender.id)
+                args["msg_id"] = cached_message_ids[
+                    str(message_source.message_chain.message_id)
+                ]
                 await self.bot.api.post_dms(**args)
             elif type(message_source) == OfficialGroupMessage:
                 # args['guild_id'] = str(message_source.sender.group.id)
                 # args['msg_id'] = cached_message_ids[str(message_source.message_chain.message_id)]
                 # await self.bot.api.post_message(**args)
-                if 'image' in args or 'file_image' in args:
+                if "image" in args or "file_image" in args:
                     continue
-                args['group_openid'] = cached_group_openids[str(message_source.sender.group.id)]
-                args['msg_id'] = cached_message_ids[str(message_source.message_chain.message_id)]
-                args['msg_seq'] = msg_seq
-                msg_seq += 1
-                await self.bot.api.post_group_message(
-                    **args
+                args["group_openid"] = self.group_openid_mapping.getkey(
+                    message_source.sender.group.id
                 )
 
+                args["msg_id"] = cached_message_ids[
+                    str(message_source.message_chain.message_id)
+                ]
+                args["msg_seq"] = msg_seq
+                msg_seq += 1
+                await self.bot.api.post_group_message(**args)
 
     async def is_muted(self, group_id: int) -> bool:
         return False
@@ -364,14 +454,22 @@ class OfficialAdapter(adapter_model.MessageSourceAdapter):
     def register_listener(
         self,
         event_type: typing.Type[mirai.Event],
-        callback: typing.Callable[[mirai.Event, adapter_model.MessageSourceAdapter], None]
+        callback: typing.Callable[
+            [mirai.Event, adapter_model.MessageSourceAdapter], None
+        ],
     ):
 
         try:
 
-            async def wrapper(message: typing.Union[botpy_message.Message, botpy_message.DirectMessage, botpy_message.GroupMessage]):
+            async def wrapper(
+                message: typing.Union[
+                    botpy_message.Message,
+                    botpy_message.DirectMessage,
+                    botpy_message.GroupMessage,
+                ]
+            ):
                 self.cached_official_messages[str(message.id)] = message
-                await callback(OfficialEventConverter.target2yiri(message), self)
+                await callback(self.event_converter.target2yiri(message), self)
 
             for event_handler in event_handler_mapping[event_type]:
                 setattr(self.bot, event_handler, wrapper)
@@ -382,15 +480,36 @@ class OfficialAdapter(adapter_model.MessageSourceAdapter):
     def unregister_listener(
         self,
         event_type: typing.Type[mirai.Event],
-        callback: typing.Callable[[mirai.Event, adapter_model.MessageSourceAdapter], None]
+        callback: typing.Callable[
+            [mirai.Event, adapter_model.MessageSourceAdapter], None
+        ],
     ):
         delattr(self.bot, event_handler_mapping[event_type])
 
     async def run_async(self):
-        self.ap.logger.info("运行 QQ 官方适配器")
-        await self.bot.start(
-            **self.cfg
+
+        self.metadata = await cfg_mgr.load_json_config(
+            "data/metadata/adapter-qq-botpy.json",
+            "templates/metadata/adapter-qq-botpy.json",
         )
 
+        self.member_openid_mapping = OpenIDMapping(
+            map=self.metadata.data["mapping"]["members"],
+            dump_func=self.metadata.dump_config_sync,
+        )
+
+        self.group_openid_mapping = OpenIDMapping(
+            map=self.metadata.data["mapping"]["groups"],
+            dump_func=self.metadata.dump_config_sync,
+        )
+
+        self.message_converter = OfficialMessageConverter()
+        self.event_converter = OfficialEventConverter(
+            self.member_openid_mapping, self.group_openid_mapping
+        )
+
+        self.ap.logger.info("运行 QQ 官方适配器")
+        await self.bot.start(**self.cfg)
+
     def kill(self) -> bool:
         return False

templates/metadata/adapter-qq-botpy.json

@@ -0,0 +1,6 @@
+{
+    "mapping": {
+        "groups": {},
+        "members": {}
+    }
+}