enh/refac: distributed crdt

backend/open_webui/socket/utils.py

@@ -1,6 +1,8 @@
 import json
 import uuid
 from open_webui.utils.redis import get_redis_connection
+from typing import Optional, List, Tuple
+import pycrdt as Y
 
 
 class RedisLock:
@@ -89,3 +91,109 @@ class RedisDict:
         if key not in self:
             self[key] = default
         return self[key]
+
+
+class YdocManager:
+    def __init__(
+        self,
+        redis=None,
+        redis_key_prefix: str = "open-webui:ydoc:documents",
+    ):
+        self._updates = {}
+        self._users = {}
+        self._redis = redis
+        self._redis_key_prefix = redis_key_prefix
+
+    async def append_to_updates(self, document_id: str, update: bytes):
+        document_id = document_id.replace(":", "_")
+        if self._redis:
+            redis_key = f"{self._redis_key_prefix}:{document_id}:updates"
+            await self._redis.rpush(redis_key, json.dumps(list(update)))
+        else:
+            if document_id not in self._updates:
+                self._updates[document_id] = []
+            self._updates[document_id].append(update)
+
+    async def get_updates(self, document_id: str) -> List[bytes]:
+        document_id = document_id.replace(":", "_")
+
+        if self._redis:
+            redis_key = f"{self._redis_key_prefix}:{document_id}:updates"
+            updates = await self._redis.lrange(redis_key, 0, -1)
+            return [bytes(json.loads(update)) for update in updates]
+        else:
+            return self._updates.get(document_id, [])
+
+    async def document_exists(self, document_id: str) -> bool:
+        document_id = document_id.replace(":", "_")
+
+        if self._redis:
+            redis_key = f"{self._redis_key_prefix}:{document_id}:updates"
+            return await self._redis.exists(redis_key) > 0
+        else:
+            return document_id in self._updates
+
+    async def get_users(self, document_id: str) -> List[str]:
+        document_id = document_id.replace(":", "_")
+
+        if self._redis:
+            redis_key = f"{self._redis_key_prefix}:{document_id}:users"
+            users = await self._redis.smembers(redis_key)
+            return list(users)
+        else:
+            return self._users.get(document_id, [])
+
+    async def add_user(self, document_id: str, user_id: str):
+        document_id = document_id.replace(":", "_")
+
+        if self._redis:
+            redis_key = f"{self._redis_key_prefix}:{document_id}:users"
+            await self._redis.sadd(redis_key, user_id)
+        else:
+            if document_id not in self._users:
+                self._users[document_id] = set()
+            self._users[document_id].add(user_id)
+
+    async def remove_user(self, document_id: str, user_id: str):
+        document_id = document_id.replace(":", "_")
+
+        if self._redis:
+            redis_key = f"{self._redis_key_prefix}:{document_id}:users"
+            await self._redis.srem(redis_key, user_id)
+        else:
+            if document_id in self._users and user_id in self._users[document_id]:
+                self._users[document_id].remove(user_id)
+
+    async def remove_user_from_all_documents(self, user_id: str):
+        if self._redis:
+            keys = await self._redis.keys(f"{self._redis_key_prefix}:*")
+            for key in keys:
+                if key.endswith(":users"):
+                    await self._redis.srem(key, user_id)
+
+                    document_id = key.split(":")[-2]
+                    if len(await self.get_users(document_id)) == 0:
+                        await self.clear_document(document_id)
+
+        else:
+            for document_id in list(self._users.keys()):
+                if user_id in self._users[document_id]:
+                    self._users[document_id].remove(user_id)
+                    if not self._users[document_id]:
+                        del self._users[document_id]
+
+                        await self.clear_document(document_id)
+
+    async def clear_document(self, document_id: str):
+        document_id = document_id.replace(":", "_")
+
+        if self._redis:
+            redis_key = f"{self._redis_key_prefix}:{document_id}:updates"
+            await self._redis.delete(redis_key)
+            redis_users_key = f"{self._redis_key_prefix}:{document_id}:users"
+            await self._redis.delete(redis_users_key)
+        else:
+            if document_id in self._updates:
+                del self._updates[document_id]
+            if document_id in self._users:
+                del self._users[document_id]