Add filters, backups, use AwareDateTimeField

jarvis_core/db/fields.py

@@ -0,0 +1,26 @@
+import bson
+import marshmallow as ma
+from marshmallow import fields as ma_fields
+from umongo import fields
+
+
+class BinaryField(fields.BaseField, ma_fields.Field):
+    default_error_messages = {"invalid": "Not a valid byte sequence."}
+
+    def _serialize(self, value, attr, data, **kwargs):
+        return bytes(value)
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        if not isinstance(value, bytes):
+            self.fail("invalid")
+        return value
+
+    def _serialize_to_mongo(self, obj):
+        return bson.binary.Binary(obj)
+
+    def _deserialize_from_mongo(self, value):
+        return bytes(value)
+
+
+class RawField(fields.BaseField, ma_fields.Raw):
+    pass

jarvis_core/db/models/__init__.py

@@ -1,29 +1,19 @@
 """JARVIS database models."""
-from datetime import datetime, timezone
+import re
+from datetime import datetime
 from typing import Any, List
 
 import marshmallow as ma
 from umongo import Document, EmbeddedDocument, fields
 
 from jarvis_core.db import CTC2_INST, JARVIS_INST
-from jarvis_core.db.models.actions import Ban, Kick, Mute, Unban, Warning
+from jarvis_core.db.fields import RawField
-from jarvis_core.db.models.modlog import Action, Modlog, Note
+from jarvis_core.db.models.actions import *
-from jarvis_core.db.models.reddit import (
+from jarvis_core.db.models.backups import *
-    Redditor,
+from jarvis_core.db.models.modlog import *
-    RedditorFollow,
+from jarvis_core.db.models.reddit import *
-    Subreddit,
+from jarvis_core.db.models.twitter import *
-    SubredditFollow,
+from jarvis_core.db.utils import get_now
-)
-from jarvis_core.db.models.twitter import TwitterAccount, TwitterFollow
-
-
-def get_now() -> datetime:
-    """Get proper timestamp."""
-    return datetime.now(tz=timezone.utc)
-
-
-class RawField(fields.BaseField, ma.fields.Raw):
-    pass
 
 
 @JARVIS_INST.register
@@ -32,7 +22,7 @@ class Autopurge(Document):
     channel: int = fields.IntegerField(required=True)
     delay: int = fields.IntegerField(default=30)
     admin: int = fields.IntegerField(required=True)
-    created_at: datetime = fields.DateTimeField(default=datetime.now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -42,7 +32,7 @@ class Autoreact(Document):
     reactions: List[str] = fields.ListField(fields.StringField())
     admin: int = fields.IntegerField(required=True)
     thread: bool = fields.BooleanField(default=True)
-    created_at: datetime = fields.DateTimeField(default=datetime.now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -53,6 +43,22 @@ class Config(Document):
     value: Any = RawField(required=True)
 
 
+@JARVIS_INST.register
+class Filter(Document):
+    """Regex Filter database object."""
+
+    def _validate_filters(value):
+        for v in value:
+            try:
+                re.compile(v)
+            except re.error:
+                raise ValueError(f"Invalid regex: {v}")
+
+    guild: int = fields.IntegerField(required=True)
+    name: str = fields.StringField(required=True)
+    filters: List[str] = fields.ListField(fields.StringField(), validate=[_validate_filters])
+
+
 @CTC2_INST.register
 class Guess(Document):
     """Guess database object."""
@@ -82,7 +88,7 @@ class Lock(Document):
     guild: int = fields.IntegerField(required=True)
     reason: str = fields.StringField(required=True)
     original_perms: Permission = fields.EmbeddedField(Permission, required=False)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -95,7 +101,7 @@ class Lockdown(Document):
     guild: int = fields.IntegerField(required=True)
     reason: str = fields.StringField(required=True)
     original_perms: int = fields.IntegerField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -106,7 +112,7 @@ class Purge(Document):
     channel: int = fields.IntegerField(required=True)
     guild: int = fields.IntegerField(required=True)
     count: int = fields.IntegerField(default=10)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -118,8 +124,8 @@ class Reminder(Document):
     guild: int = fields.IntegerField(required=True)
     channel: int = fields.IntegerField(required=True)
     message: str = fields.StringField(required=True)
-    remind_at: datetime = fields.DateTimeField(required=True)
+    remind_at: datetime = fields.AwareDateTimeField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
     private: bool = fields.BooleanField(default=False)
 
 
@@ -148,7 +154,7 @@ class Roleping(Document):
     guild: int = fields.IntegerField(required=True)
     admin: int = fields.IntegerField(required=True)
     bypass: Bypass = fields.EmbeddedField(Bypass)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -172,7 +178,7 @@ class Star(Document):
     guild: int = fields.IntegerField(required=True)
     admin: int = fields.IntegerField(required=True)
     star: int = fields.IntegerField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -182,7 +188,7 @@ class Starboard(Document):
     channel: int = fields.IntegerField(required=True)
     guild: int = fields.IntegerField(required=True)
     admin: int = fields.IntegerField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -190,9 +196,9 @@ class Tag(Document):
     """Tag database object."""
 
     creator: int = fields.IntegerField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
     editor: int = fields.IntegerField()
-    edited_at: datetime = fields.DateTimeField()
+    edited_at: datetime = fields.AwareDateTimeField()
     name: str = fields.StringField(required=True)
     content: str = fields.StringField(required=True)
     guild: int = fields.IntegerField(required=True)
@@ -206,8 +212,8 @@ class Temprole(Document):
     user: int = fields.IntegerField(required=True)
     role: int = fields.IntegerField(required=True)
     admin: int = fields.IntegerField(required=True)
-    expires_at: datetime = fields.DateTimeField(required=True)
+    expires_at: datetime = fields.AwareDateTimeField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register

jarvis_core/db/models/actions.py

@@ -4,11 +4,7 @@ from datetime import datetime, timezone
 from umongo import Document, fields
 
 from jarvis_core.db import JARVIS_INST
-
+from jarvis_core.db.utils import get_now
-
-def get_now() -> datetime:
-    """Get proper timestamp."""
-    return datetime.now(tz=timezone.utc)
 
 
 @JARVIS_INST.register
@@ -22,7 +18,7 @@ class Ban(Document):
     guild: int = fields.IntegerField(required=True)
     type: str = fields.StringField(default="perm")
     reason: str = fields.StringField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -33,7 +29,7 @@ class Kick(Document):
     guild: int = fields.IntegerField(required=True)
     reason: str = fields.StringField(required=True)
     user: int = fields.IntegerField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -46,7 +42,7 @@ class Mute(Document):
     duration: int = fields.IntegerField(default=10)
     guild: int = fields.IntegerField(required=True)
     reason: str = fields.StringField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -59,7 +55,7 @@ class Unban(Document):
     guild: int = fields.IntegerField(required=True)
     admin: int = fields.IntegerField(required=True)
     reason: str = fields.StringField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -72,5 +68,5 @@ class Warning(Document):
     guild: int = fields.IntegerField(required=True)
     duration: int = fields.IntegerField(default=24)
     reason: str = fields.StringField(required=True)
-    expires_at: datetime = fields.DateTimeField(required=True)
+    expires_at: datetime = fields.AwareDateTimeField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)

jarvis_core/db/models/backups.py

@@ -0,0 +1,122 @@
+from datetime import datetime
+from typing import List, Optional
+
+from umongo import Document, EmbeddedDocument, fields
+
+from jarvis_core import __version__
+from jarvis_core.db import JARVIS_INST
+from jarvis_core.db.fields import BinaryField
+from jarvis_core.db.utils import get_id, get_now
+
+
+@JARVIS_INST.register
+class Image(Document):
+    discord_id: int = fields.IntegerField(unique=True)
+    image_data: List[bytes] = BinaryField()
+    image_ext: str = fields.StringField()
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
+
+
+@JARVIS_INST.register
+class PermissionOverwriteBackup(EmbeddedDocument):
+    id: int = fields.IntegerField()
+    type: int = fields.IntegerField()
+    allow: int = fields.IntegerField()
+    deny: int = fields.IntegerField()
+
+
+@JARVIS_INST.register
+class WebhookBackup(EmbeddedDocument):
+    id: int = fields.IntegerField()
+    channel_id: int = fields.IntegerField()
+    type: int = fields.IntegerField()
+    avatar: Image = fields.ReferenceField(Image)
+    name: str = fields.StringField()
+
+
+@JARVIS_INST.register
+class ChannelBackup(EmbeddedDocument):
+    id: int = fields.IntegerField()
+    name: str = fields.StringField()
+    type: int = fields.IntegerField()
+    position: int = fields.IntegerField()
+    topic: Optional[str] = fields.StringField(default=None)
+    nsfw: bool = fields.BooleanField(default=False)
+    rate_limit_per_user: int = fields.IntegerField(default=None)
+    bitrate: Optional[int] = fields.IntegerField(default=None)
+    user_limit: Optional[int] = fields.IntegerField(default=None)
+    permission_overwrites: List[PermissionOverwriteBackup] = fields.ListField(
+        fields.EmbeddedField(PermissionOverwriteBackup), factory=list
+    )
+    parent_id: Optional[int] = fields.IntegerField(default=None)
+    rtc_region: Optional[str] = fields.StringField(default=None)
+    video_quality_mode: Optional[int] = fields.IntegerField(default=None)
+    default_auto_archive_duration: Optional[int] = fields.IntegerField(default=None)
+    webhooks: List[WebhookBackup] = fields.ListField(
+        fields.EmbeddedField(WebhookBackup), factory=list
+    )
+
+
+@JARVIS_INST.register
+class RoleBackup(EmbeddedDocument):
+    id: int = fields.IntegerField()
+    name: str = fields.StringField()
+    permissions: int = fields.IntegerField()
+    color: str = fields.StringField()
+    hoist: bool = fields.BooleanField()
+    mentionable: bool = fields.BooleanField()
+
+
+@JARVIS_INST.register
+class EmojiBackup(EmbeddedDocument):
+    id: int = fields.IntegerField()
+    name: str = fields.StringField()
+    image: Image = fields.ReferenceField(Image)
+
+
+@JARVIS_INST.register
+class StickerBackup(EmbeddedDocument):
+    id: int = fields.IntegerField()
+    name: str = fields.StringField()
+    format_type: int = fields.IntegerField()
+    tags: str = fields.StringField()
+    type: int = fields.IntegerField()
+    image: Image = fields.ReferenceField(Image)
+
+
+@JARVIS_INST.register
+class GuildBackup(EmbeddedDocument):
+    name: str = fields.StringField(required=True)
+    description: str = fields.StringField(default=None)
+    default_message_notifications: Optional[int] = fields.IntegerField(default=None)
+    explicit_content_filter: Optional[int] = fields.IntegerField(default=None)
+    afk_channel: Optional[int] = fields.IntegerField(default=None)
+    afk_timeout: Optional[int] = fields.IntegerField(default=None)
+    icon: Optional[Image] = fields.ReferenceField(Image, default=None)
+    owner: int = fields.IntegerField(required=True)
+    splash: Optional[Image] = fields.ReferenceField(Image, default=None)
+    discovery_splash: Optional[Image] = fields.ReferenceField(Image, default=None)
+    banner: Optional[Image] = fields.ReferenceField(Image, default=None)
+    system_channel: Optional[int] = fields.IntegerField(default=None)
+    system_channel_flags: Optional[int] = fields.IntegerField(default=None)
+    rules_channel: Optional[int] = fields.IntegerField(default=None)
+    public_updates_channel: Optional[int] = fields.IntegerField(default=None)
+    preferred_locale: Optional[str] = fields.StringField(default=None)
+    features: List[str] = fields.ListField(fields.StringField, factory=list)
+    channels: List[ChannelBackup] = fields.ListField(
+        fields.EmbeddedField(ChannelBackup), factory=list
+    )
+    roles: List[RoleBackup] = fields.ListField(fields.EmbeddedField(RoleBackup), factory=list)
+    emojis: List[EmojiBackup] = fields.ListField(fields.EmbeddedField(EmojiBackup), factory=list)
+    stickers: List[StickerBackup] = fields.ListField(
+        fields.EmbeddedField(StickerBackup), factory=list
+    )
+
+
+@JARVIS_INST.register
+class Backup(Document):
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
+    guild_id: int = fields.IntegerField()
+    bkid: str = fields.StringField(default=get_id)
+    guild: GuildBackup = fields.EmbeddedField(GuildBackup, required=True)
+    version: str = fields.StringField(default=__version__)

jarvis_core/db/models/modlog.py

@@ -2,24 +2,11 @@
 from datetime import datetime, timezone
 from typing import List
 
-import nanoid
 from bson import ObjectId
 from umongo import Document, EmbeddedDocument, fields
 
 from jarvis_core.db import JARVIS_INST
-
+from jarvis_core.db.utils import get_id, get_now
-NANOID_ALPHA = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
-
-
-def get_now() -> datetime:
-    """Get proper timestamp."""
-    return datetime.now(tz=timezone.utc)
-
-
-def get_id() -> str:
-    """Get nanoid."""
-    return nanoid.generate(NANOID_ALPHA, 12)
-
 
 @JARVIS_INST.register
 class Action(EmbeddedDocument):
@@ -36,7 +23,7 @@ class Note(EmbeddedDocument):
 
     admin: int = fields.IntegerField(required=True)
     content: str = fields.StrField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
 
 @JARVIS_INST.register
@@ -49,5 +36,5 @@ class Modlog(Document):
     admin: int = fields.IntegerField(required=True)
     actions: List[Action] = fields.ListField(fields.EmbeddedField(Action), factory=list)
     open: bool = fields.BoolField(default=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
     notes: List[Note] = fields.ListField(fields.EmbeddedField(Note), factory=list)

jarvis_core/db/models/reddit.py

@@ -4,11 +4,7 @@ from datetime import datetime, timezone
 from umongo import Document, fields
 
 from jarvis_core.db import JARVIS_INST
-
+from jarvis_core.db.utils import get_now
-
-def get_now() -> datetime:
-    """Get proper timestamp."""
-    return datetime.now(tz=timezone.utc)
 
 
 @JARVIS_INST.register
@@ -28,7 +24,7 @@ class SubredditFollow(Document):
     channel: int = fields.IntegerField(required=True)
     guild: int = fields.IntegerField(required=True)
     admin: int = fields.IntegerField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
     class Meta:
         collection_name = "subredditfollow"
@@ -50,7 +46,7 @@ class RedditorFollow(Document):
     channel: int = fields.IntegerField(required=True)
     guild: int = fields.IntegerField(required=True)
     admin: int = fields.IntegerField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
     class Meta:
         collection_name = "redditorfollow"

jarvis_core/db/models/twitter.py

@@ -4,11 +4,7 @@ from datetime import datetime, timezone
 from umongo import Document, fields
 
 from jarvis_core.db import JARVIS_INST
-
+from jarvis_core.db.utils import get_now
-
-def get_now() -> datetime:
-    """Get proper timestamp."""
-    return datetime.now(tz=timezone.utc)
 
 
 @JARVIS_INST.register
@@ -18,7 +14,7 @@ class TwitterAccount(Document):
     handle: str = fields.StringField(required=True)
     twitter_id: int = fields.IntegerField(required=True)
     last_tweet: int = fields.IntegerField(required=True)
-    last_sync: datetime = fields.DateTimeField(default=get_now)
+    last_sync: datetime = fields.AwareDateTimeField(default=get_now)
 
     class Meta:
         collection_name = "twitteraccount"
@@ -34,7 +30,7 @@ class TwitterFollow(Document):
     guild: int = fields.IntegerField(required=True)
     retweets: bool = fields.BooleanField(default=True)
     admin: int = fields.IntegerField(required=True)
-    created_at: datetime = fields.DateTimeField(default=get_now)
+    created_at: datetime = fields.AwareDateTimeField(default=get_now)
 
     class Meta:
         collection_name = "twitterfollow"

jarvis_core/db/utils.py

@@ -0,0 +1,15 @@
+from datetime import datetime, timezone
+
+import nanoid
+
+NANOID_ALPHA = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+
+
+def get_now() -> datetime:
+    """Get proper timestamp."""
+    return datetime.now(tz=timezone.utc)
+
+
+def get_id() -> str:
+    """Get nanoid."""
+    return nanoid.generate(NANOID_ALPHA, 12)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "jarvis-core"
-version = "0.12.0"
+version = "0.13.0"
 description = "JARVIS core"
 authors = ["Zevaryx <zevaryx@gmail.com>"]