big update

.gitignore

@@ -108,7 +108,7 @@ venv.bak/
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
-# User-specific stuff
+# Subscriber-specific stuff
 .idea/**/workspace.xml
 .idea/**/tasks.xml
 .idea/**/usage.statistics.xml
@@ -168,3 +168,7 @@ fabric.properties
 
 
 config.py
+users.fs*
+unknown_errors.txt
+*.session
+config.ini

.idea/lono.iml

@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="Remote Python 3.7.2 (sftp://root@bakatrouble.pw:22/srv/apps/lono/venv/bin/python)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.7 (lono)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="TestRunnerService">

.idea/misc.xml

@@ -78,5 +78,5 @@
       <textMaps />
     </LinkMapSettings>
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Remote Python 3.7.2 (sftp://root@bakatrouble.pw:22/srv/apps/lono/venv/bin/python)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (lono)" project-jdk-type="Python SDK" />
 </project>

config.example.py

@@ -1 +1,4 @@
-BOT_TOKEN = '450146961:AAGt5QRp3jS5wcHVIZOLxmqyO55iOVf6gpY'
+BOT_TOKEN = ''
+ADMIN = 98934915
+MANAGEMENT_CHAT = -1001411678110
+SENTRY_DSN = None

db.py

@@ -0,0 +1,33 @@
+from ZODB import DB
+from ZODB.Connection import Connection
+from ZODB.FileStorage import FileStorage
+from persistent import Persistent
+from persistent.mapping import PersistentMapping
+from transaction import commit
+from telegram import Message, Chat
+
+
+def get_conn(read_only=False) -> Connection:
+    storage = FileStorage('users.fs', read_only=read_only)
+    db = DB(storage)
+    conn = db.open()
+    if not hasattr(conn.root, 'subscribers'):
+        conn.root.subscribers = PersistentMapping()
+    return conn
+
+
+class Subscriber(Persistent):
+    def __init__(self, user_id, name):
+        self.uid = user_id
+        self.name = name
+
+    def update_from_message(self, m: Message):
+        self.name = Subscriber.get_name(m.chat)
+
+    @classmethod
+    def get_name(cls, chat: Chat):
+        return f'{chat.first_name or ""} {chat.last_name or ""} {chat.title or ""}'.strip()
+
+    @classmethod
+    def from_chat(cls, chat: Chat):
+        return cls(chat.id, cls.get_name(chat))

get_users_list.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-
-from html import escape
-
-from models import Subscriber
-from telegram import Bot
-
-from config import BOT_TOKEN
-
-
-bot = Bot(BOT_TOKEN)
-
-subs = Subscriber.select()
-
-messages = []
-for sub in subs:
-    chat = bot.send_message(sub.user_id, '.').chat
-    chat_name = escape(f'{chat.first_name or ""} {chat.last_name or ""} {chat.title or ""}'.strip())
-    messages.append(f'<code>#{sub.id:<4} {sub.user_id:>14} </code><a href="tg://user?id={sub.user_id}">{chat_name}</a>')
-
-
-for i in range(0, len(messages), 10):
-    sl = messages[i:i+10]
-    if not sl:
-        continue
-    bot.send_message(98934915, '\n'.join(sl), parse_mode='html')

import_users.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+from telegram import Bot
+
+from db import get_conn, Subscriber, commit
+from config import BOT_TOKEN
+
+
+if __name__ == '__main__':
+    bot = Bot(BOT_TOKEN)
+    conn = get_conn()
+    uids = input('Please input user ids split by spaces')
+    for uid in uids.split():
+        conn.root.subscribers[uid] = Subscriber.from_chat(bot.get_chat(uid))
+    commit()
+    print('Users have been successfully imported')

main.py

@@ -1,103 +1,233 @@
+#!/usr/bin/env python3
+
+import logging
 from html import escape
-from random import random
+from queue import Queue, Empty
 from time import sleep
+from threading import Thread
 
+import sentry_sdk
 from telegram.error import Unauthorized, TelegramError
-from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
-from telegram import Message, Update
+from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, CallbackQueryHandler
+from telegram import Message, Update, Bot, InlineKeyboardMarkup, InlineKeyboardButton, User
 
-import config
-from models import Subscriber
+from config import BOT_TOKEN, SENTRY_DSN, MANAGEMENT_CHAT
+from db import get_conn, Subscriber, PersistentMapping, commit
+from send_users_list import send_users_list
 
-queue = []
 
+logging.basicConfig(level=logging.WARNING)
+queue = Queue()
+sentry_sdk.init(dsn=SENTRY_DSN)
+conn = get_conn()
 
-def go_away(bot, update: Update):
-    # Subscriber(user_id=str(update.message.chat_id))
-    # update.message.reply_text('Вы были добавлены')
-    update.message.reply_text('Пожалуйста, обратитесь к @lono_contactbot')
+MAX_MESSAGE_LENGTH = 4096
+MAX_CAPTION_LENGTH = 1024
 
 
-def unsubscribe(bot, update: Update):
-    Subscriber.deleteBy(user_id=str(update.message.chat_id))
-    update.message.reply_text('Вы были отписаны от бота. Обратитесь к @lono_contactbot за добавлением обратно.')
+def _notify_access_request(bot: Bot, user: User):
+    markup = InlineKeyboardMarkup([[InlineKeyboardButton('Добавить', callback_data=f'add {user.id}')]])
+    bot.send_message(MANAGEMENT_CHAT, f'<a href="tg://user?id={user.id}">{escape(user.full_name)}</a> запросил доступ',
+                     parse_mode='html', reply_markup=markup)
 
 
-def msg(bot, update: Update):
-    queue.append(update.message)
+def welcome(bot: Bot, update: Update):
+    if update.effective_user.id in conn.root.subscribers:
+        update.message.reply_text('Вы уже являетесь участником ЛОНО')
+    else:
+        update.message.reply_text('Пожалуйста, обратитесь к @lono_contactbot')
+        _notify_access_request(bot, update.message.from_user)
 
 
-def task_queue(bot, job):
-    if not queue:
-        return
-    m = queue.pop(0)
-    current_chat = str(m.chat_id)
-    uids = set(s.user_id for s in Subscriber.select())
-    if current_chat not in uids:
-        return m.reply_text('Пожалуйста, обратитесь к @lono_contactbot')
-        # Subscriber(user_id=current_chat)
-        # m.reply_text('Вы были добавлены')
+def unsubscribe(bot: Bot, update: Update):
+    del conn.root.subscribers[update.message.chat_id]
+    commit()
+    update.message.reply_text('Вы были отписаны от бота. '
+                              'Обратитесь к @lono_contactbot если вы хотите подписаться снова.')
+    user = update.message.from_user
+    bot.send_message(MANAGEMENT_CHAT, f'<a href="tg://user?id={user.id}">{escape(user.full_name)}</a> отписался')
+
+
+def add_user(bot: Bot, update: Update, groups=(), args=()):
+    if update.callback_query:
+        update.callback_query.answer()
+
+    if groups:
+        if update.callback_query.message.chat.id != MANAGEMENT_CHAT:
+            return
+        uid = groups[0]
+    elif args:
+        uid = args[0]
+    elif update.message and update.message.reply_to_message and update.message.reply_to_message.forward_from:
+        uid = update.message.reply_to_message.forward_from.id
+    else:
+        return bot.send_message(MANAGEMENT_CHAT, 'Укажите ID пользователя или ответьте на его сообщение')
 
     try:
-        uids.remove(current_chat)
-    except KeyError:
+        uid = int(uid)
+    except (ValueError, TypeError):
         pass
 
-    for uid in uids:
+    try:
+        user = conn.root.subscribers[uid] = Subscriber.from_chat(bot.get_chat(uid))
+        commit()
+        if update.callback_query:
+            update.callback_query.message.edit_reply_markup()
+        bot.send_message(MANAGEMENT_CHAT, f'<a href="tg://user?id={uid}">{escape(user.name)}</a> был добавлен',
+                         parse_mode='html')
+        bot.send_message(uid, 'Добро пожаловать. Снова.')
+    except TelegramError as e:
+        bot.send_message(MANAGEMENT_CHAT, str(e))
+
+
+def remove_user(bot: Bot, update: Update, groups=(), args=()):
+    if update.callback_query:
+        update.callback_query.answer()
+
+    if groups:
+        if update.callback_query.message.chat.id != MANAGEMENT_CHAT:
+            return
+        uid = groups[0]
+    elif args:
+        uid = args[0]
+    elif update.message and update.message.reply_to_message and update.message.reply_to_message.forward_from:
+        uid = update.message.reply_to_message.forward_from.id
+    else:
+        return bot.send_message(MANAGEMENT_CHAT, 'Укажите ID пользователя или ответьте на его сообщение')
+
+    try:
+        uid = int(uid)
+    except (ValueError, TypeError):
+        pass
+
+    try:
+        name = conn.root.subscribers[uid].name
+        del conn.root.subscribers[uid]
+        commit()
+        bot.send_message(MANAGEMENT_CHAT, f'<a href="tg://user?id={uid}">{escape(name)}</a> был удален',
+                         parse_mode='html')
+        if update.callback_query:
+            update.callback_query.message.edit_reply_markup()
+    except KeyError:
+        bot.send_message(MANAGEMENT_CHAT, f'Пользователь id={uid} не был найден')
+
+
+def users(bot: Bot, update: Update):
+    send_users_list()
+
+
+def msg(bot: Bot, update: Update):
+    queue.put(update.message)
+
+
+def _sign_text(text, m: Message, limit):
+    if not text:
+        text = ''
+
+    if text.startswith('!sign') or text.startswith('/sign'):
+        text = text[5:] + f'\n\n____________\n' \
+               f'by <a href="tg://user?id={m.from_user.id}">{escape(m.from_user.full_name)}</a>'
+    return text[:limit]
+
+
+def _process_message(bot: Bot, m: Message):
+    current_chat = m.chat_id
+    users = conn.root.subscribers  # type: PersistentMapping
+    if current_chat not in users:
+        _notify_access_request(bot, m.from_user)
+        return m.reply_text('Пожалуйста, обратитесь к @lono_contactbot')
+
+    text = _sign_text(m.text_html, m, MAX_MESSAGE_LENGTH)
+    caption = _sign_text(m.caption_html, m, MAX_CAPTION_LENGTH)
+
+    for uid, user in users.items():
+        if uid == current_chat:
+            continue
         sleep(.02)
         try:
-            if m.forward_from or m.forward_from_chat or m.forward_from_message_id or m.forward_signature:
-                m.forward(f'{uid}')
+            r = None
+            if m.forward_date:
+                r = m.forward(uid)
             elif hasattr(m, 'audio') and m.audio:
                 a = m.audio
-                bot.send_audio(f'{uid}', a.file_id, a.duration, a.performer, a.title, m.caption_html, parse_mode='html')
+                r = bot.send_audio(uid, a.file_id, a.duration, a.performer, a.title, caption, parse_mode='html')
             elif hasattr(m, 'document') and m.document:
                 d = m.document
-                bot.send_document(f'{uid}', d.file_id, d.file_name, m.caption_html, parse_mode='html')
+                r = bot.send_document(uid, d.file_id, d.file_name, caption, parse_mode='html')
             elif hasattr(m, 'photo') and m.photo:
                 p = m.photo
-                bot.send_photo(f'{uid}', p[-1].file_id, m.caption_html, parse_mode='html')
+                r = bot.send_photo(uid, p[-1].file_id, caption, parse_mode='html')
             elif hasattr(m, 'sticker') and m.sticker:
                 s = m.sticker
-                bot.send_sticker(f'{uid}', s.file_id)
+                r = bot.send_sticker(uid, s.file_id)
             elif hasattr(m, 'video') and m.video:
                 v = m.video
-                bot.send_video(f'{uid}', v.file_id, v.duration, m.caption_html, parse_mode='html')
+                r = bot.send_video(uid, v.file_id, v.duration, caption, parse_mode='html')
             elif hasattr(m, 'voice') and m.voice:
                 v = m.voice
-                bot.send_voice(f'{uid}', v.file_id, v.duration, m.caption_html, parse_mode='html')
+                r = bot.send_voice(uid, v.file_id, v.duration, caption, parse_mode='html')
             elif hasattr(m, 'video_note') and m.video_note:
                 vn = m.video_note
-                bot.send_video_note(f'{uid}', vn.file_id, vn.duration, vn.length)
+                r = bot.send_video_note(uid, vn.file_id, vn.duration, vn.length)
             elif hasattr(m, 'contact') and m.contact:
                 c = m.contact
-                bot.send_contact(f'{uid}', c.phone_number, c.first_name, c.last_name)
+                r = bot.send_contact(uid, c.phone_number, c.first_name, c.last_name)
             elif hasattr(m, 'location') and m.location:
                 l = m.location
-                bot.send_location(f'{uid}', l.latitude, l.longitude)
+                r = bot.send_location(uid, l.latitude, l.longitude)
             elif hasattr(m, 'venue') and m.venue:
                 v = m.venue
-                bot.send_venue(f'{uid}', v.location.latitude, v.location.longitude, v.title, v.address, v.foursquare_id)
+                l = v.location
+                r = bot.send_venue(uid, l.latitude, l.longitude, v.title, v.address, v.foursquare_id)
             elif hasattr(m, 'text') and m.text:
-                txt = m.text_html
-                if txt.startswith('!sign') or txt.startswith('/sign'):
-                    txt = txt[5:] + f'\n\n____________\n' \
-                                    f'by <a href="tg://user?id={m.from_user.id}">{escape(m.from_user.full_name)}</a>'
-                bot.send_message(f'{uid}', txt, 'html')
+                r = bot.send_message(uid, text, 'html')
+            if r:
+                user.update_from_message(r)
         except Unauthorized:
-            Subscriber.deleteBy(user_id=uid)
+            name = conn.root.users[uid].name
+            del conn.root.users[uid]
+            commit()
+            bot.send_message(MANAGEMENT_CHAT, f'<a href="tg://user?id={uid}">{name}</a> был удален '
+                                              f'из-за блокировки бота')
         except TelegramError:
-            pass
+            sentry_sdk.capture_exception()
+
+    commit()
 
 
-updater = Updater(config.BOT_TOKEN, workers=4)
-updater.job_queue.run_repeating(task_queue, .04)
+def task_queue(u: Updater):
+    while True:
+        if not u.running:
+            return
 
-updater.dispatcher.add_handler(CommandHandler('start', go_away))
-updater.dispatcher.add_handler(CommandHandler('stop', unsubscribe))
-updater.dispatcher.add_handler(MessageHandler(Filters.all, msg))
+        try:
+            m = queue.get(timeout=1)  # type: Message
+            _process_message(u.bot, m)
+        except Empty:
+            pass
+        except:
+            sentry_sdk.capture_exception()
 
 
 if __name__ == '__main__':
+    updater = Updater(BOT_TOKEN, workers=4)
+
+    updater.dispatcher.add_handler(CommandHandler('start', welcome, Filters.private))
+    updater.dispatcher.add_handler(CommandHandler('stop', unsubscribe, Filters.private))
+    updater.dispatcher.add_handler(CommandHandler('add', add_user, Filters.chat(MANAGEMENT_CHAT)))
+    updater.dispatcher.add_handler(CallbackQueryHandler(add_user, pattern=r'^add (\d+)$', pass_groups=True))
+    updater.dispatcher.add_handler(CommandHandler('remove', remove_user, Filters.chat(MANAGEMENT_CHAT)))
+    updater.dispatcher.add_handler(CallbackQueryHandler(remove_user, pattern=r'^remove (\d+)$', pass_groups=True))
+    updater.dispatcher.add_handler(CommandHandler('users', users, Filters.chat(MANAGEMENT_CHAT)))
+    updater.dispatcher.add_handler(MessageHandler(Filters.private, msg))
+
     updater.start_polling()
+
+    tq = Thread(target=task_queue, args=(updater,))
+    tq.start()
+
+    logging.warning('LONO has started')
     updater.idle()
+    logging.warning('LONO is stopping...')
+    commit()
+    conn.close()

models.py

@@ -1,10 +0,0 @@
-from sqlobject import *
-
-sqlhub.processConnection = connectionForURI('sqlite:db.sqlite3')
-
-
-class Subscriber(SQLObject):
-    user_id = StringCol(length=32)
-
-
-Subscriber.createTable(ifNotExists=True)

requirements.txt

@@ -1,12 +1,23 @@
 asn1crypto==0.24.0
-certifi==2018.10.15
-cffi==1.11.5
-cryptography==2.3.1
-FormEncode==1.3.1
-future==0.17.0
-idna==2.7
+BTrees==4.5.1
+certifi==2018.11.29
+cffi==1.12.2
+cryptography==2.6.1
+future==0.17.1
+idna==2.8
+persistent==4.4.3
+pyaes==1.6.1
 pycparser==2.19
-PyDispatcher==2.0.5
+Pyrogram==0.11.0
+PySocks==1.6.8
 python-telegram-bot==11.1.0
-six==1.11.0
-SQLObject==3.7.0
+sentry-sdk==0.7.4
+six==1.12.0
+TgCrypto==1.1.1
+transaction==2.4.0
+urllib3==1.24.1
+zc.lockfile==1.4
+ZConfig==3.4.0
+ZODB==5.5.1
+zodbpickle==1.0.3
+zope.interface==4.6.0

send_users_list.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+from telegram import Bot
+
+from db import get_conn, Subscriber
+from config import BOT_TOKEN, ADMIN
+
+
+def send_users_list(bot: Bot = None):
+    conn = get_conn(read_only=True)
+
+    if not bot:
+        bot = Bot(BOT_TOKEN)
+
+    subs = conn.root.subscribers.values()
+
+    messages = [f'Count: {subs.count()}\n']
+    for sub in subs:  # type: Subscriber
+        msg = f'<code>#{sub.id:<4} {sub.uid:>14} </code>'
+        if sub.uid < 0:
+            msg += str(sub.name)
+        else:
+            msg += f'<a href="tg://user?id={sub.uid}">{sub.name}</a>'
+        messages.append(msg)
+
+    for i in range(0, len(messages), 40):
+        bot.send_message(ADMIN, '\n'.join(messages[i:i+40]), parse_mode='html')
+
+
+if __name__ == '__main__':
+    send_users_list()