From b7e68abb038a3dff10da33265bc22151761b1601 Mon Sep 17 00:00:00 2001 From: bakatrouble Date: Mon, 28 Jan 2019 00:20:09 +0300 Subject: [PATCH] add aggregator app --- .idea/telegram_bots.iml | 1 + aggregator/__init__.py | 0 aggregator/admin.py | 8 ++ aggregator/apps.py | 14 +++ aggregator/client.py | 86 +++++++++++++++ aggregator/forms.py | 55 ++++++++++ aggregator/management/__init__.py | 0 aggregator/management/commands/__init__.py | 0 .../commands/start_aggregator_client.py | 8 ++ aggregator/migrations/0001_initial.py | 54 ++++++++++ .../0002_remove_aggregationsource_last_id.py | 17 +++ aggregator/models.py | 96 +++++++++++++++++ .../cabinet/aggregator/source_form.html | 59 ++++++++++ .../cabinet/aggregator/source_list.html | 40 +++++++ aggregator/tests.py | 3 + aggregator/urls.py | 13 +++ aggregator/views.py | 60 +++++++++++ .../templates/cabinet/_includes/sidebar.html | 12 ++- cabinet/templates/cabinet/admin_config.html | 2 +- cabinet/urls.py | 1 + cabinet/utils.py | 5 +- cabinet/views.py | 3 + config/settings.py | 1 + config/utils.py | 12 +++ feeds/views.py | 3 +- requirements.txt | 5 + static/js/custom.init.js | 101 ++++++++++++++++++ static/js/theme.js | 99 ----------------- supervisor.conf | 12 ++- 29 files changed, 665 insertions(+), 105 deletions(-) create mode 100644 aggregator/__init__.py create mode 100644 aggregator/admin.py create mode 100644 aggregator/apps.py create mode 100644 aggregator/client.py create mode 100644 aggregator/forms.py create mode 100644 aggregator/management/__init__.py create mode 100644 aggregator/management/commands/__init__.py create mode 100644 aggregator/management/commands/start_aggregator_client.py create mode 100644 aggregator/migrations/0001_initial.py create mode 100644 aggregator/migrations/0002_remove_aggregationsource_last_id.py create mode 100644 aggregator/models.py create mode 100644 aggregator/templates/cabinet/aggregator/source_form.html create mode 100644 aggregator/templates/cabinet/aggregator/source_list.html create mode 100644 aggregator/tests.py create mode 100644 aggregator/urls.py create mode 100644 aggregator/views.py diff --git a/.idea/telegram_bots.iml b/.idea/telegram_bots.iml index 79203c6..688a4c0 100644 --- a/.idea/telegram_bots.iml +++ b/.idea/telegram_bots.iml @@ -24,6 +24,7 @@ diff --git a/aggregator/__init__.py b/aggregator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aggregator/admin.py b/aggregator/admin.py new file mode 100644 index 0000000..adc4f83 --- /dev/null +++ b/aggregator/admin.py @@ -0,0 +1,8 @@ +from django.contrib import admin + +from .models import AggregationSource, Chat, Message + + +admin.site.register(AggregationSource) +admin.site.register(Chat) +admin.site.register(Message) diff --git a/aggregator/apps.py b/aggregator/apps.py new file mode 100644 index 0000000..d912167 --- /dev/null +++ b/aggregator/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig + + +class AggregatorConfig(AppConfig): + name = 'aggregator' + + def ready(self): + self.register_config() + + def register_config(self): + import djconfig + from .forms import AggregatorAppConfigForm + + djconfig.register(AggregatorAppConfigForm) diff --git a/aggregator/client.py b/aggregator/client.py new file mode 100644 index 0000000..e721ab3 --- /dev/null +++ b/aggregator/client.py @@ -0,0 +1,86 @@ +import os + +from django.core.files.storage import default_storage +from djconfig import config +from pyrogram import Client, MessageHandler, DeletedMessagesHandler, \ + Message as PyrogramMessage, Messages as PyrogramMessages +from pyrogram.api.errors import ChannelPrivate +from pyrogram.session import Session + +from aggregator.models import AggregationSource, Message, Chat + + +Session.notice_displayed = True + + +def get_client(): + config._reload_maybe() + session_path = os.path.relpath(default_storage.path(config.pyrogram_session.replace('.session', ''))) + return Client(session_path, config.pyrogram_app_id, config.pyrogram_app_hash) + + +def save_message(client, message: PyrogramMessage): + if not AggregationSource.objects.filter(chat_id=message.chat.id).exists(): + return + Message.from_obj(message, client) + + +def delete_messages(client, messages: PyrogramMessages): + for message in messages.messages: + Message.objects.filter(chat__chat_id=message.chat.id, message_id=message.message_id).update(deleted=True) + + +def collect_new_messages(client, chat): + # Collecting new messages + last_message = chat.messages.order_by('-message_id').first() + if last_message: + itr = client.iter_history(chat.chat_id, reverse=True, offset_id=last_message.message_id + 1) + else: + itr = client.iter_history(chat.chat_id, reverse=True, limit=10) + for message in itr: + Message.from_obj(message, client) + + +def startup_collect(client: Client): + for chat in Chat.objects.all(): + try: + client.get_chat(chat.chat_id) + except ChannelPrivate: + print('I was banned in chat id="{}"'.format(chat.chat_id)) + continue + # Collecting edited & deleted messages + offset = 0 + qs = Message.objects.active_messages().filter(chat__chat_id=chat.chat_id) + while True: + lst = qs[200*offset:200*(offset+1)] + if not lst: + break + messages = client.get_messages(chat.chat_id, [m.message_id for m in lst]) + for message in messages.messages: + m_qs = Message.objects.active_messages() \ + .filter(chat__chat_id=chat.chat_id, message_id=message.message_id) + if message.empty: + m_qs.update(deleted=True) + elif message.edit_date: + m = m_qs.first() # type: Message + if m and (not m.edit_date or m.edit_date.timestamp() != message.edit_date): + Message.from_obj(message, client) + offset += 1 + + collect_new_messages(client, chat) + + +def start_client(): + app = get_client() + + app.add_handler(MessageHandler(save_message)) + app.add_handler(DeletedMessagesHandler(delete_messages)) + + print('Starting pyrogram client...') + app.start() + + print('Loading updates that happened while I was offline...') + startup_collect(app) + + print('Idling...') + app.idle() diff --git a/aggregator/forms.py b/aggregator/forms.py new file mode 100644 index 0000000..c27523e --- /dev/null +++ b/aggregator/forms.py @@ -0,0 +1,55 @@ +from django import forms +from django.core.exceptions import ValidationError +from django.core.files.storage import default_storage +from django.forms import ModelForm +from djconfig.forms import ConfigForm +from pyrogram.api.errors import ChannelPrivate + +from config.utils import parse_mtproto_chat +from .client import get_client +from .models import AggregationSource, Chat + + +class AggregationSourceForm(ModelForm): + invite_link = forms.CharField(help_text='Invite link (with joinchat) or username') + + def clean(self): + cleaned_data = super(AggregationSourceForm, self).clean() + invite_link = cleaned_data.pop('invite_link') + if invite_link: + with get_client() as app: + try: + upd = app.join_chat(invite_link) + chat = parse_mtproto_chat(app, upd.chats[0]) + cleaned_data['chat_id'] = chat.id + Chat.from_obj(chat, app) + except ChannelPrivate: + raise ValidationError('I was banned in this chat') + return cleaned_data + + class Meta: + model = AggregationSource + fields = 'title', 'invite_link', + + +class AggregationSourceEditForm(AggregationSourceForm): + invite_link = forms.CharField(help_text='Invite link (with joinchat) or username; ' + 'leave empty if change is not needed', required=False) + + +class AggregatorAppConfigForm(ConfigForm): + slug = 'aggregator' + title = 'Aggregator' + + pyrogram_app_id = forms.CharField() + pyrogram_app_hash = forms.CharField() + pyrogram_session = forms.FileField() + + def save_session(self): + session = self.cleaned_data.get('pyrogram_session') + if session: + session.name = default_storage.save(session.name, session) + + def save(self): + self.save_session() + super(AggregatorAppConfigForm, self).save() diff --git a/aggregator/management/__init__.py b/aggregator/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aggregator/management/commands/__init__.py b/aggregator/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aggregator/management/commands/start_aggregator_client.py b/aggregator/management/commands/start_aggregator_client.py new file mode 100644 index 0000000..51f4628 --- /dev/null +++ b/aggregator/management/commands/start_aggregator_client.py @@ -0,0 +1,8 @@ +from django.core.management import BaseCommand + +from aggregator.client import start_client + + +class Command(BaseCommand): + def handle(self, *args, **options): + start_client() diff --git a/aggregator/migrations/0001_initial.py b/aggregator/migrations/0001_initial.py new file mode 100644 index 0000000..0bc1a3c --- /dev/null +++ b/aggregator/migrations/0001_initial.py @@ -0,0 +1,54 @@ +# Generated by Django 2.1.5 on 2019-01-27 12:20 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='AggregationSource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=64)), + ('chat_id', models.IntegerField(db_index=True)), + ('last_id', models.PositiveIntegerField()), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Chat', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('chat_id', models.IntegerField(db_index=True)), + ('title', models.TextField()), + ('username', models.CharField(blank=True, max_length=64, null=True)), + ('photo', models.ImageField(blank=True, null=True, upload_to='')), + ('photo_id', models.CharField(blank=True, max_length=64, null=True)), + ], + ), + migrations.CreateModel( + name='Message', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('message_id', models.PositiveIntegerField(db_index=True)), + ('text', models.TextField(blank=True)), + ('date', models.DateTimeField()), + ('edit_date', models.DateTimeField(blank=True, null=True)), + ('deleted', models.BooleanField(default=False)), + ('chat', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='aggregator.Chat')), + ('replaced_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='aggregator.Message')), + ], + options={ + 'ordering': ('message_id', 'edit_date'), + }, + ), + ] diff --git a/aggregator/migrations/0002_remove_aggregationsource_last_id.py b/aggregator/migrations/0002_remove_aggregationsource_last_id.py new file mode 100644 index 0000000..7438674 --- /dev/null +++ b/aggregator/migrations/0002_remove_aggregationsource_last_id.py @@ -0,0 +1,17 @@ +# Generated by Django 2.1.5 on 2019-01-27 13:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('aggregator', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='aggregationsource', + name='last_id', + ), + ] diff --git a/aggregator/models.py b/aggregator/models.py new file mode 100644 index 0000000..1060d60 --- /dev/null +++ b/aggregator/models.py @@ -0,0 +1,96 @@ +import os +from datetime import datetime +from tempfile import TemporaryDirectory + +import pytz +from django.conf import settings +from django.db import models +from pyrogram import Chat as PyrogramChat, Message as PyrogramMessage, ChatPhoto as PyrogramChatPhoto +from pyrogram.api.types.chat_photo import ChatPhoto as MTProtoChatPhoto + + +class AggregationSource(models.Model): + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + title = models.CharField(max_length=64) + chat_id = models.IntegerField(db_index=True) + + def __str__(self): + return self.title + + +class Chat(models.Model): + chat_id = models.IntegerField(db_index=True) + title = models.TextField() + username = models.CharField(max_length=64, null=True, blank=True) + photo = models.ImageField(null=True, blank=True) + photo_id = models.CharField(max_length=64, null=True, blank=True) + + @classmethod + def from_obj(cls, chat: PyrogramChat, client): + obj, _ = Chat.objects.update_or_create( + chat_id=chat.id, + defaults={ + 'title': chat.title or '{} {}'.format(chat.first_name, chat.last_name).rstrip(), + 'username': chat.username, + } + ) + if chat.photo is None: + if obj.photo_id is not None: + obj.photo = obj.photo_id = None + obj.save() + else: + photo_file_id = chat.photo.small_file_id + if photo_file_id != obj.photo_id: + with TemporaryDirectory() as d: + path = client.download_media(photo_file_id, os.path.join(d, ''), block=True) + with open(path, 'rb') as f: + obj.photo.save(os.path.basename(path), f, save=True) + obj.photo_id = chat.photo.small_file_id + obj.save() + return obj + + def __str__(self): + return '{} (chat_id="{}")'.format(self.title, self.chat_id) + + +class MessageManager(models.Manager): + def active_messages(self): + return self.get_queryset().filter(deleted=False, replaced_by__isnull=True) + + +class Message(models.Model): + chat = models.ForeignKey(Chat, on_delete=models.CASCADE, related_name='messages') + message_id = models.PositiveIntegerField(db_index=True) + text = models.TextField(blank=True) + date = models.DateTimeField() + edit_date = models.DateTimeField(null=True, blank=True) + deleted = models.BooleanField(default=False) + replaced_by = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True) + + objects = MessageManager() + + @classmethod + def from_obj(cls, message: PyrogramMessage, client): + tz = pytz.timezone('UTC') + chat = Chat.from_obj(message.chat, client) + try: + old = Message.objects.active_messages().get(chat=chat, message_id=message.message_id) + except Message.DoesNotExist: + old = None + obj = Message.objects.create( + chat=chat, + message_id=message.message_id, + text=message.text.html if message.text else '', + date=tz.localize(datetime.utcfromtimestamp(message.date)), + edit_date=tz.localize(datetime.utcfromtimestamp(message.edit_date)) if message.edit_date else None, + ) + if old is not None: + old.replaced_by = obj + old.save() + return obj + + def __str__(self): + return 'id="{}" (chat_id="{}")'.format(self.message_id, self.chat.chat_id) + + class Meta: + ordering = 'message_id', 'edit_date', diff --git a/aggregator/templates/cabinet/aggregator/source_form.html b/aggregator/templates/cabinet/aggregator/source_form.html new file mode 100644 index 0000000..3616acf --- /dev/null +++ b/aggregator/templates/cabinet/aggregator/source_form.html @@ -0,0 +1,59 @@ +{% extends 'cabinet/_internal_base.html' %} +{% load bootstrap4 %} + +{% block breadcrumbs %} +
  • Aggregation source list
    • +
  • {{ title }}
    • +{% endblock %} + +{% block content %} +
    + {% csrf_token %} +
    +

    {% if source %}Source "{{ source.title }}" configuration{% else %}New source{% endif %}

    +
    +
    + {% bootstrap_form form layout='horizontal' %} + {% if source %} +
    + +
    + +
    +
    + {% endif %} +
    +
    + {% if source %}Delete{% endif %} + +
    +
    + + {% if source %} + + {% endif %} +{% endblock %} diff --git a/aggregator/templates/cabinet/aggregator/source_list.html b/aggregator/templates/cabinet/aggregator/source_list.html new file mode 100644 index 0000000..80ec869 --- /dev/null +++ b/aggregator/templates/cabinet/aggregator/source_list.html @@ -0,0 +1,40 @@ +{% extends 'cabinet/_internal_base.html' %} + +{% block breadcrumbs %} +
  • Aggregator source list
    • +{% endblock %} + +{% block content %} +
    +
    +
    + + Add new source + +
    +

    Your sources

    +
    +
    + + + + + + + + {% for source in object_list %} + + + + + {% empty %} + + + + + + {% endfor %} +
    TitleChat ID
    {{ source.title }}{{ source.chat_id }}
    No sources added
    +
    +
    +{% endblock %} diff --git a/aggregator/tests.py b/aggregator/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/aggregator/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/aggregator/urls.py b/aggregator/urls.py new file mode 100644 index 0000000..7bf3069 --- /dev/null +++ b/aggregator/urls.py @@ -0,0 +1,13 @@ +from django.urls import path + +from .views import AggregationSourceListView, AggregationSourceCreateView, AggregationSourceUpdateView, \ + AggregationSourceDeleteView + +app_name = 'feeds' +urlpatterns = [ + path('', AggregationSourceListView.as_view(), name='index'), + path('/', AggregationSourceUpdateView.as_view(), name='edit'), + path('new/', AggregationSourceCreateView.as_view(), name='new'), + path('delete//', AggregationSourceDeleteView.as_view(), name='delete'), +] + diff --git a/aggregator/views.py b/aggregator/views.py new file mode 100644 index 0000000..6f89b71 --- /dev/null +++ b/aggregator/views.py @@ -0,0 +1,60 @@ +from django.contrib import messages +from django.contrib.auth.mixins import LoginRequiredMixin +from django.shortcuts import redirect +from django.views import View +from django.views.generic import ListView, UpdateView, CreateView +from django.views.generic.detail import SingleObjectMixin + +from aggregator.forms import AggregationSourceForm, AggregationSourceEditForm +from aggregator.models import AggregationSource +from cabinet.utils import CabinetViewMixin + + +class AggregationSourceListView(CabinetViewMixin, ListView): + template_name = 'cabinet/aggregator/source_list.html' + title = 'Aggregation source list' + sidebar_section = 'aggregator' + + def get_queryset(self): + return AggregationSource.objects.filter(owner=self.request.user) + + +class AggregationSourceCreateView(CabinetViewMixin, CreateView): + template_name = 'cabinet/aggregator/source_form.html' + title = 'Create aggregation source' + sidebar_section = 'aggregator' + form_class = AggregationSourceForm + + def form_valid(self, form): + form.instance.owner = self.request.user + form.instance.chat_id = form.cleaned_data['chat_id'] + form.save() + return redirect('cabinet:aggregator:edit', pk=form.instance.pk) + + +class AggregationSourceUpdateView(CabinetViewMixin, UpdateView): + template_name = 'cabinet/aggregator/source_form.html' + title = 'Configure aggregation source' + sidebar_section = 'aggregator' + form_class = AggregationSourceEditForm + context_object_name = 'source' + + def get_queryset(self): + return AggregationSource.objects.filter(owner=self.request.user) + + def form_valid(self, form): + if 'chat_id' in form.cleaned_data: + form.instance.chat_id = form.cleaned_data['chat_id'] + form.save() + return redirect('cabinet:aggregator:edit', pk=form.instance.pk) + + +class AggregationSourceDeleteView(LoginRequiredMixin, SingleObjectMixin, View): + def get_queryset(self): + return AggregationSource.objects.filter(owner=self.request.user) + + def post(self, request, *args, **kwargs): + source = self.get_object() + messages.success(self.request, 'Source "{}" was successfully deleted'.format(source.title)) + source.delete() + return redirect('cabinet:aggregator:index') diff --git a/cabinet/templates/cabinet/_includes/sidebar.html b/cabinet/templates/cabinet/_includes/sidebar.html index 8723f74..8218888 100644 --- a/cabinet/templates/cabinet/_includes/sidebar.html +++ b/cabinet/templates/cabinet/_includes/sidebar.html @@ -15,6 +15,12 @@ Feeds +
  • + + + Aggregator + +
    • @@ -24,14 +30,16 @@
    Admin