add aggregator app

aggregator/admin.py

@@ -0,0 +1,8 @@
+from django.contrib import admin
+
+from .models import AggregationSource, Chat, Message
+
+
+admin.site.register(AggregationSource)
+admin.site.register(Chat)
+admin.site.register(Message)

aggregator/apps.py

@@ -0,0 +1,14 @@
+from django.apps import AppConfig
+
+
+class AggregatorConfig(AppConfig):
+    name = 'aggregator'
+
+    def ready(self):
+        self.register_config()
+
+    def register_config(self):
+        import djconfig
+        from .forms import AggregatorAppConfigForm
+
+        djconfig.register(AggregatorAppConfigForm)

aggregator/client.py

@@ -0,0 +1,86 @@
+import os
+
+from django.core.files.storage import default_storage
+from djconfig import config
+from pyrogram import Client, MessageHandler, DeletedMessagesHandler, \
+                     Message as PyrogramMessage, Messages as PyrogramMessages
+from pyrogram.api.errors import ChannelPrivate
+from pyrogram.session import Session
+
+from aggregator.models import AggregationSource, Message, Chat
+
+
+Session.notice_displayed = True
+
+
+def get_client():
+    config._reload_maybe()
+    session_path = os.path.relpath(default_storage.path(config.pyrogram_session.replace('.session', '')))
+    return Client(session_path, config.pyrogram_app_id, config.pyrogram_app_hash)
+
+
+def save_message(client, message: PyrogramMessage):
+    if not AggregationSource.objects.filter(chat_id=message.chat.id).exists():
+        return
+    Message.from_obj(message, client)
+
+
+def delete_messages(client, messages: PyrogramMessages):
+    for message in messages.messages:
+        Message.objects.filter(chat__chat_id=message.chat.id, message_id=message.message_id).update(deleted=True)
+
+
+def collect_new_messages(client, chat):
+    # Collecting new messages
+    last_message = chat.messages.order_by('-message_id').first()
+    if last_message:
+        itr = client.iter_history(chat.chat_id, reverse=True, offset_id=last_message.message_id + 1)
+    else:
+        itr = client.iter_history(chat.chat_id, reverse=True, limit=10)
+    for message in itr:
+        Message.from_obj(message, client)
+
+
+def startup_collect(client: Client):
+    for chat in Chat.objects.all():
+        try:
+            client.get_chat(chat.chat_id)
+        except ChannelPrivate:
+            print('I was banned in chat id="{}"'.format(chat.chat_id))
+            continue
+        # Collecting edited & deleted messages
+        offset = 0
+        qs = Message.objects.active_messages().filter(chat__chat_id=chat.chat_id)
+        while True:
+            lst = qs[200*offset:200*(offset+1)]
+            if not lst:
+                break
+            messages = client.get_messages(chat.chat_id, [m.message_id for m in lst])
+            for message in messages.messages:
+                m_qs = Message.objects.active_messages() \
+                    .filter(chat__chat_id=chat.chat_id, message_id=message.message_id)
+                if message.empty:
+                     m_qs.update(deleted=True)
+                elif message.edit_date:
+                    m = m_qs.first()  # type: Message
+                    if m and (not m.edit_date or m.edit_date.timestamp() != message.edit_date):
+                        Message.from_obj(message, client)
+            offset += 1
+
+        collect_new_messages(client, chat)
+
+
+def start_client():
+    app = get_client()
+
+    app.add_handler(MessageHandler(save_message))
+    app.add_handler(DeletedMessagesHandler(delete_messages))
+
+    print('Starting pyrogram client...')
+    app.start()
+
+    print('Loading updates that happened while I was offline...')
+    startup_collect(app)
+
+    print('Idling...')
+    app.idle()

aggregator/forms.py

@@ -0,0 +1,55 @@
+from django import forms
+from django.core.exceptions import ValidationError
+from django.core.files.storage import default_storage
+from django.forms import ModelForm
+from djconfig.forms import ConfigForm
+from pyrogram.api.errors import ChannelPrivate
+
+from config.utils import parse_mtproto_chat
+from .client import get_client
+from .models import AggregationSource, Chat
+
+
+class AggregationSourceForm(ModelForm):
+    invite_link = forms.CharField(help_text='Invite link (with joinchat) or username')
+
+    def clean(self):
+        cleaned_data = super(AggregationSourceForm, self).clean()
+        invite_link = cleaned_data.pop('invite_link')
+        if invite_link:
+            with get_client() as app:
+                try:
+                    upd = app.join_chat(invite_link)
+                    chat = parse_mtproto_chat(app, upd.chats[0])
+                    cleaned_data['chat_id'] = chat.id
+                    Chat.from_obj(chat, app)
+                except ChannelPrivate:
+                    raise ValidationError('I was banned in this chat')
+        return cleaned_data
+
+    class Meta:
+        model = AggregationSource
+        fields = 'title', 'invite_link',
+
+
+class AggregationSourceEditForm(AggregationSourceForm):
+    invite_link = forms.CharField(help_text='Invite link (with joinchat) or username; '
+                                            'leave empty if change is not needed', required=False)
+
+
+class AggregatorAppConfigForm(ConfigForm):
+    slug = 'aggregator'
+    title = 'Aggregator'
+
+    pyrogram_app_id = forms.CharField()
+    pyrogram_app_hash = forms.CharField()
+    pyrogram_session = forms.FileField()
+
+    def save_session(self):
+        session = self.cleaned_data.get('pyrogram_session')
+        if session:
+            session.name = default_storage.save(session.name, session)
+
+    def save(self):
+        self.save_session()
+        super(AggregatorAppConfigForm, self).save()

aggregator/management/commands/start_aggregator_client.py

@@ -0,0 +1,8 @@
+from django.core.management import BaseCommand
+
+from aggregator.client import start_client
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        start_client()

aggregator/migrations/0001_initial.py

@@ -0,0 +1,54 @@
+# Generated by Django 2.1.5 on 2019-01-27 12:20
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AggregationSource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=64)),
+                ('chat_id', models.IntegerField(db_index=True)),
+                ('last_id', models.PositiveIntegerField()),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Chat',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('chat_id', models.IntegerField(db_index=True)),
+                ('title', models.TextField()),
+                ('username', models.CharField(blank=True, max_length=64, null=True)),
+                ('photo', models.ImageField(blank=True, null=True, upload_to='')),
+                ('photo_id', models.CharField(blank=True, max_length=64, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Message',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('message_id', models.PositiveIntegerField(db_index=True)),
+                ('text', models.TextField(blank=True)),
+                ('date', models.DateTimeField()),
+                ('edit_date', models.DateTimeField(blank=True, null=True)),
+                ('deleted', models.BooleanField(default=False)),
+                ('chat', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='aggregator.Chat')),
+                ('replaced_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='aggregator.Message')),
+            ],
+            options={
+                'ordering': ('message_id', 'edit_date'),
+            },
+        ),
+    ]

aggregator/migrations/0002_remove_aggregationsource_last_id.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.1.5 on 2019-01-27 13:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('aggregator', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='aggregationsource',
+            name='last_id',
+        ),
+    ]

aggregator/models.py

@@ -0,0 +1,96 @@
+import os
+from datetime import datetime
+from tempfile import TemporaryDirectory
+
+import pytz
+from django.conf import settings
+from django.db import models
+from pyrogram import Chat as PyrogramChat, Message as PyrogramMessage, ChatPhoto as PyrogramChatPhoto
+from pyrogram.api.types.chat_photo import ChatPhoto as MTProtoChatPhoto
+
+
+class AggregationSource(models.Model):
+    owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
+    title = models.CharField(max_length=64)
+    chat_id = models.IntegerField(db_index=True)
+
+    def __str__(self):
+        return self.title
+
+
+class Chat(models.Model):
+    chat_id = models.IntegerField(db_index=True)
+    title = models.TextField()
+    username = models.CharField(max_length=64, null=True, blank=True)
+    photo = models.ImageField(null=True, blank=True)
+    photo_id = models.CharField(max_length=64, null=True, blank=True)
+
+    @classmethod
+    def from_obj(cls, chat: PyrogramChat, client):
+        obj, _ = Chat.objects.update_or_create(
+            chat_id=chat.id,
+            defaults={
+                'title': chat.title or '{} {}'.format(chat.first_name, chat.last_name).rstrip(),
+                'username': chat.username,
+            }
+        )
+        if chat.photo is None:
+            if obj.photo_id is not None:
+                obj.photo = obj.photo_id = None
+                obj.save()
+        else:
+            photo_file_id = chat.photo.small_file_id
+            if photo_file_id != obj.photo_id:
+                with TemporaryDirectory() as d:
+                    path = client.download_media(photo_file_id, os.path.join(d, ''), block=True)
+                    with open(path, 'rb') as f:
+                        obj.photo.save(os.path.basename(path), f, save=True)
+                obj.photo_id = chat.photo.small_file_id
+                obj.save()
+        return obj
+
+    def __str__(self):
+        return '{} (chat_id="{}")'.format(self.title, self.chat_id)
+
+
+class MessageManager(models.Manager):
+    def active_messages(self):
+        return self.get_queryset().filter(deleted=False, replaced_by__isnull=True)
+
+
+class Message(models.Model):
+    chat = models.ForeignKey(Chat, on_delete=models.CASCADE, related_name='messages')
+    message_id = models.PositiveIntegerField(db_index=True)
+    text = models.TextField(blank=True)
+    date = models.DateTimeField()
+    edit_date = models.DateTimeField(null=True, blank=True)
+    deleted = models.BooleanField(default=False)
+    replaced_by = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True)
+
+    objects = MessageManager()
+
+    @classmethod
+    def from_obj(cls, message: PyrogramMessage, client):
+        tz = pytz.timezone('UTC')
+        chat = Chat.from_obj(message.chat, client)
+        try:
+            old = Message.objects.active_messages().get(chat=chat, message_id=message.message_id)
+        except Message.DoesNotExist:
+            old = None
+        obj = Message.objects.create(
+            chat=chat,
+            message_id=message.message_id,
+            text=message.text.html if message.text else '',
+            date=tz.localize(datetime.utcfromtimestamp(message.date)),
+            edit_date=tz.localize(datetime.utcfromtimestamp(message.edit_date)) if message.edit_date else None,
+        )
+        if old is not None:
+            old.replaced_by = obj
+            old.save()
+        return obj
+
+    def __str__(self):
+        return 'id="{}" (chat_id="{}")'.format(self.message_id, self.chat.chat_id)
+
+    class Meta:
+        ordering = 'message_id', 'edit_date',

aggregator/templates/cabinet/aggregator/source_form.html

@@ -0,0 +1,59 @@
+{% extends 'cabinet/_internal_base.html' %}
+{% load bootstrap4 %}
+
+{% block breadcrumbs %}
+    <li><a href="{% url 'cabinet:aggregator:index' %}">Aggregation source list</a></li>
+    <li><span>{{ title }}</span></li>
+{% endblock %}
+
+{% block content %}
+    <form action="" method="post" class="card">
+        {% csrf_token %}
+        <header class="card-header">
+            <h2 class="card-title">{% if source %}Source "{{ source.title }}" configuration{% else %}New source{% endif %}</h2>
+        </header>
+        <div class="card-body">
+            {% bootstrap_form form layout='horizontal' %}
+            {% if source %}
+                <div class="form-group row">
+                    <label class="col-md-3 col-form-label" for="id_feed-last_check">Chat ID</label>
+                    <div class="col-md-9">
+                        <input type="text" value="{{ source.chat_id }}" class="form-control" placeholder="Chat ID" title="" id="id_chat_id" disabled>
+                    </div>
+                </div>
+            {% endif %}
+        </div>
+        <footer class="card-footer text-right">
+            {% if source %}<a href="#delete-modal" class="modal-basic btn btn-danger">Delete</a>{% endif %}
+            <button type="submit" class="btn btn-primary">Save</button>
+        </footer>
+    </form>
+
+    {% if source %}
+        <div id="delete-modal" class="modal-block modal-full-color modal-block-danger mfp-hide">
+            <section class="card">
+                <header class="card-header">
+                    <h2 class="card-title">Delete source "{{ source.title }}"</h2>
+                </header>
+                <div class="card-body">
+                    <div class="modal-wrapper">
+                        <div class="modal-icon"><i class="fas fa-times-circle"></i></div>
+                        <div class="modal-text">
+                            <h4>Are you sure?</h4>
+                            <p>This action cannot be undone.</p>
+                        </div>
+                    </div>
+                </div>
+                <footer class="card-footer">
+                    <div class="row">
+                        <form action="{% url 'cabinet:aggregator:delete' pk=source.pk %}" method="post" class="col-md-12 text-right">
+                            {% csrf_token %}
+                            <button type="button" class="btn btn-default modal-dismiss">Cancel</button>
+                            <button type="submit" class="btn btn-danger">Delete</button>
+                        </form>
+                    </div>
+                </footer>
+            </section>
+        </div>
+    {% endif %}
+{% endblock %}

aggregator/templates/cabinet/aggregator/source_list.html

@@ -0,0 +1,40 @@
+{% extends 'cabinet/_internal_base.html' %}
+
+{% block breadcrumbs %}
+    <li><span>Aggregator source list</span></li>
+{% endblock %}
+
+{% block content %}
+    <section class="card">
+        <header class="card-header">
+            <div class="card-button-actions">
+                <a href="{% url 'cabinet:aggregator:new' %}" class="btn btn-primary btn-sm" type="button">
+                    Add new source
+                </a>
+            </div>
+            <h2 class="card-title">Your sources</h2>
+        </header>
+        <div class="card-body">
+            <table class="table table-hover table-responsive-md mb-0">
+                <thead>
+                <tr>
+                    <th>Title</th>
+                    <th>Chat ID</th>
+                </tr>
+                </thead>
+                {% for source in object_list %}
+                <tr class="clickable-row" data-href="{% url 'cabinet:aggregator:edit' pk=source.pk %}">
+                    <td>{{ source.title }}</td>
+                    <td>{{ source.chat_id }}</td>
+                </tr>
+                {% empty %}
+                <tfoot>
+                <tr>
+                    <td colspan="2">No sources added</td>
+                </tr>
+                </tfoot>
+                {% endfor %}
+            </table>
+        </div>
+    </section>
+{% endblock %}

aggregator/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

aggregator/urls.py

@@ -0,0 +1,13 @@
+from django.urls import path
+
+from .views import AggregationSourceListView, AggregationSourceCreateView, AggregationSourceUpdateView, \
+                   AggregationSourceDeleteView
+
+app_name = 'feeds'
+urlpatterns = [
+    path('', AggregationSourceListView.as_view(), name='index'),
+    path('<int:pk>/', AggregationSourceUpdateView.as_view(), name='edit'),
+    path('new/', AggregationSourceCreateView.as_view(), name='new'),
+    path('delete/<int:pk>/', AggregationSourceDeleteView.as_view(), name='delete'),
+]
+

aggregator/views.py

@@ -0,0 +1,60 @@
+from django.contrib import messages
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.shortcuts import redirect
+from django.views import View
+from django.views.generic import ListView, UpdateView, CreateView
+from django.views.generic.detail import SingleObjectMixin
+
+from aggregator.forms import AggregationSourceForm, AggregationSourceEditForm
+from aggregator.models import AggregationSource
+from cabinet.utils import CabinetViewMixin
+
+
+class AggregationSourceListView(CabinetViewMixin, ListView):
+    template_name = 'cabinet/aggregator/source_list.html'
+    title = 'Aggregation source list'
+    sidebar_section = 'aggregator'
+
+    def get_queryset(self):
+        return AggregationSource.objects.filter(owner=self.request.user)
+
+
+class AggregationSourceCreateView(CabinetViewMixin, CreateView):
+    template_name = 'cabinet/aggregator/source_form.html'
+    title = 'Create aggregation source'
+    sidebar_section = 'aggregator'
+    form_class = AggregationSourceForm
+
+    def form_valid(self, form):
+        form.instance.owner = self.request.user
+        form.instance.chat_id = form.cleaned_data['chat_id']
+        form.save()
+        return redirect('cabinet:aggregator:edit', pk=form.instance.pk)
+
+
+class AggregationSourceUpdateView(CabinetViewMixin, UpdateView):
+    template_name = 'cabinet/aggregator/source_form.html'
+    title = 'Configure aggregation source'
+    sidebar_section = 'aggregator'
+    form_class = AggregationSourceEditForm
+    context_object_name = 'source'
+
+    def get_queryset(self):
+        return AggregationSource.objects.filter(owner=self.request.user)
+
+    def form_valid(self, form):
+        if 'chat_id' in form.cleaned_data:
+            form.instance.chat_id = form.cleaned_data['chat_id']
+        form.save()
+        return redirect('cabinet:aggregator:edit', pk=form.instance.pk)
+
+
+class AggregationSourceDeleteView(LoginRequiredMixin, SingleObjectMixin, View):
+    def get_queryset(self):
+        return AggregationSource.objects.filter(owner=self.request.user)
+
+    def post(self, request, *args, **kwargs):
+        source = self.get_object()
+        messages.success(self.request, 'Source "{}" was successfully deleted'.format(source.title))
+        source.delete()
+        return redirect('cabinet:aggregator:index')