add aggregator app

.idea/telegram_bots.iml

@@ -24,6 +24,7 @@
         <option value="$MODULE_DIR$/templates" />
         <option value="$MODULE_DIR$/feeds/templates" />
         <option value="$MODULE_DIR$/cabinet/templates" />
+        <option value="$MODULE_DIR$/aggregator/templates" />
       </list>
     </option>
   </component>

aggregator/admin.py

@@ -0,0 +1,8 @@
+from django.contrib import admin
+
+from .models import AggregationSource, Chat, Message
+
+
+admin.site.register(AggregationSource)
+admin.site.register(Chat)
+admin.site.register(Message)

aggregator/apps.py

@@ -0,0 +1,14 @@
+from django.apps import AppConfig
+
+
+class AggregatorConfig(AppConfig):
+    name = 'aggregator'
+
+    def ready(self):
+        self.register_config()
+
+    def register_config(self):
+        import djconfig
+        from .forms import AggregatorAppConfigForm
+
+        djconfig.register(AggregatorAppConfigForm)

aggregator/client.py

@@ -0,0 +1,86 @@
+import os
+
+from django.core.files.storage import default_storage
+from djconfig import config
+from pyrogram import Client, MessageHandler, DeletedMessagesHandler, \
+                     Message as PyrogramMessage, Messages as PyrogramMessages
+from pyrogram.api.errors import ChannelPrivate
+from pyrogram.session import Session
+
+from aggregator.models import AggregationSource, Message, Chat
+
+
+Session.notice_displayed = True
+
+
+def get_client():
+    config._reload_maybe()
+    session_path = os.path.relpath(default_storage.path(config.pyrogram_session.replace('.session', '')))
+    return Client(session_path, config.pyrogram_app_id, config.pyrogram_app_hash)
+
+
+def save_message(client, message: PyrogramMessage):
+    if not AggregationSource.objects.filter(chat_id=message.chat.id).exists():
+        return
+    Message.from_obj(message, client)
+
+
+def delete_messages(client, messages: PyrogramMessages):
+    for message in messages.messages:
+        Message.objects.filter(chat__chat_id=message.chat.id, message_id=message.message_id).update(deleted=True)
+
+
+def collect_new_messages(client, chat):
+    # Collecting new messages
+    last_message = chat.messages.order_by('-message_id').first()
+    if last_message:
+        itr = client.iter_history(chat.chat_id, reverse=True, offset_id=last_message.message_id + 1)
+    else:
+        itr = client.iter_history(chat.chat_id, reverse=True, limit=10)
+    for message in itr:
+        Message.from_obj(message, client)
+
+
+def startup_collect(client: Client):
+    for chat in Chat.objects.all():
+        try:
+            client.get_chat(chat.chat_id)
+        except ChannelPrivate:
+            print('I was banned in chat id="{}"'.format(chat.chat_id))
+            continue
+        # Collecting edited & deleted messages
+        offset = 0
+        qs = Message.objects.active_messages().filter(chat__chat_id=chat.chat_id)
+        while True:
+            lst = qs[200*offset:200*(offset+1)]
+            if not lst:
+                break
+            messages = client.get_messages(chat.chat_id, [m.message_id for m in lst])
+            for message in messages.messages:
+                m_qs = Message.objects.active_messages() \
+                    .filter(chat__chat_id=chat.chat_id, message_id=message.message_id)
+                if message.empty:
+                     m_qs.update(deleted=True)
+                elif message.edit_date:
+                    m = m_qs.first()  # type: Message
+                    if m and (not m.edit_date or m.edit_date.timestamp() != message.edit_date):
+                        Message.from_obj(message, client)
+            offset += 1
+
+        collect_new_messages(client, chat)
+
+
+def start_client():
+    app = get_client()
+
+    app.add_handler(MessageHandler(save_message))
+    app.add_handler(DeletedMessagesHandler(delete_messages))
+
+    print('Starting pyrogram client...')
+    app.start()
+
+    print('Loading updates that happened while I was offline...')
+    startup_collect(app)
+
+    print('Idling...')
+    app.idle()

aggregator/forms.py

@@ -0,0 +1,55 @@
+from django import forms
+from django.core.exceptions import ValidationError
+from django.core.files.storage import default_storage
+from django.forms import ModelForm
+from djconfig.forms import ConfigForm
+from pyrogram.api.errors import ChannelPrivate
+
+from config.utils import parse_mtproto_chat
+from .client import get_client
+from .models import AggregationSource, Chat
+
+
+class AggregationSourceForm(ModelForm):
+    invite_link = forms.CharField(help_text='Invite link (with joinchat) or username')
+
+    def clean(self):
+        cleaned_data = super(AggregationSourceForm, self).clean()
+        invite_link = cleaned_data.pop('invite_link')
+        if invite_link:
+            with get_client() as app:
+                try:
+                    upd = app.join_chat(invite_link)
+                    chat = parse_mtproto_chat(app, upd.chats[0])
+                    cleaned_data['chat_id'] = chat.id
+                    Chat.from_obj(chat, app)
+                except ChannelPrivate:
+                    raise ValidationError('I was banned in this chat')
+        return cleaned_data
+
+    class Meta:
+        model = AggregationSource
+        fields = 'title', 'invite_link',
+
+
+class AggregationSourceEditForm(AggregationSourceForm):
+    invite_link = forms.CharField(help_text='Invite link (with joinchat) or username; '
+                                            'leave empty if change is not needed', required=False)
+
+
+class AggregatorAppConfigForm(ConfigForm):
+    slug = 'aggregator'
+    title = 'Aggregator'
+
+    pyrogram_app_id = forms.CharField()
+    pyrogram_app_hash = forms.CharField()
+    pyrogram_session = forms.FileField()
+
+    def save_session(self):
+        session = self.cleaned_data.get('pyrogram_session')
+        if session:
+            session.name = default_storage.save(session.name, session)
+
+    def save(self):
+        self.save_session()
+        super(AggregatorAppConfigForm, self).save()

aggregator/management/commands/start_aggregator_client.py

@@ -0,0 +1,8 @@
+from django.core.management import BaseCommand
+
+from aggregator.client import start_client
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        start_client()

aggregator/migrations/0001_initial.py

@@ -0,0 +1,54 @@
+# Generated by Django 2.1.5 on 2019-01-27 12:20
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AggregationSource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=64)),
+                ('chat_id', models.IntegerField(db_index=True)),
+                ('last_id', models.PositiveIntegerField()),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Chat',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('chat_id', models.IntegerField(db_index=True)),
+                ('title', models.TextField()),
+                ('username', models.CharField(blank=True, max_length=64, null=True)),
+                ('photo', models.ImageField(blank=True, null=True, upload_to='')),
+                ('photo_id', models.CharField(blank=True, max_length=64, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Message',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('message_id', models.PositiveIntegerField(db_index=True)),
+                ('text', models.TextField(blank=True)),
+                ('date', models.DateTimeField()),
+                ('edit_date', models.DateTimeField(blank=True, null=True)),
+                ('deleted', models.BooleanField(default=False)),
+                ('chat', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='aggregator.Chat')),
+                ('replaced_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='aggregator.Message')),
+            ],
+            options={
+                'ordering': ('message_id', 'edit_date'),
+            },
+        ),
+    ]

aggregator/migrations/0002_remove_aggregationsource_last_id.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.1.5 on 2019-01-27 13:42
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('aggregator', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='aggregationsource',
+            name='last_id',
+        ),
+    ]

aggregator/models.py

@@ -0,0 +1,96 @@
+import os
+from datetime import datetime
+from tempfile import TemporaryDirectory
+
+import pytz
+from django.conf import settings
+from django.db import models
+from pyrogram import Chat as PyrogramChat, Message as PyrogramMessage, ChatPhoto as PyrogramChatPhoto
+from pyrogram.api.types.chat_photo import ChatPhoto as MTProtoChatPhoto
+
+
+class AggregationSource(models.Model):
+    owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
+    title = models.CharField(max_length=64)
+    chat_id = models.IntegerField(db_index=True)
+
+    def __str__(self):
+        return self.title
+
+
+class Chat(models.Model):
+    chat_id = models.IntegerField(db_index=True)
+    title = models.TextField()
+    username = models.CharField(max_length=64, null=True, blank=True)
+    photo = models.ImageField(null=True, blank=True)
+    photo_id = models.CharField(max_length=64, null=True, blank=True)
+
+    @classmethod
+    def from_obj(cls, chat: PyrogramChat, client):
+        obj, _ = Chat.objects.update_or_create(
+            chat_id=chat.id,
+            defaults={
+                'title': chat.title or '{} {}'.format(chat.first_name, chat.last_name).rstrip(),
+                'username': chat.username,
+            }
+        )
+        if chat.photo is None:
+            if obj.photo_id is not None:
+                obj.photo = obj.photo_id = None
+                obj.save()
+        else:
+            photo_file_id = chat.photo.small_file_id
+            if photo_file_id != obj.photo_id:
+                with TemporaryDirectory() as d:
+                    path = client.download_media(photo_file_id, os.path.join(d, ''), block=True)
+                    with open(path, 'rb') as f:
+                        obj.photo.save(os.path.basename(path), f, save=True)
+                obj.photo_id = chat.photo.small_file_id
+                obj.save()
+        return obj
+
+    def __str__(self):
+        return '{} (chat_id="{}")'.format(self.title, self.chat_id)
+
+
+class MessageManager(models.Manager):
+    def active_messages(self):
+        return self.get_queryset().filter(deleted=False, replaced_by__isnull=True)
+
+
+class Message(models.Model):
+    chat = models.ForeignKey(Chat, on_delete=models.CASCADE, related_name='messages')
+    message_id = models.PositiveIntegerField(db_index=True)
+    text = models.TextField(blank=True)
+    date = models.DateTimeField()
+    edit_date = models.DateTimeField(null=True, blank=True)
+    deleted = models.BooleanField(default=False)
+    replaced_by = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True)
+
+    objects = MessageManager()
+
+    @classmethod
+    def from_obj(cls, message: PyrogramMessage, client):
+        tz = pytz.timezone('UTC')
+        chat = Chat.from_obj(message.chat, client)
+        try:
+            old = Message.objects.active_messages().get(chat=chat, message_id=message.message_id)
+        except Message.DoesNotExist:
+            old = None
+        obj = Message.objects.create(
+            chat=chat,
+            message_id=message.message_id,
+            text=message.text.html if message.text else '',
+            date=tz.localize(datetime.utcfromtimestamp(message.date)),
+            edit_date=tz.localize(datetime.utcfromtimestamp(message.edit_date)) if message.edit_date else None,
+        )
+        if old is not None:
+            old.replaced_by = obj
+            old.save()
+        return obj
+
+    def __str__(self):
+        return 'id="{}" (chat_id="{}")'.format(self.message_id, self.chat.chat_id)
+
+    class Meta:
+        ordering = 'message_id', 'edit_date',

aggregator/templates/cabinet/aggregator/source_form.html

@@ -0,0 +1,59 @@
+{% extends 'cabinet/_internal_base.html' %}
+{% load bootstrap4 %}
+
+{% block breadcrumbs %}
+    <li><a href="{% url 'cabinet:aggregator:index' %}">Aggregation source list</a></li>
+    <li><span>{{ title }}</span></li>
+{% endblock %}
+
+{% block content %}
+    <form action="" method="post" class="card">
+        {% csrf_token %}
+        <header class="card-header">
+            <h2 class="card-title">{% if source %}Source "{{ source.title }}" configuration{% else %}New source{% endif %}</h2>
+        </header>
+        <div class="card-body">
+            {% bootstrap_form form layout='horizontal' %}
+            {% if source %}
+                <div class="form-group row">
+                    <label class="col-md-3 col-form-label" for="id_feed-last_check">Chat ID</label>
+                    <div class="col-md-9">
+                        <input type="text" value="{{ source.chat_id }}" class="form-control" placeholder="Chat ID" title="" id="id_chat_id" disabled>
+                    </div>
+                </div>
+            {% endif %}
+        </div>
+        <footer class="card-footer text-right">
+            {% if source %}<a href="#delete-modal" class="modal-basic btn btn-danger">Delete</a>{% endif %}
+            <button type="submit" class="btn btn-primary">Save</button>
+        </footer>
+    </form>
+
+    {% if source %}
+        <div id="delete-modal" class="modal-block modal-full-color modal-block-danger mfp-hide">
+            <section class="card">
+                <header class="card-header">
+                    <h2 class="card-title">Delete source "{{ source.title }}"</h2>
+                </header>
+                <div class="card-body">
+                    <div class="modal-wrapper">
+                        <div class="modal-icon"><i class="fas fa-times-circle"></i></div>
+                        <div class="modal-text">
+                            <h4>Are you sure?</h4>
+                            <p>This action cannot be undone.</p>
+                        </div>
+                    </div>
+                </div>
+                <footer class="card-footer">
+                    <div class="row">
+                        <form action="{% url 'cabinet:aggregator:delete' pk=source.pk %}" method="post" class="col-md-12 text-right">
+                            {% csrf_token %}
+                            <button type="button" class="btn btn-default modal-dismiss">Cancel</button>
+                            <button type="submit" class="btn btn-danger">Delete</button>
+                        </form>
+                    </div>
+                </footer>
+            </section>
+        </div>
+    {% endif %}
+{% endblock %}

aggregator/templates/cabinet/aggregator/source_list.html

@@ -0,0 +1,40 @@
+{% extends 'cabinet/_internal_base.html' %}
+
+{% block breadcrumbs %}
+    <li><span>Aggregator source list</span></li>
+{% endblock %}
+
+{% block content %}
+    <section class="card">
+        <header class="card-header">
+            <div class="card-button-actions">
+                <a href="{% url 'cabinet:aggregator:new' %}" class="btn btn-primary btn-sm" type="button">
+                    Add new source
+                </a>
+            </div>
+            <h2 class="card-title">Your sources</h2>
+        </header>
+        <div class="card-body">
+            <table class="table table-hover table-responsive-md mb-0">
+                <thead>
+                <tr>
+                    <th>Title</th>
+                    <th>Chat ID</th>
+                </tr>
+                </thead>
+                {% for source in object_list %}
+                <tr class="clickable-row" data-href="{% url 'cabinet:aggregator:edit' pk=source.pk %}">
+                    <td>{{ source.title }}</td>
+                    <td>{{ source.chat_id }}</td>
+                </tr>
+                {% empty %}
+                <tfoot>
+                <tr>
+                    <td colspan="2">No sources added</td>
+                </tr>
+                </tfoot>
+                {% endfor %}
+            </table>
+        </div>
+    </section>
+{% endblock %}

aggregator/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

aggregator/urls.py

@@ -0,0 +1,13 @@
+from django.urls import path
+
+from .views import AggregationSourceListView, AggregationSourceCreateView, AggregationSourceUpdateView, \
+                   AggregationSourceDeleteView
+
+app_name = 'feeds'
+urlpatterns = [
+    path('', AggregationSourceListView.as_view(), name='index'),
+    path('<int:pk>/', AggregationSourceUpdateView.as_view(), name='edit'),
+    path('new/', AggregationSourceCreateView.as_view(), name='new'),
+    path('delete/<int:pk>/', AggregationSourceDeleteView.as_view(), name='delete'),
+]
+

aggregator/views.py

@@ -0,0 +1,60 @@
+from django.contrib import messages
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.shortcuts import redirect
+from django.views import View
+from django.views.generic import ListView, UpdateView, CreateView
+from django.views.generic.detail import SingleObjectMixin
+
+from aggregator.forms import AggregationSourceForm, AggregationSourceEditForm
+from aggregator.models import AggregationSource
+from cabinet.utils import CabinetViewMixin
+
+
+class AggregationSourceListView(CabinetViewMixin, ListView):
+    template_name = 'cabinet/aggregator/source_list.html'
+    title = 'Aggregation source list'
+    sidebar_section = 'aggregator'
+
+    def get_queryset(self):
+        return AggregationSource.objects.filter(owner=self.request.user)
+
+
+class AggregationSourceCreateView(CabinetViewMixin, CreateView):
+    template_name = 'cabinet/aggregator/source_form.html'
+    title = 'Create aggregation source'
+    sidebar_section = 'aggregator'
+    form_class = AggregationSourceForm
+
+    def form_valid(self, form):
+        form.instance.owner = self.request.user
+        form.instance.chat_id = form.cleaned_data['chat_id']
+        form.save()
+        return redirect('cabinet:aggregator:edit', pk=form.instance.pk)
+
+
+class AggregationSourceUpdateView(CabinetViewMixin, UpdateView):
+    template_name = 'cabinet/aggregator/source_form.html'
+    title = 'Configure aggregation source'
+    sidebar_section = 'aggregator'
+    form_class = AggregationSourceEditForm
+    context_object_name = 'source'
+
+    def get_queryset(self):
+        return AggregationSource.objects.filter(owner=self.request.user)
+
+    def form_valid(self, form):
+        if 'chat_id' in form.cleaned_data:
+            form.instance.chat_id = form.cleaned_data['chat_id']
+        form.save()
+        return redirect('cabinet:aggregator:edit', pk=form.instance.pk)
+
+
+class AggregationSourceDeleteView(LoginRequiredMixin, SingleObjectMixin, View):
+    def get_queryset(self):
+        return AggregationSource.objects.filter(owner=self.request.user)
+
+    def post(self, request, *args, **kwargs):
+        source = self.get_object()
+        messages.success(self.request, 'Source "{}" was successfully deleted'.format(source.title))
+        source.delete()
+        return redirect('cabinet:aggregator:index')

cabinet/templates/cabinet/_includes/sidebar.html

@@ -15,6 +15,12 @@
                             <span>Feeds</span>
                         </a>
                     </li>
+                    <li class="{% if sidebar_section == 'aggregator' %}nav-active{% endif %}">
+                        <a href="{% url 'cabinet:aggregator:index' %}">
+                            <i class="fas fa-copy" aria-hidden="true"></i>
+                            <span>Aggregator</span>
+                        </a>
+                    </li>
                 </ul>
             </nav>
 
@@ -24,14 +30,16 @@
             <div class="nav-subtitle">Admin</div>
             <nav class="nav-main" role="navigation">
                 <ul class="nav nav-main">
-                    <li class="nav-parent {% if sidebar_section %}nav-active{% endif %}">
+                    <li class="nav-parent {% if sidebar_section|slice:':6' == 'admin_' %}nav-expanded nav-active{% endif %}">
                         <a href="#">
                             <i class="fas fa-cog" aria-hidden="true"></i>
                             <span>Configs</span>
                         </a>
                         <ul class="nav nav-children">
                             {% for slug, title in admin_configs.items %}
-                            <li><a href="{% url 'cabinet:admin_config' slug=slug %}" class="nav-link">{{ title }}</a></li>
+                            <li class="{% if sidebar_section == 'admin_'|add:slug %}nav-active{% endif %}">
+                                <a href="{% url 'cabinet:admin_config' slug=slug %}" class="nav-link">{{ title }}</a>
+                            </li>
                             {% endfor %}
                         </ul>
                     </li>

cabinet/templates/cabinet/admin_config.html

@@ -8,7 +8,7 @@
 {% endblock %}
 
 {% block content %}
-    <form action="" method="post" class="card">
+    <form action="" method="post" enctype="multipart/form-data" class="card">
         {% csrf_token %}
         <header class="card-header">
             <h2 class="card-title">Config</h2>

cabinet/urls.py

@@ -9,5 +9,6 @@ urlpatterns = [
     path('login/', LoginView.as_view(), name='login'),
     path('logout/', LogoutView.as_view(), name='logout'),
     path('feeds/', include('feeds.urls', namespace='feeds')),
+    path('aggregator/', include('aggregator.urls', namespace='aggregator')),
     path('admin/config/<slug>/', AdminConfigView.as_view(), name='admin_config'),
 ]

cabinet/utils.py

@@ -6,13 +6,16 @@ class CabinetViewMixin(LoginRequiredMixin):
     title = 'No title'
     sidebar_section = None
 
+    def get_sidebar_section(self):
+        return self.sidebar_section
+
     def get_title(self):
         return self.title
 
     def get_context_data(self, **kwargs):
         ctx = super(CabinetViewMixin, self).get_context_data(**kwargs)
         ctx['title'] = self.get_title()
-        ctx['sidebar_section'] = self.sidebar_section
+        ctx['sidebar_section'] = self.get_sidebar_section()
         return ctx
 
 

cabinet/views.py

@@ -30,6 +30,9 @@ class LoginView(BaseLoginView):
 class AdminConfigView(CabinetViewMixin, FormView):
     template_name = 'cabinet/admin_config.html'
 
+    def get_sidebar_section(self):
+        return 'admin_{}'.format(self.get_form_class().slug)
+
     def get_success_url(self):
         return ''
 

config/settings.py

@@ -33,6 +33,7 @@ INSTALLED_APPS = [
 
     'cabinet.apps.CabinetConfig',
     'feeds.apps.FeedsConfig',
+    'aggregator.apps.AggregatorConfig',
 ]
 
 MIDDLEWARE = [

config/utils.py

@@ -1,6 +1,18 @@
 from urllib.parse import urlparse
 
 from django.http import HttpRequest, HttpResponse, HttpResponseForbidden
+from pyrogram import Chat as PyrogramChat
+from pyrogram.api.types.chat import Chat as MTProtoChat
+from pyrogram.api.types.user import User as MTProtoUser
+from pyrogram.api.types.channel import Channel as MTProtoChannel
+
+
+def parse_mtproto_chat(client, chat):
+    if isinstance(chat, MTProtoChat):
+        return PyrogramChat._parse_chat_chat(client, chat)
+    elif isinstance(chat, MTProtoUser):
+        return PyrogramChat._parse_user_chat(client, chat)
+    return PyrogramChat._parse_channel_chat(client, chat)
 
 
 def same_origin(current_uri, redirect_uri):

feeds/views.py

@@ -1,4 +1,5 @@
 from django.contrib import messages
+from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.contenttypes.models import ContentType
 from django.http import HttpResponseRedirect
 from django.shortcuts import get_object_or_404, redirect
@@ -60,7 +61,7 @@ class FeedConfigCreateView(BaseFeedConfigView):
         return get_object_or_404(ContentType, model=self.kwargs['content_type'])
 
 
-class FeedConfigDeleteView(SingleObjectMixin, View):
+class FeedConfigDeleteView(LoginRequiredMixin, SingleObjectMixin, View):
     def get_queryset(self):
         return Feed.objects.filter(owner=self.request.user)
 

requirements.txt

@@ -1,4 +1,5 @@
 amqp==2.3.2
+async-generator==1.10
 beautifulsoup4==4.7.1
 billiard==3.5.0.5
 celery==4.2.1
@@ -19,7 +20,10 @@ django-yamlfield==1.0.3
 enum34==1.1.6
 idna==2.8
 kombu==4.2.2.post1
+Pillow==5.4.1
 psycopg2-binary==2.7.6.1
+pyaes==1.6.1
+-e git+https://github.com/pyrogram/pyrogram@69922e5cf905df7da5a37602be48d063dea4212e#egg=Pyrogram
 PySocks==1.6.8
 pyTelegramBotAPI==3.6.6
 python-crontab==2.3.6
@@ -31,6 +35,7 @@ requests==2.21.0
 sentry-sdk==0.6.9
 six==1.12.0
 soupsieve==1.7.3
+TgCrypto==1.1.1
 urllib3==1.24.1
 uWSGI==2.0.17.1
 vine==1.2.0

static/js/custom.init.js

@@ -51,3 +51,104 @@ $('.modal-basic').magnificPopup({
 	});
 
 }).apply(this, [jQuery]);
+
+
+
+// Navigation
+(function($) {
+
+	'use strict';
+
+	var $items = $( '.nav-main li.nav-parent' );
+
+	function expand( $li ) {
+		$li.children( 'ul.nav-children' ).slideDown( 'fast', function() {
+			$li.addClass( 'nav-expanded' );
+			$(this).css( 'display', '' );
+			ensureVisible( $li );
+		});
+	}
+
+	function collapse( $li ) {
+		$li.children('ul.nav-children' ).slideUp( 'fast', function() {
+			$(this).css( 'display', '' );
+			$li.removeClass( 'nav-expanded' );
+		});
+	}
+
+	function ensureVisible( $li ) {
+		var scroller = $li.offsetParent();
+		if ( !scroller.get(0) ) {
+			return false;
+		}
+
+		var top = $li.position().top;
+		if ( top < 0 ) {
+			scroller.animate({
+				scrollTop: scroller.scrollTop() + top
+			}, 'fast');
+		}
+	}
+
+	function buildSidebarNav( anchor, prev, next, ev ) {
+		if ( anchor.prop('href') ) {
+			var arrowWidth = parseInt(window.getComputedStyle(anchor.get(0), ':after').width, 10) || 0;
+			if (ev.offsetX > anchor.get(0).offsetWidth - arrowWidth) {
+				ev.preventDefault();
+			}
+		}
+
+		if ( prev.get( 0 ) !== next.get( 0 ) ) {
+			collapse( prev );
+			expand( next );
+		} else {
+			collapse( prev );
+		}
+	}
+
+	$items.find('> a').on('click', function( ev ) {
+
+		var $html   = $('html'),
+			$window = $(window),
+		    $anchor = $( this ),
+			$prev   = $anchor.closest('ul.nav').find('> li.nav-expanded' ),
+			$next   = $anchor.closest('li'),
+			$ev     = ev;
+
+		if( $anchor.attr('href') == '#' ) {
+			ev.preventDefault();
+		}
+
+		if( !$html.hasClass('sidebar-left-big-icons') ) {
+			buildSidebarNav( $anchor, $prev, $next, $ev );
+		} else if( $html.hasClass('sidebar-left-big-icons') && $window.width() < 768 ) {
+			buildSidebarNav( $anchor, $prev, $next, $ev );
+		}
+
+	});
+
+	// Chrome Fix
+	$.browser.chrome = /chrom(e|ium)/.test(navigator.userAgent.toLowerCase());
+	if( $.browser.chrome && !$.browser.mobile ) {
+		var flag = true;
+		$('.sidebar-left .nav-main li a').on('click', function(){
+			flag = false;
+			setTimeout(function(){
+				flag = true;
+			}, 200);
+		});
+
+		$('.nano').on('mouseenter', function(e){
+			$(this).addClass('hovered');
+		});
+
+		$('.nano').on('mouseleave', function(e){
+			if( flag ) {
+				$(this).removeClass('hovered');
+			}
+		});
+	}
+
+	$('.nav-main a').filter(':not([href])').attr('href', '#');
+
+}).apply(this, [jQuery]);

static/js/theme.js

@@ -4106,105 +4106,6 @@ window.theme.fn = {
 
 }).apply(this, [window.theme, jQuery]);
 
-// Navigation
-(function($) {
-
-	'use strict';
-
-	var $items = $( '.nav-main li.nav-parent' );
-
-	function expand( $li ) {
-		$li.children( 'ul.nav-children' ).slideDown( 'fast', function() {
-			$li.addClass( 'nav-expanded' );
-			$(this).css( 'display', '' );
-			ensureVisible( $li );
-		});
-	}
-
-	function collapse( $li ) {
-		$li.children('ul.nav-children' ).slideUp( 'fast', function() {
-			$(this).css( 'display', '' );
-			$li.removeClass( 'nav-expanded' );
-		});
-	}
-
-	function ensureVisible( $li ) {
-		var scroller = $li.offsetParent();
-		if ( !scroller.get(0) ) {
-			return false;
-		}
-
-		var top = $li.position().top;
-		if ( top < 0 ) {
-			scroller.animate({
-				scrollTop: scroller.scrollTop() + top
-			}, 'fast');
-		}
-	}
-
-	function buildSidebarNav( anchor, prev, next, ev ) {
-		if ( anchor.prop('href') ) {
-			var arrowWidth = parseInt(window.getComputedStyle(anchor.get(0), ':after').width, 10) || 0;
-			if (ev.offsetX > anchor.get(0).offsetWidth - arrowWidth) {
-				ev.preventDefault();
-			}
-		}
-
-		if ( prev.get( 0 ) !== next.get( 0 ) ) {
-			collapse( prev );
-			expand( next );
-		} else {
-			collapse( prev );
-		}
-	}
-
-	$items.find('> a').on('click', function( ev ) {
-
-		var $html   = $('html'),
-			$window = $(window),
-		    $anchor = $( this ),
-			$prev   = $anchor.closest('ul.nav').find('> li.nav-expanded' ),
-			$next   = $anchor.closest('li'),
-			$ev     = ev;
-
-		if( $anchor.attr('href') == '#' ) {
-			ev.preventDefault();
-		}
-
-		if( !$html.hasClass('sidebar-left-big-icons') ) {
-			buildSidebarNav( $anchor, $prev, $next, $ev );
-		} else if( $html.hasClass('sidebar-left-big-icons') && $window.width() < 768 ) {
-			buildSidebarNav( $anchor, $prev, $next, $ev );
-		}
-
-	});
-
-	// Chrome Fix
-	$.browser.chrome = /chrom(e|ium)/.test(navigator.userAgent.toLowerCase());
-	if( $.browser.chrome && !$.browser.mobile ) {
-		var flag = true;
-		$('.sidebar-left .nav-main li a').on('click', function(){
-			flag = false;
-			setTimeout(function(){
-				flag = true;
-			}, 200);
-		});
-
-		$('.nano').on('mouseenter', function(e){
-			$(this).addClass('hovered');
-		});
-
-		$('.nano').on('mouseleave', function(e){
-			if( flag ) {
-				$(this).removeClass('hovered');
-			}
-		});	
-	}
-
-	$('.nav-main a').filter(':not([href])').attr('href', '#');
-
-}).apply(this, [jQuery]);
-
 // Skeleton
 (function(theme, $) {
 

supervisor.conf

@@ -1,5 +1,5 @@
 [group:bots]
-programs = bots_web,bots_celeryd,bots_celerybeat
+programs = bots_web,bots_aggregator_client,bots_celeryd,bots_celerybeat
 
 [program:bots_web]
 user = http
@@ -11,6 +11,16 @@ stderr_logfile = /srv/apps/bots/logs/uwsgi.log
 stdout_logfile = /srv/apps/bots/logs/uwsgi.log
 stopsignal = INT
 
+[program:bots_aggregator_client]
+user = http
+directory = /srv/apps/bots
+command = /srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py start_aggregator_client
+autostart = true
+autorestart = true
+stderr_logfile = /srv/apps/bots/logs/aggregator_client.log
+stdout_logfile = /srv/apps/bots/logs/aggregator_client.log
+stopsignal = INT
+
 [program:bots_celeryd]
 user = http
 directory = /srv/apps/bots