parent
8643fc42f9
commit
b7e68abb03
@ -0,0 +1,8 @@
|
||||
from django.contrib import admin
|
||||
|
||||
from .models import AggregationSource, Chat, Message
|
||||
|
||||
|
||||
admin.site.register(AggregationSource)
|
||||
admin.site.register(Chat)
|
||||
admin.site.register(Message)
|
||||
@ -0,0 +1,14 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class AggregatorConfig(AppConfig):
|
||||
name = 'aggregator'
|
||||
|
||||
def ready(self):
|
||||
self.register_config()
|
||||
|
||||
def register_config(self):
|
||||
import djconfig
|
||||
from .forms import AggregatorAppConfigForm
|
||||
|
||||
djconfig.register(AggregatorAppConfigForm)
|
||||
@ -0,0 +1,86 @@
|
||||
import os
|
||||
|
||||
from django.core.files.storage import default_storage
|
||||
from djconfig import config
|
||||
from pyrogram import Client, MessageHandler, DeletedMessagesHandler, \
|
||||
Message as PyrogramMessage, Messages as PyrogramMessages
|
||||
from pyrogram.api.errors import ChannelPrivate
|
||||
from pyrogram.session import Session
|
||||
|
||||
from aggregator.models import AggregationSource, Message, Chat
|
||||
|
||||
|
||||
Session.notice_displayed = True
|
||||
|
||||
|
||||
def get_client():
|
||||
config._reload_maybe()
|
||||
session_path = os.path.relpath(default_storage.path(config.pyrogram_session.replace('.session', '')))
|
||||
return Client(session_path, config.pyrogram_app_id, config.pyrogram_app_hash)
|
||||
|
||||
|
||||
def save_message(client, message: PyrogramMessage):
|
||||
if not AggregationSource.objects.filter(chat_id=message.chat.id).exists():
|
||||
return
|
||||
Message.from_obj(message, client)
|
||||
|
||||
|
||||
def delete_messages(client, messages: PyrogramMessages):
|
||||
for message in messages.messages:
|
||||
Message.objects.filter(chat__chat_id=message.chat.id, message_id=message.message_id).update(deleted=True)
|
||||
|
||||
|
||||
def collect_new_messages(client, chat):
|
||||
# Collecting new messages
|
||||
last_message = chat.messages.order_by('-message_id').first()
|
||||
if last_message:
|
||||
itr = client.iter_history(chat.chat_id, reverse=True, offset_id=last_message.message_id + 1)
|
||||
else:
|
||||
itr = client.iter_history(chat.chat_id, reverse=True, limit=10)
|
||||
for message in itr:
|
||||
Message.from_obj(message, client)
|
||||
|
||||
|
||||
def startup_collect(client: Client):
|
||||
for chat in Chat.objects.all():
|
||||
try:
|
||||
client.get_chat(chat.chat_id)
|
||||
except ChannelPrivate:
|
||||
print('I was banned in chat id="{}"'.format(chat.chat_id))
|
||||
continue
|
||||
# Collecting edited & deleted messages
|
||||
offset = 0
|
||||
qs = Message.objects.active_messages().filter(chat__chat_id=chat.chat_id)
|
||||
while True:
|
||||
lst = qs[200*offset:200*(offset+1)]
|
||||
if not lst:
|
||||
break
|
||||
messages = client.get_messages(chat.chat_id, [m.message_id for m in lst])
|
||||
for message in messages.messages:
|
||||
m_qs = Message.objects.active_messages() \
|
||||
.filter(chat__chat_id=chat.chat_id, message_id=message.message_id)
|
||||
if message.empty:
|
||||
m_qs.update(deleted=True)
|
||||
elif message.edit_date:
|
||||
m = m_qs.first() # type: Message
|
||||
if m and (not m.edit_date or m.edit_date.timestamp() != message.edit_date):
|
||||
Message.from_obj(message, client)
|
||||
offset += 1
|
||||
|
||||
collect_new_messages(client, chat)
|
||||
|
||||
|
||||
def start_client():
|
||||
app = get_client()
|
||||
|
||||
app.add_handler(MessageHandler(save_message))
|
||||
app.add_handler(DeletedMessagesHandler(delete_messages))
|
||||
|
||||
print('Starting pyrogram client...')
|
||||
app.start()
|
||||
|
||||
print('Loading updates that happened while I was offline...')
|
||||
startup_collect(app)
|
||||
|
||||
print('Idling...')
|
||||
app.idle()
|
||||
@ -0,0 +1,55 @@
|
||||
from django import forms
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.core.files.storage import default_storage
|
||||
from django.forms import ModelForm
|
||||
from djconfig.forms import ConfigForm
|
||||
from pyrogram.api.errors import ChannelPrivate
|
||||
|
||||
from config.utils import parse_mtproto_chat
|
||||
from .client import get_client
|
||||
from .models import AggregationSource, Chat
|
||||
|
||||
|
||||
class AggregationSourceForm(ModelForm):
|
||||
invite_link = forms.CharField(help_text='Invite link (with joinchat) or username')
|
||||
|
||||
def clean(self):
|
||||
cleaned_data = super(AggregationSourceForm, self).clean()
|
||||
invite_link = cleaned_data.pop('invite_link')
|
||||
if invite_link:
|
||||
with get_client() as app:
|
||||
try:
|
||||
upd = app.join_chat(invite_link)
|
||||
chat = parse_mtproto_chat(app, upd.chats[0])
|
||||
cleaned_data['chat_id'] = chat.id
|
||||
Chat.from_obj(chat, app)
|
||||
except ChannelPrivate:
|
||||
raise ValidationError('I was banned in this chat')
|
||||
return cleaned_data
|
||||
|
||||
class Meta:
|
||||
model = AggregationSource
|
||||
fields = 'title', 'invite_link',
|
||||
|
||||
|
||||
class AggregationSourceEditForm(AggregationSourceForm):
|
||||
invite_link = forms.CharField(help_text='Invite link (with joinchat) or username; '
|
||||
'leave empty if change is not needed', required=False)
|
||||
|
||||
|
||||
class AggregatorAppConfigForm(ConfigForm):
|
||||
slug = 'aggregator'
|
||||
title = 'Aggregator'
|
||||
|
||||
pyrogram_app_id = forms.CharField()
|
||||
pyrogram_app_hash = forms.CharField()
|
||||
pyrogram_session = forms.FileField()
|
||||
|
||||
def save_session(self):
|
||||
session = self.cleaned_data.get('pyrogram_session')
|
||||
if session:
|
||||
session.name = default_storage.save(session.name, session)
|
||||
|
||||
def save(self):
|
||||
self.save_session()
|
||||
super(AggregatorAppConfigForm, self).save()
|
||||
@ -0,0 +1,8 @@
|
||||
from django.core.management import BaseCommand
|
||||
|
||||
from aggregator.client import start_client
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **options):
|
||||
start_client()
|
||||
@ -0,0 +1,54 @@
|
||||
# Generated by Django 2.1.5 on 2019-01-27 12:20
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='AggregationSource',
|
||||
fields=[
|
||||
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('title', models.CharField(max_length=64)),
|
||||
('chat_id', models.IntegerField(db_index=True)),
|
||||
('last_id', models.PositiveIntegerField()),
|
||||
('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Chat',
|
||||
fields=[
|
||||
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('chat_id', models.IntegerField(db_index=True)),
|
||||
('title', models.TextField()),
|
||||
('username', models.CharField(blank=True, max_length=64, null=True)),
|
||||
('photo', models.ImageField(blank=True, null=True, upload_to='')),
|
||||
('photo_id', models.CharField(blank=True, max_length=64, null=True)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Message',
|
||||
fields=[
|
||||
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('message_id', models.PositiveIntegerField(db_index=True)),
|
||||
('text', models.TextField(blank=True)),
|
||||
('date', models.DateTimeField()),
|
||||
('edit_date', models.DateTimeField(blank=True, null=True)),
|
||||
('deleted', models.BooleanField(default=False)),
|
||||
('chat', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='aggregator.Chat')),
|
||||
('replaced_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='aggregator.Message')),
|
||||
],
|
||||
options={
|
||||
'ordering': ('message_id', 'edit_date'),
|
||||
},
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,17 @@
|
||||
# Generated by Django 2.1.5 on 2019-01-27 13:42
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('aggregator', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='aggregationsource',
|
||||
name='last_id',
|
||||
),
|
||||
]
|
||||
@ -0,0 +1,96 @@
|
||||
import os
|
||||
from datetime import datetime
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import pytz
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
from pyrogram import Chat as PyrogramChat, Message as PyrogramMessage, ChatPhoto as PyrogramChatPhoto
|
||||
from pyrogram.api.types.chat_photo import ChatPhoto as MTProtoChatPhoto
|
||||
|
||||
|
||||
class AggregationSource(models.Model):
|
||||
owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
|
||||
title = models.CharField(max_length=64)
|
||||
chat_id = models.IntegerField(db_index=True)
|
||||
|
||||
def __str__(self):
|
||||
return self.title
|
||||
|
||||
|
||||
class Chat(models.Model):
|
||||
chat_id = models.IntegerField(db_index=True)
|
||||
title = models.TextField()
|
||||
username = models.CharField(max_length=64, null=True, blank=True)
|
||||
photo = models.ImageField(null=True, blank=True)
|
||||
photo_id = models.CharField(max_length=64, null=True, blank=True)
|
||||
|
||||
@classmethod
|
||||
def from_obj(cls, chat: PyrogramChat, client):
|
||||
obj, _ = Chat.objects.update_or_create(
|
||||
chat_id=chat.id,
|
||||
defaults={
|
||||
'title': chat.title or '{} {}'.format(chat.first_name, chat.last_name).rstrip(),
|
||||
'username': chat.username,
|
||||
}
|
||||
)
|
||||
if chat.photo is None:
|
||||
if obj.photo_id is not None:
|
||||
obj.photo = obj.photo_id = None
|
||||
obj.save()
|
||||
else:
|
||||
photo_file_id = chat.photo.small_file_id
|
||||
if photo_file_id != obj.photo_id:
|
||||
with TemporaryDirectory() as d:
|
||||
path = client.download_media(photo_file_id, os.path.join(d, ''), block=True)
|
||||
with open(path, 'rb') as f:
|
||||
obj.photo.save(os.path.basename(path), f, save=True)
|
||||
obj.photo_id = chat.photo.small_file_id
|
||||
obj.save()
|
||||
return obj
|
||||
|
||||
def __str__(self):
|
||||
return '{} (chat_id="{}")'.format(self.title, self.chat_id)
|
||||
|
||||
|
||||
class MessageManager(models.Manager):
|
||||
def active_messages(self):
|
||||
return self.get_queryset().filter(deleted=False, replaced_by__isnull=True)
|
||||
|
||||
|
||||
class Message(models.Model):
|
||||
chat = models.ForeignKey(Chat, on_delete=models.CASCADE, related_name='messages')
|
||||
message_id = models.PositiveIntegerField(db_index=True)
|
||||
text = models.TextField(blank=True)
|
||||
date = models.DateTimeField()
|
||||
edit_date = models.DateTimeField(null=True, blank=True)
|
||||
deleted = models.BooleanField(default=False)
|
||||
replaced_by = models.ForeignKey('self', on_delete=models.CASCADE, null=True, blank=True)
|
||||
|
||||
objects = MessageManager()
|
||||
|
||||
@classmethod
|
||||
def from_obj(cls, message: PyrogramMessage, client):
|
||||
tz = pytz.timezone('UTC')
|
||||
chat = Chat.from_obj(message.chat, client)
|
||||
try:
|
||||
old = Message.objects.active_messages().get(chat=chat, message_id=message.message_id)
|
||||
except Message.DoesNotExist:
|
||||
old = None
|
||||
obj = Message.objects.create(
|
||||
chat=chat,
|
||||
message_id=message.message_id,
|
||||
text=message.text.html if message.text else '',
|
||||
date=tz.localize(datetime.utcfromtimestamp(message.date)),
|
||||
edit_date=tz.localize(datetime.utcfromtimestamp(message.edit_date)) if message.edit_date else None,
|
||||
)
|
||||
if old is not None:
|
||||
old.replaced_by = obj
|
||||
old.save()
|
||||
return obj
|
||||
|
||||
def __str__(self):
|
||||
return 'id="{}" (chat_id="{}")'.format(self.message_id, self.chat.chat_id)
|
||||
|
||||
class Meta:
|
||||
ordering = 'message_id', 'edit_date',
|
||||
@ -0,0 +1,59 @@
|
||||
{% extends 'cabinet/_internal_base.html' %}
|
||||
{% load bootstrap4 %}
|
||||
|
||||
{% block breadcrumbs %}
|
||||
<li><a href="{% url 'cabinet:aggregator:index' %}">Aggregation source list</a></li>
|
||||
<li><span>{{ title }}</span></li>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<form action="" method="post" class="card">
|
||||
{% csrf_token %}
|
||||
<header class="card-header">
|
||||
<h2 class="card-title">{% if source %}Source "{{ source.title }}" configuration{% else %}New source{% endif %}</h2>
|
||||
</header>
|
||||
<div class="card-body">
|
||||
{% bootstrap_form form layout='horizontal' %}
|
||||
{% if source %}
|
||||
<div class="form-group row">
|
||||
<label class="col-md-3 col-form-label" for="id_feed-last_check">Chat ID</label>
|
||||
<div class="col-md-9">
|
||||
<input type="text" value="{{ source.chat_id }}" class="form-control" placeholder="Chat ID" title="" id="id_chat_id" disabled>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<footer class="card-footer text-right">
|
||||
{% if source %}<a href="#delete-modal" class="modal-basic btn btn-danger">Delete</a>{% endif %}
|
||||
<button type="submit" class="btn btn-primary">Save</button>
|
||||
</footer>
|
||||
</form>
|
||||
|
||||
{% if source %}
|
||||
<div id="delete-modal" class="modal-block modal-full-color modal-block-danger mfp-hide">
|
||||
<section class="card">
|
||||
<header class="card-header">
|
||||
<h2 class="card-title">Delete source "{{ source.title }}"</h2>
|
||||
</header>
|
||||
<div class="card-body">
|
||||
<div class="modal-wrapper">
|
||||
<div class="modal-icon"><i class="fas fa-times-circle"></i></div>
|
||||
<div class="modal-text">
|
||||
<h4>Are you sure?</h4>
|
||||
<p>This action cannot be undone.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<footer class="card-footer">
|
||||
<div class="row">
|
||||
<form action="{% url 'cabinet:aggregator:delete' pk=source.pk %}" method="post" class="col-md-12 text-right">
|
||||
{% csrf_token %}
|
||||
<button type="button" class="btn btn-default modal-dismiss">Cancel</button>
|
||||
<button type="submit" class="btn btn-danger">Delete</button>
|
||||
</form>
|
||||
</div>
|
||||
</footer>
|
||||
</section>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@ -0,0 +1,40 @@
|
||||
{% extends 'cabinet/_internal_base.html' %}
|
||||
|
||||
{% block breadcrumbs %}
|
||||
<li><span>Aggregator source list</span></li>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<section class="card">
|
||||
<header class="card-header">
|
||||
<div class="card-button-actions">
|
||||
<a href="{% url 'cabinet:aggregator:new' %}" class="btn btn-primary btn-sm" type="button">
|
||||
Add new source
|
||||
</a>
|
||||
</div>
|
||||
<h2 class="card-title">Your sources</h2>
|
||||
</header>
|
||||
<div class="card-body">
|
||||
<table class="table table-hover table-responsive-md mb-0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Title</th>
|
||||
<th>Chat ID</th>
|
||||
</tr>
|
||||
</thead>
|
||||
{% for source in object_list %}
|
||||
<tr class="clickable-row" data-href="{% url 'cabinet:aggregator:edit' pk=source.pk %}">
|
||||
<td>{{ source.title }}</td>
|
||||
<td>{{ source.chat_id }}</td>
|
||||
</tr>
|
||||
{% empty %}
|
||||
<tfoot>
|
||||
<tr>
|
||||
<td colspan="2">No sources added</td>
|
||||
</tr>
|
||||
</tfoot>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@ -0,0 +1,3 @@
|
||||
from django.test import TestCase
|
||||
|
||||
# Create your tests here.
|
||||
@ -0,0 +1,13 @@
|
||||
from django.urls import path
|
||||
|
||||
from .views import AggregationSourceListView, AggregationSourceCreateView, AggregationSourceUpdateView, \
|
||||
AggregationSourceDeleteView
|
||||
|
||||
app_name = 'feeds'
|
||||
urlpatterns = [
|
||||
path('', AggregationSourceListView.as_view(), name='index'),
|
||||
path('<int:pk>/', AggregationSourceUpdateView.as_view(), name='edit'),
|
||||
path('new/', AggregationSourceCreateView.as_view(), name='new'),
|
||||
path('delete/<int:pk>/', AggregationSourceDeleteView.as_view(), name='delete'),
|
||||
]
|
||||
|
||||
@ -0,0 +1,60 @@
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.mixins import LoginRequiredMixin
|
||||
from django.shortcuts import redirect
|
||||
from django.views import View
|
||||
from django.views.generic import ListView, UpdateView, CreateView
|
||||
from django.views.generic.detail import SingleObjectMixin
|
||||
|
||||
from aggregator.forms import AggregationSourceForm, AggregationSourceEditForm
|
||||
from aggregator.models import AggregationSource
|
||||
from cabinet.utils import CabinetViewMixin
|
||||
|
||||
|
||||
class AggregationSourceListView(CabinetViewMixin, ListView):
|
||||
template_name = 'cabinet/aggregator/source_list.html'
|
||||
title = 'Aggregation source list'
|
||||
sidebar_section = 'aggregator'
|
||||
|
||||
def get_queryset(self):
|
||||
return AggregationSource.objects.filter(owner=self.request.user)
|
||||
|
||||
|
||||
class AggregationSourceCreateView(CabinetViewMixin, CreateView):
|
||||
template_name = 'cabinet/aggregator/source_form.html'
|
||||
title = 'Create aggregation source'
|
||||
sidebar_section = 'aggregator'
|
||||
form_class = AggregationSourceForm
|
||||
|
||||
def form_valid(self, form):
|
||||
form.instance.owner = self.request.user
|
||||
form.instance.chat_id = form.cleaned_data['chat_id']
|
||||
form.save()
|
||||
return redirect('cabinet:aggregator:edit', pk=form.instance.pk)
|
||||
|
||||
|
||||
class AggregationSourceUpdateView(CabinetViewMixin, UpdateView):
|
||||
template_name = 'cabinet/aggregator/source_form.html'
|
||||
title = 'Configure aggregation source'
|
||||
sidebar_section = 'aggregator'
|
||||
form_class = AggregationSourceEditForm
|
||||
context_object_name = 'source'
|
||||
|
||||
def get_queryset(self):
|
||||
return AggregationSource.objects.filter(owner=self.request.user)
|
||||
|
||||
def form_valid(self, form):
|
||||
if 'chat_id' in form.cleaned_data:
|
||||
form.instance.chat_id = form.cleaned_data['chat_id']
|
||||
form.save()
|
||||
return redirect('cabinet:aggregator:edit', pk=form.instance.pk)
|
||||
|
||||
|
||||
class AggregationSourceDeleteView(LoginRequiredMixin, SingleObjectMixin, View):
|
||||
def get_queryset(self):
|
||||
return AggregationSource.objects.filter(owner=self.request.user)
|
||||
|
||||
def post(self, request, *args, **kwargs):
|
||||
source = self.get_object()
|
||||
messages.success(self.request, 'Source "{}" was successfully deleted'.format(source.title))
|
||||
source.delete()
|
||||
return redirect('cabinet:aggregator:index')
|
||||
Loading…
Reference in new issue