From 90d8c02a084f93e5bfed4ad357abd0885a32e9af Mon Sep 17 00:00:00 2001 From: bakatrouble Date: Fri, 25 Jan 2019 19:33:22 +0300 Subject: [PATCH] vk feed module --- config/settings.py | 12 ++-- feeds/forms.py | 1 + feeds/migrations/0002_vkfeedmoduleconfig.py | 25 +++++++ feeds/models.py | 4 +- feeds/modules/__init__.py | 3 +- feeds/modules/vk_feed.py | 77 +++++++++++++++++++++ feeds/tasks.py | 10 ++- feeds/utils.py | 22 +++++- requirements.txt | 6 +- 9 files changed, 147 insertions(+), 13 deletions(-) create mode 100644 feeds/migrations/0002_vkfeedmoduleconfig.py create mode 100644 feeds/modules/vk_feed.py diff --git a/config/settings.py b/config/settings.py index 471ee1c..7596002 100644 --- a/config/settings.py +++ b/config/settings.py @@ -1,7 +1,8 @@ import os import environ -import raven +import sentry_sdk +from sentry_sdk.integrations.django import DjangoIntegration BASE_DIR = environ.Path(__file__) - 2 @@ -23,7 +24,6 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', - 'raven.contrib.django.raven_compat', 'django_extensions', 'bootstrap4', 'djconfig', @@ -109,7 +109,7 @@ LOGIN_URL = 'cabinet:login' LOGIN_REDIRECT_URL = 'cabinet:index' LOGOUT_REDIRECT_URL = LOGIN_URL -RAVEN_CONFIG = { - 'dsn': env.str('SENTRY_DSN', None), - 'release': raven.fetch_git_sha(str(BASE_DIR)), -} +sentry_sdk.init( + dsn=env.str('SENTRY_DSN', None), + integrations=[DjangoIntegration()], +) diff --git a/feeds/forms.py b/feeds/forms.py index cac84b9..b4e9389 100644 --- a/feeds/forms.py +++ b/feeds/forms.py @@ -30,3 +30,4 @@ class FeedsAppConfigForm(ConfigForm): feed_bot_token = forms.CharField(required=True) vk_username = forms.CharField(required=True) vk_password = forms.CharField(required=True) + vk_auth = forms.CharField(required=False, widget=forms.Textarea()) diff --git a/feeds/migrations/0002_vkfeedmoduleconfig.py b/feeds/migrations/0002_vkfeedmoduleconfig.py new file mode 100644 index 0000000..2a5ba28 --- /dev/null +++ b/feeds/migrations/0002_vkfeedmoduleconfig.py @@ -0,0 +1,25 @@ +# Generated by Django 2.1.5 on 2019-01-25 16:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('feeds', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='VKFeedModuleConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('owner_id', models.IntegerField()), + ('send_text', models.BooleanField(default=True)), + ('send_links', models.BooleanField(default=True)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/feeds/models.py b/feeds/models.py index 0c1092d..1082836 100644 --- a/feeds/models.py +++ b/feeds/models.py @@ -28,8 +28,8 @@ class Feed(models.Model): if self.last_check and timezone.now() < self.last_check + self.check_interval: return - self.lock = True - self.save() + # self.lock = True + # self.save() execute_feed.delay(self.pk) class Meta: diff --git a/feeds/modules/__init__.py b/feeds/modules/__init__.py index db82692..cf1031a 100644 --- a/feeds/modules/__init__.py +++ b/feeds/modules/__init__.py @@ -1,3 +1,4 @@ from .echo import EchoFeedModuleConfig +from .vk_feed import VKFeedModuleConfig -FEED_MODULES = [EchoFeedModuleConfig] +FEED_MODULES = [EchoFeedModuleConfig, VKFeedModuleConfig] diff --git a/feeds/modules/vk_feed.py b/feeds/modules/vk_feed.py new file mode 100644 index 0000000..b1fe4e2 --- /dev/null +++ b/feeds/modules/vk_feed.py @@ -0,0 +1,77 @@ +import os +import tempfile + +import requests +import sentry_sdk +from django.db import models +from djconfig import config +from telebot import TeleBot +from telebot.types import InputMediaPhoto +from vk_api import VkApi +from vk_api.audio import VkAudio + +from feeds.models import FeedModuleConfig +from feeds.utils import DatabaseConfig + + +def get_vk_photo(attachment): + for size in (2560, 1280, 807, 604, 130, 75): + if f'photo_{size}' in attachment: + return attachment[f'photo_{size}'] + return None + + +def get_file(url): + fname = '?'.join(url.split('?')[:-1]) + extension = os.path.basename(fname).split('.')[-1] + f = tempfile.NamedTemporaryFile(suffix=f'.{extension}' if extension else None) + r = requests.get(url, stream=True) + for chunk in r.iter_content(1024 * 1024): + if chunk: + f.write(chunk) + f.seek(0) + return f + + +class VKFeedModuleConfig(FeedModuleConfig): + owner_id = models.IntegerField() + send_text = models.BooleanField(default=True) + send_links = models.BooleanField(default=True) + + MODULE_NAME = 'VK feed' + + def execute(self, bot: TeleBot, chat_id, last_id): + config._reload_maybe() + vk_session = VkApi(login=config.vk_username, password=config.vk_password, config=DatabaseConfig, + api_version='5.60') + vk_session.auth() + vk_audio = VkAudio(vk_session) + vk = vk_session.get_api() + wall_data = vk.wall.get(owner_id=self.owner_id) + for post in reversed(wall_data['items']): + if post['id'] > last_id: + try: + if self.send_text and post['text']: + bot.send_message(chat_id, post['text']) + if 'attachments' in post: + photos = [] + for image in filter(lambda a: 'photo' in a, post['attachments']): + url = get_vk_photo(image['photo']) + if url: + photos.append(url.replace('\\', '')) + if len(photos) == 1: + bot.send_photo(chat_id, photos[0]) + elif len(photos) > 1: + bot.send_media_group(chat_id, [InputMediaPhoto(photo) for photo in photos]) + for a in post['attachments']: + if 'audio' in a: + f = get_file(vk_audio.get_audio_by_id(a['audio']['owner_id'], a['audio']['id'])) + bot.send_audio(chat_id, f, a['audio'].get('duration'), + a['audio'].get('artist'), a['audio'].get('title')) + if self.send_links: + bot.send_message(chat_id, f"https://vk.com/wall{post['owner_id']}_{post['id']}", + disable_web_page_preview=True) + except Exception as e: + sentry_sdk.capture_exception(e) + last_id = post['id'] + return last_id diff --git a/feeds/tasks.py b/feeds/tasks.py index 1dbd576..0991e8a 100644 --- a/feeds/tasks.py +++ b/feeds/tasks.py @@ -1,3 +1,6 @@ +from pprint import pprint + +from django.conf import settings from django.utils import timezone from telebot import TeleBot from djconfig import config @@ -7,8 +10,13 @@ from config.celery import app @app.task() def execute_feed(feed_pk): - config._reload_maybe() from feeds.models import Feed + from django.db import connections + + c = connections['default'].cursor() + print(c.db.__dict__) + + # config._reload_maybe() try: feed = Feed.objects.get(pk=feed_pk) diff --git a/feeds/utils.py b/feeds/utils.py index 1e7ba70..857148b 100644 --- a/feeds/utils.py +++ b/feeds/utils.py @@ -1,7 +1,11 @@ +import yaml from django.views.generic import TemplateView +from djconfig import config +from jconfig.base import BaseConfig +from yaml.parser import ParserError from cabinet.utils import CabinetViewMixin -from feeds.forms import FeedForm, get_config_form +from feeds.forms import FeedForm, get_config_form, FeedsAppConfigForm from feeds.models import Feed @@ -36,3 +40,19 @@ class BaseFeedConfigView(CabinetViewMixin, TemplateView): ctx['feed_form'], ctx['config_form'] = self.get_forms() if forms is None else forms ctx['feed_module'] = self.get_content_type().model_class() return ctx + + +class DatabaseConfig(BaseConfig): + __slots__ = () + + def load(self, **kwargs): + config._reload_maybe() + try: + settings = yaml.load(config.vk_auth) + except (ParserError, AttributeError): + settings = {} + settings.setdefault(self.section_name, {}) + return settings + + def save(self): + FeedsAppConfigForm(data={'vk_auth': yaml.dump(self._settings)}).save() diff --git a/requirements.txt b/requirements.txt index c4d281d..aa67074 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ amqp==2.3.2 +beautifulsoup4==4.7.1 billiard==3.5.0.5 celery==4.2.1 certifi==2018.11.29 @@ -17,12 +18,13 @@ psycopg2-binary==2.7.6.1 pyTelegramBotAPI==3.6.6 pytz==2018.9 PyYAML==3.13 -raven==6.10.0 redis==3.0.1 requests==2.21.0 +sentry-sdk==0.6.9 six==1.12.0 +soupsieve==1.7.3 urllib3==1.24.1 uWSGI==2.0.17.1 vine==1.2.0 -vk-api==11.3.0 +-e git+https://github.com/python273/vk_api@442281c9392109945dd6ed85a423d2fe8d068e7d#egg=vk_api Werkzeug==0.14.1