diff --git a/_systemd/bots-bot-worker.service b/_systemd/bots-bot-worker.service new file mode 100644 index 0000000..73c1882 --- /dev/null +++ b/_systemd/bots-bot-worker.service @@ -0,0 +1,14 @@ +[Unit] +Description=bots telegram bots service +After=network.target + +[Service] +Type=simple +User=http +Group=http +WorkingDirectory=/srv/apps/bots +ExecStart=/srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py bot_worker +KillSignal=SIGINT + +[Install] +WantedBy=multi-user.target diff --git a/_systemd/bots-celeryd.service b/_systemd/bots-celeryd.service new file mode 100644 index 0000000..7e2325f --- /dev/null +++ b/_systemd/bots-celeryd.service @@ -0,0 +1,14 @@ +[Unit] +Description=bots celeryd service +After=network.target + +[Service] +Type=simple +User=http +Group=http +WorkingDirectory=/srv/apps/bots +ExecStart=/srv/apps/bots/venv/bin/celery worker --app=config -l info +KillSignal=SIGINT + +[Install] +WantedBy=multi-user.target diff --git a/_systemd/bots-periodic-bots.service b/_systemd/bots-periodic-bots.service new file mode 100644 index 0000000..d7f9714 --- /dev/null +++ b/_systemd/bots-periodic-bots.service @@ -0,0 +1,14 @@ +[Unit] +Description=bots telegram bots service +After=network.target + +[Service] +Type=simple +User=http +Group=http +WorkingDirectory=/srv/apps/bots +ExecStart=/srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py periodic_bots +KillSignal=SIGINT + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/_systemd/bots-periodic-feeds.service b/_systemd/bots-periodic-feeds.service new file mode 100644 index 0000000..bda87a4 --- /dev/null +++ b/_systemd/bots-periodic-feeds.service @@ -0,0 +1,14 @@ +[Unit] +Description=bots telegram bots service +After=network.target + +[Service] +Type=simple +User=http +Group=http +WorkingDirectory=/srv/apps/bots +ExecStart=/srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py periodic_feeds +KillSignal=SIGINT + +[Install] +WantedBy=multi-user.target diff --git a/_systemd/bots.service b/_systemd/bots.service new file mode 100644 index 0000000..5f8a88d --- /dev/null +++ b/_systemd/bots.service @@ -0,0 +1,19 @@ +[Unit] +Description=bots main service +After=network.target +Requires=bots-celeryd.service +Requires=bots-bot-worker.service +Requires=bots-periodic-feeds.service +Requires=bots-periodic-bots.service +# Requires=bots-aggregator-client.service + +[Service] +Type=simple +User=http +Group=http +WorkingDirectory=/srv/apps/bots +ExecStart=/srv/apps/bots/venv/bin/daphne -u /tmp/bots.sock config.asgi:application +KillSignal=SIGINT + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/bots/management/commands/run_bots.py b/bots/management/commands/bot_worker.py similarity index 100% rename from bots/management/commands/run_bots.py rename to bots/management/commands/bot_worker.py diff --git a/bots/utils.py b/bots/utils.py index d4a4248..89e88b9 100644 --- a/bots/utils.py +++ b/bots/utils.py @@ -1,3 +1,9 @@ +import json + +from django.core import serializers +from django.core.exceptions import ImproperlyConfigured +from django.core.serializers.json import DjangoJSONEncoder +from django.http import HttpResponse from django.views.generic import TemplateView from bots.forms import BotForm @@ -38,4 +44,83 @@ class BaseBotConfigView(CabinetViewMixin, TemplateView): ctx = super(BaseBotConfigView, self).get_context_data(**kwargs) ctx['bot_form'], ctx['config_form'] = self.get_forms() if forms is None else forms ctx['bot_module'] = self.get_content_type().model_class() - return ctx \ No newline at end of file + return ctx + + +# django-braces +class AjaxResponseMixin(object): + """ + Mixin allows you to define alternative methods for ajax requests. Similar + to the normal get, post, and put methods, you can use get_ajax, post_ajax, + and put_ajax. + """ + def dispatch(self, request, *args, **kwargs): + request_method = request.method.lower() + + if request.is_ajax() and request_method in self.http_method_names: + handler = getattr(self, "{0}_ajax".format(request_method), + self.http_method_not_allowed) + self.request = request + self.args = args + self.kwargs = kwargs + return handler(request, *args, **kwargs) + + return super(AjaxResponseMixin, self).dispatch( + request, *args, **kwargs) + + def get_ajax(self, request, *args, **kwargs): + return self.get(request, *args, **kwargs) + + def post_ajax(self, request, *args, **kwargs): + return self.post(request, *args, **kwargs) + + def put_ajax(self, request, *args, **kwargs): + return self.get(request, *args, **kwargs) + + def delete_ajax(self, request, *args, **kwargs): + return self.get(request, *args, **kwargs) + + +class JSONResponseMixin(object): + """ + A mixin that allows you to easily serialize simple data such as a dict or + Django models. + """ + content_type = None + json_dumps_kwargs = None + json_encoder_class = DjangoJSONEncoder + + def get_content_type(self): + if self.content_type is not None and not isinstance(self.content_type, str): + raise ImproperlyConfigured( + '{0} is missing a content type. Define {0}.content_type, ' + 'or override {0}.get_content_type().'.format( + self.__class__.__name__)) + return self.content_type or "application/json" + + def get_json_dumps_kwargs(self): + if self.json_dumps_kwargs is None: + self.json_dumps_kwargs = {} + self.json_dumps_kwargs.setdefault('ensure_ascii', False) + return self.json_dumps_kwargs + + def render_json_response(self, context_dict, status=200): + """ + Limited serialization for shipping plain data. Do not use for models + or other complex or custom objects. + """ + json_context = json.dumps( + context_dict, + cls=self.json_encoder_class, + **self.get_json_dumps_kwargs()).encode('utf-8') + return HttpResponse(json_context, + content_type=self.get_content_type(), + status=status) + + def render_json_object_response(self, objects, **kwargs): + """ + Serializes objects using Django's builtin JSON serializer. Additional + kwargs can be used the same way for django.core.serializers.serialize. + """ + json_data = serializers.serialize("json", objects, **kwargs) + return HttpResponse(json_data, content_type=self.get_content_type()) diff --git a/bots/views.py b/bots/views.py index fa3206a..e546fc0 100644 --- a/bots/views.py +++ b/bots/views.py @@ -1,4 +1,3 @@ -from braces.views import AjaxResponseMixin, JSONResponseMixin from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin from django.contrib.contenttypes.models import ContentType @@ -15,7 +14,7 @@ from jsonrpc import JSONRPCResponseManager from bots.models import TelegramBot from bots.modules import BOT_MODULES -from bots.utils import BaseBotConfigView +from bots.utils import BaseBotConfigView, JSONResponseMixin, AjaxResponseMixin from cabinet.utils import CabinetViewMixin from config.utils import AllowCORSMixin diff --git a/cabinet/templates/cabinet/_base.html b/cabinet/templates/cabinet/_base.html index 2a13771..53889da 100644 --- a/cabinet/templates/cabinet/_base.html +++ b/cabinet/templates/cabinet/_base.html @@ -1,4 +1,4 @@ -{% load staticfiles cabinet %} +{% load static cabinet %} diff --git a/cabinet/templates/cabinet/_includes/header.html b/cabinet/templates/cabinet/_includes/header.html index 8f3e672..602e029 100644 --- a/cabinet/templates/cabinet/_includes/header.html +++ b/cabinet/templates/cabinet/_includes/header.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %}
diff --git a/cabinet/templates/cabinet/login.html b/cabinet/templates/cabinet/login.html index 41e58ca..9f002c2 100644 --- a/cabinet/templates/cabinet/login.html +++ b/cabinet/templates/cabinet/login.html @@ -1,5 +1,5 @@ {% extends 'cabinet/_base.html' %} -{% load staticfiles bootstrap4 %} +{% load static bootstrap4 %} {% block page %}
diff --git a/config/asgi.py b/config/asgi.py new file mode 100644 index 0000000..8cf2d26 --- /dev/null +++ b/config/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for config.settings project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0-rc1/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') + +application = get_asgi_application() diff --git a/config/celery.py b/config/celery.py index 2fe282b..9c912f4 100644 --- a/config/celery.py +++ b/config/celery.py @@ -8,8 +8,6 @@ app = Celery('telegram_bots') app.config_from_object({ 'broker_url': 'redis://127.0.0.1:6379/4', - 'result_backend': 'django-db', - 'beat_scheduler': 'django_celery_beat.schedulers:DatabaseScheduler', 'worker_concurrency': 4, 'ONCE': { 'backend': 'celery_once.backends.Redis', diff --git a/config/settings.py b/config/settings.py index 862cf28..4cc2897 100644 --- a/config/settings.py +++ b/config/settings.py @@ -21,8 +21,6 @@ INSTALLED_APPS = [ 'bootstrap4', 'crispy_forms', 'djconfig', - 'django_celery_results', - 'django_celery_beat', 'django.contrib.admin', 'django.contrib.auth', @@ -93,10 +91,18 @@ STATICFILES_FINDERS = [ CACHES = { "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://127.0.0.1:6379/4", + "BACKEND": "redis_cache.RedisCache", + "LOCATION": ["127.0.0.1:6379"], "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", + 'DB': 4, + 'PARSER_CLASS': 'redis.connection.HiredisParser', + 'CONNECTION_POOL_CLASS': 'redis.BlockingConnectionPool', + 'CONNECTION_POOL_CLASS_KWARGS': { + 'max_connections': 50, + 'timeout': 20, + }, + 'MAX_CONNECTIONS': 1000, + 'PICKLE_VERSION': -1, } } } diff --git a/feeds/management/__init__.py b/feeds/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/feeds/management/commands/__init__.py b/feeds/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nginx.conf b/nginx.conf index 4566990..5e0f3e4 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,20 +1,51 @@ server { - listen 80; - server_name bots.bakatrouble.pw; + server_name ~^bots\.bakatrouble\.(pw|me)$; - access_log /srv/apps/bots/logs/nginx.access; - error_log /srv/apps/bots/logs/nginx.error; + access_log /srv/apps/bots/logs/nginx.access; + error_log /srv/apps/bots/logs/nginx.error; - location /static { - alias /srv/apps/bots/public/static; - } + location /static { + alias /srv/apps/bots/public/static; + } - location /uploads { - alias /srv/apps/bots/public/uploads; - } + location /uploads { + alias /srv/apps/bots/public/uploads; + } - location / { - include uwsgi_params; - uwsgi_pass unix:///tmp/bots.sock; - } + location / { + proxy_pass unix:///tmp/bots.sock; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + } + + listen [::]:443 http2 ssl ipv6only=on; + listen 443 http2 ssl; + ssl_certificate /etc/letsencrypt/live/bakatrouble.pw/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/bakatrouble.pw/privkey.pem; + # include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5; + + + if ($scheme != "https") { + return 301 https://$host$request_uri; + } +} + +server { + if ($host ~ ^bots\.bakatrouble\.(pw|me)$) { + return 301 https://$host$request_uri; + } + + listen [::]:80 ipv6only=on; + listen 80; + server_name ~^bots\.bakatrouble\.(pw|me)$; + return 404; } diff --git a/requirements.txt b/requirements.txt index a377e5e..711b53a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,10 @@ amqp==2.3.2 +asgiref==3.2.3 asn1crypto==0.24.0 async-generator==1.10 +attrs==19.3.0 +autobahn==19.11.1 +Automat==0.8.0 beautifulsoup4==4.7.1 billiard==3.5.0.5 bs4==0.0.1 @@ -9,31 +13,37 @@ celery-once==2.0.0 certifi==2018.11.29 cffi==1.12.3 chardet==3.0.4 +constantly==15.1.0 cryptography==2.7 -Django==2.1.5 -django-bootstrap4==0.0.7 -django-braces==1.13.0 -django-celery-beat==1.4.0 -django-celery-results==1.0.4 -django-crispy-forms==1.7.2 +daphne==2.4.0 +Django==3.0rc1 +django-bootstrap4==1.0.1 +django-crispy-forms==1.8.1 django-djconfig==0.9.0 django-environ==0.4.5 django-extensions==2.1.4 django-jet==1.0.8 -django-redis==4.10.0 +django-redis-cache==2.1.0 django-timezone-field==3.0 django-yamlfield==1.0.3 enum34==1.1.6 feedparser==5.2.1 future==0.17.1 +hiredis==1.0.1 +hyperlink==19.0.0 idna==2.8 +incremental==17.5.0 json-rpc==1.12.1 kombu==4.2.2.post1 oauthlib==3.0.1 Pillow==5.4.1 psycopg2-binary==2.7.6.1 pyaes==1.6.1 +pyasn1==0.4.8 +pyasn1-modules==0.2.7 pycparser==2.19 +PyHamcrest==1.9.0 +pyOpenSSL==19.1.0 Pyrogram==0.11.0 PySocks==1.6.8 pyTelegramBotAPI==3.6.6 @@ -48,12 +58,16 @@ redis==3.0.1 requests==2.21.0 requests-oauthlib==1.2.0 sentry-sdk==0.6.9 +service-identity==18.1.0 six==1.12.0 soupsieve==1.7.3 +sqlparse==0.3.0 TgCrypto==1.1.1 tornado==6.0.3 +Twisted==19.10.0 +txaio==18.8.1 urllib3==1.24.1 -uWSGI==2.0.17.1 vine==1.2.0 vk-api==11.6.1 Werkzeug==0.14.1 +zope.interface==4.7.1 diff --git a/supervisor.conf b/supervisor.conf deleted file mode 100644 index c3a677f..0000000 --- a/supervisor.conf +++ /dev/null @@ -1,44 +0,0 @@ -[group:bots] -programs = bots_web,bots_aggregator_client,bots_celeryd,bots_celerybeat - -[program:bots_web] -user = http -directory = /srv/apps/bots -command = /srv/apps/bots/venv/bin/uwsgi --yaml /srv/apps/bots/uwsgi.yml -autostart = true -autorestart = true -stderr_logfile = /srv/apps/bots/logs/uwsgi.log -stdout_logfile = /srv/apps/bots/logs/uwsgi.log -stopsignal = INT - -[program:bots_aggregator_client] -user = http -directory = /srv/apps/bots -command = /srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py start_aggregator_client -autostart = true -autorestart = true -stderr_logfile = /srv/apps/bots/logs/aggregator_client.log -stdout_logfile = /srv/apps/bots/logs/aggregator_client.log -stopsignal = INT - -[program:bots_celeryd] -user = http -directory = /srv/apps/bots -command=/srv/apps/bots/venv/bin/celery worker --app=config -l info -stdout_logfile=/srv/apps/bots/logs/celeryd.log -stderr_logfile=/srv/apps/bots/logs/celeryd.log -autostart=true -autorestart=true -startsecs=10 -stopwaitsecs=600 - -[program:bots_celerybeat] -user = http -directory = /srv/apps/bots -command=/srv/apps/bots/venv/bin/celery beat --app=config -l info -stdout_logfile=/srv/apps/bots/logs/celerybeat.log -stderr_logfile=/srv/apps/bots/logs/celerybeat.log -autostart=true -autorestart=true -startsecs=10 -stopwaitsecs=600 diff --git a/uwsgi.yml b/uwsgi.yml deleted file mode 100644 index b60e0db..0000000 --- a/uwsgi.yml +++ /dev/null @@ -1,7 +0,0 @@ -uwsgi: - socket: /tmp/bots.sock - module: config.wsgi:application - chmod-socket: 666 - master: true - processes: 4 - enable-threads: true