update to django 3.0 & asgi

_systemd/bots-bot-worker.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=bots telegram bots service
+After=network.target
+
+[Service]
+Type=simple
+User=http
+Group=http
+WorkingDirectory=/srv/apps/bots
+ExecStart=/srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py bot_worker
+KillSignal=SIGINT
+
+[Install]
+WantedBy=multi-user.target

_systemd/bots-celeryd.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=bots celeryd service
+After=network.target
+
+[Service]
+Type=simple
+User=http
+Group=http
+WorkingDirectory=/srv/apps/bots
+ExecStart=/srv/apps/bots/venv/bin/celery worker --app=config -l info
+KillSignal=SIGINT
+
+[Install]
+WantedBy=multi-user.target

_systemd/bots-periodic-bots.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=bots telegram bots service
+After=network.target
+
+[Service]
+Type=simple
+User=http
+Group=http
+WorkingDirectory=/srv/apps/bots
+ExecStart=/srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py periodic_bots
+KillSignal=SIGINT
+
+[Install]
+WantedBy=multi-user.target

_systemd/bots-periodic-feeds.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=bots telegram bots service
+After=network.target
+
+[Service]
+Type=simple
+User=http
+Group=http
+WorkingDirectory=/srv/apps/bots
+ExecStart=/srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py periodic_feeds
+KillSignal=SIGINT
+
+[Install]
+WantedBy=multi-user.target

_systemd/bots.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=bots main service
+After=network.target
+Requires=bots-celeryd.service
+Requires=bots-bot-worker.service
+Requires=bots-periodic-feeds.service
+Requires=bots-periodic-bots.service
+# Requires=bots-aggregator-client.service
+
+[Service]
+Type=simple
+User=http
+Group=http
+WorkingDirectory=/srv/apps/bots
+ExecStart=/srv/apps/bots/venv/bin/daphne -u /tmp/bots.sock config.asgi:application
+KillSignal=SIGINT
+
+[Install]
+WantedBy=multi-user.target

bots/utils.py

@@ -1,3 +1,9 @@
+import json
+
+from django.core import serializers
+from django.core.exceptions import ImproperlyConfigured
+from django.core.serializers.json import DjangoJSONEncoder
+from django.http import HttpResponse
 from django.views.generic import TemplateView
 
 from bots.forms import BotForm
@@ -39,3 +45,82 @@ class BaseBotConfigView(CabinetViewMixin, TemplateView):
         ctx['bot_form'], ctx['config_form'] = self.get_forms() if forms is None else forms
         ctx['bot_module'] = self.get_content_type().model_class()
         return ctx
+
+
+# django-braces
+class AjaxResponseMixin(object):
+    """
+    Mixin allows you to define alternative methods for ajax requests. Similar
+    to the normal get, post, and put methods, you can use get_ajax, post_ajax,
+    and put_ajax.
+    """
+    def dispatch(self, request, *args, **kwargs):
+        request_method = request.method.lower()
+
+        if request.is_ajax() and request_method in self.http_method_names:
+            handler = getattr(self, "{0}_ajax".format(request_method),
+                              self.http_method_not_allowed)
+            self.request = request
+            self.args = args
+            self.kwargs = kwargs
+            return handler(request, *args, **kwargs)
+
+        return super(AjaxResponseMixin, self).dispatch(
+            request, *args, **kwargs)
+
+    def get_ajax(self, request, *args, **kwargs):
+        return self.get(request, *args, **kwargs)
+
+    def post_ajax(self, request, *args, **kwargs):
+        return self.post(request, *args, **kwargs)
+
+    def put_ajax(self, request, *args, **kwargs):
+        return self.get(request, *args, **kwargs)
+
+    def delete_ajax(self, request, *args, **kwargs):
+        return self.get(request, *args, **kwargs)
+
+
+class JSONResponseMixin(object):
+    """
+    A mixin that allows you to easily serialize simple data such as a dict or
+    Django models.
+    """
+    content_type = None
+    json_dumps_kwargs = None
+    json_encoder_class = DjangoJSONEncoder
+
+    def get_content_type(self):
+        if self.content_type is not None and not isinstance(self.content_type, str):
+            raise ImproperlyConfigured(
+                '{0} is missing a content type. Define {0}.content_type, '
+                'or override {0}.get_content_type().'.format(
+                    self.__class__.__name__))
+        return self.content_type or "application/json"
+
+    def get_json_dumps_kwargs(self):
+        if self.json_dumps_kwargs is None:
+            self.json_dumps_kwargs = {}
+        self.json_dumps_kwargs.setdefault('ensure_ascii', False)
+        return self.json_dumps_kwargs
+
+    def render_json_response(self, context_dict, status=200):
+        """
+        Limited serialization for shipping plain data. Do not use for models
+        or other complex or custom objects.
+        """
+        json_context = json.dumps(
+            context_dict,
+            cls=self.json_encoder_class,
+            **self.get_json_dumps_kwargs()).encode('utf-8')
+        return HttpResponse(json_context,
+                            content_type=self.get_content_type(),
+                            status=status)
+
+    def render_json_object_response(self, objects, **kwargs):
+        """
+        Serializes objects using Django's builtin JSON serializer. Additional
+        kwargs can be used the same way for django.core.serializers.serialize.
+        """
+        json_data = serializers.serialize("json", objects, **kwargs)
+        return HttpResponse(json_data, content_type=self.get_content_type())

bots/views.py

@@ -1,4 +1,3 @@
-from braces.views import AjaxResponseMixin, JSONResponseMixin
 from django.contrib import messages
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.contenttypes.models import ContentType
@@ -15,7 +14,7 @@ from jsonrpc import JSONRPCResponseManager
 
 from bots.models import TelegramBot
 from bots.modules import BOT_MODULES
-from bots.utils import BaseBotConfigView
+from bots.utils import BaseBotConfigView, JSONResponseMixin, AjaxResponseMixin
 from cabinet.utils import CabinetViewMixin
 from config.utils import AllowCORSMixin
 

cabinet/templates/cabinet/_base.html

@@ -1,4 +1,4 @@
-{% load staticfiles cabinet %}
+{% load static cabinet %}
 
 <!DOCTYPE html>
 <html class="fixed dark" lang="en">

cabinet/templates/cabinet/_includes/header.html

@@ -1,4 +1,4 @@
-{% load staticfiles %}
+{% load static %}
 
 <header class="header">
     <div class="logo-container">

cabinet/templates/cabinet/login.html

@@ -1,5 +1,5 @@
 {% extends 'cabinet/_base.html' %}
-{% load staticfiles bootstrap4 %}
+{% load static bootstrap4 %}
 
 {% block page %}
     <section class="body-sign">

config/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for config.settings project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0-rc1/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')
+
+application = get_asgi_application()

config/celery.py

@@ -8,8 +8,6 @@ app = Celery('telegram_bots')
 
 app.config_from_object({
     'broker_url': 'redis://127.0.0.1:6379/4',
-    'result_backend': 'django-db',
-    'beat_scheduler': 'django_celery_beat.schedulers:DatabaseScheduler',
     'worker_concurrency': 4,
     'ONCE': {
         'backend': 'celery_once.backends.Redis',

config/settings.py

@@ -21,8 +21,6 @@ INSTALLED_APPS = [
     'bootstrap4',
     'crispy_forms',
     'djconfig',
-    'django_celery_results',
-    'django_celery_beat',
 
     'django.contrib.admin',
     'django.contrib.auth',
@@ -93,10 +91,18 @@ STATICFILES_FINDERS = [
 
 CACHES = {
     "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379/4",
+        "BACKEND": "redis_cache.RedisCache",
+        "LOCATION": ["127.0.0.1:6379"],
         "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            'DB': 4,
+            'PARSER_CLASS': 'redis.connection.HiredisParser',
+            'CONNECTION_POOL_CLASS': 'redis.BlockingConnectionPool',
+            'CONNECTION_POOL_CLASS_KWARGS': {
+                'max_connections': 50,
+                'timeout': 20,
+            },
+            'MAX_CONNECTIONS': 1000,
+            'PICKLE_VERSION': -1,
         }
     }
 }

nginx.conf

@@ -1,20 +1,51 @@
 server {
-  listen                80;
-  server_name           bots.bakatrouble.pw;
+    server_name             ~^bots\.bakatrouble\.(pw|me)$;
 
-  access_log            /srv/apps/bots/logs/nginx.access;
-  error_log             /srv/apps/bots/logs/nginx.error;
+    access_log              /srv/apps/bots/logs/nginx.access;
+    error_log               /srv/apps/bots/logs/nginx.error;
 
-  location /static {
-    alias               /srv/apps/bots/public/static;
-  }
+    location /static {
+        alias               /srv/apps/bots/public/static;
+    }
 
-  location /uploads {
-    alias               /srv/apps/bots/public/uploads;
-  }
+    location /uploads {
+        alias               /srv/apps/bots/public/uploads;
+    }
 
-  location / {
-    include             uwsgi_params;
-    uwsgi_pass          unix:///tmp/bots.sock;
-  }
+    location / {
+        proxy_pass          unix:///tmp/bots.sock;
+
+        proxy_http_version  1.1;
+        proxy_set_header    Upgrade $http_upgrade;
+        proxy_set_header    Connection "upgrade";
+
+        proxy_redirect      off;
+        proxy_set_header    Host $host;
+        proxy_set_header    X-Real-IP $remote_addr;
+        proxy_set_header    X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header    X-Forwarded-Host $server_name;
+    }
+
+    listen [::]:443 http2 ssl ipv6only=on;
+    listen 443 http2 ssl;
+    ssl_certificate         /etc/letsencrypt/live/bakatrouble.pw/fullchain.pem;
+    ssl_certificate_key     /etc/letsencrypt/live/bakatrouble.pw/privkey.pem;
+    # include                 /etc/letsencrypt/options-ssl-nginx.conf;
+    ssl_ciphers             EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
+
+
+    if ($scheme != "https") {
+        return 301 https://$host$request_uri;
+    }
+}
+
+server {
+    if ($host ~ ^bots\.bakatrouble\.(pw|me)$) {
+        return 301 https://$host$request_uri;
+    }
+
+    listen [::]:80 ipv6only=on;
+    listen 80;
+    server_name             ~^bots\.bakatrouble\.(pw|me)$;
+    return 404;
 }

requirements.txt

@@ -1,6 +1,10 @@
 amqp==2.3.2
+asgiref==3.2.3
 asn1crypto==0.24.0
 async-generator==1.10
+attrs==19.3.0
+autobahn==19.11.1
+Automat==0.8.0
 beautifulsoup4==4.7.1
 billiard==3.5.0.5
 bs4==0.0.1
@@ -9,31 +13,37 @@ celery-once==2.0.0
 certifi==2018.11.29
 cffi==1.12.3
 chardet==3.0.4
+constantly==15.1.0
 cryptography==2.7
-Django==2.1.5
-django-bootstrap4==0.0.7
-django-braces==1.13.0
-django-celery-beat==1.4.0
-django-celery-results==1.0.4
-django-crispy-forms==1.7.2
+daphne==2.4.0
+Django==3.0rc1
+django-bootstrap4==1.0.1
+django-crispy-forms==1.8.1
 django-djconfig==0.9.0
 django-environ==0.4.5
 django-extensions==2.1.4
 django-jet==1.0.8
-django-redis==4.10.0
+django-redis-cache==2.1.0
 django-timezone-field==3.0
 django-yamlfield==1.0.3
 enum34==1.1.6
 feedparser==5.2.1
 future==0.17.1
+hiredis==1.0.1
+hyperlink==19.0.0
 idna==2.8
+incremental==17.5.0
 json-rpc==1.12.1
 kombu==4.2.2.post1
 oauthlib==3.0.1
 Pillow==5.4.1
 psycopg2-binary==2.7.6.1
 pyaes==1.6.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.7
 pycparser==2.19
+PyHamcrest==1.9.0
+pyOpenSSL==19.1.0
 Pyrogram==0.11.0
 PySocks==1.6.8
 pyTelegramBotAPI==3.6.6
@@ -48,12 +58,16 @@ redis==3.0.1
 requests==2.21.0
 requests-oauthlib==1.2.0
 sentry-sdk==0.6.9
+service-identity==18.1.0
 six==1.12.0
 soupsieve==1.7.3
+sqlparse==0.3.0
 TgCrypto==1.1.1
 tornado==6.0.3
+Twisted==19.10.0
+txaio==18.8.1
 urllib3==1.24.1
-uWSGI==2.0.17.1
 vine==1.2.0
 vk-api==11.6.1
 Werkzeug==0.14.1
+zope.interface==4.7.1

supervisor.conf

@@ -1,44 +0,0 @@
-[group:bots]
-programs = bots_web,bots_aggregator_client,bots_celeryd,bots_celerybeat
-
-[program:bots_web]
-user = http
-directory = /srv/apps/bots
-command = /srv/apps/bots/venv/bin/uwsgi --yaml /srv/apps/bots/uwsgi.yml
-autostart = true
-autorestart = true
-stderr_logfile = /srv/apps/bots/logs/uwsgi.log
-stdout_logfile = /srv/apps/bots/logs/uwsgi.log
-stopsignal = INT
-
-[program:bots_aggregator_client]
-user = http
-directory = /srv/apps/bots
-command = /srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py start_aggregator_client
-autostart = true
-autorestart = true
-stderr_logfile = /srv/apps/bots/logs/aggregator_client.log
-stdout_logfile = /srv/apps/bots/logs/aggregator_client.log
-stopsignal = INT
-
-[program:bots_celeryd]
-user = http
-directory = /srv/apps/bots
-command=/srv/apps/bots/venv/bin/celery worker --app=config -l info
-stdout_logfile=/srv/apps/bots/logs/celeryd.log
-stderr_logfile=/srv/apps/bots/logs/celeryd.log
-autostart=true
-autorestart=true
-startsecs=10
-stopwaitsecs=600
-
-[program:bots_celerybeat]
-user = http
-directory = /srv/apps/bots
-command=/srv/apps/bots/venv/bin/celery beat --app=config -l info
-stdout_logfile=/srv/apps/bots/logs/celerybeat.log
-stderr_logfile=/srv/apps/bots/logs/celerybeat.log
-autostart=true
-autorestart=true
-startsecs=10
-stopwaitsecs=600

uwsgi.yml

@@ -1,7 +0,0 @@
-uwsgi:
-  socket: /tmp/bots.sock
-  module: config.wsgi:application
-  chmod-socket: 666
-  master: true
-  processes: 4
-  enable-threads: true