From ec01ab879f338f5e26ffa9b23b1053841c68225b Mon Sep 17 00:00:00 2001 From: bakatrouble Date: Sat, 26 Jan 2019 12:36:44 +0300 Subject: [PATCH] use celerybeat --- .../migrations}/__init__.py | 0 config/celery.py | 10 ++++++++++ config/settings.py | 4 +++- feeds/management/commands/__init__.py | 0 feeds/management/commands/worker_loop.py | 17 ----------------- feeds/tasks.py | 12 +++++++++--- requirements.txt | 6 ++++++ supervisor.conf | 8 ++++---- 8 files changed, 32 insertions(+), 25 deletions(-) rename {feeds/management => aggregator/migrations}/__init__.py (100%) delete mode 100644 feeds/management/commands/__init__.py delete mode 100644 feeds/management/commands/worker_loop.py diff --git a/feeds/management/__init__.py b/aggregator/migrations/__init__.py similarity index 100% rename from feeds/management/__init__.py rename to aggregator/migrations/__init__.py diff --git a/config/celery.py b/config/celery.py index 21e0053..ec54c60 100644 --- a/config/celery.py +++ b/config/celery.py @@ -8,6 +8,16 @@ app = Celery('telegram_bots') app.config_from_object({ 'broker_url': 'redis://127.0.0.1:6379/1', + 'result_backend': 'django-db', + 'beat_scheduler': 'django_celery_beat.schedulers:DatabaseScheduler', + 'worker_concurrency': 4, + 'ONCE': { + 'backend': 'celery_once.backends.Redis', + 'settings': { + 'url': 'redis://localhost:6379/0', + 'default_timeout': 60 * 60, + } + } }) app.autodiscover_tasks() diff --git a/config/settings.py b/config/settings.py index dd45887..47e8a2c 100644 --- a/config/settings.py +++ b/config/settings.py @@ -20,6 +20,8 @@ INSTALLED_APPS = [ 'django_extensions', 'bootstrap4', 'djconfig', + 'django_celery_results', + 'django_celery_beat', 'config.suit_config.SuitConfig', 'django.contrib.admin', @@ -73,7 +75,7 @@ DATABASES = { DATABASES['default']['ATOMIC_REQUESTS'] = True LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +TIME_ZONE = 'Europe/Moscow' USE_I18N = True USE_L10N = True USE_TZ = True diff --git a/feeds/management/commands/__init__.py b/feeds/management/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/feeds/management/commands/worker_loop.py b/feeds/management/commands/worker_loop.py deleted file mode 100644 index b45dbf7..0000000 --- a/feeds/management/commands/worker_loop.py +++ /dev/null @@ -1,17 +0,0 @@ -from time import sleep - -from django.core.management import BaseCommand - -from feeds.models import Feed - - -class Command(BaseCommand): - def handle(self, *args, **options): - try: - while True: - feeds = Feed.objects.filter(lock=False) - for feed in feeds: - feed.run_check() - sleep(5) - except KeyboardInterrupt: - pass diff --git a/feeds/tasks.py b/feeds/tasks.py index 88d92a2..58cdf8a 100644 --- a/feeds/tasks.py +++ b/feeds/tasks.py @@ -1,11 +1,17 @@ -from pprint import pprint - -from django.conf import settings +from celery_once import QueueOnce from django.utils import timezone from telebot import TeleBot from djconfig import config from config.celery import app +from feeds.models import Feed + + +@app.task(base=QueueOnce) +def check_feeds(): + feeds = Feed.objects.filter(lock=False) + for feed in feeds: + feed.run_check() @app.task() diff --git a/requirements.txt b/requirements.txt index e4669c9..36ce01f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,15 +2,19 @@ amqp==2.3.2 beautifulsoup4==4.7.1 billiard==3.5.0.5 celery==4.2.1 +celery-once==2.0.0 certifi==2018.11.29 chardet==3.0.4 Django==2.1.5 django-bootstrap4==0.0.7 +django-celery-beat==1.4.0 +django-celery-results==1.0.4 django-djconfig==0.9.0 django-environ==0.4.5 django-extensions==2.1.4 django-redis==4.10.0 -e git+https://github.com/darklow/django-suit@9941211bc7d63d94478a3c6479c51926dc75db09#egg=django_suit +django-timezone-field==3.0 django-yamlfield==1.0.3 enum34==1.1.6 idna==2.8 @@ -18,6 +22,8 @@ kombu==4.2.2.post1 psycopg2-binary==2.7.6.1 PySocks==1.6.8 pyTelegramBotAPI==3.6.6 +python-crontab==2.3.6 +python-dateutil==2.7.5 pytz==2018.9 PyYAML==3.13 redis==3.0.1 diff --git a/supervisor.conf b/supervisor.conf index 6b38a4c..fb4789f 100644 --- a/supervisor.conf +++ b/supervisor.conf @@ -22,12 +22,12 @@ autorestart=true startsecs=10 stopwaitsecs=600 -[program:bots_worker] +[program:bots_celerybeat] user = http directory = /srv/apps/bots -command=/srv/apps/bots/venv/bin/python /srv/apps/bots/manage.py worker_loop -stdout_logfile=/srv/apps/bots/logs/worker.log -stderr_logfile=/srv/apps/bots/logs/worker.log +command=/srv/apps/bots/venv/bin/celery beat --app=config -l info +stdout_logfile=/srv/apps/bots/logs/celerybeat.log +stderr_logfile=/srv/apps/bots/logs/celerybeat.log autostart=true autorestart=true startsecs=10