bakatrouble/e621_bot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

encode videos to mp4 and upload to s3

Browse Source
This commit is contained in:
bakatrouble 2023-06-01 00:32:11 +03:00
parent d10f9a52be
commit 7384497a02
4 changed files with 170 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
example.env
View File

@ -1,2 +1,6 @@
BOT_TOKEN=450146961:asd
SEND_CHAT=9893249151
USERS=9893249151
AWS_ACCESS_KEY=AKIAUIXZQT
AWS_SECRET_KEY=QyBnXOhmlc
AWS_S3_BUCKET=bucket

36
main.py
View File

@ -2,15 +2,20 @@ import asyncio
import logging
import os
from io import BytesIO
from pathlib import Path
from tempfile import TemporaryFile, TemporaryDirectory
from time import time
from typing import List
import boto3
import ffmpeg
from PIL import Image
import httpx
import redis.asyncio as aioredis
from aiogram import Bot, Dispatcher
from aiogram.dispatcher import filters
from aiogram.types import Message, ParseMode, ChatActions
from aiogram.types import Message, ParseMode, ChatActions, InputFile
from aiogram.utils import executor, exceptions
import dotenv
@ -50,18 +55,24 @@ async def send_post(post: E621Post, tag_list: List[str]):
file.name = f'file.{post.file.ext}'
file.seek(0)
if post.file.ext == 'webm':
if post.file.size < 50_000_000:
await bot.send_video(int(os.environ['SEND_CHAT']),
file,
width=post.file.width,
height=post.file.height,
thumb=post.preview.url,
caption=caption,
parse_mode=ParseMode.HTML)
else:
with TemporaryDirectory() as td:
webm_path = Path(td) / 'video.webm'
mp4_path = Path(td) / 'video.mp4'
with open(webm_path, 'wb') as webm:
webm.write(file.read())
ffmpeg\
.input(str(webm_path))\
.output(str(mp4_path), vcodec='libx264', crf='26')\
.run()
s3 = boto3.client('s3', aws_access_key_id=os.environ['AWS_ACCESS_KEY'], aws_secret_access_key=os.environ['AWS_SECRET_KEY'])
bucket = os.environ['AWS_S3_BUCKET']
upload_filename = f'e621-{post.id}-{int(time())}.mp4'
s3.upload_file(mp4_path, bucket, upload_filename, ExtraArgs={'ACL': 'public-read', 'ContentType': 'video/mp4'})
await bot.send_message(int(os.environ['SEND_CHAT']),
f'File is too large: {post.file.url}\n\n' + caption,
f'https://{bucket}.s3.amazonaws.com/{upload_filename}\n\n' + caption,
parse_mode=ParseMode.HTML)
webm_path.unlink()
mp4_path.unlink()
elif post.file.ext == 'gif':
await bot.send_animation(int(os.environ['SEND_CHAT']),
file,
@ -180,7 +191,8 @@ async def test(msg: Message):
if not args:
await msg.reply('Please provide post id')
return
post = await e621.get_posts(f'id:{args[0]}')
post = await e621.get_posts(f'id:{args}')
print(f'id:{args}')
if not post:
await msg.reply('Post not found')
return

177
poetry.lock generated
View File

@ -1,10 +1,9 @@
# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand.
# This file is automatically @generated by Poetry 1.5.0 and should not be changed by hand.
[[package]]
name = "aiogram"
version = "2.25.1"
description = "Is a pretty simple and fully asynchronous framework for Telegram Bot API"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -26,7 +25,6 @@ proxy = ["aiohttp-socks (>=0.5.3,<0.6.0)"]
name = "aiohttp"
version = "3.8.4"
description = "Async http client/server framework (asyncio)"
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -135,7 +133,6 @@ speedups = ["Brotli", "aiodns", "cchardet"]
name = "aiosignal"
version = "1.3.1"
description = "aiosignal: a list of registered asynchronous callbacks"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -150,7 +147,6 @@ frozenlist = ">=1.1.0"
name = "anyio"
version = "3.6.2"
description = "High level compatibility layer for multiple asynchronous event loop implementations"
category = "main"
optional = false
python-versions = ">=3.6.2"
files = [
@ -171,7 +167,6 @@ trio = ["trio (>=0.16,<0.22)"]
name = "async-timeout"
version = "4.0.2"
description = "Timeout context manager for asyncio programs"
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -183,7 +178,6 @@ files = [
name = "attrs"
version = "22.2.0"
description = "Classes Without Boilerplate"
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -202,7 +196,6 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy
name = "babel"
version = "2.9.1"
description = "Internationalization utilities"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
files = [
@ -213,11 +206,48 @@ files = [
[package.dependencies]
pytz = ">=2015.7"
[[package]]
name = "boto3"
version = "1.26.144"
description = "The AWS SDK for Python"
optional = false
python-versions = ">= 3.7"
files = [
{file = "boto3-1.26.144-py3-none-any.whl", hash = "sha256:2eb9e688aa86bf1fadcec0b6995b42ec9788e7cd5f1a9c8ac1b66a2506aa209f"},
{file = "boto3-1.26.144.tar.gz", hash = "sha256:5b7e9f2674fe8aa99e2d168744023a3f66da12d9c51e0624489dd0db7aafe30d"},
]
[package.dependencies]
botocore = ">=1.29.144,<1.30.0"
jmespath = ">=0.7.1,<2.0.0"
s3transfer = ">=0.6.0,<0.7.0"
[package.extras]
crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
[[package]]
name = "botocore"
version = "1.29.144"
description = "Low-level, data-driven core of boto 3."
optional = false
python-versions = ">= 3.7"
files = [
{file = "botocore-1.29.144-py3-none-any.whl", hash = "sha256:e2b970e68643cf4752cad4e45ba3319fc35707f1bff7f150f7ffcac1b1427b47"},
{file = "botocore-1.29.144.tar.gz", hash = "sha256:c60b9158cbc7447411abdec77b87a71d86d9404064702e92d317dca6a1ec9a5b"},
]
[package.dependencies]
jmespath = ">=0.7.1,<2.0.0"
python-dateutil = ">=2.1,<3.0.0"
urllib3 = ">=1.25.4,<1.27"
[package.extras]
crt = ["awscrt (==0.16.9)"]
[[package]]
name = "certifi"
version = "2022.12.7"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -229,7 +259,6 @@ files = [
name = "charset-normalizer"
version = "3.1.0"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "main"
optional = false
python-versions = ">=3.7.0"
files = [
@ -314,7 +343,6 @@ files = [
name = "dataclasses-json"
version = "0.5.7"
description = "Easily serialize dataclasses to and from JSON"
category = "main"
optional = false
python-versions = ">=3.6"
files = [
@ -330,11 +358,27 @@ typing-inspect = ">=0.4.0"
[package.extras]
dev = ["flake8", "hypothesis", "ipython", "mypy (>=0.710)", "portray", "pytest (>=6.2.3)", "simplejson", "types-dataclasses"]
[[package]]
name = "ffmpeg-python"
version = "0.2.0"
description = "Python bindings for FFmpeg - with complex filtering support"
optional = false
python-versions = "*"
files = [
{file = "ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127"},
{file = "ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5"},
]
[package.dependencies]
future = "*"
[package.extras]
dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"]
[[package]]
name = "frozenlist"
version = "1.3.3"
description = "A list-like structure which implements collections.abc.MutableSequence"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -414,11 +458,20 @@ files = [
{file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"},
]
[[package]]
name = "future"
version = "0.18.3"
description = "Clean single-source support for Python 3 and 2"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
files = [
{file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"},
]
[[package]]
name = "h11"
version = "0.14.0"
description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -430,7 +483,6 @@ files = [
name = "hiredis"
version = "2.2.2"
description = "Python wrapper for hiredis"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -529,7 +581,6 @@ files = [
name = "httpcore"
version = "0.16.3"
description = "A minimal low-level HTTP client."
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -541,17 +592,16 @@ files = [
anyio = ">=3.0,<5.0"
certifi = "*"
h11 = ">=0.13,<0.15"
sniffio = ">=1.0.0,<2.0.0"
sniffio = "==1.*"
[package.extras]
http2 = ["h2 (>=3,<5)"]
socks = ["socksio (>=1.0.0,<2.0.0)"]
socks = ["socksio (==1.*)"]
[[package]]
name = "httpx"
version = "0.23.3"
description = "The next generation HTTP client."
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -567,15 +617,14 @@ sniffio = "*"
[package.extras]
brotli = ["brotli", "brotlicffi"]
cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"]
cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<13)"]
http2 = ["h2 (>=3,<5)"]
socks = ["socksio (>=1.0.0,<2.0.0)"]
socks = ["socksio (==1.*)"]
[[package]]
name = "idna"
version = "3.4"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false
python-versions = ">=3.5"
files = [
@ -583,11 +632,21 @@ files = [
{file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"},
]
[[package]]
name = "jmespath"
version = "1.0.1"
description = "JSON Matching Expressions"
optional = false
python-versions = ">=3.7"
files = [
{file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
{file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
]
[[package]]
name = "magic-filter"
version = "1.0.9"
description = "This package provides magic filter based on dynamic attribute getter"
category = "main"
optional = false
python-versions = ">=3.7,<4.0"
files = [
@ -599,7 +658,6 @@ files = [
name = "marshmallow"
version = "3.19.0"
description = "A lightweight library for converting complex datatypes to and from native Python datatypes."
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -620,7 +678,6 @@ tests = ["pytest", "pytz", "simplejson"]
name = "marshmallow-enum"
version = "1.5.1"
description = "Enum field for Marshmallow"
category = "main"
optional = false
python-versions = "*"
files = [
@ -635,7 +692,6 @@ marshmallow = ">=2.0.0"
name = "multidict"
version = "6.0.4"
description = "multidict implementation"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -719,7 +775,6 @@ files = [
name = "mypy-extensions"
version = "1.0.0"
description = "Type system extensions for programs checked with the mypy type checker."
category = "main"
optional = false
python-versions = ">=3.5"
files = [
@ -731,7 +786,6 @@ files = [
name = "packaging"
version = "23.0"
description = "Core utilities for Python packages"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -743,7 +797,6 @@ files = [
name = "pillow"
version = "9.4.0"
description = "Python Imaging Library (Fork)"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -830,11 +883,24 @@ files = [
docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-issues (>=3.0.1)", "sphinx-removed-in", "sphinxext-opengraph"]
tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"]
[[package]]
name = "python-dateutil"
version = "2.8.2"
description = "Extensions to the standard Python datetime module"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
files = [
{file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
{file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
]
[package.dependencies]
six = ">=1.5"
[[package]]
name = "python-dotenv"
version = "1.0.0"
description = "Read key-value pairs from a .env file and set them as environment variables"
category = "main"
optional = false
python-versions = ">=3.8"
files = [
@ -849,7 +915,6 @@ cli = ["click (>=5.0)"]
name = "pytz"
version = "2022.7.1"
description = "World timezone definitions, modern and historical"
category = "main"
optional = false
python-versions = "*"
files = [
@ -861,7 +926,6 @@ files = [
name = "redis"
version = "4.5.2"
description = "Python client for Redis database and key-value store"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -881,7 +945,6 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"
name = "rfc3986"
version = "1.5.0"
description = "Validating URI References per RFC 3986"
category = "main"
optional = false
python-versions = "*"
files = [
@ -895,11 +958,38 @@ idna = {version = "*", optional = true, markers = "extra == \"idna2008\""}
[package.extras]
idna2008 = ["idna"]
[[package]]
name = "s3transfer"
version = "0.6.1"
description = "An Amazon S3 Transfer Manager"
optional = false
python-versions = ">= 3.7"
files = [
{file = "s3transfer-0.6.1-py3-none-any.whl", hash = "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346"},
{file = "s3transfer-0.6.1.tar.gz", hash = "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9"},
]
[package.dependencies]
botocore = ">=1.12.36,<2.0a.0"
[package.extras]
crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"]
[[package]]
name = "six"
version = "1.16.0"
description = "Python 2 and 3 compatibility utilities"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
files = [
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
]
[[package]]
name = "sniffio"
version = "1.3.0"
description = "Sniff out which async library your code is running under"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -911,7 +1001,6 @@ files = [
name = "typing-extensions"
version = "4.5.0"
description = "Backported and Experimental Type Hints for Python 3.7+"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -923,7 +1012,6 @@ files = [
name = "typing-inspect"
version = "0.8.0"
description = "Runtime inspection utilities for typing module."
category = "main"
optional = false
python-versions = "*"
files = [
@ -935,11 +1023,26 @@ files = [
mypy-extensions = ">=0.3.0"
typing-extensions = ">=3.7.4"
[[package]]
name = "urllib3"
version = "1.26.16"
description = "HTTP library with thread-safe connection pooling, file post, and more."
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
files = [
{file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"},
{file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"},
]
[package.extras]
brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[[package]]
name = "yarl"
version = "1.8.2"
description = "Yet another URL library"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
@ -1026,4 +1129,4 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = ">= 3.9 < 4.0"
content-hash = "5a77365de854eb4f98b1ab47cae0196bf696843e14d3577555910e545333a46c"
content-hash = "962628b868946c891bbd383ad143faa8dde31c148958fbf8c7d86e4ec26d5c09"

2
pyproject.toml
View File

@ -14,6 +14,8 @@ httpx = "^0.23.3"
dataclasses-json = "^0.5.7"
redis = {extras = ["hiredis"], version = "^4.5.2"}
pillow = "^9.4.0"
ffmpeg-python = "^0.2.0"
boto3 = "^1.26.144"
[build-system]