From bc4d70ae04a63f6703839a8e9bf0ea86f91a0c8b Mon Sep 17 00:00:00 2001 From: Pavle Portic Date: Fri, 30 Jul 2021 11:48:32 +0200 Subject: [PATCH] Initial commit --- .editorconfig | 12 ++++++++ .gitignore | 67 +++++++++++++++++++++++++++++++++++++++++++ makefile | 27 +++++++++++++++++ requirements-dev.txt | 3 ++ requirements.txt | 2 ++ setup.py | 26 +++++++++++++++++ ytrssil/__init__.py | 0 ytrssil/cli.py | 33 +++++++++++++++++++++ ytrssil/config.py | 11 +++++++ ytrssil/constants.py | 14 +++++++++ ytrssil/datatypes.py | 43 +++++++++++++++++++++++++++ ytrssil/fetch.py | 39 +++++++++++++++++++++++++ ytrssil/parser.py | 37 ++++++++++++++++++++++++ ytrssil/py.typed | 0 ytrssil/query.py | 24 ++++++++++++++++ ytrssil/repository.py | 32 +++++++++++++++++++++ 16 files changed, 370 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 makefile create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 ytrssil/__init__.py create mode 100644 ytrssil/cli.py create mode 100644 ytrssil/config.py create mode 100644 ytrssil/constants.py create mode 100644 ytrssil/datatypes.py create mode 100644 ytrssil/fetch.py create mode 100644 ytrssil/parser.py create mode 100644 ytrssil/py.typed create mode 100644 ytrssil/query.py create mode 100644 ytrssil/repository.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..f7f21cc --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[Makefile] +indent_style = tab diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9af77c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,67 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Environments +.*env +.*venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ diff --git a/makefile b/makefile new file mode 100644 index 0000000..8f404fd --- /dev/null +++ b/makefile @@ -0,0 +1,27 @@ +NAME = ytrssil +TESTS = tests + +FILES_PY = $(shell find $(CURDIR)/$(NAME) $(CURDIR)/$(TESTS) -type f -name "*.py") +TEST_PY = $(shell find $(CURDIR)/$(TESTS) -type f -name "*.py") + +setup-dev: + pip install -r requirements-dev.txt + pip install -e . + +test: + python -m unittest discover $(CURDIR)/$(TESTS) + +flake8: + @flake8 $(FILES_PY) + +mypy: + @mypy --strict $(FILES_PY) + +isort: + @isort -c $(FILES_PY) + +validate: flake8 mypy isort + +coverage: + python -m coverage run -m unittest discover $(CURDIR)/$(TESTS) + python -m coverage html diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..1c900c5 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +flake8==3.* +jedi==0.* +mypy==0.* diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..338eac9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +aiohttp==3.7.* +feedparser==6.0.* diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..2182b07 --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +from setuptools import setup # type: ignore + + +with open('requirements.txt') as f: + required = f.read().splitlines() + + +setup( + name='ytrssil', + description=( + 'Subscribe to YouTube RSS feeds and keep track of watched videos' + ), + homepage='https://gitea.theedgeofrage.com/TheEdgeOfRage/ytrssil', + repository='https://gitea.theedgeofrage.com/TheEdgeOfRage/ytrssil', + documentation='https://gitea.theedgeofrage.com/TheEdgeOfRage/ytrssil', + version='0.0.0', + packages=['ytrssil'], + package_data={'': ['py.typed']}, + include_package_data=True, + install_requires=required, + entry_points={ + 'console_scripts': [ + 'ytrssil = ytrssil.cli:main', + ], + }, +) diff --git a/ytrssil/__init__.py b/ytrssil/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ytrssil/cli.py b/ytrssil/cli.py new file mode 100644 index 0000000..9054483 --- /dev/null +++ b/ytrssil/cli.py @@ -0,0 +1,33 @@ +from os import execv, fork +from sys import stderr + +from ytrssil.constants import mpv_options +from ytrssil.repository import ChannelRepository +from ytrssil.fetch import fetch_new_videos +from ytrssil.query import query + + +def main() -> int: + with ChannelRepository() as repository: + channels, new_videos = fetch_new_videos(repository) + + selected_videos = query(new_videos) + if not selected_videos: + print('No video selected', file=stderr) + return 1 + + video_urls = [video.url for video in selected_videos] + cmd = ['/usr/bin/mpv', *mpv_options, *video_urls] + if (fork() == 0): + execv(cmd[0], cmd) + + for video in selected_videos: + selected_channel = channels[video.channel_id] + selected_channel.mark_video_as_watched(video) + repository.update_channel(selected_channel) + + return 0 + + +if __name__ == '__main__': + main() diff --git a/ytrssil/config.py b/ytrssil/config.py new file mode 100644 index 0000000..f805936 --- /dev/null +++ b/ytrssil/config.py @@ -0,0 +1,11 @@ +import os +from collections.abc import Iterator + +from ytrssil.constants import config_dir + + +def get_feed_urls() -> Iterator[str]: + file_path = os.path.join(config_dir, 'feeds') + with open(file_path, 'r') as f: + for line in f: + yield line.strip() diff --git a/ytrssil/constants.py b/ytrssil/constants.py new file mode 100644 index 0000000..9334636 --- /dev/null +++ b/ytrssil/constants.py @@ -0,0 +1,14 @@ +import os + + +config_prefix: str +try: + config_prefix = os.environ['XDG_CONFIG_HOME'] +except KeyError: + config_prefix = os.path.expanduser('~/.config') + +config_dir: str = os.path.join(config_prefix, 'ytrssil') +mpv_options: list[str] = [ + '--no-terminal', + '--ytdl-format=bestvideo[height<=?1080][vcodec!=vp9]+bestaudio/best', +] diff --git a/ytrssil/datatypes.py b/ytrssil/datatypes.py new file mode 100644 index 0000000..c54300e --- /dev/null +++ b/ytrssil/datatypes.py @@ -0,0 +1,43 @@ +from dataclasses import dataclass, field +from datetime import datetime + + +@dataclass +class Video: + video_id: str + name: str + url: str + timestamp: datetime + channel_id: str + channel_name: str + + def __str__(self) -> str: + return f'{self.channel_name} - {self.name} - {self.video_id}' + + +@dataclass +class Channel: + channel_id: str + name: str + url: str + new_videos: dict[str, Video] = field(default_factory=lambda: dict()) + watched_videos: dict[str, Video] = field(default_factory=lambda: dict()) + + def add_video(self, video: Video) -> None: + if video.video_id in self.watched_videos: + return + + self.new_videos[video.video_id] = video + + def remove_old_videos(self) -> None: + vid_list: list[Video] = sorted( + self.watched_videos.values(), + key=lambda x: x.timestamp, + ) + for video in vid_list[15:]: + self.watched_videos.pop(video.video_id) + + def mark_video_as_watched(self, video: Video) -> None: + self.new_videos.pop(video.video_id) + self.watched_videos[video.video_id] = video + self.remove_old_videos() diff --git a/ytrssil/fetch.py b/ytrssil/fetch.py new file mode 100644 index 0000000..054656a --- /dev/null +++ b/ytrssil/fetch.py @@ -0,0 +1,39 @@ +from asyncio import gather, run +from collections.abc import Iterable + +from aiohttp import ClientResponse, ClientSession + +from ytrssil.config import get_feed_urls +from ytrssil.datatypes import Channel, Video +from ytrssil.repository import ChannelRepository +from ytrssil.parser import Parser + + +async def request(session: ClientSession, url: str) -> ClientResponse: + return await session.request(method='GET', url=url) + + +async def fetch_feeds(urls: Iterable[str]) -> Iterable[str]: + async with ClientSession() as session: + responses: list[ClientResponse] = await gather(*[ + request(session, url) for url in urls + ]) + return [ + await response.text(encoding='UTF-8') + for response in responses + ] + + +def fetch_new_videos( + repository: ChannelRepository, +) -> tuple[dict[str, Channel], dict[str, Video]]: + feed_urls = get_feed_urls() + parser = Parser(repository) + channels: dict[str, Channel] = {} + new_videos: dict[str, Video] = {} + for feed in run(fetch_feeds(feed_urls)): + channel = parser(feed) + channels[channel.channel_id] = channel + new_videos.update(channel.new_videos) + + return channels, new_videos diff --git a/ytrssil/parser.py b/ytrssil/parser.py new file mode 100644 index 0000000..6e3b78d --- /dev/null +++ b/ytrssil/parser.py @@ -0,0 +1,37 @@ +from datetime import datetime + +import feedparser + +from ytrssil.datatypes import Channel, Video +from ytrssil.repository import ChannelNotFound, ChannelRepository + + +class Parser: + def __init__(self, channel_repository: ChannelRepository) -> None: + self.repository = channel_repository + + def __call__(self, feed_content: str) -> Channel: + d = feedparser.parse(feed_content) + channel_id: str = d['feed']['yt_channelid'] + try: + channel = self.repository.get_channel(channel_id) + except ChannelNotFound: + channel = Channel( + channel_id=channel_id, + name=d['feed']['title'], + url=d['feed']['link'], + ) + self.repository.update_channel(channel) + + for entry in d['entries']: + channel.add_video(Video( + video_id=entry['yt_videoid'], + name=entry['title'], + url=entry['link'], + timestamp=datetime.fromisoformat(entry['published']), + channel_id=channel.channel_id, + channel_name=channel.name, + )) + + self.repository.update_channel(channel) + return channel diff --git a/ytrssil/py.typed b/ytrssil/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/ytrssil/query.py b/ytrssil/query.py new file mode 100644 index 0000000..65e8efc --- /dev/null +++ b/ytrssil/query.py @@ -0,0 +1,24 @@ +from subprocess import Popen, PIPE + +from ytrssil.datatypes import Video + + +def query(videos: dict[str, Video]) -> list[Video]: + p = Popen( + ['fzf', '-m'], + stdout=PIPE, + stdin=PIPE, + ) + input_bytes = '\n'.join(map(str, videos.values())).encode('UTF-8') + stdout, _ = p.communicate(input=input_bytes) + videos_str: list[str] = stdout.decode('UTF-8').strip().split('\n') + ret: list[Video] = [] + for video_str in videos_str: + *_, video_id = video_str.split(' - ') + + try: + ret.append(videos[video_id]) + except KeyError: + pass + + return ret diff --git a/ytrssil/repository.py b/ytrssil/repository.py new file mode 100644 index 0000000..9e6b7ab --- /dev/null +++ b/ytrssil/repository.py @@ -0,0 +1,32 @@ +from __future__ import annotations +import os +import shelve + +from ytrssil.constants import config_dir +from ytrssil.datatypes import Channel + + +class ChannelNotFound(Exception): + pass + + +class ChannelRepository: + def __init__(self) -> None: + os.makedirs(config_dir, exist_ok=True) + self.file_path: str = os.path.join(config_dir, 'shelf') + + def __enter__(self) -> ChannelRepository: + self.repository = shelve.open(self.file_path) + return self + + def __exit__(self, exc_type, exc_value, exc_traceback) -> None: + self.repository.close() + + def get_channel(self, channel_id: str) -> Channel: + try: + return self.repository[channel_id] + except KeyError as e: + raise ChannelNotFound(e) + + def update_channel(self, channel: Channel) -> None: + self.repository[channel.channel_id] = channel