From b2d4a2789df48ca46cc5110dac28d8e98c7fccec Mon Sep 17 00:00:00 2001 From: Pavle Portic Date: Mon, 2 Aug 2021 01:38:28 +0200 Subject: [PATCH] Implement functionality to rewatch already watched videos --- setup.py | 2 +- ytrssil/bindings.py | 10 +- ytrssil/cli.py | 125 +++++++++++++++++++---- ytrssil/config.py | 1 + ytrssil/datatypes.py | 4 +- ytrssil/exceptions.py | 2 + ytrssil/fetch.py | 74 ++++++++------ ytrssil/{parser.py => parse.py} | 3 +- ytrssil/query.py | 24 ----- ytrssil/repository.py | 169 ++++++++++++++++++++++---------- 10 files changed, 283 insertions(+), 131 deletions(-) create mode 100644 ytrssil/exceptions.py rename ytrssil/{parser.py => parse.py} (94%) delete mode 100644 ytrssil/query.py diff --git a/setup.py b/setup.py index 9e2f966..5d3b588 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( ), long_description=long_description, long_description_content_type='text/markdown', - license='BSD-3-Clause', + license_files=('LICENSE',), homepage='https://gitea.theedgeofrage.com/TheEdgeOfRage/ytrssil', repository='https://gitea.theedgeofrage.com/TheEdgeOfRage/ytrssil', documentation='https://gitea.theedgeofrage.com/TheEdgeOfRage/ytrssil', diff --git a/ytrssil/bindings.py b/ytrssil/bindings.py index b27a3ff..6c044e4 100644 --- a/ytrssil/bindings.py +++ b/ytrssil/bindings.py @@ -1,7 +1,8 @@ -from inject import Binder, configure +from inject import Binder, Injector, configure, get_injector_or_die from ytrssil.config import Configuration -from ytrssil.parser import Parser, create_feed_parser +from ytrssil.fetch import Fetcher, create_fetcher +from ytrssil.parse import Parser, create_feed_parser from ytrssil.repository import ChannelRepository, create_channel_repository @@ -10,8 +11,11 @@ def dependency_configuration(binder: Binder) -> None: binder.bind(Configuration, config) binder.bind_to_provider binder.bind_to_constructor(ChannelRepository, create_channel_repository) + binder.bind_to_constructor(Fetcher, create_fetcher) binder.bind_to_constructor(Parser, create_feed_parser) -def setup_dependencies() -> None: +def setup_dependencies() -> Injector: configure(dependency_configuration) + + return get_injector_or_die() diff --git a/ytrssil/cli.py b/ytrssil/cli.py index 2b1ffb3..d495a45 100644 --- a/ytrssil/cli.py +++ b/ytrssil/cli.py @@ -1,26 +1,61 @@ +from collections.abc import Iterator +from datetime import datetime, timezone +from operator import attrgetter from os import execv, fork -from sys import stderr +from subprocess import PIPE, Popen +from sys import argv, stderr from inject import autoparams from ytrssil.bindings import setup_dependencies from ytrssil.constants import mpv_options -from ytrssil.fetch import fetch_new_videos -from ytrssil.query import query +from ytrssil.datatypes import Video +from ytrssil.fetch import Fetcher from ytrssil.repository import ChannelRepository -class NoVideoSelected(Exception): - pass +def user_query(videos: dict[str, Video], reverse: bool = False) -> list[Video]: + p = Popen( + ['fzf', '-m'], + stdout=PIPE, + stdin=PIPE, + ) + video_list: Iterator[Video] + if reverse: + video_list = reversed(videos.values()) + else: + video_list = iter(videos.values()) + + input_bytes = '\n'.join(map(str, video_list)).encode('UTF-8') + stdout, _ = p.communicate(input=input_bytes) + videos_str: list[str] = stdout.decode('UTF-8').strip().split('\n') + ret: list[Video] = [] + for video_str in videos_str: + *_, video_id = video_str.split(' - ') + + try: + ret.append(videos[video_id]) + except KeyError: + pass + + return ret @autoparams() -def run(repository_manager: ChannelRepository) -> None: +def watch_videos( + repository_manager: ChannelRepository, + fetcher: Fetcher, +) -> int: with repository_manager as repository: - channels, new_videos = fetch_new_videos() - selected_videos = query(new_videos) + channels, new_videos = fetcher.fetch_new_videos() + if not new_videos: + print('No new videos', file=stderr) + return 1 + + selected_videos = user_query(new_videos) if not selected_videos: - raise NoVideoSelected + print('No video selected', file=stderr) + return 2 video_urls = [video.url for video in selected_videos] cmd = ['/usr/bin/mpv', *mpv_options, *video_urls] @@ -30,19 +65,67 @@ def run(repository_manager: ChannelRepository) -> None: for video in selected_videos: selected_channel = channels[video.channel_id] selected_channel.mark_video_as_watched(video) - repository.update_video(video, True) - - -def main() -> int: - setup_dependencies() - try: - run() - except NoVideoSelected: - print('No video selected', file=stderr) - return 1 + watch_timestamp = datetime.utcnow().replace(tzinfo=timezone.utc) + repository.update_video(video, watch_timestamp) return 0 -if __name__ == '__main__': - main() +@autoparams() +def watch_history( + repository_manager: ChannelRepository, + fetcher: Fetcher, +) -> int: + with repository_manager as repository: + watched_videos = repository.get_watched_videos() + selected_videos = user_query(watched_videos, reverse=True) + if not selected_videos: + print('No video selected', file=stderr) + return 1 + + video_urls = [video.url for video in selected_videos] + cmd = ['/usr/bin/mpv', *mpv_options, *video_urls] + if (fork() == 0): + execv(cmd[0], cmd) + + return 0 + + +@autoparams() +def mark_as_watched( + repository_manager: ChannelRepository, + fetcher: Fetcher, + up_to_date: datetime +) -> int: + with repository_manager as repository: + channels, new_videos = fetcher.fetch_new_videos() + for video in sorted(new_videos.values(), key=attrgetter('timestamp')): + if video.timestamp >= up_to_date: + continue + + selected_channel = channels[video.channel_id] + selected_channel.mark_video_as_watched(video) + watch_timestamp = datetime.utcnow().replace(tzinfo=timezone.utc) + repository.update_video(video, watch_timestamp) + + return 0 + + +def main() -> int: + setup_dependencies() + command: str + try: + command = argv[1] + except IndexError: + command = 'watch' + + if command == 'watch': + return watch_videos() + elif command == 'history': + return watch_history() + elif command == 'mark': + up_to_date = datetime.fromisoformat(argv[2]) + return mark_as_watched(up_to_date=up_to_date) + else: + print(f'Unknown command "{command}"', file=stderr) + return 1 diff --git a/ytrssil/config.py b/ytrssil/config.py index c4982bf..fe58ddc 100644 --- a/ytrssil/config.py +++ b/ytrssil/config.py @@ -8,6 +8,7 @@ from ytrssil.constants import config_dir @dataclass class Configuration: channel_repository_type: str = 'sqlite' + fetcher_type: str = 'aiohttp' feed_parser_type: str = 'feedparser' diff --git a/ytrssil/datatypes.py b/ytrssil/datatypes.py index ee92350..c3fbb04 100644 --- a/ytrssil/datatypes.py +++ b/ytrssil/datatypes.py @@ -1,5 +1,6 @@ from dataclasses import dataclass, field from datetime import datetime +from typing import Union @dataclass @@ -7,9 +8,10 @@ class Video: video_id: str name: str url: str - timestamp: datetime channel_id: str channel_name: str + timestamp: datetime + watch_timestamp: Union[datetime, None] = None def __str__(self) -> str: return f'{self.channel_name} - {self.name} - {self.video_id}' diff --git a/ytrssil/exceptions.py b/ytrssil/exceptions.py new file mode 100644 index 0000000..164dd97 --- /dev/null +++ b/ytrssil/exceptions.py @@ -0,0 +1,2 @@ +class ChannelNotFound(Exception): + pass diff --git a/ytrssil/fetch.py b/ytrssil/fetch.py index 3483924..a2264cf 100644 --- a/ytrssil/fetch.py +++ b/ytrssil/fetch.py @@ -1,42 +1,60 @@ +from abc import ABCMeta, abstractmethod from asyncio import gather, run from collections.abc import Iterable from aiohttp import ClientResponse, ClientSession from inject import autoparams -from ytrssil.config import get_feed_urls +from ytrssil.config import Configuration, get_feed_urls from ytrssil.datatypes import Channel, Video -from ytrssil.parser import Parser +from ytrssil.parse import Parser from ytrssil.repository import ChannelRepository -async def request(session: ClientSession, url: str) -> ClientResponse: - return await session.request(method='GET', url=url) +class Fetcher(metaclass=ABCMeta): + @abstractmethod + def fetch_feeds(self, urls: Iterable[str]) -> Iterable[str]: + pass + + @autoparams('parser', 'repository') + def fetch_new_videos( + self, + parser: Parser, + repository: ChannelRepository, + ) -> tuple[dict[str, Channel], dict[str, Video]]: + feed_urls = get_feed_urls() + channels: dict[str, Channel] = {} + new_videos: dict[str, Video] = {} + for feed in self.fetch_feeds(feed_urls): + channel = parser(feed) + channels[channel.channel_id] = channel + new_videos.update(channel.new_videos) + + return channels, new_videos -async def fetch_feeds(urls: Iterable[str]) -> Iterable[str]: - async with ClientSession() as session: - responses: list[ClientResponse] = await gather(*[ - request(session, url) for url in urls - ]) - return [ - await response.text(encoding='UTF-8') - for response in responses - ] +class AioHttpFetcher(Fetcher): + async def request(self, session: ClientSession, url: str) -> ClientResponse: + return await session.request(method='GET', url=url) + + async def async_fetch_feeds(self, urls: Iterable[str]) -> Iterable[str]: + async with ClientSession() as session: + responses: list[ClientResponse] = await gather(*[ + self.request(session, url) for url in urls + ]) + return [ + await response.text(encoding='UTF-8') + for response in responses + ] + + def fetch_feeds(self, urls: Iterable[str]) -> Iterable[str]: + return run(self.async_fetch_feeds(urls)) -@autoparams('parser', 'repository') -def fetch_new_videos( - *, - parser: Parser, - repository: ChannelRepository, -) -> tuple[dict[str, Channel], dict[str, Video]]: - feed_urls = get_feed_urls() - channels: dict[str, Channel] = {} - new_videos: dict[str, Video] = {} - for feed in run(fetch_feeds(feed_urls)): - channel = parser(feed) - channels[channel.channel_id] = channel - new_videos.update(channel.new_videos) - - return channels, new_videos +@autoparams() +def create_fetcher(config: Configuration) -> Fetcher: + fetcher_type = config.fetcher_type + if fetcher_type == 'aiohttp': + return AioHttpFetcher() + else: + raise Exception(f'Unknown feed fetcher type: "{fetcher_type}"') diff --git a/ytrssil/parser.py b/ytrssil/parse.py similarity index 94% rename from ytrssil/parser.py rename to ytrssil/parse.py index ae63df6..72224fd 100644 --- a/ytrssil/parser.py +++ b/ytrssil/parse.py @@ -6,7 +6,8 @@ from inject import autoparams from ytrssil.config import Configuration from ytrssil.datatypes import Channel, Video -from ytrssil.repository import ChannelNotFound, ChannelRepository +from ytrssil.exceptions import ChannelNotFound +from ytrssil.repository import ChannelRepository class Parser(metaclass=ABCMeta): diff --git a/ytrssil/query.py b/ytrssil/query.py deleted file mode 100644 index 792663f..0000000 --- a/ytrssil/query.py +++ /dev/null @@ -1,24 +0,0 @@ -from subprocess import PIPE, Popen - -from ytrssil.datatypes import Video - - -def query(videos: dict[str, Video]) -> list[Video]: - p = Popen( - ['fzf', '-m'], - stdout=PIPE, - stdin=PIPE, - ) - input_bytes = '\n'.join(map(str, videos.values())).encode('UTF-8') - stdout, _ = p.communicate(input=input_bytes) - videos_str: list[str] = stdout.decode('UTF-8').strip().split('\n') - ret: list[Video] = [] - for video_str in videos_str: - *_, video_id = video_str.split(' - ') - - try: - ret.append(videos[video_id]) - except KeyError: - pass - - return ret diff --git a/ytrssil/repository.py b/ytrssil/repository.py index 55d6f40..e25d088 100644 --- a/ytrssil/repository.py +++ b/ytrssil/repository.py @@ -11,10 +11,7 @@ from inject import autoparams from ytrssil.config import Configuration from ytrssil.constants import config_dir from ytrssil.datatypes import Channel, Video - - -class ChannelNotFound(Exception): - pass +from ytrssil.exceptions import ChannelNotFound class ChannelRepository(metaclass=ABCMeta): @@ -35,6 +32,14 @@ class ChannelRepository(metaclass=ABCMeta): def get_channel(self, channel_id: str) -> Channel: pass + @abstractmethod + def get_all_channels(self) -> list[Channel]: + pass + + @abstractmethod + def get_watched_videos(self) -> dict[str, Video]: + pass + @abstractmethod def create_channel(self, channel: Channel) -> None: pass @@ -44,7 +49,7 @@ class ChannelRepository(metaclass=ABCMeta): pass @abstractmethod - def update_video(self, video: Video, watched: bool) -> None: + def update_video(self, video: Video, watch_timestamp: datetime) -> None: pass @@ -65,7 +70,7 @@ class SqliteChannelRepository(ChannelRepository): cursor.execute( 'CREATE TABLE IF NOT EXISTS videos (' 'video_id VARCHAR PRIMARY KEY, name VARCHAR, url VARCHAR UNIQUE, ' - 'timestamp VARCHAR, watched BOOLEAN, channel_id VARCHAR, ' + 'timestamp VARCHAR, watch_timestamp VARCHAR, channel_id VARCHAR, ' 'FOREIGN KEY(channel_id) REFERENCES channels(channel_id))' ) connection.commit() @@ -83,46 +88,67 @@ class SqliteChannelRepository(ChannelRepository): ) -> None: self.connection.close() - def get_channel_as_dict(self, channel: Channel) -> dict[str, str]: + def channel_to_params(self, channel: Channel) -> dict[str, str]: return { 'channel_id': channel.channel_id, 'name': channel.name, 'url': channel.url, } - def get_video_as_dict(self, video: Video, watched: bool) -> dict[str, Any]: + def channel_data_to_channel( + self, + channel_data: tuple[str, str, str], + ) -> Channel: + channel = Channel( + channel_id=channel_data[0], + name=channel_data[1], + url=channel_data[2], + ) + for video in self.get_videos_for_channel(channel): + if video.watch_timestamp is not None: + channel.watched_videos[video.video_id] = video + else: + channel.new_videos[video.video_id] = video + + return channel + + def video_to_params(self, video: Video) -> dict[str, Any]: + watch_timestamp: Union[str, None] + if video.watch_timestamp is not None: + watch_timestamp = video.watch_timestamp.isoformat() + else: + watch_timestamp = video.watch_timestamp + return { 'video_id': video.video_id, 'name': video.name, 'url': video.url, 'timestamp': video.timestamp.isoformat(), - 'watched': watched, + 'watch_timestamp': watch_timestamp, 'channel_id': video.channel_id, } - def get_videos(self, channel: Channel) -> list[tuple[Video, bool]]: - cursor = self.connection.cursor() - cursor.execute( - 'SELECT video_id, name, url, timestamp, watched ' - 'FROM videos WHERE channel_id=:channel_id', - {'channel_id': channel.channel_id}, - ) - ret: list[tuple[Video, bool]] = [] - video_data: tuple[str, str, str, str, bool] - for video_data in cursor.fetchall(): - ret.append(( - Video( - video_id=video_data[0], - name=video_data[1], - url=video_data[2], - timestamp=datetime.fromisoformat(video_data[3]), - channel_id=channel.channel_id, - channel_name=channel.name, - ), - video_data[4] - )) + def video_data_to_video( + self, + video_data: tuple[str, str, str, str, str], + channel_id: str, + channel_name: str, + ) -> Video: + watch_timestamp: Union[datetime, None] + if video_data[4] is not None: + watch_timestamp = datetime.fromisoformat(video_data[4]) + else: + watch_timestamp = video_data[4] - return ret + return Video( + video_id=video_data[0], + name=video_data[1], + url=video_data[2], + timestamp=datetime.fromisoformat(video_data[3]), + watch_timestamp=watch_timestamp, + channel_id=channel_id, + channel_name=channel_name, + ) def get_channel(self, channel_id: str) -> Channel: cursor = self.connection.cursor() @@ -130,28 +156,63 @@ class SqliteChannelRepository(ChannelRepository): 'SELECT * FROM channels WHERE channel_id=:channel_id', {'channel_id': channel_id}, ) - channel_data: Union[tuple[str, str, str], None] = cursor.fetchone() - if channel_data is None: + try: + return self.channel_data_to_channel(next(cursor)) + except StopIteration: raise ChannelNotFound - channel = Channel( - channel_id=channel_data[0], - name=channel_data[1], - url=channel_data[2], - ) - for video, watched in self.get_videos(channel): - if watched: - channel.watched_videos[video.video_id] = video - else: - channel.new_videos[video.video_id] = video + def get_all_channels(self) -> list[Channel]: + cursor = self.connection.cursor() + cursor.execute('SELECT * FROM channels') - return channel + return [ + self.channel_data_to_channel(channel_data) + for channel_data in cursor + ] + + def get_videos_for_channel( + self, + channel: Channel, + ) -> list[Video]: + cursor = self.connection.cursor() + cursor.execute( + 'SELECT video_id, name, url, timestamp, watch_timestamp ' + 'FROM videos WHERE channel_id=:channel_id', + {'channel_id': channel.channel_id}, + ) + + return [ + self.video_data_to_video( + video_data=video_data, + channel_id=channel.channel_id, + channel_name=channel.name, + ) + for video_data in cursor + ] + + def get_watched_videos(self) -> dict[str, Video]: + cursor = self.connection.cursor() + cursor.execute( + 'SELECT video_id, videos.name, videos.url, timestamp, ' + 'watch_timestamp, channels.channel_id, channels.name FROM videos ' + 'LEFT JOIN channels ON channels.channel_id=videos.channel_id WHERE ' + 'watch_timestamp IS NOT NULL ORDER BY timestamp' + ) + + return { + video_data[0]: self.video_data_to_video( + video_data=video_data, + channel_id=video_data[5], + channel_name=video_data[6], + ) + for video_data in cursor + } def create_channel(self, channel: Channel) -> None: cursor = self.connection.cursor() cursor.execute( 'INSERT INTO channels VALUES (:channel_id, :name, :url)', - self.get_channel_as_dict(channel), + self.channel_to_params(channel), ) self.connection.commit() @@ -160,24 +221,28 @@ class SqliteChannelRepository(ChannelRepository): cursor.execute( 'UPDATE channels SET channel_id = :channel_id, name = :name, ' 'url = :url WHERE channel_id=:channel_id', - self.get_channel_as_dict(channel), + self.channel_to_params(channel), ) self.connection.commit() def add_new_video(self, channel: Channel, video: Video) -> None: cursor = self.connection.cursor() cursor.execute( - 'INSERT INTO videos VALUES ' - '(:video_id, :name, :url, :timestamp, :watched, :channel_id)', - self.get_video_as_dict(video, False), + 'INSERT INTO videos VALUES (:video_id, :name, ' + ':url, :timestamp, :watch_timestamp, :channel_id)', + self.video_to_params(video), ) self.connection.commit() - def update_video(self, video: Video, watched: bool) -> None: + def update_video(self, video: Video, watch_timestamp: datetime) -> None: cursor = self.connection.cursor() cursor.execute( - 'UPDATE videos SET watched = :watched WHERE video_id=:video_id', - {'watched': watched, 'video_id': video.video_id}, + 'UPDATE videos SET watch_timestamp = :watch_timestamp ' + 'WHERE video_id=:video_id', + { + 'watch_timestamp': watch_timestamp.isoformat(), + 'video_id': video.video_id, + }, ) self.connection.commit()