From bd7fcb35bdd08181340c09f15070a0135809e0d6 Mon Sep 17 00:00:00 2001 From: Pavle Portic Date: Fri, 19 Nov 2021 19:44:46 +0100 Subject: [PATCH] Add dedicated fetch command instead of always checking Use Protocol instead of ABCMeta --- .mypy.ini | 3 ++ readme.md | 3 ++ requirements-dev.txt | 2 +- tests/constants.py | 6 ++-- tests/test_bindings.py | 7 ++-- tests/test_datatypes.py | 26 +++++++------- tests/test_fetch.py | 9 ++--- tests/test_parse.py | 4 +-- ytrssil/api.py | 3 +- ytrssil/bindings.py | 7 ++-- ytrssil/cli.py | 31 +++++++++++++---- ytrssil/datatypes.py | 20 +++++++++-- ytrssil/fetch.py | 16 +++++---- ytrssil/parse.py | 11 ++---- ytrssil/protocols.py | 70 ++++++++++++++++++++++++++++++++++++++ ytrssil/repository.py | 75 ++++++++++++----------------------------- ytrssil/types.py | 18 ---------- 17 files changed, 186 insertions(+), 125 deletions(-) create mode 100644 ytrssil/protocols.py delete mode 100644 ytrssil/types.py diff --git a/.mypy.ini b/.mypy.ini index 2a3ce20..ee186b2 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -4,4 +4,7 @@ ignore_missing_imports=true [mypy-feedparser.*] ignore_missing_imports=true +[mypy-pytest_mock.*] +ignore_missing_imports=true + [mypy] diff --git a/readme.md b/readme.md index 03afe08..c0a1e7d 100644 --- a/readme.md +++ b/readme.md @@ -4,6 +4,9 @@ This is a simple CLI to manage YouTube subscriptions through RSS feeds and watch new videos using `mpv`. It keeps track of watched videos in a local sqlite database. +**This tool is still in early development and breaking changes across minor +versions are expected.** + ## Configuration It looks for a list of RSS URLs in `$XDG_CONFIG_HOME/ytrssil/feeds` diff --git a/requirements-dev.txt b/requirements-dev.txt index 0eeec3a..5c9c04b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ aioresponses==0.7.* flake8==3.* -jedi==0.* mypy==0.* pytest==6.2.* +pytest-cov==3.0.0 pytest-mock==3.6.* wheel==0.37.* diff --git a/tests/constants.py b/tests/constants.py index 174c18a..1b8647f 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,7 +1,6 @@ from datetime import datetime -from ytrssil.types import ChannelData, VideoData - +from ytrssil.datatypes import ChannelData, VideoData FEED_XML: str = ''' @@ -30,5 +29,6 @@ TEST_VIDEO_DATA: VideoData = { TEST_CHANNEL_DATA: ChannelData = { 'channel_id': 'channel_id', 'name': 'channel_name', - 'new_videos': {} + 'new_videos': {}, + 'watched_videos': {}, } diff --git a/tests/test_bindings.py b/tests/test_bindings.py index d48e61c..f576d4d 100644 --- a/tests/test_bindings.py +++ b/tests/test_bindings.py @@ -1,8 +1,9 @@ from ytrssil.bindings import setup_dependencies from ytrssil.config import Configuration -from ytrssil.fetch import Fetcher, AioHttpFetcher -from ytrssil.parse import Parser, FeedparserParser -from ytrssil.repository import ChannelRepository, SqliteChannelRepository +from ytrssil.fetch import AioHttpFetcher +from ytrssil.parse import FeedparserParser +from ytrssil.protocols import ChannelRepository, Fetcher, Parser +from ytrssil.repository import SqliteChannelRepository def test_setup_dependencies() -> None: diff --git a/tests/test_datatypes.py b/tests/test_datatypes.py index 1feb0fd..ff74e66 100644 --- a/tests/test_datatypes.py +++ b/tests/test_datatypes.py @@ -1,7 +1,7 @@ from datetime import datetime from tests.constants import TEST_CHANNEL_DATA, TEST_VIDEO_DATA -from ytrssil.datatypes import Channel, Video +from ytrssil.datatypes import Channel, ChannelData, Video def test_video_str() -> None: @@ -11,8 +11,8 @@ def test_video_str() -> None: def test_channel_str() -> None: - channel = Channel.from_dict({ - **TEST_CHANNEL_DATA, + channel_data: ChannelData = TEST_CHANNEL_DATA.copy() + channel_data.update({ 'new_videos': { 'video_id': Video( video_id='video_id', @@ -25,13 +25,14 @@ def test_channel_str() -> None: ), }, }) - string = str(channel) + channel_string = str(Channel.from_dict(channel_data)) - assert string == 'channel_name - 1' + assert channel_string == 'channel_name - 1' def test_channel_add_new_video() -> None: - channel = Channel.from_dict(TEST_CHANNEL_DATA) + channel_data: ChannelData = TEST_CHANNEL_DATA.copy() + channel = Channel.from_dict(channel_data) added_video = channel.add_video(Video( video_id='video_id', name='video_name', @@ -47,8 +48,8 @@ def test_channel_add_new_video() -> None: def test_channel_add_existing_video() -> None: - channel = Channel.from_dict({ - **TEST_CHANNEL_DATA, + channel_data: ChannelData = TEST_CHANNEL_DATA.copy() + channel_data.update({ 'new_videos': { 'video_id': Video( video_id='video_id', @@ -61,6 +62,7 @@ def test_channel_add_existing_video() -> None: ), }, }) + channel = Channel.from_dict(channel_data) added_video = channel.add_video(Video( video_id='video_id', name='video_name', @@ -85,10 +87,10 @@ def test_channel_mark_video_as_watched() -> None: timestamp=datetime.fromisoformat('1970-01-01T00:00:00+00:00'), watch_timestamp=None, ) - channel = Channel.from_dict({ - **TEST_CHANNEL_DATA, - 'new_videos': {'video_id': video}, - }) + + channel_data: ChannelData = TEST_CHANNEL_DATA.copy() + channel_data.update({'new_videos': {'video_id': video}}) + channel = Channel.from_dict(channel_data) channel.mark_video_as_watched(video) diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 0ae9656..ad01097 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,4 +1,5 @@ from __future__ import annotations + from collections.abc import Iterable from aioresponses import aioresponses @@ -6,11 +7,11 @@ from aioresponses import aioresponses from tests.constants import FEED_XML, TEST_CHANNEL_DATA, TEST_VIDEO_DATA from ytrssil.config import Configuration from ytrssil.datatypes import Channel, Video -from ytrssil.fetch import AioHttpFetcher, Fetcher +from ytrssil.fetch import AioHttpFetcher, FetcherBase -def test_fetch_new_videos(): - class MockFetcher(Fetcher): +def test_fetch_new_videos() -> None: + class MockFetcher(FetcherBase): def fetch_feeds(self, urls: Iterable[str]) -> Iterable[str]: return [FEED_XML] @@ -30,7 +31,7 @@ def test_fetch_new_videos(): assert new_videos[TEST_VIDEO_DATA['video_id']] == video -def test_aiohttpfetcher_fetch_feeds(): +def test_aiohttpfetcher_fetch_feeds() -> None: feed_url = 'test_url' with aioresponses() as mocked: mocked.get( diff --git a/tests/test_parse.py b/tests/test_parse.py index c6081c5..0c17db5 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -7,8 +7,8 @@ from tests.constants import FEED_XML, TEST_CHANNEL_DATA, TEST_VIDEO_DATA from ytrssil.config import Configuration from ytrssil.datatypes import Channel, Video from ytrssil.exceptions import ChannelNotFound -from ytrssil.parse import create_feed_parser, FeedparserParser -from ytrssil.repository import ChannelRepository +from ytrssil.parse import FeedparserParser, create_feed_parser +from ytrssil.protocols import ChannelRepository def test_feedparser_channel_exists() -> None: diff --git a/ytrssil/api.py b/ytrssil/api.py index 506890b..d5d1392 100644 --- a/ytrssil/api.py +++ b/ytrssil/api.py @@ -2,8 +2,7 @@ from inject import autoparams from ytrssil.bindings import setup_dependencies from ytrssil.datatypes import Video -from ytrssil.fetch import Fetcher -from ytrssil.repository import ChannelRepository +from ytrssil.protocols import ChannelRepository, Fetcher def get_new_videos() -> list[Video]: diff --git a/ytrssil/bindings.py b/ytrssil/bindings.py index 1772076..ee94a9c 100644 --- a/ytrssil/bindings.py +++ b/ytrssil/bindings.py @@ -1,9 +1,10 @@ from inject import Binder, Injector, clear_and_configure, get_injector_or_die from ytrssil.config import Configuration -from ytrssil.fetch import Fetcher, create_fetcher -from ytrssil.parse import Parser, create_feed_parser -from ytrssil.repository import ChannelRepository, create_channel_repository +from ytrssil.fetch import create_fetcher +from ytrssil.parse import create_feed_parser +from ytrssil.protocols import ChannelRepository, Fetcher, Parser +from ytrssil.repository import create_channel_repository def dependency_configuration(binder: Binder) -> None: diff --git a/ytrssil/cli.py b/ytrssil/cli.py index 553c592..01b7037 100644 --- a/ytrssil/cli.py +++ b/ytrssil/cli.py @@ -10,8 +10,7 @@ from inject import autoparams from ytrssil.bindings import setup_dependencies from ytrssil.constants import mpv_options from ytrssil.datatypes import Video -from ytrssil.fetch import Fetcher -from ytrssil.repository import ChannelRepository +from ytrssil.protocols import ChannelRepository, Fetcher def user_query(videos: dict[str, Video], reverse: bool = False) -> list[Video]: @@ -41,13 +40,27 @@ def user_query(videos: dict[str, Video], reverse: bool = False) -> list[Video]: return ret +@autoparams() +def fetch_new_videos( + repository_manager: ChannelRepository, + fetcher: Fetcher, +) -> int: + with repository_manager as _: + _, new_videos = fetcher.fetch_new_videos() + if not new_videos: + print('No new videos', file=stderr) + return 1 + + return 0 + + @autoparams() def watch_videos( repository_manager: ChannelRepository, fetcher: Fetcher, ) -> int: with repository_manager as repository: - channels, new_videos = fetcher.fetch_new_videos() + new_videos = repository.get_new_videos() if not new_videos: print('No new videos', file=stderr) return 1 @@ -63,7 +76,7 @@ def watch_videos( execv(cmd[0], cmd) for video in selected_videos: - selected_channel = channels[video.channel_id] + selected_channel = repository.get_channel(video.channel_id) selected_channel.mark_video_as_watched(video) watch_timestamp = datetime.utcnow().replace(tzinfo=timezone.utc) repository.update_video(video, watch_timestamp) @@ -77,7 +90,7 @@ def print_url( fetcher: Fetcher, ) -> int: with repository_manager as repository: - channels, new_videos = fetcher.fetch_new_videos() + new_videos = repository.get_new_videos() if not new_videos: print('No new videos', file=stderr) return 1 @@ -88,7 +101,7 @@ def print_url( return 2 for video in selected_videos: - selected_channel = channels[video.channel_id] + selected_channel = repository.get_channel(video.channel_id) selected_channel.mark_video_as_watched(video) watch_timestamp = datetime.utcnow().replace(tzinfo=timezone.utc) repository.update_video(video, watch_timestamp) @@ -145,7 +158,9 @@ def main(args: list[str] = argv) -> int: except IndexError: command = 'watch' - if command == 'watch': + if command == 'fetch': + return fetch_new_videos() + elif command == 'watch': return watch_videos() elif command == 'print': return print_url() @@ -157,3 +172,5 @@ def main(args: list[str] = argv) -> int: else: print(f'Unknown command "{command}"', file=stderr) return 1 + + return 0 diff --git a/ytrssil/datatypes.py b/ytrssil/datatypes.py index a62797a..8113e78 100644 --- a/ytrssil/datatypes.py +++ b/ytrssil/datatypes.py @@ -1,9 +1,25 @@ from __future__ import annotations + from dataclasses import dataclass, field from datetime import datetime -from typing import Optional +from typing import Any, Optional, TypedDict -from ytrssil.types import ChannelData, VideoData + +class VideoData(TypedDict): + video_id: str + name: str + url: str + channel_id: str + channel_name: str + timestamp: datetime + watch_timestamp: Optional[datetime] + + +class ChannelData(TypedDict): + channel_id: str + name: str + new_videos: dict[str, Any] + watched_videos: dict[str, Any] @dataclass diff --git a/ytrssil/fetch.py b/ytrssil/fetch.py index 07c9c58..2f6d8b5 100644 --- a/ytrssil/fetch.py +++ b/ytrssil/fetch.py @@ -1,4 +1,3 @@ -from abc import ABCMeta, abstractmethod from asyncio import gather, run from collections.abc import Iterable @@ -7,16 +6,15 @@ from inject import autoparams from ytrssil.config import Configuration from ytrssil.datatypes import Channel, Video -from ytrssil.parse import Parser +from ytrssil.protocols import Fetcher, Parser -class Fetcher(metaclass=ABCMeta): - @abstractmethod +class FetcherBase: def fetch_feeds( self, urls: Iterable[str], ) -> Iterable[str]: # pragma: no cover - pass + raise NotImplementedError @autoparams() def fetch_new_videos( @@ -35,8 +33,12 @@ class Fetcher(metaclass=ABCMeta): return channels, new_videos -class AioHttpFetcher(Fetcher): - async def request(self, session: ClientSession, url: str) -> ClientResponse: +class AioHttpFetcher(FetcherBase): + async def request( + self, + session: ClientSession, + url: str, + ) -> ClientResponse: return await session.get(url=url) async def async_fetch_feeds(self, urls: Iterable[str]) -> Iterable[str]: diff --git a/ytrssil/parse.py b/ytrssil/parse.py index 32b0f2c..d27bc73 100644 --- a/ytrssil/parse.py +++ b/ytrssil/parse.py @@ -1,4 +1,3 @@ -from abc import ABCMeta, abstractmethod from datetime import datetime import feedparser @@ -7,20 +6,16 @@ from inject import autoparams from ytrssil.config import Configuration from ytrssil.datatypes import Channel, Video from ytrssil.exceptions import ChannelNotFound -from ytrssil.repository import ChannelRepository +from ytrssil.protocols import ChannelRepository, Parser -class Parser(metaclass=ABCMeta): +class ParserBase: @autoparams('channel_repository') def __init__(self, channel_repository: ChannelRepository) -> None: self.repository = channel_repository - @abstractmethod - def __call__(self, feed_content: str) -> Channel: # pragma: no cover - pass - -class FeedparserParser(Parser): +class FeedparserParser(ParserBase): def __call__(self, feed_content: str) -> Channel: d = feedparser.parse(feed_content) channel_id: str = d['feed']['yt_channelid'] diff --git a/ytrssil/protocols.py b/ytrssil/protocols.py new file mode 100644 index 0000000..71f52b4 --- /dev/null +++ b/ytrssil/protocols.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from datetime import datetime +from types import TracebackType +from typing import Iterable, Optional, Protocol, Type + +from ytrssil.config import Configuration +from ytrssil.datatypes import Channel, Video + + +class ChannelRepository(Protocol): + def __enter__(self) -> ChannelRepository: # pragma: no cover + ... + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: # pragma: no cover + ... + + def get_channel(self, channel_id: str) -> Channel: # pragma: no cover + ... + + def get_all_channels(self) -> list[Channel]: # pragma: no cover + ... + + def get_watched_videos(self) -> dict[str, Video]: # pragma: no cover + ... + + def get_new_videos(self) -> dict[str, Video]: # pragma: no cover + ... + + def create_channel(self, channel: Channel) -> None: # pragma: no cover + ... + + def add_new_video( + self, + channel: Channel, + video: Video, + ) -> None: # pragma: no cover + ... + + def update_video( + self, + video: Video, + watch_timestamp: datetime, + ) -> None: # pragma: no cover + ... + + +class Fetcher(Protocol): + def fetch_feeds( + self, + urls: Iterable[str], + ) -> Iterable[str]: # pragma: no cover + ... + + def fetch_new_videos( + self, + config: Optional[Configuration] = None, + parser: Optional[Parser] = None, + ) -> tuple[dict[str, Channel], dict[str, Video]]: # pragma: no cover + ... + + +class Parser(Protocol): + def __call__(self, feed_content: str) -> Channel: # pragma: no cover + ... diff --git a/ytrssil/repository.py b/ytrssil/repository.py index 7181558..22c4f1b 100644 --- a/ytrssil/repository.py +++ b/ytrssil/repository.py @@ -1,7 +1,4 @@ -from __future__ import annotations - import os -from abc import ABCMeta, abstractmethod from datetime import datetime from sqlite3 import connect from types import TracebackType @@ -13,56 +10,10 @@ from ytrssil.config import Configuration from ytrssil.constants import config_dir from ytrssil.datatypes import Channel, Video from ytrssil.exceptions import ChannelNotFound +from ytrssil.protocols import ChannelRepository -class ChannelRepository(metaclass=ABCMeta): - @abstractmethod - def __enter__(self) -> ChannelRepository: # pragma: no cover - pass - - @abstractmethod - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType], - ) -> None: # pragma: no cover - pass - - @abstractmethod - def get_channel(self, channel_id: str) -> Channel: # pragma: no cover - pass - - @abstractmethod - def get_all_channels(self) -> list[Channel]: # pragma: no cover - pass - - @abstractmethod - def get_watched_videos(self) -> dict[str, Video]: # pragma: no cover - pass - - @abstractmethod - def create_channel(self, channel: Channel) -> None: # pragma: no cover - pass - - @abstractmethod - def add_new_video( - self, - channel: Channel, - video: Video, - ) -> None: # pragma: no cover - pass - - @abstractmethod - def update_video( - self, - video: Video, - watch_timestamp: datetime, - ) -> None: # pragma: no cover - pass - - -class SqliteChannelRepository(ChannelRepository): +class SqliteChannelRepository: def __init__(self) -> None: os.makedirs(config_dir, exist_ok=True) self.file_path: str = os.path.join(config_dir, 'channels.db') @@ -202,8 +153,26 @@ class SqliteChannelRepository(ChannelRepository): cursor.execute( 'SELECT video_id, videos.name, url, timestamp, ' 'watch_timestamp, channels.channel_id, channels.name FROM videos ' - 'LEFT JOIN channels ON channels.channel_id=videos.channel_id WHERE ' - 'watch_timestamp IS NOT NULL ORDER BY timestamp' + 'LEFT JOIN channels ON channels.channel_id=videos.channel_id ' + 'WHERE watch_timestamp IS NOT NULL ORDER BY timestamp' + ) + + return { + video_data[0]: self.video_data_to_video( + video_data=video_data, + channel_id=video_data[5], + channel_name=video_data[6], + ) + for video_data in cursor + } + + def get_new_videos(self) -> dict[str, Video]: + cursor = self.connection.cursor() + cursor.execute( + 'SELECT video_id, videos.name, url, timestamp, ' + 'watch_timestamp, channels.channel_id, channels.name FROM videos ' + 'LEFT JOIN channels ON channels.channel_id=videos.channel_id ' + 'WHERE watch_timestamp IS NULL ORDER BY timestamp' ) return { diff --git a/ytrssil/types.py b/ytrssil/types.py deleted file mode 100644 index 134a88a..0000000 --- a/ytrssil/types.py +++ /dev/null @@ -1,18 +0,0 @@ -from datetime import datetime -from typing import Optional, TypedDict - - -class VideoData(TypedDict): - video_id: str - name: str - url: str - channel_id: str - channel_name: str - timestamp: datetime - watch_timestamp: Optional[datetime] - - -class ChannelData(TypedDict): - channel_id: str - name: str - new_videos: dict