ytrssil
/
cli
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Add dedicated fetch command instead of always checking 

Use Protocol instead of ABCMeta
This commit is contained in:
Pavle Portic 2021-11-19 19:44:46 +01:00
parent f410f7d302
commit bd7fcb35bd
Signed by: TheEdgeOfRage
GPG Key ID: F2AB38285780DE3D
17 changed files with 186 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.mypy.ini
View File

@ -4,4 +4,7 @@ ignore_missing_imports=true
[mypy-feedparser.*]
ignore_missing_imports=true
[mypy-pytest_mock.*]
ignore_missing_imports=true
[mypy]

3
readme.md
View File

@ -4,6 +4,9 @@ This is a simple CLI to manage YouTube subscriptions through RSS feeds
and watch new videos using `mpv`. It keeps track of watched videos in a local
sqlite database.
**This tool is still in early development and breaking changes across minor
versions are expected.**
## Configuration
It looks for a list of RSS URLs in `$XDG_CONFIG_HOME/ytrssil/feeds`

2
requirements-dev.txt
View File

@ -1,7 +1,7 @@
aioresponses==0.7.*
flake8==3.*
jedi==0.*
mypy==0.*
pytest==6.2.*
pytest-cov==3.0.0
pytest-mock==3.6.*
wheel==0.37.*

6
tests/constants.py
View File

@ -1,7 +1,6 @@
from datetime import datetime
from ytrssil.types import ChannelData, VideoData
from ytrssil.datatypes import ChannelData, VideoData
FEED_XML: str = '''
<?xml version="1.0" encoding="UTF-8"?>
@ -30,5 +29,6 @@ TEST_VIDEO_DATA: VideoData = {
TEST_CHANNEL_DATA: ChannelData = {
'channel_id': 'channel_id',
'name': 'channel_name',
'new_videos': {}
'new_videos': {},
'watched_videos': {},
}

7
tests/test_bindings.py
View File

@ -1,8 +1,9 @@
from ytrssil.bindings import setup_dependencies
from ytrssil.config import Configuration
from ytrssil.fetch import Fetcher, AioHttpFetcher
from ytrssil.parse import Parser, FeedparserParser
from ytrssil.repository import ChannelRepository, SqliteChannelRepository
from ytrssil.fetch import AioHttpFetcher
from ytrssil.parse import FeedparserParser
from ytrssil.protocols import ChannelRepository, Fetcher, Parser
from ytrssil.repository import SqliteChannelRepository
def test_setup_dependencies() -> None:

26
tests/test_datatypes.py
View File

@ -1,7 +1,7 @@
from datetime import datetime
from tests.constants import TEST_CHANNEL_DATA, TEST_VIDEO_DATA
from ytrssil.datatypes import Channel, Video
from ytrssil.datatypes import Channel, ChannelData, Video
def test_video_str() -> None:
@ -11,8 +11,8 @@ def test_video_str() -> None:
def test_channel_str() -> None:
channel = Channel.from_dict({
**TEST_CHANNEL_DATA,
channel_data: ChannelData = TEST_CHANNEL_DATA.copy()
channel_data.update({
'new_videos': {
'video_id': Video(
video_id='video_id',
@ -25,13 +25,14 @@ def test_channel_str() -> None:
),
},
})
string = str(channel)
channel_string = str(Channel.from_dict(channel_data))
assert string == 'channel_name - 1'
assert channel_string == 'channel_name - 1'
def test_channel_add_new_video() -> None:
channel = Channel.from_dict(TEST_CHANNEL_DATA)
channel_data: ChannelData = TEST_CHANNEL_DATA.copy()
channel = Channel.from_dict(channel_data)
added_video = channel.add_video(Video(
video_id='video_id',
name='video_name',
@ -47,8 +48,8 @@ def test_channel_add_new_video() -> None:
def test_channel_add_existing_video() -> None:
channel = Channel.from_dict({
**TEST_CHANNEL_DATA,
channel_data: ChannelData = TEST_CHANNEL_DATA.copy()
channel_data.update({
'new_videos': {
'video_id': Video(
video_id='video_id',
@ -61,6 +62,7 @@ def test_channel_add_existing_video() -> None:
),
},
})
channel = Channel.from_dict(channel_data)
added_video = channel.add_video(Video(
video_id='video_id',
name='video_name',
@ -85,10 +87,10 @@ def test_channel_mark_video_as_watched() -> None:
timestamp=datetime.fromisoformat('1970-01-01T00:00:00+00:00'),
watch_timestamp=None,
)
channel = Channel.from_dict({
**TEST_CHANNEL_DATA,
'new_videos': {'video_id': video},
})
channel_data: ChannelData = TEST_CHANNEL_DATA.copy()
channel_data.update({'new_videos': {'video_id': video}})
channel = Channel.from_dict(channel_data)
channel.mark_video_as_watched(video)

9
tests/test_fetch.py
View File

@ -1,4 +1,5 @@
from __future__ import annotations
from collections.abc import Iterable
from aioresponses import aioresponses
@ -6,11 +7,11 @@ from aioresponses import aioresponses
from tests.constants import FEED_XML, TEST_CHANNEL_DATA, TEST_VIDEO_DATA
from ytrssil.config import Configuration
from ytrssil.datatypes import Channel, Video
from ytrssil.fetch import AioHttpFetcher, Fetcher
from ytrssil.fetch import AioHttpFetcher, FetcherBase
def test_fetch_new_videos():
class MockFetcher(Fetcher):
def test_fetch_new_videos() -> None:
class MockFetcher(FetcherBase):
def fetch_feeds(self, urls: Iterable[str]) -> Iterable[str]:
return [FEED_XML]
@ -30,7 +31,7 @@ def test_fetch_new_videos():
assert new_videos[TEST_VIDEO_DATA['video_id']] == video
def test_aiohttpfetcher_fetch_feeds():
def test_aiohttpfetcher_fetch_feeds() -> None:
feed_url = 'test_url'
with aioresponses() as mocked:
mocked.get(

4
tests/test_parse.py
View File

@ -7,8 +7,8 @@ from tests.constants import FEED_XML, TEST_CHANNEL_DATA, TEST_VIDEO_DATA
from ytrssil.config import Configuration
from ytrssil.datatypes import Channel, Video
from ytrssil.exceptions import ChannelNotFound
from ytrssil.parse import create_feed_parser, FeedparserParser
from ytrssil.repository import ChannelRepository
from ytrssil.parse import FeedparserParser, create_feed_parser
from ytrssil.protocols import ChannelRepository
def test_feedparser_channel_exists() -> None:

3
ytrssil/api.py
View File

@ -2,8 +2,7 @@ from inject import autoparams
from ytrssil.bindings import setup_dependencies
from ytrssil.datatypes import Video
from ytrssil.fetch import Fetcher
from ytrssil.repository import ChannelRepository
from ytrssil.protocols import ChannelRepository, Fetcher
def get_new_videos() -> list[Video]:

7
ytrssil/bindings.py
View File

@ -1,9 +1,10 @@
from inject import Binder, Injector, clear_and_configure, get_injector_or_die
from ytrssil.config import Configuration
from ytrssil.fetch import Fetcher, create_fetcher
from ytrssil.parse import Parser, create_feed_parser
from ytrssil.repository import ChannelRepository, create_channel_repository
from ytrssil.fetch import create_fetcher
from ytrssil.parse import create_feed_parser
from ytrssil.protocols import ChannelRepository, Fetcher, Parser
from ytrssil.repository import create_channel_repository
def dependency_configuration(binder: Binder) -> None:

31
ytrssil/cli.py
View File

@ -10,8 +10,7 @@ from inject import autoparams
from ytrssil.bindings import setup_dependencies
from ytrssil.constants import mpv_options
from ytrssil.datatypes import Video
from ytrssil.fetch import Fetcher
from ytrssil.repository import ChannelRepository
from ytrssil.protocols import ChannelRepository, Fetcher
def user_query(videos: dict[str, Video], reverse: bool = False) -> list[Video]:
@ -41,13 +40,27 @@ def user_query(videos: dict[str, Video], reverse: bool = False) -> list[Video]:
return ret
@autoparams()
def fetch_new_videos(
repository_manager: ChannelRepository,
fetcher: Fetcher,
) -> int:
with repository_manager as _:
_, new_videos = fetcher.fetch_new_videos()
if not new_videos:
print('No new videos', file=stderr)
return 1
return 0
@autoparams()
def watch_videos(
repository_manager: ChannelRepository,
fetcher: Fetcher,
) -> int:
with repository_manager as repository:
channels, new_videos = fetcher.fetch_new_videos()
new_videos = repository.get_new_videos()
if not new_videos:
print('No new videos', file=stderr)
return 1
@ -63,7 +76,7 @@ def watch_videos(
execv(cmd[0], cmd)
for video in selected_videos:
selected_channel = channels[video.channel_id]
selected_channel = repository.get_channel(video.channel_id)
selected_channel.mark_video_as_watched(video)
watch_timestamp = datetime.utcnow().replace(tzinfo=timezone.utc)
repository.update_video(video, watch_timestamp)
@ -77,7 +90,7 @@ def print_url(
fetcher: Fetcher,
) -> int:
with repository_manager as repository:
channels, new_videos = fetcher.fetch_new_videos()
new_videos = repository.get_new_videos()
if not new_videos:
print('No new videos', file=stderr)
return 1
@ -88,7 +101,7 @@ def print_url(
return 2
for video in selected_videos:
selected_channel = channels[video.channel_id]
selected_channel = repository.get_channel(video.channel_id)
selected_channel.mark_video_as_watched(video)
watch_timestamp = datetime.utcnow().replace(tzinfo=timezone.utc)
repository.update_video(video, watch_timestamp)
@ -145,7 +158,9 @@ def main(args: list[str] = argv) -> int:
except IndexError:
command = 'watch'
if command == 'watch':
if command == 'fetch':
return fetch_new_videos()
elif command == 'watch':
return watch_videos()
elif command == 'print':
return print_url()
@ -157,3 +172,5 @@ def main(args: list[str] = argv) -> int:
else:
print(f'Unknown command "{command}"', file=stderr)
return 1
return 0

20
ytrssil/datatypes.py
View File

@ -1,9 +1,25 @@
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from typing import Optional
from typing import Any, Optional, TypedDict
from ytrssil.types import ChannelData, VideoData
class VideoData(TypedDict):
video_id: str
name: str
url: str
channel_id: str
channel_name: str
timestamp: datetime
watch_timestamp: Optional[datetime]
class ChannelData(TypedDict):
channel_id: str
name: str
new_videos: dict[str, Any]
watched_videos: dict[str, Any]
@dataclass

16
ytrssil/fetch.py
View File

@ -1,4 +1,3 @@
from abc import ABCMeta, abstractmethod
from asyncio import gather, run
from collections.abc import Iterable
@ -7,16 +6,15 @@ from inject import autoparams
from ytrssil.config import Configuration
from ytrssil.datatypes import Channel, Video
from ytrssil.parse import Parser
from ytrssil.protocols import Fetcher, Parser
class Fetcher(metaclass=ABCMeta):
@abstractmethod
class FetcherBase:
def fetch_feeds(
self,
urls: Iterable[str],
) -> Iterable[str]: # pragma: no cover
pass
raise NotImplementedError
@autoparams()
def fetch_new_videos(
@ -35,8 +33,12 @@ class Fetcher(metaclass=ABCMeta):
return channels, new_videos
class AioHttpFetcher(Fetcher):
async def request(self, session: ClientSession, url: str) -> ClientResponse:
class AioHttpFetcher(FetcherBase):
async def request(
self,
session: ClientSession,
url: str,
) -> ClientResponse:
return await session.get(url=url)
async def async_fetch_feeds(self, urls: Iterable[str]) -> Iterable[str]:

11
ytrssil/parse.py
View File

@ -1,4 +1,3 @@
from abc import ABCMeta, abstractmethod
from datetime import datetime
import feedparser
@ -7,20 +6,16 @@ from inject import autoparams
from ytrssil.config import Configuration
from ytrssil.datatypes import Channel, Video
from ytrssil.exceptions import ChannelNotFound
from ytrssil.repository import ChannelRepository
from ytrssil.protocols import ChannelRepository, Parser
class Parser(metaclass=ABCMeta):
class ParserBase:
@autoparams('channel_repository')
def __init__(self, channel_repository: ChannelRepository) -> None:
self.repository = channel_repository
@abstractmethod
def __call__(self, feed_content: str) -> Channel: # pragma: no cover
pass
class FeedparserParser(Parser):
class FeedparserParser(ParserBase):
def __call__(self, feed_content: str) -> Channel:
d = feedparser.parse(feed_content)
channel_id: str = d['feed']['yt_channelid']

70
ytrssil/protocols.py Normal file
View File

@ -0,0 +1,70 @@
from __future__ import annotations
from datetime import datetime
from types import TracebackType
from typing import Iterable, Optional, Protocol, Type
from ytrssil.config import Configuration
from ytrssil.datatypes import Channel, Video
class ChannelRepository(Protocol):
def __enter__(self) -> ChannelRepository: # pragma: no cover
...
def __exit__(
self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType],
) -> None: # pragma: no cover
...
def get_channel(self, channel_id: str) -> Channel: # pragma: no cover
...
def get_all_channels(self) -> list[Channel]: # pragma: no cover
...
def get_watched_videos(self) -> dict[str, Video]: # pragma: no cover
...
def get_new_videos(self) -> dict[str, Video]: # pragma: no cover
...
def create_channel(self, channel: Channel) -> None: # pragma: no cover
...
def add_new_video(
self,
channel: Channel,
video: Video,
) -> None: # pragma: no cover
...
def update_video(
self,
video: Video,
watch_timestamp: datetime,
) -> None: # pragma: no cover
...
class Fetcher(Protocol):
def fetch_feeds(
self,
urls: Iterable[str],
) -> Iterable[str]: # pragma: no cover
...
def fetch_new_videos(
self,
config: Optional[Configuration] = None,
parser: Optional[Parser] = None,
) -> tuple[dict[str, Channel], dict[str, Video]]: # pragma: no cover
...
class Parser(Protocol):
def __call__(self, feed_content: str) -> Channel: # pragma: no cover
...

75
ytrssil/repository.py
View File

@ -1,7 +1,4 @@
from __future__ import annotations
import os
from abc import ABCMeta, abstractmethod
from datetime import datetime
from sqlite3 import connect
from types import TracebackType
@ -13,56 +10,10 @@ from ytrssil.config import Configuration
from ytrssil.constants import config_dir
from ytrssil.datatypes import Channel, Video
from ytrssil.exceptions import ChannelNotFound
from ytrssil.protocols import ChannelRepository
class ChannelRepository(metaclass=ABCMeta):
@abstractmethod
def __enter__(self) -> ChannelRepository: # pragma: no cover
pass
@abstractmethod
def __exit__(
self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType],
) -> None: # pragma: no cover
pass
@abstractmethod
def get_channel(self, channel_id: str) -> Channel: # pragma: no cover
pass
@abstractmethod
def get_all_channels(self) -> list[Channel]: # pragma: no cover
pass
@abstractmethod
def get_watched_videos(self) -> dict[str, Video]: # pragma: no cover
pass
@abstractmethod
def create_channel(self, channel: Channel) -> None: # pragma: no cover
pass
@abstractmethod
def add_new_video(
self,
channel: Channel,
video: Video,
) -> None: # pragma: no cover
pass
@abstractmethod
def update_video(
self,
video: Video,
watch_timestamp: datetime,
) -> None: # pragma: no cover
pass
class SqliteChannelRepository(ChannelRepository):
class SqliteChannelRepository:
def __init__(self) -> None:
os.makedirs(config_dir, exist_ok=True)
self.file_path: str = os.path.join(config_dir, 'channels.db')
@ -202,8 +153,26 @@ class SqliteChannelRepository(ChannelRepository):
cursor.execute(
'SELECT video_id, videos.name, url, timestamp, '
'watch_timestamp, channels.channel_id, channels.name FROM videos '
'LEFT JOIN channels ON channels.channel_id=videos.channel_id WHERE '
'watch_timestamp IS NOT NULL ORDER BY timestamp'
'LEFT JOIN channels ON channels.channel_id=videos.channel_id '
'WHERE watch_timestamp IS NOT NULL ORDER BY timestamp'
)
return {
video_data[0]: self.video_data_to_video(
video_data=video_data,
channel_id=video_data[5],
channel_name=video_data[6],
)
for video_data in cursor
}
def get_new_videos(self) -> dict[str, Video]:
cursor = self.connection.cursor()
cursor.execute(
'SELECT video_id, videos.name, url, timestamp, '
'watch_timestamp, channels.channel_id, channels.name FROM videos '
'LEFT JOIN channels ON channels.channel_id=videos.channel_id '
'WHERE watch_timestamp IS NULL ORDER BY timestamp'
)
return {

18
ytrssil/types.py
View File

@ -1,18 +0,0 @@
from datetime import datetime
from typing import Optional, TypedDict
class VideoData(TypedDict):
video_id: str
name: str
url: str
channel_id: str
channel_name: str
timestamp: datetime
watch_timestamp: Optional[datetime]
class ChannelData(TypedDict):
channel_id: str
name: str
new_videos: dict