Switch to SQLite3 as storage backend and add dependency injection

This commit is contained in:
Pavle Portic 2021-08-01 17:02:15 +02:00
parent bc4d70ae04
commit a031f90765
Signed by: TheEdgeOfRage
GPG Key ID: F2AB38285780DE3D
11 changed files with 265 additions and 33 deletions

4
.mypy.ini Normal file
View File

@ -0,0 +1,4 @@
[mypy-feedparser.*]
ignore_missing_imports=true
[mypy]

View File

@ -1,2 +1,3 @@
aiohttp==3.7.*
feedparser==6.0.*
inject==4.3.*

17
ytrssil/bindings.py Normal file
View File

@ -0,0 +1,17 @@
from inject import Binder, configure
from ytrssil.config import Configuration
from ytrssil.parser import Parser, create_feed_parser
from ytrssil.repository import ChannelRepository, create_channel_repository
def dependency_configuration(binder: Binder) -> None:
config = Configuration()
binder.bind(Configuration, config)
binder.bind_to_provider
binder.bind_to_constructor(ChannelRepository, create_channel_repository)
binder.bind_to_constructor(Parser, create_feed_parser)
def setup_dependencies() -> None:
configure(dependency_configuration)

View File

@ -1,20 +1,26 @@
from os import execv, fork
from sys import stderr
from inject import autoparams
from ytrssil.bindings import setup_dependencies
from ytrssil.constants import mpv_options
from ytrssil.repository import ChannelRepository
from ytrssil.fetch import fetch_new_videos
from ytrssil.query import query
from ytrssil.repository import ChannelRepository
def main() -> int:
with ChannelRepository() as repository:
channels, new_videos = fetch_new_videos(repository)
class NoVideoSelected(Exception):
pass
@autoparams()
def run(repository_manager: ChannelRepository) -> None:
with repository_manager as repository:
channels, new_videos = fetch_new_videos()
selected_videos = query(new_videos)
if not selected_videos:
print('No video selected', file=stderr)
return 1
raise NoVideoSelected
video_urls = [video.url for video in selected_videos]
cmd = ['/usr/bin/mpv', *mpv_options, *video_urls]
@ -24,7 +30,16 @@ def main() -> int:
for video in selected_videos:
selected_channel = channels[video.channel_id]
selected_channel.mark_video_as_watched(video)
repository.update_channel(selected_channel)
repository.update_video(video, True)
def main() -> int:
setup_dependencies()
try:
run()
except NoVideoSelected:
print('No video selected', file=stderr)
return 1
return 0

View File

@ -1,9 +1,16 @@
import os
from collections.abc import Iterator
from dataclasses import dataclass
from ytrssil.constants import config_dir
@dataclass
class Configuration:
channel_repository_type: str = 'sqlite'
feed_parser_type: str = 'feedparser'
def get_feed_urls() -> Iterator[str]:
file_path = os.path.join(config_dir, 'feeds')
with open(file_path, 'r') as f:

View File

@ -1,6 +1,5 @@
import os
config_prefix: str
try:
config_prefix = os.environ['XDG_CONFIG_HOME']
@ -10,5 +9,5 @@ except KeyError:
config_dir: str = os.path.join(config_prefix, 'ytrssil')
mpv_options: list[str] = [
'--no-terminal',
'--ytdl-format=bestvideo[height<=?1080][vcodec!=vp9]+bestaudio/best',
'--ytdl-format=bestvideo[height<=?1080]+bestaudio/best',
]

View File

@ -23,11 +23,15 @@ class Channel:
new_videos: dict[str, Video] = field(default_factory=lambda: dict())
watched_videos: dict[str, Video] = field(default_factory=lambda: dict())
def add_video(self, video: Video) -> None:
if video.video_id in self.watched_videos:
return
def add_video(self, video: Video) -> bool:
if (
video.video_id in self.watched_videos
or video.video_id in self.new_videos
):
return False
self.new_videos[video.video_id] = video
return True
def remove_old_videos(self) -> None:
vid_list: list[Video] = sorted(
@ -41,3 +45,6 @@ class Channel:
self.new_videos.pop(video.video_id)
self.watched_videos[video.video_id] = video
self.remove_old_videos()
def __str__(self) -> str:
return f'{self.name} - {len(self.new_videos)}'

View File

@ -2,11 +2,12 @@ from asyncio import gather, run
from collections.abc import Iterable
from aiohttp import ClientResponse, ClientSession
from inject import autoparams
from ytrssil.config import get_feed_urls
from ytrssil.datatypes import Channel, Video
from ytrssil.repository import ChannelRepository
from ytrssil.parser import Parser
from ytrssil.repository import ChannelRepository
async def request(session: ClientSession, url: str) -> ClientResponse:
@ -24,11 +25,13 @@ async def fetch_feeds(urls: Iterable[str]) -> Iterable[str]:
]
@autoparams('parser', 'repository')
def fetch_new_videos(
*,
parser: Parser,
repository: ChannelRepository,
) -> tuple[dict[str, Channel], dict[str, Video]]:
feed_urls = get_feed_urls()
parser = Parser(repository)
channels: dict[str, Channel] = {}
new_videos: dict[str, Video] = {}
for feed in run(fetch_feeds(feed_urls)):

View File

@ -1,15 +1,25 @@
from abc import ABCMeta, abstractmethod
from datetime import datetime
import feedparser
from inject import autoparams
from ytrssil.config import Configuration
from ytrssil.datatypes import Channel, Video
from ytrssil.repository import ChannelNotFound, ChannelRepository
class Parser:
class Parser(metaclass=ABCMeta):
@autoparams('channel_repository')
def __init__(self, channel_repository: ChannelRepository) -> None:
self.repository = channel_repository
@abstractmethod
def __call__(self, feed_content: str) -> Channel:
pass
class FeedparserParser(Parser):
def __call__(self, feed_content: str) -> Channel:
d = feedparser.parse(feed_content)
channel_id: str = d['feed']['yt_channelid']
@ -21,17 +31,27 @@ class Parser:
name=d['feed']['title'],
url=d['feed']['link'],
)
self.repository.update_channel(channel)
self.repository.create_channel(channel)
for entry in d['entries']:
channel.add_video(Video(
video = Video(
video_id=entry['yt_videoid'],
name=entry['title'],
url=entry['link'],
timestamp=datetime.fromisoformat(entry['published']),
channel_id=channel.channel_id,
channel_name=channel.name,
))
)
if channel.add_video(video):
self.repository.add_new_video(channel, video)
self.repository.update_channel(channel)
return channel
@autoparams()
def create_feed_parser(config: Configuration) -> Parser:
parser_type = config.feed_parser_type
if parser_type == 'feedparser':
return FeedparserParser()
else:
raise Exception(f'Unknown feed parser type: "{parser_type}"')

View File

@ -1,4 +1,4 @@
from subprocess import Popen, PIPE
from subprocess import PIPE, Popen
from ytrssil.datatypes import Video

View File

@ -1,32 +1,191 @@
from __future__ import annotations
import os
import shelve
import os
from abc import ABCMeta, abstractmethod
from datetime import datetime
from sqlite3 import connect
from typing import Any, Union
from inject import autoparams
from ytrssil.config import Configuration
from ytrssil.constants import config_dir
from ytrssil.datatypes import Channel
from ytrssil.datatypes import Channel, Video
class ChannelNotFound(Exception):
pass
class ChannelRepository:
class ChannelRepository(metaclass=ABCMeta):
@abstractmethod
def __enter__(self) -> ChannelRepository:
pass
@abstractmethod
def __exit__(
self,
exc_type: Any,
exc_value: Any,
exc_traceback: Any,
) -> None:
pass
@abstractmethod
def get_channel(self, channel_id: str) -> Channel:
pass
@abstractmethod
def create_channel(self, channel: Channel) -> None:
pass
@abstractmethod
def add_new_video(self, channel: Channel, video: Video) -> None:
pass
@abstractmethod
def update_video(self, video: Video, watched: bool) -> None:
pass
class SqliteChannelRepository(ChannelRepository):
def __init__(self) -> None:
os.makedirs(config_dir, exist_ok=True)
self.file_path: str = os.path.join(config_dir, 'shelf')
self.file_path: str = os.path.join(config_dir, 'channels.db')
self.setup_database()
def setup_database(self) -> None:
connection = connect(self.file_path)
cursor = connection.cursor()
cursor.execute('PRAGMA foreign_keys = ON')
cursor.execute(
'CREATE TABLE IF NOT EXISTS channels ('
'channel_id VARCHAR PRIMARY KEY, name VARCHAR, url VARCHAR UNIQUE)'
)
cursor.execute(
'CREATE TABLE IF NOT EXISTS videos ('
'video_id VARCHAR PRIMARY KEY, name VARCHAR, url VARCHAR UNIQUE, '
'timestamp VARCHAR, watched BOOLEAN, channel_id VARCHAR, '
'FOREIGN KEY(channel_id) REFERENCES channels(channel_id))'
)
connection.commit()
connection.close()
def __enter__(self) -> ChannelRepository:
self.repository = shelve.open(self.file_path)
self.connection = connect(self.file_path)
return self
def __exit__(self, exc_type, exc_value, exc_traceback) -> None:
self.repository.close()
def __exit__(
self,
exc_type: Any,
exc_value: Any,
exc_traceback: Any,
) -> None:
self.connection.close()
def get_channel_as_dict(self, channel: Channel) -> dict[str, str]:
return {
'channel_id': channel.channel_id,
'name': channel.name,
'url': channel.url,
}
def get_video_as_dict(self, video: Video, watched: bool) -> dict[str, Any]:
return {
'video_id': video.video_id,
'name': video.name,
'url': video.url,
'timestamp': video.timestamp.isoformat(),
'watched': watched,
'channel_id': video.channel_id,
}
def get_videos(self, channel: Channel) -> list[tuple[Video, bool]]:
cursor = self.connection.cursor()
cursor.execute(
'SELECT video_id, name, url, timestamp, watched '
'FROM videos WHERE channel_id=:channel_id',
{'channel_id': channel.channel_id},
)
ret: list[tuple[Video, bool]] = []
video_data: tuple[str, str, str, str, bool]
for video_data in cursor.fetchall():
ret.append((
Video(
video_id=video_data[0],
name=video_data[1],
url=video_data[2],
timestamp=datetime.fromisoformat(video_data[3]),
channel_id=channel.channel_id,
channel_name=channel.name,
),
video_data[4]
))
return ret
def get_channel(self, channel_id: str) -> Channel:
try:
return self.repository[channel_id]
except KeyError as e:
raise ChannelNotFound(e)
cursor = self.connection.cursor()
cursor.execute(
'SELECT * FROM channels WHERE channel_id=:channel_id',
{'channel_id': channel_id},
)
channel_data: Union[tuple[str, str, str], None] = cursor.fetchone()
if channel_data is None:
raise ChannelNotFound
channel = Channel(
channel_id=channel_data[0],
name=channel_data[1],
url=channel_data[2],
)
for video, watched in self.get_videos(channel):
if watched:
channel.watched_videos[video.video_id] = video
else:
channel.new_videos[video.video_id] = video
return channel
def create_channel(self, channel: Channel) -> None:
cursor = self.connection.cursor()
cursor.execute(
'INSERT INTO channels VALUES (:channel_id, :name, :url)',
self.get_channel_as_dict(channel),
)
self.connection.commit()
def update_channel(self, channel: Channel) -> None:
self.repository[channel.channel_id] = channel
cursor = self.connection.cursor()
cursor.execute(
'UPDATE channels SET channel_id = :channel_id, name = :name, '
'url = :url WHERE channel_id=:channel_id',
self.get_channel_as_dict(channel),
)
self.connection.commit()
def add_new_video(self, channel: Channel, video: Video) -> None:
cursor = self.connection.cursor()
cursor.execute(
'INSERT INTO videos VALUES '
'(:video_id, :name, :url, :timestamp, :watched, :channel_id)',
self.get_video_as_dict(video, False),
)
self.connection.commit()
def update_video(self, video: Video, watched: bool) -> None:
cursor = self.connection.cursor()
cursor.execute(
'UPDATE videos SET watched = :watched WHERE video_id=:video_id',
{'watched': watched, 'video_id': video.video_id},
)
self.connection.commit()
@autoparams()
def create_channel_repository(config: Configuration) -> ChannelRepository:
repo_type = config.channel_repository_type
if repo_type == 'sqlite':
return SqliteChannelRepository()
else:
raise Exception(f'Unknown channel repository type: "{repo_type}"')