Switch to SQLite3 as storage backend and add dependency injection
This commit is contained in:
parent
bc4d70ae04
commit
a031f90765
|
@ -0,0 +1,4 @@
|
||||||
|
[mypy-feedparser.*]
|
||||||
|
ignore_missing_imports=true
|
||||||
|
|
||||||
|
[mypy]
|
|
@ -1,2 +1,3 @@
|
||||||
aiohttp==3.7.*
|
aiohttp==3.7.*
|
||||||
feedparser==6.0.*
|
feedparser==6.0.*
|
||||||
|
inject==4.3.*
|
||||||
|
|
|
@ -0,0 +1,17 @@
|
||||||
|
from inject import Binder, configure
|
||||||
|
|
||||||
|
from ytrssil.config import Configuration
|
||||||
|
from ytrssil.parser import Parser, create_feed_parser
|
||||||
|
from ytrssil.repository import ChannelRepository, create_channel_repository
|
||||||
|
|
||||||
|
|
||||||
|
def dependency_configuration(binder: Binder) -> None:
|
||||||
|
config = Configuration()
|
||||||
|
binder.bind(Configuration, config)
|
||||||
|
binder.bind_to_provider
|
||||||
|
binder.bind_to_constructor(ChannelRepository, create_channel_repository)
|
||||||
|
binder.bind_to_constructor(Parser, create_feed_parser)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_dependencies() -> None:
|
||||||
|
configure(dependency_configuration)
|
|
@ -1,20 +1,26 @@
|
||||||
from os import execv, fork
|
from os import execv, fork
|
||||||
from sys import stderr
|
from sys import stderr
|
||||||
|
|
||||||
|
from inject import autoparams
|
||||||
|
|
||||||
|
from ytrssil.bindings import setup_dependencies
|
||||||
from ytrssil.constants import mpv_options
|
from ytrssil.constants import mpv_options
|
||||||
from ytrssil.repository import ChannelRepository
|
|
||||||
from ytrssil.fetch import fetch_new_videos
|
from ytrssil.fetch import fetch_new_videos
|
||||||
from ytrssil.query import query
|
from ytrssil.query import query
|
||||||
|
from ytrssil.repository import ChannelRepository
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
class NoVideoSelected(Exception):
|
||||||
with ChannelRepository() as repository:
|
pass
|
||||||
channels, new_videos = fetch_new_videos(repository)
|
|
||||||
|
|
||||||
|
|
||||||
|
@autoparams()
|
||||||
|
def run(repository_manager: ChannelRepository) -> None:
|
||||||
|
with repository_manager as repository:
|
||||||
|
channels, new_videos = fetch_new_videos()
|
||||||
selected_videos = query(new_videos)
|
selected_videos = query(new_videos)
|
||||||
if not selected_videos:
|
if not selected_videos:
|
||||||
print('No video selected', file=stderr)
|
raise NoVideoSelected
|
||||||
return 1
|
|
||||||
|
|
||||||
video_urls = [video.url for video in selected_videos]
|
video_urls = [video.url for video in selected_videos]
|
||||||
cmd = ['/usr/bin/mpv', *mpv_options, *video_urls]
|
cmd = ['/usr/bin/mpv', *mpv_options, *video_urls]
|
||||||
|
@ -24,7 +30,16 @@ def main() -> int:
|
||||||
for video in selected_videos:
|
for video in selected_videos:
|
||||||
selected_channel = channels[video.channel_id]
|
selected_channel = channels[video.channel_id]
|
||||||
selected_channel.mark_video_as_watched(video)
|
selected_channel.mark_video_as_watched(video)
|
||||||
repository.update_channel(selected_channel)
|
repository.update_video(video, True)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
setup_dependencies()
|
||||||
|
try:
|
||||||
|
run()
|
||||||
|
except NoVideoSelected:
|
||||||
|
print('No video selected', file=stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
|
@ -1,9 +1,16 @@
|
||||||
import os
|
import os
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from ytrssil.constants import config_dir
|
from ytrssil.constants import config_dir
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Configuration:
|
||||||
|
channel_repository_type: str = 'sqlite'
|
||||||
|
feed_parser_type: str = 'feedparser'
|
||||||
|
|
||||||
|
|
||||||
def get_feed_urls() -> Iterator[str]:
|
def get_feed_urls() -> Iterator[str]:
|
||||||
file_path = os.path.join(config_dir, 'feeds')
|
file_path = os.path.join(config_dir, 'feeds')
|
||||||
with open(file_path, 'r') as f:
|
with open(file_path, 'r') as f:
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
|
||||||
config_prefix: str
|
config_prefix: str
|
||||||
try:
|
try:
|
||||||
config_prefix = os.environ['XDG_CONFIG_HOME']
|
config_prefix = os.environ['XDG_CONFIG_HOME']
|
||||||
|
@ -10,5 +9,5 @@ except KeyError:
|
||||||
config_dir: str = os.path.join(config_prefix, 'ytrssil')
|
config_dir: str = os.path.join(config_prefix, 'ytrssil')
|
||||||
mpv_options: list[str] = [
|
mpv_options: list[str] = [
|
||||||
'--no-terminal',
|
'--no-terminal',
|
||||||
'--ytdl-format=bestvideo[height<=?1080][vcodec!=vp9]+bestaudio/best',
|
'--ytdl-format=bestvideo[height<=?1080]+bestaudio/best',
|
||||||
]
|
]
|
||||||
|
|
|
@ -23,11 +23,15 @@ class Channel:
|
||||||
new_videos: dict[str, Video] = field(default_factory=lambda: dict())
|
new_videos: dict[str, Video] = field(default_factory=lambda: dict())
|
||||||
watched_videos: dict[str, Video] = field(default_factory=lambda: dict())
|
watched_videos: dict[str, Video] = field(default_factory=lambda: dict())
|
||||||
|
|
||||||
def add_video(self, video: Video) -> None:
|
def add_video(self, video: Video) -> bool:
|
||||||
if video.video_id in self.watched_videos:
|
if (
|
||||||
return
|
video.video_id in self.watched_videos
|
||||||
|
or video.video_id in self.new_videos
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
|
||||||
self.new_videos[video.video_id] = video
|
self.new_videos[video.video_id] = video
|
||||||
|
return True
|
||||||
|
|
||||||
def remove_old_videos(self) -> None:
|
def remove_old_videos(self) -> None:
|
||||||
vid_list: list[Video] = sorted(
|
vid_list: list[Video] = sorted(
|
||||||
|
@ -41,3 +45,6 @@ class Channel:
|
||||||
self.new_videos.pop(video.video_id)
|
self.new_videos.pop(video.video_id)
|
||||||
self.watched_videos[video.video_id] = video
|
self.watched_videos[video.video_id] = video
|
||||||
self.remove_old_videos()
|
self.remove_old_videos()
|
||||||
|
|
||||||
|
def __str__(self) -> str:
|
||||||
|
return f'{self.name} - {len(self.new_videos)}'
|
||||||
|
|
|
@ -2,11 +2,12 @@ from asyncio import gather, run
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
|
||||||
from aiohttp import ClientResponse, ClientSession
|
from aiohttp import ClientResponse, ClientSession
|
||||||
|
from inject import autoparams
|
||||||
|
|
||||||
from ytrssil.config import get_feed_urls
|
from ytrssil.config import get_feed_urls
|
||||||
from ytrssil.datatypes import Channel, Video
|
from ytrssil.datatypes import Channel, Video
|
||||||
from ytrssil.repository import ChannelRepository
|
|
||||||
from ytrssil.parser import Parser
|
from ytrssil.parser import Parser
|
||||||
|
from ytrssil.repository import ChannelRepository
|
||||||
|
|
||||||
|
|
||||||
async def request(session: ClientSession, url: str) -> ClientResponse:
|
async def request(session: ClientSession, url: str) -> ClientResponse:
|
||||||
|
@ -24,11 +25,13 @@ async def fetch_feeds(urls: Iterable[str]) -> Iterable[str]:
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@autoparams('parser', 'repository')
|
||||||
def fetch_new_videos(
|
def fetch_new_videos(
|
||||||
|
*,
|
||||||
|
parser: Parser,
|
||||||
repository: ChannelRepository,
|
repository: ChannelRepository,
|
||||||
) -> tuple[dict[str, Channel], dict[str, Video]]:
|
) -> tuple[dict[str, Channel], dict[str, Video]]:
|
||||||
feed_urls = get_feed_urls()
|
feed_urls = get_feed_urls()
|
||||||
parser = Parser(repository)
|
|
||||||
channels: dict[str, Channel] = {}
|
channels: dict[str, Channel] = {}
|
||||||
new_videos: dict[str, Video] = {}
|
new_videos: dict[str, Video] = {}
|
||||||
for feed in run(fetch_feeds(feed_urls)):
|
for feed in run(fetch_feeds(feed_urls)):
|
||||||
|
|
|
@ -1,15 +1,25 @@
|
||||||
|
from abc import ABCMeta, abstractmethod
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import feedparser
|
import feedparser
|
||||||
|
from inject import autoparams
|
||||||
|
|
||||||
|
from ytrssil.config import Configuration
|
||||||
from ytrssil.datatypes import Channel, Video
|
from ytrssil.datatypes import Channel, Video
|
||||||
from ytrssil.repository import ChannelNotFound, ChannelRepository
|
from ytrssil.repository import ChannelNotFound, ChannelRepository
|
||||||
|
|
||||||
|
|
||||||
class Parser:
|
class Parser(metaclass=ABCMeta):
|
||||||
|
@autoparams('channel_repository')
|
||||||
def __init__(self, channel_repository: ChannelRepository) -> None:
|
def __init__(self, channel_repository: ChannelRepository) -> None:
|
||||||
self.repository = channel_repository
|
self.repository = channel_repository
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def __call__(self, feed_content: str) -> Channel:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class FeedparserParser(Parser):
|
||||||
def __call__(self, feed_content: str) -> Channel:
|
def __call__(self, feed_content: str) -> Channel:
|
||||||
d = feedparser.parse(feed_content)
|
d = feedparser.parse(feed_content)
|
||||||
channel_id: str = d['feed']['yt_channelid']
|
channel_id: str = d['feed']['yt_channelid']
|
||||||
|
@ -21,17 +31,27 @@ class Parser:
|
||||||
name=d['feed']['title'],
|
name=d['feed']['title'],
|
||||||
url=d['feed']['link'],
|
url=d['feed']['link'],
|
||||||
)
|
)
|
||||||
self.repository.update_channel(channel)
|
self.repository.create_channel(channel)
|
||||||
|
|
||||||
for entry in d['entries']:
|
for entry in d['entries']:
|
||||||
channel.add_video(Video(
|
video = Video(
|
||||||
video_id=entry['yt_videoid'],
|
video_id=entry['yt_videoid'],
|
||||||
name=entry['title'],
|
name=entry['title'],
|
||||||
url=entry['link'],
|
url=entry['link'],
|
||||||
timestamp=datetime.fromisoformat(entry['published']),
|
timestamp=datetime.fromisoformat(entry['published']),
|
||||||
channel_id=channel.channel_id,
|
channel_id=channel.channel_id,
|
||||||
channel_name=channel.name,
|
channel_name=channel.name,
|
||||||
))
|
)
|
||||||
|
if channel.add_video(video):
|
||||||
|
self.repository.add_new_video(channel, video)
|
||||||
|
|
||||||
self.repository.update_channel(channel)
|
|
||||||
return channel
|
return channel
|
||||||
|
|
||||||
|
|
||||||
|
@autoparams()
|
||||||
|
def create_feed_parser(config: Configuration) -> Parser:
|
||||||
|
parser_type = config.feed_parser_type
|
||||||
|
if parser_type == 'feedparser':
|
||||||
|
return FeedparserParser()
|
||||||
|
else:
|
||||||
|
raise Exception(f'Unknown feed parser type: "{parser_type}"')
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from subprocess import Popen, PIPE
|
from subprocess import PIPE, Popen
|
||||||
|
|
||||||
from ytrssil.datatypes import Video
|
from ytrssil.datatypes import Video
|
||||||
|
|
||||||
|
|
|
@ -1,32 +1,191 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import os
|
|
||||||
import shelve
|
|
||||||
|
|
||||||
|
import os
|
||||||
|
from abc import ABCMeta, abstractmethod
|
||||||
|
from datetime import datetime
|
||||||
|
from sqlite3 import connect
|
||||||
|
from typing import Any, Union
|
||||||
|
|
||||||
|
from inject import autoparams
|
||||||
|
|
||||||
|
from ytrssil.config import Configuration
|
||||||
from ytrssil.constants import config_dir
|
from ytrssil.constants import config_dir
|
||||||
from ytrssil.datatypes import Channel
|
from ytrssil.datatypes import Channel, Video
|
||||||
|
|
||||||
|
|
||||||
class ChannelNotFound(Exception):
|
class ChannelNotFound(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class ChannelRepository:
|
class ChannelRepository(metaclass=ABCMeta):
|
||||||
|
@abstractmethod
|
||||||
|
def __enter__(self) -> ChannelRepository:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def __exit__(
|
||||||
|
self,
|
||||||
|
exc_type: Any,
|
||||||
|
exc_value: Any,
|
||||||
|
exc_traceback: Any,
|
||||||
|
) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_channel(self, channel_id: str) -> Channel:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def create_channel(self, channel: Channel) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def add_new_video(self, channel: Channel, video: Video) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def update_video(self, video: Video, watched: bool) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class SqliteChannelRepository(ChannelRepository):
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
os.makedirs(config_dir, exist_ok=True)
|
os.makedirs(config_dir, exist_ok=True)
|
||||||
self.file_path: str = os.path.join(config_dir, 'shelf')
|
self.file_path: str = os.path.join(config_dir, 'channels.db')
|
||||||
|
self.setup_database()
|
||||||
|
|
||||||
|
def setup_database(self) -> None:
|
||||||
|
connection = connect(self.file_path)
|
||||||
|
cursor = connection.cursor()
|
||||||
|
cursor.execute('PRAGMA foreign_keys = ON')
|
||||||
|
cursor.execute(
|
||||||
|
'CREATE TABLE IF NOT EXISTS channels ('
|
||||||
|
'channel_id VARCHAR PRIMARY KEY, name VARCHAR, url VARCHAR UNIQUE)'
|
||||||
|
)
|
||||||
|
cursor.execute(
|
||||||
|
'CREATE TABLE IF NOT EXISTS videos ('
|
||||||
|
'video_id VARCHAR PRIMARY KEY, name VARCHAR, url VARCHAR UNIQUE, '
|
||||||
|
'timestamp VARCHAR, watched BOOLEAN, channel_id VARCHAR, '
|
||||||
|
'FOREIGN KEY(channel_id) REFERENCES channels(channel_id))'
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
connection.close()
|
||||||
|
|
||||||
def __enter__(self) -> ChannelRepository:
|
def __enter__(self) -> ChannelRepository:
|
||||||
self.repository = shelve.open(self.file_path)
|
self.connection = connect(self.file_path)
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def __exit__(self, exc_type, exc_value, exc_traceback) -> None:
|
def __exit__(
|
||||||
self.repository.close()
|
self,
|
||||||
|
exc_type: Any,
|
||||||
|
exc_value: Any,
|
||||||
|
exc_traceback: Any,
|
||||||
|
) -> None:
|
||||||
|
self.connection.close()
|
||||||
|
|
||||||
|
def get_channel_as_dict(self, channel: Channel) -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
'channel_id': channel.channel_id,
|
||||||
|
'name': channel.name,
|
||||||
|
'url': channel.url,
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_video_as_dict(self, video: Video, watched: bool) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
'video_id': video.video_id,
|
||||||
|
'name': video.name,
|
||||||
|
'url': video.url,
|
||||||
|
'timestamp': video.timestamp.isoformat(),
|
||||||
|
'watched': watched,
|
||||||
|
'channel_id': video.channel_id,
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_videos(self, channel: Channel) -> list[tuple[Video, bool]]:
|
||||||
|
cursor = self.connection.cursor()
|
||||||
|
cursor.execute(
|
||||||
|
'SELECT video_id, name, url, timestamp, watched '
|
||||||
|
'FROM videos WHERE channel_id=:channel_id',
|
||||||
|
{'channel_id': channel.channel_id},
|
||||||
|
)
|
||||||
|
ret: list[tuple[Video, bool]] = []
|
||||||
|
video_data: tuple[str, str, str, str, bool]
|
||||||
|
for video_data in cursor.fetchall():
|
||||||
|
ret.append((
|
||||||
|
Video(
|
||||||
|
video_id=video_data[0],
|
||||||
|
name=video_data[1],
|
||||||
|
url=video_data[2],
|
||||||
|
timestamp=datetime.fromisoformat(video_data[3]),
|
||||||
|
channel_id=channel.channel_id,
|
||||||
|
channel_name=channel.name,
|
||||||
|
),
|
||||||
|
video_data[4]
|
||||||
|
))
|
||||||
|
|
||||||
|
return ret
|
||||||
|
|
||||||
def get_channel(self, channel_id: str) -> Channel:
|
def get_channel(self, channel_id: str) -> Channel:
|
||||||
try:
|
cursor = self.connection.cursor()
|
||||||
return self.repository[channel_id]
|
cursor.execute(
|
||||||
except KeyError as e:
|
'SELECT * FROM channels WHERE channel_id=:channel_id',
|
||||||
raise ChannelNotFound(e)
|
{'channel_id': channel_id},
|
||||||
|
)
|
||||||
|
channel_data: Union[tuple[str, str, str], None] = cursor.fetchone()
|
||||||
|
if channel_data is None:
|
||||||
|
raise ChannelNotFound
|
||||||
|
|
||||||
|
channel = Channel(
|
||||||
|
channel_id=channel_data[0],
|
||||||
|
name=channel_data[1],
|
||||||
|
url=channel_data[2],
|
||||||
|
)
|
||||||
|
for video, watched in self.get_videos(channel):
|
||||||
|
if watched:
|
||||||
|
channel.watched_videos[video.video_id] = video
|
||||||
|
else:
|
||||||
|
channel.new_videos[video.video_id] = video
|
||||||
|
|
||||||
|
return channel
|
||||||
|
|
||||||
|
def create_channel(self, channel: Channel) -> None:
|
||||||
|
cursor = self.connection.cursor()
|
||||||
|
cursor.execute(
|
||||||
|
'INSERT INTO channels VALUES (:channel_id, :name, :url)',
|
||||||
|
self.get_channel_as_dict(channel),
|
||||||
|
)
|
||||||
|
self.connection.commit()
|
||||||
|
|
||||||
def update_channel(self, channel: Channel) -> None:
|
def update_channel(self, channel: Channel) -> None:
|
||||||
self.repository[channel.channel_id] = channel
|
cursor = self.connection.cursor()
|
||||||
|
cursor.execute(
|
||||||
|
'UPDATE channels SET channel_id = :channel_id, name = :name, '
|
||||||
|
'url = :url WHERE channel_id=:channel_id',
|
||||||
|
self.get_channel_as_dict(channel),
|
||||||
|
)
|
||||||
|
self.connection.commit()
|
||||||
|
|
||||||
|
def add_new_video(self, channel: Channel, video: Video) -> None:
|
||||||
|
cursor = self.connection.cursor()
|
||||||
|
cursor.execute(
|
||||||
|
'INSERT INTO videos VALUES '
|
||||||
|
'(:video_id, :name, :url, :timestamp, :watched, :channel_id)',
|
||||||
|
self.get_video_as_dict(video, False),
|
||||||
|
)
|
||||||
|
self.connection.commit()
|
||||||
|
|
||||||
|
def update_video(self, video: Video, watched: bool) -> None:
|
||||||
|
cursor = self.connection.cursor()
|
||||||
|
cursor.execute(
|
||||||
|
'UPDATE videos SET watched = :watched WHERE video_id=:video_id',
|
||||||
|
{'watched': watched, 'video_id': video.video_id},
|
||||||
|
)
|
||||||
|
self.connection.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@autoparams()
|
||||||
|
def create_channel_repository(config: Configuration) -> ChannelRepository:
|
||||||
|
repo_type = config.channel_repository_type
|
||||||
|
if repo_type == 'sqlite':
|
||||||
|
return SqliteChannelRepository()
|
||||||
|
else:
|
||||||
|
raise Exception(f'Unknown channel repository type: "{repo_type}"')
|
||||||
|
|
Loading…
Reference in New Issue