From 6b873ac6cf9eeabe0f222cfb36466d279b883d91 Mon Sep 17 00:00:00 2001 From: tropicoo Date: Sun, 25 Sep 2022 23:58:34 +0300 Subject: [PATCH] Some refinements --- README.md | 100 +++++++++--------- bot.py | 2 +- hikcamerabot/bot_setup.py | 2 +- hikcamerabot/callbacks.py | 4 +- hikcamerabot/camera.py | 4 +- .../version_checker.py | 5 +- hikcamerabot/clients/hikvision/api_client.py | 23 +--- hikcamerabot/clients/hikvision/api_wrapper.py | 14 +-- .../clients/hikvision/endpoints/abstract.py | 10 +- .../{helpers.py => config_switch.py} | 4 +- .../clients/hikvision/endpoints/endpoints.py | 4 +- hikcamerabot/common/video/tasks/abstract.py | 5 - .../common/video/tasks/ffprobe_context.py | 5 +- hikcamerabot/common/video/tasks/thumbnail.py | 22 +++- hikcamerabot/common/video/tasks/videogif.py | 62 ++++++----- hikcamerabot/decorators.py | 2 +- hikcamerabot/event_engine/handlers/inbound.py | 2 +- .../event_engine/handlers/outbound.py | 2 +- hikcamerabot/event_engine/workers/tasks.py | 2 +- hikcamerabot/services/stream/abstract.py | 4 +- .../services/stream/dvr/file_wrapper.py | 4 +- hikcamerabot/services/stream/dvr/service.py | 4 +- .../services/stream/dvr/tasks/file_delete.py | 2 +- .../stream/dvr/tasks/file_lock_check.py | 16 +-- .../stream/dvr/tasks/file_monitoring.py | 2 +- .../services/stream/dvr/upload/engine.py | 8 +- hikcamerabot/services/tasks/livestream.py | 2 +- hikcamerabot/utils/image.py | 2 +- hikcamerabot/utils/process.py | 6 ++ hikcamerabot/utils/{utils.py => shared.py} | 0 hikcamerabot/version.py | 2 +- 31 files changed, 170 insertions(+), 156 deletions(-) rename hikcamerabot/clients/{github_version => github}/version_checker.py (87%) rename hikcamerabot/clients/hikvision/endpoints/{helpers.py => config_switch.py} (95%) create mode 100644 hikcamerabot/utils/process.py rename hikcamerabot/utils/{utils.py => shared.py} (100%) diff --git a/README.md b/README.md index 8a34cda..bd06fdb 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,12 @@ Telegram Bot which sends snapshots from your Hikvision cameras. Version: 1.5. [Release details](releases/release_1.5.md). ## Features -1. Send full/resized pictures on request -2. Auto-send pictures on **Motion**, **Line Crossing** and **Intrusion (Field) Detection** -3. Send so-called Telegram video-gifs on request and alert events from previous paragraph -4. YouTube, Telegram and Icecast direct or re-encoded livestreams -5. DVR to local storage with upload to Telegram group -6. SRS re-stream server +1. Send full/resized pictures on request. +2. Auto-send pictures on **Motion**, **Line Crossing** and **Intrusion (Field) Detection**. +3. Send so-called Telegram video-gifs on request and alert events from the previous paragraph. +4. YouTube, Telegram, and Icecast direct or re-encoded livestreams. +5. DVR to local storage with upload to Telegram group. +6. SRS re-stream server. ![frames](img/screenshot-1.png) @@ -23,13 +23,13 @@ cd hikvision-camera-bot ``` # Configuration -Configuration files are stored in JSON format and can be found in `configs` directory. +Configuration files are stored in JSON format and can be found in the `configs` directory. ## Quick Setup 1. [Create and start Telegram Bot](https://core.telegram.org/bots#6-botfather) and get its API token -2. [Get your own Telegram API key](https://my.telegram.org/apps) (`api_id` and `api_hash`) -3. Copy 3 default configuration files with predefined templates in `configs` directory: +2. [Get your Telegram API key](https://my.telegram.org/apps) (`api_id` and `api_hash`) +3. Copy 3 default configuration files with predefined templates in the `configs` directory: ```bash cd configs @@ -38,21 +38,21 @@ Configuration files are stored in JSON format and can be found in `configs` dire cp livestream_templates-template.json livestream_templates.json ``` 4. Edit **config.json**: - 1. Put the obtained `api_id` and `api_hash` strings to same keys - 2. Put the obtained bot API token string to `token` key + 1. Put the obtained `api_id` and `api_hash` strings to the same keys + 2. Put the obtained bot API token string to the `token` key 3. [Find](https://stackoverflow.com/a/32777943) your Telegram user id and put it to `chat_users`, `alert_users` and `startup_message_users` lists as integer value. Multiple ids can be used, just separate them with a comma. - 4. Hikvision camera settings are placed inside the `camera_list` section. Template + 4. Hikvision camera settings are placed inside the `camera_list` section. The template comes with two cameras - **Camera names should start with `cam_` prefix and end with + **Camera names should start with the `cam_` prefix and end with digit suffix**: `cam_1`, `cam_2`, `cam_` with any description. 5. Write authentication credentials in `user` and `password` keys for every camera - 6. Same for `host`, which should include protocol, e.g. `http://192.168.1.1` - 7. In `alert` section you can enable sending picture on alert (Motion, - Line Crossing and Intrusion (Field) Detection). Configure `delay` setting + 6. Same for `host`, which should include protocol e.g., `http://192.168.1.1` + 7. In the `alert` section you can enable sending pictures on alert (Motion, + Line Crossing and Intrusion (Field) Detection). Configure the `delay` setting in seconds between pushing alert pictures. To send resized picture change `fullpic` to `false` @@ -188,10 +188,10 @@ Configuration files are stored in JSON format and can be found in `configs` dire # Usage ## Launch by using Docker and Docker Compose -1. Set your timezone by editing `.env` file (`TZ=Europe/Kiev`). -Currently, there is Ukrainian timezone because I live there. +1. Set your timezone by editing the `.env` file (`TZ=Europe/Kiev`). +Currently, there is a Ukrainian timezone because I live there. Look for your timezone here [http://www.timezoneconverter.com/cgi-bin/zoneinfo](http://www.timezoneconverter.com/cgi-bin/zoneinfo). -If you want to use default UTC time format, set Greenwich Mean Time timezone `TZ=GMT` +If you want to use the default UTC time format, set Greenwich Mean Time timezone `TZ=GMT` 2. Build an image and run a container in a detached mode ```bash @@ -201,12 +201,12 @@ If you want to use default UTC time format, set Greenwich Mean Time timezone `TZ # Commands | Command | Description | |---|---| -| `/start` | Start the bot (one-time action during first start) and show help | +| `/start` | Start the bot (one-time action during the first start) and show help | | `/help` | Show help message | | `/list_cams` | List all your cameras | | `/cmds_cam_*` | List commands for particular camera | | `/getpic_cam_*` | Get resized picture from your Hikvision camera | -| `/getfullpic_cam_*` | Get full-sized picture from your Hikvision camera | +| `/getfullpic_cam_*` | Get a full-sized picture from your Hikvision camera | | `/ir_on_cam_*` | Turn on Infrared mode | | `/ir_off_cam_*` | Turn off Infrared mode | | `/ir_auto_cam_*` | Turn on Infrared auto mode | @@ -216,20 +216,20 @@ If you want to use default UTC time format, set Greenwich Mean Time timezone `TZ | `/ld_off_cam_*` | Disable Line Crossing Detection | | `/intr_on_cam_*` | Enable Intrusion (Field) Detection | | `/intr_off_cam_*` | Disable Intrusion (Field) Detection | -| `/alert_on_cam_*` | Enable Alert (Alarm) mode. It means it will send respective alert to your account in Telegram | -| `/alert_off_cam_*` | Disable Alert (Alarm) mode, no alerts will be sent when something detected | +| `/alert_on_cam_*` | Enable Alert (Alarm) mode. It means it will send a respective alert to your account in Telegram | +| `/alert_off_cam_*` | Disable Alert (Alarm) mode, no alerts will be sent when something is detected | | `/yt_on_cam_*` | Enable YouTube stream | | `/yt_off_cam_*` | Disable YouTube stream | | `/icecast_on_cam_*` | Enable Icecast stream | | `/icecast_off_cam_*` | Disable Icecast stream | -`*` - camera digit id, e.g. `cam_1`. +`*` - camera digit id e.g., `cam_1`. # Advanced Configuration ## SRS -[SRS](https://github.com/ossrs/srs/tree/4.0release) (Simple Realtime Server) is a re-stream server which takes a stream from your camera and re-streams it -to any destination without touching native camera stream multiple times. -SRS release version used in the bot is `4.0`. +[SRS](https://github.com/ossrs/srs/tree/4.0release) (Simple Realtime Server) is a re-stream server that takes a stream from your camera and re-streams it +to any destination without touching the native camera stream multiple times. +The SRS release version used in the bot is `4.0`. SRS decreases CPU time and network load on the camera when you enable something like DVR, YouTube Livestream or try to get Video GIF at the same time. Pictures are taken @@ -237,17 +237,17 @@ directly from the camera stream, not from the SRS. How it works - if you have two cameras with enabled SRS for both, there will be two running 24/7 bot tasks taking streams from the cameras to the SRS server. Eventually, when you -request Video Gif, or it's triggered by some alert, video will be taken from SRS server. +request Video Gif, or it's triggered by some alert, the video will be taken from the SRS server. -You can also connect to SRS server with any video player like VLC and watch the stream +You can also connect to the SRS server with any video player like VLC and watch the stream without any interruptions. URL looks like this: `rtmp://192.168.1.100/live/livestream_101_cam_2`, where: 1. `192.168.1.100` is an IP address or a host of your server. -2. `101` is camera's configured stream channel. -3. `cam_2` is ID of your second configured camera. +2. `101` is the camera's configured stream channel. +3. `cam_2` is the ID of your second configured camera. SRS runs in a separate docker container. SRS config and `Dockerfile` are placed -in `srs_prod` directory. Service name is `hikvision-srs-server` in `docker-compose.yml`. +in the `srs_prod` directory. The service name is `hikvision-srs-server` in `docker-compose.yml`. @@ -255,22 +255,22 @@ If `docker-compose.yml` is a list of forwarded and open SRS ports to the world: ```yaml # If you don't plan to use anything from this, just comment out the whole section. ports: - - "1935:1935" # SRS RTMP port, if you comment this out, you won't be able to connect with video player + - "1935:1935" # SRS RTMP port, if you comment this out, you won't be able to connect with the video player - "1985:1985" # SRS API port, can be commented out since not used - "8080:8080" # SRS WebUI port ``` ## DVR -You can record your videos from the camera to a local storage mounted as volume in +You can record your videos from the camera to local storage mounted as a volume in volumes section of `hikvision-camera-bot` service in `docker-compose.yml`. DVR configuration is per camera in `config.json` with livestream template name from `livestream_templates.json`. It's very simple: -1. Use `enabled` key to turn on/off this feature. +1. Use the `enabled` key to turn on/off this feature. 2. `local_storage_path` is a path inside the container to which videos will be recorded. Don't change this default value (`/data/dvr`) since it's written in the volumes mapping section. -If you need to change it for some reason - you need to change both here and in the volumes mapping. +If you need to change it for some reason - you must change it both here and in the volumes mapping. 3. `livestream_template` has a template name located inside the `livestream_templates.json` file with DVR stream settings: ```json @@ -284,11 +284,11 @@ file with DVR stream settings: } } ``` - a) `segment_time` is time in seconds when DVR record file will be split to a new one. + a) `segment_time` is the time in seconds when the DVR record file will be split into a new one. b) `1800` seconds mean every file will have 30 minutes of video recording. - c) File is named as `cam_1_101_1800_2022-04-15_21-19-32.mp4` with cam ID, channel name, segment time, and record start datetime. + c) File is named `cam_1_101_1800_2022-04-15_21-19-32.mp4` with cam ID, channel name, segment time, and record start datetime. 4. Configuration part from the `config.json`: ```json @@ -308,12 +308,12 @@ file with DVR stream settings: } } ``` - Recorded files can be uploaded to Telegram group. Right now upload will work only - if `delete_after_upload` is set to `true` meaning uploaded file will be deleted + Recorded files can be uploaded to the Telegram group. Right now, the upload will work only + if `delete_after_upload` is set to `true` meaning the uploaded file will be deleted from the local storage. You need to make sure your file size will be up to 2GB since Telegram rejects larger ones. Just experiment with segment time. 5. Local storage (the real one, not in the container) by default is `/data/dvr` in volumes mapping (the first path string, not the last). - Change it to any location you need, e.g. to `- "D:\Videos:/data/dvr"` if you're on Windows. + Change it to any location you need e.g., `- "D:\Videos:/data/dvr"` if you're on Windows. ```yaml volumes: - "/data/dvr:/data/dvr" @@ -332,22 +332,22 @@ To enable YouTube Live Stream enable it in the `youtube` key. **Livestream templates** -To start particular livestream, user needs to set both *livestream* and +To start a particular livestream, a user needs to set both *livestream* and *encoding* templates with stream settings and encoding type/arguments. Encoding templates -`direct` means that video stream will not be re-encoded (transcoded) and will +`direct` means that the video stream will not be re-encoded (transcoded) and will be sent to YouTube/Icecast servers "as is", only audio can be disabled. -`x264` or `vp9` means that video stream will be re-encoded on your machine/server -where bot is running using respective encoding codecs. +`x264` or `vp9` means that the video stream will be re-encoded on your machine/server +where the bot is running using respective encoding codecs. -User can create its own templates in file named `livestream_templates.json` +User can create their templates in a file named `livestream_templates.json` and `encoding_templates.json`. -Default dummy template file is named `livestream_templates_template.json` -(not very funny name but anyway) which should be copied or renamed to +The default dummy template file is named `livestream_templates_template.json` +(not a very funny name but anyway) which should be copied or renamed to `livestream_templates.json`. Same for `encoding_templates-template.json` -> `encoding_templates.json` @@ -411,7 +411,7 @@ Where: | Parameter | Value | Description | |---|---|---| -| `channel` | `101` | camera channel. 101 is main stream, 102 is substream. | +| `channel` | `101` | camera channel. 101 is the main stream, and 102 is the substream. | | `restart_period` | `39600` | stream restart period in seconds | | `restart_pause` | `10` | stream pause before starting on restart | | `url` | `"rtmp://a.rtmp.youtube.com/live2"` | YouTube rtmp server | @@ -420,7 +420,7 @@ Where: | `ice_name` | `"Default"` | Icecast stream name | | `ice_description` | `"Default"` | Icecast stream description | | `ice_public` | `0` | Icecast public switch, default 0 | -| `url` | `"icecast://source@x.x.x.x:8000/video.webm"` | Icecast server URL, Port and media mount point | +| `url` | `"icecast://source@x.x.x.x:8000/video.webm"` | Icecast server URL, Port, and media mount point | | `password` | `"xxxx"` | Icecast authentication password | | `content_type` | `"video/webm"` | FFMPEG content-type for Icecast stream | diff --git a/bot.py b/bot.py index ba6ed9e..e318da3 100755 --- a/bot.py +++ b/bot.py @@ -4,7 +4,7 @@ import asyncio from hikcamerabot.launcher import BotLauncher -from hikcamerabot.utils.utils import setup_logging +from hikcamerabot.utils.shared import setup_logging async def main() -> None: diff --git a/hikcamerabot/bot_setup.py b/hikcamerabot/bot_setup.py index 5ee9b33..49081c9 100644 --- a/hikcamerabot/bot_setup.py +++ b/hikcamerabot/bot_setup.py @@ -10,7 +10,7 @@ from hikcamerabot.camerabot import CameraBot from hikcamerabot.commands import setup_commands from hikcamerabot.config.config import get_main_config -from hikcamerabot.utils.utils import build_command_presentation +from hikcamerabot.utils.shared import build_command_presentation class BotSetup: diff --git a/hikcamerabot/callbacks.py b/hikcamerabot/callbacks.py index c81d32e..d16fd7d 100644 --- a/hikcamerabot/callbacks.py +++ b/hikcamerabot/callbacks.py @@ -5,7 +5,7 @@ from hikcamerabot.camera import HikvisionCam from hikcamerabot.camerabot import CameraBot -from hikcamerabot.clients.github_version.version_checker import ( +from hikcamerabot.clients.github.version_checker import ( HikCameraBotVersionChecker, ) from hikcamerabot.clients.hikvision.enums import IrcutFilterType @@ -19,7 +19,7 @@ IrcutConfEvent, StreamEvent, ) -from hikcamerabot.utils.utils import bold, send_text +from hikcamerabot.utils.shared import bold, send_text log = logging.getLogger(__name__) diff --git a/hikcamerabot/camera.py b/hikcamerabot/camera.py index 70f86e6..f5b9e50 100644 --- a/hikcamerabot/camera.py +++ b/hikcamerabot/camera.py @@ -102,7 +102,7 @@ def __init__(self, id: str, conf: Dict, bot: 'CameraBot') -> None: self.hashtag = f'#{conf.hashtag.lower() if conf.hashtag else self.id}' self.group = conf.group or 'Default group' self.bot = bot - self._log.debug('Initializing %s', self.description) + self._log.debug('Initializing camera %s', self.description) self._api = HikvisionAPI(api_client=HikvisionAPIClient(conf=conf.api)) self._img_processor = ImageProcessor() @@ -119,7 +119,7 @@ def __init__(self, id: str, conf: Dict, bot: 'CameraBot') -> None: self._videogif = VideoGifRecorder(cam=self) def __repr__(self) -> str: - return f'' + return f'' async def start_videogif_record( self, diff --git a/hikcamerabot/clients/github_version/version_checker.py b/hikcamerabot/clients/github/version_checker.py similarity index 87% rename from hikcamerabot/clients/github_version/version_checker.py rename to hikcamerabot/clients/github/version_checker.py index 1096665..26a42fc 100644 --- a/hikcamerabot/clients/github_version/version_checker.py +++ b/hikcamerabot/clients/github/version_checker.py @@ -8,6 +8,8 @@ @dataclass class BotVersion: + """Bot version DTO class.""" + current: str latest: str @@ -27,11 +29,12 @@ def get_current_version(self) -> str: return __version__ async def get_latest_version(self) -> str: + """Get latest version number from latest GitHub tag URL.""" self._log.info('Get latest hikvision-camera-bot version number') client: AsyncClient async with AsyncClient() as client: response = await client.head(self.LATEST_TAG_URL) - version = response.headers.get('location').split('/')[-1] + version: str = response.headers.get('location').split('/')[-1] self._log.info('Latest hikvision-camera-bot version number: %s', version) return version diff --git a/hikcamerabot/clients/hikvision/api_client.py b/hikcamerabot/clients/hikvision/api_client.py index 5577dfb..c2c7543 100644 --- a/hikcamerabot/clients/hikvision/api_client.py +++ b/hikcamerabot/clients/hikvision/api_client.py @@ -1,5 +1,4 @@ """Hikvision camera API client module.""" -import abc import logging from typing import Any from urllib.parse import urljoin @@ -13,29 +12,13 @@ from hikcamerabot.exceptions import APIBadResponseCodeError, APIRequestError -class AbstractHikvisionAPIClient(metaclass=abc.ABCMeta): +class HikvisionAPIClient: + """Hikvision API Class.""" + def __init__(self, conf: Dict) -> None: self._log = logging.getLogger(self.__class__.__name__) self._conf = conf self.host: str = self._conf.host - - @abc.abstractmethod - async def request( - self, - endpoint: str, - data: Any = None, - headers: dict = None, - method: str = 'GET', - timeout: float = CONN_TIMEOUT, - ) -> Any: - pass - - -class HikvisionAPIClient(AbstractHikvisionAPIClient): - """Hikvision API Class.""" - - def __init__(self, conf: Dict) -> None: - super().__init__(conf) self.session = httpx.AsyncClient( auth=DigestAuthCached( username=self._conf.auth.user, diff --git a/hikcamerabot/clients/hikvision/api_wrapper.py b/hikcamerabot/clients/hikvision/api_wrapper.py index c965060..59311e8 100644 --- a/hikcamerabot/clients/hikvision/api_wrapper.py +++ b/hikcamerabot/clients/hikvision/api_wrapper.py @@ -1,6 +1,6 @@ import logging -from hikcamerabot.clients.hikvision.api_client import AbstractHikvisionAPIClient +from hikcamerabot.clients.hikvision import HikvisionAPIClient from hikcamerabot.clients.hikvision.endpoints.endpoints import ( AlertStreamEndpoint, ExposureEndpoint, @@ -13,12 +13,12 @@ class HikvisionAPI: """Hikvision API Wrapper. API methods are Endpoint instances.""" - def __init__(self, api_client: AbstractHikvisionAPIClient) -> None: + def __init__(self, api_client: HikvisionAPIClient) -> None: self._log = logging.getLogger(self.__class__.__name__) self._api_client = api_client - self.alert_stream = AlertStreamEndpoint(self._api_client) - self.take_snapshot = TakeSnapshotEndpoint(self._api_client) - self.set_ircut_filter = IrcutFilterEndpoint(self._api_client) - self.set_exposure = ExposureEndpoint(self._api_client) - self.switch = SwitchEndpoint(self._api_client) + self.alert_stream = AlertStreamEndpoint(api_client) + self.take_snapshot = TakeSnapshotEndpoint(api_client) + self.set_ircut_filter = IrcutFilterEndpoint(api_client) + self.set_exposure = ExposureEndpoint(api_client) + self.switch = SwitchEndpoint(api_client) diff --git a/hikcamerabot/clients/hikvision/endpoints/abstract.py b/hikcamerabot/clients/hikvision/endpoints/abstract.py index e043fba..23e5017 100644 --- a/hikcamerabot/clients/hikvision/endpoints/abstract.py +++ b/hikcamerabot/clients/hikvision/endpoints/abstract.py @@ -6,21 +6,27 @@ import xmltodict from addict import Dict -from hikcamerabot.clients.hikvision.api_client import AbstractHikvisionAPIClient +from hikcamerabot.clients.hikvision import HikvisionAPIClient from hikcamerabot.clients.hikvision.enums import Endpoint from hikcamerabot.exceptions import HikvisionAPIError class AbstractEndpoint(metaclass=abc.ABCMeta): + """API Endpoint class. + + Used to decompose API methods since they are too complex to store in one API class. + """ + _XML_PAYLOAD_TPL: Optional[str] = None _XML_HEADERS = {'Content-Type': 'application/xml'} - def __init__(self, api_client: AbstractHikvisionAPIClient) -> None: + def __init__(self, api_client: HikvisionAPIClient) -> None: self._api_client = api_client self._log = logging.getLogger(self.__class__.__name__) @abc.abstractmethod async def __call__(self, *args, **kwargs) -> Any: + """Real API call starts here.""" pass async def _get_channel_capabilities(self) -> Dict: diff --git a/hikcamerabot/clients/hikvision/endpoints/helpers.py b/hikcamerabot/clients/hikvision/endpoints/config_switch.py similarity index 95% rename from hikcamerabot/clients/hikvision/endpoints/helpers.py rename to hikcamerabot/clients/hikvision/endpoints/config_switch.py index 44e6ab7..698baa3 100644 --- a/hikcamerabot/clients/hikvision/endpoints/helpers.py +++ b/hikcamerabot/clients/hikvision/endpoints/config_switch.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: - from hikcamerabot.clients.hikvision.api_client import AbstractHikvisionAPIClient + from hikcamerabot.clients.hikvision.api_client import HikvisionAPIClient class CameraConfigSwitch: @@ -25,7 +25,7 @@ class CameraConfigSwitch: ) XML_HEADERS = {'Content-Type': 'application/xml'} - def __init__(self, api: 'AbstractHikvisionAPIClient') -> None: + def __init__(self, api: 'HikvisionAPIClient') -> None: self._log = logging.getLogger(self.__class__.__name__) self._api = api diff --git a/hikcamerabot/clients/hikvision/endpoints/endpoints.py b/hikcamerabot/clients/hikvision/endpoints/endpoints.py index 8c7789e..ddef81d 100644 --- a/hikcamerabot/clients/hikvision/endpoints/endpoints.py +++ b/hikcamerabot/clients/hikvision/endpoints/endpoints.py @@ -5,6 +5,8 @@ import httpx from addict import Dict +from hikcamerabot.clients.hikvision.endpoints.abstract import AbstractEndpoint +from hikcamerabot.clients.hikvision.endpoints.config_switch import CameraConfigSwitch from hikcamerabot.clients.hikvision.enums import ( Endpoint, ExposureType, @@ -12,8 +14,6 @@ OverexposeSuppressEnabledType, OverexposeSuppressType, ) -from hikcamerabot.clients.hikvision.endpoints.abstract import AbstractEndpoint -from hikcamerabot.clients.hikvision.endpoints.helpers import CameraConfigSwitch from hikcamerabot.constants import CONN_TIMEOUT, Detection from hikcamerabot.exceptions import APIRequestError diff --git a/hikcamerabot/common/video/tasks/abstract.py b/hikcamerabot/common/video/tasks/abstract.py index 5505b1e..b5ba490 100644 --- a/hikcamerabot/common/video/tasks/abstract.py +++ b/hikcamerabot/common/video/tasks/abstract.py @@ -33,11 +33,6 @@ async def _run_proc(self, cmd: str) -> Optional[asyncio.subprocess.Process]: await self._killpg(os.getpgid(proc.pid), signal.SIGINT) return None - @staticmethod - async def _get_stdout_stderr(proc: asyncio.subprocess.Process) -> tuple[str, str]: - stdout, stderr = await proc.stdout.read(), await proc.stderr.read() - return stdout.decode().strip(), stderr.decode().strip() - @abc.abstractmethod async def run(self) -> None: """Main entry point.""" diff --git a/hikcamerabot/common/video/tasks/ffprobe_context.py b/hikcamerabot/common/video/tasks/ffprobe_context.py index d058c04..0238cbc 100644 --- a/hikcamerabot/common/video/tasks/ffprobe_context.py +++ b/hikcamerabot/common/video/tasks/ffprobe_context.py @@ -2,6 +2,7 @@ from typing import Optional from hikcamerabot.common.video.tasks.abstract import AbstractFfBinaryTask +from hikcamerabot.utils.process import get_stdout_stderr class GetFfprobeContextTask(AbstractFfBinaryTask): @@ -16,9 +17,9 @@ async def _get_context(self) -> Optional[dict]: if not proc: return None - stdout, stderr = await self._get_stdout_stderr(proc) + stdout, stderr = await get_stdout_stderr(proc) self._log.debug( - 'Process %s returncode: %d, stderr: %s', cmd, proc.returncode, stderr + 'Process "%s" returncode: %d, stderr: %s', cmd, proc.returncode, stderr ) if proc.returncode: self._log.error( diff --git a/hikcamerabot/common/video/tasks/thumbnail.py b/hikcamerabot/common/video/tasks/thumbnail.py index 9d806a0..5278eea 100644 --- a/hikcamerabot/common/video/tasks/thumbnail.py +++ b/hikcamerabot/common/video/tasks/thumbnail.py @@ -1,4 +1,7 @@ +import os + from hikcamerabot.common.video.tasks.abstract import AbstractFfBinaryTask +from hikcamerabot.utils.process import get_stdout_stderr class MakeThumbnailTask(AbstractFfBinaryTask): @@ -17,9 +20,9 @@ async def _make_thumbnail(self) -> bool: if not proc: return False - stdout, stderr = await self._get_stdout_stderr(proc) + stdout, stderr = await get_stdout_stderr(proc) self._log.debug( - 'Process %s returncode: %d, stdout: %s, stderr: %s', + 'Process "%s" returncode: %d, stdout: %s, stderr: %s', cmd, proc.returncode, stdout, @@ -27,5 +30,20 @@ async def _make_thumbnail(self) -> bool: ) if proc.returncode: self._log.error('Failed to make thumbnail for %s', self._file_path) + self._err_cleanup() return False return True + + def _err_cleanup(self) -> None: + """Cleanup errored thumbnail if any. + + For example, zero-size thumbnail could be created when no space left on device. + """ + if not os.path.exists(self._thumbnail_path): + return + + self._log.info('Cleaning up errored "%s"', self._thumbnail_path) + try: + os.remove(self._thumbnail_path) + except Exception: + self._log.exception('Cleanup failed for errored "%s"', self._thumbnail_path) diff --git a/hikcamerabot/common/video/tasks/videogif.py b/hikcamerabot/common/video/tasks/videogif.py index 1905a4e..6da9d17 100644 --- a/hikcamerabot/common/video/tasks/videogif.py +++ b/hikcamerabot/common/video/tasks/videogif.py @@ -28,8 +28,8 @@ VideoOutboundEvent, ) from hikcamerabot.event_engine.queue import get_result_queue +from hikcamerabot.utils.shared import bold, format_ts, gen_random_str from hikcamerabot.utils.task import wrap -from hikcamerabot.utils.utils import format_ts, gen_random_str, bold if TYPE_CHECKING: from hikcamerabot.camera import HikvisionCam @@ -68,7 +68,7 @@ def __init__( self._filename = self._get_filename() self._file_path: str = os.path.join(self._tmp_storage_path, self._filename) self._thumb_path: str = os.path.join( - self._tmp_storage_path, f'{self._filename}.jpg' + self._tmp_storage_path, f'{self._filename}-thumb.jpg' ) self._thumb_created = False @@ -93,35 +93,40 @@ async def run(self) -> None: async def _record(self) -> None: """Start Ffmpeg subprocess and return file path and video type.""" self._log.debug( - 'Recording %s video gif from %s: %s', + 'Recording "%s" video gif from "%s": "%s"', self._video_type.value, self._cam.conf.description, self._ffmpeg_cmd, ) await self._start_ffmpeg_subprocess() - is_validated = await self._validate_file() - if not is_validated: - err_msg = f'Failed to record {self._file_path} on {self._cam.description}' - self._log.error(err_msg) - await self._result_queue.put( - SendTextOutboundEvent( - event=Event.SEND_TEXT, - text=f'{err_msg}.\nEvent type: {self._event.value}\nCheck logs.', - message=self._message, - ) + if await self._validate_file(): + await self._post_process_successful_record() + else: + await self._post_process_failed_record() + + async def _post_process_successful_record(self): + await asyncio.gather(self._get_probe_ctx(), self._make_thumbnail_frame()) + await self._send_result() + + async def _post_process_failed_record(self): + self._post_err_cleanup() + err_msg = f'Failed to record {self._file_path} on {self._cam.description}' + self._log.error(err_msg) + await self._result_queue.put( + SendTextOutboundEvent( + event=Event.SEND_TEXT, + text=f'{err_msg}.\nEvent type: {self._event.value}\nCheck logs.', + message=self._message, ) - if is_validated: - await asyncio.gather(self._get_probe_ctx(), self._make_thumbnail_frame()) - await self._send_result() + ) async def _make_thumbnail_frame(self) -> None: # TODO: Refactor duplicate code. Move to mixin. - if not await MakeThumbnailTask(self._thumb_path, self._file_path).run(): - self._log.error( - 'Error during making thumbnail context of %s', self._file_path - ) - return - self._thumb_created = True + self._thumb_created = await MakeThumbnailTask( + self._thumb_path, self._file_path + ).run() + if not self._thumb_created: + self._log.error('Error during making thumbnail of %s', self._file_path) async def _get_probe_ctx(self) -> None: # TODO: Refactor duplicate code. Move to mixin. @@ -140,10 +145,11 @@ async def _get_probe_ctx(self) -> None: def _post_err_cleanup(self) -> None: """Delete video file and thumb if they exist after exception.""" for file_path in (self._file_path, self._thumb_path): - try: - os.remove(file_path) - except Exception as err: - self._log.warning('File path %s not deleted: %s', file_path, err) + if os.path.exists(file_path): + try: + os.remove(file_path) + except Exception as err: + self._log.warning('File path %s not deleted: %s', file_path, err) async def _start_ffmpeg_subprocess(self) -> None: proc_timeout = ( @@ -155,7 +161,8 @@ async def _start_ffmpeg_subprocess(self) -> None: except asyncio.TimeoutError: self._log.error( 'Failed to record %s: FFMPEG process ran longer than ' - 'expected and was killed', + 'expected (%ds) and was killed', + proc_timeout, self._file_path, ) await self._killpg(os.getpgid(proc.pid), signal.SIGINT) @@ -176,7 +183,6 @@ async def _validate_file(self) -> bool: if is_empty: self._log.error('Failed to validate %s: File is empty', self._file_path) - self._post_err_cleanup() return not is_empty async def _send_result(self) -> None: diff --git a/hikcamerabot/decorators.py b/hikcamerabot/decorators.py index 16d14b1..7f83880 100644 --- a/hikcamerabot/decorators.py +++ b/hikcamerabot/decorators.py @@ -9,7 +9,7 @@ from pyrogram.types import Message from hikcamerabot.constants import CMD_CAM_ID_REGEX -from hikcamerabot.utils.utils import get_user_info +from hikcamerabot.utils.shared import get_user_info if TYPE_CHECKING: from hikcamerabot.camerabot import CameraBot diff --git a/hikcamerabot/event_engine/handlers/inbound.py b/hikcamerabot/event_engine/handlers/inbound.py index bc20b0d..bb405a4 100644 --- a/hikcamerabot/event_engine/handlers/inbound.py +++ b/hikcamerabot/event_engine/handlers/inbound.py @@ -24,7 +24,7 @@ ) from hikcamerabot.event_engine.queue import get_result_queue from hikcamerabot.exceptions import ServiceRuntimeError -from hikcamerabot.utils.utils import bold +from hikcamerabot.utils.shared import bold if TYPE_CHECKING: from hikcamerabot.camerabot import CameraBot diff --git a/hikcamerabot/event_engine/handlers/outbound.py b/hikcamerabot/event_engine/handlers/outbound.py index abd9b4c..b5ca0a9 100644 --- a/hikcamerabot/event_engine/handlers/outbound.py +++ b/hikcamerabot/event_engine/handlers/outbound.py @@ -22,7 +22,7 @@ StreamOutboundEvent, VideoOutboundEvent, ) -from hikcamerabot.utils.utils import format_ts, bold, send_text +from hikcamerabot.utils.shared import bold, format_ts, send_text if TYPE_CHECKING: from hikcamerabot.camerabot import CameraBot diff --git a/hikcamerabot/event_engine/workers/tasks.py b/hikcamerabot/event_engine/workers/tasks.py index 360fe84..fc86c14 100644 --- a/hikcamerabot/event_engine/workers/tasks.py +++ b/hikcamerabot/event_engine/workers/tasks.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING from hikcamerabot.event_engine.queue import get_result_queue -from hikcamerabot.utils.utils import shallow_sleep_async +from hikcamerabot.utils.shared import shallow_sleep_async if TYPE_CHECKING: from hikcamerabot.event_engine.dispatchers.outbound import OutboundEventDispatcher diff --git a/hikcamerabot/services/stream/abstract.py b/hikcamerabot/services/stream/abstract.py index cfb0bf0..22aa648 100644 --- a/hikcamerabot/services/stream/abstract.py +++ b/hikcamerabot/services/stream/abstract.py @@ -30,8 +30,8 @@ FfmpegStdoutReaderTask, ServiceStreamerTask, ) +from hikcamerabot.utils.shared import shallow_sleep_async from hikcamerabot.utils.task import create_task, wrap -from hikcamerabot.utils.utils import shallow_sleep_async if TYPE_CHECKING: from hikcamerabot.camera import HikvisionCam @@ -133,7 +133,7 @@ def _start_stream_task(self) -> None: ) async def _start_ffmpeg_process(self) -> None: - self._log.debug('%s ffmpeg command: %s', self._cls_name, self._cmd) + self._log.debug('%s ffmpeg command: "%s"', self._cls_name, self._cmd) try: self._proc = await asyncio.create_subprocess_shell( self._cmd, diff --git a/hikcamerabot/services/stream/dvr/file_wrapper.py b/hikcamerabot/services/stream/dvr/file_wrapper.py index 499acbe..861eb89 100644 --- a/hikcamerabot/services/stream/dvr/file_wrapper.py +++ b/hikcamerabot/services/stream/dvr/file_wrapper.py @@ -56,9 +56,7 @@ async def _get_probe_ctx(self) -> None: async def _make_thumbnail_frame(self) -> None: if not await MakeThumbnailTask(self._thumbnail, self.full_path).run(): - self._log.error( - 'Error during making thumbnail context of %s', self.full_path - ) + self._log.error('Error during making thumbnail for %s', self.full_path) async def make_context(self) -> None: await asyncio.gather(self._get_probe_ctx(), self._make_thumbnail_frame()) diff --git a/hikcamerabot/services/stream/dvr/service.py b/hikcamerabot/services/stream/dvr/service.py index 4d44a47..8e4314c 100644 --- a/hikcamerabot/services/stream/dvr/service.py +++ b/hikcamerabot/services/stream/dvr/service.py @@ -59,13 +59,13 @@ async def start(self, *args, **kwargs) -> None: async def _start_upload_engine(self) -> None: """Start Upload Engine only if at least one storage is enabled.""" # TODO: Right now upload engine will start only if DVR records are set - # TODO: to be deleted since there is no uploaded files tracking. + # TODO: to be deleted since there is no tracking for uploaded files. for storage_settings in self._conf.upload.storage.values(): if ( storage_settings.enabled and self.cam.conf.livestream.dvr.upload.delete_after_upload ): - self._log.info('Starting Upload Engine for %s', self.cam.description) + self._log.info('Starting Upload Engine for %s', self.cam) await self._upload_engine.start() return self._log.info('Upload Engine not started.') diff --git a/hikcamerabot/services/stream/dvr/tasks/file_delete.py b/hikcamerabot/services/stream/dvr/tasks/file_delete.py index 8ca3c73..289571c 100644 --- a/hikcamerabot/services/stream/dvr/tasks/file_delete.py +++ b/hikcamerabot/services/stream/dvr/tasks/file_delete.py @@ -3,7 +3,7 @@ import os from typing import TYPE_CHECKING -from hikcamerabot.utils.utils import shallow_sleep_async +from hikcamerabot.utils.shared import shallow_sleep_async if TYPE_CHECKING: from hikcamerabot.services.stream.dvr.file_wrapper import DvrFile diff --git a/hikcamerabot/services/stream/dvr/tasks/file_lock_check.py b/hikcamerabot/services/stream/dvr/tasks/file_lock_check.py index 7444b6e..1a00ac2 100644 --- a/hikcamerabot/services/stream/dvr/tasks/file_lock_check.py +++ b/hikcamerabot/services/stream/dvr/tasks/file_lock_check.py @@ -3,6 +3,7 @@ import os import signal +from hikcamerabot.utils.process import get_stdout_stderr from hikcamerabot.utils.task import wrap @@ -22,6 +23,9 @@ async def run(self) -> list[str]: return await self._get_unlocked_files() async def _get_unlocked_files(self) -> list[str]: + """Return list with absolute file paths that are not locked by ffmpeg process + during write operation. + """ proc = await asyncio.create_subprocess_shell( self._LOCKED_FILES_CMD, stdout=asyncio.subprocess.PIPE, @@ -32,15 +36,16 @@ async def _get_unlocked_files(self) -> list[str]: except asyncio.TimeoutError: self._log.error( 'Failed to execute %s: process ran longer than ' - 'expected and was killed', + 'expected (%ds) and was killed', self._LOCKED_FILES_CMD, + self._PROCESS_TIMEOUT, ) await self._killpg(os.getpgid(proc.pid), signal.SIGINT) return [] - stdout, stderr = await self._get_stdout_stderr(proc) + stdout, stderr = await get_stdout_stderr(proc) self._log.debug( - 'Process %s returncode: %d, stdout: %s, stderr: %s', + 'Process "%s" returncode: %d, stdout: %s, stderr: %s', self._LOCKED_FILES_CMD, proc.returncode, stdout, @@ -49,8 +54,3 @@ async def _get_unlocked_files(self) -> list[str]: unlocked_files = [f for f in self._files if f not in stdout] unlocked_files.sort() return unlocked_files - - @staticmethod - async def _get_stdout_stderr(proc: asyncio.subprocess.Process) -> tuple[str, str]: - stdout, stderr = await proc.stdout.read(), await proc.stderr.read() - return stdout.decode().strip(), stderr.decode().strip() diff --git a/hikcamerabot/services/stream/dvr/tasks/file_monitoring.py b/hikcamerabot/services/stream/dvr/tasks/file_monitoring.py index e2a092b..1471c45 100644 --- a/hikcamerabot/services/stream/dvr/tasks/file_monitoring.py +++ b/hikcamerabot/services/stream/dvr/tasks/file_monitoring.py @@ -5,7 +5,7 @@ from addict import Dict from hikcamerabot.services.stream.dvr.tasks.file_lock_check import FileLockCheckTask -from hikcamerabot.utils.utils import shallow_sleep_async +from hikcamerabot.utils.shared import shallow_sleep_async if TYPE_CHECKING: from hikcamerabot.services.stream.dvr.upload.engine import DvrUploadEngine diff --git a/hikcamerabot/services/stream/dvr/upload/engine.py b/hikcamerabot/services/stream/dvr/upload/engine.py index a621cbc..377c9d6 100644 --- a/hikcamerabot/services/stream/dvr/upload/engine.py +++ b/hikcamerabot/services/stream/dvr/upload/engine.py @@ -60,7 +60,7 @@ async def _wrap_as_dvr_files(self, files: list[str]) -> list[DvrFile]: async def start(self) -> None: await self._start_tasks() - self._log.debug('Upload Engine for %s has started', self._cam.description) + self._log.debug('Upload Engine for %s has started', self._cam) async def _start_tasks(self) -> None: await asyncio.gather( @@ -72,7 +72,7 @@ async def _start_tasks(self) -> None: async def _start_storage_tasks(self) -> None: for storage, queue in self._storage_queues.items(): self._log.debug( - 'Starting %s upload task for %s storage', self._cam.description, storage + 'Starting %s upload task for %s storage', self._cam, storage ) task = self._UPLOAD_TASKS[DvrUploadType(storage)] create_task( @@ -99,9 +99,7 @@ async def _start_file_monitoring_task_(self) -> None: async def _start_file_deletion_task_(self) -> None: if self._conf.upload.delete_after_upload: - self._log.debug( - 'Starting DVR file deletion task for %s', self._cam.description - ) + self._log.debug('Starting DVR file deletion task for %s', self._cam) create_task( self._FILE_DELETE_TASK_CLS( queue=self._delete_candidates_queue, diff --git a/hikcamerabot/services/tasks/livestream.py b/hikcamerabot/services/tasks/livestream.py index b1b825a..c8c80f5 100644 --- a/hikcamerabot/services/tasks/livestream.py +++ b/hikcamerabot/services/tasks/livestream.py @@ -6,7 +6,7 @@ from hikcamerabot.enums import ServiceType from hikcamerabot.exceptions import HikvisionCamError from hikcamerabot.services.abstract import AbstractServiceTask -from hikcamerabot.utils.utils import shallow_sleep_async +from hikcamerabot.utils.shared import shallow_sleep_async class FfmpegStdoutReaderTask: diff --git a/hikcamerabot/utils/image.py b/hikcamerabot/utils/image.py index 8132b15..f3688fc 100644 --- a/hikcamerabot/utils/image.py +++ b/hikcamerabot/utils/image.py @@ -6,7 +6,7 @@ from PIL import Image from hikcamerabot.constants import Img -from hikcamerabot.utils.utils import Singleton +from hikcamerabot.utils.shared import Singleton class ImageProcessor(metaclass=Singleton): diff --git a/hikcamerabot/utils/process.py b/hikcamerabot/utils/process.py new file mode 100644 index 0000000..8a34e90 --- /dev/null +++ b/hikcamerabot/utils/process.py @@ -0,0 +1,6 @@ +from asyncio.subprocess import Process + + +async def get_stdout_stderr(proc: Process) -> tuple[str, str]: + stdout, stderr = await proc.stdout.read(), await proc.stderr.read() + return stdout.decode().strip(), stderr.decode().strip() diff --git a/hikcamerabot/utils/utils.py b/hikcamerabot/utils/shared.py similarity index 100% rename from hikcamerabot/utils/utils.py rename to hikcamerabot/utils/shared.py diff --git a/hikcamerabot/version.py b/hikcamerabot/version.py index fcb6b5d..51ed7c4 100644 --- a/hikcamerabot/version.py +++ b/hikcamerabot/version.py @@ -1 +1 @@ -__version__ = '1.5' +__version__ = '1.5.1'