From 414b1c97fe051bd65b3a5ff7181a8523f09be335 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 6 Oct 2023 11:44:24 +0200 Subject: [PATCH] Rework youtube flags descriptions + flags ordering + set api_key as secret --- .../src/common/schemas/offliners/youtube.py | 313 ++++++++++-------- 1 file changed, 172 insertions(+), 141 deletions(-) diff --git a/dispatcher/backend/src/common/schemas/offliners/youtube.py b/dispatcher/backend/src/common/schemas/offliners/youtube.py index 46b65cc4..6bad3e71 100644 --- a/dispatcher/backend/src/common/schemas/offliners/youtube.py +++ b/dispatcher/backend/src/common/schemas/offliners/youtube.py @@ -17,74 +17,136 @@ class Meta: falsy=[False], metadata={ "label": "Playlists mode", - "description": "Build one ZIM per playlist of the channel or user", + "description": "If set, playlists mode is activated and one ZIM will be " + "built per playlist of the channel or user", }, data_key="indiv-playlists", ) + ident = String( + metadata={ + "label": "Youtube ID(s)", + "description": "Youtube ID(s) of the user, channel or playlist(s) to ZIM " + "(depending on the Type chosen below). Only playlist Type support multiple " + "Youtube IDs and they must be separated by commas.", + }, + data_key="id", + required=True, + ) + kind = StringEnum( metadata={ "label": "Type", - "description": "Type of collection. Only `playlist` accepts multiple IDs.", + "description": "Type of Youtube ID.", }, validate=validate.OneOf(["channel", "playlist", "user"]), data_key="type", required=True, ) - ident = String( + + language = String( metadata={ - "label": "Youtube ID", - "description": "Youtube ID of the collection. " - "Separate multiple playlists with commas.", - }, - data_key="id", - required=True, - ) - api_key = String( - metadata={"label": "API Key", "description": "Youtube API Token"}, - data_key="api-key", - required=True, + "label": "Language", + "description": "ISO-639-3 (3 chars) language code of content", + } ) name = String( metadata={ "label": "ZIM Name", - "description": "Used as identifier and filename (date will be appended)", + "description": "Normal mode: Used as identifier and filename (date will " + "be appended)", "placeholder": "mychannel_eng_all", }, ) + + zim_file = String( + metadata={ + "label": "ZIM Filename", + "description": "Normal mode: ZIM file name (optional, based on ZIM Name " + "if not provided). Include {period} to insert date period dynamically", + }, + data_key="zim-file", + validate=validate_zim_filename, + ) + + title = String( + metadata={ + "label": "ZIM Title", + "description": "Normal mode: Custom title for your project and ZIM. " + "Default to Channel name (of first video if playlists)", + } + ) + + description = String( + metadata={ + "label": "ZIM Description", + "description": "Normal mode: Description for ZIM", + }, + validate=validate_zim_description, + ) + playlists_name = String( metadata={ - "label": "Playlists name", - "description": "Format for building individual --name argument. " - "Required in playlist mode. Variables: {title}, {description}, " + "label": "Playlists ZIM Name", + "description": "Playlists mode: custom format for building each " + "ZIM Name argument for each playlists. Required in playlist mode. " + "You might use these placeholders: {title}, {description}, " "{playlist_id}, {slug} (from title), {creator_id}, {creator_name}", }, data_key="playlists-name", ) - video_format = StringEnum( + playlists_zim_file = String( metadata={ - "label": "Video format", - "description": "Format to download/transcode video to. webm is smaller", + "label": "Playlists ZIM Filename", + "description": "Playlists mode: custom filename format for building " + "each ZIM Filename argument. Same placeholders as Playlists ZIM Name.", }, - validate=validate.OneOf(["webm", "mp4"]), - data_key="format", + data_key="playlists-zim-file", ) - low_quality = fields.Boolean( - truthy=[True], - falsy=[False], + + playlists_title = String( metadata={ - "label": "Low Quality", - "description": "Re-encode video using stronger compression", + "label": "Playlists ZIM Title", + "description": "Playlists mode: custom title format for building each " + "ZIM Title. Same placeholders as Playlists ZIM Name.", }, - data_key="low-quality", + data_key="playlists-title", ) - concurrency = fields.Integer( + + playlists_description = String( metadata={ - "label": "Concurrency", - "description": "Number of concurrent threads to use", + "label": "Playlists ZIM description", + "description": "Playlists mode: custom description format for building " + "each ZIM Description. Same placeholders as Playlists ZIM Name.", }, + data_key="playlists-description", + ) + + creator = String( + metadata={ + "label": "Content Creator", + "description": "Name of content creator. Defaults to Channel name " + "or “Youtube Channels”", + } + ) + + tags = String( + metadata={ + "label": "ZIM Tags", + "description": "List of Tags for the ZIM file. " + "_videos:yes added automatically", + } + ) + + locale = String( + metadata={ + "label": "Locale", + "description": "Locale name to use for translations (if avail) " + "and time representations. Defaults to Language flag value or English if " + "flag is not set.", + } ) dateafter = String( @@ -96,13 +158,23 @@ class Meta: } ) - optimization_cache = fields.Url( + video_format = StringEnum( metadata={ - "label": "Optimization Cache URL", - "description": "S3 Storage URL including credentials and bucket", - "secret": True, + "label": "Video format", + "description": "Format to download/transcode video to. webm is smaller", }, - data_key="optimization-cache", + validate=validate.OneOf(["webm", "mp4"]), + data_key="format", + ) + + low_quality = fields.Boolean( + truthy=[True], + falsy=[False], + metadata={ + "label": "Low Quality", + "description": "Re-encode video using stronger compression", + }, + data_key="low-quality", ) use_any_optimized_version = fields.Boolean( @@ -124,12 +196,14 @@ class Meta: }, data_key="all-subtitles", ) + pagination = fields.Integer( metadata={ "label": "Pagination", "description": "Number of videos per page (40 otherwise)", }, ) + autoplay = fields.Boolean( truthy=[True], falsy=[False], @@ -139,146 +213,103 @@ class Meta: "(home never have autoplay).", }, ) - output = String( - metadata={ - "label": "Output folder", - "placeholder": "/output", - "description": "Output folder for ZIM file(s). Leave it as `/output`", - }, - load_default="/output", - dump_default="/output", - validate=validate_output, - ) - tmp_dir = String( - metadata={ - "label": "Temp folder", - "placeholder": "/output", - "description": "Where to create temporay build folder. " - "Leave it as `/output`", - }, - load_default="/output", - dump_default="/output", - validate=validate_output, - data_key="tmp-dir", - ) - - zim_file = String( - metadata={ - "label": "ZIM filename", - "description": "ZIM file name (based on --name if not provided). " - "Include {period} to insert date period dynamically", - }, - data_key="zim-file", - validate=validate_zim_filename, - ) - playlists_zim_file = String( - metadata={ - "label": "Playlists ZIM filename", - "description": "Format for building individual --zim-file argument. " - "Uses --playlists-name otherwise", - }, - data_key="playlists-zim-file", - ) - language = String( - metadata={ - "label": "Language", - "description": "ISO-639-3 (3 chars) language code of content", - } - ) - locale = String( + profile = fields.Url( metadata={ - "label": "Locale", - "description": "Locale name to use for translations (if avail) " - "and time representations. Defaults to --language or English.", + "label": "Profile Image", + "description": "Custom profile image. Squared. " + "Will be resized to 100x100px", } ) - title = String( + banner = fields.Url( metadata={ - "label": "Title", - "description": "Custom title for your project and ZIM. Default to " - "Channel name (of first video if playlists)", + "label": "Banner Image", + "description": "Custom banner image. Will be resized to 1060x175px", } ) - playlists_title = String( + + main_color = HexColor( metadata={ - "label": "Playlists title", - "description": "Custom title format for individual playlist ZIM", + "label": "Main Color", + "description": "Custom color. Hex/HTML syntax (#DEDEDE). " + "Default to main color of profile image.", }, - data_key="playlists-title", + data_key="main-color", ) - description = String( - metadata={"label": "Description", "description": "Description for ZIM"}, - validate=validate_zim_description, - ) - playlists_description = String( + secondary_color = HexColor( metadata={ - "label": "Playlists description", - "description": "Custom description format for individual playlist ZIM", + "label": "Secondary Color", + "description": "Custom secondary color. Hex/HTML syntax (#DEDEDE). " + "Default to secondary color of profile image.", }, - data_key="playlists-description", + data_key="secondary-color", ) - creator = String( - metadata={ - "label": "Content Creator", - "description": "Name of content creator. Defaults to Channel name " - "or “Youtue Channels”", - } - ) - tags = String( - metadata={ - "label": "ZIM Tags", - "description": "List of Tags for the ZIM file. " - "_videos:yes added automatically", - } + debug = fields.Boolean( + truthy=[True], + falsy=[False], + metadata={"label": "Debug", "description": "Enable verbose output"}, ) metadata_from = String( metadata={ "label": "Metadata JSON", - "description": "File path or URL to a JSON file holding custom metadata " - "for individual playlists", + "description": "Expert flag: File path or URL to a JSON file holding " + "custom metadata for individual playlists", }, data_key="metadata-from", ) - profile = fields.Url( + api_key = String( metadata={ - "label": "Profile Image", - "description": "Custom profile image. Squared. " - "Will be resized to 100x100px", - } + "label": "API Key", + "description": "Expert flag: Youtube API Token", + "secret": True, + }, + data_key="api-key", + required=True, ) - banner = fields.Url( + + optimization_cache = fields.Url( metadata={ - "label": "Banner Image", - "description": "Custom banner image. Will be resized to 1060x175px", - } + "label": "Optimization Cache URL", + "description": "Expert Flag: S3 Storage URL including credentials and bucket", + "secret": True, + }, + data_key="optimization-cache", ) - main_color = HexColor( + + concurrency = fields.Integer( metadata={ - "label": "Main Color", - "description": "Custom color. Hex/HTML syntax (#DEDEDE). " - "Default to main color of profile image.", + "label": "Concurrency", + "description": "Expert flag: Number of concurrent threads to use", }, - data_key="main-color", ) - secondary_color = HexColor( + + output = String( metadata={ - "label": "Secondary Color", - "description": "Custom secondary color. Hex/HTML syntax (#DEDEDE). " - "Default to secondary color of profile image.", + "label": "Output folder", + "placeholder": "/output", + "description": "Technical flag: Output folder for ZIM file(s). Leave it as `/output`", }, - data_key="secondary-color", + load_default="/output", + dump_default="/output", + validate=validate_output, ) - debug = fields.Boolean( - truthy=[True], - falsy=[False], - metadata={"label": "Debug", "description": "Enable verbose output"}, + tmp_dir = String( + metadata={ + "label": "Temp folder", + "placeholder": "/output", + "description": "Expert flag: Where to create temporay build folder. " + "Leave it as `/output`", + }, + load_default="/output", + dump_default="/output", + validate=validate_output, + data_key="tmp-dir", ) @validates_schema