Skip to content

Commit

Permalink
Adding import of OPML files
Browse files Browse the repository at this point in the history
  • Loading branch information
vscode iu2frl.it committed Nov 2, 2024
1 parent c4be42d commit 784c01a
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 37 deletions.
11 changes: 8 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

This bot reads articles from a list of RSS feeds, translate them to a custom language and sends the output to a Telegram chat (can be any chat, group or channel)

## Live example

See a live example of this code at [IU2FRL Blog channel](https://t.me/iu2frl_news) on Telegram

## Configuration

Working variables are set via environments, these are the used ones:
Expand All @@ -19,10 +23,11 @@ Working variables are set via environments, these are the used ones:
- `/addfeed [url]`: adds a new URL to the RSS feeds list
- `/rmfeed [id]`: removes the specified ID from the RSS list
- `/force`: forces a bot execution
- `/rmoldnews`: removes old news from the DB (older than `MAX_NEWS_AGE` days)
- `/rmoldnews`: removes old news from the DB (older than `MAX_NEWS_AGE` days - generally not needed, this is done automatically)
- `/addcsv [url],[url],[...]`: adds a list of RSS feeds separated by commas
- `/dbcleanup`: check if RSS feeds are valid or duplicated
- `/sqlitebackup`: makes a backup of the SQLite database
- `/dbcleanup`: check if any of the RSS feeds are invalid or duplicated and deletes them from DB
- `/sqlitebackup`: makes a backup of the SQLite database and sends it to the chat
- `/importopml [opml file url]`: to import a list of feeds from an OPML file (example: `https://git.dk1mi.radio/mclemens/Ham-Radio-RSS-Feeds/raw/branch/main/hamradio.opml`)

## Functioning

Expand Down
104 changes: 70 additions & 34 deletions frlbot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@
import getopt
import threading
from googletrans import Translator
import requests
import xml.dom.minidom
import emoji
import requests
import xml.etree.ElementTree as ET
import csv

# Specify logging level
logging.basicConfig(level=logging.DEBUG)
Expand Down Expand Up @@ -458,6 +459,43 @@ def telegram_loop():
logging.info("Starting telegram loop")
telegramBot.infinity_polling()

def file_download(url):
response = requests.get(url)
response.raise_for_status() # Check if the request was successful
return response.text

def opml_import_xmlfeeds(opml_content) -> int:
root = ET.fromstring(opml_content)
imported_feeds = 0
for outline in root.findall(".//outline"):
xmlFeed = outline.get("xmlUrl") # Use "xmlUrl" to find xmlFeed
if xmlFeed:
if add_feed_if_not_duplicate(xmlFeed):
imported_feeds += 1
return imported_feeds

def add_feed_if_not_duplicate(feed_url) -> bool:
sqlCon = get_sql_connector()
if sqlCon.execute("SELECT * FROM feeds WHERE url=?", [feed_url]).fetchone() is not None:
logging.warning("Duplicate URL [" + feed_url + "]")
return False
else:
try:
logging.info("Adding [" + feed_url + "] to DB")
if valid_xml(feed_url):
sqlCon.execute("INSERT INTO feeds(url) VALUES(?)", [feed_url])
logging.debug("Added [" + feed_url + "] to DB")
else:
logging.warning("RSS feed [" + feed_url + "] cannot be validated")
return False
except Exception as retExc:
logging.warning(retExc)
return False
# Commit changes to DB
sqlCon.commit()
sqlCon.close()
return True

# Main method invocation
if __name__ == "__main__":
logging.info("Starting frlbot at " + str(datetime.now()))
Expand Down Expand Up @@ -511,21 +549,10 @@ def HandleAddMessage(inputMessage: telebot.types.Message):
return
logging.debug("Feed add requested from [" + str(inputMessage.from_user.id) + "]")
# Check if feed already exists
if sqlCon.execute("SELECT * FROM feeds WHERE url=?", [splitText[1]]).fetchone() is not None:
logging.warning("Duplicate URL [" + splitText[1] + "]")
telegramBot.reply_to(inputMessage, "URL exists in the DB")
return
# Add it to the store
try:
logging.info("Adding [" + splitText[1] + "] to DB")
if valid_xml(splitText[1]):
sqlCon.execute("INSERT INTO feeds(url) VALUES(?)", [splitText[1]])
sqlCon.commit()
telegramBot.reply_to(inputMessage, "Added successfully!")
else:
telegramBot.reply_to(inputMessage, "RSS feed cannot be validated (invalid syntax or unreachable)")
except Exception as retExc:
telegramBot.reply_to(inputMessage, retExc)
if add_feed_if_not_duplicate(splitText[1]):
telegramBot.reply_to(inputMessage, "Added successfully!")
else:
telegramBot.reply_to(inputMessage, "RSS feed cannot be validated (invalid syntax, unreachable or duplicated)")
else:
logging.warning("Invalid AddFeed arguments [" + inputMessage.text + "]")
telegramBot.reply_to(inputMessage, "Expecting only one argument")
Expand Down Expand Up @@ -591,7 +618,7 @@ def HandleAddCsvList(inputMessage: telebot.types.Message):
if inputMessage.from_user.id == get_admin_chat_from_env():
logging.debug("Adding news from CSV list")
global telegramBot
sqlCon = get_sql_connector()

splitMessage = inputMessage.text.split("/addcsv")
# Invalid syntax
if len(splitMessage) <= 1:
Expand All @@ -607,23 +634,9 @@ def HandleAddCsvList(inputMessage: telebot.types.Message):
for singleUrl in splitCsv:
# Clean input string
singleUrl = singleUrl.strip()
# Check if feed already exists
if sqlCon.execute("SELECT * FROM feeds WHERE url=?", [singleUrl]).fetchone() is not None:
logging.warning("Duplicate URL [" + singleUrl + "]")
else:
try:
logging.info("Adding [" + singleUrl + "] to DB")
if valid_xml(singleUrl):
sqlCon.execute("INSERT INTO feeds(url) VALUES(?)", [singleUrl])
newFeedsCnt += 1
logging.debug("Added [" + singleUrl + "] to DB")
else:
logging.warning("RSS feed [" + singleUrl + "] cannot be validated")
except Exception as retExc:
continue
# Commit changes to DB
sqlCon.commit()
sqlCon.close()
# Add feed if not existing
if (add_feed_if_not_duplicate(singleUrl)):
newFeedsCnt += 1
# Send reply
telegramBot.reply_to(inputMessage, "[" + str(newFeedsCnt) + "] out of [" + str(len(splitCsv)) + "] feeds were added to DB")
else:
Expand Down Expand Up @@ -682,6 +695,29 @@ def HandleSqliteBackup(inputMessage: telebot.types.Message):
telegramBot.reply_to(inputMessage, "Error: " + str(retExc))
else:
logging.debug("Ignoring message from [" + str(inputMessage.from_user.id) + "]")
# Parse OPML file
@telegramBot.message_handler(content_types=["text"], commands=['importopml'])
def HandleImportOPML(inputMessage: telebot.types.Message):
global telegramBot
if inputMessage.from_user.id == get_admin_chat_from_env():
logging.debug("OPML file import requested from [" + str(inputMessage.from_user.id) + "]")
splitText = inputMessage.text.split(" ")
if len(splitText) != 2:
telegramBot.reply_to(inputMessage, f"Command length is invalid, found {len(splitText)} arguments")
logging.warning(f"Invalid OPML import command received: {inputMessage}")
return
else:
telegramBot.reply_to(inputMessage, f"Starting OPML file import, please wait")
logging.info(f"Starting OPML import of [{splitText[1]}]")
# Parse OPML file
try:
opml_content = file_download(splitText[1])
imported_feeds = opml_import_xmlfeeds(opml_content)
telegramBot.reply_to(inputMessage, f"Imported {imported_feeds} feeds")
except Exception as retExc:
telegramBot.reply_to(inputMessage, "Error: " + str(retExc))
else:
logging.debug("Ignoring message from [" + str(inputMessage.from_user.id) + "]")
# Prepare DB object
prepare_db()
if forceRun:
Expand Down

0 comments on commit 784c01a

Please sign in to comment.