Skip to content

Commit

Permalink
chore:result cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@Guovin
Guovin committed Sep 24, 2024
1 parent fb30ba2 commit bcc8a7e
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 44 deletions.
10 changes: 5 additions & 5 deletions main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
write_channel_to_file,
setup_logging,
cleanup_logging,
get_channel_data_with_cache_compare,
get_channel_data_cache_with_compare,
)
from utils.tools import (
update_file,
Expand Down Expand Up @@ -155,7 +155,7 @@ async def main(self):
self.subscribe_result,
self.online_search_result,
)
channel_data_with_cache = self.channel_data
channel_data_cache = self.channel_data
self.total = self.get_urls_len(filter=True)
sort_callback = lambda: self.pbar_update(name="测速")
open_sort = config.getboolean("Settings", "open_sort")
Expand Down Expand Up @@ -190,11 +190,11 @@ async def main(self):
shutil.copy(user_final_file, result_file)
if config.getboolean("Settings", "open_use_old_result"):
if open_sort:
channel_data_with_cache = get_channel_data_with_cache_compare(
channel_data_with_cache, self.channel_data
channel_data_cache = get_channel_data_cache_with_compare(
channel_data_cache, self.channel_data
)
with open(resource_path("output/result_cache.pkl"), "wb") as file:
pickle.dump(channel_data_with_cache, file)
pickle.dump(channel_data_cache, file)
if open_sort:
user_log_file = "output/" + (
"user_result.log"
Expand Down
70 changes: 31 additions & 39 deletions utils/channel.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,29 +50,28 @@ def cleanup_logging():
os.remove(log_path)


def get_channel_data_from_file(channels=None, file=None, use_old=False):
def get_channel_data_from_file(channels, file, use_old):
"""
Get the channel data from the file
"""
current_category = ""
pattern = r"^(.*?)(,(?!#genre#)(.*?))?$"
pattern = re.compile(r"^(.*?)(,(?!#genre#)(.*?))?$")

for line in file:
line = line.strip()
if "#genre#" in line:
# This is a new channel, create a new key in the dictionary.
current_category = line.split(",")[0]
else:
# This is a url, add it to the list of urls for the current channel.
match = re.search(pattern, line)
match = pattern.search(line)
if match is not None and match.group(1):
name = match.group(1).strip()
if name not in channels[current_category]:
channels[current_category][name] = []
category_dict = channels[current_category]
if name not in category_dict:
category_dict[name] = []
if use_old and match.group(3):
url = match.group(3).strip()
if url and url not in channels[current_category][name]:
channels[current_category][name].append(url)
info = (match.group(3).strip(), None, None)
if info[0] and info not in category_dict[name]:
category_dict[name].append(info)
return channels


Expand All @@ -86,23 +85,20 @@ def get_channel_items():

if os.path.exists(resource_path(user_source_file)):
with open(resource_path(user_source_file), "r", encoding="utf-8") as file:
channels = get_channel_data_from_file(
channels=channels, file=file, use_old=open_use_old_result
)

if open_use_old_result and os.path.exists(resource_path("output/result_cache.pkl")):
with open(resource_path("output/result_cache.pkl"), "rb") as file:
old_result = pickle.load(file)
for cate, data in channels.items():
if cate in old_result:
for name, urls in data.items():
if name in old_result[cate]:
old_urls = [
url
for info in old_result[cate][name]
for url, _, _ in info
]
channels[cate][name] = set(urls + old_urls)
channels = get_channel_data_from_file(channels, file, open_use_old_result)

if open_use_old_result:
result_cache_path = resource_path("output/result_cache.pkl")
if os.path.exists(result_cache_path):
with open(resource_path("output/result_cache.pkl"), "rb") as file:
old_result = pickle.load(file)
for cate, data in channels.items():
if cate in old_result:
for name, info_list in data.items():
if name in old_result[cate]:
for info in old_result[cate][name]:
if info not in info_list:
channels[cate][name].append(info)
return channels


Expand Down Expand Up @@ -520,7 +516,7 @@ def append_all_method_data(
Append all method data to total info data
"""
for cate, channel_obj in items:
for name, old_urls in channel_obj.items():
for name, old_info_list in channel_obj.items():
for method, result in [
("hotel_fofa", hotel_fofa_result),
("multicast", multicast_result),
Expand Down Expand Up @@ -553,9 +549,9 @@ def append_all_method_data(
data,
cate,
name,
[(url, None, None) for url in old_urls],
old_info_list,
)
print(name, "using old num:", len(old_urls))
print(name, "using old num:", len(old_info_list))
print(
name,
"total num:",
Expand Down Expand Up @@ -593,14 +589,14 @@ def append_all_method_data_keep_all(
data = append_data_to_info_data(data, cate, name, urls)
print(name, f"{method.capitalize()} num:", len(urls))
if config.getboolean("Settings", "open_use_old_result"):
old_urls = channel_obj.get(name, [])
old_info_list = channel_obj.get(name, [])
data = append_data_to_info_data(
data,
cate,
name,
[(url, None, None) for url in old_urls],
old_info_list,
)
print(name, "using old num:", len(old_urls))
print(name, "using old num:", len(old_info_list))
return data


Expand Down Expand Up @@ -740,7 +736,7 @@ def get_multicast_fofa_search_urls():
return search_urls


def get_channel_data_with_cache_compare(data, new_data):
def get_channel_data_cache_with_compare(data, new_data):
"""
Get channel data with cache compare new data
"""
Expand All @@ -754,10 +750,6 @@ def match_url(url, sort_urls):
if url_info and cate in data and name in data[cate]:
new_urls = {new_url for new_url, _, _ in url_info}
data[cate][name] = [
url
for info in data[cate][name]
for url, _, _ in info
if match_url(url, new_urls)
info for info in data[cate][name] if match_url(info[0], new_urls)
]

return data

0 comments on commit bcc8a7e

Please sign in to comment.