Skip to content

Commit

Permalink
pluginupdate.py: fix bugs and add improvements
Browse files Browse the repository at this point in the history 
I fixed many hidden bugs and made some small improvements. The main
reason this was separated from NixOS#336137 is to fix the sorting issue.

Before this commit, sorting for Vim plugins was broken and worked by
what was fetched first. This is because the sorting was done by empty
strings (the default value in CSV is not None but an empty string). This
PR fixes it and also moves sorting from the user to the library (from
`vim/plugins/update.py` to `pluginupdate.py`) to prevent such weird
issues and duplication of code.
  • Loading branch information
@PerchunPak
PerchunPak committed Nov 5, 2024
1 parent e2e4131 commit 08a39df
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 134 deletions.
101 changes: 52 additions & 49 deletions maintainers/scripts/pluginupdate-py/pluginupdate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# - pkgs/development/lua-modules/updater/updater.py

# format:
# $ nix run nixpkgs#black maintainers/scripts/pluginupdate.py
# $ nix run nixpkgs#ruff maintainers/scripts/pluginupdate.py
# type-check:
# $ nix run nixpkgs#python3.pkgs.mypy maintainers/scripts/pluginupdate.py
# linted:
Expand Down Expand Up @@ -142,7 +142,7 @@ def _prefetch(self, ref: Optional[str]):
return loaded

def prefetch(self, ref: Optional[str]) -> str:
print("Prefetching %s", self.uri)
log.info("Prefetching %s", self.uri)
loaded = self._prefetch(ref)
return loaded["sha256"]

Expand Down Expand Up @@ -195,7 +195,7 @@ def latest_commit(self) -> Tuple[str, datetime]:
xml = req.read()

# Filter out illegal XML characters
illegal_xml_regex = re.compile(b"[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]")
illegal_xml_regex = re.compile(b"[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]")
xml = illegal_xml_regex.sub(b"", xml)

root = ET.fromstring(xml)
Expand Down Expand Up @@ -256,21 +256,20 @@ class PluginDesc:

@property
def name(self):
if self.alias is None:
return self.repo.name
else:
return self.alias

def __lt__(self, other):
return self.repo.name < other.repo.name
return self.alias or self.repo.name

@staticmethod
def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> "PluginDesc":
log.debug("Loading row %s", row)
branch = row["branch"]
repo = make_repo(row["repo"], branch.strip())
repo.token = config.github_token
return PluginDesc(repo, branch.strip(), row["alias"])
return PluginDesc(
repo,
branch.strip(),
# alias is usually an empty string
row["alias"] if row["alias"] else None,
)

@staticmethod
def load_from_string(config: FetchConfig, line: str) -> "PluginDesc":
Expand Down Expand Up @@ -328,12 +327,11 @@ def load_plugins_from_csv(
return plugins



def run_nix_expr(expr, nixpkgs: str, **args):
'''
"""
:param expr nix expression to fetch current plugins
:param nixpkgs Path towards a nixpkgs checkout
'''
"""
with CleanEnvironment(nixpkgs) as nix_path:
cmd = [
"nix",
Expand Down Expand Up @@ -382,16 +380,14 @@ def add(self, args):
fetch_config = FetchConfig(args.proc, args.github_token)
editor = self
for plugin_line in args.add_plugins:
log.debug("using plugin_line", plugin_line)
log.debug("using plugin_line %s", plugin_line)
pdesc = PluginDesc.load_from_string(fetch_config, plugin_line)
log.debug("loaded as pdesc", pdesc)
log.debug("loaded as pdesc %s", pdesc)
append = [pdesc]
editor.rewrite_input(
fetch_config, args.input_file, editor.deprecated, append=append
)
plugin, _ = prefetch_plugin(
pdesc,
)
plugin, _ = prefetch_plugin(pdesc)
autocommit = not args.no_commit
if autocommit:
commit(
Expand All @@ -406,9 +402,9 @@ def add(self, args):
# Expects arguments generated by 'update' subparser
def update(self, args):
"""CSV spec"""
print("the update member function should be overriden in subclasses")
print("the update member function should be overridden in subclasses")

def get_current_plugins(self, nixpkgs) -> List[Plugin]:
def get_current_plugins(self, nixpkgs: str) -> List[Plugin]:
"""To fill the cache"""
data = run_nix_expr(self.get_plugins, nixpkgs)
plugins = []
Expand Down Expand Up @@ -440,6 +436,7 @@ def update() -> dict:

plugins, redirects = check_results(results)

plugins = sorted(plugins, key=lambda v: v[1].normalized_name)
self.generate_nix(plugins, outfile)

return redirects
Expand Down Expand Up @@ -559,6 +556,7 @@ def run(
parser = self.create_parser()
args = parser.parse_args()
command = args.command or "update"
logging.basicConfig()
log.setLevel(LOG_LEVELS[args.debug])
log.info("Chose to run command: %s", command)
self.nixpkgs = args.nixpkgs
Expand Down Expand Up @@ -591,25 +589,24 @@ def prefetch_plugin(
p: PluginDesc,
cache: "Optional[Cache]" = None,
) -> Tuple[Plugin, Optional[Repo]]:
repo, branch, alias = p.repo, p.branch, p.alias
name = alias or p.repo.name
commit = None
log.info(f"Fetching last commit for plugin {name} from {repo.uri}@{branch}")
commit, date = repo.latest_commit()
log.info(f"Fetching last commit for plugin {p.name} from {p.repo.uri}@{p.branch}")
commit, date = p.repo.latest_commit()

cached_plugin = cache[commit] if cache else None
if cached_plugin is not None:
log.debug("Cache hit !")
cached_plugin.name = name
log.debug(f"Cache hit for {p.name}!")
cached_plugin.name = p.name
cached_plugin.date = date
return cached_plugin, repo.redirect
return cached_plugin, p.repo.redirect

has_submodules = repo.has_submodules()
log.debug(f"prefetch {name}")
sha256 = repo.prefetch(commit)
has_submodules = p.repo.has_submodules()
log.debug(f"prefetch {p.name}")
sha256 = p.repo.prefetch(commit)

return (
Plugin(name, commit, has_submodules, sha256, date=date),
repo.redirect,
Plugin(p.name, commit, has_submodules, sha256, date=date),
p.repo.redirect,
)


Expand All @@ -624,7 +621,7 @@ def print_download_error(plugin: PluginDesc, ex: Exception):


def check_results(
results: List[Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]]
results: List[Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]],
) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]:
""" """
failures: List[Tuple[PluginDesc, Exception]] = []
Expand All @@ -642,10 +639,9 @@ def check_results(

print(f"{len(results) - len(failures)} plugins were checked", end="")
if len(failures) == 0:
print()
return plugins, redirects
else:
print(f", {len(failures)} plugin(s) could not be downloaded:\n")
log.error(f", {len(failures)} plugin(s) could not be downloaded:\n")

for plugin, exception in failures:
print_download_error(plugin, exception)
Expand Down Expand Up @@ -738,10 +734,7 @@ def rewrite_input(
append: List[PluginDesc] = [],
):
log.info("Rewriting input file %s", input_file)
plugins = load_plugins_from_csv(
config,
input_file,
)
plugins = load_plugins_from_csv(config, input_file)

plugins.extend(append)

Expand All @@ -753,15 +746,25 @@ def rewrite_input(
deprecations = json.load(f)
# TODO parallelize this step
for pdesc, new_repo in redirects.items():
log.info("Rewriting input file %s", input_file)
log.info("Resolving deprecated plugin %s -> %s", pdesc.name, new_repo.name)
new_pdesc = PluginDesc(new_repo, pdesc.branch, pdesc.alias)

old_plugin, _ = prefetch_plugin(pdesc)
new_plugin, _ = prefetch_plugin(new_pdesc)

if old_plugin.normalized_name != new_plugin.normalized_name:
deprecations[old_plugin.normalized_name] = {
"new": new_plugin.normalized_name,
"date": cur_date_iso,
}

# remove plugin from index file, so we won't add it to deprecations again
for i, plugin in enumerate(plugins):
if plugin.name == pdesc.name:
plugins.pop(i)
break
plugins.append(new_pdesc)

with open(deprecated, "w") as f:
json.dump(deprecations, f, indent=4, sort_keys=True)
f.write("\n")
Expand All @@ -772,7 +775,7 @@ def rewrite_input(
fieldnames = ["repo", "branch", "alias"]
writer = csv.DictWriter(f, fieldnames, dialect="unix", quoting=csv.QUOTE_NONE)
writer.writeheader()
for plugin in sorted(plugins):
for plugin in sorted(plugins, key=lambda x: x.name):
writer.writerow(asdict(plugin))


Expand All @@ -792,9 +795,11 @@ def update_plugins(editor: Editor, args):

log.info("Start updating plugins")
if args.proc > 1 and args.github_token == None:
log.warning("You have enabled parallel updates but haven't set a github token.\n"
"You may be hit with `HTTP Error 429: too many requests` as a consequence."
"Either set --proc=1 or --github-token=YOUR_TOKEN. ")
log.warning(
"You have enabled parallel updates but haven't set a github token.\n"
"You may be hit with `HTTP Error 429: too many requests` as a consequence."
"Either set --proc=1 or --github-token=YOUR_TOKEN. "
)

fetch_config = FetchConfig(args.proc, args.github_token)
update = editor.get_update(args.input_file, args.outfile, fetch_config)
Expand All @@ -810,11 +815,9 @@ def update_plugins(editor: Editor, args):
if autocommit:
try:
repo = git.Repo(os.getcwd())
updated = datetime.now(tz=UTC).strftime('%Y-%m-%d')
updated = datetime.now(tz=UTC).strftime("%Y-%m-%d")
print(args.outfile)
commit(repo,
f"{editor.attr_path}: update on {updated}", [args.outfile]
)
commit(repo, f"{editor.attr_path}: update on {updated}", [args.outfile])
except git.InvalidGitRepositoryError as e:
print(f"Not in a git repository: {e}", file=sys.stderr)
sys.exit(1)
Expand Down
58 changes: 36 additions & 22 deletions pkgs/applications/editors/kakoune/plugins/update.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,63 @@
#!nix-shell update-shell.nix -i python3

# format:
# $ nix run nixpkgs.python3Packages.black -c black update.py
# $ nix run nixpkgs#python3Packages.ruff -- update.py
# type-check:
# $ nix run nixpkgs.python3Packages.mypy -c mypy update.py
# $ nix run nixpkgs#python3Packages.mypy -- update.py
# linted:
# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265,E402 update.py
# $ nix run nixpkgs#python3Packages.flake8 -- --ignore E501,E265,E402 update.py

import inspect
import os
import sys
from typing import List, Tuple
from pathlib import Path
from typing import List, Tuple

# Import plugin update library from maintainers/scripts/pluginupdate.py
ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))) # type: ignore
ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))) # type: ignore
sys.path.insert(
0, os.path.join(ROOT.parent.parent.parent.parent.parent, "maintainers", "scripts")
)
import pluginupdate

GET_PLUGINS = f"""(with import <localpkgs> {{}};
GET_PLUGINS = f"""(
with import <localpkgs> {{ }};
let
inherit (kakouneUtils.override {{}}) buildKakounePluginFrom2Nix;
inherit (kakouneUtils.override {{ }}) buildKakounePluginFrom2Nix;
generated = callPackage {ROOT}/generated.nix {{
inherit buildKakounePluginFrom2Nix;
}};
hasChecksum = value: lib.isAttrs value && lib.hasAttrByPath ["src" "outputHash"] value;
getChecksum = name: value:
if hasChecksum value then {{
submodules = value.src.fetchSubmodules or false;
sha256 = value.src.outputHash;
rev = value.src.rev;
}} else null;
hasChecksum =
value:
lib.isAttrs value
&& lib.hasAttrByPath [
"src"
"outputHash"
] value;
getChecksum =
name: value:
if hasChecksum value then
{{
submodules = value.src.fetchSubmodules or false;
sha256 = value.src.outputHash;
rev = value.src.rev;
}}
else
null;
checksums = lib.mapAttrs getChecksum generated;
in lib.filterAttrs (n: v: v != null) checksums)"""

HEADER = "# This file has been generated by ./pkgs/applications/editors/kakoune/plugins/update.py. Do not edit!"

class KakouneEditor(pluginupdate.Editor):
in
lib.filterAttrs (n: v: v != null) checksums
)"""

HEADER = "# This file has been @generated by ./pkgs/applications/editors/kakoune/plugins/update.py. Do not edit!"

def generate_nix(self, plugins: List[Tuple[pluginupdate.PluginDesc, pluginupdate.Plugin]], outfile: str):
sorted_plugins = sorted(plugins, key=lambda v: v[1].name.lower())

class KakouneEditor(pluginupdate.Editor):
def generate_nix(
self,
plugins: List[Tuple[pluginupdate.PluginDesc, pluginupdate.Plugin]],
outfile: str,
):
with open(outfile, "w+") as f:
f.write(HEADER)
f.write(
Expand All @@ -54,7 +68,7 @@ def generate_nix(self, plugins: List[Tuple[pluginupdate.PluginDesc, pluginupdate
packages = ( self:
{"""
)
for pluginDesc, plugin in sorted_plugins:
for pluginDesc, plugin in plugins:
f.write(
f"""
{plugin.normalized_name} = buildKakounePluginFrom2Nix {{
Expand Down
Loading

0 comments on commit 08a39df

Please sign in to comment.