From e1170343f2a5cc5cde22d12fe5f78204b7c6f4bf Mon Sep 17 00:00:00 2001 From: Moraxyc Date: Mon, 16 Dec 2024 18:46:18 +0800 Subject: [PATCH 1/3] feat: add support for generating package-lock.json --- README.md | 4 +-- nix_update/update.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ea7313c..5778a0f 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ work with nixpkgs but also other package sets. - update mixRelease's mixFodDeps - update fetchYarnDeps offlineCache output hash - update flake outputs (see `--flake`) -- generate the following lockfile, Cargo.lock (see `--generate-lockfile` and - `--lockfile-metadata-path`) +- generate the following lockfile, Cargo.lock, package-lock.json (see + `--generate-lockfile` and `--lockfile-metadata-path`) - build and run the resulting package (see `--build`, `--run` or `--shell` - commit updated files (see `--commit` flag) - run update scripts (`passthru.updateScript`, see `--use-update-script` flag) diff --git a/nix_update/update.py b/nix_update/update.py index 16e6b85..27f8513 100644 --- a/nix_update/update.py +++ b/nix_update/update.py @@ -340,6 +340,81 @@ def disable_copystat(): shutil.copy(lockfile, Path(filename).parent / "Cargo.lock") +def generate_npm_lock(opts: Options, filename: str) -> None: + @contextmanager + def disable_copystat(): + _orig = shutil.copystat + shutil.copystat = lambda *args, **kwargs: None + try: + yield + finally: + shutil.copystat = _orig + + getSrcAndNpm = textwrap.dedent( + f""" + {get_package(opts)}.overrideAttrs (old: {{ + npmDeps = null; + npmDepsHash = null; + postUnpack = '' + cp -pr --reflink=auto -- $sourceRoot $out + mkdir -p "$out/nix-support" + command -v npm > $out/nix-support/npm-bin || {{ + echo "no npm executable found in native build inputs" >&2 + exit 1 + }} + exit + ''; + outputs = [ "out" ]; + separateDebugInfo = false; + }}) + """ + ) + + res = run( + [ + "nix", + "build", + "-L", + "--no-link", + "--impure", + "--print-out-paths", + "--expr", + getSrcAndNpm, + ] + + opts.extra_flags, + ) + src = Path(res.stdout.strip()) + + with tempfile.TemporaryDirectory() as tempdir: + with disable_copystat(): + shutil.copytree(src, tempdir, dirs_exist_ok=True, copy_function=shutil.copy) + + npm_bin = (src / "nix-support" / "npm-bin").read_text().rstrip("\n") + + run( + [ + npm_bin, + "install", + "--package-lock-only", + "--prefix", + opts.lockfile_metadata_path, + ], + cwd=tempdir, + ) + + if ( + lockfile_in_subdir := Path(tempdir) + / opts.lockfile_metadata_path + / "package-lock.json" + ).exists(): + # if package.json is outside a workspace, package0lock.json is generated in the same directory as package.json + lockfile = lockfile_in_subdir + else: + lockfile = Path(tempdir) / "package-lock.json" + + shutil.copy(lockfile, Path(filename).parent / "package-lock.json") + + def update_composer_deps_hash(opts: Options, filename: str, current_hash: str) -> None: target_hash = nix_prefetch(opts, "composerVendor") replace_hash(filename, current_hash, target_hash) @@ -509,6 +584,8 @@ def update(opts: Options) -> Package: ) if package.npm_deps: + if opts.generate_lockfile: + generate_npm_lock(opts, package.filename) update_npm_deps_hash(opts, package.filename, package.npm_deps) if package.pnpm_deps: From f763ab329a7502e00d93c8cc11871bd79cd00c21 Mon Sep 17 00:00:00 2001 From: Moraxyc Date: Mon, 16 Dec 2024 20:08:17 +0800 Subject: [PATCH 2/3] refactor: unify lockfile generation functions - combine cargo and npm lockfile generation into a single function - use parameters to handle different types and configurations - reduce code duplication and improve maintainability --- nix_update/update.py | 128 +++++++++++++------------------------------ 1 file changed, 37 insertions(+), 91 deletions(-) diff --git a/nix_update/update.py b/nix_update/update.py index 27f8513..8f55f08 100644 --- a/nix_update/update.py +++ b/nix_update/update.py @@ -268,79 +268,33 @@ def update_cargo_lock( print(line, end="") -def generate_cargo_lock(opts: Options, filename: str) -> None: - @contextmanager - def disable_copystat(): - _orig = shutil.copystat - shutil.copystat = lambda *args, **kwargs: None - try: - yield - finally: - shutil.copystat = _orig - - getSrcAndCargo = textwrap.dedent(f""" - {get_package(opts)}.overrideAttrs (old: {{ - cargoDeps = null; - cargoVendorDir = "."; - postUnpack = '' - cp -pr --reflink=auto -- $sourceRoot $out - mkdir -p "$out/nix-support" - command -v cargo > $out/nix-support/cargo-bin || {{ - echo "no cargo executable found in native build inputs" >&2 - exit 1 - }} - exit - ''; - outputs = [ "out" ]; - separateDebugInfo = false; - }}) - """) - - res = run( - [ - "nix", - "build", - "-L", - "--no-link", - "--impure", - "--print-out-paths", - "--expr", - getSrcAndCargo, +def generate_lockfile(opts: Options, filename: str, type: str) -> None: + if type == "cargo": + cmd = [ + "generate-lockfile", + "--manifest-path", + f"{opts.lockfile_metadata_path}/Cargo.toml", ] - + opts.extra_flags, - ) - src = Path(res.stdout.strip()) - - with tempfile.TemporaryDirectory() as tempdir: - with disable_copystat(): - shutil.copytree(src, tempdir, dirs_exist_ok=True, copy_function=shutil.copy) - - cargo_bin = (src / "nix-support" / "cargo-bin").read_text().rstrip("\n") - - run( - [ - cargo_bin, - "generate-lockfile", - "--manifest-path", - f"{opts.lockfile_metadata_path}/Cargo.toml", - ], - cwd=tempdir, - ) - - if ( - lockfile_in_subdir := Path(tempdir) - / opts.lockfile_metadata_path - / "Cargo.lock" - ).exists(): - # if Cargo.toml is outside a workspace, Cargo.lock is generated in the same directory as Cargo.toml - lockfile = lockfile_in_subdir - else: - lockfile = Path(tempdir) / "Cargo.lock" - - shutil.copy(lockfile, Path(filename).parent / "Cargo.lock") - + bin_name = "cargo" + lockfile_name = "Cargo.lock" + extra_nix_override = """ + cargoDeps = null; + cargoVendorDir = "."; + """ + elif type == "npm": + cmd = [ + "install", + "--package-lock-only", + "--prefix", + opts.lockfile_metadata_path, + ] + bin_name = "npm" + lockfile_name = "package-lock.json" + extra_nix_override = """ + npmDeps = null; + npmDepsHash = null; + """ -def generate_npm_lock(opts: Options, filename: str) -> None: @contextmanager def disable_copystat(): _orig = shutil.copystat @@ -350,16 +304,15 @@ def disable_copystat(): finally: shutil.copystat = _orig - getSrcAndNpm = textwrap.dedent( + getSrcAndBin = textwrap.dedent( f""" {get_package(opts)}.overrideAttrs (old: {{ - npmDeps = null; - npmDepsHash = null; + {extra_nix_override} postUnpack = '' cp -pr --reflink=auto -- $sourceRoot $out mkdir -p "$out/nix-support" - command -v npm > $out/nix-support/npm-bin || {{ - echo "no npm executable found in native build inputs" >&2 + command -v {bin_name} > $out/nix-support/{bin_name}-bin || {{ + echo "no {bin_name} executable found in native build inputs" >&2 exit 1 }} exit @@ -379,7 +332,7 @@ def disable_copystat(): "--impure", "--print-out-paths", "--expr", - getSrcAndNpm, + getSrcAndBin, ] + opts.extra_flags, ) @@ -389,30 +342,23 @@ def disable_copystat(): with disable_copystat(): shutil.copytree(src, tempdir, dirs_exist_ok=True, copy_function=shutil.copy) - npm_bin = (src / "nix-support" / "npm-bin").read_text().rstrip("\n") + bin_path = (src / "nix-support" / f"{bin_name}-bin").read_text().rstrip("\n") run( - [ - npm_bin, - "install", - "--package-lock-only", - "--prefix", - opts.lockfile_metadata_path, - ], + [bin_path] + cmd, cwd=tempdir, ) if ( lockfile_in_subdir := Path(tempdir) / opts.lockfile_metadata_path - / "package-lock.json" + / lockfile_name ).exists(): - # if package.json is outside a workspace, package0lock.json is generated in the same directory as package.json lockfile = lockfile_in_subdir else: - lockfile = Path(tempdir) / "package-lock.json" + lockfile = Path(tempdir) / lockfile_name - shutil.copy(lockfile, Path(filename).parent / "package-lock.json") + shutil.copy(lockfile, Path(filename).parent / lockfile_name) def update_composer_deps_hash(opts: Options, filename: str, current_hash: str) -> None: @@ -585,7 +531,7 @@ def update(opts: Options) -> Package: if package.npm_deps: if opts.generate_lockfile: - generate_npm_lock(opts, package.filename) + generate_lockfile(opts, package.filename, "npm") update_npm_deps_hash(opts, package.filename, package.npm_deps) if package.pnpm_deps: @@ -607,7 +553,7 @@ def update(opts: Options) -> Package: package.cargo_lock, CargoLockInStore ): if opts.generate_lockfile: - generate_cargo_lock(opts, package.filename) + generate_lockfile(opts, package.filename, "cargo") else: update_cargo_lock(opts, package.filename, package.cargo_lock) From 403966c9455d0cf0a81f3f371af034bb267ddb91 Mon Sep 17 00:00:00 2001 From: Moraxyc Date: Mon, 16 Dec 2024 20:57:30 +0800 Subject: [PATCH 3/3] test: add a test for npm lock generation --- tests/test_npm_lock_generate.py | 47 +++++++++++++++++++ tests/testpkgs/default.nix | 1 + tests/testpkgs/npm-lock-generate/default.nix | 23 +++++++++ .../npm-lock-generate/package-lock.json | 0 4 files changed, 71 insertions(+) create mode 100644 tests/test_npm_lock_generate.py create mode 100644 tests/testpkgs/npm-lock-generate/default.nix create mode 100644 tests/testpkgs/npm-lock-generate/package-lock.json diff --git a/tests/test_npm_lock_generate.py b/tests/test_npm_lock_generate.py new file mode 100644 index 0000000..544d00c --- /dev/null +++ b/tests/test_npm_lock_generate.py @@ -0,0 +1,47 @@ +import subprocess + +import conftest + +from nix_update import main + + +def test_update(helpers: conftest.Helpers) -> None: + with helpers.testpkgs(init_git=True) as path: + main( + [ + "--file", + str(path), + "--commit", + "npm-lock-generate", + "--version", + "v2.6.0", + "--generate-lockfile", + ] + ) + npm_deps_name = subprocess.run( + [ + "nix", + "eval", + "--raw", + "--extra-experimental-features", + "nix-command", + "-f", + path, + "npm-lock-generate.npmDeps.name", + ], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + diff = subprocess.run( + ["git", "-C", path, "show"], + text=True, + stdout=subprocess.PIPE, + check=True, + ).stdout.strip() + print(diff) + assert "2.6.0" in npm_deps_name + assert ( + "https://github.com/olrtg/emmet-language-server/compare/v2.5.0...v2.6.0" + in diff + ) diff --git a/tests/testpkgs/default.nix b/tests/testpkgs/default.nix index 96dd0e5..a1cf273 100644 --- a/tests/testpkgs/default.nix +++ b/tests/testpkgs/default.nix @@ -27,6 +27,7 @@ savanna = pkgs.python3.pkgs.callPackage ./savanna.nix { }; npm = pkgs.callPackage ./npm.nix { }; npm-package = pkgs.callPackage ./npm-package.nix { }; + npm-lock-generate = pkgs.callPackage ./npm-lock-generate { }; pnpm = pkgs.callPackage ./pnpm.nix { }; maven = pkgs.callPackage ./maven.nix { }; mix = pkgs.callPackage ./mix.nix { }; diff --git a/tests/testpkgs/npm-lock-generate/default.nix b/tests/testpkgs/npm-lock-generate/default.nix new file mode 100644 index 0000000..551def1 --- /dev/null +++ b/tests/testpkgs/npm-lock-generate/default.nix @@ -0,0 +1,23 @@ +{ + buildNpmPackage, + fetchFromGitHub, +}: + +buildNpmPackage rec { + pname = "emmet-language-server"; + version = "2.5.0"; + + src = fetchFromGitHub { + owner = "olrtg"; + repo = "emmet-language-server"; + rev = "v${version}"; + hash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }; + + npmDepsHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + + postPatch = '' + cp ${./package-lock.json} ./package-lock.json + ''; + +} diff --git a/tests/testpkgs/npm-lock-generate/package-lock.json b/tests/testpkgs/npm-lock-generate/package-lock.json new file mode 100644 index 0000000..e69de29