From 3d7a06286f65c1e0deedeb54f505d4e5a4f698f5 Mon Sep 17 00:00:00 2001 From: mahiki Date: Tue, 23 Jul 2024 22:56:03 +0200 Subject: [PATCH 1/4] testing formual locally, with poetry install --- Formula/desertislandutils.rb | 6 +- notes/create-a-homebrew-tap.md | 9 +- notes/homebrew-dev.log.md | 196 ++++++++++++++++++++++++++++++++- 3 files changed, 207 insertions(+), 4 deletions(-) diff --git a/Formula/desertislandutils.rb b/Formula/desertislandutils.rb index 470481e..67acc91 100644 --- a/Formula/desertislandutils.rb +++ b/Formula/desertislandutils.rb @@ -8,9 +8,13 @@ class Desertislandutils < Formula license "MIT" depends_on "python@3.11" + depends_on "poetry" def install - virtualenv_install_with_resources + venv = virtualenv_create(libexec, "python3") + + system "poetry", "install", "--no-root" + end test do diff --git a/notes/create-a-homebrew-tap.md b/notes/create-a-homebrew-tap.md index a96303f..223e660 100644 --- a/notes/create-a-homebrew-tap.md +++ b/notes/create-a-homebrew-tap.md @@ -1,6 +1,7 @@ # Homebrew Tap: mahiki - [Homebrew Tap: mahiki](#homebrew-tap-mahiki) + - [RSRC](#rsrc) - [IOYCSWISWYE: TOPLINES](#ioycswiswye-toplines) - [UNICORN DREAM: BUILT URL IS `/downloads/` LINK](#unicorn-dream-built-url-is-downloads-link) - [UNICORN DREAM: UPDATE BREW URL](#unicorn-dream-update-brew-url) @@ -14,6 +15,10 @@ - [BLUSH-RESPONSE: ok create homebrew thingy now](#blush-response-ok-create-homebrew-thingy-now) - [EMPATHY](#empathy) +## RSRC +[Homebrew docs: Python Formula](https://docs.brew.sh/Python-for-Formula-Authors) +[Homebrew docs: Why not install with pip inside homebrew](https://docs.brew.sh/Acceptable-Formulae#we-dont-like-install-scripts-that-download-unversioned-things) + ## IOYCSWISWYE: TOPLINES NODO: trigger brew build on upstream updates at pypi @@ -38,7 +43,7 @@ First lets try updating the formula with the new URL. NOTE: the SHA of the tar.gz file locally is different than the github release tar.gz: ```sh brew edit mahiki/tap/desertislandutils -# opens the desertislandtils.rb file +# opens the desertislandutils.rb file # github tar file created via Release UI from `main v0.1.0` curl -Ls https://github.com/mahiki/desertislandutils/archive/refs/tags/v0.1.0.tar.gz | shasum -a 256 @@ -110,7 +115,7 @@ brew create --tap=mahiki/tap * [ASCIINEMA has pyproject definition and pypi hosting.][asci] * Maybe just make a pypi package and try the homebrew defaults -[asciinema brew formula simple](https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/asciinema.rb) +[asciinema brew formula simple](https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula`/asciinema.rb) ```rb url "https://files.pythonhosted.org/packages/2c/31/492da48c9d7d23cd26f16c8f459aeb443ff056258bed592b5ba28ed271ea/asciinema-2.1.0.tar.gz" diff --git a/notes/homebrew-dev.log.md b/notes/homebrew-dev.log.md index deb852f..751737b 100644 --- a/notes/homebrew-dev.log.md +++ b/notes/homebrew-dev.log.md @@ -1,5 +1,142 @@ # Homebrew Dev Log +## 2024-07-23: HOMEBREW INSTALL FROM FILE +```sh +cd ./the-others/homebrew-tap +HOMEBREW_NO_INSTALL_FROM_API=1 brew install --build-from-source --verbose Formula/desertislandutils.rb + +# it pulls down a whole lotta objects. +``` +It works! + +## 2024-07-23: POETRY INSTALL IN HOMEBREW +Chatgpt narrows down a potentially workable idea. +The crux is that when poetry install is run inside a virtual environment, it doesn't create a new random path. + +```sh +cd "$(mktemp -d)" +python3 -m venv libexec +source libexec/bin/activate +tar -xf desertislandutils-0.3.10.tar.gz +cd desertislandutils-0.3.10.tar.gz +poetry install +poetry env info + # /private/var/folders/x9/d983fsw11ls6y88p7f2_2zrc0000gn/T/tmp.rrm9NYOBXR/libexec + +l /private/var/folders/x9/d983fsw11ls6y88p7f2_2zrc0000gn/T/tmp.rrm9NYOBXR/libexec/bin/ + # -rwxr-xr-x 1 186 2024-07-23 20:49 wn + # -rwxr-xr-x 1 192 2024-07-23 20:49 too + +bat --plain /private/var/folders/x9/d983fsw11ls6y88p7f2_2zrc0000gn/T/tmp.rrm9NYOBXR/libexec/bin/wn + #!/private/var/folders/x9/d983fsw11ls6y88p7f2_2zrc0000gn/T/tmp.rrm9NYOBXR/libexec/bin/python + import sys + from src.weeknumber.wn import app + + if __name__ == '__main__': + sys.exit(app()) + +# Destroy the virtual environment +deactivate +rm -rf venv +``` +GREAT! The install is within the existing virtual environment. No random path like: + + ../Caches/pypoetry/virtualenvs/templisher-4hu7XBP5-py3.11 + +**Problem:** maybe homebrew doesn't activate the virtual enviro. Then what? + +```sh +cd "$(mktemp -d)" +python3 -m venv libexec +curl -LO https://github.com/mahiki/desertislandutils/releases/download/v0.3.10/desertislandutils-0.3.10.tar.gz +# untar and cd +export VIRTUAL_ENV=../libexec +cd desertislandutils-0.3.10 # I hope homebrew is smart enough to do this +poetry install +l ../libexec/bin + # -rwxr-xr-x 1 117 2024-07-23 22:03 wn + # -rwxr-xr-x 1 123 2024-07-23 22:03 too + # lrwxr-xr-x 1 10 2024-07-23 21:30 python -> python3.11 + +poetry env info --executable + # ../libexec/bin/python + +bat --plain ../libexec/bin/wn -l py + #!../libexec/bin/python + import sys + from src.weeknumber.wn import app + + if __name__ == '__main__': + sys.exit(app()) +``` +WOW. So if homebrew handles with absolute paths it should be no problem. +Obv the relative assignment I did there wasnt good. + + +## 2024-07-23: CAPITULATE - SPECIFY ALL DEPENDENCIES +```sh +brew update-python-resources --print-only Formula/desertislandutils.rb +# long list of packages, they looks like a subset of the lock file. +``` + +I guesss that plus the way it can rewrite the formula file opens up a good option +for triggering a GHA job that pulls the repo, updates the formula, build and test, merge to main. + +* And the links will work correctly +* And the uninstall process will work, where maybe not with poetry + +### How to use homebrew-pypi-poet +```sh +# Use a temporary directory for the virtual environment +cd "$(mktemp -d)" + +# Create and source a new virtual environment in the venv/ directory +python3 -m venv venv +source venv/bin/activate + +# Install the package of interest as well as homebrew-pypi-poet +pip install some_package homebrew-pypi-poet + # I did: + # pip install https://github.com/mahiki/desertislandutils/releases/download/v0.3.10/desertislandutils-0.3.10.tar.gz homebrew-pypi-poet + # Building wheel for desertislandutils (pyproject.toml) ... done + # Created wheel for desertislandutils: filename=desertislandutils-0.3.10-py3-none-any.whl size=5531 sha256=ee2a20015a6329e3a298ffd7a562e519161599331c0424c99c4c77afa3af68c3 + # Stored in directory: /Users/hans/Library/Caches/pip/wheels/d0/56/ee/2b008b2c75cf901cf5894ae81a4fee4f7b105cf658cd653555 + # Successfully built desertislandutils + +poet desertislandutils +# pretty much the same as update-python-resources +# with also: + # resource "desertislandutils" do + # url "https://files.pythonhosted.org/packages/62/0f/db9abf3d5d7513b50f618d634cf666278cd6deb0e73f5880bfcc838b5c59/desertislandutils-0.1.0.tar.gz" + # sha256 "573c103661d99ff73a3f9749f5c3343f2e8255e36a66928a7192aaabecd056ef" + # end + +# Destroy the virtual environment +deactivate +rm -rf venv +``` + +## 2024-07-23: DEEPER AND DEEPER - POETRY IN HOMEBREW +I asked chat GPT if i can just do poetry install inside the homebrew installation. I suspect the fine points are the symlinks and the script with special shebang that homebrew creates as a local executable. + +Brew docs say dont use pip or setup.py to install dependencies, which are unversioned ie from master branch. +But look at `poetry.lock`: these are versioned and checksummed in poetry: + +```toml +[[package]] +name = "argparse" +version = "1.4.0" +description = "Python command-line parsing library" +optional = false +python-versions = "*" +files = [ + {file = "argparse-1.4.0-py2.py3-none-any.whl", hash = "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314"}, + {file = "argparse-1.4.0.tar.gz", hash = "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4"}, +] +``` +Doesnt have the URL, but see PyPi: https://pypi.org/project/argparse/1.4.0/#files +Hashes are there, tar and .whl just like GH releases. + ## 2024-07-23: MINIMAL FORMULA I just want to generate teh formula @@ -25,8 +162,65 @@ brew create --no-fetch --tap "mahiki/homebrew-tap" $url # Receiving objects: 1% (20682/2068170), 12.13 MiB | 2.12 MiB/s ``` Well this sucks a##. +This gets me nowhere. The example formula comes out as Ruby install commands. ### ChatGPT says try `brew-pypi-poet` ```sh brew install brew-pypi-poet -poet mypackage > MyPackage.rb \ No newline at end of file +poet mypackage > MyPackage.rb +``` +There is no such brew package. U can install with pip. I don't want all that entails. + +### Just change the file, merge to mahiki/homebrew-tap:main +```sh +brew update +brew uninstall desertislandutils +brew info desertislandutils + # ==> mahiki/tap/desertislandutils: stable 0.3.10 + # Be here, thy collection of personal convenience utilities + # https://github.com/mahiki/homebrew-tap + # Not installed +brew install desertislandutils +``` + +### FAILURE: just not working go back to the tarball with explicit resource dependencies. + +## 2024-07-22: THE EXECUTABLE +I found in the 0.3.9 deployment that the src/weeknumber module wasn't added to pyproject.toml. +Investigating uncovered the format of the executable script that homebrew makes: + +```sh +# FILE: /opt/homebrew/bin/wn + +#!/opt/homebrew/Cellar/desertislandutils/0.3.9/libexec/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from src.weeknumber.wn import app +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(app()) +``` + +And Poetry creates an executable script as well in the virtual environment: + +```sh +# bin/wn: +bat --style plain /Users/hans/Library/Caches/pypoetry/virtualenvs/desertislandutils-zyraM7-y-py3.12/bin/wn +#!/Users/hans/Library/Caches/pypoetry/virtualenvs/desertislandutils-zyraM7-y-py3.12/bin/python +import sys +from src.weeknumber.wn import app + +if __name__ == '__main__': + sys.exit(app()) +``` + +# bin/too: +bat --style plain /Users/hans/Library/Caches/pypoetry/virtualenvs/desertislandutils-zyraM7-y-py3.12/bin/too +#!/Users/hans/Library/Caches/pypoetry/virtualenvs/desertislandutils-zyraM7-y-py3.12/bin/python +import sys +from src.toobigdatadoc.too import main + +if __name__ == '__main__': + sys.exit(main()) +``` \ No newline at end of file From 761c3750c62717d677cf7623ae692feae80cf289 Mon Sep 17 00:00:00 2001 From: mahiki Date: Tue, 23 Jul 2024 23:35:14 +0200 Subject: [PATCH 2/4] poetry install formula WORKS --- Formula/desertislandutils.rb | 10 +++++++++- notes/homebrew-dev.log.md | 27 ++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/Formula/desertislandutils.rb b/Formula/desertislandutils.rb index 67acc91..d203691 100644 --- a/Formula/desertislandutils.rb +++ b/Formula/desertislandutils.rb @@ -13,8 +13,15 @@ class Desertislandutils < Formula def install venv = virtualenv_create(libexec, "python3") + ENV["VIRTUAL_ENV"] = libexec + + puts "libexec: #{libexec}" + # this comes as: libexec: /opt/homebrew/Cellar/desertislandutils/0.3.10/libexec + system "poetry", "install", "--no-root" + venv.pip_install_and_link buildpath + end test do @@ -28,7 +35,8 @@ def install # # The installed folder is not in the path, so use the entire path to any # executables being tested: `system "#{bin}/program", "do", "something"`. - system "false" + system "#{bin}/wn", "--help" + # TODO: basic test of built function # system "#{bin}/wn", "--help" # system "#{bin}/too", "--help" diff --git a/notes/homebrew-dev.log.md b/notes/homebrew-dev.log.md index 751737b..d9738d7 100644 --- a/notes/homebrew-dev.log.md +++ b/notes/homebrew-dev.log.md @@ -1,13 +1,38 @@ # Homebrew Dev Log ## 2024-07-23: HOMEBREW INSTALL FROM FILE + +### First Try: wn installed and linked, but dependencies installed to poetry venv ```sh cd ./the-others/homebrew-tap HOMEBREW_NO_INSTALL_FROM_API=1 brew install --build-from-source --verbose Formula/desertislandutils.rb # it pulls down a whole lotta objects. + +==> Finishing up +ln -s ../Cellar/desertislandutils/0.3.10/bin/too too +ln -s ../Cellar/desertislandutils/0.3.10/bin/wn wn +==> Summary +🍺 /opt/homebrew/Cellar/desertislandutils/0.3.10: 22 files, 35.6KB, built in 9 seconds + +# Install works! symlinks created. + +which wn +/Users/hans/bin/utility/wn +/opt/homebrew/bin/wn + +/opt/homebrew/bin/wn --help + # Traceback (most recent call last): + # File "/opt/homebrew/bin/wn", line 5, in + # from src.weeknumber.wn import app + # File "/opt/homebrew/Cellar/desertislandutils/0.3.10/libexec/lib/python3.12/site-packages/src/weeknumber/wn.py", line 9, in + # import pendulum + # ModuleNotFoundError: No module named 'pendulum' ``` -It works! +Module not found, because it installed in the usual poetry env: + + ==> poetry install --no-root + Creating virtualenv desertislandutils-wMi-OlyN-py3.12 in /private/tmp/desertislandutils-20240723-43941-or2nph/desertislandutils-0.3.10/.brew_home/Library/Caches/pypoetry/virtualenvs ## 2024-07-23: POETRY INSTALL IN HOMEBREW Chatgpt narrows down a potentially workable idea. From 4497a6759d369ac166bdfb1dbbff086dd014a944 Mon Sep 17 00:00:00 2001 From: mahiki Date: Tue, 23 Jul 2024 23:59:22 +0200 Subject: [PATCH 3/4] finalized formula using poetry install --- .../examples/desertislandutils.poetry-poc.rb | 38 +++++++++++++++++++ notes/homebrew-dev.log.md | 9 +++++ 2 files changed, 47 insertions(+) create mode 100644 notes/examples/desertislandutils.poetry-poc.rb diff --git a/notes/examples/desertislandutils.poetry-poc.rb b/notes/examples/desertislandutils.poetry-poc.rb new file mode 100644 index 0000000..a07b108 --- /dev/null +++ b/notes/examples/desertislandutils.poetry-poc.rb @@ -0,0 +1,38 @@ +# Demonstrates using poetry install to bring dependencies without +# specifying every single resource URL and SHA like usual, which sucks. +# author: mahiki with help from chatgpt4, a lot of back and forth. + +class YourPackage < Formula + include Language::Python::Virtualenv + + desc "Description of your package" + homepage "https://your.package.homepage" + url "https://your.package.url/your_package-0.3.9.tar.gz" + sha256 "SHA256_CHECKSUM_OF_YOUR_PACKAGE" + + depends_on "python@3.9" + depends_on "poetry" + + def install + # Create a virtual environment and folder + venv = virtualenv_create(libexec, "python3") + + # Set the VIRTUAL_ENV, poetry uses this instead of generating a random path + # libexec is a variable, the full path. + # /opt/homebrew/Cellar/desertislandutils/0.3.10/libexec + + ENV["VIRTUAL_ENV"] = libexec + + # Install the package dependencies using poetry, not the package source + system "poetry", "install", "--no-root" + + # Install the package itself using pip and create necessary symlinks + venv.pip_install_and_link buildpath + end + + test do + # Test the installation + system "#{bin}/your_executable", "--version" + end + end + \ No newline at end of file diff --git a/notes/homebrew-dev.log.md b/notes/homebrew-dev.log.md index d9738d7..5c70c6d 100644 --- a/notes/homebrew-dev.log.md +++ b/notes/homebrew-dev.log.md @@ -1,6 +1,15 @@ # Homebrew Dev Log ## 2024-07-23: HOMEBREW INSTALL FROM FILE +### Totally Works! +```ruby + def install + venv = virtualenv_create(libexec, "python3") + ENV["VIRTUAL_ENV"] = libexec + system "poetry", "install", "--no-root" + venv.pip_install_and_link buildpath + end +``` ### First Try: wn installed and linked, but dependencies installed to poetry venv ```sh From a8d10fcad132a86584f0b1a1ad4ba9edb65176ba Mon Sep 17 00:00:00 2001 From: mahiki Date: Wed, 24 Jul 2024 00:03:24 +0200 Subject: [PATCH 4/4] formula finally good on v0.3.9 --- Formula/desertislandutils.rb | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/Formula/desertislandutils.rb b/Formula/desertislandutils.rb index d203691..0486465 100644 --- a/Formula/desertislandutils.rb +++ b/Formula/desertislandutils.rb @@ -12,33 +12,13 @@ class Desertislandutils < Formula def install venv = virtualenv_create(libexec, "python3") - ENV["VIRTUAL_ENV"] = libexec - - puts "libexec: #{libexec}" - # this comes as: libexec: /opt/homebrew/Cellar/desertislandutils/0.3.10/libexec - system "poetry", "install", "--no-root" - venv.pip_install_and_link buildpath - end test do - # TODO: you should like test the brew install and function test of your cli things - # `test do` will create, run in and delete a temporary directory. - # - # This test will fail and we won't accept that! For Homebrew/homebrew-core - # this will need to be a test that verifies the functionality of the - # software. Run the test with `brew test desertislandutils`. Options passed - # to `brew install` such as `--HEAD` also need to be provided to `brew test`. - # - # The installed folder is not in the path, so use the entire path to any - # executables being tested: `system "#{bin}/program", "do", "something"`. system "#{bin}/wn", "--help" - - # TODO: basic test of built function - # system "#{bin}/wn", "--help" - # system "#{bin}/too", "--help" + system "#{bin}/too", "--help" end end