From 8d7e43b4b1316c6b3b6b39ecacaae4969b2b9457 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 20 Oct 2024 14:24:33 -0700 Subject: [PATCH 01/51] Renamed test_complete to test_end_to_end. The name 'test_complete' confused me that we are waiting for some completion of the test. No semantic change. --- test/{test_complete.py => test_end_to_end.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename test/{test_complete.py => test_end_to_end.py} (97%) diff --git a/test/test_complete.py b/test/test_end_to_end.py similarity index 97% rename from test/test_complete.py rename to test/test_end_to_end.py index c4625d03..5e69e0fb 100644 --- a/test/test_complete.py +++ b/test/test_end_to_end.py @@ -41,7 +41,7 @@ def validate_dir_leds(folder=""): assert leds_dir.is_dir() and nfiles > 0 -def test_complete(clirunner, validate_cliresult, configenv, offline): +def test_end_to_end1(clirunner, validate_cliresult, configenv, offline): """Test the installation of the examples package""" # -- If the option 'offline' is passed, the test is skip @@ -99,7 +99,7 @@ def test_complete(clirunner, validate_cliresult, configenv, offline): assert "examples" in result.output -def test_complete2(clirunner, validate_cliresult, configenv, offline): +def test_end_to_end2(clirunner, validate_cliresult, configenv, offline): """Test more 'apio examples' commands""" # -- If the option 'offline' is passed, the test is skip @@ -172,7 +172,7 @@ def test_complete2(clirunner, validate_cliresult, configenv, offline): validate_dir_leds() -def test_complete3(clirunner, validate_cliresult, configenv, offline): +def test_end_to_end3(clirunner, validate_cliresult, configenv, offline): """Test more 'apio examples' commands""" # -- If the option 'offline' is passed, the test is skip From 691b7bc821ea01d083a32c6cec519af8c1ce7a42 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 20 Oct 2024 21:42:14 -0700 Subject: [PATCH 02/51] Improved the scons dependency of the command 'apio report'. Instead of a fake builder for hardware.pnr, using now a scons emitter that declares that the nextpnr command generates also the file 'hardware.pnr'. The command 'apio report' now reconstructs correctly the file 'hardware.pnr' if deleted (while preserving the main nextpnr artifact file). --- apio/scons/ecp5/SConstruct | 34 +++++++++++++++++++--------------- apio/scons/gowin/SConstruct | 32 ++++++++++++++++++-------------- apio/scons/ice40/SConstruct | 34 +++++++++++++++++++--------------- 3 files changed, 56 insertions(+), 44 deletions(-) diff --git a/apio/scons/ecp5/SConstruct b/apio/scons/ecp5/SConstruct index 7c3218de..d8e1aceb 100644 --- a/apio/scons/ecp5/SConstruct +++ b/apio/scons/ecp5/SConstruct @@ -87,6 +87,7 @@ VERILATOR_NO_STYLE = arg_bool(env, "nostyle", False) NOWARNS = arg_str(env, "nowarn", "").split(",") WARNS = arg_str(env, "warn", "").split(",") + # -- Resources paths IVL_PATH = os.environ["IVL"] if "IVL" in os.environ else "" TRELLIS_PATH = os.environ["TRELLIS"] if "TRELLIS" in os.environ else "" @@ -120,6 +121,19 @@ synth_builder = Builder( ) env.Append(BUILDERS={"Synth": synth_builder}) +# -- The name of the report file generated by nextpnr. +PNR_REPORT_FILE: str = TARGET + ".pnr" + + +# -- Apio report. +# -- emmiter (nextpnr, Place and route). +# -- hardware.json -> hardware.pnr. +def pnr_builder_emitter(target, source, env): + """A scons emmiter function for the pnr builder. It declares that the + nextpnr builder creates also a second file called 'hardware.pnr'.""" + target.append(PNR_REPORT_FILE) + return target, source + # -- Apio build/upload/time/report. # -- builder (nextpnr, Place and route). @@ -127,29 +141,20 @@ env.Append(BUILDERS={"Synth": synth_builder}) pnr_builder = Builder( action=( "nextpnr-ecp5 --{0} --package {1} --json $SOURCE --textcfg $TARGET " - "--report {2}.pnr --lpf {3} {4} --timing-allow-fail --force" + "--report {2} --lpf {3} {4} --timing-allow-fail --force" ).format( "25k" if (FPGA_TYPE == "12k") else FPGA_TYPE, FPGA_PACK, - TARGET, + PNR_REPORT_FILE, LPF, "" if VERBOSE_ALL or VERBOSE_PNR else "-q", ), suffix=".config", src_suffix=".json", + emitter = pnr_builder_emitter, ) env.Append(BUILDERS={"PnR": pnr_builder}) -# -- Apio report. -# -- builder (fake, the .pnr file is generated by pnr_builder) -# -- hardware.config -> hardware.pnr. -pnr_builder = Builder( - action="", - suffix=".pnr", - src_suffix=".config", -) -env.Append(BUILDERS={"PnrReport": pnr_builder}) - # -- Apio build/upload. # -- Builder (icepack, bitstream generator). @@ -183,11 +188,10 @@ if VERBOSE_ALL: # -- Apio report. # -- Targets. # -- hardware.config -> hardware.pnr -> (report) -pnr_report_target = env.PnrReport(TARGET, pnr_target) report_action = get_report_action( env, SConstructId.SCONSTRUCT_ECP5, VERBOSE_PNR ) -report_target = env.Alias("report", pnr_report_target, report_action) +report_target = env.Alias("report", PNR_REPORT_FILE, report_action) AlwaysBuild(report_target) @@ -409,7 +413,7 @@ if GetOption("clean"): env.Default( [ - pnr_report_target, + report_target, build_target, synth_target, pnr_target, diff --git a/apio/scons/gowin/SConstruct b/apio/scons/gowin/SConstruct index b4409774..88382155 100644 --- a/apio/scons/gowin/SConstruct +++ b/apio/scons/gowin/SConstruct @@ -118,6 +118,19 @@ synth_builder = Builder( ) env.Append(BUILDERS={"Synth": synth_builder}) +# -- The name of the report file generated by nextpnr. +PNR_REPORT_FILE: str = TARGET + ".pnr" + + +# -- Apio report. +# -- emmiter (nextpnr, Place and route). +# -- hardware.json -> hardware.pnr. +def pnr_builder_emitter(target, source, env): + """A scons emmiter function for the pnr builder. It declares that the + nextpnr builder creates also a second file called 'hardware.pnr'.""" + target.append(PNR_REPORT_FILE) + return target, source + # -- Apio build/upload/time/report. # -- builder (nextpnr, Place and route). @@ -125,28 +138,20 @@ env.Append(BUILDERS={"Synth": synth_builder}) pnr_builder = Builder( action=( "nextpnr-himbaechel --device {0} --json $SOURCE --write $TARGET " - "--vopt family={1} --vopt cst={2} {3}" + "--report {1} --vopt family={2} --vopt cst={3} {4}" ).format( FPGA_MODEL, + PNR_REPORT_FILE, FPGA_TYPE, CST, "" if VERBOSE_ALL or VERBOSE_PNR else "-q", ), suffix=".pnr.json", src_suffix=".json", + emitter=pnr_builder_emitter, ) env.Append(BUILDERS={"PnR": pnr_builder}) -# -- Apio report. -# -- builder (fake, the .pnr file is generated by pnr_builder) -# -- hardware..pnr.json -> hardware.pnr. -pnr_builder = Builder( - action="", - suffix=".pnr.json", - src_suffix=".asc", -) -env.Append(BUILDERS={"PnrReport": pnr_builder}) - # -- Apio build/upload. # -- Builder (icepack, bitstream generator). @@ -177,11 +182,10 @@ if VERBOSE_ALL: # -- Apio report. # -- Targets. # -- hardware..pnr.json -> hardware.pnr -> (report) -pnr_report_target = env.PnrReport(TARGET, pnr_target) report_action = get_report_action( env, SConstructId.SCONSTRUCT_GOWIN, VERBOSE_PNR ) -report_target = env.Alias("report", pnr_report_target, report_action) +report_target = env.Alias("report", PNR_REPORT_FILE, report_action) AlwaysBuild(report_target) @@ -400,7 +404,7 @@ if GetOption("clean"): env.Default( [ - pnr_report_target, + report_target, build_target, verify_out_target, graph_target, diff --git a/apio/scons/ice40/SConstruct b/apio/scons/ice40/SConstruct index fd73e53e..f8bed764 100644 --- a/apio/scons/ice40/SConstruct +++ b/apio/scons/ice40/SConstruct @@ -121,6 +121,19 @@ synth_builder = Builder( ) env.Append(BUILDERS={"Synth": synth_builder}) +# -- The name of the report file generated by nextpnr. +PNR_REPORT_FILE: str = TARGET + ".pnr" + + +# -- Apio report. +# -- emmiter (nextpnr, Place and route). +# -- hardware.json -> hardware.pnr. +def pnr_builder_emitter(target, source, env): + """A scons emmiter function for the pnr builder. It declares that the + nextpnr builder creates also a second file called 'hardware.pnr'.""" + target.append(PNR_REPORT_FILE) + return target, source + # -- Apio build/upload/time/report. # -- builder (nextpnr, Place and route). @@ -128,30 +141,21 @@ env.Append(BUILDERS={"Synth": synth_builder}) pnr_builder = Builder( action=( "nextpnr-ice40 --{0}{1} --package {2} --json $SOURCE --asc $TARGET " - "--report {3}.pnr --pcf {4} {5}" + "--report {3} --pcf {4} {5}" ).format( FPGA_TYPE, FPGA_SIZE, FPGA_PACK, - TARGET, + PNR_REPORT_FILE, PCF, "" if VERBOSE_ALL or VERBOSE_PNR else "-q", ), suffix=".asc", src_suffix=".json", + emitter=pnr_builder_emitter, ) env.Append(BUILDERS={"PnR": pnr_builder}) -# -- Apio report. -# -- builder (fake, the .pnr file is generated by pnr_builder) -# -- hardware.asc -> hardware.pnr. -pnr_builder = Builder( - action="", - suffix=".pnr", - src_suffix=".asc", -) -env.Append(BUILDERS={"PnrReport": pnr_builder}) - # -- Apio build/upload. # -- Builder (icepack, bitstream generator). @@ -193,11 +197,11 @@ if VERBOSE_ALL: # -- Apio report. # -- Targets. # -- hardware.asc -> hardware.pnr -> (report) -pnr_report_target = env.PnrReport(TARGET, pnr_target) +# pnr_report_target = env.PnrReport(TARGET, pnr_target) report_action = get_report_action( env, SConstructId.SCONSTRUCT_ICE40, VERBOSE_PNR ) -report_target = env.Alias("report", pnr_report_target, report_action) +report_target = env.Alias("report", PNR_REPORT_FILE, report_action) AlwaysBuild(report_target) @@ -426,7 +430,7 @@ if GetOption("clean"): env.Default( [ - pnr_report_target, + report_target, time_target, build_target, verify_out_target, From 924ebf2e2f70d5bb0d90b410b07af668ec2fcaa9 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 20 Oct 2024 21:44:34 -0700 Subject: [PATCH 03/51] Fixed a lint error in the example test-examples/ColorLight-5A-75B-V8/Blinky. --- apio/scons/ecp5/SConstruct | 2 +- test-examples/ColorLight-5A-75B-V8/Blinky/Blinky.v | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apio/scons/ecp5/SConstruct b/apio/scons/ecp5/SConstruct index d8e1aceb..00803e0d 100644 --- a/apio/scons/ecp5/SConstruct +++ b/apio/scons/ecp5/SConstruct @@ -151,7 +151,7 @@ pnr_builder = Builder( ), suffix=".config", src_suffix=".json", - emitter = pnr_builder_emitter, + emitter=pnr_builder_emitter, ) env.Append(BUILDERS={"PnR": pnr_builder}) diff --git a/test-examples/ColorLight-5A-75B-V8/Blinky/Blinky.v b/test-examples/ColorLight-5A-75B-V8/Blinky/Blinky.v index f662a47e..39616f08 100644 --- a/test-examples/ColorLight-5A-75B-V8/Blinky/Blinky.v +++ b/test-examples/ColorLight-5A-75B-V8/Blinky/Blinky.v @@ -4,7 +4,7 @@ module Test ( input CLK, // 25MHz clock - output led, // LED to blink + output led // LED to blink ); reg [23:0] counter = 0; From 364efeaeaef7fedb64de5f2efcff9567f050d31b Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 20 Oct 2024 21:54:52 -0700 Subject: [PATCH 04/51] Suppressed the Verilator version messages on each 'apio lint'. --- apio/scons/scons_util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index a05ee562..9c4d3baf 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -525,8 +525,9 @@ def make_verilator_action( """ action = ( - "verilator --lint-only --bbox-unsup --timing -Wno-TIMESCALEMOD " - "-Wno-MULTITOP {0} {1} {2} {3} {4} {5} {6} {7} {8} $SOURCES" + "verilator --lint-only --quiet --bbox-unsup --timing " + "-Wno-TIMESCALEMOD -Wno-MULTITOP " + "{0} {1} {2} {3} {4} {5} {6} {7} {8} $SOURCES" ).format( "-Wall" if warnings_all else "", "-Wno-style" if warnings_no_style else "", From fc1a317c325699a493ad9b72ee38576e729738d7 Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 21 Oct 2024 19:40:49 -0700 Subject: [PATCH 05/51] Minore name change. Renamed pnr_builder_emmiter to pnr_emitter. --- apio/scons/ecp5/SConstruct | 4 ++-- apio/scons/gowin/SConstruct | 4 ++-- apio/scons/ice40/SConstruct | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apio/scons/ecp5/SConstruct b/apio/scons/ecp5/SConstruct index 00803e0d..8cb30604 100644 --- a/apio/scons/ecp5/SConstruct +++ b/apio/scons/ecp5/SConstruct @@ -128,7 +128,7 @@ PNR_REPORT_FILE: str = TARGET + ".pnr" # -- Apio report. # -- emmiter (nextpnr, Place and route). # -- hardware.json -> hardware.pnr. -def pnr_builder_emitter(target, source, env): +def pnr_emitter(target, source, env): """A scons emmiter function for the pnr builder. It declares that the nextpnr builder creates also a second file called 'hardware.pnr'.""" target.append(PNR_REPORT_FILE) @@ -151,7 +151,7 @@ pnr_builder = Builder( ), suffix=".config", src_suffix=".json", - emitter=pnr_builder_emitter, + emitter=pnr_emitter, ) env.Append(BUILDERS={"PnR": pnr_builder}) diff --git a/apio/scons/gowin/SConstruct b/apio/scons/gowin/SConstruct index 88382155..c765ad7a 100644 --- a/apio/scons/gowin/SConstruct +++ b/apio/scons/gowin/SConstruct @@ -125,7 +125,7 @@ PNR_REPORT_FILE: str = TARGET + ".pnr" # -- Apio report. # -- emmiter (nextpnr, Place and route). # -- hardware.json -> hardware.pnr. -def pnr_builder_emitter(target, source, env): +def pnr_emitter(target, source, env): """A scons emmiter function for the pnr builder. It declares that the nextpnr builder creates also a second file called 'hardware.pnr'.""" target.append(PNR_REPORT_FILE) @@ -148,7 +148,7 @@ pnr_builder = Builder( ), suffix=".pnr.json", src_suffix=".json", - emitter=pnr_builder_emitter, + emitter=pnr_emitter, ) env.Append(BUILDERS={"PnR": pnr_builder}) diff --git a/apio/scons/ice40/SConstruct b/apio/scons/ice40/SConstruct index f8bed764..97a21cbd 100644 --- a/apio/scons/ice40/SConstruct +++ b/apio/scons/ice40/SConstruct @@ -128,7 +128,7 @@ PNR_REPORT_FILE: str = TARGET + ".pnr" # -- Apio report. # -- emmiter (nextpnr, Place and route). # -- hardware.json -> hardware.pnr. -def pnr_builder_emitter(target, source, env): +def pnr_emitter(target, source, env): """A scons emmiter function for the pnr builder. It declares that the nextpnr builder creates also a second file called 'hardware.pnr'.""" target.append(PNR_REPORT_FILE) @@ -152,7 +152,7 @@ pnr_builder = Builder( ), suffix=".asc", src_suffix=".json", - emitter=pnr_builder_emitter, + emitter=pnr_emitter, ) env.Append(BUILDERS={"PnR": pnr_builder}) From 1864504260cbe484fd262c5170007b982fb22a77 Mon Sep 17 00:00:00 2001 From: Zapta Date: Tue, 22 Oct 2024 16:51:01 -0700 Subject: [PATCH 06/51] Added a workaround for the ""UnicodeEncodeError: 'charmap'" crash on Windows. Experienced in on Windows Pro 10. --- apio/__main__.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/apio/__main__.py b/apio/__main__.py index 1efd5e6a..d6395bba 100644 --- a/apio/__main__.py +++ b/apio/__main__.py @@ -12,6 +12,8 @@ import string import re +import platform +import sys from typing import List from click.core import Context import click @@ -236,7 +238,25 @@ def context_settings(): https://github.com/FPGAwars/apio/wiki/Apio """ - +def pre_click_init(cli_func): + """A decorator that provides an early initialization. The + init() function is called before click or any command starts. + To be invoked as early as possible, this decorator must be + the first decorator of the top level command""" + def init(*args, **kwargs): + """Program initialization. """ + if platform.system() == "Windows": + """ For windows, force the stdout/err to use utf-8 encoding. + This is to avoid the "UnicodeEncodeError: 'charmap'" crash. + See https://tinyurl.com/charmap-bug for details. + """ + sys.stdin.reconfigure(encoding='utf-8') + sys.stdout.reconfigure(encoding='utf-8') + # Invoke the command as usual. + cli_func(*args, **kwargs) + return init + +@pre_click_init @click.command( cls=ApioCLI, help=HELP, From 4c5a4d9afc2f63586af928b5a20d2f9626e00aca Mon Sep 17 00:00:00 2001 From: Zapta Date: Tue, 22 Oct 2024 19:27:03 -0700 Subject: [PATCH 07/51] Removed the ""UnicodeEncodeError: 'charmap'" workaround from previous commit. I upgraded my git/bash to the latest one and the problem disappeared. Since this was not an issue for apio users so far, it's probably not needed. --- apio/__main__.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/apio/__main__.py b/apio/__main__.py index d6395bba..90f71ff3 100644 --- a/apio/__main__.py +++ b/apio/__main__.py @@ -12,8 +12,6 @@ import string import re -import platform -import sys from typing import List from click.core import Context import click @@ -238,25 +236,6 @@ def context_settings(): https://github.com/FPGAwars/apio/wiki/Apio """ -def pre_click_init(cli_func): - """A decorator that provides an early initialization. The - init() function is called before click or any command starts. - To be invoked as early as possible, this decorator must be - the first decorator of the top level command""" - def init(*args, **kwargs): - """Program initialization. """ - if platform.system() == "Windows": - """ For windows, force the stdout/err to use utf-8 encoding. - This is to avoid the "UnicodeEncodeError: 'charmap'" crash. - See https://tinyurl.com/charmap-bug for details. - """ - sys.stdin.reconfigure(encoding='utf-8') - sys.stdout.reconfigure(encoding='utf-8') - # Invoke the command as usual. - cli_func(*args, **kwargs) - return init - -@pre_click_init @click.command( cls=ApioCLI, help=HELP, From 78e42c0bb6b1a8f4460748fb8170870d4389747a Mon Sep 17 00:00:00 2001 From: Zapta Date: Wed, 23 Oct 2024 13:14:13 -0700 Subject: [PATCH 08/51] Changed the configuration to debug SConstruct scripts. Instead of running them as top level process, which was tedious due to their prerequisites such as env variables, they are now debugged by running apio from the command line and attaching the VCS debugger at the begining of the SConstruct script. --- .vscode/launch.json | 24 ++++++------------------ DEVELOPER.md | 7 +++++++ apio/__main__.py | 1 + apio/scons/ecp5/SConstruct | 4 ++++ apio/scons/gowin/SConstruct | 4 ++++ apio/scons/ice40/SConstruct | 5 +++++ apio/scons/scons_util.py | 24 ++++++++++++++++++++++++ 7 files changed, 51 insertions(+), 18 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 5032b83d..afb360dd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -268,26 +268,14 @@ "cwd": "${workspaceFolder}/test-examples/Alhambra-II/02-jumping-LED" }, { - "name": "Scon (direct)", + "name": "Attach remote", "type": "debugpy", - "request": "launch", - "program": "${workspaceFolder}/scons_run.py", - "args": [ - "-Q", - "build", - "fpga_model=iCE40-HX4K-TQ144", - "fpga_arch=ice40", - "fpga_size=4k", - "fpga_type=hx", - "fpga_pack=tq144", - "top_module=main", - "-f", - "${workspaceFolder}/apio/scons/ice40/SConstruct", - "force_colors=True" - ], - "console": "integratedTerminal", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, "justMyCode": false, - "cwd": "${workspaceFolder}/test-examples/Alhambra-II/02-jumping-LED" }, ] } \ No newline at end of file diff --git a/DEVELOPER.md b/DEVELOPER.md index ef669c24..9bfd6674 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -33,9 +33,16 @@ python apio_run.py build --project_dir ~/projects/fpga/repo/hdl The ``apio`` repository contains at its root the file ``.vscode/launch.json`` with debug target for most of the ``apio`` commands. Make sure to open the roo folder of the repository for VSC to recognize the targets file. To select the debug target, click on the debug icon on the left sidebar and this will display above a pull down menu with the available debug target and a start icon. +[NOTE] This method doesn't not work for debugging the SConstruct scripts since they are run as subprocesses of the apio process. For debugging SConstruct scripts see the next section. + The debug target can be viewed here https://github.com/FPGAwars/apio/blob/develop/.vscode/launch.json +## Debugging SConstruct scripts (subprocesses) with Visual Studio Code. + +To debug the scons scripts, which are run as apio subprocesses, we use a different method or remote debugging. Enable the ``wait_for_remote_debugger(env)`` call in the SConstruct script, run apio from the command line, and once it reports that it waits for a debugger, run the VCS ``Attach remote`` debug target to connect to the SConstruct process. + + ## Using the dev repository for apio commands. You can tell pip to youse your apio dev repository for apio commands instead of the standard apio release. This allows quick edit/test cycles where you the modify code in your apio dev repository and immediately test it by running ``apio`` commands in the console.. diff --git a/apio/__main__.py b/apio/__main__.py index 90f71ff3..1efd5e6a 100644 --- a/apio/__main__.py +++ b/apio/__main__.py @@ -236,6 +236,7 @@ def context_settings(): https://github.com/FPGAwars/apio/wiki/Apio """ + @click.command( cls=ApioCLI, help=HELP, diff --git a/apio/scons/ecp5/SConstruct b/apio/scons/ecp5/SConstruct index 8cb30604..53a2d801 100644 --- a/apio/scons/ecp5/SConstruct +++ b/apio/scons/ecp5/SConstruct @@ -72,6 +72,10 @@ from apio.scons.scons_util import ( # -- Create the environment env = create_construction_env(ARGUMENTS) +# -- Uncomment for debugging of the scons subprocess using a remote debugger. +# from apio.scons import scons_util +# scons_util.wait_for_remote_debugger(env) + # -- Get arguments. FPGA_SIZE = arg_str(env, "fpga_size", "") FPGA_TYPE = arg_str(env, "fpga_type", "") diff --git a/apio/scons/gowin/SConstruct b/apio/scons/gowin/SConstruct index c765ad7a..94c6f504 100644 --- a/apio/scons/gowin/SConstruct +++ b/apio/scons/gowin/SConstruct @@ -72,6 +72,10 @@ from apio.scons.scons_util import ( # -- Create the environment env = create_construction_env(ARGUMENTS) +# -- Uncomment for debugging of the scons subprocess using a remote debugger. +# from apio.scons import scons_util +# scons_util.wait_for_remote_debugger(env) + # -- Get arguments. FPGA_MODEL = arg_str(env, "fpga_model", "") FPGA_SIZE = arg_str(env, "fpga_size", "") diff --git a/apio/scons/ice40/SConstruct b/apio/scons/ice40/SConstruct index 97a21cbd..b1c15ff3 100644 --- a/apio/scons/ice40/SConstruct +++ b/apio/scons/ice40/SConstruct @@ -69,9 +69,14 @@ from apio.scons.scons_util import ( get_report_action, ) + # -- Create the environment env = create_construction_env(ARGUMENTS) +# -- Uncomment for debugging of the scons subprocess using a remote debugger. +# from apio.scons import scons_util +# scons_util.wait_for_remote_debugger(env) + # -- Get arguments. FPGA_SIZE = arg_str(env, "fpga_size", "") FPGA_TYPE = arg_str(env, "fpga_type", "") diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index 9c4d3baf..49661eee 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -617,3 +617,27 @@ def print_pnr_report( _print_pnr_report(env, json_txt, script_id, verbose) return env.Action(print_pnr_report, "Formatting pnr report.") + + +def wait_for_remote_debugger(env: SConsEnvironment): + """For developement only. Useful for debugging SConstruct scripts that apio + runs as a subprocesses. Call this from the SCconstruct script, run apio + from a command line, and then connect with the Visual Studio Code debugger + using the launch.json debug target. Can also be used to debug apio itself, + without having to create or modify the Visual Studio Code debug targets + in launch.json""" + + # -- We require this import only when using the debugger. + import debugpy + + # -- 5678 is the default debugger port. + port = 5678 + msg( + env, + f"Waiting for remote debugger on port localhost:{port}.", + fg="magenta", + ) + debugpy.listen(port) + msg(env, "Attach with the Visual Studio Code debugger.") + debugpy.wait_for_client() + msg(env, "Remote debugger is attached.", fg="green") From 66f64a9053072f7c4d7121cbeac3071d4604811e Mon Sep 17 00:00:00 2001 From: Zapta Date: Wed, 23 Oct 2024 18:57:32 -0700 Subject: [PATCH 09/51] Added a comment regarding a click bug. No code change. --- apio/scons/scons_util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index 49661eee..93675629 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -164,6 +164,11 @@ def force_colors(env: SConsEnvironment) -> bool: for example from the scons subprocess to the apio app. To preserve the sconstruct text colors, the apio app passes to the sconstract scripts a flag to force the preservation of colors. + + NOTE: As of Oct 2024, forcing colors from the scons subprocess does not + work on Windows and as result, scons output is colorless. + For more details see the click issue at + https://github.com/pallets/click/issues/2791. """ flag = env["FORCE_COLORS"] assert isinstance(flag, bool) From d762bfc0a34c10f27c259bb48b80c4a353f7a3c9 Mon Sep 17 00:00:00 2001 From: Zapta Date: Tue, 29 Oct 2024 21:53:14 -0700 Subject: [PATCH 10/51] Removed the verilator --quiet flag which the verilator rejects. Not sure how it passes the tests when I added it. This fixes the apio lint command. --- apio/scons/scons_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index 93675629..45f65dd6 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -530,7 +530,7 @@ def make_verilator_action( """ action = ( - "verilator --lint-only --quiet --bbox-unsup --timing " + "verilator --lint-only --bbox-unsup --timing " "-Wno-TIMESCALEMOD -Wno-MULTITOP " "{0} {1} {2} {3} {4} {5} {6} {7} {8} $SOURCES" ).format( From c979b38949cbfc34d10d1726ac43b9893372db21 Mon Sep 17 00:00:00 2001 From: Zapta Date: Wed, 30 Oct 2024 09:48:19 -0700 Subject: [PATCH 11/51] First installement of cleaning up the packages logic. 1. Move the package required util functions from util.py to a new pkg_util.py file. 2. Now having env setting function for each package. 3. Package checking/setting is not table driver (in pkg_util.py) 4. Now setting env for all packages, not just the installed one. More changes in follow up commits. --- apio/commands/raw.py | 3 +- apio/managers/drivers.py | 21 ++- apio/managers/examples.py | 15 +- apio/managers/installer.py | 17 +- apio/managers/scons.py | 37 ++-- apio/managers/system.py | 19 +- apio/pkg_util.py | 331 +++++++++++++++++++++++++++++++++ apio/resources.py | 23 +-- apio/resources/packages.json | 8 +- apio/util.py | 351 +---------------------------------- test/commands/test_clean.py | 4 +- 11 files changed, 412 insertions(+), 417 deletions(-) create mode 100644 apio/pkg_util.py diff --git a/apio/commands/raw.py b/apio/commands/raw.py index 9fc5acdd..f6cb826e 100644 --- a/apio/commands/raw.py +++ b/apio/commands/raw.py @@ -10,6 +10,7 @@ import click from click.core import Context from apio import util +from apio import pkg_util from apio import cmd_util @@ -50,6 +51,6 @@ def cli( """Implements the apio raw command which executes user specified commands from apio installed tools. """ - + pkg_util.set_env_for_packages() exit_code = util.call(cmd) ctx.exit(exit_code) diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index 7bcac1e5..d0efb542 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -12,6 +12,7 @@ from pathlib import Path import click from apio import util +from apio import pkg_util from apio.profile import Profile from apio.resources import Resources @@ -185,8 +186,10 @@ def _setup_windows(self): # -- On windows the zadig driver installer should be # -- execute. Get the package version self.name = "drivers" - self.version = util.get_package_version(self.name, profile) - self.spec_version = util.get_package_spec_version(self.name, resources) + self.version = pkg_util.get_package_version(self.name, profile) + self.spec_version = pkg_util.get_package_spec_version( + self.name, resources + ) def _ftdi_enable_linux(self): """Drivers enable on Linux. It copies the .rules file into @@ -401,19 +404,19 @@ def _check_ftdi_driver_darwin(driver): def _ftdi_enable_windows(self): # -- Get the drivers apio package base folder - drivers_base_dir = util.get_package_dir("tools-drivers") + drivers_base_dir = pkg_util.get_package_dir("drivers") # -- No folder --> package not installer (or not correctly installed) - if not drivers_base_dir: - util.show_package_path_error(self.name) - util.show_package_install_instructions(self.name) + if not drivers_base_dir.exists(): + pkg_util.show_package_path_error(self.name) + pkg_util.show_package_install_instructions(self.name) sys.exit(1) # -- Build the drivers base bin dir drivers_bin_dir = drivers_base_dir / "bin" # -- Check if the driver packages is installed - package_ok = util.check_package( + package_ok = pkg_util.check_package( self.name, self.version, self.spec_version, drivers_bin_dir ) @@ -474,11 +477,11 @@ def _ftdi_disable_windows(): # W0703: Catching too general exception Exception (broad-except) # pylint: disable=W0703 def _serial_enable_windows(self): - drivers_base_dir = util.get_package_dir("tools-drivers") + drivers_base_dir = pkg_util.get_package_dir("drivers") drivers_bin_dir = drivers_base_dir / "bin" try: - if util.check_package( + if pkg_util.check_package( self.name, self.version, self.spec_version, drivers_bin_dir ): click.secho("Launch drivers configuration tool") diff --git a/apio/managers/examples.py b/apio/managers/examples.py index e3bf90a8..da98f4c3 100644 --- a/apio/managers/examples.py +++ b/apio/managers/examples.py @@ -12,6 +12,7 @@ from typing import Optional, Tuple, List import click from apio import util +from apio import pkg_util from apio.profile import Profile from apio.resources import Resources @@ -55,20 +56,22 @@ def __init__(self): self.name = "examples" # -- Folder where the example packages was installed - self.examples_dir = util.get_package_dir(self.name) + self.examples_dir = pkg_util.get_package_dir(self.name) # -- Get the example package version - self.version = util.get_package_version(self.name, profile) + self.version = pkg_util.get_package_version(self.name, profile) # -- Get the version restrictions - self.spec_version = util.get_package_spec_version(self.name, resources) + self.spec_version = pkg_util.get_package_spec_version( + self.name, resources + ) def get_examples_infos(self) -> Optional[List[ExampleInfo]]: """Scans the examples and returns a list of ExampleInfos. Returns null if an error.""" # -- Check if the example package is installed - installed = util.check_package( + installed = pkg_util.check_package( self.name, self.version, self.spec_version, self.examples_dir ) @@ -168,7 +171,7 @@ def copy_example_dir(self, example: str, project_dir: Path, sayno: bool): """ # -- Check if the example package is installed - installed = util.check_package( + installed = pkg_util.check_package( self.name, self.version, self.spec_version, self.examples_dir ) @@ -232,7 +235,7 @@ def copy_example_files(self, example: str, project_dir: Path, sayno: bool): """ # -- Check if the example package is installed - installed = util.check_package( + installed = pkg_util.check_package( self.name, self.version, self.spec_version, self.examples_dir ) diff --git a/apio/managers/installer.py b/apio/managers/installer.py index 3386479f..98c5c6fb 100644 --- a/apio/managers/installer.py +++ b/apio/managers/installer.py @@ -14,6 +14,7 @@ import requests from apio import util +from apio import pkg_util from apio.resources import Resources from apio.profile import Profile from apio.managers.downloader import FileDownloader @@ -60,7 +61,7 @@ def __init__( self.resources = resources self.profile = None self.spec_version = None - self.package_name = None + self.package_folder_name = None self.extension = None self.download_urls = None self.compressed_name = None @@ -108,8 +109,8 @@ def __init__( # Get the spectec package version self.spec_version = distribution["packages"][self.package] - # Get the package name (from resources/package.json file) - self.package_name = data["release"]["package_name"] + # Get the package folder name (from resources/package.json file) + self.package_folder_name = data["release"]["folder_name"] # Get the extension given to the toolchain. Tipically tar.gz self.extension = data["release"]["extension"] @@ -157,7 +158,7 @@ def __init__( ): self.packages_dir = util.get_home_dir() / dirname - self.package_name = "toolchain-" + package + self.package_folder_name = "toolchain-" + package # -- If the Installer.package_dir property was not assigned, # -- is because the package was not known. Abort! @@ -280,7 +281,7 @@ def _install_package(self, dlpath: Path): # -- Build the destination path # -- Ex. '/home/obijuan/.apio/packages/examples' - package_dir = self.packages_dir / self.package_name + package_dir = self.packages_dir / self.package_folder_name if self.verbose: click.secho(f"Package dir: {package_dir.absolute()}") @@ -344,7 +345,7 @@ def _rename_unpacked_dir(self): # -- New folder # -. Ex, '/home/obijuan/.apio/packages/examples' - package_dir = self.packages_dir / self.package_name + package_dir = self.packages_dir / self.package_folder_name # -- Rename it! if unpack_dir.is_dir(): @@ -354,7 +355,7 @@ def uninstall(self): """Uninstall the apio package""" # -- Build the package filename - file = self.packages_dir / self.package_name + file = self.packages_dir / self.package_folder_name if self.verbose: click.secho(f"Package dir: {file.absolute()}") @@ -377,7 +378,7 @@ def uninstall(self): ) else: # -- Package not installed! - util.show_package_path_error(self.package) + pkg_util.show_package_path_error(self.package) # -- Remove the package from the profile file self.profile.remove_package(self.package) diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 4dc00256..09bf7488 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -22,10 +22,10 @@ import semantic_version from apio import util +from apio import pkg_util from apio.managers.arguments import process_arguments from apio.managers.arguments import serialize_scons_flags from apio.managers.system import System -from apio.profile import Profile from apio.resources import Resources from apio.managers.project import Project from apio.managers.scons_filter import SconsFilter @@ -86,9 +86,6 @@ def __init__(self, project_dir: Path): self.project = Project(project_dir) self.project.read() - # -- Read the apio profile file - self.profile = Profile() - # -- Read the apio resources self.resources = Resources(project_dir=project_dir) @@ -114,7 +111,9 @@ def clean(self, args) -> int: ) # --Clean the project: run scons -c (with aditional arguments) - return self._run("-c", arch=arch, variables=variables, packages=[]) + return self._run( + "-c", arch=arch, variables=variables, packages_names=[] + ) @on_exception(exit_code=1) def verify(self, args) -> int: @@ -132,7 +131,7 @@ def verify(self, args) -> int: "verify", variables=variables, arch=arch, - packages=["oss-cad-suite"], + packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -151,7 +150,7 @@ def graph(self, args) -> int: "graph", variables=variables, arch=arch, - packages=["oss-cad-suite"], + packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -174,7 +173,7 @@ def lint(self, args) -> int: "lint", variables=variables, arch=arch, - packages=["oss-cad-suite"], + packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -191,7 +190,7 @@ def sim(self, args) -> int: "sim", variables=variables, arch=arch, - packages=["oss-cad-suite", "gtkwave"], + packages_names=["oss-cad-suite", "gtkwave"], ) @on_exception(exit_code=1) @@ -208,7 +207,7 @@ def test(self, args) -> int: "test", variables=variables, arch=arch, - packages=["oss-cad-suite"], + packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -228,7 +227,7 @@ def build(self, args) -> int: variables=variables, board=board, arch=arch, - packages=["oss-cad-suite"], + packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -253,7 +252,7 @@ def time(self, args) -> int: variables=variables, board=board, arch=arch, - packages=["oss-cad-suite"], + packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -270,7 +269,7 @@ def report(self, args) -> int: variables=variables, board=board, arch=arch, - packages=["oss-cad-suite"], + packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -311,7 +310,7 @@ def upload(self, config: dict, prog: dict) -> int: exit_code = self._run( "upload", variables=flags, - packages=["oss-cad-suite"], + packages_names=["oss-cad-suite"], board=board, arch=arch, ) @@ -963,7 +962,7 @@ def _check_ftdi( # pylint: disable=too-many-arguments # pylint: disable=too-many-positional-arguments - def _run(self, command, variables, packages, board=None, arch=None): + def _run(self, command, variables, packages_names, board=None, arch=None): """Executes scons""" # -- Construct the path to the SConstruct file. @@ -986,14 +985,12 @@ def _run(self, command, variables, packages, board=None, arch=None): else: # Run on `default` config mode # -- Check if the necessary packages are installed - if not util.resolve_packages( - packages, - self.profile.packages, - self.resources.distribution.get("packages"), - ): + if not pkg_util.check_packages(packages_names, self.resources): # Exit if a package is not installed raise AttributeError("Package not installed") + pkg_util.set_env_for_packages() + # -- Execute scons return self._execute_scons(command, variables, board) diff --git a/apio/managers/system.py b/apio/managers/system.py index 29220d45..e5d2ba91 100644 --- a/apio/managers/system.py +++ b/apio/managers/system.py @@ -12,6 +12,7 @@ import click from apio import util +from apio import pkg_util from apio.profile import Profile @@ -32,10 +33,12 @@ def __init__(self, resources: dict): self.package_name = "oss-cad-suite" # -- Get the installed package versions - self.version = util.get_package_version(self.name, profile) + self.version = pkg_util.get_package_version(self.name, profile) # -- Get the spected versions - self.spec_version = util.get_package_spec_version(self.name, resources) + self.spec_version = pkg_util.get_package_spec_version( + self.name, resources + ) # -- Windows: Executables should end with .exe self.ext = "" @@ -158,14 +161,14 @@ def _run_command( # -- Get the package base dir # -- Ex. "/home/obijuan/.apio/packages/tools-oss-cad-suite" - system_base_dir = util.get_package_dir("tools-oss-cad-suite") + system_base_dir = pkg_util.get_package_dir("oss-cad-suite") # -- Package not found - if not system_base_dir: + if not system_base_dir.exists(): # -- Show the error message and a hint # -- on how to install the package - util.show_package_path_error(self.package_name) - util.show_package_install_instructions(self.package_name) + pkg_util.show_package_path_error(self.package_name) + pkg_util.show_package_install_instructions(self.package_name) raise util.ApioException() # -- Get the folder were the binary file is located (PosixPath) @@ -186,8 +189,8 @@ def _run_command( # -- Show the error message and a hint # -- on how to install the package - util.show_package_path_error(self.package_name) - util.show_package_install_instructions(self.package_name) + pkg_util.show_package_path_error(self.package_name) + pkg_util.show_package_install_instructions(self.package_name) # -- Command not executed. return None diff --git a/apio/pkg_util.py b/apio/pkg_util.py new file mode 100644 index 00000000..ed99b67e --- /dev/null +++ b/apio/pkg_util.py @@ -0,0 +1,331 @@ +# -*- coding: utf-8 -*- +# -- This file is part of the Apio project +# -- (C) 2016-2018 FPGAwars +# -- Author Jesús Arroyo +# -- Licence GPLv2 +# -- Derived from: +# ---- Platformio project +# ---- (C) 2014-2016 Ivan Kravets +# ---- Licence Apache v2 +"""Utility functions related to apio packages.""" + +from typing import List, Callable, Dict +from pathlib import Path +import os +import platform +from dataclasses import dataclass +import click +import semantic_version +from apio import util +from apio.profile import Profile +from apio.resources import Resources + + +def _add_env_path(path: Path) -> None: + """Prepends given path to env PATH variable. Not checking for dupes.""" + print(f" * Adding path: {path}") + old_val = os.environ["PATH"] + new_val = os.pathsep.join([str(path), old_val]) + os.environ["PATH"] = new_val + + +def _set_env_var(name: str, value: str) -> None: + """Sets env var to given value.""" + print(f" * Setting env: {name} = {value}") + os.environ[name] = value + + +def _set_oss_cad_package_env(package_path: Path) -> None: + """Sets the environment variasbles for the oss-cad-suite package.""" + + _add_env_path(package_path / "bin") + _add_env_path(package_path / "lib") + _add_env_path(package_path / "libexec") + + _set_env_var("IVL", str(package_path / "lib" / "ivl")) + _set_env_var("ICEBOX", str(package_path / "share" / "icebox")) + _set_env_var("TRELLIS", str(package_path / "share" / "trellis")) + _set_env_var("YOSYS_LIB", str(package_path / "share" / "yosys")) + + +def _set_examples_package_env(_: Path) -> None: + """Sets the environment variasbles for the examples package.""" + # -- Nothing to set here. + + +def _set_gtkwave_package_env(package_path: Path) -> None: + """Sets the environment variasbles for the gtkwave package.""" + _add_env_path(package_path / "bin") + + +def _set_drivers_package_env(package_path: Path) -> None: + """Sets the environment variasbles for the drivers package.""" + _add_env_path(package_path / "bin") + + +@dataclass(frozen=True) +class _PackageDesc: + """Represents an entry in the packages table.""" + + # -- Package folder name. E.g. "tools-oss-cad-suite" + folder_name: str + # -- True if the package is available for this platform. + platform_match: bool + # -- A function to set the env for this package. + env_setting_func: Callable[[Path], None] + + +# -- Package names to package properties. Using lambda as a workaround for +# -- forward reference to the env functions. +_PACKAGES: Dict[str, _PackageDesc] = { + "oss-cad-suite": _PackageDesc( + folder_name="tools-oss-cad-suite", + platform_match=True, + env_setting_func=_set_oss_cad_package_env, + ), + "examples": _PackageDesc( + folder_name="examples", + platform_match=True, + env_setting_func=_set_examples_package_env, + ), + "gtkwave": _PackageDesc( + folder_name="tool-gtkwave", + platform_match=platform.system() == "Windows", + env_setting_func=_set_gtkwave_package_env, + ), + "drivers": _PackageDesc( + folder_name="tools-drivers", + platform_match=platform.system() == "Windows", + env_setting_func=_set_drivers_package_env, + ), +} + + +def set_env_for_packages() -> None: + """Sets the environment variables for using the packages.""" + # print(f"*** set_env_for_packages()") + # base_path = get_packages_dir() + for package_name, package_desc in _PACKAGES.items(): + if package_desc.platform_match: + print(f"*** Setting env for package: {package_name}") + package_path = get_package_dir(package_name) + package_desc.env_setting_func(package_path) + + +def check_packages(packages_names: List[str], resources: Resources) -> None: + """Tesks if the given packages have proper versions installed. + Returns True if OK. + """ + + profile = Profile() + installed_packages = profile.packages + spec_packages = resources.distribution.get("packages") + + # -- Check packages + check = True + for package_name in packages_names: + package_desc = _PACKAGES[package_name] + if package_desc.platform_match: + version = installed_packages.get(package_name, {}).get( + "version", "" + ) + spec_version = spec_packages.get(package_name, "") + + _bin = get_package_dir(package_name) / "bin" + + # -- Check this package + check &= check_package(package_name, version, spec_version, _bin) + + return check + + +def check_package( + name: str, version: str, spec_version: str, path: Path +) -> bool: + """Check if the given package is ok + (and can be installed without problems) + * INPUTS: + - name: Package name + - version: Package version + - spec_version: semantic version constraint + - path: path where the binary files of the package are stored + + * OUTPUT: + - True: Package + """ + + # Check package path + if path and not path.is_dir(): + show_package_path_error(name) + show_package_install_instructions(name) + return False + + # Check package version + if not _check_package_version(version, spec_version): + _show_package_version_error(name, version, spec_version) + show_package_install_instructions(name) + return False + + return True + + +def _check_package_version(version: str, spec_version: str) -> bool: + """Check if a given version satisfy the semantic version constraints + * INPUTS: + - version: Package version (Ex. '0.0.9') + - spec_version: semantic version constraint (Ex. '>=0.0.1') + * OUTPUT: + - True: Version ok! + - False: Version not ok! or incorrect version number + """ + + # -- Build a semantic version object + spec = semantic_version.SimpleSpec(spec_version) + + # -- Check it! + try: + semver = semantic_version.Version(version) + + # -- Incorrect version number + except ValueError: + return False + + # -- Check the version (True if the semantic version is satisfied) + return semver in spec + + +def _show_package_version_error( + name: str, current_version: str, spec_version: str +): + """Print error message: a package is missing or has a worng version.""" + + if current_version: + message = ( + ( + f"Error: package '{name}' version {current_version} does not\n" + f"match the requirement for version {spec_version}." + ), + ) + + else: + message = f"Error: package '{name}' is missing." + click.secho(message, fg="red") + + +def show_package_path_error(name: str): + """Display an error: package Not installed + * INPUTs: + - name: Package name + """ + + message = f"Error: package '{name}' is not installed" + click.secho(message, fg="red") + + +def show_package_install_instructions(name: str): + """Print the package install instructions + * INPUTs: + - name: Package name + """ + + click.secho(f"Please run:\n apio install {name}", fg="yellow") + + +def get_package_version(name: str, profile: dict) -> str: + """Get the version of a given package + * INPUTs: + - name: Package name + - profile: Dictrionary with the profile information + * OUTPUT: + - The version (Ex. '0.0.9') + """ + + # -- Default version + version = "" + + # -- Check if the package is intalled + if name in profile.packages: + version = profile.packages[name]["version"] + + # -- Return the version + return version + + +def get_package_spec_version(name: str, resources: dict) -> str: + """Get the version restrictions for a given package + * INPUTs: + * name: Package name + * resources: Apio resources object + * OUTPUT: version restrictions for that package + Ex. '>=1.1.0,<1.2.0' + """ + + # -- No restrictions by default + spec_version = "" + + # -- Check that the package is valid + if name in resources.distribution["packages"]: + + # -- Get the package restrictions + spec_version = resources.distribution["packages"][name] + + # -- Return the restriction + return spec_version + + +def get_packages_dir() -> Path: + """Return the base directory of apio packages. + Packages are installed in the following folder: + * Default: $APIO_HOME_DIR/packages + * $APIO_PKG_DIR/packages: if the APIO_PKG_DIR env variable is set + * INPUT: + - pkg_name: Package name (Ex. 'examples') + * OUTPUT: + - The package folder (PosixPath) + (Ex. '/home/obijuan/.apio/packages/examples')) + - or None if the packageis not installed + """ + + # -- Get the apio home dir: + # -- Ex. '/home/obijuan/.apio' + apio_home_dir = util.get_home_dir() + + # -- Get the APIO_PKG_DIR env variable + # -- It returns None if it was not defined + apio_pkg_dir_env = util.get_projconf_option_dir("pkg_dir") + + # -- Get the pkg base dir. It is what the APIO_PKG_DIR env variable + # -- says, or the default folder if None + if apio_pkg_dir_env: + pkg_home_dir = Path(apio_pkg_dir_env) + + # -- Default value + else: + pkg_home_dir = apio_home_dir + + # -- Create the package folder + # -- Ex '/home/obijuan/.apio/packages/tools-oss-cad-suite' + package_dir = pkg_home_dir / "packages" + + # -- Return the folder if it exists + # if package_dir.exists(): + return package_dir + + +def get_package_dir(package_name: str) -> Path: + """Return the APIO package dir of a given package + Packages are installed in the following folder: + * Default: $APIO_HOME_DIR/packages + * $APIO_PKG_DIR/packages: if the APIO_PKG_DIR env variable is set + * INPUT: + - pkg_name: Package name (Ex. 'examples') + * OUTPUT: + - The package folder (PosixPath) + (Ex. '/home/obijuan/.apio/packages/examples')) + - or None if the packageis not installed + """ + + package_folder = _PACKAGES[package_name].folder_name + package_dir = get_packages_dir() / package_folder + + return package_dir diff --git a/apio/resources.py b/apio/resources.py index fc1e74cb..22953d9f 100644 --- a/apio/resources.py +++ b/apio/resources.py @@ -105,9 +105,6 @@ def __init__( sorted(self.fpgas.items(), key=lambda t: t[0]) ) - # -- Default profile file - self.profile = None - def _load_resource(self, name: str, allow_custom: bool = False) -> dict: """Load the resources from a given json file * INPUTS: @@ -195,11 +192,13 @@ def _load_resource_file(filepath: Path) -> dict: # -- Return the object for the resource return resource - def get_package_release_name(self, package: str) -> str: - """return the package name""" + def get_package_folder_name(self, package: str) -> str: + """return the package folder name""" try: - package_name = self.packages[package]["release"]["package_name"] + package_folder_name = self.packages[package]["release"][ + "folder_name" + ] # -- This error should never ocurr except KeyError as excp: @@ -224,7 +223,7 @@ def get_package_release_name(self, package: str) -> str: sys.exit(1) # -- Return the name - return package_name + return package_folder_name def get_packages(self) -> tuple[list, list]: """Get all the packages, classified in installed and @@ -237,6 +236,8 @@ def get_packages(self) -> tuple[list, list]: installed_packages = [] notinstalled_packages = [] + profile = Profile() + # -- Go though all the apio packages for package in self.packages: @@ -248,10 +249,10 @@ def get_packages(self) -> tuple[list, list]: } # -- Check if this package is installed - if package in self.profile.packages: + if package in profile.packages: # -- Get the installed version - version = self.profile.packages[package]["version"] + version = profile.packages[package]["version"] # -- Store the version data["version"] = version @@ -265,7 +266,7 @@ def get_packages(self) -> tuple[list, list]: # -- Check the installed packages and update # -- its information - for package in self.profile.packages: + for package in profile.packages: # -- The package is not known! # -- Strange case @@ -283,7 +284,7 @@ def get_packages(self) -> tuple[list, list]: def list_packages(self, installed=True, notinstalled=True): """Return a list with all the installed/notinstalled packages""" - self.profile = Profile() + # profile = Profile() # Classify packages installed_packages, notinstalled_packages = self.get_packages() diff --git a/apio/resources/packages.json b/apio/resources/packages.json index 3b8daacf..5d4221d5 100644 --- a/apio/resources/packages.json +++ b/apio/resources/packages.json @@ -8,7 +8,7 @@ "tag_name": "%V", "compressed_name": "apio-examples-%V", "uncompressed_name": "apio-examples-%V", - "package_name": "examples", + "folder_name": "examples", "extension": "zip", "url_version": "https://github.com/FPGAwars/apio-examples/raw/master/VERSION" }, @@ -24,7 +24,7 @@ "tag_name": "v%V", "compressed_name": "tools-oss-cad-suite-%P-%V", "uncompressed_name": "", - "package_name": "tools-oss-cad-suite", + "folder_name": "tools-oss-cad-suite", "extension": "tar.gz", "url_version": "https://github.com/FPGAwars/tools-oss-cad-suite/raw/main/VERSION_DEV" }, @@ -40,7 +40,7 @@ "tag_name": "v%V", "compressed_name": "tool-gtkwave-%P-%V", "uncompressed_name": "", - "package_name": "tool-gtkwave", + "folder_name": "tool-gtkwave", "extension": "tar.gz", "url_version": "https://github.com/FPGAwars/tool-gtkwave/raw/master/VERSION", "available_platforms": [ @@ -61,7 +61,7 @@ "tag_name": "v%V", "compressed_name": "tools-drivers-%P-%V", "uncompressed_name": "", - "package_name": "tools-drivers", + "folder_name": "tools-drivers", "extension": "tar.gz", "url_version": "https://github.com/FPGAwars/tools-drivers/raw/master/VERSION", "available_platforms": [ diff --git a/apio/util.py b/apio/util.py index 055c7162..0caa8f05 100644 --- a/apio/util.py +++ b/apio/util.py @@ -21,7 +21,6 @@ from threading import Thread from pathlib import Path import click -import semantic_version from serial.tools.list_ports import comports import requests @@ -29,20 +28,6 @@ # -- Constants # ---------------------------------------- -# -- Packages names -# -- If you need to create new packages, you should -# -- define first the constants here -# -- -OSS_CAD_SUITE = "oss-cad-suite" -GTKWAVE = "gtkwave" - -# -- Name of the subfolder to store de executable files -BIN = "bin" - -# -- Folder names. They are built from the -# -- packages names -OSS_CAD_SUITE_FOLDER = f"tools-{OSS_CAD_SUITE}" -GTKWAVE_FOLDER = f"tool-{GTKWAVE}" # -- AVAILABLE PLATFORMS PLATFORMS = [ @@ -206,7 +191,7 @@ def get_systype() -> str: return platform_str -def _get_projconf_option_dir(name: str, default=None): +def get_projconf_option_dir(name: str, default=None): """Return the project option with the given name These options are place either on environment variables or into the /etc/apio.json file in the case of debian distributions @@ -251,7 +236,7 @@ def get_home_dir() -> Path: # -- Get the APIO_HOME_DIR env variable # -- It returns None if it was not defined - apio_home_dir_env = _get_projconf_option_dir("home_dir") + apio_home_dir_env = get_projconf_option_dir("home_dir") # -- Get the home dir. It is what the APIO_HOME_DIR env variable # -- says, or the default folder if None @@ -271,54 +256,8 @@ def get_home_dir() -> Path: return home_dir -def get_package_dir(pkg_name: str) -> Path: - """Return the APIO package dir of a given package - Packages are installed in the following folder: - * Default: $APIO_HOME_DIR/packages - * $APIO_PKG_DIR/packages: if the APIO_PKG_DIR env variable is set - * INPUT: - - pkg_name: Package name (Ex. 'examples') - * OUTPUT: - - The package folder (PosixPath) - (Ex. '/home/obijuan/.apio/packages/examples')) - - or None if the packageis not installed - """ - - # -- Get the apio home dir: - # -- Ex. '/home/obijuan/.apio' - apio_home_dir = get_home_dir() - - # -- Get the APIO_PKG_DIR env variable - # -- It returns None if it was not defined - apio_pkg_dir_env = _get_projconf_option_dir("pkg_dir") - - # -- Get the pkg base dir. It is what the APIO_PKG_DIR env variable - # -- says, or the default folder if None - if apio_pkg_dir_env: - pkg_home_dir = Path(apio_pkg_dir_env) - - # -- Default value - else: - pkg_home_dir = apio_home_dir - - # -- Create the package folder - # -- Ex '/home/obijuan/.apio/packages/tools-oss-cad-suite' - package_dir = pkg_home_dir / "packages" / pkg_name - - # -- Return the folder if it exists - if package_dir.exists(): - return package_dir - - # -- No path... - return None - - def call(cmd): - """Execute the given command from the installed apio packages""" - - # -- Set the PATH environment variable for finding the - # -- executables on the apio package folders first - setup_environment() + """Execute the given command.""" # -- Execute the command from the shell result = subprocess.call(cmd, shell=True) @@ -331,290 +270,6 @@ def call(cmd): return result -def setup_environment(): - """Set the environment variables and the system PATH""" - - # --- Get the table with the paths of all the apio packages - base_dirs = get_base_dirs() - - # --- Get the table with the paths of all the executables - # --- of the apio packages - bin_dirs = get_bin_dirs(base_dirs) - - # --- Set the system env. variables - set_env_variables(base_dirs, bin_dirs) - - return bin_dirs - - -def set_env_variables(base_dirs: dict, bin_dirs: dict): - """Set the environment variables""" - - # -- Get the current system PATH - path = os.environ["PATH"] - - # -- Add the packages to the path. The first packages added - # -- have the lowest priority. The latest the highest - - # -- Add the gtkwave to the path if installed, - # -- but only for windows platforms - if platform.system() == "Windows": - # -- Gtkwave package is installed - if bin_dirs[GTKWAVE]: - path = os.pathsep.join([str(bin_dirs[GTKWAVE]), path]) - - # -- Add the binary folders of the installed packages - # -- to the path, except for the OSS_CAD_SUITE package - for pack in base_dirs: - if base_dirs[pack] and pack != OSS_CAD_SUITE: - path = os.pathsep.join([str(bin_dirs[pack]), path]) - - # -- Add the OSS_CAD_SUITE package to the path - # -- if installed (Maximum priority) - if base_dirs[OSS_CAD_SUITE]: - # -- Get the lib folder (where the shared libraries are located) - oss_cad_suite_lib = str(base_dirs[OSS_CAD_SUITE] / "lib") - - # -- Add the lib folder - path = os.pathsep.join([oss_cad_suite_lib, path]) - path = os.pathsep.join([str(bin_dirs[OSS_CAD_SUITE]), path]) - - # Add the virtual python environment to the path - os.environ["PATH"] = path - - # Add other environment variables - - os.environ["IVL"] = str(base_dirs[OSS_CAD_SUITE] / "lib" / "ivl") - - os.environ["ICEBOX"] = str(base_dirs[OSS_CAD_SUITE] / "share" / "icebox") - - os.environ["TRELLIS"] = str(base_dirs[OSS_CAD_SUITE] / "share" / "trellis") - - os.environ["YOSYS_LIB"] = str(base_dirs[OSS_CAD_SUITE] / "share" / "yosys") - - -def resolve_packages( - packages: list, installed_packages: list, spec_packages: dict -) -> bool: - """Check the given packages - * make sure they all are installed - * make sure they versions are ok and have no conflicts... - * INPUTS - * package: List of package names to check - * installed_packages: Dictionry with all the apio packages installed - * spec_packages: Dictionary with the spec version: - (Ex. {'drivers': '>=1.1.0,<1.2.0'....}) - - * OUTPUT: - * True: All the packages are ok! - * False: There is an error... - """ - - # --- Get the table with the paths of all the apio packages - base_dirs = get_base_dirs() - - # --- Get the table with the paths of all the executables - # --- of the apio packages - bin_dirs = get_bin_dirs(base_dirs) - - # -- Check packages - check = True - for package in packages: - version = installed_packages.get(package, {}).get("version", "") - - spec_version = spec_packages.get(package, "") - - # -- Get the package binary dir as a PosixPath object - _bin = bin_dirs[package] - - # -- Check this package - check &= check_package(package, version, spec_version, _bin) - - # -- Load packages - if check: - # --- Set the system env. variables - set_env_variables(base_dirs, bin_dirs) - - return check - - -def get_base_dirs(): - """Return a dictionary with the local paths of the apio packages - installed on the system. If the packages is not installed, - the path is '' - """ - - # -- Create the dictionary: - # -- Package Name : Folder (string) - base_dirs = { - OSS_CAD_SUITE: get_package_dir(OSS_CAD_SUITE_FOLDER), - GTKWAVE: get_package_dir(GTKWAVE_FOLDER), - } - - return base_dirs - - -def get_bin_dirs(base_dirs: dict): - """Return a table with the package name and the folder were - the executable files are stored - * INPUT - -base_dirs: A Dict with the package base_dir - """ - - if base_dirs[GTKWAVE]: - gtkwave_path = base_dirs[GTKWAVE] / BIN - else: - gtkwave_path = None - - if base_dirs[OSS_CAD_SUITE]: - oss_cad_suite_path = base_dirs[OSS_CAD_SUITE] / BIN - else: - oss_cad_suite_path = None - - bin_dir = {OSS_CAD_SUITE: oss_cad_suite_path, GTKWAVE: gtkwave_path} - - return bin_dir - - -def check_package( - name: str, version: str, spec_version: str, path: Path -) -> bool: - """Check if the given package is ok - (and can be installed without problems) - * INPUTS: - - name: Package name - - version: Package version - - spec_version: semantic version constraint - - path: path where the binary files of the package are stored - - * OUTPUT: - - True: Package - """ - - # Apio package 'gtkwave' only exists for Windows. - # Linux and MacOS user must install the native GTKWave. - if name == "gtkwave" and platform.system() != "Windows": - return True - - # Check package path - if path and not path.is_dir(): - show_package_path_error(name) - show_package_install_instructions(name) - return False - - # Check package version - if not check_package_version(version, spec_version): - show_package_version_error(name, version, spec_version) - show_package_install_instructions(name) - return False - - return True - - -def check_package_version(version: str, spec_version: str) -> bool: - """Check if a given version satisfy the semantic version constraints - * INPUTS: - - version: Package version (Ex. '0.0.9') - - spec_version: semantic version constraint (Ex. '>=0.0.1') - * OUTPUT: - - True: Version ok! - - False: Version not ok! or incorrect version number - """ - - # -- Build a semantic version object - spec = semantic_version.SimpleSpec(spec_version) - - # -- Check it! - try: - semver = semantic_version.Version(version) - - # -- Incorrect version number - except ValueError: - return False - - # -- Check the version (True if the semantic version is satisfied) - return semver in spec - - -def show_package_version_error( - name: str, current_version: str, spec_version: str -): - """Print error message: a package is missing or has a worng version.""" - - if current_version: - message = ( - ( - f"Error: package '{name}' version {current_version} does not\n" - f"match the requirement for version {spec_version}." - ), - ) - - else: - message = f"Error: package '{name}' is missing." - click.secho(message, fg="red") - - -def show_package_path_error(name: str): - """Display an error: package Not installed - * INPUTs: - - name: Package name - """ - - message = f"Error: package '{name}' is not installed" - click.secho(message, fg="red") - - -def show_package_install_instructions(name: str): - """Print the package install instructions - * INPUTs: - - name: Package name - """ - - click.secho(f"Please run:\n apio install {name}", fg="yellow") - - -def get_package_version(name: str, profile: dict) -> str: - """Get the version of a given package - * INPUTs: - - name: Package name - - profile: Dictrionary with the profile information - * OUTPUT: - - The version (Ex. '0.0.9') - """ - - # -- Default version - version = "" - - # -- Check if the package is intalled - if name in profile.packages: - version = profile.packages[name]["version"] - - # -- Return the version - return version - - -def get_package_spec_version(name: str, resources: dict) -> str: - """Get the version restrictions for a given package - * INPUTs: - * name: Package name - * resources: Apio resources object - * OUTPUT: version restrictions for that package - Ex. '>=1.1.0,<1.2.0' - """ - - # -- No restrictions by default - spec_version = "" - - # -- Check that the package is valid - if name in resources.distribution["packages"]: - - # -- Get the package restrictions - spec_version = resources.distribution["packages"][name] - - # -- Return the restriction - return spec_version - - @dataclass(frozen=True) class CommandResult: """Contains the results of a command (subprocess) execution.""" diff --git a/test/commands/test_clean.py b/test/commands/test_clean.py index 30392488..f8db0ad3 100644 --- a/test/commands/test_clean.py +++ b/test/commands/test_clean.py @@ -30,7 +30,7 @@ def test_clean(clirunner, configenv): # -- Execute "apio clean --board alhambra-ii" result = clirunner.invoke(cmd_clean, ["--board", "alhambra-ii"]) - assert result.exit_code != 0, result.output + assert result.exit_code == 0, result.output def test_clean_create(clirunner, configenv): @@ -51,4 +51,4 @@ def test_clean_create(clirunner, configenv): # --- Execute "apio clean" result = clirunner.invoke(cmd_clean) - assert result.exit_code != 0, result.output + assert result.exit_code == 0, result.output From 2a023648a1654efba8d67b86928b27bb155dc2c2 Mon Sep 17 00:00:00 2001 From: Zapta Date: Wed, 30 Oct 2024 20:00:16 -0700 Subject: [PATCH 12/51] Next installment of cleaning up the apio packages logic. With this change, all command check installed packages via a single function check_required_packages(). --- apio/commands/drivers.py | 4 +- apio/commands/examples.py | 4 +- apio/commands/upload.py | 4 +- apio/managers/drivers.py | 286 +++++++++++++++++-------------------- apio/managers/examples.py | 62 ++------ apio/managers/installer.py | 7 +- apio/managers/scons.py | 6 +- apio/managers/system.py | 76 ++-------- apio/pkg_util.py | 221 ++++++++++++---------------- apio/resources.py | 14 +- test/test_end_to_end.py | 2 +- 11 files changed, 268 insertions(+), 418 deletions(-) diff --git a/apio/commands/drivers.py b/apio/commands/drivers.py index 05f5bfa3..0f62af75 100644 --- a/apio/commands/drivers.py +++ b/apio/commands/drivers.py @@ -12,6 +12,7 @@ from click.core import Context from apio.managers.drivers import Drivers from apio import cmd_util +from apio.resources import Resources # --------------------------- # -- COMMAND SPECIFIC OPTIONS @@ -96,7 +97,8 @@ def cli( ) # -- Access to the Drivers - drivers = Drivers() + resources = Resources() + drivers = Drivers(resources) # -- FTDI enable option if ftdi_enable: diff --git a/apio/commands/examples.py b/apio/commands/examples.py index e8a1f688..fe3b4b69 100644 --- a/apio/commands/examples.py +++ b/apio/commands/examples.py @@ -14,6 +14,7 @@ from apio.managers.examples import Examples from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- # -- COMMAND SPECIFIC OPTIONS @@ -87,7 +88,8 @@ def cli( cmd_util.check_exclusive_params(ctx, nameof(list_, dir_, files)) # -- Access to the Drivers - examples = Examples() + resources = Resources() + examples = Examples(resources) # -- Option: List all the available examples if list_: diff --git a/apio/commands/upload.py b/apio/commands/upload.py index 017cfdc0..28847443 100644 --- a/apio/commands/upload.py +++ b/apio/commands/upload.py @@ -14,6 +14,7 @@ from apio.managers.drivers import Drivers from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -92,7 +93,8 @@ def cli( """Implements the upload command.""" # -- Create a drivers object - drivers = Drivers() + resources = Resources() + drivers = Drivers(resources) # -- Only for MAC # -- Operation to do before uploading a design in MAC diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index d0efb542..67bf9443 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -13,7 +13,6 @@ import click from apio import util from apio import pkg_util -from apio.profile import Profile from apio.resources import Resources FTDI_INSTALL_DRIVER_INSTRUCTIONS = """ @@ -65,101 +64,101 @@ class Drivers: # -- to the /etc/udev/rules.d folder # -- FTDI source rules file paths - resources = util.get_path_in_apio_package("resources") - ftdi_rules_local_path = resources / "80-fpga-ftdi.rules" + resources_dir = util.get_path_in_apio_package("resources") + ftdi_rules_local_path = resources_dir / "80-fpga-ftdi.rules" # -- Target rule file ftdi_rules_system_path = Path("/etc/udev/rules.d/80-fpga-ftdi.rules") # Serial rules files paths - serial_rules_local_path = resources / "80-fpga-serial.rules" + serial_rules_local_path = resources_dir / "80-fpga-serial.rules" serial_rules_system_path = Path("/etc/udev/rules.d/80-fpga-serial.rules") # Driver to restore: mac os driver_c = "" - def __init__(self) -> None: - self.profile = None - self.name = None - self.version = None - self.spec_version = None + def __init__(self, resources: Resources) -> None: # -- Get the platform (a string) self.platform = util.get_systype() - def ftdi_enable(self): - """Enable the FTDI driver. It depends on the platform""" + # self.profile = Profile() + self.resources = resources + + def ftdi_enable(self) -> int: + """Enables the FTDI driver. Function is platform dependent. + Returns a process exit code. + """ - # -- Driver enabling on Linux if "linux" in self.platform: return self._ftdi_enable_linux() - # -- Driver enabling on MAC if "darwin" in self.platform: - self._setup_darwin() return self._ftdi_enable_darwin() - # -- Driver enabling on Windows if "windows" in self.platform: - self._setup_windows() return self._ftdi_enable_windows() - return None - def ftdi_disable(self): - """Disable the FTDI driver. It depends on the platform""" + click.secho("Error: unknown platform '{self.platform}'.") + return 1 + + def ftdi_disable(self) -> int: + """Disables the FTDI driver. Function is platform dependent. + Returns a process exit code. + """ - # -- Linux platforms if "linux" in self.platform: return self._ftdi_disable_linux() - # -- MAC if "darwin" in self.platform: - self._setup_darwin() + # self._setup_darwin() return self._ftdi_disable_darwin() - # -- Windows if "windows" in self.platform: - self._setup_windows() + # self._setup_windows() return self._ftdi_disable_windows() - return None + click.secho("Error: unknown platform '{self.platform}'.") + return 1 - def serial_enable(self): - """Enable the Serial driver. It depends on the platform""" + def serial_enable(self) -> int: + """Enables the serial driver. Function is platform dependent. + Returns a process exit code. + """ if "linux" in self.platform: return self._serial_enable_linux() if "darwin" in self.platform: - self._setup_darwin() return self._serial_enable_darwin() if "windows" in self.platform: - self._setup_windows() return self._serial_enable_windows() - return None - def serial_disable(self): - """Disable the Serial driver. It depends on the platform""" + click.secho("Error: unknown platform '{self.platform}'.") + return 1 + def serial_disable(self) -> int: + """Disables the serial driver. Function is platform dependent. + Returns a process exit code. + """ if "linux" in self.platform: return self._serial_disable_linux() if "darwin" in self.platform: - self._setup_darwin() return self._serial_disable_darwin() if "windows" in self.platform: - self._setup_windows() return self._serial_disable_windows() - return None + + click.secho("Error: unknown platform '{self.platform}'.") + return 1 def pre_upload(self): """Operations to do before uploading a design Only for mac platforms""" if "darwin" in self.platform: - self._setup_darwin() self._pre_upload_darwin() def post_upload(self): @@ -167,33 +166,11 @@ def post_upload(self): Only for mac platforms""" if "darwin" in self.platform: - self._setup_darwin() self._post_upload_darwin() - def _setup_darwin(self): - """Setup operation on Mac""" - - # -- Just read the profile file - self.profile = Profile() - - def _setup_windows(self): - """Setup operations on Windows""" - - # -- Read the Profile and Resources files - profile = Profile() - resources = Resources() - - # -- On windows the zadig driver installer should be - # -- execute. Get the package version - self.name = "drivers" - self.version = pkg_util.get_package_version(self.name, profile) - self.spec_version = pkg_util.get_package_spec_version( - self.name, resources - ) - - def _ftdi_enable_linux(self): + def _ftdi_enable_linux(self) -> int: """Drivers enable on Linux. It copies the .rules file into - the corresponding folder""" + the corresponding folder. Return process exit code.""" click.secho("Configure FTDI drivers for FPGA") @@ -212,15 +189,17 @@ def _ftdi_enable_linux(self): ) # -- Execute the commands for reloading the udev system - self._reload_rules() + self._reload_rules_linux() click.secho("FTDI drivers enabled", fg="green") click.secho("Unplug and reconnect your board", fg="yellow") else: click.secho("Already enabled", fg="yellow") + return 0 + def _ftdi_disable_linux(self): - """Disable the FTDI drivers on linux""" + """Disable the FTDI drivers on linux. Returns process exist code.""" # -- For disabling the FTDI driver the .rules files should be # -- removed from the /etc/udev/rules.d/ folder @@ -233,15 +212,17 @@ def _ftdi_disable_linux(self): subprocess.call(["sudo", "rm", str(self.ftdi_rules_system_path)]) # -- # -- Execute the commands for reloading the udev system - self._reload_rules() + self._reload_rules_linux() click.secho("FTDI drivers disabled", fg="green") click.secho("Unplug and reconnect your board", fg="yellow") else: click.secho("Already disabled", fg="yellow") + return 0 + def _serial_enable_linux(self): - """Serial drivers enable on Linux""" + """Serial drivers enable on Linux. Returns process exit code.""" click.secho("Configure Serial drivers for FPGA") @@ -249,7 +230,7 @@ def _serial_enable_linux(self): if not self.serial_rules_system_path.exists(): # -- Add the user to the dialout group for # -- having access to the serial port - group_added = self._add_dialout_group() + group_added = self._add_dialout_group_linux() # -- The file does not exist. Copy! # -- Execute the cmd: sudo cp src_file target_file @@ -263,7 +244,7 @@ def _serial_enable_linux(self): ) # -- Execute the commands for reloading the udev system - self._reload_rules() + self._reload_rules_linux() click.secho("Serial drivers enabled", fg="green") click.secho("Unplug and reconnect your board", fg="yellow") @@ -275,8 +256,10 @@ def _serial_enable_linux(self): else: click.secho("Already enabled", fg="yellow") - def _serial_disable_linux(self): - """Disable the serial driver on Linux""" + return 0 + + def _serial_disable_linux(self) -> int: + """Disable the serial driver on Linux. Return process exit code.""" # -- For disabling the serial driver the corresponding .rules file # -- should be removed, it it exists @@ -287,14 +270,15 @@ def _serial_disable_linux(self): subprocess.call(["sudo", "rm", str(self.serial_rules_system_path)]) # -- Execute the commands for reloading the udev system - self._reload_rules() + self._reload_rules_linux() click.secho("Serial drivers disabled", fg="green") click.secho("Unplug and reconnect your board", fg="yellow") else: click.secho("Already disabled", fg="yellow") - @staticmethod - def _reload_rules(): + return 0 + + def _reload_rules_linux(self): """Execute the commands for reloading the udev system""" # -- These are Linux commands that should be executed on @@ -303,8 +287,7 @@ def _reload_rules(): subprocess.call(["sudo", "udevadm", "trigger"]) subprocess.call(["sudo", "service", "udev", "restart"]) - @staticmethod - def _add_dialout_group(): + def _add_dialout_group_linux(self): """Add the current user to the dialout group on Linux systems""" # -- This operation is needed for granting access to the serial port @@ -319,66 +302,73 @@ def _add_dialout_group(): return True return None - def _ftdi_enable_darwin(self): + def _ftdi_enable_darwin(self) -> int: + """Enables FTDI driver on darwin. Returns process status code.""" # Check homebrew brew = subprocess.call("which brew > /dev/null", shell=True) if brew != 0: click.secho("Error: homebrew is required", fg="red") - else: - click.secho("Enable FTDI drivers for FPGA") - subprocess.call(["brew", "update"]) - self._brew_install("libffi") - self._brew_install("libftdi") - self.profile.add_setting("macos_ftdi_drivers", True) - self.profile.save() - click.secho("FTDI drivers enabled", fg="green") + return 1 + + click.secho("Enable FTDI drivers for FPGA") + subprocess.call(["brew", "update"]) + self._brew_install_darwin("libffi") + self._brew_install_darwin("libftdi") + self.resources.profile.add_setting("macos_ftdi_drivers", True) + self.resources.profile.save() + click.secho("FTDI drivers enabled", fg="green") + return 0 def _ftdi_disable_darwin(self): + """Disables FTDI driver on darwin. Returns process status code.""" click.secho("Disable FTDI drivers configuration") - self.profile.add_setting("macos_ftdi_drivers", False) - self.profile.save() + self.resources.profile.add_setting("macos_ftdi_drivers", False) + self.resources.profile.save() click.secho("FTDI drivers disabled", fg="green") + return 0 def _serial_enable_darwin(self): + """Enables serial driver on darwin. Returns process status code.""" # Check homebrew brew = subprocess.call("which brew > /dev/null", shell=True) if brew != 0: click.secho("Error: homebrew is required", fg="red") - else: - click.secho("Enable Serial drivers for FPGA") - subprocess.call(["brew", "update"]) - self._brew_install("libffi") - self._brew_install("libusb") - # self._brew_install_serial_drivers() - click.secho("Serial drivers enabled", fg="green") - - @staticmethod - def _serial_disable_darwin(): + return 1 + + click.secho("Enable Serial drivers for FPGA") + subprocess.call(["brew", "update"]) + self._brew_install_darwin("libffi") + self._brew_install_darwin("libusb") + # self._brew_install_serial_drivers_darwin() + click.secho("Serial drivers enabled", fg="green") + return 0 + + def _serial_disable_darwin(self): + """Disables serial driver on darwin. Returns process status code.""" click.secho("Disable Serial drivers configuration") click.secho("Serial drivers disabled", fg="green") - - @staticmethod - def _brew_install(package): - subprocess.call(["brew", "install", "--force", package]) - subprocess.call(["brew", "unlink", package]) - subprocess.call(["brew", "link", "--force", package]) - - @staticmethod - def _brew_install_serial_drivers(): - subprocess.call( - [ - "brew", - "tap", - "mengbo/ch340g-ch34g-ch34x-mac-os-x-driver", - "https://github.com/mengbo/ch340g-ch34g-ch34x-mac-os-x-driver", - ] - ) - subprocess.call( - ["brew", "cask", "install", "wch-ch34x-usb-serial-driver"] - ) + return 0 + + def _brew_install_darwin(self, brew_package): + subprocess.call(["brew", "install", "--force", brew_package]) + subprocess.call(["brew", "unlink", brew_package]) + subprocess.call(["brew", "link", "--force", brew_package]) + + # def _brew_install_serial_drivers_darwin(self): + # subprocess.call( + # [ + # "brew", + # "tap", + # "mengbo/ch340g-ch34g-ch34x-mac-os-x-driver", + # "https://github.com/mengbo/ch340g-ch34g-ch34x-mac-os-x-driver", + # ] + # ) + # subprocess.call( + # ["brew", "cask", "install", "wch-ch34x-usb-serial-driver"] + # ) def _pre_upload_darwin(self): - if self.profile.settings.get("macos_ftdi_drivers", False): + if self.resources.profile.settings.get("macos_ftdi_drivers", False): # Check and unload the drivers driver_a = "com.FTDI.driver.FTDIUSBSerialDriver" driver_b = "com.apple.driver.AppleUSBFTDI" @@ -390,40 +380,25 @@ def _pre_upload_darwin(self): self.driver_c = driver_b def _post_upload_darwin(self): - if self.profile.settings.get("macos_ftdi_drivers", False): + if self.resources.profile.settings.get("macos_ftdi_drivers", False): # Restore previous driver configuration if self.driver_c: subprocess.call(["sudo", "kextload", "-b", self.driver_c]) - @staticmethod - def _check_ftdi_driver_darwin(driver): + def _check_ftdi_driver_darwin(self, driver): return driver in str(subprocess.check_output(["kextstat"])) # W0703: Catching too general exception Exception (broad-except) # pylint: disable=W0703 - def _ftdi_enable_windows(self): + def _ftdi_enable_windows(self) -> int: + pkg_util.check_required_packages(["drivers"], self.resources) + + # -- + pkg_util.check_required_packages(["drivers"], self.resources) # -- Get the drivers apio package base folder drivers_base_dir = pkg_util.get_package_dir("drivers") - # -- No folder --> package not installer (or not correctly installed) - if not drivers_base_dir.exists(): - pkg_util.show_package_path_error(self.name) - pkg_util.show_package_install_instructions(self.name) - sys.exit(1) - - # -- Build the drivers base bin dir - drivers_bin_dir = drivers_base_dir / "bin" - - # -- Check if the driver packages is installed - package_ok = pkg_util.check_package( - self.name, self.version, self.spec_version, drivers_bin_dir - ) - - # -- Not installed. Exit - if not package_ok: - sys.exit(1) - # -- Path to the zadig.ini file # -- It is the zadig config file zadig_ini_src = drivers_base_dir / "share" / "zadig.ini" @@ -466,8 +441,9 @@ def _ftdi_enable_windows(self): return result.exit_code - @staticmethod - def _ftdi_disable_windows(): + def _ftdi_disable_windows(self) -> int: + pkg_util.check_required_packages(["drivers"], self.resources) + click.secho("Launch device manager") click.secho(FTDI_UNINSTALL_DRIVER_INSTRUCTIONS, fg="yellow") @@ -476,32 +452,32 @@ def _ftdi_disable_windows(): # W0703: Catching too general exception Exception (broad-except) # pylint: disable=W0703 - def _serial_enable_windows(self): + def _serial_enable_windows(self) -> int: + pkg_util.check_required_packages(["drivers"], self.resources) + drivers_base_dir = pkg_util.get_package_dir("drivers") drivers_bin_dir = drivers_base_dir / "bin" + pkg_util.check_required_packages(["drivers"], self.resources) + try: - if pkg_util.check_package( - self.name, self.version, self.spec_version, drivers_bin_dir - ): - click.secho("Launch drivers configuration tool") - click.secho(SERIAL_INSTALL_DRIVER_INSTRUCTIONS, fg="yellow") - result = util.exec_command( - str(Path(drivers_bin_dir) / "serial_install.exe") - ) - click.secho( - "Serial drivers configuration finished", fg="green" - ) - else: - result = util.CommandResult(exit_code=1) + + click.secho("Launch drivers configuration tool") + click.secho(SERIAL_INSTALL_DRIVER_INSTRUCTIONS, fg="yellow") + result = util.exec_command( + str(Path(drivers_bin_dir) / "serial_install.exe") + ) + click.secho("Serial drivers configuration finished", fg="green") + except Exception as exc: click.secho("Error: " + str(exc), fg="red") result = util.CommandResult(exit_code=1) return result.exit_code - @staticmethod - def _serial_disable_windows(): + def _serial_disable_windows(self) -> int: + pkg_util.check_required_packages(["drivers"], self.resources) + click.secho("Launch device manager") click.secho(SERIAL_UNINSTALL_DRIVER_INSTRUCTIONS, fg="yellow") diff --git a/apio/managers/examples.py b/apio/managers/examples.py index da98f4c3..bc7d99d5 100644 --- a/apio/managers/examples.py +++ b/apio/managers/examples.py @@ -13,7 +13,6 @@ import click from apio import util from apio import pkg_util -from apio.profile import Profile from apio.resources import Resources # -- Error messages @@ -23,7 +22,7 @@ USAGE_EXAMPLE = """ To fetch example files: - apio examples -f example-name + apio examples -f Example of use: apio examples -f icesum/leds @@ -44,40 +43,20 @@ class ExampleInfo: class Examples: """Manage the apio examples""" - def __init__(self): - - # -- Access to the profile information - profile = Profile() + def __init__(self, resources: Resources): # -- Access to the resources - resources = Resources() - - # -- Apio examples package name - self.name = "examples" + self.resources = resources # -- Folder where the example packages was installed - self.examples_dir = pkg_util.get_package_dir(self.name) - - # -- Get the example package version - self.version = pkg_util.get_package_version(self.name, profile) - - # -- Get the version restrictions - self.spec_version = pkg_util.get_package_spec_version( - self.name, resources - ) + self.examples_dir = pkg_util.get_package_dir("examples") def get_examples_infos(self) -> Optional[List[ExampleInfo]]: """Scans the examples and returns a list of ExampleInfos. Returns null if an error.""" - # -- Check if the example package is installed - installed = pkg_util.check_package( - self.name, self.version, self.spec_version, self.examples_dir - ) - - if not installed: - # -- A message was already printed. - return None + # -- Check that the example package is installed + pkg_util.check_required_packages(["examples"], self.resources) # -- Collect the examples home dir each board. boards_dirs: List[PosixPath] = [] @@ -119,6 +98,9 @@ def list_examples(self) -> None: """Print all the examples available. Return a process exit code, 0 if ok, non zero otherwise.""" + # -- Check that the examples package is installed. + pkg_util.check_required_packages(["examples"], self.resources) + # -- Get list of examples. examples: List[ExampleInfo] = self.get_examples_infos() if examples is None: @@ -170,14 +152,8 @@ def copy_example_dir(self, example: str, project_dir: Path, sayno: bool): * sayno: Automatically answer no """ - # -- Check if the example package is installed - installed = pkg_util.check_package( - self.name, self.version, self.spec_version, self.examples_dir - ) - - # -- No package installed: return - if not installed: - return 1 + # -- Check that the examples package is installed. + pkg_util.check_required_packages(["examples"], self.resources) # -- Get the working dir (current or given) project_dir = util.get_project_dir(project_dir, create_if_missing=True) @@ -234,14 +210,8 @@ def copy_example_files(self, example: str, project_dir: Path, sayno: bool): * sayno: Automatically answer no """ - # -- Check if the example package is installed - installed = pkg_util.check_package( - self.name, self.version, self.spec_version, self.examples_dir - ) - - # -- No package installed: return - if not installed: - return 1 + # -- Check that the examples package is installed. + pkg_util.check_required_packages(["examples"], self.resources) # -- Get the working dir (current or given) dst_example_path = util.get_project_dir( @@ -264,9 +234,8 @@ def copy_example_files(self, example: str, project_dir: Path, sayno: bool): return exit_code - @staticmethod def _copy_files( - example: str, src_path: Path, dest_path: Path, sayno: bool + self, example: str, src_path: Path, dest_path: Path, sayno: bool ): """Copy the example files to the destination folder * INPUTS: @@ -332,8 +301,7 @@ def _copy_files( return 0 - @staticmethod - def _copy_dir(example: str, src_path: Path, dest_path: Path): + def _copy_dir(self, example: str, src_path: Path, dest_path: Path): """Copy example of the src_path on the dest_path * INPUT * example: Name of the example (Ex. 'Alhambra-II/ledon') diff --git a/apio/managers/installer.py b/apio/managers/installer.py index 98c5c6fb..46e8f99c 100644 --- a/apio/managers/installer.py +++ b/apio/managers/installer.py @@ -14,7 +14,6 @@ import requests from apio import util -from apio import pkg_util from apio.resources import Resources from apio.profile import Profile from apio.managers.downloader import FileDownloader @@ -265,7 +264,7 @@ def install(self): # Try os name # dlpath = self._install_os_package(platform_download_url) except util.ApioException: - click.secho("Error: Package not found\n", fg="red") + click.secho("Error: package not found\n", fg="red") # -- Second step: Install downloaded package self._install_package(dlpath) @@ -378,7 +377,9 @@ def uninstall(self): ) else: # -- Package not installed! - pkg_util.show_package_path_error(self.package) + click.secho( + f"Error: package '{self.package}' is not installed", fg="red" + ) # -- Remove the package from the profile file self.profile.remove_package(self.package) diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 09bf7488..62ce304c 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -984,10 +984,8 @@ def _run(self, command, variables, packages_names, board=None, arch=None): ) else: # Run on `default` config mode - # -- Check if the necessary packages are installed - if not pkg_util.check_packages(packages_names, self.resources): - # Exit if a package is not installed - raise AttributeError("Package not installed") + # -- Check that the required packages are installed + pkg_util.check_required_packages(packages_names, self.resources) pkg_util.set_env_for_packages() diff --git a/apio/managers/system.py b/apio/managers/system.py index e5d2ba91..887ee48b 100644 --- a/apio/managers/system.py +++ b/apio/managers/system.py @@ -13,37 +13,15 @@ from apio import util from apio import pkg_util -from apio.profile import Profile +from apio.resources import Resources class System: # pragma: no cover """System class. Managing and execution of the system commands""" - def __init__(self, resources: dict): - # -- Read the profile from the file - profile = Profile() + def __init__(self, resources: Resources): - # -- This command is called system - self.name = "system" - - # -- Package name: apio package were all the system commands - # -- are located - # -- From apio > 0.7 the system tools are located inside the - # -- oss-cad-suite - self.package_name = "oss-cad-suite" - - # -- Get the installed package versions - self.version = pkg_util.get_package_version(self.name, profile) - - # -- Get the spected versions - self.spec_version = pkg_util.get_package_spec_version( - self.name, resources - ) - - # -- Windows: Executables should end with .exe - self.ext = "" - if platform.system() == "Windows": - self.ext = ".exe" + self.resources = resources def lsusb(self): """Run the lsusb system command""" @@ -64,7 +42,7 @@ def lsserial(): """DOC: TODO""" serial_ports = util.get_serial_ports() - click.secho(f"Number of Serial devices found: {len(serial_ports)}\n") + click.secho(f"Number of Serial devices found: {len(serial_ports)}") for serial_port in serial_ports: port = serial_port.get("port") @@ -156,46 +134,14 @@ def _run_command( In case of not executing the command it returns none! """ - # The system tools are locate in the - # oss-cad-suite package - - # -- Get the package base dir - # -- Ex. "/home/obijuan/.apio/packages/tools-oss-cad-suite" - system_base_dir = pkg_util.get_package_dir("oss-cad-suite") + # -- Check that the required package exists. + pkg_util.check_required_packages(["oss-cad-suite"], self.resources) - # -- Package not found - if not system_base_dir.exists(): - # -- Show the error message and a hint - # -- on how to install the package - pkg_util.show_package_path_error(self.package_name) - pkg_util.show_package_install_instructions(self.package_name) - raise util.ApioException() + # -- Set system env for using the packages. + pkg_util.set_env_for_packages() - # -- Get the folder were the binary file is located (PosixPath) - system_bin_dir = system_base_dir / "bin" - - # -- Get the executable filename - # -- Ex. Posix('/home/obijuan/.apio/packages/tools-oss-cad-suite/ - # -- bin/lsusb') - executable_file = system_bin_dir / (command + self.ext) - - # -- Check if the file exist! - if not executable_file.exists(): - - # -- The command was not in the oss-cad-suit package - # -- Print an error message - click.secho("Error!\n", fg="red") - click.secho(f"Command not fount: {executable_file}", fg="red") - - # -- Show the error message and a hint - # -- on how to install the package - pkg_util.show_package_path_error(self.package_name) - pkg_util.show_package_install_instructions(self.package_name) - - # -- Command not executed. - return None - - # -- The command exist! Let's execute it! + if platform.system() == "Windows": + command = command + ".ext" # -- Set the stdout and stderr callbacks, when executing the command # -- Silent mode (True): No callback @@ -204,7 +150,7 @@ def _run_command( # -- Execute the command! result = util.exec_command( - executable_file, + command, stdout=util.AsyncPipe(on_stdout), stderr=util.AsyncPipe(on_stderr), ) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index ed99b67e..9254f36e 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -9,9 +9,10 @@ # ---- Licence Apache v2 """Utility functions related to apio packages.""" -from typing import List, Callable, Dict +from typing import List, Callable, Dict, Optional from pathlib import Path import os +import sys import platform from dataclasses import dataclass import click @@ -22,21 +23,22 @@ def _add_env_path(path: Path) -> None: - """Prepends given path to env PATH variable. Not checking for dupes.""" - print(f" * Adding path: {path}") + """Prepends to given path to env variable PATH. Does not check for + duplicates.""" + # print(f" * Adding path: {path}") old_val = os.environ["PATH"] new_val = os.pathsep.join([str(path), old_val]) os.environ["PATH"] = new_val def _set_env_var(name: str, value: str) -> None: - """Sets env var to given value.""" - print(f" * Setting env: {name} = {value}") + """Sets the env var with given name to given value.""" + # print(f" * Setting env: {name} = {value}") os.environ[name] = value def _set_oss_cad_package_env(package_path: Path) -> None: - """Sets the environment variasbles for the oss-cad-suite package.""" + """Sets the environment variasbles for using the oss-cad-suite package.""" _add_env_path(package_path / "bin") _add_env_path(package_path / "lib") @@ -49,17 +51,21 @@ def _set_oss_cad_package_env(package_path: Path) -> None: def _set_examples_package_env(_: Path) -> None: - """Sets the environment variasbles for the examples package.""" + """Sets the environment variasbles for using the examples package.""" # -- Nothing to set here. def _set_gtkwave_package_env(package_path: Path) -> None: - """Sets the environment variasbles for the gtkwave package.""" + """Sets the environment variasbles for using the gtkwave package. + Called only on Windows, where this package is available. + """ _add_env_path(package_path / "bin") def _set_drivers_package_env(package_path: Path) -> None: - """Sets the environment variasbles for the drivers package.""" + """Sets the environment variasbles for the drivers package. + Called only on Windows, where this package is available. + """ _add_env_path(package_path / "bin") @@ -69,14 +75,17 @@ class _PackageDesc: # -- Package folder name. E.g. "tools-oss-cad-suite" folder_name: str - # -- True if the package is available for this platform. + # -- True if the package is available for the current platform. platform_match: bool # -- A function to set the env for this package. env_setting_func: Callable[[Path], None] -# -- Package names to package properties. Using lambda as a workaround for -# -- forward reference to the env functions. +# pylint: disable=fixme +# -- TODO: Harmonize this table with the packages.json resource file. +# -- currently they are updated independent, with some overlap. +# -- +# -- A dictionary that maps package names to package entries. _PACKAGES: Dict[str, _PackageDesc] = { "oss-cad-suite": _PackageDesc( folder_name="tools-oss-cad-suite", @@ -102,19 +111,23 @@ class _PackageDesc: def set_env_for_packages() -> None: - """Sets the environment variables for using the packages.""" - # print(f"*** set_env_for_packages()") - # base_path = get_packages_dir() + """Sets the environment variables for using all the that are + available for this platform, even if currently not installed. + """ for package_name, package_desc in _PACKAGES.items(): if package_desc.platform_match: - print(f"*** Setting env for package: {package_name}") + # print(f"*** Setting env for package: {package_name}") package_path = get_package_dir(package_name) package_desc.env_setting_func(package_path) -def check_packages(packages_names: List[str], resources: Resources) -> None: - """Tesks if the given packages have proper versions installed. - Returns True if OK. +def check_required_packages( + packages_names: List[str], resources: Resources +) -> None: + """Checks that the packages whose names are in 'packages_names' are + installed and have a version that meets the requirements. If any error, + it prints an error message and aborts the program with an error status + code. """ profile = Profile() @@ -122,55 +135,64 @@ def check_packages(packages_names: List[str], resources: Resources) -> None: spec_packages = resources.distribution.get("packages") # -- Check packages - check = True for package_name in packages_names: package_desc = _PACKAGES[package_name] if package_desc.platform_match: - version = installed_packages.get(package_name, {}).get( - "version", "" + current_version = installed_packages.get(package_name, {}).get( + "version", None ) spec_version = spec_packages.get(package_name, "") + _check_required_package( + package_name, current_version, spec_version + ) - _bin = get_package_dir(package_name) / "bin" - - # -- Check this package - check &= check_package(package_name, version, spec_version, _bin) - - return check -def check_package( - name: str, version: str, spec_version: str, path: Path -) -> bool: - """Check if the given package is ok - (and can be installed without problems) - * INPUTS: - - name: Package name - - version: Package version - - spec_version: semantic version constraint - - path: path where the binary files of the package are stored +def _check_required_package( + package_name: str, + current_version: Optional[str], + spec_version: str, +) -> None: + """Checks that the package with the given packages is installed and + has a version that meets the requirements. If any error, it prints an + error message and exists with an error code. - * OUTPUT: - - True: Package + 'package_name' - the package name, e.g. 'oss-cad-suite'. + 'current_version' - the version of the install package or None if not + installed. + 'spec_version' - a specification of the required version. """ + # -- Case 1: Package is not installed. + if current_version is None: + click.secho( + f"Error: package '{package_name}' is not installed.", fg="red" + ) + _show_package_install_instructions(package_name) + sys.exit(1) + + # -- Case 2: Version does not match requirmeents. + if not _version_matches(current_version, spec_version): + click.secho( + f"Error: package '{package_name}' version {current_version}" + " does not\n" + f"match the requirement for version {spec_version}.", + fg="red", + ) - # Check package path - if path and not path.is_dir(): - show_package_path_error(name) - show_package_install_instructions(name) - return False - - # Check package version - if not _check_package_version(version, spec_version): - _show_package_version_error(name, version, spec_version) - show_package_install_instructions(name) - return False + _show_package_install_instructions(package_name) + sys.exit(1) - return True + # -- Case 3: The package's directory does not exist. + package_dir = get_package_dir(package_name) + if package_dir and not package_dir.is_dir(): + message = f"Error: package '{package_name}' is installed but missing" + click.secho(message, fg="red") + _show_package_install_instructions(package_name) + sys.exit(1) -def _check_package_version(version: str, spec_version: str) -> bool: - """Check if a given version satisfy the semantic version constraints +def _version_matches(current_version: str, spec_version: str) -> bool: + """Tests if a given version satisfy the semantic version constraints * INPUTS: - version: Package version (Ex. '0.0.9') - spec_version: semantic version constraint (Ex. '>=0.0.1') @@ -184,7 +206,7 @@ def _check_package_version(version: str, spec_version: str) -> bool: # -- Check it! try: - semver = semantic_version.Version(version) + semver = semantic_version.Version(current_version) # -- Incorrect version number except ValueError: @@ -194,86 +216,19 @@ def _check_package_version(version: str, spec_version: str) -> bool: return semver in spec -def _show_package_version_error( - name: str, current_version: str, spec_version: str -): - """Print error message: a package is missing or has a worng version.""" - - if current_version: - message = ( - ( - f"Error: package '{name}' version {current_version} does not\n" - f"match the requirement for version {spec_version}." - ), - ) - - else: - message = f"Error: package '{name}' is missing." - click.secho(message, fg="red") - - -def show_package_path_error(name: str): - """Display an error: package Not installed - * INPUTs: - - name: Package name - """ - - message = f"Error: package '{name}' is not installed" - click.secho(message, fg="red") - - -def show_package_install_instructions(name: str): - """Print the package install instructions - * INPUTs: - - name: Package name - """ - - click.secho(f"Please run:\n apio install {name}", fg="yellow") - - -def get_package_version(name: str, profile: dict) -> str: - """Get the version of a given package - * INPUTs: - - name: Package name - - profile: Dictrionary with the profile information - * OUTPUT: - - The version (Ex. '0.0.9') - """ - - # -- Default version - version = "" - - # -- Check if the package is intalled - if name in profile.packages: - version = profile.packages[name]["version"] - - # -- Return the version - return version - - -def get_package_spec_version(name: str, resources: dict) -> str: - """Get the version restrictions for a given package - * INPUTs: - * name: Package name - * resources: Apio resources object - * OUTPUT: version restrictions for that package - Ex. '>=1.1.0,<1.2.0' - """ - - # -- No restrictions by default - spec_version = "" - - # -- Check that the package is valid - if name in resources.distribution["packages"]: - - # -- Get the package restrictions - spec_version = resources.distribution["packages"][name] +def _show_package_install_instructions(package_name: str): + """Prints hints on how to install a package with a given name.""" - # -- Return the restriction - return spec_version + click.secho( + "Please run:\n" + f" apio install {package_name} --force\n" + "or:\n" + " apio install --all --force", + fg="yellow", + ) -def get_packages_dir() -> Path: +def _get_packages_dir() -> Path: """Return the base directory of apio packages. Packages are installed in the following folder: * Default: $APIO_HOME_DIR/packages @@ -326,6 +281,6 @@ def get_package_dir(package_name: str) -> Path: """ package_folder = _PACKAGES[package_name].folder_name - package_dir = get_packages_dir() / package_folder + package_dir = _get_packages_dir() / package_folder return package_dir diff --git a/apio/resources.py b/apio/resources.py index 22953d9f..bb8a47be 100644 --- a/apio/resources.py +++ b/apio/resources.py @@ -68,9 +68,11 @@ class Resources: def __init__( self, *, platform: str = "", project_dir: Optional[Path] = None ): - project_dir = util.get_project_dir(project_dir) + # -- Maps the optional project_dir option to a path. + self._project_dir: Path = util.get_project_dir(project_dir) - self._project_dir = project_dir + # -- Profile information, from ~/.apio/profile.json + self.profile = Profile() # -- Read the apio packages information self.packages = self._load_resource(PACKAGES_JSON) @@ -236,8 +238,6 @@ def get_packages(self) -> tuple[list, list]: installed_packages = [] notinstalled_packages = [] - profile = Profile() - # -- Go though all the apio packages for package in self.packages: @@ -249,10 +249,10 @@ def get_packages(self) -> tuple[list, list]: } # -- Check if this package is installed - if package in profile.packages: + if package in self.profile.packages: # -- Get the installed version - version = profile.packages[package]["version"] + version = self.profile.packages[package]["version"] # -- Store the version data["version"] = version @@ -266,7 +266,7 @@ def get_packages(self) -> tuple[list, list]: # -- Check the installed packages and update # -- its information - for package in profile.packages: + for package in self.profile.packages: # -- The package is not known! # -- Strange case diff --git a/test/test_end_to_end.py b/test/test_end_to_end.py index 5e69e0fb..fbf488a0 100644 --- a/test/test_end_to_end.py +++ b/test/test_end_to_end.py @@ -61,7 +61,7 @@ def test_end_to_end1(clirunner, validate_cliresult, configenv, offline): # -- Execute "apio install examples@X" result = clirunner.invoke(cmd_install, ["examples@X"]) - assert "Error: Package not found" in result.output + assert "Error: package not found" in result.output # -- Execute "apio install examples@0.0.34" result = clirunner.invoke(cmd_install, ["examples@0.0.34"]) From b736297def9df7ace8861b5bbc9016b3fe8a7e34 Mon Sep 17 00:00:00 2001 From: Zapta Date: Wed, 30 Oct 2024 21:06:07 -0700 Subject: [PATCH 13/51] More cleanup around the usage of the check_required_packages() function. --- apio/commands/graph.py | 23 ----------------------- apio/commands/raw.py | 12 +++++++++--- apio/managers/drivers.py | 9 ++++----- apio/managers/scons.py | 33 +++++++++++++++++++++------------ apio/pkg_util.py | 5 ++--- 5 files changed, 36 insertions(+), 46 deletions(-) diff --git a/apio/commands/graph.py b/apio/commands/graph.py index 3f1faab8..99c66207 100644 --- a/apio/commands/graph.py +++ b/apio/commands/graph.py @@ -8,7 +8,6 @@ """Implementation of 'apio graph' command""" from pathlib import Path -import shutil import click from click.core import Context from apio.managers.scons import SCons @@ -57,38 +56,16 @@ @click.pass_context @options.project_dir_option @options.top_module_option_gen(help="Set the name of the top module to graph.") -@options.force_option_gen(help="Force execution despite no 'dot' command.") @options.verbose_option def cli( ctx: Context, # Options project_dir: Path, - force: bool, verbose: bool, top_module: str, ): """Implements the apio graph command.""" - # -- This program requires a user install graphviz 'dot' available on - # -- the path. Verify it. - dot_path = shutil.which("dot") - if not dot_path: - if force: - # -- Just print a warning and continue. - click.secho( - "Warning: Skipping the check for the 'dot' command.", - fg="yellow", - ) - else: - # -- Print an error message and abort. - click.secho() - click.secho( - "Error: The 'dot' command was not found on the system path.", - fg="red", - ) - click.secho(DOT_HELP, fg="yellow") - ctx.exit(1) - # -- Crete the scons object scons = SCons(project_dir) diff --git a/apio/commands/raw.py b/apio/commands/raw.py index f6cb826e..22d41f80 100644 --- a/apio/commands/raw.py +++ b/apio/commands/raw.py @@ -9,9 +9,8 @@ import click from click.core import Context -from apio import util -from apio import pkg_util -from apio import cmd_util +from apio import util, pkg_util, cmd_util +from apio.resources import Resources # --------------------------- @@ -51,6 +50,13 @@ def cli( """Implements the apio raw command which executes user specified commands from apio installed tools. """ + + # -- Make sure the oss-cad-suite is installed. + resources = Resources() + pkg_util.check_required_packages(["oss-cad-suite"], resources) + + # -- Set the system env for using the packages. pkg_util.set_env_for_packages() + exit_code = util.call(cmd) ctx.exit(exit_code) diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index 67bf9443..9e7a658b 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -391,9 +391,7 @@ def _check_ftdi_driver_darwin(self, driver): # W0703: Catching too general exception Exception (broad-except) # pylint: disable=W0703 def _ftdi_enable_windows(self) -> int: - pkg_util.check_required_packages(["drivers"], self.resources) - - # -- + # -- Check that the required packages are installed. pkg_util.check_required_packages(["drivers"], self.resources) # -- Get the drivers apio package base folder @@ -442,6 +440,7 @@ def _ftdi_enable_windows(self) -> int: return result.exit_code def _ftdi_disable_windows(self) -> int: + # -- Check that the required packages exist. pkg_util.check_required_packages(["drivers"], self.resources) click.secho("Launch device manager") @@ -453,13 +452,12 @@ def _ftdi_disable_windows(self) -> int: # W0703: Catching too general exception Exception (broad-except) # pylint: disable=W0703 def _serial_enable_windows(self) -> int: + # -- Check that the required packages exist. pkg_util.check_required_packages(["drivers"], self.resources) drivers_base_dir = pkg_util.get_package_dir("drivers") drivers_bin_dir = drivers_base_dir / "bin" - pkg_util.check_required_packages(["drivers"], self.resources) - try: click.secho("Launch drivers configuration tool") @@ -476,6 +474,7 @@ def _serial_enable_windows(self) -> int: return result.exit_code def _serial_disable_windows(self) -> int: + # -- Check that the required packages exist. pkg_util.check_required_packages(["drivers"], self.resources) click.secho("Launch device manager") diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 62ce304c..4a72eed8 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -112,7 +112,7 @@ def clean(self, args) -> int: # --Clean the project: run scons -c (with aditional arguments) return self._run( - "-c", arch=arch, variables=variables, packages_names=[] + "-c", arch=arch, variables=variables, required_packages_names=[] ) @on_exception(exit_code=1) @@ -131,7 +131,7 @@ def verify(self, args) -> int: "verify", variables=variables, arch=arch, - packages_names=["oss-cad-suite"], + required_packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -150,7 +150,7 @@ def graph(self, args) -> int: "graph", variables=variables, arch=arch, - packages_names=["oss-cad-suite"], + required_packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -173,7 +173,7 @@ def lint(self, args) -> int: "lint", variables=variables, arch=arch, - packages_names=["oss-cad-suite"], + required_packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -190,7 +190,7 @@ def sim(self, args) -> int: "sim", variables=variables, arch=arch, - packages_names=["oss-cad-suite", "gtkwave"], + required_packages_names=["oss-cad-suite", "gtkwave"], ) @on_exception(exit_code=1) @@ -207,7 +207,7 @@ def test(self, args) -> int: "test", variables=variables, arch=arch, - packages_names=["oss-cad-suite"], + required_packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -227,7 +227,7 @@ def build(self, args) -> int: variables=variables, board=board, arch=arch, - packages_names=["oss-cad-suite"], + required_packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -252,7 +252,7 @@ def time(self, args) -> int: variables=variables, board=board, arch=arch, - packages_names=["oss-cad-suite"], + required_packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -269,7 +269,7 @@ def report(self, args) -> int: variables=variables, board=board, arch=arch, - packages_names=["oss-cad-suite"], + required_packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) @@ -310,7 +310,7 @@ def upload(self, config: dict, prog: dict) -> int: exit_code = self._run( "upload", variables=flags, - packages_names=["oss-cad-suite"], + required_packages_names=["oss-cad-suite"], board=board, arch=arch, ) @@ -962,7 +962,14 @@ def _check_ftdi( # pylint: disable=too-many-arguments # pylint: disable=too-many-positional-arguments - def _run(self, command, variables, packages_names, board=None, arch=None): + def _run( + self, + command, + variables, + required_packages_names, + board=None, + arch=None, + ): """Executes scons""" # -- Construct the path to the SConstruct file. @@ -985,7 +992,9 @@ def _run(self, command, variables, packages_names, board=None, arch=None): else: # Run on `default` config mode # -- Check that the required packages are installed - pkg_util.check_required_packages(packages_names, self.resources) + pkg_util.check_required_packages( + required_packages_names, self.resources + ) pkg_util.set_env_for_packages() diff --git a/apio/pkg_util.py b/apio/pkg_util.py index 9254f36e..e791857c 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -122,7 +122,7 @@ def set_env_for_packages() -> None: def check_required_packages( - packages_names: List[str], resources: Resources + required_packages_names: List[str], resources: Resources ) -> None: """Checks that the packages whose names are in 'packages_names' are installed and have a version that meets the requirements. If any error, @@ -135,7 +135,7 @@ def check_required_packages( spec_packages = resources.distribution.get("packages") # -- Check packages - for package_name in packages_names: + for package_name in required_packages_names: package_desc = _PACKAGES[package_name] if package_desc.platform_match: current_version = installed_packages.get(package_name, {}).get( @@ -147,7 +147,6 @@ def check_required_packages( ) - def _check_required_package( package_name: str, current_version: Optional[str], From 3db638d90e31580b9724d2d9ed5978c441a8556e Mon Sep 17 00:00:00 2001 From: Zapta Date: Wed, 30 Oct 2024 21:13:55 -0700 Subject: [PATCH 14/51] Removed the -p alias from the --platform option to avoid a conflict with the --project-dir option. --- apio/commands/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apio/commands/options.py b/apio/commands/options.py index 90957589..3961c972 100644 --- a/apio/commands/options.py +++ b/apio/commands/options.py @@ -197,9 +197,9 @@ def pack_option_gen( ) +# -- NOTE: Not using -p to avoid conflict with --project-dir. platform_option = click.option( "platform", # Var name. - "-p", "--platform", type=click.Choice(util.PLATFORMS), help=("(Advanced, for developers) Set the platform."), From 702ef04e34b4397e39ead0f7e03760b9ecf211be Mon Sep 17 00:00:00 2001 From: Zapta Date: Wed, 30 Oct 2024 21:35:04 -0700 Subject: [PATCH 15/51] Changed the selection of the constraint file. It's now a fatal error to have more than one (previously selected the first one alphabetically). --- .gitignore | 1 + apio/scons/scons_util.py | 11 ++++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 89fa0429..a12ebc20 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.swp *~ .coverage +.coverage.* .tox/ .cache/ .pytest_cache/ diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index 45f65dd6..fa3499f6 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -226,14 +226,11 @@ def get_constraint_file( # Case 2: Exactly one file found. if n == 1: result = str(files[0]) - info(env, f"Found constraint file '{result}'.") return result - # Case 3: Multiple matching files. Pick the first file (alphabetically). - # We could improve the heuristic here, e.g. to prefer a file with - # the top_module name, if exists. - result = str(files[0]) - warning(env, f"Found multiple {file_ext} files, using '{result}'.") - return result + # Case 3: Multiple matching files. + fatal_error( + env, f"Found multiple '*{file_ext}' constrain files, expecting one." + ) def dump_env_vars(env: SConsEnvironment) -> None: From aaa45d4f4184913b1acd5e1ec43bc15586ffc666 Mon Sep 17 00:00:00 2001 From: Zapta Date: Thu, 31 Oct 2024 11:46:00 -0700 Subject: [PATCH 16/51] Removed the PATH of libexec. It caused problems on windows. --- apio/pkg_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index e791857c..23e28fbb 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -42,7 +42,6 @@ def _set_oss_cad_package_env(package_path: Path) -> None: _add_env_path(package_path / "bin") _add_env_path(package_path / "lib") - _add_env_path(package_path / "libexec") _set_env_var("IVL", str(package_path / "lib" / "ivl")) _set_env_var("ICEBOX", str(package_path / "share" / "icebox")) From 8ab8bbaadb6a40683d709c09dad3611fce6e3d5e Mon Sep 17 00:00:00 2001 From: Zapta Date: Thu, 31 Oct 2024 14:05:22 -0700 Subject: [PATCH 17/51] Changed the packages env settings to preserve package order. --- apio/pkg_util.py | 122 +++++++++++++++++++++++++++++++---------------- 1 file changed, 80 insertions(+), 42 deletions(-) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index 23e28fbb..23371549 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -11,6 +11,7 @@ from typing import List, Callable, Dict, Optional from pathlib import Path +from dataclasses import dataclass import os import sys import platform @@ -22,50 +23,60 @@ from apio.resources import Resources -def _add_env_path(path: Path) -> None: - """Prepends to given path to env variable PATH. Does not check for - duplicates.""" - # print(f" * Adding path: {path}") - old_val = os.environ["PATH"] - new_val = os.pathsep.join([str(path), old_val]) - os.environ["PATH"] = new_val +@dataclass(frozen=True) +class EnvMutations: + """Contains mutations to the system env.""" + # -- PATH items to add. + paths: List[str] + # -- Vars name/value paris. + vars: Dict[str, str] -def _set_env_var(name: str, value: str) -> None: - """Sets the env var with given name to given value.""" - # print(f" * Setting env: {name} = {value}") - os.environ[name] = value -def _set_oss_cad_package_env(package_path: Path) -> None: - """Sets the environment variasbles for using the oss-cad-suite package.""" - _add_env_path(package_path / "bin") - _add_env_path(package_path / "lib") - _set_env_var("IVL", str(package_path / "lib" / "ivl")) - _set_env_var("ICEBOX", str(package_path / "share" / "icebox")) - _set_env_var("TRELLIS", str(package_path / "share" / "trellis")) - _set_env_var("YOSYS_LIB", str(package_path / "share" / "yosys")) +def _oss_cad_suite_package_env(package_path: Path) -> EnvMutations: + """Returns the env mutations for the oss-cad-suite package.""" + return EnvMutations( + paths=[ + str(package_path / "bin"), + str(package_path / "lib"), + ], + vars=[ + ("IVL", str(package_path / "lib" / "ivl")), + ("ICEBOX", str(package_path / "share" / "icebox")), + ("TRELLIS", str(package_path / "share" / "trellis")), + ("YOSYS_LIB", str(package_path / "share" / "yosys")), + ], + ) -def _set_examples_package_env(_: Path) -> None: - """Sets the environment variasbles for using the examples package.""" - # -- Nothing to set here. +def _examples_package_env(_: Path) -> None: + """Returns the env mutations for the examples package.""" + return EnvMutations( + paths=[], + vars=[], + ) -def _set_gtkwave_package_env(package_path: Path) -> None: - """Sets the environment variasbles for using the gtkwave package. - Called only on Windows, where this package is available. - """ - _add_env_path(package_path / "bin") +def _gtkwave_package_env(package_path: Path) -> None: + """Returns the env mutations for the gtkwave package.""" -def _set_drivers_package_env(package_path: Path) -> None: - """Sets the environment variasbles for the drivers package. - Called only on Windows, where this package is available. - """ - _add_env_path(package_path / "bin") + return EnvMutations( + paths=[str(package_path / "bin")], + vars=[], + ) + + +def _drivers_package_env(package_path: Path) -> None: + """Returns the env mutations for the drivers package.""" + + return EnvMutations( + paths=[str(package_path / "bin")], + vars=[], + ) @dataclass(frozen=True) @@ -77,7 +88,7 @@ class _PackageDesc: # -- True if the package is available for the current platform. platform_match: bool # -- A function to set the env for this package. - env_setting_func: Callable[[Path], None] + env_func: Callable[[Path], EnvMutations] # pylint: disable=fixme @@ -85,39 +96,66 @@ class _PackageDesc: # -- currently they are updated independent, with some overlap. # -- # -- A dictionary that maps package names to package entries. +# -- The order determines the order of their respective paths in the +# -- system env PATH variable, and it may matter. _PACKAGES: Dict[str, _PackageDesc] = { "oss-cad-suite": _PackageDesc( folder_name="tools-oss-cad-suite", platform_match=True, - env_setting_func=_set_oss_cad_package_env, + env_func=_oss_cad_suite_package_env, ), "examples": _PackageDesc( folder_name="examples", platform_match=True, - env_setting_func=_set_examples_package_env, + env_func=_examples_package_env, ), "gtkwave": _PackageDesc( folder_name="tool-gtkwave", platform_match=platform.system() == "Windows", - env_setting_func=_set_gtkwave_package_env, + env_func=_gtkwave_package_env, ), "drivers": _PackageDesc( folder_name="tools-drivers", platform_match=platform.system() == "Windows", - env_setting_func=_set_drivers_package_env, + env_func=_drivers_package_env, ), } +def _get_env_mutations_for_packages() -> EnvMutations: + """Collects the env mutation for each of the defined packages, + in the order they are defined.""" + result = EnvMutations([], []) + for package_name, package_desc in _PACKAGES.items(): + if package_desc.platform_match: + package_path = get_package_dir(package_name) + mutations = package_desc.env_func(package_path) + result.paths.extend(mutations.paths) + result.vars.extend(mutations.vars) + return result + + +def _apply_env_mutations(mutations: EnvMutations) -> None: + """Apply a given set of env mutations, while preserving their order.""" + + # -- Apply the path mutations, while preserving order. + old_val = os.environ["PATH"] + items = mutations.paths + [old_val] + new_val = os.pathsep.join(items) + os.environ["PATH"] = new_val + + # -- Apply the vars mutations, while preserving order. + for name, value in mutations.vars: + os.environ[name] = value + + def set_env_for_packages() -> None: """Sets the environment variables for using all the that are available for this platform, even if currently not installed. """ - for package_name, package_desc in _PACKAGES.items(): - if package_desc.platform_match: - # print(f"*** Setting env for package: {package_name}") - package_path = get_package_dir(package_name) - package_desc.env_setting_func(package_path) + mutations = _get_env_mutations_for_packages() + # print(f"{mutations = }") + _apply_env_mutations(mutations) def check_required_packages( From 11b04539feb4e53da72169fb5f5f916632bdfe90 Mon Sep 17 00:00:00 2001 From: Zapta Date: Thu, 31 Oct 2024 17:31:49 -0700 Subject: [PATCH 18/51] Eliminated the gtkwave package, added the graphviz packages (both are windows only packages). --- apio/managers/scons.py | 4 ++-- apio/pkg_util.py | 18 +++++++----------- apio/resources/distribution.json | 2 +- apio/resources/packages.json | 18 +++++++++--------- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 4a72eed8..72469803 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -150,7 +150,7 @@ def graph(self, args) -> int: "graph", variables=variables, arch=arch, - required_packages_names=["oss-cad-suite"], + required_packages_names=["oss-cad-suite", "graphviz"], ) @on_exception(exit_code=1) @@ -190,7 +190,7 @@ def sim(self, args) -> int: "sim", variables=variables, arch=arch, - required_packages_names=["oss-cad-suite", "gtkwave"], + required_packages_names=["oss-cad-suite"], ) @on_exception(exit_code=1) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index 23371549..e78ba259 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -9,13 +9,12 @@ # ---- Licence Apache v2 """Utility functions related to apio packages.""" -from typing import List, Callable, Dict, Optional +from typing import List, Callable, Dict, Tuple, Optional from pathlib import Path from dataclasses import dataclass import os import sys import platform -from dataclasses import dataclass import click import semantic_version from apio import util @@ -30,10 +29,7 @@ class EnvMutations: # -- PATH items to add. paths: List[str] # -- Vars name/value paris. - vars: Dict[str, str] - - - + vars: List[Tuple[str, str]] def _oss_cad_suite_package_env(package_path: Path) -> EnvMutations: @@ -61,8 +57,8 @@ def _examples_package_env(_: Path) -> None: ) -def _gtkwave_package_env(package_path: Path) -> None: - """Returns the env mutations for the gtkwave package.""" +def _graphviz_package_env(package_path: Path) -> None: + """Returns the env mutations for the graphviz package.""" return EnvMutations( paths=[str(package_path / "bin")], @@ -109,10 +105,10 @@ class _PackageDesc: platform_match=True, env_func=_examples_package_env, ), - "gtkwave": _PackageDesc( - folder_name="tool-gtkwave", + "graphviz": _PackageDesc( + folder_name="tool-graphviz", platform_match=platform.system() == "Windows", - env_func=_gtkwave_package_env, + env_func=_graphviz_package_env, ), "drivers": _PackageDesc( folder_name="tools-drivers", diff --git a/apio/resources/distribution.json b/apio/resources/distribution.json index c1abdf9c..f600505b 100644 --- a/apio/resources/distribution.json +++ b/apio/resources/distribution.json @@ -2,7 +2,7 @@ "packages": { "drivers": ">=1.1.0", "examples": ">=0.0.7", - "gtkwave": ">=3.3.77,<3.4.0", + "graphviz": ">=12.1.2", "oss-cad-suite": ">=0.0.1" }, "pip_packages": { diff --git a/apio/resources/packages.json b/apio/resources/packages.json index 5d4221d5..91f36300 100644 --- a/apio/resources/packages.json +++ b/apio/resources/packages.json @@ -18,7 +18,7 @@ "oss-cad-suite": { "repository": { "name": "tools-oss-cad-suite", - "organization": "FPGAwars" + "organization": "zapta" }, "release": { "tag_name": "v%V", @@ -26,30 +26,30 @@ "uncompressed_name": "", "folder_name": "tools-oss-cad-suite", "extension": "tar.gz", - "url_version": "https://github.com/FPGAwars/tools-oss-cad-suite/raw/main/VERSION_DEV" + "url_version": "https://github.com/zapta/tools-oss-cad-suite/raw/main/VERSION_DEV" }, "description": "YosysHQ/oss-cad-suite" }, - "gtkwave": { + "graphviz": { "repository": { - "name": "tool-gtkwave", - "organization": "FPGAwars" + "name": "tool-graphviz", + "organization": "zapta" }, "release": { "tag_name": "v%V", - "compressed_name": "tool-gtkwave-%P-%V", + "compressed_name": "tool-graphviz-%P-%V", "uncompressed_name": "", - "folder_name": "tool-gtkwave", + "folder_name": "tool-graphviz", "extension": "tar.gz", - "url_version": "https://github.com/FPGAwars/tool-gtkwave/raw/master/VERSION", + "url_version": "https://github.com/zapta/tool-graphviz/raw/master/VERSION", "available_platforms": [ "windows", "windows_x86", "windows_amd64" ] }, - "description": "GTKWave tool for Windows" + "description": "Graphviz tool for Windows" }, "drivers": { From e650bdbdf8110b38861e449c797cb0e8a45e5822 Mon Sep 17 00:00:00 2001 From: Zapta Date: Thu, 31 Oct 2024 18:52:34 -0700 Subject: [PATCH 19/51] Updated graphviz package url version. --- apio/resources/packages.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apio/resources/packages.json b/apio/resources/packages.json index 91f36300..1eb9428f 100644 --- a/apio/resources/packages.json +++ b/apio/resources/packages.json @@ -42,7 +42,7 @@ "uncompressed_name": "", "folder_name": "tool-graphviz", "extension": "tar.gz", - "url_version": "https://github.com/zapta/tool-graphviz/raw/master/VERSION", + "url_version": "https://github.com/zapta/tool-graphviz/raw/main/VERSION", "available_platforms": [ "windows", "windows_x86", From f24308bb516869af6d6a2076a6d3b1320d889083 Mon Sep 17 00:00:00 2001 From: Zapta Date: Fri, 1 Nov 2024 11:21:41 -0700 Subject: [PATCH 20/51] Fixed windows binary extension from '.ext' to '.exe'. --- apio/managers/system.py | 4 +++- apio/pkg_util.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/apio/managers/system.py b/apio/managers/system.py index 887ee48b..70917709 100644 --- a/apio/managers/system.py +++ b/apio/managers/system.py @@ -140,8 +140,10 @@ def _run_command( # -- Set system env for using the packages. pkg_util.set_env_for_packages() + # TODO: Is this necessary or does windows accepts commands without + # the '.exe' extension? if platform.system() == "Windows": - command = command + ".ext" + command = command + ".exe" # -- Set the stdout and stderr callbacks, when executing the command # -- Silent mode (True): No callback diff --git a/apio/pkg_util.py b/apio/pkg_util.py index e78ba259..9809ad70 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -131,6 +131,18 @@ def _get_env_mutations_for_packages() -> EnvMutations: return result +# def _dump_env_mutations_for_batch(mutations: EnvMutations) -> None: +# """For debugging. Delete once stabalizing the new oss-cad-suite on +# windows.""" +# print("--- mutations:") +# for p in reversed(mutations.paths): +# print(f"@set PATH={p};%PATH%") +# print() +# for name, val in mutations.vars: +# print(f"@set {name}={val}") +# print("---") + + def _apply_env_mutations(mutations: EnvMutations) -> None: """Apply a given set of env mutations, while preserving their order.""" @@ -150,6 +162,9 @@ def set_env_for_packages() -> None: available for this platform, even if currently not installed. """ mutations = _get_env_mutations_for_packages() + + # _dump_env_mutations_for_batch(mutations) + # print(f"{mutations = }") _apply_env_mutations(mutations) From a715cd3f3b24db56605d168313374810fcdafc90 Mon Sep 17 00:00:00 2001 From: Zapta Date: Fri, 1 Nov 2024 11:48:04 -0700 Subject: [PATCH 21/51] Minor message and comment tweaking. --- apio/managers/scons.py | 2 +- apio/resources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 72469803..3cf1634b 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -79,7 +79,7 @@ class SCons: def __init__(self, project_dir: Path): """Initialization: * project_dir: path where the sources are located - If not given, the curent working dir is used + If not given, the current working dir is used """ # -- Read the project file (apio.ini) diff --git a/apio/resources.py b/apio/resources.py index bb8a47be..09f3a058 100644 --- a/apio/resources.py +++ b/apio/resources.py @@ -127,7 +127,7 @@ def _load_resource(self, name: str, allow_custom: bool = False) -> dict: if filepath.exists(): if allow_custom: - click.secho(f"Using project's custom '{name}' file.") + click.secho(f"Loading project's custom '{name}' file.") return self._load_resource_file(filepath) # -- Load the stock resource file from the APIO package. From 7847b0ba1074218f38df34a53b5858748c9f7cb5 Mon Sep 17 00:00:00 2001 From: Zapta Date: Fri, 1 Nov 2024 12:58:28 -0700 Subject: [PATCH 22/51] This change eliminate a double loading of Resource and a double message to the user in the upload command. Resources is now created once early in the scons command and is propagated from there. --- .gitignore | 2 ++ apio/commands/build.py | 4 +++- apio/commands/clean.py | 4 +++- apio/commands/graph.py | 4 +++- apio/commands/lint.py | 4 +++- apio/commands/report.py | 4 +++- apio/commands/sim.py | 5 ++++- apio/commands/test.py | 4 +++- apio/commands/time.py | 4 +++- apio/commands/upload.py | 4 ++-- apio/commands/verify.py | 4 +++- apio/managers/project.py | 4 +--- apio/managers/scons.py | 27 +++++++-------------------- apio/managers/system.py | 1 + apio/pkg_util.py | 6 +++--- apio/resources.py | 4 ++-- apio/util.py | 4 +++- 17 files changed, 49 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index a12ebc20..01df3227 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ hardware.vlt hardware.bit hardware.config hardware.pnr +hardware.dot +hardware.svg *.out *.vcd .DS_Store diff --git a/apio/commands/build.py b/apio/commands/build.py index c505950f..4f7b9b3a 100644 --- a/apio/commands/build.py +++ b/apio/commands/build.py @@ -13,6 +13,7 @@ from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -75,7 +76,8 @@ def cli( # https://www.scons.org/documentation.html # -- Create the scons object - scons = SCons(project_dir) + resources = Resources(project_dir=project_dir) + scons = SCons(resources) # R0801: Similar lines in 2 files # pylint: disable=R0801 diff --git a/apio/commands/clean.py b/apio/commands/clean.py index 2e66ec5a..2caa9ceb 100644 --- a/apio/commands/clean.py +++ b/apio/commands/clean.py @@ -13,6 +13,7 @@ from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -56,7 +57,8 @@ def cli( """ # -- Create the scons object - scons = SCons(project_dir) + resources = Resources(project_dir=project_dir) + scons = SCons(resources) # -- Build the project with the given parameters exit_code = scons.clean({"board": board, "verbose": {"all": verbose}}) diff --git a/apio/commands/graph.py b/apio/commands/graph.py index 99c66207..e6a225f9 100644 --- a/apio/commands/graph.py +++ b/apio/commands/graph.py @@ -13,6 +13,7 @@ from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -67,7 +68,8 @@ def cli( """Implements the apio graph command.""" # -- Crete the scons object - scons = SCons(project_dir) + resources = Resources(project_dir=project_dir) + scons = SCons(resources) # -- Graph the project with the given parameters exit_code = scons.graph( diff --git a/apio/commands/lint.py b/apio/commands/lint.py index d6787694..764d0317 100644 --- a/apio/commands/lint.py +++ b/apio/commands/lint.py @@ -13,6 +13,7 @@ from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -97,7 +98,8 @@ def cli( """Lint the verilog code.""" # -- Create the scons object - scons = SCons(project_dir) + resources = Resources(project_dir=project_dir) + scons = SCons(resources) # -- Lint the project with the given parameters exit_code = scons.lint( diff --git a/apio/commands/report.py b/apio/commands/report.py index d96cedb4..86d8b241 100644 --- a/apio/commands/report.py +++ b/apio/commands/report.py @@ -13,6 +13,7 @@ from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -66,7 +67,8 @@ def cli( """Analyze the design and report timing.""" # -- Create the scons object - scons = SCons(project_dir) + resources = Resources(project_dir=project_dir) + scons = SCons(resources) # Run scons exit_code = scons.report( diff --git a/apio/commands/sim.py b/apio/commands/sim.py index 67d15232..2f016009 100644 --- a/apio/commands/sim.py +++ b/apio/commands/sim.py @@ -12,6 +12,8 @@ from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options +from apio.resources import Resources + # --------------------------- # -- COMMAND @@ -59,7 +61,8 @@ def cli( """ # -- Create the scons object - scons = SCons(project_dir) + resources = Resources(project_dir=project_dir) + scons = SCons(resources) # -- Simulate the project with the given parameters exit_code = scons.sim({"testbench": testbench}) diff --git a/apio/commands/test.py b/apio/commands/test.py index f4b35c17..8f2ce4fa 100644 --- a/apio/commands/test.py +++ b/apio/commands/test.py @@ -13,6 +13,7 @@ from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -59,7 +60,8 @@ def cli( """Implements the test command.""" # -- Create the scons object - scons = SCons(project_dir) + resources = Resources(project_dir=project_dir) + scons = SCons(resources) exit_code = scons.test({"testbench": testbench_file}) ctx.exit(exit_code) diff --git a/apio/commands/time.py b/apio/commands/time.py index 6c6542b7..997746b0 100644 --- a/apio/commands/time.py +++ b/apio/commands/time.py @@ -13,6 +13,7 @@ from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -62,7 +63,8 @@ def cli( """Analyze the design and report timing.""" # -- Create the scons object - scons = SCons(project_dir) + resources = Resources(project_dir=project_dir) + scons = SCons(resources) # Run scons exit_code = scons.time( diff --git a/apio/commands/upload.py b/apio/commands/upload.py index 28847443..3f2d8ab4 100644 --- a/apio/commands/upload.py +++ b/apio/commands/upload.py @@ -93,7 +93,7 @@ def cli( """Implements the upload command.""" # -- Create a drivers object - resources = Resources() + resources = Resources(project_dir=project_dir) drivers = Drivers(resources) # -- Only for MAC @@ -101,7 +101,7 @@ def cli( drivers.pre_upload() # -- Create the SCons object - scons = SCons(project_dir) + scons = SCons(resources) # -- Construct the configuration params to pass to SCons # -- from the arguments diff --git a/apio/commands/verify.py b/apio/commands/verify.py index f164dbdf..2a7598ad 100644 --- a/apio/commands/verify.py +++ b/apio/commands/verify.py @@ -13,6 +13,7 @@ from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -55,7 +56,8 @@ def cli( """Implements the verify command.""" # -- Crete the scons object - scons = SCons(project_dir) + resources = Resources(project_dir=project_dir) + scons = SCons(resources) # -- Verify the project with the given parameters exit_code = scons.verify( diff --git a/apio/managers/project.py b/apio/managers/project.py index 78ce76b7..2ba3543f 100644 --- a/apio/managers/project.py +++ b/apio/managers/project.py @@ -31,9 +31,7 @@ class Project: """Class for managing apio projects""" - def __init__(self, project_dir: Path): - # pylint: disable=fixme - # TODO: Make these __private and provide getter methods. + def __init__(self, project_dir: Optional[Path]): self.project_dir = util.get_project_dir(project_dir) self.board: str = None self.top_module: str = None diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 3cf1634b..04febc55 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -14,7 +14,6 @@ import time import datetime import shutil -from pathlib import Path from functools import wraps import importlib.metadata @@ -76,29 +75,17 @@ def wrapper(*args, **kwargs): class SCons: """Class for managing the scons tools""" - def __init__(self, project_dir: Path): - """Initialization: - * project_dir: path where the sources are located - If not given, the current working dir is used - """ + def __init__(self, resources: Resources): + """Initialization.""" + # -- Cache resources. + self.resources = resources # -- Read the project file (apio.ini) - self.project = Project(project_dir) + self.project = Project(resources.project_dir) self.project.read() - # -- Read the apio resources - self.resources = Resources(project_dir=project_dir) - - # -- Project path is given - if project_dir: - # Check if it is a correct folder - # (or create a new one) - project_dir = util.get_project_dir( - project_dir, create_if_missing=False - ) - - # Change to that folder - os.chdir(project_dir) + # -- Change to the project's folder. + os.chdir(resources.project_dir) @on_exception(exit_code=1) def clean(self, args) -> int: diff --git a/apio/managers/system.py b/apio/managers/system.py index 70917709..0efe704d 100644 --- a/apio/managers/system.py +++ b/apio/managers/system.py @@ -140,6 +140,7 @@ def _run_command( # -- Set system env for using the packages. pkg_util.set_env_for_packages() + # pylint: disable=fixme # TODO: Is this necessary or does windows accepts commands without # the '.exe' extension? if platform.system() == "Windows": diff --git a/apio/pkg_util.py b/apio/pkg_util.py index 9809ad70..910b5ffe 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -162,10 +162,10 @@ def set_env_for_packages() -> None: available for this platform, even if currently not installed. """ mutations = _get_env_mutations_for_packages() - + + # -- For debugging. # _dump_env_mutations_for_batch(mutations) - - # print(f"{mutations = }") + _apply_env_mutations(mutations) diff --git a/apio/resources.py b/apio/resources.py index 09f3a058..dd8e95d5 100644 --- a/apio/resources.py +++ b/apio/resources.py @@ -69,7 +69,7 @@ def __init__( self, *, platform: str = "", project_dir: Optional[Path] = None ): # -- Maps the optional project_dir option to a path. - self._project_dir: Path = util.get_project_dir(project_dir) + self.project_dir: Path = util.get_project_dir(project_dir) # -- Profile information, from ~/.apio/profile.json self.profile = Profile() @@ -123,7 +123,7 @@ def _load_resource(self, name: str, allow_custom: bool = False) -> dict: In case of error it raises an exception and finish """ # -- Try loading a custom resource file from the project directory. - filepath = self._project_dir / name + filepath = self.project_dir / name if filepath.exists(): if allow_custom: diff --git a/apio/util.py b/apio/util.py index 0caa8f05..dfa20b78 100644 --- a/apio/util.py +++ b/apio/util.py @@ -413,7 +413,9 @@ def print_exception_developers(e): click.secho(f"{e}\n", fg="yellow") -def get_project_dir(_dir: Path, create_if_missing: bool = False) -> Path: +def get_project_dir( + _dir: Optional[Path], create_if_missing: bool = False +) -> Path: """Check if the given path is a folder. It it does not exists and create_if_missing is true, folder is created, otherwise a fatal error. If no path is given the current working directory is used. From 9c1b92c8498247ae52769c1d2550e16e7eb701cf Mon Sep 17 00:00:00 2001 From: Zapta Date: Fri, 1 Nov 2024 18:25:50 -0700 Subject: [PATCH 23/51] Added an abstractions of is_linux(), is_darwin() and is_windows. --- apio/commands/system.py | 4 ++-- apio/managers/drivers.py | 43 ++++++++++++++++---------------------- apio/managers/installer.py | 2 +- apio/managers/scons.py | 4 ++-- apio/managers/system.py | 9 ++++---- apio/pkg_util.py | 11 +++++----- apio/resources.py | 2 +- apio/scons/scons_util.py | 4 ++-- apio/util.py | 17 ++++++++++++++- 9 files changed, 51 insertions(+), 45 deletions(-) diff --git a/apio/commands/system.py b/apio/commands/system.py index 9c016315..978ef21b 100644 --- a/apio/commands/system.py +++ b/apio/commands/system.py @@ -13,7 +13,7 @@ from click.core import Context from apio import util from apio import cmd_util -from apio.util import get_systype +from apio.util import get_system_type from apio.managers.system import System from apio.resources import Resources from apio.commands import options @@ -128,7 +128,7 @@ def cli( if info: # -- Print platform id. click.secho("Platform: ", nl=False) - click.secho(get_systype(), fg="yellow") + click.secho(get_system_type(), fg="yellow") # -- Print apio package directory. click.secho("Package: ", nl=False) diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index 9e7a658b..5319655f 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -79,10 +79,6 @@ class Drivers: def __init__(self, resources: Resources) -> None: - # -- Get the platform (a string) - self.platform = util.get_systype() - - # self.profile = Profile() self.resources = resources def ftdi_enable(self) -> int: @@ -90,35 +86,32 @@ def ftdi_enable(self) -> int: Returns a process exit code. """ - if "linux" in self.platform: + if util.is_linux(): return self._ftdi_enable_linux() - if "darwin" in self.platform: + if util.is_darwin(): return self._ftdi_enable_darwin() - if "windows" in self.platform: + if util.is_windows(): return self._ftdi_enable_windows() - click.secho("Error: unknown platform '{self.platform}'.") + click.secho(f"Error: unknown platform '{util.get_system_type()}'.") return 1 def ftdi_disable(self) -> int: """Disables the FTDI driver. Function is platform dependent. Returns a process exit code. """ - - if "linux" in self.platform: + if util.is_linux(): return self._ftdi_disable_linux() - if "darwin" in self.platform: - # self._setup_darwin() + if util.is_darwin(): return self._ftdi_disable_darwin() - if "windows" in self.platform: - # self._setup_windows() + if util.is_windows(): return self._ftdi_disable_windows() - click.secho("Error: unknown platform '{self.platform}'.") + click.secho(f"Error: unknown platform '{util.get_system_type()}'.") return 1 def serial_enable(self) -> int: @@ -126,46 +119,46 @@ def serial_enable(self) -> int: Returns a process exit code. """ - if "linux" in self.platform: + if util.is_linux(): return self._serial_enable_linux() - if "darwin" in self.platform: + if util.is_darwin(): return self._serial_enable_darwin() - if "windows" in self.platform: + if util.is_windows(): return self._serial_enable_windows() - click.secho("Error: unknown platform '{self.platform}'.") + click.secho(f"Error: unknown platform '{util.get_system_type()}'.") return 1 def serial_disable(self) -> int: """Disables the serial driver. Function is platform dependent. Returns a process exit code. """ - if "linux" in self.platform: + if util.is_linux(): return self._serial_disable_linux() - if "darwin" in self.platform: + if util.is_darwin(): return self._serial_disable_darwin() - if "windows" in self.platform: + if util.is_windows(): return self._serial_disable_windows() - click.secho("Error: unknown platform '{self.platform}'.") + click.secho(f"Error: unknown platform '{util.get_system_type()}'.") return 1 def pre_upload(self): """Operations to do before uploading a design Only for mac platforms""" - if "darwin" in self.platform: + if util.is_darwin(): self._pre_upload_darwin() def post_upload(self): """Operations to do after uploading a design Only for mac platforms""" - if "darwin" in self.platform: + if util.is_darwin(): self._post_upload_darwin() def _ftdi_enable_linux(self) -> int: diff --git a/apio/managers/installer.py b/apio/managers/installer.py index 46e8f99c..102b2062 100644 --- a/apio/managers/installer.py +++ b/apio/managers/installer.py @@ -115,7 +115,7 @@ def __init__( self.extension = data["release"]["extension"] # Get the current platform (if not forced by the user) - platform = platform or util.get_systype() + platform = platform or util.get_system_type() # Check if the version is ok (It is only done if the # checkversion flag has been activated) diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 04febc55..3be3baf2 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -351,7 +351,7 @@ def _get_programmer(self, board: str, prog: dict) -> str: if "tinyprog" in board_data: # -- Get the platform - platform = util.get_systype() + platform = util.get_system_type() # -- darwin / darwin_arm64 platforms if "darwin" in platform or "darwin_arm64" in platform: @@ -456,7 +456,7 @@ def _check_platform(board_data: dict) -> None: platform = board_data["platform"] # -- Get the current platform - current_platform = util.get_systype() + current_platform = util.get_system_type() # -- Check if they are not compatible! if platform != current_platform: diff --git a/apio/managers/system.py b/apio/managers/system.py index 0efe704d..81da828a 100644 --- a/apio/managers/system.py +++ b/apio/managers/system.py @@ -7,7 +7,6 @@ # -- Licence GPLv2 import re -import platform from typing import Optional import click @@ -101,10 +100,10 @@ def get_ftdi_devices(self) -> list: # -- Initial empty ftdi devices list ftdi_devices = [] - # -- Run the "lsftdi" command! + # -- Run the "lsftdi" command. result = self._run_command("lsftdi", silent=True) - # -- Sucess in executing the command + # -- Success in executing the command if result and result.exit_code == 0: # -- Get the list of the ftdi devices. It is read @@ -118,7 +117,7 @@ def get_ftdi_devices(self) -> list: # -- It was not possible to run the "lsftdi" command # -- for reading the ftdi devices - raise RuntimeError("Error executing lsftdi") + raise RuntimeError(f"Error executing lsftdi.\n{result.err_text}") def _run_command( self, command: str, silent=False @@ -143,7 +142,7 @@ def _run_command( # pylint: disable=fixme # TODO: Is this necessary or does windows accepts commands without # the '.exe' extension? - if platform.system() == "Windows": + if util.is_windows(): command = command + ".exe" # -- Set the stdout and stderr callbacks, when executing the command diff --git a/apio/pkg_util.py b/apio/pkg_util.py index 910b5ffe..11cc9948 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -14,7 +14,6 @@ from dataclasses import dataclass import os import sys -import platform import click import semantic_version from apio import util @@ -107,12 +106,12 @@ class _PackageDesc: ), "graphviz": _PackageDesc( folder_name="tool-graphviz", - platform_match=platform.system() == "Windows", + platform_match=util.is_windows(), env_func=_graphviz_package_env, ), "drivers": _PackageDesc( folder_name="tools-drivers", - platform_match=platform.system() == "Windows", + platform_match=util.is_windows(), env_func=_drivers_package_env, ), } @@ -134,12 +133,12 @@ def _get_env_mutations_for_packages() -> EnvMutations: # def _dump_env_mutations_for_batch(mutations: EnvMutations) -> None: # """For debugging. Delete once stabalizing the new oss-cad-suite on # windows.""" -# print("--- mutations:") +# print("--- Env Mutations:") # for p in reversed(mutations.paths): -# print(f"@set PATH={p};%PATH%") +# print(f" @set PATH={p};%PATH%") # print() # for name, val in mutations.vars: -# print(f"@set {name}={val}") +# print(f" @set {name}={val}") # print("---") diff --git a/apio/resources.py b/apio/resources.py index dd8e95d5..3cbedec4 100644 --- a/apio/resources.py +++ b/apio/resources.py @@ -487,7 +487,7 @@ def _filter_packages(self, given_platform): # -- If not given platform, use the current if not given_platform: - given_platform = util.get_systype() + given_platform = util.get_system_type() # -- Check all the packages for pkg in self.packages.keys(): diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index fa3499f6..d27a9f61 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -21,7 +21,7 @@ import re from enum import Enum import json -from platform import system +import platform from typing import Dict, Tuple, List, Optional from dataclasses import dataclass import click @@ -90,7 +90,7 @@ def is_testbench(env: SConsEnvironment, file_name: str) -> bool: def is_windows(env: SConsEnvironment) -> bool: """Returns True if running on Windows.""" - return "Windows" == system() + return "windows" in platform.system().lower() def create_construction_env(args: Dict[str, str]) -> SConsEnvironment: diff --git a/apio/util.py b/apio/util.py index dfa20b78..f2edbcd1 100644 --- a/apio/util.py +++ b/apio/util.py @@ -166,7 +166,7 @@ def get_path_in_apio_package(subpath: str) -> Path: return path -def get_systype() -> str: +def get_system_type() -> str: """Return a String with the current platform: ex. linux_x86_64 ex. windows_amd64""" @@ -191,6 +191,21 @@ def get_systype() -> str: return platform_str +def is_linux() -> bool: + """Returns True iff running on linux.""" + return "linux" in platform.system().lower() + + +def is_darwin() -> bool: + """Returns True iff running on darwin (Mac OSX).""" + return "darwin" in platform.system().lower() + + +def is_windows() -> bool: + """Returns True iff running on windows.""" + return "windows" in platform.system().lower() + + def get_projconf_option_dir(name: str, default=None): """Return the project option with the given name These options are place either on environment variables or From 510d265f85e09d08d04effea54f7c27c2de1c5a9 Mon Sep 17 00:00:00 2001 From: Zapta Date: Fri, 1 Nov 2024 18:55:45 -0700 Subject: [PATCH 24/51] Added a special error message for the case zadig configuration seems to be needed on windows. It provides a hint to run apio drivers --ftdi-enable. --- apio/managers/system.py | 20 +++++++++++++++++++- apio/util.py | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/apio/managers/system.py b/apio/managers/system.py index 81da828a..7ea448e9 100644 --- a/apio/managers/system.py +++ b/apio/managers/system.py @@ -7,6 +7,7 @@ # -- Licence GPLv2 import re +import sys from typing import Optional import click @@ -103,6 +104,23 @@ def get_ftdi_devices(self) -> list: # -- Run the "lsftdi" command. result = self._run_command("lsftdi", silent=True) + # If the signs suggest that a zadig configuration is required, + # print an error message with a hint. + if ( + util.is_windows + and result + and result.exit_code != 0 + and "libusb" in result.err_text + ): + click.secho("Error executing lsftdi.", fg="red") + click.secho( + "Hint:\n" + " FTDI driver may not be enabled yet.\n" + " Try running: apio drivers --ftdi-enable", + fg="yellow", + ) + sys.exit(1) + # -- Success in executing the command if result and result.exit_code == 0: @@ -117,7 +135,7 @@ def get_ftdi_devices(self) -> list: # -- It was not possible to run the "lsftdi" command # -- for reading the ftdi devices - raise RuntimeError(f"Error executing lsftdi.\n{result.err_text}") + raise RuntimeError("lsftdi failed.") def _run_command( self, command: str, silent=False diff --git a/apio/util.py b/apio/util.py index f2edbcd1..f8e7d4f9 100644 --- a/apio/util.py +++ b/apio/util.py @@ -317,7 +317,7 @@ def exec_command(*args, **kwargs) -> CommandResult: # -- Set the default arguments to pass to subprocess.Popen() # -- for executing the command flags = { - # -- Catpure the command output + # -- Capture the command output "stdout": subprocess.PIPE, "stderr": subprocess.PIPE, # -- Execute it directly, without using the shell From 7fca3b0ef91c50f993bb047e3dd39c724399bcef Mon Sep 17 00:00:00 2001 From: Zapta Date: Fri, 1 Nov 2024 22:36:29 -0700 Subject: [PATCH 25/51] Changed the folder names of the graphviz package from tool-graphviz to tools-graphviz --- apio/pkg_util.py | 2 +- apio/resources/packages.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index 11cc9948..8374f8a5 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -105,7 +105,7 @@ class _PackageDesc: env_func=_examples_package_env, ), "graphviz": _PackageDesc( - folder_name="tool-graphviz", + folder_name="tools-graphviz", platform_match=util.is_windows(), env_func=_graphviz_package_env, ), diff --git a/apio/resources/packages.json b/apio/resources/packages.json index 1eb9428f..476b1ed8 100644 --- a/apio/resources/packages.json +++ b/apio/resources/packages.json @@ -33,16 +33,16 @@ "graphviz": { "repository": { - "name": "tool-graphviz", + "name": "tools-graphviz", "organization": "zapta" }, "release": { "tag_name": "v%V", - "compressed_name": "tool-graphviz-%P-%V", + "compressed_name": "tools-graphviz-%P-%V", "uncompressed_name": "", - "folder_name": "tool-graphviz", + "folder_name": "tools-graphviz", "extension": "tar.gz", - "url_version": "https://github.com/zapta/tool-graphviz/raw/main/VERSION", + "url_version": "https://github.com/zapta/tools-graphviz/raw/main/VERSION", "available_platforms": [ "windows", "windows_x86", From d9ae2dac4949af40f60738546b8d196465c5ccc9 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 2 Nov 2024 13:24:56 -0700 Subject: [PATCH 26/51] Stream line the zadig configration on windows. Zadig is now started properly with elevated permissions and apio system --lsfdti now gives a hint if the zadig confituration seems to be missing. --- apio/managers/drivers.py | 90 ++++++++++++++++++------------- apio/managers/system.py | 113 ++++++++++++++++++++------------------- 2 files changed, 111 insertions(+), 92 deletions(-) diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index 5319655f..db5048d6 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -6,7 +6,7 @@ """Manage board drivers""" -import sys +import os import shutil import subprocess from pathlib import Path @@ -15,18 +15,37 @@ from apio import pkg_util from apio.resources import Resources -FTDI_INSTALL_DRIVER_INSTRUCTIONS = """ - FTDI driver installation: - Usage instructions +FTDI_ENABLE_INSTRUCTIONS_WINDOWS = """ + Please follow these steps: + + 1. Make sure your FPGA board is connected to the computer. + + 2. Accept the Zadig request to make changes to your computer. + + 3. Find the Zadig window on your screen. + + 4. Select your FPGA board from the drop down list, For example + 'Alhambra II v1.0A - B09-335 (Interface 0)'. + + **VERY IMPORTANT** + If your board appears multiple time, select its 'interface 0' entry. + + 5. Make sure that 'libusbk' is selected. For example + 'libusbK (v3.1.0.0)'. + + 6. Click the 'Replace Driver' button and wait for a successful + completion, this can take a minute or two. + + 7. Close the zadig window. + + 8. Disconnect and reconnect your FPGA board for the new driver + to take affect. - 1. Connect the FTDI FPGA board - 2. Select (Interface 0) - 3. Replace driver by "libusbK" - 4. Reconnect the board - 5. Check `apio system --lsftdi` + 9. Run the command `apio system --lsftdi` and verify that + your board is listed. """ -FTDI_UNINSTALL_DRIVER_INSTRUCTIONS = """ +FTDI_DISABLE_INSTRUCTIONS_WINDOWS = """ FTDI driver uninstallation: Usage instructions @@ -36,7 +55,7 @@ 4. Accept the dialog """ -SERIAL_INSTALL_DRIVER_INSTRUCTIONS = """ +SERIAL_ENABLE_INSTRUCTIONS_WINDOWS = """ Serial driver installation: Usage instructions @@ -46,7 +65,7 @@ 4. Check `apio system --lsserial` """ -SERIAL_UNINSTALL_DRIVER_INSTRUCTIONS = """ +SERIAL_DISABLE_INSTRUCTIONS_WINDOWS = """ Serial driver uninstallation: Usage instructions @@ -399,45 +418,44 @@ def _ftdi_enable_windows(self) -> int: # -- so that zadig open it when executed shutil.copyfile(zadig_ini_src, zadig_ini_dst) - # -- Show messages for the user - click.secho("Launch drivers configuration tool") - click.secho(FTDI_INSTALL_DRIVER_INSTRUCTIONS, fg="yellow") - # -- Zadig exe file with full path: zadig_exe = drivers_base_dir / "bin" / "zadig.exe" + # -- Show messages for the user + click.secho( + "\nStarting the interactive config tool zadig.exe.", fg="green" + ) + click.secho(FTDI_ENABLE_INSTRUCTIONS_WINDOWS, fg="yellow") + try: # -- Execute zadig! - result = util.exec_command(str(zadig_exe)) + # -- We execute it using os.system() rather than by + # -- util.exec_command() because zadig required permissions + # -- elevation. + exit_code = os.system(str(zadig_exe)) click.secho("FTDI drivers configuration finished", fg="green") # -- It was not possible to execute Zadig... except OSError as exc: - click.secho("Error: " + str(exc), fg="red") - click.secho( - "Trying to execute zadig.exe in command line, " - "but an error ocurred", - fg="red", - ) - click.secho( - "Please, execute the command again in the command line with" - " administrator privilegdes", - fg="red", - ) - sys.exit(1) + msg = str(exc) + click.secho("\n" + msg, fg="red") + click.secho("Error: zadig.exe failed.", fg="red") + return 1 - # -- Remove zadig.ini from the current folder. It is no longer needed - if zadig_ini_dst.exists(): - zadig_ini_dst.unlink() + finally: + # -- Remove zadig.ini from the current folder. It is no longer + # -- needed + if zadig_ini_dst.exists(): + zadig_ini_dst.unlink() - return result.exit_code + return exit_code def _ftdi_disable_windows(self) -> int: # -- Check that the required packages exist. pkg_util.check_required_packages(["drivers"], self.resources) click.secho("Launch device manager") - click.secho(FTDI_UNINSTALL_DRIVER_INSTRUCTIONS, fg="yellow") + click.secho(FTDI_DISABLE_INSTRUCTIONS_WINDOWS, fg="yellow") result = util.exec_command("mmc devmgmt.msc") return result.exit_code @@ -454,7 +472,7 @@ def _serial_enable_windows(self) -> int: try: click.secho("Launch drivers configuration tool") - click.secho(SERIAL_INSTALL_DRIVER_INSTRUCTIONS, fg="yellow") + click.secho(SERIAL_ENABLE_INSTRUCTIONS_WINDOWS, fg="yellow") result = util.exec_command( str(Path(drivers_bin_dir) / "serial_install.exe") ) @@ -471,7 +489,7 @@ def _serial_disable_windows(self) -> int: pkg_util.check_required_packages(["drivers"], self.resources) click.secho("Launch device manager") - click.secho(SERIAL_UNINSTALL_DRIVER_INSTRUCTIONS, fg="yellow") + click.secho(SERIAL_DISABLE_INSTRUCTIONS_WINDOWS, fg="yellow") result = util.exec_command("mmc devmgmt.msc") return result.exit_code diff --git a/apio/managers/system.py b/apio/managers/system.py index 7ea448e9..d8e8d0aa 100644 --- a/apio/managers/system.py +++ b/apio/managers/system.py @@ -8,7 +8,6 @@ import re import sys -from typing import Optional import click from apio import util @@ -23,23 +22,46 @@ def __init__(self, resources: Resources): self.resources = resources - def lsusb(self): - """Run the lsusb system command""" + def _lsftdi_fatal_error(self, result: util.CommandResult) -> None: + """Handles a failure of a 'lsftdi' command. Print message and exits.""" + # + assert result.exit_code != 0, result + click.secho(result.out_text) + click.secho(f"{result.err_text}", fg="red") + click.secho("Error: the 'lsftdi' command failed.", fg="red") - result = self._run_command("lsusb") + # -- A special hint for zadig on windows. + if util.is_windows() and "libusb" in result.err_text: + click.secho( + "\n" + "Hint:\n" + " The FTDI driver may not be enabled yet.\n" + " Try running the command 'apio drivers --ftdi-enable'", + fg="yellow", + ) + sys.exit(1) + + def lsusb(self) -> int: + """Run the lsusb command. Returns exit code.""" + + result = self._run_command("lsusb", silent=False) return result.exit_code if result else 1 - def lsftdi(self): - """DOC: TODO""" + def lsftdi(self) -> int: + """Runs the lsftdi command. Returns exit code.""" - result = self._run_command("lsftdi") + result = self._run_command("lsftdi", silent=True) - return result.exit_code if result else 1 + if result.exit_code != 0: + # -- Print error message and exit. + self._lsftdi_fatal_error(result) - @staticmethod - def lsserial(): - """DOC: TODO""" + click.secho(result.out_text) + return 0 + + def lsserial(self) -> int: + """List the serial ports. Returns exit code.""" serial_ports = util.get_serial_ports() click.secho(f"Number of Serial devices found: {len(serial_ports)}") @@ -71,20 +93,18 @@ def get_usb_devices(self) -> list: # -- Run the "lsusb" command! result = self._run_command("lsusb", silent=True) - # -- Sucess in executing the command - if result and result.exit_code == 0: - - # -- Get the list of the usb devices. It is read - # -- from the command stdout - # -- Ex: [{'hwid':'1d6b:0003'}, {'hwid':'04f2:b68b'}...] - usb_devices = self._parse_usb_devices(result.out_text) + if result.exit_code != 0: + click.secho(result.out_text) + click.secho(result.err_text, fg="red") + raise RuntimeError("Error executing lsusb") - # -- Return the devices - return usb_devices + # -- Get the list of the usb devices. It is read + # -- from the command stdout + # -- Ex: [{'hwid':'1d6b:0003'}, {'hwid':'04f2:b68b'}...] + usb_devices = self._parse_usb_devices(result.out_text) - # -- It was not possible to run the "lsusb" command - # -- for reading the usb devices - raise RuntimeError("Error executing lsusb") + # -- Return the devices + return usb_devices def get_ftdi_devices(self) -> list: """Return a list of the connected FTDI devices @@ -104,42 +124,24 @@ def get_ftdi_devices(self) -> list: # -- Run the "lsftdi" command. result = self._run_command("lsftdi", silent=True) - # If the signs suggest that a zadig configuration is required, - # print an error message with a hint. - if ( - util.is_windows - and result - and result.exit_code != 0 - and "libusb" in result.err_text - ): - click.secho("Error executing lsftdi.", fg="red") - click.secho( - "Hint:\n" - " FTDI driver may not be enabled yet.\n" - " Try running: apio drivers --ftdi-enable", - fg="yellow", - ) - sys.exit(1) - - # -- Success in executing the command - if result and result.exit_code == 0: + # -- Exit if error. + if result.exit_code != 0: + self._lsftdi_fatal_error(result) - # -- Get the list of the ftdi devices. It is read - # -- from the command stdout - # -- Ex: [{'index': '0', 'manufacturer': 'AlhambraBits', - # -- 'description': 'Alhambra II v1.0A - B07-095'}] - ftdi_devices = self._parse_ftdi_devices(result.out_text) + # -- Get the list of the ftdi devices. It is read + # -- from the command stdout + # -- Ex: [{'index': '0', 'manufacturer': 'AlhambraBits', + # -- 'description': 'Alhambra II v1.0A - B07-095'}] + ftdi_devices = self._parse_ftdi_devices(result.out_text) - # -- Return the devices - return ftdi_devices + # -- Return the devices + return ftdi_devices - # -- It was not possible to run the "lsftdi" command - # -- for reading the ftdi devices - raise RuntimeError("lsftdi failed.") + # -- Print error message and exit. def _run_command( - self, command: str, silent=False - ) -> Optional[util.CommandResult]: + self, command: str, *, silent: bool + ) -> util.CommandResult: """Execute the given system command * INPUT: * command: Command to execute (Ex. "lsusb") @@ -148,7 +150,6 @@ def _run_command( * True --> Print on the console * OUTPUT: An ExecResult with the command's outcome. - In case of not executing the command it returns none! """ # -- Check that the required package exists. @@ -166,7 +167,7 @@ def _run_command( # -- Set the stdout and stderr callbacks, when executing the command # -- Silent mode (True): No callback on_stdout = None if silent else self._on_stdout - on_stderr = self._on_stderr + on_stderr = None if silent else self._on_stderr # -- Execute the command! result = util.exec_command( From fddc0b06c047fa38504cb2010dcec9293d55bcf9 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 2 Nov 2024 14:12:06 -0700 Subject: [PATCH 27/51] Directed the tools-graphviz package loading to FPGAwars (was zapta's temp repo). --- apio/resources/packages.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apio/resources/packages.json b/apio/resources/packages.json index 476b1ed8..d74de724 100644 --- a/apio/resources/packages.json +++ b/apio/resources/packages.json @@ -34,7 +34,7 @@ "graphviz": { "repository": { "name": "tools-graphviz", - "organization": "zapta" + "organization": "FPGAwars" }, "release": { "tag_name": "v%V", @@ -42,7 +42,7 @@ "uncompressed_name": "", "folder_name": "tools-graphviz", "extension": "tar.gz", - "url_version": "https://github.com/zapta/tools-graphviz/raw/main/VERSION", + "url_version": "https://github.com/FPGAwars/tools-graphviz/raw/main/VERSION", "available_platforms": [ "windows", "windows_x86", From bbe44b0ec838f30841e0f9ed7622aa94303adf0a Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 2 Nov 2024 18:29:37 -0700 Subject: [PATCH 28/51] Refactored the 'clean' logig from the three SConstruct scripts to a common function in scons_util.py. Also, added 'zadig.ini' to the list of cleaned up files. --- apio/scons/ecp5/SConstruct | 18 ++++++------------ apio/scons/gowin/SConstruct | 21 ++++++++------------- apio/scons/ice40/SConstruct | 21 ++++++++------------- apio/scons/scons_util.py | 24 ++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 38 deletions(-) diff --git a/apio/scons/ecp5/SConstruct b/apio/scons/ecp5/SConstruct index 53a2d801..45d76c27 100644 --- a/apio/scons/ecp5/SConstruct +++ b/apio/scons/ecp5/SConstruct @@ -45,7 +45,6 @@ from SCons.Script import ( GetOption, COMMAND_LINE_TARGETS, ARGUMENTS, - Glob, ) from apio.scons.scons_util import ( TARGET, @@ -67,6 +66,7 @@ from apio.scons.scons_util import ( make_iverilog_action, make_verilator_action, get_report_action, + set_up_cleanup, ) # -- Create the environment @@ -405,23 +405,17 @@ lint_target = env.Alias("lint", lint_out_target) AlwaysBuild(lint_target) -# -- These is for cleaning the artifact files. +# -- Handle the cleanu of the artifact files. if GetOption("clean"): - # Identify additional files that may not be associated with targets and - # associate them with a target such that they will be cleaned up as well. - # This cleans for example artifacts of past simulation since the testbench - # target are dynamic and changes with the selected testbench. - for glob_pattern in ["*.out", "*.vcd"]: - for node in Glob(glob_pattern): - env.Clean(build_target, str(node)) - - env.Default( + set_up_cleanup( + env, [ report_target, build_target, synth_target, + verify_target, pnr_target, graph_target, lint_target, - ] + ], ) diff --git a/apio/scons/gowin/SConstruct b/apio/scons/gowin/SConstruct index 94c6f504..015084a8 100644 --- a/apio/scons/gowin/SConstruct +++ b/apio/scons/gowin/SConstruct @@ -45,7 +45,6 @@ from SCons.Script import ( GetOption, COMMAND_LINE_TARGETS, ARGUMENTS, - Glob, ) from apio.scons.scons_util import ( TARGET, @@ -67,6 +66,7 @@ from apio.scons.scons_util import ( make_iverilog_action, make_verilator_action, get_report_action, + set_up_cleanup, ) # -- Create the environment @@ -396,22 +396,17 @@ lint_target = env.Alias("lint", lint_out_target) AlwaysBuild(lint_target) -# -- These is for cleaning the artifact files. +# -- Handle the cleanu of the artifact files. if GetOption("clean"): - # Identify additional files that may not be associated with targets and - # associate them with a target such that they will be cleaned up as well. - # This cleans for example artifacts of past simulation since the testbench - # target are dynamic and changes with the selected testbench. - for glob_pattern in ["*.out", "*.vcd"]: - for node in Glob(glob_pattern): - env.Clean(build_target, str(node)) - - env.Default( + set_up_cleanup( + env, [ report_target, build_target, - verify_out_target, + synth_target, + verify_target, + pnr_target, graph_target, lint_target, - ] + ], ) diff --git a/apio/scons/ice40/SConstruct b/apio/scons/ice40/SConstruct index b1c15ff3..ef268a39 100644 --- a/apio/scons/ice40/SConstruct +++ b/apio/scons/ice40/SConstruct @@ -45,7 +45,6 @@ from SCons.Script import ( GetOption, COMMAND_LINE_TARGETS, ARGUMENTS, - Glob, ) from apio.scons.scons_util import ( TARGET, @@ -67,6 +66,7 @@ from apio.scons.scons_util import ( make_iverilog_action, make_verilator_action, get_report_action, + set_up_cleanup, ) @@ -423,23 +423,18 @@ lint_target = env.Alias("lint", lint_out_target) AlwaysBuild(lint_target) -# -- These is for cleaning the artifact files. +# -- Handle the cleanu of the artifact files. if GetOption("clean"): - # Identify additional files that may not be associated with targets and - # associate them with a target such that they will be cleaned up as well. - # This cleans for example artifacts of past simulation since the testbench - # target are dynamic and changes with the selected testbench. - for glob_pattern in ["*.out", "*.vcd"]: - for node in Glob(glob_pattern): - env.Clean(build_target, str(node)) - - env.Default( + set_up_cleanup( + env, [ report_target, time_target, build_target, - verify_out_target, + synth_target, + verify_target, + pnr_target, graph_target, lint_target, - ] + ], ) diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index d27a9f61..9d1b45a1 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -643,3 +643,27 @@ def wait_for_remote_debugger(env: SConsEnvironment): msg(env, "Attach with the Visual Studio Code debugger.") debugpy.wait_for_client() msg(env, "Remote debugger is attached.", fg="green") + + +def set_up_cleanup(env: SConsEnvironment, targets) -> None: + """Should be called only when the "clean" target is specified. Configures + in env the set of targets that should be cleaned up. 'targets' is a list + of top level targets and aliases defined in SConstruct. + """ + + # -- Should be called only when the 'clean' target is specified. + assert env.GetOption("clean") + + # -- Patterns for target files that may not be defined in every invocation + # -- of SConstruct. For example xyz_tb.vcd appears only when simulating + # -- benchmark xyz_tb.vcd. + dynamic_targets = ["*.out", "*.vcd", "zadig.ini"] + + # -- Attach all the existing files that match the dynamic targets to the + # -- first given target (this is an arbitrary choice). + for dynamic_target in dynamic_targets: + for node in env.Glob(dynamic_target): + env.Clean(targets[0], str(node)) + + # -- Tell SCons to cleanup the given targets and all of their dependencies. + env.Default(targets) From e9dfd4234ce444f350a124e471a311ac9ec805b4 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 2 Nov 2024 20:04:57 -0700 Subject: [PATCH 29/51] Streamedline the 'apio system --ftdi-disable' on windows. Improved the instructions to the user and the device manager is now elevated properly by launching it with os.system(). --- apio/managers/drivers.py | 36 +++++++++++++++++++++++++++--------- apio/util.py | 5 ++++- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index db5048d6..cb400754 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -46,13 +46,28 @@ """ FTDI_DISABLE_INSTRUCTIONS_WINDOWS = """ - FTDI driver uninstallation: - Usage instructions + Please follow these steps: - 1. Find the FPGA USB Device - 2. Right click - 3. Select "Uninstall" - 4. Accept the dialog + 1. Make sure your FPGA board is NOT connected to the computer. + + 2. If asked, allow the Device Manager to make changes to your system. + + 3. Find the Device Manager window. + + 4. Connect the board to your computer and a new entry will be added + to the device list (though sometimes it may be collapsed). + + 5. Identify the entry of your board (e.g. in the 'libusbK USB Devices' + section). + + 6. Right click on your board entry and select 'Uninstall device'. + + 7. If available, check the box 'Delete the driver software for this + device'. + + 8. Click the 'Uninstall' button. + + 9. Close the Device Manager window. """ SERIAL_ENABLE_INSTRUCTIONS_WINDOWS = """ @@ -454,11 +469,14 @@ def _ftdi_disable_windows(self) -> int: # -- Check that the required packages exist. pkg_util.check_required_packages(["drivers"], self.resources) - click.secho("Launch device manager") + click.secho("\nStarting the interactive Device Manager..", fg="green") click.secho(FTDI_DISABLE_INSTRUCTIONS_WINDOWS, fg="yellow") - result = util.exec_command("mmc devmgmt.msc") - return result.exit_code + # -- We launch the device manager using os.system() rather than with + # -- util.exec_command() because util.exec_command() does not support + # -- elevation. + exit_code = os.system("mmc devmgmt.msc") + return exit_code # W0703: Catching too general exception Exception (broad-except) # pylint: disable=W0703 diff --git a/apio/util.py b/apio/util.py index f8e7d4f9..4d6a57c0 100644 --- a/apio/util.py +++ b/apio/util.py @@ -295,7 +295,10 @@ class CommandResult: def exec_command(*args, **kwargs) -> CommandResult: - """Execute the given command: + """Execute the given command. + + NOTE: When running on windows, this function does not support + privilege elevation, to achieve that, use os.system() instead. INPUTS: *args: List with the command and its arguments to execute From abefe229e31362dbbeccae1a38dd3ec1f9c2168c Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 2 Nov 2024 20:10:22 -0700 Subject: [PATCH 30/51] Added a note to the 'apio drivers --ftdi-disable' instructions. --- apio/managers/drivers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index cb400754..528253c9 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -60,6 +60,9 @@ 5. Identify the entry of your board (e.g. in the 'libusbK USB Devices' section). + NOTE: If your board does not show up or if it's listed as a + COM port, it may not have the FTDI driver enabled for it. + 6. Right click on your board entry and select 'Uninstall device'. 7. If available, check the box 'Delete the driver software for this From 4c1e58d53d0d7619655099bf742bb43cb17ce4a4 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 2 Nov 2024 20:42:43 -0700 Subject: [PATCH 31/51] Tweaked the four commands of 'apio drivers' (enable/disable ftdi/serial) on windows and they all should work now, as far as I could test on my windows box. The main change is the use of os.system() to launch commands that require elevation. --- apio/managers/drivers.py | 169 +++++++++++++++++++++------------------ 1 file changed, 90 insertions(+), 79 deletions(-) diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index 528253c9..4583f7f1 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -16,81 +16,101 @@ from apio.resources import Resources FTDI_ENABLE_INSTRUCTIONS_WINDOWS = """ - Please follow these steps: +Please follow these steps: - 1. Make sure your FPGA board is connected to the computer. + 1. Make sure your FPGA board is connected to the computer. - 2. Accept the Zadig request to make changes to your computer. + 2. Accept the Zadig request to make changes to your computer. - 3. Find the Zadig window on your screen. + 3. Find the Zadig window on your screen. - 4. Select your FPGA board from the drop down list, For example - 'Alhambra II v1.0A - B09-335 (Interface 0)'. + 4. Select your FPGA board from the drop down list, For example + 'Alhambra II v1.0A - B09-335 (Interface 0)'. - **VERY IMPORTANT** - If your board appears multiple time, select its 'interface 0' entry. + **VERY IMPORTANT** + If your board appears multiple time, select its 'interface 0' entry. - 5. Make sure that 'libusbk' is selected. For example - 'libusbK (v3.1.0.0)'. + 5. Make sure that 'libusbk' is selected. For example + 'libusbK (v3.1.0.0)'. - 6. Click the 'Replace Driver' button and wait for a successful - completion, this can take a minute or two. + 6. Click the 'Replace Driver' button and wait for a successful + completion, this can take a minute or two. - 7. Close the zadig window. + 7. Close the zadig window. - 8. Disconnect and reconnect your FPGA board for the new driver - to take affect. + 8. Disconnect and reconnect your FPGA board for the new driver + to take affect. - 9. Run the command `apio system --lsftdi` and verify that - your board is listed. + 9. Run the command `apio system --lsftdi` and verify that + your board is listed. """ FTDI_DISABLE_INSTRUCTIONS_WINDOWS = """ - Please follow these steps: +Please follow these steps: - 1. Make sure your FPGA board is NOT connected to the computer. + 1. Make sure your FPGA board is NOT connected to the computer. - 2. If asked, allow the Device Manager to make changes to your system. + 2. If asked, allow the Device Manager to make changes to your system. - 3. Find the Device Manager window. + 3. Find the Device Manager window. - 4. Connect the board to your computer and a new entry will be added - to the device list (though sometimes it may be collapsed). + 4. Connect the board to your computer and a new entry will be added + to the device list (though sometimes it may be collapsed). - 5. Identify the entry of your board (e.g. in the 'libusbK USB Devices' - section). + 5. Identify the entry of your board (e.g. in the 'libusbK USB Devices' + section). - NOTE: If your board does not show up or if it's listed as a - COM port, it may not have the FTDI driver enabled for it. + NOTE: If your board does not show up or if it's listed as a + COM port, it may not have the FTDI driver enabled for it. - 6. Right click on your board entry and select 'Uninstall device'. + 6. Right click on your board entry and select 'Uninstall device'. - 7. If available, check the box 'Delete the driver software for this - device'. + 7. If available, check the box 'Delete the driver software for this + device'. - 8. Click the 'Uninstall' button. + 8. Click the 'Uninstall' button. - 9. Close the Device Manager window. + 9. Close the Device Manager window. """ SERIAL_ENABLE_INSTRUCTIONS_WINDOWS = """ - Serial driver installation: - Usage instructions +Please follow these steps: - 1. Connect the Serial FPGA board - 2. Install the driver - 3. Reconnect the board - 4. Check `apio system --lsserial` + 1. Make sure your FPGA board is connected to the computer. + + 2. Accept the Serial Installer request to make changes to your computer. + + 3. Find the Serial installer window and follow the instructions. + + 4. To verify, disconnect and reconnect the board and run the command + 'apio system --lsserial'. """ SERIAL_DISABLE_INSTRUCTIONS_WINDOWS = """ - Serial driver uninstallation: - Usage instructions +Please follow these steps: + + 1. Make sure your FPGA board is NOT connected to the computer. + + 2. If asked, allow the Device Manager to make changes to your system. + + 3. Find the Device Manager window. + + 4. Connect the board to your computer and a new entry will be added + to the device list (though sometimes it may be collapsed). - 1. Find the FPGA USB Device - 2. Right click - 3. Select "Uninstall" - 4. Accept the dialog + 5. Identify the entry of your board (typically in the Ports section). + + NOTE: If your board does not show up as a COM port, it may not + have the 'apio drivers --serial-enable' applied to it. + + 6. Right click on your board entry and select 'Uninstall device'. + + 7. If available, check the box 'Delete the driver software for this + device'. + + 8. Click the 'Uninstall' button. + + 9. Close the Device Manager window. """ @@ -445,26 +465,17 @@ def _ftdi_enable_windows(self) -> int: ) click.secho(FTDI_ENABLE_INSTRUCTIONS_WINDOWS, fg="yellow") - try: - # -- Execute zadig! - # -- We execute it using os.system() rather than by - # -- util.exec_command() because zadig required permissions - # -- elevation. - exit_code = os.system(str(zadig_exe)) - click.secho("FTDI drivers configuration finished", fg="green") - - # -- It was not possible to execute Zadig... - except OSError as exc: - msg = str(exc) - click.secho("\n" + msg, fg="red") - click.secho("Error: zadig.exe failed.", fg="red") - return 1 + # -- Execute zadig! + # -- We execute it using os.system() rather than by + # -- util.exec_command() because zadig required permissions + # -- elevation. + exit_code = os.system(str(zadig_exe)) + click.secho("FTDI drivers configuration finished", fg="green") - finally: - # -- Remove zadig.ini from the current folder. It is no longer - # -- needed - if zadig_ini_dst.exists(): - zadig_ini_dst.unlink() + # -- Remove zadig.ini from the current folder. It is no longer + # -- needed + if zadig_ini_dst.exists(): + zadig_ini_dst.unlink() return exit_code @@ -472,7 +483,7 @@ def _ftdi_disable_windows(self) -> int: # -- Check that the required packages exist. pkg_util.check_required_packages(["drivers"], self.resources) - click.secho("\nStarting the interactive Device Manager..", fg="green") + click.secho("\nStarting the interactive Device Manager.", fg="green") click.secho(FTDI_DISABLE_INSTRUCTIONS_WINDOWS, fg="yellow") # -- We launch the device manager using os.system() rather than with @@ -490,27 +501,27 @@ def _serial_enable_windows(self) -> int: drivers_base_dir = pkg_util.get_package_dir("drivers") drivers_bin_dir = drivers_base_dir / "bin" - try: - - click.secho("Launch drivers configuration tool") - click.secho(SERIAL_ENABLE_INSTRUCTIONS_WINDOWS, fg="yellow") - result = util.exec_command( - str(Path(drivers_bin_dir) / "serial_install.exe") - ) - click.secho("Serial drivers configuration finished", fg="green") + click.secho("\nStarting the Serial Installer.", fg="green") + click.secho(SERIAL_ENABLE_INSTRUCTIONS_WINDOWS, fg="yellow") - except Exception as exc: - click.secho("Error: " + str(exc), fg="red") - result = util.CommandResult(exit_code=1) + # -- We launch the device manager using os.system() rather than with + # -- util.exec_command() because util.exec_command() does not support + # -- elevation. + exit_code = os.system( + str(Path(drivers_bin_dir) / "serial_install.exe") + ) - return result.exit_code + return exit_code def _serial_disable_windows(self) -> int: # -- Check that the required packages exist. pkg_util.check_required_packages(["drivers"], self.resources) - click.secho("Launch device manager") + click.secho("\nStarting the interactive Device Manager.", fg="green") click.secho(SERIAL_DISABLE_INSTRUCTIONS_WINDOWS, fg="yellow") - result = util.exec_command("mmc devmgmt.msc") - return result.exit_code + # -- We launch the device manager using os.system() rather than with + # -- util.exec_command() because util.exec_command() does not support + # -- elevation. + exit_code = os.system("mmc devmgmt.msc") + return exit_code From 4023c55ae96b6edf69e6154609a982caa345f25d Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 2 Nov 2024 20:45:31 -0700 Subject: [PATCH 32/51] Minor message text tweak for consistency. --- apio/managers/drivers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index 4583f7f1..74780778 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -501,7 +501,7 @@ def _serial_enable_windows(self) -> int: drivers_base_dir = pkg_util.get_package_dir("drivers") drivers_bin_dir = drivers_base_dir / "bin" - click.secho("\nStarting the Serial Installer.", fg="green") + click.secho("\nStarting the interactive Serial Installer.", fg="green") click.secho(SERIAL_ENABLE_INSTRUCTIONS_WINDOWS, fg="yellow") # -- We launch the device manager using os.system() rather than with From 54af27e52bd4154ecf32bb51ab571443d3d48394 Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 4 Nov 2024 17:27:20 -0800 Subject: [PATCH 33/51] Now printing a message to the user when setting the environment. The goal is to convey that the printed command run in an environment different than the user's environment. --- apio/pkg_util.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index 8374f8a5..c96ec971 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -160,11 +160,19 @@ def set_env_for_packages() -> None: """Sets the environment variables for using all the that are available for this platform, even if currently not installed. """ + + # -- Be transparent to the user about setting the environment, in case + # -- they will try to run the commands from a regular shell. + click.secho("Setting the envinronment.") + + # -- Collect the env mutations for all packages. mutations = _get_env_mutations_for_packages() # -- For debugging. # _dump_env_mutations_for_batch(mutations) + # -- Apply the env mutations. These mutations are temporary and does not + # -- affect the user's shell environment. _apply_env_mutations(mutations) From 4938658e0a1d9d36eceefddf7e5fc6bd30cda248 Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 4 Nov 2024 17:28:15 -0800 Subject: [PATCH 34/51] Bumped the min oss-cad-tools version to 0.2.0. --- apio/resources/distribution.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apio/resources/distribution.json b/apio/resources/distribution.json index f600505b..a6a67ff1 100644 --- a/apio/resources/distribution.json +++ b/apio/resources/distribution.json @@ -3,7 +3,7 @@ "drivers": ">=1.1.0", "examples": ">=0.0.7", "graphviz": ">=12.1.2", - "oss-cad-suite": ">=0.0.1" + "oss-cad-suite": ">=0.2.0" }, "pip_packages": { "blackiceprog": ">=2.0.0,<3.0.0", From b3eca5de4c0c5df87c7a7b68e370cf250e42285d Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 4 Nov 2024 18:38:16 -0800 Subject: [PATCH 35/51] Added to the raw command an option --env which shows the env mutation apio does before running the command. This will allow the user to replicated apio command in their own batch or shell scripts. --- apio/commands/raw.py | 37 ++++++++++++++++++++++++++++------- apio/pkg_util.py | 46 ++++++++++++++++++++++++++++---------------- 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/apio/commands/raw.py b/apio/commands/raw.py index 22d41f80..9a7415ef 100644 --- a/apio/commands/raw.py +++ b/apio/commands/raw.py @@ -19,7 +19,9 @@ HELP = """ The raw command allows to bypass apio and run underlying tools directly. This is an advanced command that requires familiarity -with the underlying tools. +with the underlying tools. Before running the command, apio changes the +internal env settings to provide access to its packages. To view the +env changes, use the --env option. \b Examples: @@ -28,11 +30,23 @@ apio raw "yosys -p 'read_verilog leds.v; show' -q" # Graph a module apio raw "verilator --lint-only leds.v" # Lint a module apio raw "icepll -i 12 -o 30" # ICE PLL parameters + apio raw --env # Show env changes + apio raw --env "yosys --version" # Show also env changes + [Note] If you find a raw command that would benefit other apio users consider suggesting it as an apio feature request. """ +verbose_option = click.option( + "env", # Var name. + "-e", + "--env", + is_flag=True, + help="Show env changes.", + cls=cmd_util.ApioOption, +) + @click.command( "raw", @@ -41,22 +55,31 @@ cls=cmd_util.ApioCommand, ) @click.pass_context -@click.argument("cmd") +@click.argument("cmd", metavar="COMMAND", required=False) +@verbose_option def cli( ctx: Context, # Arguments cmd: str, + # Options + env: bool, ): """Implements the apio raw command which executes user specified commands from apio installed tools. """ - # -- Make sure the oss-cad-suite is installed. - resources = Resources() - pkg_util.check_required_packages(["oss-cad-suite"], resources) + if not cmd and not env: + cmd_util.fatal_usage_error(ctx, "Missing an option or a command") # -- Set the system env for using the packages. - pkg_util.set_env_for_packages() + pkg_util.set_env_for_packages(verbose=env) + + # -- Make sure the oss-cad-suite is installed. + if cmd: + resources = Resources() + pkg_util.check_required_packages(["oss-cad-suite"], resources) + exit_code = util.call(cmd) + else: + exit_code = 0 - exit_code = util.call(cmd) ctx.exit(exit_code) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index c96ec971..a96f52a2 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -130,16 +130,28 @@ def _get_env_mutations_for_packages() -> EnvMutations: return result -# def _dump_env_mutations_for_batch(mutations: EnvMutations) -> None: -# """For debugging. Delete once stabalizing the new oss-cad-suite on -# windows.""" -# print("--- Env Mutations:") -# for p in reversed(mutations.paths): -# print(f" @set PATH={p};%PATH%") -# print() -# for name, val in mutations.vars: -# print(f" @set {name}={val}") -# print("---") +def _dump_env_mutations(mutations: EnvMutations) -> None: + """For debugging. Delete once stabalizing the new oss-cad-suite on + windows.""" + # batch = util.is_windows() + click.secho("Env Mutations:", fg="magenta") + + # -- Print PATH mutations. + windows = False + for p in reversed(mutations.paths): + styled_name = click.style("PATH", fg="magenta") + if windows: + print(f" @set {styled_name}={p};%PATH%") + else: + print(f' {styled_name}="{p}:$PATH"') + + # -- Print vars mutations. + for name, val in mutations.vars: + styled_name = click.style(name, fg="magenta") + if windows: + print(f" @set {styled_name}={val}") + else: + print(f' {styled_name}="{val}"') def _apply_env_mutations(mutations: EnvMutations) -> None: @@ -156,20 +168,20 @@ def _apply_env_mutations(mutations: EnvMutations) -> None: os.environ[name] = value -def set_env_for_packages() -> None: +def set_env_for_packages(verbose: bool = False) -> None: """Sets the environment variables for using all the that are available for this platform, even if currently not installed. """ - # -- Be transparent to the user about setting the environment, in case - # -- they will try to run the commands from a regular shell. - click.secho("Setting the envinronment.") - # -- Collect the env mutations for all packages. mutations = _get_env_mutations_for_packages() - # -- For debugging. - # _dump_env_mutations_for_batch(mutations) + if verbose: + _dump_env_mutations(mutations) + else: + # -- Be transparent to the user about setting the environment, in case + # -- they will try to run the commands from a regular shell. + click.secho("Setting the envinronment.") # -- Apply the env mutations. These mutations are temporary and does not # -- affect the user's shell environment. From e48492dc734a5beebc7669a28f8edf1b4a38d5a4 Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 4 Nov 2024 19:52:00 -0800 Subject: [PATCH 36/51] This change formalizes the dual mode of Resources. In project scope it's loads project resources such as boards.json override (e.g. for apio build) while in global scope it just the defaults resources (e.g. for apio install). This will further be cleaned up in follow up changes and potentially moving the Project object as well into the resources object. --- apio/commands/boards.py | 5 +++-- apio/commands/build.py | 2 +- apio/commands/clean.py | 2 +- apio/commands/create.py | 8 +++++++- apio/commands/drivers.py | 2 +- apio/commands/examples.py | 2 +- apio/commands/graph.py | 2 +- apio/commands/install.py | 12 ++++++++---- apio/commands/lint.py | 2 +- apio/commands/modify.py | 7 ++++--- apio/commands/raw.py | 2 +- apio/commands/report.py | 2 +- apio/commands/sim.py | 2 +- apio/commands/system.py | 2 +- apio/commands/test.py | 2 +- apio/commands/time.py | 2 +- apio/commands/uninstall.py | 10 +++++++--- apio/commands/upload.py | 2 +- apio/commands/verify.py | 2 +- apio/managers/installer.py | 11 ----------- apio/managers/project.py | 14 ++++++++------ apio/resources.py | 26 +++++++++++++++++++++----- 22 files changed, 72 insertions(+), 49 deletions(-) diff --git a/apio/commands/boards.py b/apio/commands/boards.py index 0a9a3e1b..83d38855 100644 --- a/apio/commands/boards.py +++ b/apio/commands/boards.py @@ -74,8 +74,9 @@ def cli( # Make sure these params are exclusive. cmd_util.check_exclusive_params(ctx, nameof(list_, fpgas)) - # -- Access to the apio resources - resources = Resources(project_dir=project_dir) + # -- Access to the apio resources. We need project scope since the project + # -- may override the list of boards. + resources = Resources(project_dir=project_dir, project_scope=True) # -- Option 1: List boards if list_: diff --git a/apio/commands/build.py b/apio/commands/build.py index 4f7b9b3a..a86b6b5c 100644 --- a/apio/commands/build.py +++ b/apio/commands/build.py @@ -76,7 +76,7 @@ def cli( # https://www.scons.org/documentation.html # -- Create the scons object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) # R0801: Similar lines in 2 files diff --git a/apio/commands/clean.py b/apio/commands/clean.py index 2caa9ceb..42e45e1d 100644 --- a/apio/commands/clean.py +++ b/apio/commands/clean.py @@ -57,7 +57,7 @@ def cli( """ # -- Create the scons object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) # -- Build the project with the given parameters diff --git a/apio/commands/create.py b/apio/commands/create.py index 6c327493..14e02be5 100644 --- a/apio/commands/create.py +++ b/apio/commands/create.py @@ -14,6 +14,7 @@ from apio import util from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -74,10 +75,15 @@ def cli( if not top_module: top_module = DEFAULT_TOP_MODULE + # -- Load resources. We use project scope in case the project dir + # -- already has a custom boards.json file so we validate 'board' + # -- against that board list. + resources = Resources(project_dir=project_dir, project_scope=True) + project_dir = util.get_project_dir(project_dir) # Create the apio.ini file - ok = Project.create_ini(project_dir, board, top_module, sayyes) + ok = Project.create_ini(resources, board, top_module, sayyes) exit_code = 0 if ok else 1 ctx.exit(exit_code) diff --git a/apio/commands/drivers.py b/apio/commands/drivers.py index 0f62af75..61db34fc 100644 --- a/apio/commands/drivers.py +++ b/apio/commands/drivers.py @@ -97,7 +97,7 @@ def cli( ) # -- Access to the Drivers - resources = Resources() + resources = Resources(project_scope=False) drivers = Drivers(resources) # -- FTDI enable option diff --git a/apio/commands/examples.py b/apio/commands/examples.py index fe3b4b69..b17af94e 100644 --- a/apio/commands/examples.py +++ b/apio/commands/examples.py @@ -88,7 +88,7 @@ def cli( cmd_util.check_exclusive_params(ctx, nameof(list_, dir_, files)) # -- Access to the Drivers - resources = Resources() + resources = Resources(project_scope=False) examples = Examples(resources) # -- Option: List all the available examples diff --git a/apio/commands/graph.py b/apio/commands/graph.py index e6a225f9..91dec5d9 100644 --- a/apio/commands/graph.py +++ b/apio/commands/graph.py @@ -68,7 +68,7 @@ def cli( """Implements the apio graph command.""" # -- Crete the scons object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) # -- Graph the project with the given parameters diff --git a/apio/commands/install.py b/apio/commands/install.py index 24252477..fb41fd17 100644 --- a/apio/commands/install.py +++ b/apio/commands/install.py @@ -12,7 +12,7 @@ from varname import nameof import click from click.core import Context -from apio.managers.installer import Installer, list_packages +from apio.managers.installer import Installer from apio.resources import Resources from apio import cmd_util from apio.commands import options @@ -98,8 +98,12 @@ def cli( # Make sure these params are exclusive. cmd_util.check_exclusive_params(ctx, nameof(packages, all_, list_)) - # -- Load the resources. - resources = Resources(platform=platform, project_dir=project_dir) + # -- Load the resources. We don't care about project specific resources. + resources = Resources( + platform=platform, + project_dir=project_dir, + project_scope=False, + ) # -- Install the given apio packages if packages: @@ -116,7 +120,7 @@ def cli( # -- List all the packages (installed or not) if list_: - list_packages(platform) + resources.list_packages() ctx.exit(0) # -- Invalid option. Just show the help diff --git a/apio/commands/lint.py b/apio/commands/lint.py index 764d0317..5efe1776 100644 --- a/apio/commands/lint.py +++ b/apio/commands/lint.py @@ -98,7 +98,7 @@ def cli( """Lint the verilog code.""" # -- Create the scons object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) # -- Lint the project with the given parameters diff --git a/apio/commands/modify.py b/apio/commands/modify.py index 66c08b15..7a55f867 100644 --- a/apio/commands/modify.py +++ b/apio/commands/modify.py @@ -12,9 +12,9 @@ import click from click.core import Context from apio.managers.project import Project -from apio import util from apio import cmd_util from apio.commands import options +from apio.resources import Resources # --------------------------- @@ -62,10 +62,11 @@ def cli( # At least one of these options are required. cmd_util.check_required_params(ctx, nameof(board, top_module)) - project_dir = util.get_project_dir(project_dir) + # Load resources. + resources = Resources(project_dir=project_dir, project_scope=True) # Create the apio.ini file - ok = Project.modify_ini_file(project_dir, board, top_module) + ok = Project.modify_ini_file(resources, board, top_module) exit_code = 0 if ok else 1 ctx.exit(exit_code) diff --git a/apio/commands/raw.py b/apio/commands/raw.py index 9a7415ef..ec755ea3 100644 --- a/apio/commands/raw.py +++ b/apio/commands/raw.py @@ -76,7 +76,7 @@ def cli( # -- Make sure the oss-cad-suite is installed. if cmd: - resources = Resources() + resources = Resources(project_scope=False) pkg_util.check_required_packages(["oss-cad-suite"], resources) exit_code = util.call(cmd) else: diff --git a/apio/commands/report.py b/apio/commands/report.py index 86d8b241..0e3080ab 100644 --- a/apio/commands/report.py +++ b/apio/commands/report.py @@ -67,7 +67,7 @@ def cli( """Analyze the design and report timing.""" # -- Create the scons object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) # Run scons diff --git a/apio/commands/sim.py b/apio/commands/sim.py index 2f016009..29cc158d 100644 --- a/apio/commands/sim.py +++ b/apio/commands/sim.py @@ -61,7 +61,7 @@ def cli( """ # -- Create the scons object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) # -- Simulate the project with the given parameters diff --git a/apio/commands/system.py b/apio/commands/system.py index 978ef21b..626f87f2 100644 --- a/apio/commands/system.py +++ b/apio/commands/system.py @@ -104,7 +104,7 @@ def cli( cmd_util.check_exclusive_params(ctx, nameof(lsftdi, lsusb, lsserial, info)) # Load the various resource files. - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=False) # -- Create the system object system = System(resources) diff --git a/apio/commands/test.py b/apio/commands/test.py index 8f2ce4fa..d9dfd42c 100644 --- a/apio/commands/test.py +++ b/apio/commands/test.py @@ -60,7 +60,7 @@ def cli( """Implements the test command.""" # -- Create the scons object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) exit_code = scons.test({"testbench": testbench_file}) diff --git a/apio/commands/time.py b/apio/commands/time.py index 997746b0..7d20b35b 100644 --- a/apio/commands/time.py +++ b/apio/commands/time.py @@ -63,7 +63,7 @@ def cli( """Analyze the design and report timing.""" # -- Create the scons object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) # Run scons diff --git a/apio/commands/uninstall.py b/apio/commands/uninstall.py index 3b4032bb..9f89fe96 100644 --- a/apio/commands/uninstall.py +++ b/apio/commands/uninstall.py @@ -12,7 +12,7 @@ from varname import nameof import click from click.core import Context -from apio.managers.installer import Installer, list_packages +from apio.managers.installer import Installer from apio.profile import Profile from apio import cmd_util from apio.resources import Resources @@ -95,7 +95,11 @@ def cli( cmd_util.check_exclusive_params(ctx, nameof(packages, list_, all_)) # -- Load the resources. - resources = Resources(platform=platform, project_dir=project_dir) + resources = Resources( + platform=platform, + project_dir=project_dir, + project_scope=False, + ) # -- Uninstall the given apio packages if packages: @@ -112,7 +116,7 @@ def cli( # -- List all the packages (installed or not) if list_: - list_packages(platform) + resources.list_packages() ctx.exit(0) # -- Invalid option. Just show the help diff --git a/apio/commands/upload.py b/apio/commands/upload.py index 3f2d8ab4..da7b0909 100644 --- a/apio/commands/upload.py +++ b/apio/commands/upload.py @@ -93,7 +93,7 @@ def cli( """Implements the upload command.""" # -- Create a drivers object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) drivers = Drivers(resources) # -- Only for MAC diff --git a/apio/commands/verify.py b/apio/commands/verify.py index 2a7598ad..5c95fceb 100644 --- a/apio/commands/verify.py +++ b/apio/commands/verify.py @@ -56,7 +56,7 @@ def cli( """Implements the verify command.""" # -- Crete the scons object - resources = Resources(project_dir=project_dir) + resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) # -- Verify the project with the given parameters diff --git a/apio/managers/installer.py b/apio/managers/installer.py index 102b2062..d40c28be 100644 --- a/apio/managers/installer.py +++ b/apio/managers/installer.py @@ -14,7 +14,6 @@ import requests from apio import util -from apio.resources import Resources from apio.profile import Profile from apio.managers.downloader import FileDownloader from apio.managers.unpacker import FileUnpacker @@ -522,13 +521,3 @@ def _unpack(pkgpath: Path, pkgdir: Path): success = fileu.start() return success - - -def list_packages(platform: str): - """List all the available packages""" - - # -- Get all the resources - resources = Resources(platform=platform) - - # -- List the packages - resources.list_packages() diff --git a/apio/managers/project.py b/apio/managers/project.py index 2ba3543f..3cc04aec 100644 --- a/apio/managers/project.py +++ b/apio/managers/project.py @@ -38,14 +38,16 @@ def __init__(self, project_dir: Optional[Path]): self.native_exe_mode: bool = None @staticmethod - def create_ini(project_dir, board, top_module, sayyes=False) -> bool: + def create_ini( + resources: Resources, board: str, top_module: str, sayyes: bool = False + ) -> bool: """Creates a new apio project file. Returns True if ok.""" # -- Construct the path - ini_path = project_dir / PROJECT_FILENAME + ini_path = resources.project_dir / PROJECT_FILENAME # -- Verify that the board id is valid. - boards = Resources().boards + boards = resources.boards if board not in boards.keys(): click.secho(f"Error: no such board '{board}'", fg="red") return False @@ -85,17 +87,17 @@ def create_ini(project_dir, board, top_module, sayyes=False) -> bool: @staticmethod def modify_ini_file( - project_dir: Path, board: Optional[str], top_module: Optional[str] + resources: Resources, board: Optional[str], top_module: Optional[str] ) -> bool: """Update the current ini file with the given optional parameters. Returns True if ok.""" # -- construct the file path. - ini_path = project_dir / PROJECT_FILENAME + ini_path = resources.project_dir / PROJECT_FILENAME # -- Verify that the board id is valid. if board: - boards = Resources().boards + boards = resources.boards if board not in boards.keys(): click.secho( f"Error: no such board '{board}'.\n" diff --git a/apio/resources.py b/apio/resources.py index 3cbedec4..60e13544 100644 --- a/apio/resources.py +++ b/apio/resources.py @@ -63,11 +63,23 @@ class Resources: - """Resource manager. Class for accesing to all the resources""" + """Resource manager. Class for accesing to all the resources.""" def __init__( - self, *, platform: str = "", project_dir: Optional[Path] = None + self, + *, + project_scope: bool, + platform: str = "", + project_dir: Optional[Path] = None, ): + """Initializes the Resources object. 'project dir' is an optional path + to the project dir, otherwise, the current directory is used. + 'project_scope' indicates if project specfic resources such as + boards.json should be loaded, if available' or that the global + default resources should be used instead. Some commands such as + 'apio install' uses the global scope while commands such as + 'apio build' use the project scope. + """ # -- Maps the optional project_dir option to a path. self.project_dir: Path = util.get_project_dir(project_dir) @@ -78,14 +90,18 @@ def __init__( self.packages = self._load_resource(PACKAGES_JSON) # -- Read the boards information - self.boards = self._load_resource(BOARDS_JSON, allow_custom=True) + self.boards = self._load_resource( + BOARDS_JSON, allow_custom=project_scope + ) # -- Read the FPGAs information - self.fpgas = self._load_resource(FPGAS_JSON, allow_custom=True) + self.fpgas = self._load_resource( + FPGAS_JSON, allow_custom=project_scope + ) # -- Read the programmers information self.programmers = self._load_resource( - PROGRAMMERS_JSON, allow_custom=True + PROGRAMMERS_JSON, allow_custom=project_scope ) # -- Read the distribution information From ad7fe837d2280bd28ceac1e5daa9297584d0964a Mon Sep 17 00:00:00 2001 From: Zapta Date: Tue, 5 Nov 2024 08:42:29 -0800 Subject: [PATCH 37/51] Changed the invocation of 'verilator' (a perl script) to 'verilator_bin' (a binary) to avoid the dependency on perl. --- apio/pkg_util.py | 3 ++- apio/scons/scons_util.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index a96f52a2..b405f877 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -40,6 +40,7 @@ def _oss_cad_suite_package_env(package_path: Path) -> EnvMutations: str(package_path / "lib"), ], vars=[ + ("VERILATOR_ROOT", str(package_path / "share" / "verilator")), ("IVL", str(package_path / "lib" / "ivl")), ("ICEBOX", str(package_path / "share" / "icebox")), ("TRELLIS", str(package_path / "share" / "trellis")), @@ -134,7 +135,7 @@ def _dump_env_mutations(mutations: EnvMutations) -> None: """For debugging. Delete once stabalizing the new oss-cad-suite on windows.""" # batch = util.is_windows() - click.secho("Env Mutations:", fg="magenta") + click.secho("Env settings:", fg="magenta") # -- Print PATH mutations. windows = False diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index 9d1b45a1..dd3de379 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -527,7 +527,7 @@ def make_verilator_action( """ action = ( - "verilator --lint-only --bbox-unsup --timing " + "verilator_bin --lint-only --bbox-unsup --timing " "-Wno-TIMESCALEMOD -Wno-MULTITOP " "{0} {1} {2} {3} {4} {5} {6} {7} {8} $SOURCES" ).format( From ec4dd10dd57d385d12d4409e281acdc58c115ccc Mon Sep 17 00:00:00 2001 From: Zapta Date: Tue, 5 Nov 2024 08:50:01 -0800 Subject: [PATCH 38/51] Tweaked the environment setting dump. --- apio/pkg_util.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index b405f877..b2c946bb 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -134,25 +134,24 @@ def _get_env_mutations_for_packages() -> EnvMutations: def _dump_env_mutations(mutations: EnvMutations) -> None: """For debugging. Delete once stabalizing the new oss-cad-suite on windows.""" - # batch = util.is_windows() - click.secho("Env settings:", fg="magenta") + click.secho("Envirnment settings:", fg="magenta") # -- Print PATH mutations. - windows = False + windows = util.is_windows() for p in reversed(mutations.paths): styled_name = click.style("PATH", fg="magenta") if windows: - print(f" @set {styled_name}={p};%PATH%") + print(f"@set {styled_name}={p};%PATH%") else: - print(f' {styled_name}="{p}:$PATH"') + print(f'{styled_name}="{p}:$PATH"') # -- Print vars mutations. for name, val in mutations.vars: styled_name = click.style(name, fg="magenta") if windows: - print(f" @set {styled_name}={val}") + print(f"@set {styled_name}={val}") else: - print(f' {styled_name}="{val}"') + print(f'{styled_name}="{val}"') def _apply_env_mutations(mutations: EnvMutations) -> None: From ca9b44318e54e773afaa7db520acd0795171a6a5 Mon Sep 17 00:00:00 2001 From: Zapta Date: Tue, 5 Nov 2024 08:54:56 -0800 Subject: [PATCH 39/51] Added the flag '--quiet' to the verilator command line (lint) to supress the banner. --- apio/scons/scons_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index dd3de379..1e798d65 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -527,7 +527,7 @@ def make_verilator_action( """ action = ( - "verilator_bin --lint-only --bbox-unsup --timing " + "verilator_bin --lint-only --quiet --bbox-unsup --timing " "-Wno-TIMESCALEMOD -Wno-MULTITOP " "{0} {1} {2} {3} {4} {5} {6} {7} {8} $SOURCES" ).format( From 0160fefe689d8eebc80fff8a72dc81a1421f7fc3 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 9 Nov 2024 19:20:59 -0800 Subject: [PATCH 40/51] The time and the verify command now prints a message about their deprecation. No change in functionality. --- apio/commands/time.py | 6 ++++++ apio/commands/verify.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/apio/commands/time.py b/apio/commands/time.py index 7d20b35b..219dc0e3 100644 --- a/apio/commands/time.py +++ b/apio/commands/time.py @@ -62,6 +62,12 @@ def cli( ): """Analyze the design and report timing.""" + click.secho( + "The apio 'time' command is deprecated. " + "Please use the 'report' command instead.", + fg="yellow", + ) + # -- Create the scons object resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) diff --git a/apio/commands/verify.py b/apio/commands/verify.py index 5c95fceb..373d0d13 100644 --- a/apio/commands/verify.py +++ b/apio/commands/verify.py @@ -55,6 +55,12 @@ def cli( ): """Implements the verify command.""" + click.secho( + "The apio 'verify' command is deprecated. " + "Please use the 'lint' command instead.", + fg="yellow", + ) + # -- Crete the scons object resources = Resources(project_dir=project_dir, project_scope=True) scons = SCons(resources) From 283dd122fae398e0118fd710142afdfd15f2fa64 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 9 Nov 2024 22:38:01 -0800 Subject: [PATCH 41/51] Added the command 'apio packages' to list/install/uninstall packages and mark as deprecated the commands apio install/uninstall (they are still fully functional). --- .vscode/launch.json | 14 +++ apio/__main__.py | 1 + apio/cmd_util.py | 23 +++- apio/commands/examples.py | 2 +- apio/commands/install.py | 22 ++-- apio/commands/packages.py | 196 +++++++++++++++++++++++++++++++++ apio/commands/time.py | 4 +- apio/commands/uninstall.py | 21 ++-- apio/commands/verify.py | 4 +- apio/managers/installer.py | 2 +- apio/pkg_util.py | 4 +- apio/resources.py | 2 +- test/commands/test_build.py | 16 +-- test/commands/test_examples.py | 6 +- test/commands/test_packages.py | 41 +++++++ test/commands/test_report.py | 2 +- test/commands/test_system.py | 6 +- test/commands/test_time.py | 2 +- test/commands/test_verify.py | 2 +- test/test_end_to_end.py | 64 ++++++----- 20 files changed, 358 insertions(+), 76 deletions(-) create mode 100644 apio/commands/packages.py create mode 100644 test/commands/test_packages.py diff --git a/.vscode/launch.json b/.vscode/launch.json index afb360dd..84f46efa 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -144,6 +144,20 @@ "justMyCode": false, "cwd": "${workspaceFolder}/test-examples/Alhambra-II/02-jumping-LED" }, + { + "name": "Apio packages", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/apio_run.py", + "args": [ + "packages", + "--install", + "--force" + ], + "console": "integratedTerminal", + "justMyCode": false, + "cwd": "${workspaceFolder}/test-examples/Alhambra-II/02-jumping-LED" + }, { "name": "Apio raw", "type": "debugpy", diff --git a/apio/__main__.py b/apio/__main__.py index 1efd5e6a..01e2cd77 100644 --- a/apio/__main__.py +++ b/apio/__main__.py @@ -38,6 +38,7 @@ "Setup commands": [ "create", "modify", + "packages", "drivers", "install", "uninstall", diff --git a/apio/cmd_util.py b/apio/cmd_util.py index 28170338..5a9fbe78 100644 --- a/apio/cmd_util.py +++ b/apio/cmd_util.py @@ -26,7 +26,7 @@ def fatal_usage_error(ctx: click.Context, msg: str) -> None: msg: A single line short error message. """ # Mimiking the usage error message from click/exceptions.py. - # E.g. "Try 'apio install -h' for help." + # E.g. "Try 'apio packages -h' for help." click.secho(ctx.get_usage()) click.secho( f"Try '{ctx.command_path} {ctx.help_option_names[0]}' for help." @@ -135,6 +135,27 @@ def check_exclusive_params(ctx: click.Context, param_ids: List[str]) -> None: fatal_usage_error(ctx, f"{aliases_str} are mutually exclusive.") +def check_exactly_one_param(ctx: click.Context, param_ids: List[str]) -> None: + """Checks that at exactly one of given params is specified in + the command line. If more or less than one params is specified, exits the + program with a message and error status. + + Params are click options and arguments that are passed to a command. + Param ids are the names of variables that are used to pass options and + argument values to the command. A safe way to construct param_ids + is nameof(param_var1, param_var2, ...) + """ + # The the subset of ids of params that where used in the command. + specified_param_ids = _specified_params(ctx, param_ids) + # If more 2 or more print an error and exit. + if len(specified_param_ids) != 1: + canonical_aliases = _params_ids_to_aliases(ctx, param_ids) + aliases_str = ", ".join(canonical_aliases) + fatal_usage_error( + ctx, f"Exactly one of of {aliases_str} must be specified." + ) + + def check_required_params(ctx: click.Context, param_ids: List[str]) -> None: """Checks that at least one of given params is specified in the command line. If none of the params is specified, exits the diff --git a/apio/commands/examples.py b/apio/commands/examples.py index b17af94e..b8e7afd9 100644 --- a/apio/commands/examples.py +++ b/apio/commands/examples.py @@ -82,7 +82,7 @@ def cli( sayno: bool, ): """Manage verilog examples.\n - Install with `apio install examples`""" + Install with `apio packages --install examples`""" # Make sure these params are exclusive. cmd_util.check_exclusive_params(ctx, nameof(list_, dir_, files)) diff --git a/apio/commands/install.py b/apio/commands/install.py index fb41fd17..daf76560 100644 --- a/apio/commands/install.py +++ b/apio/commands/install.py @@ -18,6 +18,8 @@ from apio.commands import options +# R0801: Similar lines in 2 files +# pylint: disable=R0801 def install_packages( packages: list, platform: str, @@ -49,16 +51,8 @@ def install_packages( # -- COMMAND # --------------------------- HELP = """ -The install command lists and installs the apio packages. - -\b -Examples: - apio install --list # List packages - apio install --all # Install all packages - apio install --all -f # Force the re/installation of all packages - apio install examples # Install the examples package - -For packages uninstallation see the apio uninstall command. +The install command has been deprecated. Please use the 'apio packages' command +instead. """ @@ -67,7 +61,7 @@ def install_packages( # pylint: disable=too-many-positional-arguments @click.command( "install", - short_help="Install apio packages.", + short_help="[Depreciated] Install apio packages.", help=HELP, cls=cmd_util.ApioCommand, ) @@ -95,6 +89,12 @@ def cli( manage the installation of apio packages. """ + click.secho( + "The 'apio install' command is deprecated. " + "Please use the 'apio packages' command instead.", + fg="yellow", + ) + # Make sure these params are exclusive. cmd_util.check_exclusive_params(ctx, nameof(packages, all_, list_)) diff --git a/apio/commands/packages.py b/apio/commands/packages.py new file mode 100644 index 00000000..e2e56796 --- /dev/null +++ b/apio/commands/packages.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +# -- This file is part of the Apio project +# -- (C) 2016-2024 FPGAwars +# -- Authors +# -- * Jesús Arroyo (2016-2019) +# -- * Juan Gonzalez (obijuan) (2019-2024) +# -- Licence GPLv2 +"""Implementation of 'apio packages' command""" + +from pathlib import Path +from typing import Tuple +from varname import nameof +import click +from click.core import Context +from apio.managers.installer import Installer +from apio.resources import Resources +from apio import cmd_util +from apio.commands import options + + +# R0801: Similar lines in 2 files +# pylint: disable=R0801 +def _install( + packages: list, + platform: str, + resources: Resources, + force: bool, + verbose: bool, +): + """Install the apio packages passed as a list + * INPUTS: + - packages: List of packages (Ex. ['examples', 'oss-cad-suite']) + - platform: Specific platform (Advanced, just for developers) + - force: Force package installation + - verbose: Show detailed output. + """ + # -- Install packages, one by one... + for package in packages: + + # -- The instalation is performed by the Installer object + modifiers = Installer.Modifiers( + force=force, checkversion=True, verbose=verbose + ) + installer = Installer(package, platform, resources, modifiers) + + # -- Install the package! + installer.install() + + +# R0801: Similar lines in 2 files +# pylint: disable=R0801 +def _uninstall( + packages: list, platform: str, resources: Resources, sayyes, verbose: bool +): + """Uninstall the given list of packages""" + + # -- Ask the user for confirmation + num_packages = ( + "1 package" if len(packages) == 1 else f"{len(packages)} packages" + ) + if sayyes or click.confirm(f"Do you want to uninstall {num_packages}?"): + + # -- Uninstall packages, one by one + for package in packages: + + # -- The uninstalation is performed by the Installer object + modifiers = Installer.Modifiers( + force=False, checkversion=False, verbose=verbose + ) + installer = Installer(package, platform, resources, modifiers) + + # -- Uninstall the package! + installer.uninstall() + + # -- User quit! + else: + click.secho("Abort!", fg="red") + + +# --------------------------- +# -- COMMAND +# --------------------------- +HELP = """ +The packages command manages the apio packages which are required by most +of the apio commands. These are not python packages but apio +specific packages that contain various tools and data and they can be installed +after the apip python package is installed using 'pip install apip' or +similar command. Also note that some apio packages are available and required +only of some platforms but not on others. + +\b +Examples: + apio packages --list # List the apio packages. + apio packages --install # Install all missing packages. + apio packages --install --force # Re/install all missing packages. + apio packages --install oss-cad-suite # Install a specific package. + apio packages --install examples@0.0.32 # Install a specific version. + apio packages --uninstall # Uninstall all packages. + apio packages --uninstall oss-cad-suite # Uninstall only given package(s). + +Adding --force to --install forces the reinstallation of existing packages, +otherwise, packages that are already installed correctly are left with no +change. + +[Hint] In case of doubt, run 'apio packages --install --force' to reinstall +all packages from scratch. +""" + +install_option = click.option( + "install", # Var name. Deconflicting from Python'g builtin 'all'. + "-i", + "--install", + is_flag=True, + help="Install packages.", + cls=cmd_util.ApioOption, +) + +uninstall_option = click.option( + "uninstall", # Var name. Deconflicting from Python'g builtin 'all'. + "-u", + "--uninstall", + is_flag=True, + help="Uninstall packages.", + cls=cmd_util.ApioOption, +) + + +# pylint: disable=duplicate-code +# pylint: disable=too-many-arguments +# pylint: disable=too-many-positional-arguments +@click.command( + "install", + short_help="Manage the apio packages.", + help=HELP, + cls=cmd_util.ApioCommand, +) +@click.pass_context +@click.argument("packages", nargs=-1, required=False) +@options.list_option_gen(help="List packages.") +@install_option +@uninstall_option +@options.force_option_gen(help="Force installation.") +@options.project_dir_option +@options.platform_option +@options.sayyes +@options.verbose_option +def cli( + ctx: Context, + # Arguments + packages: Tuple[str], + # Options + list_: bool, + install, + uninstall, + force: bool, + platform: str, + project_dir: Path, + sayyes: bool, + verbose: bool, +): + """Implements the packages command which allows to manage the + apio packages. + """ + + # Validate the option combination. + cmd_util.check_exactly_one_param(ctx, nameof(list_, install, uninstall)) + cmd_util.check_exclusive_params(ctx, nameof(list_, force)) + cmd_util.check_exclusive_params(ctx, nameof(uninstall, force)) + + # -- Load the resources. We don't care about project specific resources. + resources = Resources( + platform=platform, + project_dir=project_dir, + project_scope=False, + ) + + # -- List all the packages (installed or not) + + if install: + # -- If packages not specified, use all. + if not packages: + packages = resources.packages + # -- Install the packages. + _install(packages, platform, resources, force, verbose) + ctx.exit(0) + + if uninstall: + # -- If packages not specified, use all. + if not packages: + packages = resources.packages + _uninstall(packages, platform, resources, sayyes, verbose) + ctx.exit(0) + + # -- Here it must be --list. + resources.list_packages() + ctx.exit(0) diff --git a/apio/commands/time.py b/apio/commands/time.py index 219dc0e3..ad88e73e 100644 --- a/apio/commands/time.py +++ b/apio/commands/time.py @@ -63,8 +63,8 @@ def cli( """Analyze the design and report timing.""" click.secho( - "The apio 'time' command is deprecated. " - "Please use the 'report' command instead.", + "The 'apio time' command is deprecated. " + "Please use the 'apio report' command instead.", fg="yellow", ) diff --git a/apio/commands/uninstall.py b/apio/commands/uninstall.py index 9f89fe96..257ca6dd 100644 --- a/apio/commands/uninstall.py +++ b/apio/commands/uninstall.py @@ -19,6 +19,8 @@ from apio.commands import options +# R0801: Similar lines in 2 files +# pylint: disable=R0801 def _uninstall( packages: list, platform: str, resources: Resources, sayyes, verbose: bool ): @@ -48,15 +50,8 @@ def _uninstall( # -- COMMAND # --------------------------- HELP = """ -The uninstall command lists and installs apio packages. - -\b -Examples: - apio uninstall --list # List packages - apio uninstall --all # Uninstall all packages - apio uninstall examples # Uninstall the examples package - -For packages installation see the apio install command. +The uninstall command has been deprecated. Please use the 'apio packages' +command instead. """ @@ -65,7 +60,7 @@ def _uninstall( # pylint: disable=too-many-positional-arguments @click.command( "uninstall", - short_help="Uninstall apio packages.", + short_help="[Depreciated] Uninstall apio packages.", help=HELP, cls=cmd_util.ApioCommand, ) @@ -91,6 +86,12 @@ def cli( ): """Implements the uninstall command.""" + click.secho( + "The 'apio uninstall' command is deprecated. " + "Please use the 'apio packages' command instead.", + fg="yellow", + ) + # Make sure these params are exclusive. cmd_util.check_exclusive_params(ctx, nameof(packages, list_, all_)) diff --git a/apio/commands/verify.py b/apio/commands/verify.py index 373d0d13..6ec5232e 100644 --- a/apio/commands/verify.py +++ b/apio/commands/verify.py @@ -56,8 +56,8 @@ def cli( """Implements the verify command.""" click.secho( - "The apio 'verify' command is deprecated. " - "Please use the 'lint' command instead.", + "The 'apio verify' command is deprecated. " + "Please use the 'apio lint' command instead.", fg="yellow", ) diff --git a/apio/managers/installer.py b/apio/managers/installer.py index d40c28be..5d55cd55 100644 --- a/apio/managers/installer.py +++ b/apio/managers/installer.py @@ -3,7 +3,7 @@ # -- (C) 2016-2021 FPGAwars # -- Author Jesús Arroyo # -- Licence GPLv2 -"""Implementation for the apio INSTALL command""" +"""Implementation for the apio PACKAGES command""" import sys import shutil diff --git a/apio/pkg_util.py b/apio/pkg_util.py index b2c946bb..5380d6a6 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -287,9 +287,9 @@ def _show_package_install_instructions(package_name: str): click.secho( "Please run:\n" - f" apio install {package_name} --force\n" + f" apio packages --install --force {package_name}\n" "or:\n" - " apio install --all --force", + " apio packages --install --force", fg="yellow", ) diff --git a/apio/resources.py b/apio/resources.py index 60e13544..02df5100 100644 --- a/apio/resources.py +++ b/apio/resources.py @@ -77,7 +77,7 @@ def __init__( 'project_scope' indicates if project specfic resources such as boards.json should be loaded, if available' or that the global default resources should be used instead. Some commands such as - 'apio install' uses the global scope while commands such as + 'apio packages' uses the global scope while commands such as 'apio build' use the project scope. """ # -- Maps the optional project_dir option to a path. diff --git a/test/commands/test_build.py b/test/commands/test_build.py index b2a7ee7f..f5b6d519 100644 --- a/test/commands/test_build.py +++ b/test/commands/test_build.py @@ -47,7 +47,7 @@ def test_build_board(clirunner, configenv): # -- Check the result assert result.exit_code != 0, result.output - assert "install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output def test_build_complete1(clirunner, configenv): @@ -61,12 +61,12 @@ def test_build_complete1(clirunner, configenv): # apio build --board icestick result = clirunner.invoke(cmd_build, ["--board", "icestick"]) assert result.exit_code == 1, result.output - assert "install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # apio build --fpga iCE40-HX1K-VQ100 result = clirunner.invoke(cmd_build, ["--fpga", "iCE40-HX1K-VQ100"]) assert result.exit_code == 1, result.output - assert "install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # apio build --type lp --size 8k --pack cm225:4k result = clirunner.invoke( @@ -80,7 +80,7 @@ def test_build_complete1(clirunner, configenv): cmd_build, ["--board", "icezum", "--size", "1k"] ) assert result.exit_code != 0, result.output - assert "install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # apio build --board icezum --fpga iCE40-HX1K-TQ144 --type hx result = clirunner.invoke( @@ -95,14 +95,14 @@ def test_build_complete1(clirunner, configenv): ], ) assert result.exit_code != 0, result.output - assert "install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # apio build --board icezum --pack tq144 result = clirunner.invoke( cmd_build, ["--board", "icezum", "--pack", "tq144"] ) assert result.exit_code != 0, result.output - assert "install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # apio build --fpga iCE40-HX1K-TQ144 --pack tq144 --size 1k result = clirunner.invoke( @@ -110,14 +110,14 @@ def test_build_complete1(clirunner, configenv): ["--fpga", "iCE40-HX1K-TQ144", "--pack", "tq144", "--size", "1k"], ) assert result.exit_code != 0, result.output - assert "install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # apio build --fpga iCE40-HX1K-TQ144 --type hx result = clirunner.invoke( cmd_build, ["--fpga", "iCE40-HX1K-TQ144", "--type", "hx"] ) assert result.exit_code != 0, result.output - assert "install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # apio build --board icezum --size 8k result = clirunner.invoke( diff --git a/test/commands/test_examples.py b/test/commands/test_examples.py index be1f6c48..9b6be1bc 100644 --- a/test/commands/test_examples.py +++ b/test/commands/test_examples.py @@ -21,14 +21,14 @@ def test_examples(clirunner, validate_cliresult, configenv): # -- Execute "apio examples --list" result = clirunner.invoke(cmd_examples, ["--list"]) assert result.exit_code == 1, result.output - assert "apio install examples" in result.output + assert "apio packages --install --force examples" in result.output # -- Execute "apio examples --dir dir" result = clirunner.invoke(cmd_examples, ["--dir", "dir"]) assert result.exit_code == 1, result.output - assert "apio install examples" in result.output + assert "apio packages --install --force examples" in result.output # -- Execute "apio examples --files file" result = clirunner.invoke(cmd_examples, ["--files", "file"]) assert result.exit_code == 1, result.output - assert "apio install examples" in result.output + assert "apio packages --install --force examples" in result.output diff --git a/test/commands/test_packages.py b/test/commands/test_packages.py new file mode 100644 index 00000000..ffc15622 --- /dev/null +++ b/test/commands/test_packages.py @@ -0,0 +1,41 @@ +""" + Test for the "apio packages" command +""" + +# -- apio packages entry point +from apio.commands.packages import cli as cmd_packages + + +def test_packages(clirunner, configenv, validate_cliresult): + """Test "apio packages" with different parameters""" + + with clirunner.isolated_filesystem(): + + # -- Config the environment (conftest.configenv()) + configenv() + + # -- Execute "apio packages" + result = clirunner.invoke(cmd_packages) + assert result.exit_code == 1, result.output + assert ( + "Exactly one of of --list, --install, --uninstall " + "must be specified" in result.output + ) + + # -- Execute "apio packages --list" + result = clirunner.invoke(cmd_packages, ["--list"]) + validate_cliresult(result) + + # -- Execute "apio packages --install missing_package" + result = clirunner.invoke( + cmd_packages, ["--install", "missing_package"] + ) + assert result.exit_code == 1, result.output + assert "Error: no such package" in result.output + + # -- Execute "apio packages --uninstall --sayyes missing_package" + result = clirunner.invoke( + cmd_packages, ["--uninstall", "--sayyes", "missing_package"] + ) + assert result.exit_code == 1, result.output + assert "Error: no such package" in result.output diff --git a/test/commands/test_report.py b/test/commands/test_report.py index 578ef7c9..4aa4296c 100644 --- a/test/commands/test_report.py +++ b/test/commands/test_report.py @@ -43,4 +43,4 @@ def test_report_board(clirunner, configenv): # -- Check the result assert result.exit_code != 0, result.output - assert "apio install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output diff --git a/test/commands/test_system.py b/test/commands/test_system.py index f4aacbae..d0f6ff10 100644 --- a/test/commands/test_system.py +++ b/test/commands/test_system.py @@ -21,17 +21,17 @@ def test_system(clirunner, validate_cliresult, configenv): # -- Execute "apio system --lsftdi" result = clirunner.invoke(cmd_system, ["--lsftdi"]) assert result.exit_code == 1, result.output - assert "apio install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # -- Execute "apio system --lsusb" result = clirunner.invoke(cmd_system, ["--lsusb"]) assert result.exit_code == 1, result.output - assert "apio install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # -- Execute "apio system --lsserial" clirunner.invoke(cmd_system, ["--lsserial"]) assert result.exit_code == 1, result.output - assert "apio install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output # -- Execute "apio system --info" result = clirunner.invoke(cmd_system, ["--info"]) diff --git a/test/commands/test_time.py b/test/commands/test_time.py index ea48411d..57f8fa57 100644 --- a/test/commands/test_time.py +++ b/test/commands/test_time.py @@ -43,4 +43,4 @@ def test_time_board(clirunner, configenv): # -- Check the result assert result.exit_code != 0, result.output - assert "apio install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output diff --git a/test/commands/test_verify.py b/test/commands/test_verify.py index 9c2cdc52..05fb66da 100644 --- a/test/commands/test_verify.py +++ b/test/commands/test_verify.py @@ -22,4 +22,4 @@ def test_verify(clirunner, configenv): # -- Check the result assert result.exit_code != 0, result.output - assert "apio install oss-cad-suite" in result.output + assert "apio packages --install --force oss-cad-suite" in result.output diff --git a/test/test_end_to_end.py b/test/test_end_to_end.py index fbf488a0..cd21f6ef 100644 --- a/test/test_end_to_end.py +++ b/test/test_end_to_end.py @@ -6,10 +6,8 @@ import pytest -# -- Entry point for the apio install, apio uninstall -# -- apio create, apio upload, apio examples -from apio.commands.install import cli as cmd_install -from apio.commands.uninstall import cli as cmd_uninstall +# -- Entry point for apio commands. +from apio.commands.packages import cli as cmd_packages from apio.commands.create import cli as cmd_create from apio.commands.upload import cli as cmd_upload from apio.commands.examples import cli as cmd_examples @@ -54,46 +52,52 @@ def test_end_to_end1(clirunner, validate_cliresult, configenv, offline): # -- Config the environment (conftest.configenv()) configenv() - # -- Execute "apio uninstall examples" - result = clirunner.invoke(cmd_uninstall, ["examples"], input="y") - assert "Do you want to uninstall?" in result.output + # -- Execute "apio packages --uninstall examples" + result = clirunner.invoke( + cmd_packages, ["--uninstall", "examples"], input="y" + ) + assert "Do you want to uninstall 1 package?" in result.output assert "Error: package 'examples' is not installed" in result.output - # -- Execute "apio install examples@X" - result = clirunner.invoke(cmd_install, ["examples@X"]) + # -- Execute "apio packages --install examples@X" + result = clirunner.invoke(cmd_packages, ["--install", "examples@X"]) assert "Error: package not found" in result.output - # -- Execute "apio install examples@0.0.34" - result = clirunner.invoke(cmd_install, ["examples@0.0.34"]) + # -- Execute "apio packages --install examples@0.0.34" + result = clirunner.invoke( + cmd_packages, ["--install", "examples@0.0.34"] + ) validate_cliresult(result) assert "Installing examples package" in result.output assert "Download" in result.output assert "has been successfully installed!" in result.output - # -- Execute "apio install examples" - result = clirunner.invoke(cmd_install, ["examples"]) + # -- Execute "apio packages --install examples" + result = clirunner.invoke(cmd_packages, ["--install", "examples"]) validate_cliresult(result) assert "Installing examples package" in result.output assert "Download" in result.output assert "has been successfully installed!" in result.output - # -- Execute "apio install examples" again - result = clirunner.invoke(cmd_install, ["examples"]) + # -- Execute "apio packages --install examples" again + result = clirunner.invoke(cmd_packages, ["--install", "examples"]) validate_cliresult(result) assert "Installing examples package" in result.output assert "Already installed. Version " in result.output - # -- Execute "apio install examples --platform windows --force" + # -- Execute + # -- "apio packages --install examples --platform windows --force" result = clirunner.invoke( - cmd_install, ["examples", "--platform", "windows", "--force"] + cmd_packages, + ["--install", "examples", "--platform", "windows", "--force"], ) validate_cliresult(result) assert "Installing examples package" in result.output assert "Download" in result.output assert "has been successfully installed!" in result.output - # -- Execute "apio install --list" - result = clirunner.invoke(cmd_install, ["--list"]) + # -- Execute "apio packages --install --list" + result = clirunner.invoke(cmd_packages, ["--list"]) validate_cliresult(result) assert "Installed packages:" in result.output assert "examples" in result.output @@ -123,8 +127,8 @@ def test_end_to_end2(clirunner, validate_cliresult, configenv, offline): assert result.exit_code == 1, result.output assert "package 'oss-cad-suite' is not installed" in result.output - # -- Execute "apio install examples" - result = clirunner.invoke(cmd_install, ["examples"]) + # -- Execute "apio packages --install examples" + result = clirunner.invoke(cmd_packages, ["--install", "examples"]) validate_cliresult(result) assert "Installing examples package" in result.output assert "Download" in result.output @@ -185,8 +189,8 @@ def test_end_to_end3(clirunner, validate_cliresult, configenv, offline): # -- Config the environment (conftest.configenv()) configenv() - # -- Execute "apio install examples" - result = clirunner.invoke(cmd_install, ["examples"]) + # -- Execute "apio packages --install examples" + result = clirunner.invoke(cmd_packages, ["--install", "examples"]) validate_cliresult(result) assert "Installing examples package" in result.output assert "Download" in result.output @@ -220,14 +224,18 @@ def test_end_to_end3(clirunner, validate_cliresult, configenv, offline): assert "has been successfully created!" in result.output validate_dir_leds("tmp") - # -- Execute "apio uninstall examples" - result = clirunner.invoke(cmd_uninstall, ["examples"], input="n") + # -- Execute "apio packages --uninstall examples" + result = clirunner.invoke( + cmd_packages, ["--uninstall", "examples"], input="n" + ) validate_cliresult(result) assert "Abort!" in result.output - # -- Execute "apio uninstall examples" - result = clirunner.invoke(cmd_uninstall, ["examples"], input="y") + # -- Execute "apio packages --uninstall examples" + result = clirunner.invoke( + cmd_packages, ["--uninstall", "examples"], input="y" + ) validate_cliresult(result) assert "Uninstalling examples package" in result.output - assert "Do you want to uninstall?" in result.output + assert "Do you want to uninstall 1 package?" in result.output assert "has been successfully uninstalled!" in result.output From b7c26d8a84cf1768f3b73d54676733d3e2158475 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sat, 9 Nov 2024 22:42:18 -0800 Subject: [PATCH 42/51] Renamed two param checking functions for clarity. No change in functionality. --- apio/cmd_util.py | 4 ++-- apio/commands/boards.py | 2 +- apio/commands/drivers.py | 2 +- apio/commands/examples.py | 2 +- apio/commands/install.py | 2 +- apio/commands/modify.py | 2 +- apio/commands/packages.py | 4 ++-- apio/commands/system.py | 4 +++- apio/commands/uninstall.py | 2 +- 9 files changed, 13 insertions(+), 11 deletions(-) diff --git a/apio/cmd_util.py b/apio/cmd_util.py index 5a9fbe78..d5f1fbaa 100644 --- a/apio/cmd_util.py +++ b/apio/cmd_util.py @@ -116,7 +116,7 @@ def _specified_params(ctx: click.Context, param_ids: List[str]) -> List[str]: return result -def check_exclusive_params(ctx: click.Context, param_ids: List[str]) -> None: +def check_at_most_one_param(ctx: click.Context, param_ids: List[str]) -> None: """Checks that at most one of given params were specified in the command line. If more than one param was specified, exits the program with a message and error status. @@ -156,7 +156,7 @@ def check_exactly_one_param(ctx: click.Context, param_ids: List[str]) -> None: ) -def check_required_params(ctx: click.Context, param_ids: List[str]) -> None: +def check_at_least_one_param(ctx: click.Context, param_ids: List[str]) -> None: """Checks that at least one of given params is specified in the command line. If none of the params is specified, exits the program with a message and error status. diff --git a/apio/commands/boards.py b/apio/commands/boards.py index 83d38855..aae03ba2 100644 --- a/apio/commands/boards.py +++ b/apio/commands/boards.py @@ -72,7 +72,7 @@ def cli( """ # Make sure these params are exclusive. - cmd_util.check_exclusive_params(ctx, nameof(list_, fpgas)) + cmd_util.check_at_most_one_param(ctx, nameof(list_, fpgas)) # -- Access to the apio resources. We need project scope since the project # -- may override the list of boards. diff --git a/apio/commands/drivers.py b/apio/commands/drivers.py index 61db34fc..6da130e3 100644 --- a/apio/commands/drivers.py +++ b/apio/commands/drivers.py @@ -92,7 +92,7 @@ def cli( """Implements the drivers command.""" # Make sure these params are exclusive. - cmd_util.check_exclusive_params( + cmd_util.check_at_most_one_param( ctx, nameof(ftdi_enable, ftdi_disable, serial_enable, serial_disable) ) diff --git a/apio/commands/examples.py b/apio/commands/examples.py index b8e7afd9..3ddba1b4 100644 --- a/apio/commands/examples.py +++ b/apio/commands/examples.py @@ -85,7 +85,7 @@ def cli( Install with `apio packages --install examples`""" # Make sure these params are exclusive. - cmd_util.check_exclusive_params(ctx, nameof(list_, dir_, files)) + cmd_util.check_at_most_one_param(ctx, nameof(list_, dir_, files)) # -- Access to the Drivers resources = Resources(project_scope=False) diff --git a/apio/commands/install.py b/apio/commands/install.py index daf76560..efb1ddd5 100644 --- a/apio/commands/install.py +++ b/apio/commands/install.py @@ -96,7 +96,7 @@ def cli( ) # Make sure these params are exclusive. - cmd_util.check_exclusive_params(ctx, nameof(packages, all_, list_)) + cmd_util.check_at_most_one_param(ctx, nameof(packages, all_, list_)) # -- Load the resources. We don't care about project specific resources. resources = Resources( diff --git a/apio/commands/modify.py b/apio/commands/modify.py index 7a55f867..92a8b3d5 100644 --- a/apio/commands/modify.py +++ b/apio/commands/modify.py @@ -60,7 +60,7 @@ def cli( """Modify the project file.""" # At least one of these options are required. - cmd_util.check_required_params(ctx, nameof(board, top_module)) + cmd_util.check_at_least_one_param(ctx, nameof(board, top_module)) # Load resources. resources = Resources(project_dir=project_dir, project_scope=True) diff --git a/apio/commands/packages.py b/apio/commands/packages.py index e2e56796..a08681b7 100644 --- a/apio/commands/packages.py +++ b/apio/commands/packages.py @@ -164,8 +164,8 @@ def cli( # Validate the option combination. cmd_util.check_exactly_one_param(ctx, nameof(list_, install, uninstall)) - cmd_util.check_exclusive_params(ctx, nameof(list_, force)) - cmd_util.check_exclusive_params(ctx, nameof(uninstall, force)) + cmd_util.check_at_most_one_param(ctx, nameof(list_, force)) + cmd_util.check_at_most_one_param(ctx, nameof(uninstall, force)) # -- Load the resources. We don't care about project specific resources. resources = Resources( diff --git a/apio/commands/system.py b/apio/commands/system.py index 626f87f2..3fec83ac 100644 --- a/apio/commands/system.py +++ b/apio/commands/system.py @@ -101,7 +101,9 @@ def cli( system tools""" # Make sure these params are exclusive. - cmd_util.check_exclusive_params(ctx, nameof(lsftdi, lsusb, lsserial, info)) + cmd_util.check_at_most_one_param( + ctx, nameof(lsftdi, lsusb, lsserial, info) + ) # Load the various resource files. resources = Resources(project_dir=project_dir, project_scope=False) diff --git a/apio/commands/uninstall.py b/apio/commands/uninstall.py index 257ca6dd..eeab00b3 100644 --- a/apio/commands/uninstall.py +++ b/apio/commands/uninstall.py @@ -93,7 +93,7 @@ def cli( ) # Make sure these params are exclusive. - cmd_util.check_exclusive_params(ctx, nameof(packages, list_, all_)) + cmd_util.check_at_most_one_param(ctx, nameof(packages, list_, all_)) # -- Load the resources. resources = Resources( From 141478bc9fa1e01394b62aba23269c87f04781b9 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 10 Nov 2024 12:10:33 -0800 Subject: [PATCH 43/51] Move the definition of the packages env settings from pkg_util.py to the package specifications in packages.json. This eliminate the redundant table in pkg_util.py. To enable it, Resources now has two lists of packages, self.all_packages with all the packages from packages.json and self.platform_packages with the subset of packages that are applicable to this platform. --- apio/commands/install.py | 2 +- apio/commands/packages.py | 4 +- apio/commands/raw.py | 18 +-- apio/managers/drivers.py | 4 +- apio/managers/examples.py | 2 +- apio/managers/installer.py | 4 +- apio/managers/scons.py | 2 +- apio/managers/system.py | 2 +- apio/pkg_util.py | 229 ++++++++++++----------------------- apio/resources.py | 72 ++++++----- apio/resources/packages.json | 37 ++++-- apio/util.py | 39 ++++++ 12 files changed, 206 insertions(+), 209 deletions(-) diff --git a/apio/commands/install.py b/apio/commands/install.py index efb1ddd5..0ba9b29a 100644 --- a/apio/commands/install.py +++ b/apio/commands/install.py @@ -114,7 +114,7 @@ def cli( if all_: # -- Install all the available packages for this platform! install_packages( - resources.packages, platform, resources, force, verbose + resources.platform_packages, platform, resources, force, verbose ) ctx.exit(0) diff --git a/apio/commands/packages.py b/apio/commands/packages.py index a08681b7..e3d42bc8 100644 --- a/apio/commands/packages.py +++ b/apio/commands/packages.py @@ -179,7 +179,7 @@ def cli( if install: # -- If packages not specified, use all. if not packages: - packages = resources.packages + packages = resources.platform_packages # -- Install the packages. _install(packages, platform, resources, force, verbose) ctx.exit(0) @@ -187,7 +187,7 @@ def cli( if uninstall: # -- If packages not specified, use all. if not packages: - packages = resources.packages + packages = resources.platform_packages _uninstall(packages, platform, resources, sayyes, verbose) ctx.exit(0) diff --git a/apio/commands/raw.py b/apio/commands/raw.py index ec755ea3..9fecec68 100644 --- a/apio/commands/raw.py +++ b/apio/commands/raw.py @@ -71,15 +71,19 @@ def cli( if not cmd and not env: cmd_util.fatal_usage_error(ctx, "Missing an option or a command") - # -- Set the system env for using the packages. - pkg_util.set_env_for_packages(verbose=env) + # -- Set the system env for the packages. This both dumps the env settings + # -- if --env option is specifies and prepare the env for the command + # -- execution below. + if cmd or env: + resources = Resources(project_scope=False) + pkg_util.set_env_for_packages(resources, verbose=env) - # -- Make sure the oss-cad-suite is installed. if cmd: - resources = Resources(project_scope=False) + # -- Make sure that at least the oss-cad-suite is installed. pkg_util.check_required_packages(["oss-cad-suite"], resources) + + # -- Invoke the command. exit_code = util.call(cmd) - else: - exit_code = 0 + ctx.exit(exit_code) - ctx.exit(exit_code) + ctx.exit(0) diff --git a/apio/managers/drivers.py b/apio/managers/drivers.py index 74780778..b2f53eb8 100644 --- a/apio/managers/drivers.py +++ b/apio/managers/drivers.py @@ -445,7 +445,7 @@ def _ftdi_enable_windows(self) -> int: pkg_util.check_required_packages(["drivers"], self.resources) # -- Get the drivers apio package base folder - drivers_base_dir = pkg_util.get_package_dir("drivers") + drivers_base_dir = self.resources.get_package_dir("drivers") # -- Path to the zadig.ini file # -- It is the zadig config file @@ -498,7 +498,7 @@ def _serial_enable_windows(self) -> int: # -- Check that the required packages exist. pkg_util.check_required_packages(["drivers"], self.resources) - drivers_base_dir = pkg_util.get_package_dir("drivers") + drivers_base_dir = self.resources.get_package_dir("drivers") drivers_bin_dir = drivers_base_dir / "bin" click.secho("\nStarting the interactive Serial Installer.", fg="green") diff --git a/apio/managers/examples.py b/apio/managers/examples.py index bc7d99d5..46e594a4 100644 --- a/apio/managers/examples.py +++ b/apio/managers/examples.py @@ -49,7 +49,7 @@ def __init__(self, resources: Resources): self.resources = resources # -- Folder where the example packages was installed - self.examples_dir = pkg_util.get_package_dir("examples") + self.examples_dir = resources.get_package_dir("examples") def get_examples_infos(self) -> Optional[List[ExampleInfo]]: """Scans the examples and returns a list of ExampleInfos. diff --git a/apio/managers/installer.py b/apio/managers/installer.py index 5d55cd55..ff1a20bc 100644 --- a/apio/managers/installer.py +++ b/apio/managers/installer.py @@ -94,12 +94,12 @@ def __init__( # -- If the package is known... # --(It is defined in the resources/packages.json file) - if self.package in self.resources.packages: + if self.package in self.resources.platform_packages: # -- Store the package dir self.packages_dir = util.get_home_dir() / dirname # Get the data of the given package - data = self.resources.packages[self.package] + data = self.resources.platform_packages[self.package] # Get the information about the valid versions distribution = self.resources.distribution diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 3be3baf2..1c5bfdd8 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -983,7 +983,7 @@ def _run( required_packages_names, self.resources ) - pkg_util.set_env_for_packages() + pkg_util.set_env_for_packages(self.resources) # -- Execute scons return self._execute_scons(command, variables, board) diff --git a/apio/managers/system.py b/apio/managers/system.py index d8e8d0aa..52e0e81e 100644 --- a/apio/managers/system.py +++ b/apio/managers/system.py @@ -156,7 +156,7 @@ def _run_command( pkg_util.check_required_packages(["oss-cad-suite"], self.resources) # -- Set system env for using the packages. - pkg_util.set_env_for_packages() + pkg_util.set_env_for_packages(self.resources) # pylint: disable=fixme # TODO: Is this necessary or does windows accepts commands without diff --git a/apio/pkg_util.py b/apio/pkg_util.py index 5380d6a6..dc0682a8 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -9,7 +9,7 @@ # ---- Licence Apache v2 """Utility functions related to apio packages.""" -from typing import List, Callable, Dict, Tuple, Optional +from typing import List, Callable, Tuple, Optional from pathlib import Path from dataclasses import dataclass import os @@ -31,50 +31,6 @@ class EnvMutations: vars: List[Tuple[str, str]] -def _oss_cad_suite_package_env(package_path: Path) -> EnvMutations: - """Returns the env mutations for the oss-cad-suite package.""" - - return EnvMutations( - paths=[ - str(package_path / "bin"), - str(package_path / "lib"), - ], - vars=[ - ("VERILATOR_ROOT", str(package_path / "share" / "verilator")), - ("IVL", str(package_path / "lib" / "ivl")), - ("ICEBOX", str(package_path / "share" / "icebox")), - ("TRELLIS", str(package_path / "share" / "trellis")), - ("YOSYS_LIB", str(package_path / "share" / "yosys")), - ], - ) - - -def _examples_package_env(_: Path) -> None: - """Returns the env mutations for the examples package.""" - return EnvMutations( - paths=[], - vars=[], - ) - - -def _graphviz_package_env(package_path: Path) -> None: - """Returns the env mutations for the graphviz package.""" - - return EnvMutations( - paths=[str(package_path / "bin")], - vars=[], - ) - - -def _drivers_package_env(package_path: Path) -> None: - """Returns the env mutations for the drivers package.""" - - return EnvMutations( - paths=[str(package_path / "bin")], - vars=[], - ) - - @dataclass(frozen=True) class _PackageDesc: """Represents an entry in the packages table.""" @@ -87,47 +43,59 @@ class _PackageDesc: env_func: Callable[[Path], EnvMutations] -# pylint: disable=fixme -# -- TODO: Harmonize this table with the packages.json resource file. -# -- currently they are updated independent, with some overlap. -# -- -# -- A dictionary that maps package names to package entries. -# -- The order determines the order of their respective paths in the -# -- system env PATH variable, and it may matter. -_PACKAGES: Dict[str, _PackageDesc] = { - "oss-cad-suite": _PackageDesc( - folder_name="tools-oss-cad-suite", - platform_match=True, - env_func=_oss_cad_suite_package_env, - ), - "examples": _PackageDesc( - folder_name="examples", - platform_match=True, - env_func=_examples_package_env, - ), - "graphviz": _PackageDesc( - folder_name="tools-graphviz", - platform_match=util.is_windows(), - env_func=_graphviz_package_env, - ), - "drivers": _PackageDesc( - folder_name="tools-drivers", - platform_match=util.is_windows(), - env_func=_drivers_package_env, - ), -} - - -def _get_env_mutations_for_packages() -> EnvMutations: +def _expand_env_template(template: str, package_path: Path) -> str: + """Fills a packages env value template as they appear in packages.json. + Currently it recognizes only a single place holder '%p' representing the + package absolute path. The '%p" can appear only at the begigning of the + template. + + E.g. '%p/bin' -> '/users/user/.apio/packages/drivers/bin' + """ + + # Case 1: No place holder. + if "%p" not in template: + return template + + # Case 2: The template contains only the placeholder. + if template == "%p": + return str(package_path) + + # Case 3: The place holder is the prefix of the template's path. + if template.startswith("%p/"): + return str(package_path / template[3:]) + + # Case 4: Unsupported. + raise RuntimeError(f"Invalid env template: [{template}]") + + +def _get_env_mutations_for_packages(resources: Resources) -> EnvMutations: """Collects the env mutation for each of the defined packages, in the order they are defined.""" + result = EnvMutations([], []) - for package_name, package_desc in _PACKAGES.items(): - if package_desc.platform_match: - package_path = get_package_dir(package_name) - mutations = package_desc.env_func(package_path) - result.paths.extend(mutations.paths) - result.vars.extend(mutations.vars) + for package_name, package_config in resources.platform_packages.items(): + + # -- Get the package root dir. + package_path = resources.get_package_dir(package_name) + + # -- Get the json env section. We require it, even if it's empty, + # -- for clarity reasons. + package_env = package_config["env"] + + # -- Collect the path values. + path_section = package_env.get("path", {}) + for path_template in path_section: + # -- Replaces place holders, if nay. + path_value = _expand_env_template(path_template, package_path) + result.paths.append(path_value) + + # -- Collect the vars (name, value) pairs. + vars_section = package_env.get("vars", {}) + for var_name, var_template in vars_section.items(): + # -- Replaces place holders, if nay. + var_value = _expand_env_template(var_template, package_path) + result.vars.append((var_name, var_value)) + return result @@ -168,13 +136,13 @@ def _apply_env_mutations(mutations: EnvMutations) -> None: os.environ[name] = value -def set_env_for_packages(verbose: bool = False) -> None: +def set_env_for_packages(resources: Resources, verbose: bool = False) -> None: """Sets the environment variables for using all the that are available for this platform, even if currently not installed. """ # -- Collect the env mutations for all packages. - mutations = _get_env_mutations_for_packages() + mutations = _get_env_mutations_for_packages(resources) if verbose: _dump_env_mutations(mutations) @@ -203,21 +171,33 @@ def check_required_packages( # -- Check packages for package_name in required_packages_names: - package_desc = _PACKAGES[package_name] - if package_desc.platform_match: - current_version = installed_packages.get(package_name, {}).get( - "version", None - ) - spec_version = spec_packages.get(package_name, "") - _check_required_package( - package_name, current_version, spec_version - ) + # -- Package name must be in all_packages. Otherwise it's a programming + # -- error. + if package_name not in resources.all_packages: + raise RuntimeError(f"Unknown package named [{package_name}]") + + # -- Skip if packages is not applicable to this platform. + if package_name not in resources.platform_packages: + continue + + # -- The package is applicable to this platform. Check installed + # -- version, if at all. + current_version = installed_packages.get(package_name, {}).get( + "version", None + ) + + # -- Check the installed version against the required version. + spec_version = spec_packages.get(package_name, "") + _check_required_package( + package_name, current_version, spec_version, resources + ) def _check_required_package( package_name: str, current_version: Optional[str], spec_version: str, + resources: Resources, ) -> None: """Checks that the package with the given packages is installed and has a version that meets the requirements. If any error, it prints an @@ -227,6 +207,7 @@ def _check_required_package( 'current_version' - the version of the install package or None if not installed. 'spec_version' - a specification of the required version. + 'resources' - the apio resources. """ # -- Case 1: Package is not installed. if current_version is None: @@ -249,7 +230,7 @@ def _check_required_package( sys.exit(1) # -- Case 3: The package's directory does not exist. - package_dir = get_package_dir(package_name) + package_dir = resources.get_package_dir(package_name) if package_dir and not package_dir.is_dir(): message = f"Error: package '{package_name}' is installed but missing" click.secho(message, fg="red") @@ -292,61 +273,3 @@ def _show_package_install_instructions(package_name: str): " apio packages --install --force", fg="yellow", ) - - -def _get_packages_dir() -> Path: - """Return the base directory of apio packages. - Packages are installed in the following folder: - * Default: $APIO_HOME_DIR/packages - * $APIO_PKG_DIR/packages: if the APIO_PKG_DIR env variable is set - * INPUT: - - pkg_name: Package name (Ex. 'examples') - * OUTPUT: - - The package folder (PosixPath) - (Ex. '/home/obijuan/.apio/packages/examples')) - - or None if the packageis not installed - """ - - # -- Get the apio home dir: - # -- Ex. '/home/obijuan/.apio' - apio_home_dir = util.get_home_dir() - - # -- Get the APIO_PKG_DIR env variable - # -- It returns None if it was not defined - apio_pkg_dir_env = util.get_projconf_option_dir("pkg_dir") - - # -- Get the pkg base dir. It is what the APIO_PKG_DIR env variable - # -- says, or the default folder if None - if apio_pkg_dir_env: - pkg_home_dir = Path(apio_pkg_dir_env) - - # -- Default value - else: - pkg_home_dir = apio_home_dir - - # -- Create the package folder - # -- Ex '/home/obijuan/.apio/packages/tools-oss-cad-suite' - package_dir = pkg_home_dir / "packages" - - # -- Return the folder if it exists - # if package_dir.exists(): - return package_dir - - -def get_package_dir(package_name: str) -> Path: - """Return the APIO package dir of a given package - Packages are installed in the following folder: - * Default: $APIO_HOME_DIR/packages - * $APIO_PKG_DIR/packages: if the APIO_PKG_DIR env variable is set - * INPUT: - - pkg_name: Package name (Ex. 'examples') - * OUTPUT: - - The package folder (PosixPath) - (Ex. '/home/obijuan/.apio/packages/examples')) - - or None if the packageis not installed - """ - - package_folder = _PACKAGES[package_name].folder_name - package_dir = _get_packages_dir() / package_folder - - return package_dir diff --git a/apio/resources.py b/apio/resources.py index 02df5100..2339b95e 100644 --- a/apio/resources.py +++ b/apio/resources.py @@ -62,6 +62,7 @@ DISTRIBUTION_JSON = "distribution.json" +# pylint: disable=too-many-instance-attributes class Resources: """Resource manager. Class for accesing to all the resources.""" @@ -87,7 +88,12 @@ def __init__( self.profile = Profile() # -- Read the apio packages information - self.packages = self._load_resource(PACKAGES_JSON) + self.all_packages = self._load_resource(PACKAGES_JSON) + + # The subset of packages that are applicable to this platform. + self.platform_packages = self._select_platform_packages( + self.all_packages, platform + ) # -- Read the boards information self.boards = self._load_resource( @@ -107,13 +113,13 @@ def __init__( # -- Read the distribution information self.distribution = self._load_resource(DISTRIBUTION_JSON) - # -- Filter packages: Store only the packages for - # -- the given platform - self._filter_packages(platform) + # --------- Sort resources for consistency and intunitiveness. + self.all_packages = OrderedDict( + sorted(self.all_packages.items(), key=lambda t: t[0]) + ) - # --------- Sort resources - self.packages = OrderedDict( - sorted(self.packages.items(), key=lambda t: t[0]) + self.platform_packages = OrderedDict( + sorted(self.platform_packages.items(), key=lambda t: t[0]) ) self.boards = OrderedDict( @@ -214,7 +220,7 @@ def get_package_folder_name(self, package: str) -> str: """return the package folder name""" try: - package_folder_name = self.packages[package]["release"][ + package_folder_name = self.platform_packages[package]["release"][ "folder_name" ] @@ -243,9 +249,9 @@ def get_package_folder_name(self, package: str) -> str: # -- Return the name return package_folder_name - def get_packages(self) -> tuple[list, list]: - """Get all the packages, classified in installed and - not installed + def get_platform_packages_lists(self) -> tuple[list, list]: + """Get all the packages that are applicable to this platform, + grouped as installed and not installed * OUTPUT: - A tuple of two lists: Installed and not installed packages """ @@ -255,13 +261,13 @@ def get_packages(self) -> tuple[list, list]: notinstalled_packages = [] # -- Go though all the apio packages - for package in self.packages: + for package in self.platform_packages: # -- Collect information about the package data = { "name": package, "version": None, - "description": self.packages[package]["description"], + "description": self.platform_packages[package]["description"], } # -- Check if this package is installed @@ -286,7 +292,7 @@ def get_packages(self) -> tuple[list, list]: # -- The package is not known! # -- Strange case - if package not in self.packages: + if package not in self.platform_packages: data = { "name": package, "version": "Unknown", @@ -303,7 +309,9 @@ def list_packages(self, installed=True, notinstalled=True): # profile = Profile() # Classify packages - installed_packages, notinstalled_packages = self.get_packages() + installed_packages, notinstalled_packages = ( + self.get_platform_packages_lists() + ) # -- Calculate the terminal width terminal_width, _ = shutil.get_terminal_size() @@ -353,6 +361,14 @@ def list_packages(self, installed=True, notinstalled=True): click.echo("\n") + def get_package_dir(self, package_name: str) -> Path: + """Returns the root path of a package with given name.""" + + package_folder = self.get_package_folder_name(package_name) + package_dir = util.get_packages_dir() / package_folder + + return package_dir + # R0914: Too many local variables (17/15) # pylint: disable=R0914 def list_boards(self): @@ -485,17 +501,11 @@ def list_fpgas(self): click.echo(seperator_line) click.echo(f"Total: {len(self.fpgas)} fpgas\n") - def _filter_packages(self, given_platform): - """Filter the apio packages available for the given platform. - Some platforms has special packages (Ex. package Drivers is - only for windows) - * INPUT: - * packages: All the apio packages - * given_platform: Platform used for filtering the packages. - If not given,the current system platform is used - - self.packages is updated. It now contains only the packages - for the given platform + @staticmethod + def _select_platform_packages(all_packages, given_platform): + """Given a dictionary with the packages.json packages configurations, + returns subset dictionary with packages that are applicable to the + this platform. """ # -- Final dict with the output packages @@ -506,10 +516,10 @@ def _filter_packages(self, given_platform): given_platform = util.get_system_type() # -- Check all the packages - for pkg in self.packages.keys(): + for pkg in all_packages.keys(): # -- Get the information about the package - release = self.packages[pkg]["release"] + release = all_packages[pkg]["release"] # -- This packages is available only for certain platforms if "available_platforms" in release: @@ -524,13 +534,13 @@ def _filter_packages(self, given_platform): if given_platform in platform: # -- Add it to the output dictionary - filtered_packages[pkg] = self.packages[pkg] + filtered_packages[pkg] = all_packages[pkg] # -- Package for all the platforms else: # -- Add it to the output dictionary - filtered_packages[pkg] = self.packages[pkg] + filtered_packages[pkg] = all_packages[pkg] # -- Update the current packages! - self.packages = filtered_packages + return filtered_packages diff --git a/apio/resources/packages.json b/apio/resources/packages.json index d74de724..bc5baa19 100644 --- a/apio/resources/packages.json +++ b/apio/resources/packages.json @@ -12,9 +12,9 @@ "extension": "zip", "url_version": "https://github.com/FPGAwars/apio-examples/raw/master/VERSION" }, - "description": "Verilog examples" + "description": "Verilog examples", + "env": {} }, - "oss-cad-suite": { "repository": { "name": "tools-oss-cad-suite", @@ -28,9 +28,21 @@ "extension": "tar.gz", "url_version": "https://github.com/zapta/tools-oss-cad-suite/raw/main/VERSION_DEV" }, - "description": "YosysHQ/oss-cad-suite" + "description": "YosysHQ/oss-cad-suite", + "env": { + "path": [ + "%p/bin", + "%p/lib" + ], + "vars": { + "VERILATOR_ROOT": "%p/share/verilator", + "IVL": "%p/lib/ivl", + "ICEBOX": "%p/share/icebox", + "TRELLIS": "%p/share/trellis", + "YOSYS_LIB": "%p/share/yosys" + } + } }, - "graphviz": { "repository": { "name": "tools-graphviz", @@ -49,9 +61,13 @@ "windows_amd64" ] }, - "description": "Graphviz tool for Windows" + "description": "Graphviz tool for Windows", + "env": { + "path": [ + "%p/bin" + ] + } }, - "drivers": { "repository": { "name": "tools-drivers", @@ -70,6 +86,11 @@ "windows_amd64" ] }, - "description": "Drivers tools for Windows" + "description": "Drivers tools for Windows", + "env": { + "path": [ + "%p/bin" + ] + } } -} +} \ No newline at end of file diff --git a/apio/util.py b/apio/util.py index 4d6a57c0..fcdfa003 100644 --- a/apio/util.py +++ b/apio/util.py @@ -271,6 +271,45 @@ def get_home_dir() -> Path: return home_dir +def get_packages_dir() -> Path: + """Return the base directory of apio packages. + Packages are installed in the following folder: + * Default: $APIO_HOME_DIR/packages + * $APIO_PKG_DIR/packages: if the APIO_PKG_DIR env variable is set + * INPUT: + - pkg_name: Package name (Ex. 'examples') + * OUTPUT: + - The package folder (PosixPath) + (Ex. '/home/obijuan/.apio/packages/examples')) + - or None if the packageis not installed + """ + + # -- Get the apio home dir: + # -- Ex. '/home/obijuan/.apio' + apio_home_dir = get_home_dir() + + # -- Get the APIO_PKG_DIR env variable + # -- It returns None if it was not defined + apio_pkg_dir_env = get_projconf_option_dir("pkg_dir") + + # -- Get the pkg base dir. It is what the APIO_PKG_DIR env variable + # -- says, or the default folder if None + if apio_pkg_dir_env: + pkg_home_dir = Path(apio_pkg_dir_env) + + # -- Default value + else: + pkg_home_dir = apio_home_dir + + # -- Create the package folder + # -- Ex '/home/obijuan/.apio/packages/tools-oss-cad-suite' + package_dir = pkg_home_dir / "packages" + + # -- Return the folder if it exists + # if package_dir.exists(): + return package_dir + + def call(cmd): """Execute the given command.""" From 72a4469ba2b602b58a0215a26ce817d045891ce9 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 10 Nov 2024 12:54:22 -0800 Subject: [PATCH 44/51] Cleaned up the UI of the apio examples command. This includes adding help text and renaming options for clarity. Functionality should stay the same. --- .vscode/launch.json | 3 +- apio/commands/examples.py | 50 ++++++++++++++++++++-------------- test/commands/test_examples.py | 12 +++++--- test/test_end_to_end.py | 31 +++++++++++++-------- 4 files changed, 59 insertions(+), 37 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 84f46efa..a48544ee 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -83,7 +83,8 @@ "request": "launch", "program": "${workspaceFolder}/apio_run.py", "args": [ - "examples" + "examples", + "--list" ], "console": "integratedTerminal", "justMyCode": false, diff --git a/apio/commands/examples.py b/apio/commands/examples.py index 3ddba1b4..2df7d912 100644 --- a/apio/commands/examples.py +++ b/apio/commands/examples.py @@ -20,22 +20,22 @@ # -- COMMAND SPECIFIC OPTIONS # --------------------------- dir_option = click.option( - "dir_", # Var name. Deconflicting with python builtin 'dir'. + "fetch_dir", # Var name. "-d", - "--dir", + "--fetch-dir", type=str, metavar="name", - help="Copy the selected example directory.", + help="Fetch the selected example directory.", cls=cmd_util.ApioOption, ) files_option = click.option( - "files", # Var name. + "fetch_files", # Var name. "-f", - "--files", + "--fetch-files", type=str, metavar="name", - help="Copy the selected example files.", + help="Fetch the selected example files.", cls=cmd_util.ApioOption, ) @@ -57,6 +57,12 @@ apio examples -d icezum # Fetch all board examples """ +EPILOG = """ +The format of 'name' is [/], where is a board +name (e.g. 'icezum') and is a name of an example of that +board (e.g. 'leds'). +""" + # pylint: disable=too-many-arguments # pylint: disable=too-many-positional-arguments @@ -64,6 +70,7 @@ "examples", short_help="List and fetch apio examples.", help=HELP, + epilog=EPILOG, cls=cmd_util.ApioCommand, ) @click.pass_context @@ -76,35 +83,38 @@ def cli( ctx: Context, # Options list_: bool, - dir_: str, - files: str, + fetch_dir: str, + fetch_files: str, project_dir: Path, sayno: bool, ): """Manage verilog examples.\n Install with `apio packages --install examples`""" + ctx.get_help() + # Make sure these params are exclusive. - cmd_util.check_at_most_one_param(ctx, nameof(list_, dir_, files)) + cmd_util.check_exactly_one_param( + ctx, nameof(list_, fetch_dir, fetch_files) + ) # -- Access to the Drivers resources = Resources(project_scope=False) examples = Examples(resources) - # -- Option: List all the available examples - if list_: - exit_code = examples.list_examples() - ctx.exit(exit_code) - # -- Option: Copy the directory - if dir_: - exit_code = examples.copy_example_dir(dir_, project_dir, sayno) + if fetch_dir: + exit_code = examples.copy_example_dir(fetch_dir, project_dir, sayno) ctx.exit(exit_code) # -- Option: Copy only the example files (not the initial folders) - if files: - exit_code = examples.copy_example_files(files, project_dir, sayno) + if fetch_files: + exit_code = examples.copy_example_files( + fetch_files, project_dir, sayno + ) ctx.exit(exit_code) - # -- no options: Show help! - click.secho(ctx.get_help()) + # -- Option: List all the available examples + assert list_ + exit_code = examples.list_examples() + ctx.exit(exit_code) diff --git a/test/commands/test_examples.py b/test/commands/test_examples.py index 9b6be1bc..94b0b0fd 100644 --- a/test/commands/test_examples.py +++ b/test/commands/test_examples.py @@ -6,7 +6,7 @@ from apio.commands.examples import cli as cmd_examples -def test_examples(clirunner, validate_cliresult, configenv): +def test_examples(clirunner, configenv): """Test "apio examples" with different parameters""" with clirunner.isolated_filesystem(): @@ -16,7 +16,11 @@ def test_examples(clirunner, validate_cliresult, configenv): # -- Execute "apio examples" result = clirunner.invoke(cmd_examples) - validate_cliresult(result) + assert result.exit_code == 1, result.output + assert ( + "Exactly one of of --list, --fetch-dir, --fetch-files " + "must be specified" in result.output + ) # -- Execute "apio examples --list" result = clirunner.invoke(cmd_examples, ["--list"]) @@ -24,11 +28,11 @@ def test_examples(clirunner, validate_cliresult, configenv): assert "apio packages --install --force examples" in result.output # -- Execute "apio examples --dir dir" - result = clirunner.invoke(cmd_examples, ["--dir", "dir"]) + result = clirunner.invoke(cmd_examples, ["--fetch-dir", "dir"]) assert result.exit_code == 1, result.output assert "apio packages --install --force examples" in result.output # -- Execute "apio examples --files file" - result = clirunner.invoke(cmd_examples, ["--files", "file"]) + result = clirunner.invoke(cmd_examples, ["--fetch-files", "file"]) assert result.exit_code == 1, result.output assert "apio packages --install --force examples" in result.output diff --git a/test/test_end_to_end.py b/test/test_end_to_end.py index cd21f6ef..8e515cf3 100644 --- a/test/test_end_to_end.py +++ b/test/test_end_to_end.py @@ -140,30 +140,34 @@ def test_end_to_end2(clirunner, validate_cliresult, configenv, offline): assert "leds" in result.output assert "icezum" in result.output - # -- Execute "apio examples --files missing_example" - result = clirunner.invoke(cmd_examples, ["--files", "missing_example"]) + # -- Execute "apio examples --fetch-files missing_example" + result = clirunner.invoke( + cmd_examples, ["--fetch-files", "missing_example"] + ) assert result.exit_code == 1, result.output assert "Warning: this example does not exist" in result.output - # -- Execute "apio examples --files Alhambra-II/ledon" + # -- Execute "apio examples --fetch-files Alhambra-II/ledon" result = clirunner.invoke( - cmd_examples, ["--files", "Alhambra-II/ledon"] + cmd_examples, ["--fetch-files", "Alhambra-II/ledon"] ) validate_cliresult(result) assert "Copying Alhambra-II/ledon example files ..." in result.output assert "have been successfully created!" in result.output validate_files_leds(pathlib.Path()) - # -- Execute "apio examples --dir Alhambra-II/ledon" - result = clirunner.invoke(cmd_examples, ["--dir", "Alhambra-II/ledon"]) + # -- Execute "apio examples --fetch-dir Alhambra-II/ledon" + result = clirunner.invoke( + cmd_examples, ["--fetch-dir", "Alhambra-II/ledon"] + ) validate_cliresult(result) assert "Creating Alhambra-II/ledon directory ..." in result.output assert "has been successfully created!" in result.output validate_dir_leds() - # -- Execute "apio examples --dir Alhambra-II/ledon" + # -- Execute "apio examples --fetch-dir Alhambra-II/ledon" result = clirunner.invoke( - cmd_examples, ["--dir", "Alhambra-II/ledon"], input="y" + cmd_examples, ["--fetch-dir", "Alhambra-II/ledon"], input="y" ) validate_cliresult(result) assert ( @@ -203,10 +207,11 @@ def test_end_to_end3(clirunner, validate_cliresult, configenv, offline): p = pathlib.Path("tmp/") p.mkdir(parents=True, exist_ok=True) - # -- Execute "apio examples --files Alhambra-II/ledon + # -- Execute "apio examples --fetch-files Alhambra-II/ledon # -- --project-dir=tmp" result = clirunner.invoke( - cmd_examples, ["--files", "Alhambra-II/ledon", "--project-dir=tmp"] + cmd_examples, + ["--fetch-files", "Alhambra-II/ledon", "--project-dir=tmp"], ) validate_cliresult(result) assert "Copying Alhambra-II/ledon example files ..." in result.output @@ -215,9 +220,11 @@ def test_end_to_end3(clirunner, validate_cliresult, configenv, offline): # -- Check the files in the tmp folder validate_files_leds(p) - # -- Execute "apio examples --dir Alhambra-II/ledon --project-dir=tmp" + # -- Execute + # -- "apio examples --fetch-dir Alhambra-II/ledon --project-dir=tmp" result = clirunner.invoke( - cmd_examples, ["--dir", "Alhambra-II/ledon", "--project-dir=tmp"] + cmd_examples, + ["--fetch-dir", "Alhambra-II/ledon", "--project-dir=tmp"], ) validate_cliresult(result) assert "Creating Alhambra-II/ledon directory ..." in result.output From dff9016b2e76704aa8c9ca55b79fe03425faa628 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 10 Nov 2024 12:57:50 -0800 Subject: [PATCH 45/51] Fixed the environment setting dumping code to strip ansi colors when writing to a pipe. click.echo() does strip colors, print() doesn't. --- apio/pkg_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index dc0682a8..b8103578 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -109,17 +109,17 @@ def _dump_env_mutations(mutations: EnvMutations) -> None: for p in reversed(mutations.paths): styled_name = click.style("PATH", fg="magenta") if windows: - print(f"@set {styled_name}={p};%PATH%") + click.echo(f"@set {styled_name}={p};%PATH%") else: - print(f'{styled_name}="{p}:$PATH"') + click.echo(f'{styled_name}="{p}:$PATH"') # -- Print vars mutations. for name, val in mutations.vars: styled_name = click.style(name, fg="magenta") if windows: - print(f"@set {styled_name}={val}") + click.echo(f"@set {styled_name}={val}") else: - print(f'{styled_name}="{val}"') + click.echo(f'{styled_name}="{val}"') def _apply_env_mutations(mutations: EnvMutations) -> None: From 1e2c3d69a8c6d23d9f309fe76c146b16f8bda840 Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 11 Nov 2024 08:31:30 -0800 Subject: [PATCH 46/51] Added to the verilator command the --quiet flag to avoid the banner printing. --- apio/pkg_util.py | 47 +++----------------- apio/resources.py | 107 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 91 insertions(+), 63 deletions(-) diff --git a/apio/pkg_util.py b/apio/pkg_util.py index b8103578..f0e99b45 100644 --- a/apio/pkg_util.py +++ b/apio/pkg_util.py @@ -43,57 +43,24 @@ class _PackageDesc: env_func: Callable[[Path], EnvMutations] -def _expand_env_template(template: str, package_path: Path) -> str: - """Fills a packages env value template as they appear in packages.json. - Currently it recognizes only a single place holder '%p' representing the - package absolute path. The '%p" can appear only at the begigning of the - template. - - E.g. '%p/bin' -> '/users/user/.apio/packages/drivers/bin' - """ - - # Case 1: No place holder. - if "%p" not in template: - return template - - # Case 2: The template contains only the placeholder. - if template == "%p": - return str(package_path) - - # Case 3: The place holder is the prefix of the template's path. - if template.startswith("%p/"): - return str(package_path / template[3:]) - - # Case 4: Unsupported. - raise RuntimeError(f"Invalid env template: [{template}]") - - def _get_env_mutations_for_packages(resources: Resources) -> EnvMutations: """Collects the env mutation for each of the defined packages, in the order they are defined.""" result = EnvMutations([], []) - for package_name, package_config in resources.platform_packages.items(): - - # -- Get the package root dir. - package_path = resources.get_package_dir(package_name) - - # -- Get the json env section. We require it, even if it's empty, + for _, package_config in resources.platform_packages.items(): + # -- Get the json 'env' section. We require it, even if it's empty, # -- for clarity reasons. + assert "env" in package_config package_env = package_config["env"] # -- Collect the path values. - path_section = package_env.get("path", {}) - for path_template in path_section: - # -- Replaces place holders, if nay. - path_value = _expand_env_template(path_template, package_path) - result.paths.append(path_value) + path_list = package_env.get("path", []) + result.paths.extend(path_list) - # -- Collect the vars (name, value) pairs. + # -- Collect the env vars (name, value) pairs. vars_section = package_env.get("vars", {}) - for var_name, var_template in vars_section.items(): - # -- Replaces place holders, if nay. - var_value = _expand_env_template(var_template, package_path) + for var_name, var_value in vars_section.items(): result.vars.append((var_name, var_value)) return result diff --git a/apio/resources.py b/apio/resources.py index 2339b95e..1b1c3be3 100644 --- a/apio/resources.py +++ b/apio/resources.py @@ -11,7 +11,7 @@ from collections import OrderedDict import shutil from pathlib import Path -from typing import Optional +from typing import Optional, Dict import click from apio import util from apio.profile import Profile @@ -90,6 +90,9 @@ def __init__( # -- Read the apio packages information self.all_packages = self._load_resource(PACKAGES_JSON) + # -- Expand in place the env templates in all_packages. + Resources._resolve_package_envs(self.all_packages) + # The subset of packages that are applicable to this platform. self.platform_packages = self._select_platform_packages( self.all_packages, platform @@ -113,14 +116,11 @@ def __init__( # -- Read the distribution information self.distribution = self._load_resource(DISTRIBUTION_JSON) - # --------- Sort resources for consistency and intunitiveness. - self.all_packages = OrderedDict( - sorted(self.all_packages.items(), key=lambda t: t[0]) - ) - - self.platform_packages = OrderedDict( - sorted(self.platform_packages.items(), key=lambda t: t[0]) - ) + # -- Sort resources for consistency and intunitiveness. + # -- + # -- We don't sort the all_packages and platform_packages dictionaries + # -- because that will affect the order of the env path items. + # -- Instead we preserve the order from the packages.json file. self.boards = OrderedDict( sorted(self.boards.items(), key=lambda t: t[0]) @@ -216,6 +216,67 @@ def _load_resource_file(filepath: Path) -> dict: # -- Return the object for the resource return resource + @staticmethod + def _expand_env_template(template: str, package_path: Path) -> str: + """Fills a packages env value template as they appear in packages.json. + Currently it recognizes only a single place holder '%p' representing + the package absolute path. The '%p" can appear only at the begigning + of the template. + + E.g. '%p/bin' -> '/users/user/.apio/packages/drivers/bin' + + NOTE: This format is very basic but is sufficient for the current + needs. If needed, extend or modify it. + """ + + # Case 1: No place holder. + if "%p" not in template: + return template + + # Case 2: The template contains only the placeholder. + if template == "%p": + return str(package_path) + + # Case 3: The place holder is the prefix of the template's path. + if template.startswith("%p/"): + return str(package_path / template[3:]) + + # Case 4: Unsupported. + raise RuntimeError(f"Invalid env template: [{template}]") + + @staticmethod + def _resolve_package_envs(packages: Dict[str, Dict]) -> None: + """Resolve in place the path and var value templates in the + given packages dictionary. For example, %p is replaced with + the package's absolute path.""" + + packages_dir = util.get_packages_dir() + for _, package_config in packages.items(): + + # -- Get the package root dir. + package_path = ( + packages_dir / package_config["release"]["folder_name"] + ) + + # -- Get the json 'env' section. We require it, even if empty, + # -- for clarity reasons. + assert "env" in package_config + package_env = package_config["env"] + + # -- Expand the values in the "path" section, if any. + path_section = package_env.get("path", []) + for i, path_template in enumerate(path_section): + path_section[i] = Resources._expand_env_template( + path_template, package_path + ) + + # -- Expand the values in the "vars" section, if any. + vars_section = package_env.get("vars", {}) + for var_name, val_template in vars_section.items(): + vars_section[var_name] = Resources._expand_env_template( + val_template, package_path + ) + def get_package_folder_name(self, package: str) -> str: """return the package folder name""" @@ -260,21 +321,22 @@ def get_platform_packages_lists(self) -> tuple[list, list]: installed_packages = [] notinstalled_packages = [] - # -- Go though all the apio packages - for package in self.platform_packages: + # -- Go though all the apio packages and add them to the installed + # -- or uninstalled lists. + for package_name, package_config in self.platform_packages.items(): # -- Collect information about the package data = { - "name": package, + "name": package_name, "version": None, - "description": self.platform_packages[package]["description"], + "description": package_config["description"], } # -- Check if this package is installed - if package in self.profile.packages: + if package_name in self.profile.packages: # -- Get the installed version - version = self.profile.packages[package]["version"] + version = self.profile.packages[package_name]["version"] # -- Store the version data["version"] = version @@ -286,15 +348,14 @@ def get_platform_packages_lists(self) -> tuple[list, list]: else: notinstalled_packages += [data] - # -- Check the installed packages and update - # -- its information - for package in self.profile.packages: - - # -- The package is not known! - # -- Strange case - if package not in self.platform_packages: + # -- If there are in the profile packages that are not in the + # -- platform packages, add them well to the uninstalled list, as + # -- 'unknown'. These must be some left overs, e.g. if apio is + # -- upgraded. + for package_name in self.profile.packages: + if package_name not in self.platform_packages: data = { - "name": package, + "name": package_name, "version": "Unknown", "description": "Unknown deprecated package", } From 8987615acd1d30ebfa92f8b86ddf0c9646b85b29 Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 11 Nov 2024 11:59:24 -0800 Subject: [PATCH 47/51] Added to the 'apio graph' command ability to open an interactive viewer or generate svg, pdf, or png output files. This commit is for ice40 only. Once tested on windows, aill implement also for ecp5 and gowin. --- .vscode/launch.json | 3 +- apio/commands/graph.py | 71 ++++++++++++++++++++++++++++--------- apio/managers/arguments.py | 4 +++ apio/managers/scons.py | 2 +- apio/scons/ice40/SConstruct | 31 +++------------- apio/scons/scons_util.py | 66 ++++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 45 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index a48544ee..05a2b4f1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -96,7 +96,8 @@ "request": "launch", "program": "${workspaceFolder}/apio_run.py", "args": [ - "graph" + "graph", + "--svg" ], "console": "integratedTerminal", "justMyCode": false, diff --git a/apio/commands/graph.py b/apio/commands/graph.py index 91dec5d9..6f454207 100644 --- a/apio/commands/graph.py +++ b/apio/commands/graph.py @@ -10,11 +10,39 @@ from pathlib import Path import click from click.core import Context +from varname import nameof from apio.managers.scons import SCons from apio import cmd_util from apio.commands import options from apio.resources import Resources +# --------------------------- +# -- COMMAND SPECIFIC OPTIONS +# --------------------------- +svg_option = click.option( + "svg", # Var name. + "--svg", + is_flag=True, + help="Generate a svg file.", + cls=cmd_util.ApioOption, +) + +pdf_option = click.option( + "pdf", # Var name. + "--pdf", + is_flag=True, + help="Generate a pdf file.", + cls=cmd_util.ApioOption, +) + +png_option = click.option( + "png", # Var name. + "--png", + is_flag=True, + help="Generate a png file.", + cls=cmd_util.ApioOption, +) + # --------------------------- # -- COMMAND @@ -27,27 +55,15 @@ \b Examples: - apio graph # Graph the top module + apio graph --svg # Generate a svg file. + apio graph # Open an interactive viewer. apio graph -t my_module # Graph the selected module -The graph command generates the graph in .dot format and then invokes -the dot command from the path to convert it to a .svg format. The dot -command is not included with the apio distribution and needed to be -installed seperatly. See https://graphviz.org for more details. - -[Hint] If you need the graph in other formats, convert the .dot file -to the desired format using the dot command. -""" - -DOT_HELP = """ -The 'dot' command is part of the 'graphviz' suite. Please install -it per the instructions at https://graphviz.org/download and run -this command again. If you think that the 'dot' command is available -on the system path, you can try supressing this error message by -adding the --force flag to the apio graph command. """ +# pylint: disable=too-many-arguments +# pylint: disable=too-many-positional-arguments @click.command( "graph", short_help="Generate a visual graph of the code.", @@ -55,20 +71,40 @@ cls=cmd_util.ApioCommand, ) @click.pass_context +@svg_option +@pdf_option +@png_option @options.project_dir_option @options.top_module_option_gen(help="Set the name of the top module to graph.") @options.verbose_option def cli( ctx: Context, # Options + svg: bool, + pdf: bool, + png: bool, project_dir: Path, verbose: bool, top_module: str, ): """Implements the apio graph command.""" + # -- Sanity check the options. + cmd_util.check_at_most_one_param(ctx, nameof(svg, pdf, png)) + + # -- Determien graph type. An empty string for an interactive viewer. + if svg: + graph_type = "svg" + elif pdf: + graph_type = "pdf" + elif png: + graph_type = "png" + else: + graph_type = "" - # -- Crete the scons object + # -- Load apio resources. resources = Resources(project_dir=project_dir, project_scope=True) + + # -- Create the scons object. scons = SCons(resources) # -- Graph the project with the given parameters @@ -76,6 +112,7 @@ def cli( { "verbose": {"all": verbose, "yosys": False, "pnr": False}, "top-module": top_module, + "graph_type": graph_type, } ) diff --git a/apio/managers/arguments.py b/apio/managers/arguments.py index 94414973..a065a40c 100644 --- a/apio/managers/arguments.py +++ b/apio/managers/arguments.py @@ -28,6 +28,7 @@ PNR = "pnr" # -- Key for Verbose-pnr TOP_MODULE = "top-module" # -- Key for top-module TESTBENCH = "testbench" # -- Key for testbench file name +GRAPH_TYPE = "graph_type" # -- Key for graph type name def debug_params(fun): @@ -125,6 +126,7 @@ def process_arguments( VERBOSE: {ALL: False, "yosys": False, "pnr": False}, TOP_MODULE: None, TESTBENCH: None, + GRAPH_TYPE: None, } # -- Merge the initial configuration to the current configuration @@ -258,6 +260,7 @@ def process_arguments( ), "top_module": config[TOP_MODULE], "testbench": config[TESTBENCH], + "graph_type": config[GRAPH_TYPE], } ) @@ -340,6 +343,7 @@ def print_configuration(config: dict) -> None: print(f" idcode: {config[IDCODE]}") print(f" top-module: {config[TOP_MODULE]}") print(f" testbench: {config[TESTBENCH]}") + print(f" graph_type: {config[GRAPH_TYPE]}") print(" verbose:") print(f" all: {config[VERBOSE][ALL]}") # These two flags appear only in some of the commands. diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 1c5bfdd8..29981777 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -123,7 +123,7 @@ def verify(self, args) -> int: @on_exception(exit_code=1) def graph(self, args) -> int: - """Runs a scons subprocess with the 'verify' target. Returns process + """Runs a scons subprocess with the 'graph' target. Returns process exit code, 0 if ok.""" # -- Split the arguments diff --git a/apio/scons/ice40/SConstruct b/apio/scons/ice40/SConstruct index ef268a39..d825d9d2 100644 --- a/apio/scons/ice40/SConstruct +++ b/apio/scons/ice40/SConstruct @@ -59,6 +59,7 @@ from apio.scons.scons_util import ( get_programmer_cmd, make_verilog_src_scanner, make_verilator_config_builder, + make_dot_builder, get_source_files, get_sim_config, get_tests_configs, @@ -90,6 +91,7 @@ VERILATOR_ALL = arg_bool(env, "all", False) VERILATOR_NO_STYLE = arg_bool(env, "nostyle", False) NOWARNS = arg_str(env, "nowarn", "").split(",") WARNS = arg_str(env, "warn", "").split(",") +GRAPH_TYPE = arg_str(env, "graph_type", "") # -- Resources paths @@ -286,33 +288,12 @@ env.Append(BUILDERS={"IVerilogTestbench": iverilog_tb_builder}) # -- Apio graph. # -- Builder (yosys, .dot graph generator). # -- hardware.v -> hardware.dot. -dot_builder = Builder( - action=( - 'yosys -f verilog -p "show -format dot -colors 1 ' - '-prefix hardware {0}" {1} $SOURCES' - ).format( - TOP_MODULE if TOP_MODULE else "unknown_top", - "" if VERBOSE_ALL else "-q", - ), - suffix=".dot", - src_suffix=".v", - source_scanner=verilog_src_scanner, +dot_builder = make_dot_builder( + env, TOP_MODULE, GRAPH_TYPE, verilog_src_scanner, VERBOSE_ALL ) env.Append(BUILDERS={"DOT": dot_builder}) -# -- Apio graph. -# -- Builder (dot, svg renderer). -# -- hardware.dot -> hardware.svg. -svg_builder = Builder( - # Expecting graphviz dot to be installed and in the path. - action="dot -Tsvg $SOURCES -o $TARGET", - suffix=".svg", - src_suffix=".dot", -) -env.Append(BUILDERS={"SVG": svg_builder}) - - # -- Apio sim/test. # -- Builder (vvp, simulator). # -- (testbench).out -> (testbench).vcd. @@ -341,9 +322,7 @@ verify_target = env.Alias("verify", verify_out_target) # TODO: Launch a portable SVG (or differentn format) viewer. dot_target = env.DOT(TARGET, synth_srcs) AlwaysBuild(dot_target) -svg_target = env.SVG(TARGET, dot_target) -AlwaysBuild(svg_target) -graph_target = env.Alias("graph", svg_target) +graph_target = env.Alias("graph", dot_target) # -- Apio sim. diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index 1e798d65..b010bd34 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -357,6 +357,72 @@ def verilator_config_func(target, source, env): ) +def make_dot_builder( + env: SConsEnvironment, + top_module: str, + graph_type: str, + verilog_src_scanner, + verbose: bool, +): + """Creates and returns an SCons dot builder. The builder has two modes, + interactive viewer (graph_type = "") and batch file generation + (e.g. graph_type = "svg). + + 'verilog_src_scanner' is a verilog file scanner that identify additional + dependencies for the build, for example, icestudio proprietry includes. + + In batch mode, we add a small action to print a message with the + generated file name. + """ + + def print_graph_completion(source, target, env): + """Action function that prints the generated file name. Used only + when file_type != "" + """ + # -- SCons prints a blank line with the action description so we + # -- move the cursor one line up before printing. + cursor_up = "\033[F" + msg(env, f"{cursor_up}Generated {TARGET}.{graph_type}", fg="green") + + def dot_emitter(target, source, env): + """Tells scons to clean all the possible graph file types.""" + supported_types = ["svg", "pdf", "png"] + assert not graph_type or graph_type in supported_types, graph_type + for supported_type in supported_types: + target.append(TARGET + f".{supported_type}") + return target, source + + actions = [ + # -- The actual dot action. Uses Yosys to open the viewer or to + # -- generate the output file. + ( + 'yosys -f verilog -p "show -format {0} {1} -colors 1 ' + '-prefix hardware {2}" {3} $SOURCES' + ).format( + graph_type if graph_type else "dot", + "" if graph_type else "-viewer xdot", + top_module if top_module else "unknown_top", + "" if verbose else "-q", + ) + ] + + # -- If generating a file, add an action to print a message with the + # -- file name. + if graph_type: + actions.append(env.Action(print_graph_completion, " ")) + + # -- The builder. + dot_builder = env.Builder( + action=actions, + suffix=f".{graph_type}" if graph_type else ".dot", + src_suffix=".v", + source_scanner=verilog_src_scanner, + emitter=dot_emitter, + ) + + return dot_builder + + def get_source_files(env: SConsEnvironment) -> Tuple[List[str], List[str]]: """Get the list of *.v files, splitted into synth and testbench lists. If a .v file has the suffix _tb.v it's is classified st a testbench, From 43e55af503666ed744853e090290fd02587e461b Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 11 Nov 2024 12:12:57 -0800 Subject: [PATCH 48/51] Tweked the scons of the apio graph command. Now it doesn't delete one output file when generating another output file. --- apio/scons/scons_util.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index b010bd34..45195ad5 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -37,6 +37,8 @@ # -- Target name. This is the base file name for various build artifacts. TARGET = "hardware" +SUPPORTED_GRAPH_TYPES = ["svg", "pdf", "png"] + class SConstructId(Enum): """Identifies the SConstruct script that is running. Used to select @@ -384,14 +386,6 @@ def print_graph_completion(source, target, env): cursor_up = "\033[F" msg(env, f"{cursor_up}Generated {TARGET}.{graph_type}", fg="green") - def dot_emitter(target, source, env): - """Tells scons to clean all the possible graph file types.""" - supported_types = ["svg", "pdf", "png"] - assert not graph_type or graph_type in supported_types, graph_type - for supported_type in supported_types: - target.append(TARGET + f".{supported_type}") - return target, source - actions = [ # -- The actual dot action. Uses Yosys to open the viewer or to # -- generate the output file. @@ -406,8 +400,7 @@ def dot_emitter(target, source, env): ) ] - # -- If generating a file, add an action to print a message with the - # -- file name. + # -- If generating a file, add an action to print completion message. if graph_type: actions.append(env.Action(print_graph_completion, " ")) @@ -417,7 +410,6 @@ def dot_emitter(target, source, env): suffix=f".{graph_type}" if graph_type else ".dot", src_suffix=".v", source_scanner=verilog_src_scanner, - emitter=dot_emitter, ) return dot_builder @@ -731,5 +723,9 @@ def set_up_cleanup(env: SConsEnvironment, targets) -> None: for node in env.Glob(dynamic_target): env.Clean(targets[0], str(node)) + # -- Do the same for the apio graph output files. + for graph_type in SUPPORTED_GRAPH_TYPES: + env.Clean(targets[0], TARGET + "." + graph_type) + # -- Tell SCons to cleanup the given targets and all of their dependencies. env.Default(targets) From 875e05cb32d460cd184812e31824e1a5808a8353 Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 11 Nov 2024 18:56:04 -0800 Subject: [PATCH 49/51] Removed the viewer option of the 'apio graph' command. We need to research more to find a reliable way to view graphs on all supported platforms. For example 'xdot' works on Max OS but fails to install on Windows. --- apio/commands/graph.py | 48 +++++++++---------- apio/managers/arguments.py | 8 ++-- apio/scons/ice40/SConstruct | 17 +++++-- apio/scons/scons_util.py | 95 +++++++++++++++++++++---------------- 4 files changed, 94 insertions(+), 74 deletions(-) diff --git a/apio/commands/graph.py b/apio/commands/graph.py index 6f454207..98bd0f0e 100644 --- a/apio/commands/graph.py +++ b/apio/commands/graph.py @@ -19,13 +19,7 @@ # --------------------------- # -- COMMAND SPECIFIC OPTIONS # --------------------------- -svg_option = click.option( - "svg", # Var name. - "--svg", - is_flag=True, - help="Generate a svg file.", - cls=cmd_util.ApioOption, -) + pdf_option = click.option( "pdf", # Var name. @@ -48,17 +42,23 @@ # -- COMMAND # --------------------------- HELP = """ -The graph command generates a graphical representation of the -verilog code in the project. -The commands is typically used in the root directory -of the project that contains the apio.ini file. +The graph command generates a graphical representation of +the verilog code of the project. The commands is typically +used in the root directory of the project that contains +the apio.ini file. \b Examples: - apio graph --svg # Generate a svg file. - apio graph # Open an interactive viewer. - apio graph -t my_module # Graph the selected module + apio graph # Generate a svg file. + apio graph --pdf # Generate a pdf file. + apio graph --png # Generate a png file. + apio graph -t my_module # Graph my_module module. + +""" +EPILOG = """ +[Hint] On windows, type 'explorer hardware.svg' to +view the graph, and on Mac OS type 'open hardware.svg'. """ @@ -68,10 +68,10 @@ "graph", short_help="Generate a visual graph of the code.", help=HELP, + epilog=EPILOG, cls=cmd_util.ApioCommand, ) @click.pass_context -@svg_option @pdf_option @png_option @options.project_dir_option @@ -80,7 +80,6 @@ def cli( ctx: Context, # Options - svg: bool, pdf: bool, png: bool, project_dir: Path, @@ -89,17 +88,16 @@ def cli( ): """Implements the apio graph command.""" # -- Sanity check the options. - cmd_util.check_at_most_one_param(ctx, nameof(svg, pdf, png)) + cmd_util.check_at_most_one_param(ctx, nameof(pdf, png)) - # -- Determien graph type. An empty string for an interactive viewer. - if svg: - graph_type = "svg" - elif pdf: - graph_type = "pdf" + # -- Construct the graph spec to pass to scons. + # -- For now it's trivial. + if pdf: + graph_spec = "pdf" elif png: - graph_type = "png" + graph_spec = "png" else: - graph_type = "" + graph_spec = "svg" # -- Load apio resources. resources = Resources(project_dir=project_dir, project_scope=True) @@ -112,7 +110,7 @@ def cli( { "verbose": {"all": verbose, "yosys": False, "pnr": False}, "top-module": top_module, - "graph_type": graph_type, + "graph_spec": graph_spec, } ) diff --git a/apio/managers/arguments.py b/apio/managers/arguments.py index a065a40c..cd97054e 100644 --- a/apio/managers/arguments.py +++ b/apio/managers/arguments.py @@ -28,7 +28,7 @@ PNR = "pnr" # -- Key for Verbose-pnr TOP_MODULE = "top-module" # -- Key for top-module TESTBENCH = "testbench" # -- Key for testbench file name -GRAPH_TYPE = "graph_type" # -- Key for graph type name +GRAPH_SPEC = "graph_spec" # -- Key for graph specification def debug_params(fun): @@ -126,7 +126,7 @@ def process_arguments( VERBOSE: {ALL: False, "yosys": False, "pnr": False}, TOP_MODULE: None, TESTBENCH: None, - GRAPH_TYPE: None, + GRAPH_SPEC: None, } # -- Merge the initial configuration to the current configuration @@ -260,7 +260,7 @@ def process_arguments( ), "top_module": config[TOP_MODULE], "testbench": config[TESTBENCH], - "graph_type": config[GRAPH_TYPE], + "graph_spec": config[GRAPH_SPEC], } ) @@ -343,7 +343,7 @@ def print_configuration(config: dict) -> None: print(f" idcode: {config[IDCODE]}") print(f" top-module: {config[TOP_MODULE]}") print(f" testbench: {config[TESTBENCH]}") - print(f" graph_type: {config[GRAPH_TYPE]}") + print(f" graph_spec: {config[GRAPH_SPEC]}") print(" verbose:") print(f" all: {config[VERBOSE][ALL]}") # These two flags appear only in some of the commands. diff --git a/apio/scons/ice40/SConstruct b/apio/scons/ice40/SConstruct index d825d9d2..c72d9ec0 100644 --- a/apio/scons/ice40/SConstruct +++ b/apio/scons/ice40/SConstruct @@ -60,6 +60,7 @@ from apio.scons.scons_util import ( make_verilog_src_scanner, make_verilator_config_builder, make_dot_builder, + make_graphviz_builder, get_source_files, get_sim_config, get_tests_configs, @@ -91,7 +92,7 @@ VERILATOR_ALL = arg_bool(env, "all", False) VERILATOR_NO_STYLE = arg_bool(env, "nostyle", False) NOWARNS = arg_str(env, "nowarn", "").split(",") WARNS = arg_str(env, "warn", "").split(",") -GRAPH_TYPE = arg_str(env, "graph_type", "") +GRAPH_SPEC = arg_str(env, "graph_spec", "") # -- Resources paths @@ -289,10 +290,16 @@ env.Append(BUILDERS={"IVerilogTestbench": iverilog_tb_builder}) # -- Builder (yosys, .dot graph generator). # -- hardware.v -> hardware.dot. dot_builder = make_dot_builder( - env, TOP_MODULE, GRAPH_TYPE, verilog_src_scanner, VERBOSE_ALL + env, TOP_MODULE, verilog_src_scanner, VERBOSE_ALL ) env.Append(BUILDERS={"DOT": dot_builder}) +# -- Apio graph. +# -- Builder (dot, svg/pdf/png renderer). +# -- hardware.dot -> hardware.svg/pdf/png. +graphviz_builder = make_graphviz_builder(env, GRAPH_SPEC) +env.Append(BUILDERS={"GRAPHVIZ": graphviz_builder}) + # -- Apio sim/test. # -- Builder (vvp, simulator). @@ -318,11 +325,11 @@ verify_target = env.Alias("verify", verify_out_target) # -- Apio graph. # -- Targets. # -- (modules).v -> hardware.dot -> hardware.svg. -# -# TODO: Launch a portable SVG (or differentn format) viewer. dot_target = env.DOT(TARGET, synth_srcs) AlwaysBuild(dot_target) -graph_target = env.Alias("graph", dot_target) +graphviz_target = env.GRAPHVIZ(TARGET, dot_target) +AlwaysBuild(graphviz_target) +graph_target = env.Alias("graph", graphviz_target) # -- Apio sim. diff --git a/apio/scons/scons_util.py b/apio/scons/scons_util.py index 45195ad5..ddde0d72 100644 --- a/apio/scons/scons_util.py +++ b/apio/scons/scons_util.py @@ -31,7 +31,8 @@ from SCons.Node.Alias import Alias from SCons.Script import DefaultEnvironment from SCons.Script.SConscript import SConsEnvironment -from SCons.Action import FunctionAction +from SCons.Action import FunctionAction, Action +from SCons.Builder import Builder # -- Target name. This is the base file name for various build artifacts. @@ -351,8 +352,8 @@ def verilator_config_func(target, source, env): target_file.write(config_text) return 0 - return env.Builder( - action=env.Action( + return Builder( + action=Action( verilator_config_func, "Creating verilator config file." ), suffix=".vlt", @@ -362,52 +363,26 @@ def verilator_config_func(target, source, env): def make_dot_builder( env: SConsEnvironment, top_module: str, - graph_type: str, verilog_src_scanner, verbose: bool, ): - """Creates and returns an SCons dot builder. The builder has two modes, - interactive viewer (graph_type = "") and batch file generation - (e.g. graph_type = "svg). + """Creates and returns an SCons dot builder that generates the graph + in .dot format. 'verilog_src_scanner' is a verilog file scanner that identify additional - dependencies for the build, for example, icestudio proprietry includes. - - In batch mode, we add a small action to print a message with the - generated file name. + dependencies for the build, for example, icestudio propriety includes. """ - def print_graph_completion(source, target, env): - """Action function that prints the generated file name. Used only - when file_type != "" - """ - # -- SCons prints a blank line with the action description so we - # -- move the cursor one line up before printing. - cursor_up = "\033[F" - msg(env, f"{cursor_up}Generated {TARGET}.{graph_type}", fg="green") - - actions = [ - # -- The actual dot action. Uses Yosys to open the viewer or to - # -- generate the output file. - ( - 'yosys -f verilog -p "show -format {0} {1} -colors 1 ' - '-prefix hardware {2}" {3} $SOURCES' + # -- The builder. + dot_builder = Builder( + action=( + 'yosys -f verilog -p "show -format dot -colors 1 ' + '-prefix hardware {0}" {1} $SOURCES' ).format( - graph_type if graph_type else "dot", - "" if graph_type else "-viewer xdot", top_module if top_module else "unknown_top", "" if verbose else "-q", - ) - ] - - # -- If generating a file, add an action to print completion message. - if graph_type: - actions.append(env.Action(print_graph_completion, " ")) - - # -- The builder. - dot_builder = env.Builder( - action=actions, - suffix=f".{graph_type}" if graph_type else ".dot", + ), + suffix=".dot", src_suffix=".v", source_scanner=verilog_src_scanner, ) @@ -415,6 +390,46 @@ def print_graph_completion(source, target, env): return dot_builder +def make_graphviz_builder( + env: SConsEnvironment, + graph_spec: str, +): + """Creates and returns an SCons graphviz builder that renders + a .dot file to one of the supported formats. + + 'graph_spec' contains the rendering specification and currently + it includes a single value which is the target file format". + """ + + # --Decode the graphic spec. Currently it's trivial since it + # -- contains a single value. + if graph_spec: + # -- This is the case when scons target is 'graph'. + graph_type = graph_spec + assert graph_type in SUPPORTED_GRAPH_TYPES, graph_type + else: + # -- This is the case when scons target is not 'graph'. + graph_type = "svg" + + def completion_action(source, target, env): + """Action function that prints a completion message.""" + msg(env, f"Generated {TARGET}.{graph_type}", fg="green") + + actions = [ + f"dot -T{graph_type} $SOURCES -o $TARGET", + Action(completion_action, "completion_action"), + ] + + graphviz_builder = Builder( + # Expecting graphviz dot to be installed and in the path. + action=actions, + suffix=f".{graph_type}", + src_suffix=".dot", + ) + + return graphviz_builder + + def get_source_files(env: SConsEnvironment) -> Tuple[List[str], List[str]]: """Get the list of *.v files, splitted into synth and testbench lists. If a .v file has the suffix _tb.v it's is classified st a testbench, @@ -676,7 +691,7 @@ def print_pnr_report( json_txt: str = json_file.get_text_contents() _print_pnr_report(env, json_txt, script_id, verbose) - return env.Action(print_pnr_report, "Formatting pnr report.") + return Action(print_pnr_report, "Formatting pnr report.") def wait_for_remote_debugger(env: SConsEnvironment): From 905d8abe27230d56e6b6c8edc0b3011aa480fc46 Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 11 Nov 2024 21:52:35 -0800 Subject: [PATCH 50/51] Updated the help text of the sim/test/build commands to clarify the naming conventions of test benches. --- apio/commands/build.py | 8 ++++++-- apio/commands/sim.py | 5 +++-- apio/commands/test.py | 6 +++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/apio/commands/build.py b/apio/commands/build.py index a86b6b5c..28075204 100644 --- a/apio/commands/build.py +++ b/apio/commands/build.py @@ -28,8 +28,12 @@ \b Examples: - apio build - apio build -v + apio build # Build + apio build -v # Build with verbose info + +The build command builds all the .v files (e.g. my_module.v) in the project +directory except for those whose name ends with _tb (e.g. my_module_tb.v) to +indicate that they are testbenches. """ diff --git a/apio/commands/sim.py b/apio/commands/sim.py index 29cc158d..bffd9437 100644 --- a/apio/commands/sim.py +++ b/apio/commands/sim.py @@ -21,8 +21,9 @@ HELP = """ The sim command simulates a testbench file and shows -the simulation results a GTKWave graphical window. -The commands is typically used in the root directory +the simulation results a GTKWave graphical window. The testbench is expected +to have a name ending with _tb (e.g. my_module_tb.v) and the +commands is typically used in the root directory of the project that contains the apio.ini file and it accepts the testbench file name as an argument. For example: diff --git a/apio/commands/test.py b/apio/commands/test.py index d9dfd42c..58db58ca 100644 --- a/apio/commands/test.py +++ b/apio/commands/test.py @@ -22,9 +22,9 @@ HELP = """ The sim command simulates one or all the testbenches in the project and is useful for automatic unit testing of the code. Testbenches -are expected to exist with the $fatal directive if any error is -detected. The commands is typically used in the root directory -of the project that contains the apio.ini. +are expected have a name ending with _tb (e.g my_module_tb.v) and to exit +with the $fatal directive if any error is detected. The commands is typically +used in the root directory of the project that contains the apio.ini. \b Examples From d7739e1219959d461b2610f26a1916dd529f4e8d Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 11 Nov 2024 22:25:23 -0800 Subject: [PATCH 51/51] Minor tweak of the 'apio boards' command help text. --- apio/commands/boards.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apio/commands/boards.py b/apio/commands/boards.py index aae03ba2..97daeaad 100644 --- a/apio/commands/boards.py +++ b/apio/commands/boards.py @@ -45,8 +45,8 @@ apio boards -f | grep gowin # Filter FPGA results. [Advanced] Boards with wide availability can be added by contacting the -apio team. A custom one-of board can be added to your project by -placing a boards.json file next to apio.ini. +apio team. Custom one-of boards can be added to your project by +placing an alternative boards.json file in your apio project directory. """