diff --git a/README.md b/README.md index 3c12db7..a00c3f2 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ slurmutils package include: #### `from slurmutils.editors import ...` * `acctgatherconfig`: An editor for _acct_gather.conf_ configuration files. -* `gresconfig`: An editor for _gres.conf_ configuration files. * `cgroupconfig`: An editor for _cgroup.conf_ configuration files. +* `gresconfig`: An editor for _gres.conf_ configuration files. * `slurmconfig`: An editor for _slurm.conf_ configuration files. * `slurmdbdconfig`: An editor for _slurmdbd.conf_ configuration files. @@ -94,13 +94,13 @@ from slurmutils.editors import gresconfig from slurmutils.models import GRESName, GRESNode with gresconfig.edit("/etc/slurm/gres.conf") as config: - name = GRESName( + new_gres = GRESName( Name="gpu", Type="epyc", File="/dev/amd4", Cores=["0", "1"], ) - node = GRESNode( + new_node = GRESNode( NodeName="juju-abc654-[1-20]", Name="gpu", Type="epyc", @@ -108,8 +108,8 @@ with gresconfig.edit("/etc/slurm/gres.conf") as config: Count="12G", ) config.auto_detect = "rsmi" - config.names.append(name.dict()) - config.nodes.updaten(node.dict()) + config.names[new_gres.name] = [new_gres] + config.nodes[new_node.node_name] = [new_node] ``` ##### `slurmconfig` diff --git a/poetry.lock b/poetry.lock index eb7f21c..383a448 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,7 +1,253 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. -package = [] +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. + +[[package]] +name = "attrs" +version = "24.3.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.8" +files = [ + {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"}, + {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"}, +] + +[package.extras] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] + +[[package]] +name = "importlib-resources" +version = "6.4.5" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"}, + {file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] +type = ["pytest-mypy"] + +[[package]] +name = "jsonschema" +version = "4.23.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, + {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +jsonschema-specifications = ">=2023.03.6" +pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} +referencing = ">=0.28.4" +rpds-py = ">=0.7.1" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=24.6.0)"] + +[[package]] +name = "jsonschema-specifications" +version = "2023.12.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"}, + {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"}, +] + +[package.dependencies] +importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +referencing = ">=0.31.0" + +[[package]] +name = "pkgutil-resolve-name" +version = "1.3.10" +description = "Resolve a name to an object." +optional = false +python-versions = ">=3.6" +files = [ + {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"}, + {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, +] + +[[package]] +name = "referencing" +version = "0.35.1" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"}, + {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + +[[package]] +name = "rpds-py" +version = "0.20.1" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a649dfd735fff086e8a9d0503a9f0c7d01b7912a333c7ae77e1515c08c146dad"}, + {file = "rpds_py-0.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f16bc1334853e91ddaaa1217045dd7be166170beec337576818461268a3de67f"}, + {file = "rpds_py-0.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14511a539afee6f9ab492b543060c7491c99924314977a55c98bfa2ee29ce78c"}, + {file = "rpds_py-0.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3ccb8ac2d3c71cda472b75af42818981bdacf48d2e21c36331b50b4f16930163"}, + {file = "rpds_py-0.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c142b88039b92e7e0cb2552e8967077e3179b22359e945574f5e2764c3953dcf"}, + {file = "rpds_py-0.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f19169781dddae7478a32301b499b2858bc52fc45a112955e798ee307e294977"}, + {file = "rpds_py-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13c56de6518e14b9bf6edde23c4c39dac5b48dcf04160ea7bce8fca8397cdf86"}, + {file = "rpds_py-0.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:925d176a549f4832c6f69fa6026071294ab5910e82a0fe6c6228fce17b0706bd"}, + {file = "rpds_py-0.20.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:78f0b6877bfce7a3d1ff150391354a410c55d3cdce386f862926a4958ad5ab7e"}, + {file = "rpds_py-0.20.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3dd645e2b0dcb0fd05bf58e2e54c13875847687d0b71941ad2e757e5d89d4356"}, + {file = "rpds_py-0.20.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4f676e21db2f8c72ff0936f895271e7a700aa1f8d31b40e4e43442ba94973899"}, + {file = "rpds_py-0.20.1-cp310-none-win32.whl", hash = "sha256:648386ddd1e19b4a6abab69139b002bc49ebf065b596119f8f37c38e9ecee8ff"}, + {file = "rpds_py-0.20.1-cp310-none-win_amd64.whl", hash = "sha256:d9ecb51120de61e4604650666d1f2b68444d46ae18fd492245a08f53ad2b7711"}, + {file = "rpds_py-0.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:762703bdd2b30983c1d9e62b4c88664df4a8a4d5ec0e9253b0231171f18f6d75"}, + {file = "rpds_py-0.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0b581f47257a9fce535c4567782a8976002d6b8afa2c39ff616edf87cbeff712"}, + {file = "rpds_py-0.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:842c19a6ce894493563c3bd00d81d5100e8e57d70209e84d5491940fdb8b9e3a"}, + {file = "rpds_py-0.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42cbde7789f5c0bcd6816cb29808e36c01b960fb5d29f11e052215aa85497c93"}, + {file = "rpds_py-0.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c8e9340ce5a52f95fa7d3b552b35c7e8f3874d74a03a8a69279fd5fca5dc751"}, + {file = "rpds_py-0.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ba6f89cac95c0900d932c9efb7f0fb6ca47f6687feec41abcb1bd5e2bd45535"}, + {file = "rpds_py-0.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a916087371afd9648e1962e67403c53f9c49ca47b9680adbeef79da3a7811b0"}, + {file = "rpds_py-0.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:200a23239781f46149e6a415f1e870c5ef1e712939fe8fa63035cd053ac2638e"}, + {file = "rpds_py-0.20.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:58b1d5dd591973d426cbb2da5e27ba0339209832b2f3315928c9790e13f159e8"}, + {file = "rpds_py-0.20.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6b73c67850ca7cae0f6c56f71e356d7e9fa25958d3e18a64927c2d930859b8e4"}, + {file = "rpds_py-0.20.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d8761c3c891cc51e90bc9926d6d2f59b27beaf86c74622c8979380a29cc23ac3"}, + {file = "rpds_py-0.20.1-cp311-none-win32.whl", hash = "sha256:cd945871335a639275eee904caef90041568ce3b42f402c6959b460d25ae8732"}, + {file = "rpds_py-0.20.1-cp311-none-win_amd64.whl", hash = "sha256:7e21b7031e17c6b0e445f42ccc77f79a97e2687023c5746bfb7a9e45e0921b84"}, + {file = "rpds_py-0.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:36785be22066966a27348444b40389f8444671630063edfb1a2eb04318721e17"}, + {file = "rpds_py-0.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:142c0a5124d9bd0e2976089484af5c74f47bd3298f2ed651ef54ea728d2ea42c"}, + {file = "rpds_py-0.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dbddc10776ca7ebf2a299c41a4dde8ea0d8e3547bfd731cb87af2e8f5bf8962d"}, + {file = "rpds_py-0.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:15a842bb369e00295392e7ce192de9dcbf136954614124a667f9f9f17d6a216f"}, + {file = "rpds_py-0.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be5ef2f1fc586a7372bfc355986226484e06d1dc4f9402539872c8bb99e34b01"}, + {file = "rpds_py-0.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbcf360c9e3399b056a238523146ea77eeb2a596ce263b8814c900263e46031a"}, + {file = "rpds_py-0.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecd27a66740ffd621d20b9a2f2b5ee4129a56e27bfb9458a3bcc2e45794c96cb"}, + {file = "rpds_py-0.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0b937b2a1988f184a3e9e577adaa8aede21ec0b38320d6009e02bd026db04fa"}, + {file = "rpds_py-0.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6889469bfdc1eddf489729b471303739bf04555bb151fe8875931f8564309afc"}, + {file = "rpds_py-0.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:19b73643c802f4eaf13d97f7855d0fb527fbc92ab7013c4ad0e13a6ae0ed23bd"}, + {file = "rpds_py-0.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3c6afcf2338e7f374e8edc765c79fbcb4061d02b15dd5f8f314a4af2bdc7feb5"}, + {file = "rpds_py-0.20.1-cp312-none-win32.whl", hash = "sha256:dc73505153798c6f74854aba69cc75953888cf9866465196889c7cdd351e720c"}, + {file = "rpds_py-0.20.1-cp312-none-win_amd64.whl", hash = "sha256:8bbe951244a838a51289ee53a6bae3a07f26d4e179b96fc7ddd3301caf0518eb"}, + {file = "rpds_py-0.20.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6ca91093a4a8da4afae7fe6a222c3b53ee4eef433ebfee4d54978a103435159e"}, + {file = "rpds_py-0.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b9c2fe36d1f758b28121bef29ed1dee9b7a2453e997528e7d1ac99b94892527c"}, + {file = "rpds_py-0.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f009c69bc8c53db5dfab72ac760895dc1f2bc1b62ab7408b253c8d1ec52459fc"}, + {file = "rpds_py-0.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6740a3e8d43a32629bb9b009017ea5b9e713b7210ba48ac8d4cb6d99d86c8ee8"}, + {file = "rpds_py-0.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:32b922e13d4c0080d03e7b62991ad7f5007d9cd74e239c4b16bc85ae8b70252d"}, + {file = "rpds_py-0.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe00a9057d100e69b4ae4a094203a708d65b0f345ed546fdef86498bf5390982"}, + {file = "rpds_py-0.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49fe9b04b6fa685bd39237d45fad89ba19e9163a1ccaa16611a812e682913496"}, + {file = "rpds_py-0.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:aa7ac11e294304e615b43f8c441fee5d40094275ed7311f3420d805fde9b07b4"}, + {file = "rpds_py-0.20.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aa97af1558a9bef4025f8f5d8c60d712e0a3b13a2fe875511defc6ee77a1ab7"}, + {file = "rpds_py-0.20.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:483b29f6f7ffa6af845107d4efe2e3fa8fb2693de8657bc1849f674296ff6a5a"}, + {file = "rpds_py-0.20.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:37fe0f12aebb6a0e3e17bb4cd356b1286d2d18d2e93b2d39fe647138458b4bcb"}, + {file = "rpds_py-0.20.1-cp313-none-win32.whl", hash = "sha256:a624cc00ef2158e04188df5e3016385b9353638139a06fb77057b3498f794782"}, + {file = "rpds_py-0.20.1-cp313-none-win_amd64.whl", hash = "sha256:b71b8666eeea69d6363248822078c075bac6ed135faa9216aa85f295ff009b1e"}, + {file = "rpds_py-0.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:5b48e790e0355865197ad0aca8cde3d8ede347831e1959e158369eb3493d2191"}, + {file = "rpds_py-0.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3e310838a5801795207c66c73ea903deda321e6146d6f282e85fa7e3e4854804"}, + {file = "rpds_py-0.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249280b870e6a42c0d972339e9cc22ee98730a99cd7f2f727549af80dd5a963"}, + {file = "rpds_py-0.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e79059d67bea28b53d255c1437b25391653263f0e69cd7dec170d778fdbca95e"}, + {file = "rpds_py-0.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2b431c777c9653e569986ecf69ff4a5dba281cded16043d348bf9ba505486f36"}, + {file = "rpds_py-0.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da584ff96ec95e97925174eb8237e32f626e7a1a97888cdd27ee2f1f24dd0ad8"}, + {file = "rpds_py-0.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02a0629ec053fc013808a85178524e3cb63a61dbc35b22499870194a63578fb9"}, + {file = "rpds_py-0.20.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fbf15aff64a163db29a91ed0868af181d6f68ec1a3a7d5afcfe4501252840bad"}, + {file = "rpds_py-0.20.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:07924c1b938798797d60c6308fa8ad3b3f0201802f82e4a2c41bb3fafb44cc28"}, + {file = "rpds_py-0.20.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:4a5a844f68776a7715ecb30843b453f07ac89bad393431efbf7accca3ef599c1"}, + {file = "rpds_py-0.20.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:518d2ca43c358929bf08f9079b617f1c2ca6e8848f83c1225c88caeac46e6cbc"}, + {file = "rpds_py-0.20.1-cp38-none-win32.whl", hash = "sha256:3aea7eed3e55119635a74bbeb80b35e776bafccb70d97e8ff838816c124539f1"}, + {file = "rpds_py-0.20.1-cp38-none-win_amd64.whl", hash = "sha256:7dca7081e9a0c3b6490a145593f6fe3173a94197f2cb9891183ef75e9d64c425"}, + {file = "rpds_py-0.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b41b6321805c472f66990c2849e152aff7bc359eb92f781e3f606609eac877ad"}, + {file = "rpds_py-0.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a90c373ea2975519b58dece25853dbcb9779b05cc46b4819cb1917e3b3215b6"}, + {file = "rpds_py-0.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16d4477bcb9fbbd7b5b0e4a5d9b493e42026c0bf1f06f723a9353f5153e75d30"}, + {file = "rpds_py-0.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:84b8382a90539910b53a6307f7c35697bc7e6ffb25d9c1d4e998a13e842a5e83"}, + {file = "rpds_py-0.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4888e117dd41b9d34194d9e31631af70d3d526efc363085e3089ab1a62c32ed1"}, + {file = "rpds_py-0.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5265505b3d61a0f56618c9b941dc54dc334dc6e660f1592d112cd103d914a6db"}, + {file = "rpds_py-0.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e75ba609dba23f2c95b776efb9dd3f0b78a76a151e96f96cc5b6b1b0004de66f"}, + {file = "rpds_py-0.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1791ff70bc975b098fe6ecf04356a10e9e2bd7dc21fa7351c1742fdeb9b4966f"}, + {file = "rpds_py-0.20.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d126b52e4a473d40232ec2052a8b232270ed1f8c9571aaf33f73a14cc298c24f"}, + {file = "rpds_py-0.20.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c14937af98c4cc362a1d4374806204dd51b1e12dded1ae30645c298e5a5c4cb1"}, + {file = "rpds_py-0.20.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3d089d0b88996df627693639d123c8158cff41c0651f646cd8fd292c7da90eaf"}, + {file = "rpds_py-0.20.1-cp39-none-win32.whl", hash = "sha256:653647b8838cf83b2e7e6a0364f49af96deec64d2a6578324db58380cff82aca"}, + {file = "rpds_py-0.20.1-cp39-none-win_amd64.whl", hash = "sha256:fa41a64ac5b08b292906e248549ab48b69c5428f3987b09689ab2441f267d04d"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a07ced2b22f0cf0b55a6a510078174c31b6d8544f3bc00c2bcee52b3d613f74"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:68cb0a499f2c4a088fd2f521453e22ed3527154136a855c62e148b7883b99f9a"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa3060d885657abc549b2a0f8e1b79699290e5d83845141717c6c90c2df38311"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:95f3b65d2392e1c5cec27cff08fdc0080270d5a1a4b2ea1d51d5f4a2620ff08d"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2cc3712a4b0b76a1d45a9302dd2f53ff339614b1c29603a911318f2357b04dd2"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d4eea0761e37485c9b81400437adb11c40e13ef513375bbd6973e34100aeb06"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f5179583d7a6cdb981151dd349786cbc318bab54963a192692d945dd3f6435d"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fbb0ffc754490aff6dabbf28064be47f0f9ca0b9755976f945214965b3ace7e"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:a94e52537a0e0a85429eda9e49f272ada715506d3b2431f64b8a3e34eb5f3e75"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:92b68b79c0da2a980b1c4197e56ac3dd0c8a149b4603747c4378914a68706979"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:93da1d3db08a827eda74356f9f58884adb254e59b6664f64cc04cdff2cc19b0d"}, + {file = "rpds_py-0.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:754bbed1a4ca48479e9d4182a561d001bbf81543876cdded6f695ec3d465846b"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ca449520e7484534a2a44faf629362cae62b660601432d04c482283c47eaebab"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:9c4cb04a16b0f199a8c9bf807269b2f63b7b5b11425e4a6bd44bd6961d28282c"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63804105143c7e24cee7db89e37cb3f3941f8e80c4379a0b355c52a52b6780"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:55cd1fa4ecfa6d9f14fbd97ac24803e6f73e897c738f771a9fe038f2f11ff07c"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0f8f741b6292c86059ed175d80eefa80997125b7c478fb8769fd9ac8943a16c0"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fc212779bf8411667234b3cdd34d53de6c2b8b8b958e1e12cb473a5f367c338"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ad56edabcdb428c2e33bbf24f255fe2b43253b7d13a2cdbf05de955217313e6"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0a3a1e9ee9728b2c1734f65d6a1d376c6f2f6fdcc13bb007a08cc4b1ff576dc5"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:e13de156137b7095442b288e72f33503a469aa1980ed856b43c353ac86390519"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:07f59760ef99f31422c49038964b31c4dfcfeb5d2384ebfc71058a7c9adae2d2"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:59240685e7da61fb78f65a9f07f8108e36a83317c53f7b276b4175dc44151684"}, + {file = "rpds_py-0.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:83cba698cfb3c2c5a7c3c6bac12fe6c6a51aae69513726be6411076185a8b24a"}, + {file = "rpds_py-0.20.1.tar.gz", hash = "sha256:e1791c4aabd117653530dccd24108fa03cc6baf21f58b950d0a73c3b3b29a350"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "zipp" +version = "3.20.2" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, + {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.8" -content-hash = "dedbcc8ad01960ccbef8502c70bda77771c2826a438e1e94ef27a36c71acd91a" +content-hash = "342925a738dde64896ffd29777de5f04d758a9a17699ce2b2d02a893eb22ee17" diff --git a/pyproject.toml b/pyproject.toml index 4f81dc0..a615c1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "slurmutils" -version = "0.10.0" +version = "0.11.0" description = "Utilities and APIs for interfacing with the Slurm workload manager." repository = "https://github.com/charmed-hpc/slurmutils" authors = ["Jason C. Nucciarone "] @@ -42,6 +42,8 @@ classifiers=[ [tool.poetry.dependencies] python = ">=3.8" +jsonschema = "~=4.23.0" +typing-extensions = "4.12.2" [tool.poetry.urls] "Bug Tracker" = "https://github.com/charmed-hpc/slurmutils/issues" @@ -64,7 +66,7 @@ skip = "build,lib,venv,icon.svg,.tox,.git,.mypy_cache,.ruff_cache,.vscode,.cover # Formatting tools configuration [tool.black] line-length = 99 -target-version = ["py38"] +target-version = ["py310"] # Linting tools configuration [tool.ruff] diff --git a/slurmutils/models/__init__.py b/slurmutils/models/__init__.py index f43c0e9..e7db9f5 100644 --- a/slurmutils/models/__init__.py +++ b/slurmutils/models/__init__.py @@ -18,7 +18,9 @@ from .cgroup import CgroupConfig as CgroupConfig from .gres import GRESConfig as GRESConfig from .gres import GRESName as GRESName +from .gres import GRESNameMapping as GRESNameMapping from .gres import GRESNode as GRESNode +from .gres import GRESNodeMapping as GRESNodeMapping from .slurm import DownNodes as DownNodes from .slurm import FrontendNode as FrontendNode from .slurm import Node as Node diff --git a/slurmutils/models/gres.py b/slurmutils/models/gres.py index 142f5f3..10c6344 100644 --- a/slurmutils/models/gres.py +++ b/slurmutils/models/gres.py @@ -14,20 +14,60 @@ """Data models for `gres.conf` configuration file.""" -__all__ = ["GRESConfig", "GRESName", "GRESNode"] +__all__ = ["GRESConfig", "GRESName", "GRESNode", "GRESNodeMapping", "GRESNameMapping"] -import copy +from abc import ABC +from collections.abc import MutableMapping, Sequence +from itertools import chain from typing import Any -from .model import BaseModel, clean, marshall_content, parse_line +from jsonschema import ValidationError, validate + +from .model import BaseMapping, BaseModel, clean, marshall_content, parse_line from .option import GRESConfigOptionSet, GRESNameOptionSet, GRESNodeOptionSet +from .schema import ( + GRES_NAME_MAPPING_SCHEMA, + GRES_NAME_SCHEMA, + GRES_NODE_MAPPING_SCHEMA, + GRES_NODE_SCHEMA, +) + + +def _gres_name_decoder(o: Any) -> Any: + """Decode `gres.conf` data model within JSON object. + + Args: + o: JSON object to decode. + """ + try: + validate(o, schema=GRES_NAME_SCHEMA) + return GRESName.from_dict(o) + except ValidationError: + pass + + return o + + +def _gres_node_decoder(o: Any) -> Any: + """Decode `gres.conf` node data model within a JSON object. + + Args: + o: JSON object in to decode. + """ + try: + validate(o, schema=GRES_NODE_SCHEMA) + return GRESNode.from_dict(o) + except ValidationError: + pass + + return o class GRESName(BaseModel): """`gres.conf` name data model.""" - def __init__(self, *, Name, **kwargs) -> None: # noqa N803 - super().__init__(GRESNameOptionSet, Name=Name, **kwargs) + def __init__(self, **kwargs) -> None: # noqa N803 + super().__init__(GRESNameOptionSet, **kwargs) @classmethod def from_str(cls, content: str) -> "GRESName": @@ -81,7 +121,7 @@ def cores(self) -> list[str] | None: return self.data.get("Cores", None) @cores.setter - def cores(self, value: list[str]) -> None: + def cores(self, value: Sequence[str]) -> None: self.data["Cores"] = value @cores.deleter @@ -113,7 +153,7 @@ def flags(self) -> list[str] | None: return self.data.get("Flags", None) @flags.setter - def flags(self, value: list[str]) -> None: + def flags(self, value: Sequence[str]) -> None: self.data["Flags"] = value @flags.deleter @@ -129,7 +169,7 @@ def links(self) -> list[str] | None: return self.data.get("Links", None) @links.setter - def links(self, value: list[str]) -> None: + def links(self, value: Sequence[str]) -> None: self.data["Links"] = value @links.deleter @@ -189,43 +229,70 @@ def type(self) -> None: class GRESNode(GRESName): """`gres.conf` node data model.""" - def __init__(self, *, NodeName: str, **kwargs): # noqa N803 - self.__node_name = NodeName + def __init__(self, **kwargs): # noqa N803 # Want to share `GRESName` descriptors, but not constructor. BaseModel.__init__(self, GRESNodeOptionSet, **kwargs) @classmethod - def from_dict(cls, data: dict[str, Any]) -> "GRESNode": + def from_dict(cls, data: MutableMapping[str, Any]) -> "GRESNode": """Construct `GRESNode` data model from dictionary object.""" - node_name = list(data.keys())[0] - return cls(NodeName=node_name, **data[node_name]) + return cls(**data) @classmethod def from_str(cls, content: str) -> "GRESNode": """Construct `GRESNode` data model from a gres.conf configuration line.""" return cls(**parse_line(GRESNodeOptionSet, content)) - def dict(self) -> dict[str, Any]: - """Return `GRESNode` data model as a dictionary object.""" - return copy.deepcopy({self.__node_name: self.data}) - def __str__(self) -> str: """Return `GRESNode` data model as a gres.conf configuration line.""" - line = [f"NodeName={self.__node_name}"] - line.extend(marshall_content(GRESNodeOptionSet, self.data)) - return " ".join(line) + return " ".join( + [f"NodeName={self.node_name}"] + + marshall_content(GRESNodeOptionSet, self._slice(["NodeName"])) + ) @property def node_name(self) -> str: """Node(s) the generic resource configuration will be applied to. - Value `NodeName` specification can use a Slurm hostlist specification. + Format of the value for `NodeName` can be in the Slurm hostlist specification format. """ - return self.__node_name + return self.data["NodeName"] @node_name.setter def node_name(self, value: str) -> None: - self.__node_name = value + self.data["NodeName"] = value + + +class _GRESBaseMapping(BaseMapping, ABC): + """Base `gres.conf` data model mapping.""" + + def __str__(self) -> str: + """Return `gres.conf` data model mapping as gres.conf configuration block.""" + return "\n".join(str(gres) for gres in chain.from_iterable(self.values())) + + +class GRESNameMapping(_GRESBaseMapping): + """Map of generic resource names to `gres.conf` name data models.""" + + @property + def _decoder(self) -> Any: + return _gres_name_decoder + + @property + def _schema(self) -> dict[str, Any]: + return GRES_NAME_MAPPING_SCHEMA + + +class GRESNodeMapping(_GRESBaseMapping): + """Map of node names to list of `gres.conf` node data models.""" + + @property + def _decoder(self) -> Any: + return _gres_node_decoder + + @property + def _schema(self) -> dict[str, Any]: + return GRES_NODE_MAPPING_SCHEMA class GRESConfig(BaseModel): @@ -234,51 +301,44 @@ class GRESConfig(BaseModel): def __init__( self, *, - Names: list[str] | None = None, # noqa N803 - Nodes: dict[str, Any] | None = None, # noqa N803 + Names: MutableMapping[str, Sequence[GRESName]] | None = None, # noqa N803 + Nodes: MutableMapping[str, Sequence[GRESNode]] | None = None, # noqa N803 **kwargs, ) -> None: super().__init__(GRESConfigOptionSet, **kwargs) - self.data["Names"] = Names or [] - self.data["Nodes"] = Nodes or {} + self.data["Names"] = GRESNameMapping(Names) + self.data["Nodes"] = GRESNodeMapping(Nodes) @classmethod def from_str(cls, content: str) -> "GRESConfig": """Construct `gres.conf` data model from a gres.conf configuration file.""" - data = {} - lines = content.splitlines() - for line in lines: - config = clean(line) - if config is None: + config = {"Names": GRESNameMapping(), "Nodes": GRESNodeMapping()} + for line in [clean(line) for line in content.splitlines()]: + if line is None: continue - if config.startswith("Name"): - data["Names"] = data.get("Names", []) + [GRESName.from_str(config).dict()] - elif config.startswith("NodeName"): - nodes = data.get("Nodes", {}) - nodes.update(GRESNode.from_str(config).dict()) - data["Nodes"] = nodes + if line.startswith("Name"): + new = GRESName.from_str(line) + config["Names"][new.name] = config["Names"].get(new.name, []) + [new] + elif line.startswith("NodeName"): + new = GRESNode.from_str(line) + config["Nodes"][new.node_name] = config["Nodes"].get(new.node_name, []) + [new] else: - data.update(parse_line(GRESConfigOptionSet, config)) + config.update(parse_line(GRESConfigOptionSet, line)) - return GRESConfig.from_dict(data) + return GRESConfig(**config) def __str__(self) -> str: - """Return `gres.conf` data model in gres.conf format.""" - data = self.dict() - global_auto_detect = data.pop("AutoDetect", None) - names = data.pop("Names", []) - nodes = data.pop("Nodes", {}) - - content = [] - if global_auto_detect: - content.append(f"AutoDetect={global_auto_detect}") - if names: - content.extend([str(GRESName(**name)) for name in names]) - if nodes: - content.extend([str(GRESNode(NodeName=k, **v)) for k, v in nodes.items()]) - - return "\n".join(content) + "\n" + """Return `gres.conf` data model in gres.conf configuration format.""" + out = [] + if self.auto_detect: + out.append(f"AutoDetect={self.auto_detect}") + if self.names: + out.append(str(self.names)) + if self.nodes: + out.append(str(self.nodes)) + + return "\n".join(out) + "\n" @property def auto_detect(self) -> str | None: @@ -290,7 +350,7 @@ def auto_detect(self) -> str | None: `GRESNode` and`GRESName` to override the global automatic hardware detection mechanism for specific nodes or resource names. """ - return self.data.get("AutoDetect", None) + return self.data.get("AutoDetect") @auto_detect.setter def auto_detect(self, value: str) -> None: @@ -304,27 +364,27 @@ def auto_detect(self) -> None: pass @property - def names(self) -> list[dict[str, Any]] | None: - """List of configured generic resources.""" - return self.data.get("Names", None) + def names(self) -> GRESNameMapping: + """Get map of configured generic resources.""" + return self.data.get("Names") @names.setter - def names(self, value: list[dict[str, Any]]) -> None: + def names(self, value: GRESNameMapping) -> None: self.data["Names"] = value @names.deleter def names(self) -> None: - self.data["Names"] = [] + self.data["Names"] = GRESNameMapping() @property - def nodes(self) -> dict[str, dict[str, Any]]: - """Map of nodes with configured generic resources.""" - return self.data["Nodes"] + def nodes(self) -> GRESNodeMapping: + """Get map of node names with configured generic resources.""" + return self.data.get("Nodes") @nodes.setter - def nodes(self, value: dict[str, GRESNode]) -> None: + def nodes(self, value: GRESNodeMapping) -> None: self.data["Nodes"] = value @nodes.deleter def nodes(self) -> None: - self.data["Nodes"] = {} + self.data["Nodes"] = GRESNodeMapping() diff --git a/slurmutils/models/model.py b/slurmutils/models/model.py index 00dbaf0..2ea03b4 100644 --- a/slurmutils/models/model.py +++ b/slurmutils/models/model.py @@ -15,6 +15,7 @@ """Base classes and methods for composing Slurm data models.""" __all__ = [ + "BaseMapping", "BaseModel", "clean", "format_key", @@ -28,7 +29,11 @@ import re import shlex from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Optional, Tuple +from collections.abc import Callable, Iterable, MutableMapping +from typing import Any + +from jsonschema import ValidationError, validate +from typing_extensions import Self from ..exceptions import ModelError @@ -57,7 +62,52 @@ def format_key(key: str) -> str: return _camelize.sub(r"_", key).lower() -def generate_descriptors(opt: str) -> Tuple[Callable, Callable, Callable]: +def _expand_iter(i: Iterable[Any]) -> list[Any]: + """Recursively expand a complex iterable with nested Slurm data models. + + Args: + i: Iterable to expand. + """ + out = [] + for v in i: + if issubclass(type(v), BaseModel): + out.append(_expand_dict(v.dict())) + elif issubclass(type(v), MutableMapping): + out.append(_expand_dict(v)) + elif isinstance(v, list | tuple): + out.append(_expand_iter(v)) + else: + out.append(v) + + return out + + +def _expand_dict(d: MutableMapping[str, Any]) -> dict[str, Any]: + """Recursively expand a complex dictionary with nested Slurm data models. + + Args: + d: Dictionary to expand. + """ + out = {} + for k, v in d.items(): + if issubclass(type(v), BaseModel): + out.update({k: _expand_dict(v.dict())}) + elif issubclass(type(v), MutableMapping): + out.update({k: _expand_dict(v)}) + elif isinstance(v, list | tuple): + out.update({k: _expand_iter(v)}) + else: + out[k] = v + + return out + + +def expand(d: MutableMapping[str, Any]) -> dict[str, Any]: + """Expand a complex dictionary with nested Slurm data models.""" + return _expand_dict(d) + + +def generate_descriptors(opt: str) -> tuple[Callable, Callable, Callable]: """Generate descriptors for retrieving and mutating configuration options. Args: @@ -76,7 +126,7 @@ def deleter(self): return getter, setter, deleter -def clean(line: str) -> Optional[str]: +def clean(line: str) -> str | None: """Clean line before further processing. Returns: @@ -85,7 +135,7 @@ def clean(line: str) -> Optional[str]: return cleaned if (cleaned := line.split("#", maxsplit=1)[0]) != "" else None -def parse_line(options, line: str) -> Dict[str, Any]: +def parse_line(options, line: str) -> dict[str, Any]: """Parse configuration line. Args: @@ -110,7 +160,7 @@ def parse_line(options, line: str) -> Dict[str, Any]: return data -def marshall_content(options, line: Dict[str, Any]) -> List[str]: +def marshall_content(options, line: MutableMapping[str, Any]) -> list[str]: """Marshall data model content back into configuration line. Args: @@ -148,8 +198,16 @@ def __init__(self, validator=None, /, **kwargs) -> None: self.data = kwargs + def _slice(self, exclude: Iterable[str]) -> dict[str, Any]: + """Slice the internal data store by excluding specific keys. + + Args: + exclude: List of keys to exclude from slice. + """ + return {k: v for k, v in self.data.items() if k not in exclude} + @classmethod - def from_dict(cls, data: Dict[str, Any]): + def from_dict(cls, data: MutableMapping[str, Any]): """Construct new model from dictionary.""" return cls(**data) @@ -168,7 +226,7 @@ def from_str(cls, content: str): def __str__(self) -> str: """Return model as configuration string.""" - def dict(self) -> Dict[str, Any]: + def dict(self) -> dict[str, Any]: """Return model as dictionary.""" return copy.deepcopy(self.data) @@ -179,3 +237,66 @@ def json(self) -> str: def update(self, other) -> None: """Update current data model content with content of other data model.""" self.data.update(other.data) + + +class BaseMapping(MutableMapping[str, Any], ABC): + """Base map for Slurm data model mappings.""" + + def __init__(self, d: MutableMapping[str, Any] | None = None) -> None: + if not d: + self._data = {} + return + + try: + d = expand(d) + validate(d, schema=self._schema) + self._data = json.loads(json.dumps(d), object_hook=self._decoder) + except ValidationError as e: + raise ModelError(e.message) + + @property + @abstractmethod + def _schema(self) -> dict[str, Any]: + """Get data model JSON schema.""" + + @property + @abstractmethod + def _decoder(self) -> Any: + """Get data model decoder.""" + + @classmethod + def from_dict(cls, d: MutableMapping[str, Any]) -> Self: + """Create model from dictionary.""" + return cls(d) + + @classmethod + def from_json(cls, s: str | bytes | bytearray) -> Self: + """Create model from JSON object.""" + return cls(json.loads(s)) + + def dict(self) -> dict[str, Any]: + """Return data model mapping as an expanded dictionary object.""" + return expand(self._data) + + def json(self) -> str: + """Return model mapping as a JSON object.""" + return json.dumps(expand(self._data)) + + @abstractmethod + def __str__(self) -> str: # noqa D105 + pass + + def __getitem__(self, key: str, /) -> Any: # noqa D105 + return self._data[key] + + def __setitem__(self, key: str, value: Any, /) -> None: # noqa D105 + self._data[key] = value + + def __delitem__(self, key: str, /) -> None: # noqa D105 + del self._data[key] + + def __len__(self) -> int: # noqa D105 + return len(self._data) + + def __iter__(self) -> Iterable[str]: # noqa D105 + return iter(self._data) diff --git a/slurmutils/models/schema.py b/slurmutils/models/schema.py new file mode 100644 index 0000000..a7403b7 --- /dev/null +++ b/slurmutils/models/schema.py @@ -0,0 +1,97 @@ +# Copyright 2024 Canonical Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""JSON schemas for validating Slurm data models.""" + +__all__ = [ + "GRES_NAME_SCHEMA", + "GRES_NODE_SCHEMA", + "GRES_NAME_MAPPING_SCHEMA", + "GRES_NODE_MAPPING_SCHEMA", +] + +_GLOBAL_SCHEMA_VERSION = "https://json-schema.org/draft/2020-12/schema" + +# `gres.conf` data model schemas. +GRES_NAME_SCHEMA = { + "$schema": _GLOBAL_SCHEMA_VERSION, + "type": "object", + "properties": { + "AutoDetect": {"type": "string"}, + "Count": {"type": "string"}, + "Cores": {"type": "array", "items": {"type": "string"}, "uniqueItems": True}, + "File": {"type": "string"}, + "Flags": {"type": "array", "items": {"type": "string"}, "uniqueItems": True}, + "Links": {"type": "array", "items": {"type": "string"}}, + "MultipleFiles": {"type": "string"}, + "Name": {"type": "string"}, + "Type": {"type": "string"}, + }, + "additionalProperties": False, +} + +GRES_NODE_SCHEMA = { + "$schema": _GLOBAL_SCHEMA_VERSION, + "type": "object", + "properties": { + "NodeName": {"type": "string"}, + **GRES_NAME_SCHEMA["properties"], + }, + "additionalProperties": False, +} + +GRES_NAME_MAPPING_SCHEMA = { + "$schema": _GLOBAL_SCHEMA_VERSION, + "type": "object", + "patternProperties": { + "^.+$": { + "type": "array", + "items": { + "$ref": "#/$defs/GRESName", + }, + "uniqueItems": True, + }, + }, + "$defs": {"GRESName": GRES_NAME_SCHEMA}, +} + +GRES_NODE_MAPPING_SCHEMA = { + "$schema": _GLOBAL_SCHEMA_VERSION, + "type": "object", + "patternProperties": { + "^.+$": { + "type": "array", + "items": {"$ref": "#/$defs/GRESNode"}, + "uniqueItems": True, + } + }, + "$defs": { + "GRESNode": GRES_NODE_SCHEMA, + }, +} + +GRES_CONFIG_SCHEMA = { + "$schema": _GLOBAL_SCHEMA_VERSION, + "type": "object", + "properties": { + "AutoDetect": {"type": "string"}, + "Names": {"$ref", "#/$defs/GRESNameMapping"}, + "Nodes": {"$ref", "#/$defs/GRESNodeMapping"}, + }, + "additionalProperties": False, + "$defs": { + "GRESNameMapping": GRES_NAME_MAPPING_SCHEMA, + "GRESNodeMapping": GRES_NODE_MAPPING_SCHEMA, + }, +} diff --git a/tests/unit/editors/constants.py b/tests/unit/editors/constants.py index caa3403..8ec2a6a 100644 --- a/tests/unit/editors/constants.py +++ b/tests/unit/editors/constants.py @@ -26,7 +26,8 @@ Name=mps Count=100 File=/dev/nvidia3 Name=bandwidth Type=lustre Count=4G Flags=CountOnly -NodeName=juju-c9c6f-[1-10] Name=gpu Type=rtx File=/dev/nvidia[0-3] Count=8G +NodeName=juju-abc654-1 Name=gpu Type=tesla_t4 File=/dev/nvidia[0-1] Count=8G +NodeName=juju-abc654-1 Name=gpu Type=l40s File=/dev/nvidia[2-3] Count=12G """ EXAMPLE_SLURM_CONFIG = """# diff --git a/tests/unit/editors/test_gresconfig.py b/tests/unit/editors/test_gresconfig.py index f9513d4..5326783 100644 --- a/tests/unit/editors/test_gresconfig.py +++ b/tests/unit/editors/test_gresconfig.py @@ -17,7 +17,7 @@ from pyfakefs.fake_filesystem_unittest import TestCase from slurmutils.editors import gresconfig -from slurmutils.models import GRESName, GRESNode +from slurmutils.models import GRESName, GRESNameMapping, GRESNode, GRESNodeMapping class TestGRESConfigEditor(TestCase): @@ -31,29 +31,45 @@ def test_loads(self) -> None: """Test `loads` function from the `gresconfig` editor module.""" config = gresconfig.loads(EXAMPLE_GRES_CONFIG) self.assertEqual(config.auto_detect, "nvml") - self.assertListEqual( - config.names, - [ - {"Name": "gpu", "Type": "gp100", "File": "/dev/nvidia0", "Cores": ["0", "1"]}, - {"Name": "gpu", "Type": "gp100", "File": "/dev/nvidia1", "Cores": ["0", "1"]}, - {"Name": "gpu", "Type": "p6000", "File": "/dev/nvidia2", "Cores": ["2", "3"]}, - {"Name": "gpu", "Type": "p6000", "File": "/dev/nvidia3", "Cores": ["2", "3"]}, - {"Name": "mps", "Count": "200", "File": "/dev/nvidia0"}, - {"Name": "mps", "Count": "200", "File": "/dev/nvidia1"}, - {"Name": "mps", "Count": "100", "File": "/dev/nvidia2"}, - {"Name": "mps", "Count": "100", "File": "/dev/nvidia3"}, - {"Name": "bandwidth", "Type": "lustre", "Count": "4G", "Flags": ["CountOnly"]}, - ], + self.assertDictEqual( + config.names.dict(), + { + "gpu": [ + {"Name": "gpu", "Type": "gp100", "File": "/dev/nvidia0", "Cores": ["0", "1"]}, + {"Name": "gpu", "Type": "gp100", "File": "/dev/nvidia1", "Cores": ["0", "1"]}, + {"Name": "gpu", "Type": "p6000", "File": "/dev/nvidia2", "Cores": ["2", "3"]}, + {"Name": "gpu", "Type": "p6000", "File": "/dev/nvidia3", "Cores": ["2", "3"]}, + ], + "mps": [ + {"Name": "mps", "Count": "200", "File": "/dev/nvidia0"}, + {"Name": "mps", "Count": "200", "File": "/dev/nvidia1"}, + {"Name": "mps", "Count": "100", "File": "/dev/nvidia2"}, + {"Name": "mps", "Count": "100", "File": "/dev/nvidia3"}, + ], + "bandwidth": [ + {"Name": "bandwidth", "Type": "lustre", "Count": "4G", "Flags": ["CountOnly"]}, + ], + }, ) self.assertDictEqual( - config.nodes, + config.nodes.dict(), { - "juju-c9c6f-[1-10]": { - "Name": "gpu", - "Type": "rtx", - "File": "/dev/nvidia[0-3]", - "Count": "8G", - } + "juju-abc654-1": [ + { + "NodeName": "juju-abc654-1", + "Name": "gpu", + "Type": "tesla_t4", + "File": "/dev/nvidia[0-1]", + "Count": "8G", + }, + { + "NodeName": "juju-abc654-1", + "Name": "gpu", + "Type": "l40s", + "File": "/dev/nvidia[2-3]", + "Count": "12G", + }, + ] }, ) @@ -66,13 +82,13 @@ def test_dumps(self) -> None: def test_edit(self) -> None: """Test `edit` context manager from the `gresconfig` editor module.""" - name = GRESName( + new_name = GRESName( Name="gpu", Type="epyc", File="/dev/amd4", Cores=["0", "1"], ) - node = GRESNode( + new_node = GRESNode( NodeName="juju-abc654-[1-20]", Name="gpu", Type="epyc", @@ -83,13 +99,13 @@ def test_edit(self) -> None: # Set new values with each accessor. with gresconfig.edit("/etc/slurm/gres.conf") as config: config.auto_detect = "rsmi" - config.names = [name.dict()] - config.nodes = node.dict() + config.names["gpu"].append(new_name) + config.nodes[new_node.node_name] = [new_node] config = gresconfig.load("/etc/slurm/gres.conf") self.assertEqual(config.auto_detect, "rsmi") - self.assertListEqual(config.names, [name.dict()]) - self.assertDictEqual(config.nodes, node.dict()) + self.assertDictEqual(config.names["gpu"][-1].dict(), new_name.dict()) + self.assertDictEqual(config.nodes["juju-abc654-[1-20]"][0].dict(), new_node.dict()) # Delete all configured values from GRES configuration. with gresconfig.edit("/etc/slurm/gres.conf") as config: @@ -99,5 +115,5 @@ def test_edit(self) -> None: config = gresconfig.load("/etc/slurm/gres.conf") self.assertIsNone(config.auto_detect) - self.assertListEqual(config.names, []) - self.assertDictEqual(config.nodes, {}) + self.assertEqual(config.names, GRESNameMapping()) + self.assertEqual(config.nodes, GRESNodeMapping()) diff --git a/tests/unit/models/test_gres.py b/tests/unit/models/test_gres.py index 3cf72ab..01e06c7 100644 --- a/tests/unit/models/test_gres.py +++ b/tests/unit/models/test_gres.py @@ -16,6 +16,7 @@ from unittest import TestCase from slurmutils.models import GRESConfig, GRESName, GRESNode +from slurmutils.models.gres import GRESNameMapping, GRESNodeMapping class TestGRESConfig(TestCase): @@ -28,12 +29,11 @@ def setUp(self) -> None: ) self.nodes = GRESNode.from_dict( { - "juju-c9c6f-[1-10]": { - "Name": "gpu", - "Type": "rtx", - "File": "/dev/nvidia[0-3]", - "Count": "8G", - } + "NodeName": "juju-c9c6f-[1-10]", + "Name": "gpu", + "Type": "rtx", + "File": "/dev/nvidia[0-3]", + "Count": "8G", } ) @@ -47,17 +47,19 @@ def test_auto_detect(self) -> None: def test_names(self) -> None: """Test `Names` descriptor.""" - self.config.names = [self.names.dict()] - self.assertListEqual(self.config.names, [self.names.dict()]) + new = GRESNameMapping({self.names.name: [self.names]}) + self.config.names = new + self.assertEqual(self.config.names, new) del self.config.names - self.assertListEqual(self.config.names, []) + self.assertEqual(self.config.names, GRESNameMapping()) def test_nodes(self) -> None: """Test `Nodes` descriptor.""" - self.config.nodes = self.nodes.dict() - self.assertDictEqual(self.config.nodes, self.nodes.dict()) + new = GRESNodeMapping({self.nodes.node_name: [self.nodes]}) + self.config.nodes = new + self.assertEqual(self.config.nodes, new) del self.config.nodes - self.assertDictEqual(self.config.nodes, {}) + self.assertEqual(self.config.nodes, GRESNodeMapping()) class TestGRESName(TestCase): diff --git a/tox.ini b/tox.ini index 885750c..4dffeb5 100644 --- a/tox.ini +++ b/tox.ini @@ -54,6 +54,8 @@ deps = pytest pyfakefs coverage[toml] + jsonschema + typing-extensions commands = coverage run \ --source={[vars]src_path} \