From 47e18d944abbb14a8205acd90fb68faf6a99a6fb Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 16 Jul 2024 06:01:49 +0300 Subject: [PATCH 1/6] Replace custom roles with styled extlinks (#1351) --- .editorconfig | 2 +- _extensions/custom_roles.py | 43 ---------------------------------- _static/devguide_overrides.css | 11 ++++++++- conf.py | 9 +++---- internals/parser.rst | 2 +- requirements.txt | 2 +- 6 files changed, 16 insertions(+), 53 deletions(-) delete mode 100644 _extensions/custom_roles.py diff --git a/.editorconfig b/.editorconfig index 0169eed951..5230b76950 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,5 +11,5 @@ indent_size = 4 [*.rst] indent_size = 3 -[*.yml] +[*.{css,yml}] indent_size = 2 diff --git a/_extensions/custom_roles.py b/_extensions/custom_roles.py deleted file mode 100644 index f8c9bb8951..0000000000 --- a/_extensions/custom_roles.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Sphinx extension to add custom roles. - -Based on https://protips.readthedocs.io/link-roles.html -""" -import urllib.parse - -from docutils import nodes - - -def setup(app): - # role to link to cpython files - app.add_role( - "cpy-file", - autolink("https://github.com/python/cpython/blob/main/{}"), - ) - # role to link to cpython labels - app.add_role( - "gh-label", - autolink("https://github.com/python/cpython/labels/{}"), - ) - # Parallel safety: - # https://www.sphinx-doc.org/en/master/extdev/index.html#extension-metadata - return {"parallel_read_safe": True, "parallel_write_safe": True} - - -def autolink(pattern): - def role(name, rawtext, text, lineno, inliner, _options=None, _content=None): - """Combine literal + reference (unless the text is prefixed by a !).""" - if " " in text: - url_text = urllib.parse.quote(text) - else: - url_text = text - url = pattern.format(url_text) - # don't create a reference if the text starts with ! - if text.startswith('!'): - node = nodes.literal(rawtext, text[1:]) - else: - node = nodes.reference( - rawtext, '', nodes.literal(rawtext, text), refuri=url, internal=False - ) - return [node], [] - - return role diff --git a/_static/devguide_overrides.css b/_static/devguide_overrides.css index e86b6c1776..9237445d2f 100644 --- a/_static/devguide_overrides.css +++ b/_static/devguide_overrides.css @@ -38,7 +38,7 @@ .release-cycle-chart .release-cycle-blob { stroke-width: 1.6px; - /* default colours, overriden below for individual statuses */ + /* default colours, overridden below for individual statuses */ fill: var(--color-background-primary); stroke: var(--color-foreground-primary); } @@ -85,3 +85,12 @@ .bad pre { border-left: 3px solid rgb(244, 76, 78); } + +.extlink-cpy-file, +.extlink-gh-label { + border: 1px solid var(--color-background-border); + border-radius: .2em; + font-family: var(--font-stack--monospace); + font-size: var(--font-size--small--2); + padding: .1em .2em; +} diff --git a/conf.py b/conf.py index 6de594b235..fc46f32234 100644 --- a/conf.py +++ b/conf.py @@ -1,12 +1,6 @@ -import os -import sys import time -# Location of custom extensions. -sys.path.insert(0, os.path.abspath(".") + "/_extensions") - extensions = [ - 'custom_roles', 'notfound.extension', 'sphinx.ext.extlinks', 'sphinx.ext.intersphinx', @@ -176,7 +170,10 @@ # This config is a dictionary of external sites, # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html +_repo = "https://github.com/python/cpython" extlinks = { + "cpy-file": (f"{_repo}/blob/main/%s", "%s"), + "gh-label": (f"{_repo}/labels/%s", "%s"), "github": ("https://github.com/%s", "%s"), "github-user": ("https://github.com/%s", "@%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), diff --git a/internals/parser.rst b/internals/parser.rst index d9378dbd73..8ad47c1f74 100644 --- a/internals/parser.rst +++ b/internals/parser.rst @@ -537,7 +537,7 @@ parser (the one used by the interpreter) just execute: :: make regen-pegen -using the :cpy-file:`!Makefile` in the main directory. If you are on Windows you can +using the ``Makefile`` in the main directory. If you are on Windows you can use the Visual Studio project files to regenerate the parser or to execute: :: ./PCbuild/build.bat --regen diff --git a/requirements.txt b/requirements.txt index 7b98f183dd..10619e8617 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ sphinx-notfound-page>=1.0.0 sphinx_copybutton>=0.3.3 sphinxext-opengraph>=0.7.1 sphinxext-rediraffe -Sphinx~=7.3.7 +Sphinx~=7.4.0 From 7be740e6bbad2f1eac9b8d4fedb376514fc0c9d6 Mon Sep 17 00:00:00 2001 From: Tomas R Date: Tue, 16 Jul 2024 21:42:01 +0200 Subject: [PATCH 2/6] Fix broken link to LCOV (#1356) --- testing/coverage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/coverage.rst b/testing/coverage.rst index 6820a341af..d01141a7dc 100644 --- a/testing/coverage.rst +++ b/testing/coverage.rst @@ -306,5 +306,5 @@ about 20 to 30 minutes on a modern computer. .. _issue tracker: https://github.com/python/cpython/issues .. _gcov: https://gcc.gnu.org/onlinedocs/gcc/Gcov.html -.. _lcov: https://ltp.sourceforge.net/coverage/lcov.php +.. _lcov: https://github.com/linux-test-project/lcov .. _coverage.py: https://coverage.readthedocs.io/en/latest/ From 890564fec0f1d495194333fa773ddce869ffd6c4 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 16 Jul 2024 16:03:27 -0500 Subject: [PATCH 3/6] Add PSRT coordination process and messaging templates (#1348) Co-authored-by: Ezio Melotti Co-authored-by: Steve Dower Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Barry Warsaw --- developer-workflow/index.rst | 1 + developer-workflow/psrt.rst | 160 +++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 developer-workflow/psrt.rst diff --git a/developer-workflow/index.rst b/developer-workflow/index.rst index 7b069021b6..ca39a72756 100644 --- a/developer-workflow/index.rst +++ b/developer-workflow/index.rst @@ -14,3 +14,4 @@ Development workflow grammar porting sbom + psrt diff --git a/developer-workflow/psrt.rst b/developer-workflow/psrt.rst new file mode 100644 index 0000000000..9347a9a81f --- /dev/null +++ b/developer-workflow/psrt.rst @@ -0,0 +1,160 @@ +Python Security Response Team (PSRT) +==================================== + +The Python Security Response Team (PSRT) is responsible for handling +vulnerability reports for CPython and pip. + +Vulnerability report triage +--------------------------- + +Vulnerability reports are sent to one of two locations, +the long-standing ``security@python.org`` mailing list +or using the private vulnerability reporting feature +of GitHub Security Advisories (GHSA). + +For reports sent to ``security@python.org``, a PSRT admin +will triage the report and if the report seems plausible +(that is, not spam and for the correct project) will reply with +instructions on how to report the vulnerability on GitHub. + +If the reporter doesn't want to use GitHub's Security Advisories feature +then the PSRT admins can create a draft report on behalf of the reporter. + +Coordinating a vulnerability report +----------------------------------- + +Each report will have a member of the PSRT assigned as the "coordinator". +The coordinator will be responsible for following the below process and +will be publicly credited on vulnerability records post-publication. + +If a coordinator can't complete the process for any reason (time obligation, +vacation, etc.) they must find a replacement coordinator in the PSRT +and reassign the vulnerability report appropriately. + +Coordinators are expected to collaborate with other PSRT members and core developers +when needed for guidance on whether the report is an actual vulnerability, +severity, advisory text, and fixes. + +**The vulnerability coordination process is:** + +* Coordinator will determine whether the report constitutes a vulnerability. If the report isn't a vulnerability, + the reporter should be notified appropriately. Close the GHSA report, the report can be reopened if + sufficient evidence is later obtained that the report is a vulnerability. + +* After a vulnerability report is accepted, a Common Vulnerabilities and Exposures (CVE) ID must be assigned. If this is not done + automatically, then a CVE ID can be obtained by the coordinator sending an email to ``cna@python.org``. + No details about the vulnerability report need to be shared with the PSF CVE Numbering Authority (CNA) for a CVE ID to be reserved. + +* If the report is a vulnerability, the coordinator will determine the severity of the vulnerability. Severity is one of: + **Low**, **Medium**, **High**, and **Critical**. Coordinators can use their knowledge of the code, how the code is likely used, + or another mechanism like Common Vulnerability Scoring System (CVSS) for determining a severity. Add this information to the GitHub Security Advisory. + +* Once a CVE ID is assigned, the coordinator will share the acceptance and CVE ID with the reporter. + Use this CVE ID for referencing the vulnerability. The coordinator will ask the reporter + if the reporter would like to be credited publicly for the report and if so, how they would like to be credited. + Add this information to the GitHub Security Advisory. + +* The coordinator authors the vulnerability advisory text. The advisory must include the following information: + + * Title should be a brief description of the vulnerability and affected component + (for example, "Buffer over-read in SSLContext.set_npn_protocols()") + + * Short description of the vulnerability, impact, and the conditions where the affected component is vulnerable, if applicable. + + * Affected versions. This could be "all versions", but if the vulnerability exists in a new feature + or removed feature then this could be different. Include versions that are end-of-life in this calculation + (for example, "Python 3.9 and earlier", "Python 3.10 and later", "all versions of Python"). + + * Affected components and APIs. The module, function, class, or method must be specified so users can + search their codebase for usage. For issues affecting the entire project, this can be omitted. + + * Mitigations for the vulnerability beyond upgrading to a patched version, if applicable. + + This can all be done within the GitHub Security Advisory UI for easier collaboration between reporter and coordinator. + +* The coordinator determines the fix approach and who will provide a patch. + Some reporters are willing to provide or collaborate to create a patch, + otherwise relevant core developers can be invited to collaborate by + the coordinator. + + * For **Low** and **Medium** severity vulnerabilities it is acceptable + to develop a patch in public. + The pull request must be marked with the ``security`` and ``release-blocker`` + labels so that a release is not created without including the patch. + + * For **High** and **Critical** severity vulnerabilities the patch must be + developed privately using GitHub Security Advisories' "Private Forks" feature. + Core developers can be added to the GitHub Security Advisory via "collaborators" + to work on the fix together. Once a patch is approved privately and tested, + a public issue and pull request can be created with + the ``security`` and ``release-blocker`` labels. + +* Once the pull request is merged the advisory can be published. The coordinator will send the advisory by email + to ``security-announce@python.org`` using the below template. Backport labels must be added as appropriate. + After the advisory is published a CVE record can be created. + +Template responses +------------------ + +These template responses should be used as guidance for messaging +in various points in the process above. They are not required to be sent as-is, +please feel free to adapt them as needed for the current context. + +**Directing to GitHub Security Advisories:** + +.. highlight:: none + +:: + + Thanks for submitting this report. + We use GitHub Security Advisories for triaging vulnerability reports, + are you able to submit your report directly to GitHub? + + https://github.com/python/cpython/security/advisories/new + + If you're unable to submit a report to GitHub (due to not having a GitHub + account or something else) let me know and I will create a GitHub Security + Advisory on your behalf, although you won't be able to participate directly + in discussions. + +**Rejecting a vulnerability report:** + +:: + + Thanks for your report. We've determined that the report doesn't constitute + a vulnerability. Let us know if you disagree with this determination. + If you are interested in working on this further, you can optionally open a + public issue on GitHub. + +**Accepting a vulnerability report:** + +:: + + Thanks for your report. We've determined that the report + is a vulnerability. We've assigned {CVE-YYYY-XXXX} and determined + a severity of {Low,Medium,High,Critical}. Let us know if you disagree + with the determined severity. + + If you would like to be publicly credited for this vulnerability as the + reporter, please indicate that, along with how you would like to be + credited (name or organization). + + Please keep this vulnerability report private until we've published + an advisory to ``security-announce@python.org``. + +**Advisory email:** + +:: + + Title: [{CVE-YYYY-XXXX}] {title} + + There is a {LOW, MEDIUM, HIGH, CRITICAL} severity vulnerability + affecting {project}. + + {description} + + Please see the linked CVE ID for the latest information on + affected versions: + + * https://www.cve.org/CVERecord?id={CVE-YYYY-XXXX} + * {pull request URL} From 7bd3ae67a04a7b56ef2bc97a8b56507db44c676f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 18 Jul 2024 19:34:01 +0300 Subject: [PATCH 4/6] Add Ned Batchelder to core team (#1357) --- core-developers/developers.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/core-developers/developers.csv b/core-developers/developers.csv index a87f58df24..9a6a2ee3e2 100644 --- a/core-developers/developers.csv +++ b/core-developers/developers.csv @@ -1,3 +1,4 @@ +Ned Batchelder,nedbat,2024-07-16,, Michael Droetboom,mdboom,2024-06-06,, Tian Gao,gaogaotiantian,2024-06-06,, Russell Keith-Magee,freakboy3742,2024-05-30,, From c3950af5c406c548d292dab9d993e5a821dd6ffa Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 18 Jul 2024 13:02:24 -0400 Subject: [PATCH 5/6] Clarify 'time zone' in a Specific words section (#1352) * Clarify 'time zone' in a Specific words section * Use a reference that matches the section title * mention markup use for timezone --- documentation/style-guide.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/documentation/style-guide.rst b/documentation/style-guide.rst index af5c88860f..8ebcec6633 100644 --- a/documentation/style-guide.rst +++ b/documentation/style-guide.rst @@ -54,10 +54,15 @@ starting it with a lowercase letter should be avoided. Many special names are used in the Python documentation, including the names of operating systems, programming languages, standards bodies, and the like. Most of these entities are not assigned any special markup, but the preferred -spellings are given here to aid authors in maintaining the consistency of -presentation in the Python documentation. +spellings are given in :ref:`specific words` to aid authors in maintaining the +consistency of presentation in the Python documentation. -Other terms and words deserve special mention as well; these conventions should +.. _specific words: + +Specific words +============== + +Some terms and words deserve special mention. These conventions should be used to ensure consistency throughout the documentation: C API @@ -79,6 +84,12 @@ reST used to produce Python documentation. When spelled out, it is always one word and both forms start with a lowercase 'r'. +time zone + When referring to a Python term like a module, class, or argument spell it + as one word with appropriate markup (for example, ``:mod:`timezone```). + When talking about the real-world concept spell it as two words with no + markup. + Unicode The name of a character coding system. This is always written capitalized. From d2a63a9285fcd3dd53767446698273cf820dc7db Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 31 Jul 2024 16:36:30 +0100 Subject: [PATCH 6/6] gh-119786: remove part of devguide documentation which is duplicated in InternalDocs (#1334) --- internals/interpreter.rst | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/internals/interpreter.rst b/internals/interpreter.rst index d6feff544e..7405ff2461 100644 --- a/internals/interpreter.rst +++ b/internals/interpreter.rst @@ -178,38 +178,15 @@ Then the interpreter function (``_PyEval_EvalFrameDefault()``) returns ``NULL``. However, if an exception is raised in a ``try`` block, the interpreter must jump to the corresponding ``except`` or ``finally`` block. In 3.10 and before, there was a separate "block stack" which was used to keep track of nesting ``try`` blocks. -In 3.11, this mechanism has been replaced by a statically generated table, ``code->co_exceptiontable``. -The advantage of this approach is that entering and leaving a ``try`` block normally does not execute any code, making execution faster. -But of course, this table needs to be generated by the compiler, and decoded (by ``get_exception_handler``) when an exception happens. - -Exception table format ----------------------- - -The table is conceptually a list of records, each containing four variable-length integer fields (in a unique format, see below): - -- start: start of ``try`` block, in code units from the start of the bytecode -- length: size of the ``try`` block, in code units -- target: start of the first instruction of the ``except`` or ``finally`` block, in code units from the start of the bytecode -- depth_and_lasti: the low bit gives the "lasti" flag, the remaining bits give the stack depth - -The stack depth is used to clean up evaluation stack entries above this depth. -The "lasti" flag indicates whether, after stack cleanup, the instruction offset of the raising instruction should be pushed (as a ``PyLongObject *``). -For more information on the design, see :cpy-file:`Objects/exception_handling_notes.txt`. - -Each varint is encoded as one or more bytes. -The high bit (bit 7) is reserved for random access -- it is set for the first varint of a record. -The second bit (bit 6) indicates whether this is the last byte or not -- it is set for all but the last bytes of a varint. -The low 6 bits (bits 0-5) are used for the integer value, in big-endian order. - -To find the table entry (if any) for a given instruction offset, we can use bisection without decoding the whole table. -We bisect the raw bytes, at each probe finding the start of the record by scanning back for a byte with the high bit set, and then decode the first varint. -See ``get_exception_handler()`` in :cpy-file:`Python/ceval.c` for the exact code (like all bisection algorithms, the code is a bit subtle). +In 3.11, this mechanism has been replaced by a statically generated table, ``code->co_exceptiontable``, +which is described in detail in the `internals documentation +`_. The locations table ------------------- Whenever an exception is raised, we add a traceback entry to the exception. -The ``tb_lineno`` field of a traceback entry must be set to the line number of the instruction that raised it. +The ``tb_lineno`` field of a traceback entry is (lazily) set to the line number of the instruction that raised it. This field is computed from the locations table, ``co_linetable`` (this name is an understatement), using :c:func:`PyCode_Addr2Line`. This table has an entry for every instruction rather than for every ``try`` block, so a compact format is very important.