From 56385108375534b81205ceefc53ec2a0093188be Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Wed, 11 Feb 2026 15:57:49 +0000 Subject: [PATCH 01/86] fetch2/svn: Use server certificates going forward In the past, broken SSL certificates were common on subversion servers. As such, the subversion fetcher used to ignore these issues. Cert infrastructure has massively improved since that decision was made and things like self signed certificates should no longer be common place. We should follow good security practises and not have this as a default anymore, remove the --trust-server-cert commandline option by default. Signed-off-by: Richard Purdie Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- lib/bb/fetch2/svn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bb/fetch2/svn.py b/lib/bb/fetch2/svn.py index 0852108e7d9..a097ffb76b5 100644 --- a/lib/bb/fetch2/svn.py +++ b/lib/bb/fetch2/svn.py @@ -34,7 +34,7 @@ def urldata_init(self, ud, d): if not "module" in ud.parm: raise MissingParameterError('module', ud.url) - ud.basecmd = d.getVar("FETCHCMD_svn") or "/usr/bin/env svn --non-interactive --trust-server-cert" + ud.basecmd = d.getVar("FETCHCMD_svn") or "/usr/bin/env svn --non-interactive" ud.module = ud.parm["module"] From 171aa68fcffa042a03fd75aadf86f97e800f0e94 Mon Sep 17 00:00:00 2001 From: Jose Quaresma Date: Thu, 12 Feb 2026 17:23:28 +0000 Subject: [PATCH 02/86] utils: is_path_on_nfs: strip not existing path When the user provide a path that does not exist it fails with: | stat: cannot read file system information for '/path/to/my/hashserve': No such file or directory We can strip the non exising part in the path and pass just that to the stat. Signed-off-by: Jose Quaresma Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- lib/bb/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/bb/utils.py b/lib/bb/utils.py index 974611bc7e0..b2bda08cb7d 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -2269,6 +2269,11 @@ def is_path_on_nfs(path): """ Returns True if ``path`` argument is on a NFS mount. """ + # strip not existing path + if os.path.isabs(path): + while not os.path.exists(path): + path = os.path.dirname(path) + import bb.process fstype = bb.process.run("stat -f -c %T {}".format(path))[0].strip() return fstype == "nfs" From c5311e24c6f3f0a3b02934520bc1e431455371e1 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Tue, 17 Feb 2026 14:36:27 +0100 Subject: [PATCH 03/86] doc: bitbake-user-manual-ref-variables: clarify BBMASK directory matching BBMASK matches files in the directories matching the regex, and also in their respective subdirectories, so make that clear in the wording leading to the example. Signed-off-by: Quentin Schulz Reviewed-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index d523880dcb8..6c21ddf7bd3 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -1051,8 +1051,8 @@ overview of their function and contents. documentation at http://docs.python.org/3/library/re.html. The following example uses a complete regular expression to tell - BitBake to ignore all recipe and recipe append files in the - ``meta-ti/recipes-misc/`` directory:: + BitBake to ignore all recipe and recipe append files in + ``meta-ti/recipes-misc/`` directories (and their subdirectories):: BBMASK = "meta-ti/recipes-misc/" From 3831d4813d70dc15d0a987bcb86762eab56fc2b0 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Tue, 17 Feb 2026 14:36:28 +0100 Subject: [PATCH 04/86] doc: bitbake-user-manual-ref-variables: have BBMASK directory examples be consistent It's a bit confusing to have a mix of paths with and without a leading slash. The behavior is actually different. The next commit will explain when/why to add a leading slash. Reported-by: Robert P. J. Day Signed-off-by: Quentin Schulz Reviewed-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index 6c21ddf7bd3..aee677657a8 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -1054,13 +1054,13 @@ overview of their function and contents. BitBake to ignore all recipe and recipe append files in ``meta-ti/recipes-misc/`` directories (and their subdirectories):: - BBMASK = "meta-ti/recipes-misc/" + BBMASK = "/meta-ti/recipes-misc/" If you want to mask out multiple directories or recipes, you can specify multiple regular expression fragments. This next example masks out multiple directories and individual recipes:: - BBMASK += "/meta-ti/recipes-misc/ meta-ti/recipes-ti/packagegroup/" + BBMASK += "/meta-ti/recipes-ti/packagegroup/" BBMASK += "/meta-oe/recipes-support/" BBMASK += "/meta-foo/.*/openldap" BBMASK += "opencv.*\.bbappend" From ee3b2e0b7fd675af0da3ad9e8c0c25d6a20d9e66 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Tue, 17 Feb 2026 14:36:29 +0100 Subject: [PATCH 05/86] doc: bitbake-user-manual-ref-variables: expand and reorganize BBMASK explanations The documentation doesn't explain the side-effect of putting a leading slash, only the trailing slash. The leading slash is not for making the regular expression match an absolute path, but to force the match on the directory or file that exactly starts (and ends if there is a trailing slash) with the specified string. So let's explain that. This also explains that this doesn't prevent more than the intended path to be caught, specifically because it is NOT a regular expression matching an absolute path. Because any pattern not starting with a ^ character can match multiple directories from multiple layers, let's make the usage of BBFILE_PATTERN_my-layer the recommended one. Keep the rest as hints at what can happen when not using the variable but reiterate that users should be really be using that variable. Reported-by: Robert P. J. Day Signed-off-by: Quentin Schulz Reviewed-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- .../bitbake-user-manual-ref-variables.rst | 91 +++++++++++++++++-- 1 file changed, 82 insertions(+), 9 deletions(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index aee677657a8..aeef640a48b 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -1052,24 +1052,97 @@ overview of their function and contents. The following example uses a complete regular expression to tell BitBake to ignore all recipe and recipe append files in - ``meta-ti/recipes-misc/`` directories (and their subdirectories):: + ``recipes-bsp`` directory (recursively) of ``meta-ti``:: - BBMASK = "/meta-ti/recipes-misc/" + BBMASK = "${BBFILE_PATTERN_meta-ti}/recipes-bsp/" If you want to mask out multiple directories or recipes, you can specify multiple regular expression fragments. This next example masks out multiple directories and individual recipes:: - BBMASK += "/meta-ti/recipes-ti/packagegroup/" - BBMASK += "/meta-oe/recipes-support/" - BBMASK += "/meta-foo/.*/openldap" - BBMASK += "opencv.*\.bbappend" - BBMASK += "lzma" + BBMASK += "${BBFILE_PATTERN_meta-ti}/recipes-graphics/libgal/" + BBMASK += "${BBFILE_PATTERN_openembedded-layer}/recipes-support/" + BBMASK += "${BBFILE_PATTERN_openembedded-layer}/.*/openldap" + BBMASK += "${BBFILE_PATTERN_meta-ti}/.*/optee.*\.bbappend" + + This masks: + + - everything under the ``recipes-graphics/libgal/`` directory from + ``meta-ti``, + - everything under the ``recipes-support/`` directory in ``meta-oe``, + - everything under a directory whose name starts with ``openldap``, and + every file with the same naming scheme, in ``meta-oe`` at any directory + depth > 1 (e.g. in ``meta-oe``, ``recipes-foo/openldap-stuff/`` or + ``recipes-bar/baz/openldap_0.1.bb`` but not ``openldap/``), + - every append file whose name starts with ``optee`` in ``meta-ti`` at any + directory depth > 1 (e.g. ``optee/optee-examples_%.bbappend`` and + ``recipes-security/optee/optee-client_%.bbappend``). + + .. note:: + + Because these are complete regular expressions, if you want to match a + directory and not a file, you must end the expression with a trailing + slash. That is:: + + BBMASK += "${BBFILE_PATTERN_meta-ti}/recipes-graphics/libgal/" + + Will match anything under ``recipes-graphics/ligbal/`` directory of + ``meta-ti``. And:: + + BBMASK += "${BBFILE_PATTERN_meta-ti}/recipes-graphics/libgal" + + Will match in ``meta-ti`` any file prefixed with ``libgal`` in + ``recipes-graphics/`` and any directory (recursively; and its + recipes and recipe append files regardless how they are named) prefixed + with ``libgal`` in ``recipes-graphics/``. That is, provided your layers + are available at ``/bitbake-builds/poky-master/layers/``, it'll match:: + + /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal.bb + /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal_%.bbappend + /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal-foo/foo.bb + /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal-foo/foo/bz.bbappend + /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal/bar.bb .. note:: - When specifying a directory name, use the trailing slash character - to ensure you match just that directory name. + Because these are complete regular expressions, failing to start the + pattern with a ``^`` sign (which is usually the first character in + :term:`BBFILE_PATTERN`) means it can match *any* portion of a path. + Take the following as an example:: + + BBMASK = "recipes-graphics/libgal/" + + This will match subdirectories and files in any path containing + ``recipes-graphics/libgal/``, meaning (considering your layers are + available at ``/bitbake-builds/poky-master/layers/``):: + + /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal/ + /bitbake-builds/poky-master/layers/my-layer/foo-recipes-graphics/libgal/ + + will be both matched. This may be a more relaxed way of matching + directories, recipes and recipe append files from any third party layer + but is generally discouraged as it may be casting too wide of a net. + + .. note:: + + Because these are complete regular expressions, a leading slash does + not mean the path is absolute. It simply forces the directory to be + named exactly that. Take:: + + BBMASK = "recipes-graphics/libgal/" + + If you happen to have a directory ``foo-recipes-graphics/libgal/``, it + will be matched. + + Leading with a slash:: + + BBMASK = "/recipes-graphics/libgal/" + + makes sure that doesn't happen. However, this doesn't prevent matching + a directory ``recipes-graphics/libgal/`` from another layer. + + Again, we highly recommend using :term:`BBFILE_PATTERN` in the + expressions to specify absolute paths specific to one layer. :term:`BBMULTICONFIG` Enables BitBake to perform multiple configuration builds and lists From 4c2a5d4e1abf06e14a9148f5c22390c865f7db20 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Tue, 17 Feb 2026 14:36:30 +0100 Subject: [PATCH 06/86] doc: bitbake-user-manual-ref-variables: update BBMASK example with current meta-ti meta-ti git repo now has multiple layers in it since kirkstone, so let's update the examples to match the current state of meta-ti by using meta-ti-bsp. Signed-off-by: Quentin Schulz Reviewed-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- .../bitbake-user-manual-ref-variables.rst | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index aeef640a48b..d1ee7f99b49 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -1052,30 +1052,30 @@ overview of their function and contents. The following example uses a complete regular expression to tell BitBake to ignore all recipe and recipe append files in - ``recipes-bsp`` directory (recursively) of ``meta-ti``:: + ``recipes-bsp`` directory (recursively) of ``meta-ti-bsp``:: - BBMASK = "${BBFILE_PATTERN_meta-ti}/recipes-bsp/" + BBMASK = "${BBFILE_PATTERN_meta-ti-bsp}/recipes-bsp/" If you want to mask out multiple directories or recipes, you can specify multiple regular expression fragments. This next example masks out multiple directories and individual recipes:: - BBMASK += "${BBFILE_PATTERN_meta-ti}/recipes-graphics/libgal/" + BBMASK += "${BBFILE_PATTERN_meta-ti-bsp}/recipes-graphics/libgal/" BBMASK += "${BBFILE_PATTERN_openembedded-layer}/recipes-support/" BBMASK += "${BBFILE_PATTERN_openembedded-layer}/.*/openldap" - BBMASK += "${BBFILE_PATTERN_meta-ti}/.*/optee.*\.bbappend" + BBMASK += "${BBFILE_PATTERN_meta-ti-bsp}/.*/optee.*\.bbappend" This masks: - everything under the ``recipes-graphics/libgal/`` directory from - ``meta-ti``, + ``meta-ti-bsp``, - everything under the ``recipes-support/`` directory in ``meta-oe``, - everything under a directory whose name starts with ``openldap``, and every file with the same naming scheme, in ``meta-oe`` at any directory depth > 1 (e.g. in ``meta-oe``, ``recipes-foo/openldap-stuff/`` or ``recipes-bar/baz/openldap_0.1.bb`` but not ``openldap/``), - - every append file whose name starts with ``optee`` in ``meta-ti`` at any - directory depth > 1 (e.g. ``optee/optee-examples_%.bbappend`` and + - every append file whose name starts with ``optee`` in ``meta-ti-bsp`` at + any directory depth > 1 (e.g. ``optee/optee-examples_%.bbappend`` and ``recipes-security/optee/optee-client_%.bbappend``). .. note:: @@ -1084,24 +1084,24 @@ overview of their function and contents. directory and not a file, you must end the expression with a trailing slash. That is:: - BBMASK += "${BBFILE_PATTERN_meta-ti}/recipes-graphics/libgal/" + BBMASK += "${BBFILE_PATTERN_meta-ti-bsp}/recipes-graphics/libgal/" Will match anything under ``recipes-graphics/ligbal/`` directory of - ``meta-ti``. And:: + ``meta-ti-bsp``. And:: - BBMASK += "${BBFILE_PATTERN_meta-ti}/recipes-graphics/libgal" + BBMASK += "${BBFILE_PATTERN_meta-ti-bsp}/recipes-graphics/libgal" - Will match in ``meta-ti`` any file prefixed with ``libgal`` in + Will match in ``meta-ti-bsp`` any file prefixed with ``libgal`` in ``recipes-graphics/`` and any directory (recursively; and its recipes and recipe append files regardless how they are named) prefixed with ``libgal`` in ``recipes-graphics/``. That is, provided your layers are available at ``/bitbake-builds/poky-master/layers/``, it'll match:: - /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal.bb - /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal_%.bbappend - /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal-foo/foo.bb - /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal-foo/foo/bz.bbappend - /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal/bar.bb + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal.bb + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal_%.bbappend + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal-foo/foo.bb + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal-foo/foo/bz.bbappend + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal/bar.bb .. note:: @@ -1116,7 +1116,7 @@ overview of their function and contents. ``recipes-graphics/libgal/``, meaning (considering your layers are available at ``/bitbake-builds/poky-master/layers/``):: - /bitbake-builds/poky-master/layers/meta-ti/recipes-graphics/libgal/ + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal/ /bitbake-builds/poky-master/layers/my-layer/foo-recipes-graphics/libgal/ will be both matched. This may be a more relaxed way of matching From 47e4a24cb14ccc0b980076f12e407d62d82fcfe2 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Tue, 17 Feb 2026 14:36:31 +0100 Subject: [PATCH 07/86] doc: bitbake-user-manual-ref-variables: update Python re doc link to HTTPS It's 2026, use HTTPS for the link. Signed-off-by: Quentin Schulz Reviewed-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index d1ee7f99b49..3a01144ff49 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -1048,7 +1048,7 @@ overview of their function and contents. compiler. Consequently, the syntax follows Python's Regular Expression (re) syntax. The expressions are compared against the full paths to the files. For complete syntax information, see Python's - documentation at http://docs.python.org/3/library/re.html. + documentation at https://docs.python.org/3/library/re.html. The following example uses a complete regular expression to tell BitBake to ignore all recipe and recipe append files in From 787053384b4fd0e685b18397a1244c9e91125362 Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Wed, 18 Feb 2026 14:15:26 +0100 Subject: [PATCH 08/86] bitbake-setup: define a color_enabled() helper function This will be re-used in the next commits to force subprocess commands to color their outputs, as passing --color=auto in them always render without coloring. Signed-off-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- bin/bitbake-setup | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 927c03a1033..3e17e631bfb 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -35,6 +35,13 @@ GLOBAL_ONLY_SETTINGS = ( "top-dir-name", ) +def color_enabled() -> bool: + """ + Our logger has a BBLogFormatter formatter which holds whether color is + enabled or not. Return this value. + """ + return logger.handlers[0].formatter.color_enabled + # If bitbake is from a release tarball or somewhere like pypi where # updates may not be straightforward, prefer to use the git repo as the # default registry From d9700632bd6b627d1124fdc83ddf7bfb4199228d Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Wed, 18 Feb 2026 14:15:27 +0100 Subject: [PATCH 09/86] bitbake-setup: improve readability of choices Bold the configuration names and align the descriptions for improved readability. For this, define a print_configs() function which should also help factorizing the code as this is done in multiple places. If color_enabled() is false, keep the text plain. Signed-off-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- bin/bitbake-setup | 74 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 3e17e631bfb..9b0ecc8d0e4 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -42,6 +42,33 @@ def color_enabled() -> bool: """ return logger.handlers[0].formatter.color_enabled +def print_configs(prompt: str, choices: list[str], descriptions: list[str] = []): + """ + Helper function to print a list of choices and align the output. + Each option name is made bold to stand out, unless color is not enabled in + our logger. + """ + if not prompt.endswith(':'): + prompt += ":" + logger.plain(prompt) + + if not descriptions: + descriptions = ["" for _ in choices] + + # maximum size of all choices, for alignment + cmax = max([len(c) for c in choices]) + 1 + + for n, c in enumerate(choices): + msg = f"{n + 1}. " + if color_enabled(): + # make it bold + msg += "\033[1m" + msg += f"{c:<{cmax}}" + if color_enabled(): + msg += "\033[0m" + msg += f" {descriptions[n]}" + logger.plain(msg) + # If bitbake is from a release tarball or somewhere like pypi where # updates may not be straightforward, prefer to use the git repo as the # default registry @@ -497,36 +524,39 @@ def choose_bitbake_config(configs, parameters, non_interactive): if non_interactive: raise Exception("Unable to choose from bitbake configurations in non-interactive mode: {}".format(configs_dict)) - logger.plain("\nAvailable bitbake configurations:") - for n, config_data in enumerated_configs: - logger.plain("{}. {}\t{}".format(n, config_data["name"], config_data["description"])) + logger.plain("") + print_configs("Available bitbake configurations", + [c["name"] for c in flattened_configs], + [c["description"] for c in flattened_configs]) config_n = int_input([i[0] for i in enumerated_configs], "\nPlease select one of the above bitbake configurations by its number: ") - 1 return flattened_configs[config_n] def choose_config(configs, non_interactive): not_expired_configs = [k for k in sorted(configs.keys()) if not has_expired(configs[k].get("expires", None))] - config_list = list(enumerate(not_expired_configs, 1)) - if len(config_list) == 1: - only_config = config_list[0][1] + if len(not_expired_configs) == 1: + only_config = not_expired_configs[0] logger.plain("\nSelecting the only available configuration {}\n".format(only_config)) return only_config if non_interactive: raise Exception("Unable to choose from configurations in non-interactive mode: {}".format(not_expired_configs)) - logger.plain("\nAvailable Configuration Templates:") - for n, config_name in config_list: - config_data = configs[config_name] - expiry_date = config_data.get("expires", None) - config_desc = config_data["description"] + descs = [] + for c in not_expired_configs: + d = configs[c]["description"] + expiry_date = configs[c].get("expires", None) if expiry_date: - logger.plain("{}. {}\t{} (supported until {})".format(n, config_name, config_desc, expiry_date)) - else: - logger.plain("{}. {}\t{}".format(n, config_name, config_desc)) - config_n = int_input([i[0] for i in config_list], - "\nPlease select one of the above Configuration Templates by its number: ") - 1 - return config_list[config_n][1] + d += f" (supported until {expiry_date})" + descs.append(d) + + logger.plain("") + print_configs("Available Configuration Templates", + [c for c in not_expired_configs], + descs) + config_n = int_input([i[0] for i in list(enumerate(not_expired_configs, 1))], + "\nPlease select one of the above configurations by its number: ") - 1 + return not_expired_configs[config_n] def choose_fragments(possibilities, parameters, non_interactive, skip_selection): choices = {} @@ -552,13 +582,11 @@ def choose_fragments(possibilities, parameters, non_interactive, skip_selection) if non_interactive: raise Exception(f"Unable to choose from options in non-interactive mode: {[o['name'] for o in options]}") - logger.plain("\n" + v["description"] + ":") + logger.plain("") + print_configs(v["description"], + [o['name'] for o in options], + [o['description'] for o in options]) options_enumerated = list(enumerate(options, 1)) - for n,o in options_enumerated: - opt_str = f"{n}. {o['name']}" - if o["description"]: - opt_str += f"\t{o['description']}" - logger.plain(opt_str) option_n = int_input([i[0] for i in options_enumerated], "\nPlease select one of the above options by its number: ") - 1 choices[k] = options_enumerated[option_n][1]["name"] From 1d0d4a0066603461eacb92a766fec616c1e91257 Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Thu, 5 Feb 2026 16:25:45 +0100 Subject: [PATCH 10/86] doc: fix the switchers menu Fix the switchers.js script of Bitbake to show what are the supported versions and be able to switch between them. The default landing page is the stable branch and shows the BitBake version along with the corresponding Yocto Project codename. This hopefully makes it easier to remember the correspondance between the BitBake version and the Yocto Project version. This works thanks to a setversions.py script, imported and executed in conf.py, which is largely inspired from the one in yocto-docs. It reads the tags from the repository and tries to guess the currently checked out version of BitBake on which we are. The "obsolete" warning is now also shown when browsing outdated manuals, meaning any version not part of activereleases in setversions.py and "dev"/"next". Signed-off-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- doc/.gitignore | 1 + doc/Makefile | 2 +- doc/conf.py | 20 +-- doc/setversions.py | 132 ++++++++++++++++ doc/sphinx-static/switchers.js | 233 ---------------------------- doc/sphinx-static/switchers.js.in | 249 ++++++++++++++++++++++++++++++ 6 files changed, 393 insertions(+), 244 deletions(-) create mode 100755 doc/setversions.py delete mode 100644 doc/sphinx-static/switchers.js create mode 100644 doc/sphinx-static/switchers.js.in diff --git a/doc/.gitignore b/doc/.gitignore index 69fa449dd96..dee9494dcaf 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1 +1,2 @@ _build/ +sphinx-static/switchers.js diff --git a/doc/Makefile b/doc/Makefile index 996f01b7d5c..5e1632314c5 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -27,7 +27,7 @@ publish: Makefile html singlehtml sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html clean: - @rm -rf $(BUILDDIR) + @rm -rf $(BUILDDIR) sphinx-static/switchers.js # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/doc/conf.py b/doc/conf.py index bce386624e2..9318358731c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -6,20 +6,16 @@ # -- Path setup -------------------------------------------------------------- -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - -import sys import datetime +import os +import sys from pathlib import Path -current_version = "dev" +sys.path.insert(0, os.path.abspath('.')) +import setversions + +current_version = setversions.get_current_version() # String used in sidebar version = 'Version: ' + current_version @@ -28,6 +24,10 @@ # Version seen in documentation_options.js and hence in js switchers code release = current_version +setversions.write_switchers_js("sphinx-static/switchers.js.in", + "sphinx-static/switchers.js", + current_version) + # -- Project information ----------------------------------------------------- project = 'Bitbake' diff --git a/doc/setversions.py b/doc/setversions.py new file mode 100755 index 00000000000..9e4138025ac --- /dev/null +++ b/doc/setversions.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# +# This file defines is used in doc/conf.py to setup the version information for +# the documentation: +# - get_current_version() used in doc/conf.py computes the current version by +# trying to guess the approximate versions we're at using git tags and +# branches from the repository. +# - write_switchers_js() write the switchers.js file used for switching between +# versions of the documentation. +# +# Copyright (c) 2026 Antonin Godard +# +# SPDX-License-Identifier: MIT +# + +import itertools +import re +import subprocess +import sys + +DEVBRANCH = "2.18" +LTSSERIES = ["2.8", "2.0"] +ACTIVERELEASES = ["2.16"] + LTSSERIES + +YOCTO_MAPPING = { + "2.18": "wrynose", + "2.16": "whinlatter", + "2.12": "walnascar", + "2.10": "styhead", + "2.8": "scarthgap", + "2.6": "nanbield", + "2.4": "mickledore", + "2.2": "langdale", + "2.0": "kirkstone", + "1.52": "honister", + "1.50": "hardknott", + "1.48": "gatesgarth", + "1.46": "dunfell", +} + +BB_RELEASE_TAG_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") + +def get_current_version(): + ourversion = None + + # Test that we are building from a Git repository + try: + subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + except subprocess.CalledProcessError: + sys.exit("Building bitbake's documentation must be done from its Git repository.\n" + "Clone the repository with the following command:\n" + "git clone https://git.openembedded.org/bitbake ") + + # Test tags exist and inform the user to fetch if not + try: + subprocess.run(["git", "show", f"{LTSSERIES[0]}.0"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + except subprocess.CalledProcessError: + sys.exit("Please run 'git fetch --tags' before building the documentation") + + + # Try and figure out what we are + tags = subprocess.run(["git", "tag", "--points-at", "HEAD"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout + for t in tags.split(): + if re.match(BB_RELEASE_TAG_RE, t): + ourversion = t + break + + if ourversion: + # We're a tagged release + components = ourversion.split(".") + else: + # We're floating on a branch + branch = subprocess.run(["git", "branch", "--show-current"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout.strip() + + if branch == "" or branch not in list(YOCTO_MAPPING.keys()) + ["master", "master-next"]: + # We're not on a known release branch so we have to guess. Compare the + # numbers of commits from each release branch and assume the smallest + # number of commits is the one we're based off + possible_branch = None + branch_count = 0 + for b in itertools.chain(YOCTO_MAPPING.keys(), ["master"]): + result = subprocess.run(["git", "log", "--format=oneline", "HEAD..origin/" + b], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if result.returncode == 0: + count = result.stdout.count('\n') + if not possible_branch or count < branch_count: + print("Branch %s has count %s" % (b, count)) + possible_branch = b + branch_count = count + if possible_branch: + branch = possible_branch + else: + branch = "master" + print("Nearest release branch estimated to be %s" % branch) + + if branch == "master": + ourversion = "dev" + elif branch == "master-next": + ourversion = "next" + else: + ourversion = branch + head_commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout.strip() + branch_commit = subprocess.run(["git", "rev-parse", "--short", branch], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout.strip() + if head_commit != branch_commit: + ourversion += f" ({head_commit})" + + print("Version calculated to be %s" % ourversion) + return ourversion + +def write_switchers_js(js_in, js_out, current_version): + with open(js_in, "r") as r, open(js_out, "w") as w: + lines = r.readlines() + for line in lines: + if "VERSIONS_PLACEHOLDER" in line: + if current_version != "dev": + w.write(f" 'dev': 'Unstable (dev)',\n") + for series in ACTIVERELEASES: + w.write(f" '{series}': '{series} ({YOCTO_MAPPING[series]})',\n") + else: + w.write(line) + print("switchers.js generated from switchers.js.in") diff --git a/doc/sphinx-static/switchers.js b/doc/sphinx-static/switchers.js deleted file mode 100644 index 32113cfa960..00000000000 --- a/doc/sphinx-static/switchers.js +++ /dev/null @@ -1,233 +0,0 @@ -(function() { - 'use strict'; - - var all_versions = { - 'dev': 'dev (3.2)', - '3.1.2': '3.1.2', - '3.0.3': '3.0.3', - '2.7.4': '2.7.4', - }; - - var all_doctypes = { - 'single': 'Individual Webpages', - 'mega': "All-in-one 'Mega' Manual", - }; - - // Simple version comparision - // Return 1 if a > b - // Return -1 if a < b - // Return 0 if a == b - function ver_compare(a, b) { - if (a == "dev") { - return 1; - } - - if (a === b) { - return 0; - } - - var a_components = a.split("."); - var b_components = b.split("."); - - var len = Math.min(a_components.length, b_components.length); - - // loop while the components are equal - for (var i = 0; i < len; i++) { - // A bigger than B - if (parseInt(a_components[i]) > parseInt(b_components[i])) { - return 1; - } - - // B bigger than A - if (parseInt(a_components[i]) < parseInt(b_components[i])) { - return -1; - } - } - - // If one's a prefix of the other, the longer one is greater. - if (a_components.length > b_components.length) { - return 1; - } - - if (a_components.length < b_components.length) { - return -1; - } - - // Otherwise they are the same. - return 0; - } - - function build_version_select(current_series, current_version) { - var buf = [''); - return buf.join(''); - } - - function build_doctype_select(current_doctype) { - var buf = [''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - var url = urls.shift(); - - // Web browsers won't redirect file:// urls to file urls using ajax but - // its useful for local testing - if (url.startsWith("file://")) { - window.location.href = url; - return; - } - - if (urls.length == 0) { - window.location.href = url; - return; - } - $.ajax({ - url: url, - success: function() { - window.location.href = url; - }, - error: function() { - navigate_to_first_existing(urls); - } - }); - } - - function get_docroot_url() { - var url = window.location.href; - var root = DOCUMENTATION_OPTIONS.URL_ROOT; - - var urlarray = url.split('/'); - // Trim off anything after '/' - urlarray.pop(); - var depth = (root.match(/\.\.\//g) || []).length; - for (var i = 0; i < depth; i++) { - urlarray.pop(); - } - - return urlarray.join('/') + '/'; - } - - function on_version_switch() { - var selected_version = $(this).children('option:selected').attr('value'); - var url = window.location.href; - var current_version = DOCUMENTATION_OPTIONS.VERSION; - var docroot = get_docroot_url() - - var new_versionpath = selected_version + '/'; - if (selected_version == "dev") - new_versionpath = ''; - - // dev versions have no version prefix - if (current_version == "dev") { - var new_url = docroot + new_versionpath + url.replace(docroot, ""); - var fallback_url = docroot + new_versionpath; - } else { - var new_url = url.replace('/' + current_version + '/', '/' + new_versionpath); - var fallback_url = new_url.replace(url.replace(docroot, ""), ""); - } - - console.log(get_docroot_url()) - console.log(url + " to url " + new_url); - console.log(url + " to fallback " + fallback_url); - - if (new_url != url) { - navigate_to_first_existing([ - new_url, - fallback_url, - 'https://www.yoctoproject.org/docs/', - ]); - } - } - - function on_doctype_switch() { - var selected_doctype = $(this).children('option:selected').attr('value'); - var url = window.location.href; - if (selected_doctype == 'mega') { - var docroot = get_docroot_url() - var current_version = DOCUMENTATION_OPTIONS.VERSION; - // Assume manuals before 3.2 are using old docbook mega-manual - if (ver_compare(current_version, "3.2") < 0) { - var new_url = docroot + "mega-manual/mega-manual.html"; - } else { - var new_url = docroot + "singleindex.html"; - } - } else { - var new_url = url.replace("singleindex.html", "index.html") - } - - if (new_url != url) { - navigate_to_first_existing([ - new_url, - 'https://www.yoctoproject.org/docs/', - ]); - } - } - - // Returns the current doctype based upon the url - function doctype_segment_from_url(url) { - if (url.includes("singleindex") || url.includes("mega-manual")) - return "mega"; - return "single"; - } - - $(document).ready(function() { - var release = DOCUMENTATION_OPTIONS.VERSION; - var current_doctype = doctype_segment_from_url(window.location.href); - var current_series = release.substr(0, 3); - var version_select = build_version_select(current_series, release); - - $('.version_switcher_placeholder').html(version_select); - $('.version_switcher_placeholder select').bind('change', on_version_switch); - - var doctype_select = build_doctype_select(current_doctype); - - $('.doctype_switcher_placeholder').html(doctype_select); - $('.doctype_switcher_placeholder select').bind('change', on_doctype_switch); - - if (ver_compare(release, "3.1") < 0) { - $('#outdated-warning').html('Version ' + release + ' of the project is now considered obsolete, please select and use a more recent version'); - $('#outdated-warning').css('padding', '.5em'); - } else if (release != "dev") { - $.each(all_versions, function(version, title) { - var series = version.substr(0, 3); - if (series == current_series && version != release) { - $('#outdated-warning').html('This document is for outdated version ' + release + ', you should select the latest release version in this series, ' + version + '.'); - $('#outdated-warning').css('padding', '.5em'); - } - }); - } - }); -})(); diff --git a/doc/sphinx-static/switchers.js.in b/doc/sphinx-static/switchers.js.in new file mode 100644 index 00000000000..0b209e959da --- /dev/null +++ b/doc/sphinx-static/switchers.js.in @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: MIT +(function () { + "use strict"; + + var all_versions = { + VERSIONS_PLACEHOLDER, + }; + + var all_doctypes = { + single: "Individual Webpages", + mega: "All-in-one 'Mega' Manual", + }; + + // Simple version comparision + // Return 1 if a > b + // Return -1 if a < b + // Return 0 if a == b + function ver_compare(a, b) { + if (a == "dev") { + return 1; + } + + if (a === b) { + return 0; + } + + var a_components = a.split("."); + var b_components = b.split("."); + + var len = Math.min(a_components.length, b_components.length); + + // loop while the components are equal + for (var i = 0; i < len; i++) { + // A bigger than B + if (parseInt(a_components[i]) > parseInt(b_components[i])) { + return 1; + } + + // B bigger than A + if (parseInt(a_components[i]) < parseInt(b_components[i])) { + return -1; + } + } + + // If one's a prefix of the other, the longer one is greater. + if (a_components.length > b_components.length) { + return 1; + } + + if (a_components.length < b_components.length) { + return -1; + } + + // Otherwise they are the same. + return 0; + } + + function build_version_select(current_version) { + var buf = [""); + return buf.join(""); + } + + function build_doctype_select(current_doctype) { + var buf = [""); + return buf.join(""); + } + + function navigate_to_first_existing(urls) { + // Navigate to the first existing URL in urls. + var url = urls.shift(); + + // Web browsers won't redirect file:// urls to file urls using ajax but + // its useful for local testing + if (url.startsWith("file://")) { + window.location.href = url; + return; + } + + if (urls.length == 0) { + window.location.href = url; + return; + } + $.ajax({ + url: url, + success: function () { + window.location.href = url; + }, + error: function () { + navigate_to_first_existing(urls); + }, + }); + } + + function get_docroot_url() { + var url = window.location.href; + // Try to get the variable from documentation_options.js + var root = DOCUMENTATION_OPTIONS.URL_ROOT; + if (root == null) { + // In recent versions of Sphinx, URL_ROOT was removed from + // documentation_options.js, so get it like searchtools.js does. + root = document.documentElement.dataset.content_root; + } + + var urlarray = url.split("/"); + // Trim off anything after '/' + urlarray.pop(); + var depth = (root.match(/\.\.\//g) || []).length; + for (var i = 0; i < depth; i++) { + urlarray.pop(); + } + + return urlarray.join("/") + "/"; + } + + function on_version_switch() { + var selected_version = $(this).children("option:selected").attr("value"); + var url = window.location.href; + var current_version = DOCUMENTATION_OPTIONS.VERSION; + var docroot = get_docroot_url(); + + var new_versionpath = selected_version + "/"; + + // latest tag is also the default page (without version information) + if (docroot.endsWith("dev/")) { + var new_url = url.replace("/dev/", "/" + new_versionpath); + var fallback_url = new_url.replace(url.replace(docroot, ""), ""); + } else if (docroot.endsWith(current_version + "/") == false) { + var new_url = docroot + new_versionpath + url.replace(docroot, ""); + var fallback_url = docroot + new_versionpath; + } else { + var new_url = url.replace( + "/" + current_version + "/", + "/" + new_versionpath, + ); + var fallback_url = new_url.replace(url.replace(docroot, ""), ""); + } + + console.log(url + " to url " + new_url); + console.log(url + " to fallback " + fallback_url); + + if (new_url != url) { + navigate_to_first_existing([ + new_url, + fallback_url, + "https://www.yoctoproject.org/bitbake/", + ]); + } + } + + function on_doctype_switch() { + var selected_doctype = $(this).children("option:selected").attr("value"); + var url = window.location.href; + if (selected_doctype == "mega") { + var docroot = get_docroot_url(); + var current_version = DOCUMENTATION_OPTIONS.VERSION; + var new_url = docroot + "singleindex.html"; + } else { + var new_url = url.replace("singleindex.html", "index.html"); + } + + if (new_url != url) { + navigate_to_first_existing([ + new_url, + "https://www.yoctoproject.org/docs/", + ]); + } + } + + // Returns the current doctype based upon the url + function doctype_segment_from_url(url) { + if (url.includes("singleindex") || url.includes("mega-manual")) + return "mega"; + return "single"; + } + + $(document).ready(function () { + var release = DOCUMENTATION_OPTIONS.VERSION; + var current_doctype = doctype_segment_from_url(window.location.href); + var current_series = release.substr(0, 3); + var version_select = build_version_select(release); + + $(".version_switcher_placeholder").html(version_select); + $(".version_switcher_placeholder select").bind("change", on_version_switch); + + var doctype_select = build_doctype_select(current_doctype); + + $(".doctype_switcher_placeholder").html(doctype_select); + $(".doctype_switcher_placeholder select").bind("change", on_doctype_switch); + + // if release = "X.Y ()", remove the "()" so that only X.Y is compared + release = release.split(" ")[0]; + if (!(["dev", "next"].includes(release)) && !(release in all_versions)) { + $("#outdated-warning").html( + "Version " + release + " of the project is now considered obsolete, please select and use a more recent version", + ); + $("#outdated-warning").css("padding", ".5em"); + } + }); +})(); From 2408492f9143c6020d7152ce0daf149ead700be7 Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Fri, 20 Feb 2026 10:14:05 +0100 Subject: [PATCH 11/86] bitbake-setup: print colored diffs Define a get_diff_color_param() function that returns the --color value for diff commands. It function also uses color_enabled() to force showing color or not in the spawned subprocess (--color=auto would never show color). Signed-off-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- bin/bitbake-setup | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 9b0ecc8d0e4..553164b575d 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -42,6 +42,9 @@ def color_enabled() -> bool: """ return logger.handlers[0].formatter.color_enabled +def get_diff_color_param() -> str: + return "--color=always" if color_enabled() else "--color=never" + def print_configs(prompt: str, choices: list[str], descriptions: list[str] = []): """ Helper function to print a list of choices and align the output. @@ -202,7 +205,7 @@ be preserved in a backup directory.""".format(r_name, r_path)) status = bb.process.run('git -C {} status --porcelain'.format(r_path))[0] if status: return True - diff = bb.process.run('git -C {} diff {}'.format(r_path, rev))[0] + diff = bb.process.run('git -C {} diff {} {}'.format(r_path, get_diff_color_param(), rev))[0] if diff: return True return False @@ -728,7 +731,7 @@ def init_config(top_dir, settings, args): def get_diff(file1, file2): try: - bb.process.run('diff -uNr {} {}'.format(file1, file2)) + bb.process.run('diff {} -uNr {} {}'.format(get_diff_color_param(), file1, file2)) except bb.process.ExecutionError as e: if e.exitcode == 1: return e.stdout @@ -786,7 +789,7 @@ def build_status(top_dir, settings, args, d, update=False): new_upstream_config = obtain_config(top_dir, registry, args, source_overrides, d) write_upstream_config(confdir, new_upstream_config) - config_diff = bb.process.run('git -C {} diff'.format(confdir))[0] + config_diff = bb.process.run('git -C {} diff {}'.format(confdir, get_diff_color_param()))[0] if config_diff: logger.plain('\nConfiguration in {} has changed:\n{}'.format(setupdir, config_diff)) From 2fde7821fd0e84fc9f621f78a5ca08c4505d40fe Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 20 Feb 2026 12:58:43 -0700 Subject: [PATCH 12/86] cookerdata: Include "originating" recipe name when parsing When a bbappend file is parsed, the FILE variable is set to the name of the actual file being parsed (e.g. the name of the bbappend). Since PN/PV etc. are derived from FILE, this means that they can be misleading in the event that bbappend is using wildcards (e.g. PV might have the value of "%" instead of the actual version number). In order to allow bbappends to derived the actual information of the recipe, capture the name of the original recipe being parsed as __BB_RECIPE_FILE when parsing a new recipe. The value of this variable doesn't change when parsing .bbappend or .inc file associated with the recipe Signed-off-by: Joshua Watt Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- lib/bb/cookerdata.py | 2 + lib/bb/tests/parse-tests/classes/base.bbclass | 5 + .../classes/recipe-file-class.bbclass | 2 + lib/bb/tests/parse-tests/conf/bitbake.conf | 15 ++ lib/bb/tests/parse.py | 132 +++++++++++++++++- 5 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 lib/bb/tests/parse-tests/classes/base.bbclass create mode 100644 lib/bb/tests/parse-tests/classes/recipe-file-class.bbclass create mode 100644 lib/bb/tests/parse-tests/conf/bitbake.conf diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py index 22ac95eac9f..59f808c9694 100644 --- a/lib/bb/cookerdata.py +++ b/lib/bb/cookerdata.py @@ -512,6 +512,8 @@ def _parse_recipe(bb_data, bbfile, appends, mc, layername): bb_data.setVar("__BBMULTICONFIG", mc) bb_data.setVar("FILE_LAYERNAME", layername) + bb_data.setVar("__BB_RECIPE_FILE", bbfile) + bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) bb.parse.cached_mtime_noerror(bbfile_loc) diff --git a/lib/bb/tests/parse-tests/classes/base.bbclass b/lib/bb/tests/parse-tests/classes/base.bbclass new file mode 100644 index 00000000000..db8898e12f2 --- /dev/null +++ b/lib/bb/tests/parse-tests/classes/base.bbclass @@ -0,0 +1,5 @@ +# At least one task is required for bitbake to parse +do_fetch() { + : +} +addtask do_fetch diff --git a/lib/bb/tests/parse-tests/classes/recipe-file-class.bbclass b/lib/bb/tests/parse-tests/classes/recipe-file-class.bbclass new file mode 100644 index 00000000000..4682dc6c3d8 --- /dev/null +++ b/lib/bb/tests/parse-tests/classes/recipe-file-class.bbclass @@ -0,0 +1,2 @@ +BBCLASS_RECIPE_FILE := "${@os.path.basename(d.getVar('__BB_RECIPE_FILE'))}" +BBCLASS_FILE := "${@os.path.basename(d.getVar('FILE'))}" diff --git a/lib/bb/tests/parse-tests/conf/bitbake.conf b/lib/bb/tests/parse-tests/conf/bitbake.conf new file mode 100644 index 00000000000..e03625061bb --- /dev/null +++ b/lib/bb/tests/parse-tests/conf/bitbake.conf @@ -0,0 +1,15 @@ +CACHE = "${TOPDIR}/cache" +THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" +COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}" +EXTRA_BBFILES ?= "" +BBFILES = "${COREBASE}/recipes/*.bb ${COREBASE}/recipes/*.bbappend ${EXTRA_BBFILES}" +PROVIDES = "${PN}" +PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" +PF = "${BB_CURRENT_MC}:${PN}" +export PATH +TMPDIR ??= "${TOPDIR}" +STAMP = "${TMPDIR}/stamps/${PN}" +T = "${TMPDIR}/workdir/${PN}/temp" +BB_NUMBER_THREADS = "4" + +BB_BASEHASH_IGNORE_VARS = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" diff --git a/lib/bb/tests/parse.py b/lib/bb/tests/parse.py index d3867ece988..6ac2137e07a 100644 --- a/lib/bb/tests/parse.py +++ b/lib/bb/tests/parse.py @@ -11,6 +11,8 @@ import logging import bb import os +import subprocess +import textwrap logger = logging.getLogger('BitBake.TestParse') @@ -210,7 +212,7 @@ def test_parse_combinations(self): # # Test based upon a real world data corruption issue. One # data store changing a variable poked through into a different data - # store. This test case replicates that issue where the value 'B' would + # store. This test case replicates that issue where the value 'B' would # become unset/disappear. # def test_parse_classextend_contamination(self): @@ -508,3 +510,131 @@ def test_helper(content, result): test_helper("require some3.conf", " foobar") test_helper("include_all some.conf", " bar foo") test_helper("include_all some3.conf", " foobar") + + def test_file_variables(self): + # Tests the values of FILE and __BB_RECIPE_FILE in different + # combinations of bbappends, includes, and inherits + + def write_file(path, data): + with open(path, "w") as f: + f.write(textwrap.dedent(data)) + + def run_bitbake(cmd, builddir, extraenv={}): + env = os.environ.copy() + env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "parse-tests")) + env["BB_ENV_PASSTHROUGH_ADDITIONS"] = "TOPDIR" + env["TOPDIR"] = builddir + for k, v in extraenv.items(): + env[k] = v + env["BB_ENV_PASSTHROUGH_ADDITIONS"] = env["BB_ENV_PASSTHROUGH_ADDITIONS"] + " " + k + try: + return subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT, universal_newlines=True, cwd=builddir) + except subprocess.CalledProcessError as e: + self.fail("Command %s failed with %s" % (cmd, e.output)) + + with tempfile.TemporaryDirectory(prefix="parserecipes") as recipes, tempfile.TemporaryDirectory(prefix="parsetest") as builddir: + extraenv = { + "EXTRA_BBFILES": f"{recipes}/*.bb {recipes}/*.bbappend", + } + + inc_path = f"{recipes}/recipe-file.inc" + bbappend_path = f"{recipes}/recipe-%.bbappend" + recipe_path = f"{recipes}/recipe-file1.bb" + + # __BB_RECIPE_FILE should always be the name of .bb file, even + # when set in a bbappend. FILE is the name of the bbappend + write_file(recipe_path, "") + write_file(bbappend_path, + """\ + BBAPPEND_RECIPE_FILE := "${@os.path.basename(d.getVar('__BB_RECIPE_FILE'))}" + BBAPPEND_FILE := "${@os.path.basename(d.getVar('FILE'))}" + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBAPPEND_FILE="recipe-%.bbappend"', output) + self.assertIn(f'BBAPPEND_RECIPE_FILE="recipe-file1.bb"', output) + + # __BB_RECIPE_FILE should always be the name of .bb file, even when + # set in an include file. FILE is the name of the include + write_file(recipe_path, + """\ + require recipe-file.inc + """ + ) + write_file(inc_path, + """\ + INC_RECIPE_FILE := "${@os.path.basename(d.getVar('__BB_RECIPE_FILE'))}" + INC_FILE := "${@os.path.basename(d.getVar('FILE'))}" + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('INC_FILE="recipe-file.inc"', output) + self.assertIn(f'INC_RECIPE_FILE="recipe-file1.bb"', output) + + # Test when the include file is included from a bbappend + write_file(recipe_path, "") + write_file(bbappend_path, + """\ + require recipe-file.inc + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('INC_FILE="recipe-file.inc"', output) + self.assertIn(f'INC_RECIPE_FILE="recipe-file1.bb"', output) + + # Test the variables in a bbclass when inherited directly in the + # recipe. Note that FILE still refers to the recipe in a bbclass + write_file(recipe_path, + """\ + inherit recipe-file-class + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBCLASS_FILE="recipe-file1.bb"', output) + self.assertIn(f'BBCLASS_RECIPE_FILE="recipe-file1.bb"', output) + + # Test the variables when the inherit is in a bbappend. In this + # case, FILE is the bbappend + write_file(recipe_path, "") + write_file(bbappend_path, + """\ + inherit recipe-file-class + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBCLASS_FILE="recipe-%.bbappend"', output) + self.assertIn(f'BBCLASS_RECIPE_FILE="recipe-file1.bb"', output) + + # Test the variables when the inherit is in a include. In this + # case, FILE is the include file + write_file(recipe_path, + """\ + require recipe-file.inc + """ + ) + write_file(bbappend_path, "") + write_file(inc_path, + """\ + inherit recipe-file-class + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBCLASS_FILE="recipe-file.inc"', output) + self.assertIn(f'BBCLASS_RECIPE_FILE="recipe-file1.bb"', output) + + # Test the variables when the inherit is in a include included from + # a bbappend. In this case, FILE is the include file + write_file(recipe_path, "") + write_file(bbappend_path, + """\ + require recipe-file.inc + """ + ) + write_file(inc_path, + """\ + inherit recipe-file-class + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBCLASS_FILE="recipe-file.inc"', output) + self.assertIn(f'BBCLASS_RECIPE_FILE="recipe-file1.bb"', output) From 1f8fa7c25e71cd0f230a2f6bfd9d5153c694da81 Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Fri, 20 Feb 2026 15:30:07 +0100 Subject: [PATCH 13/86] tinfoil: show close matches when no providers are found When calling 'bitbake-getvar -r somerecipe FOO', show close matches when 'somerecipe' doesn't exist but close matches are found. This replicates the behavior of 'bitbake -e'. Example output: $ bitbake-getvar -r binutils-cross FOO Unable to find any recipe file matching "binutils-cross". Close matches: binutils-cross-aarch64 binutils util-macros Signed-off-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- lib/bb/tinfoil.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/bb/tinfoil.py b/lib/bb/tinfoil.py index d9e985c612c..9c1768942ab 100644 --- a/lib/bb/tinfoil.py +++ b/lib/bb/tinfoil.py @@ -685,7 +685,14 @@ def get_recipe_file(self, pn): if skipreasons: raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons))) else: - raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) + msg = f'Unable to find any recipe file matching "{pn}"' + import difflib + providers = self.get_all_providers() + close_matches = difflib.get_close_matches(pn, providers, cutoff=0.7) + if close_matches: + close_matches = "\n ".join(close_matches) + msg += f'. Close matches:\n {close_matches}' + raise bb.providers.NoProvider(msg) return best[3] def get_file_appends(self, fn, mc=''): From bc8be83aef0a6de85cd33a6f132f281d518594f7 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Wed, 14 Jan 2026 16:41:20 +0100 Subject: [PATCH 14/86] doc: bitbake-user-manual-ref-variables: BB_ENV_PASSTHROUGH is not in default BB_ENV_PASSTHROUGH_ADDITIONS (anymore) BB_ENV_PASSTHROUGH_ADDITIONS variable doesn't contain BB_ENV_PASSTHROUGH anymore since commit 0a33b560233b ("utils: remove BB_ENV_PASSTHROUGH from preserved_envvars()") so let's fix this oversight. Reported-by: Robert P. J. Day Suggested-by: Richard Purdie Signed-off-by: Quentin Schulz Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index 3a01144ff49..fcd93aadc67 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -286,8 +286,8 @@ overview of their function and contents. Specifies the internal list of variables to allow through from the external environment into BitBake's datastore. If the value of this variable is not specified (which is the default), the following - list is used: :term:`BBPATH`, :term:`BB_PRESERVE_ENV`, - :term:`BB_ENV_PASSTHROUGH`, and :term:`BB_ENV_PASSTHROUGH_ADDITIONS`. + list is used: :term:`BBPATH`, :term:`BB_PRESERVE_ENV`, and + :term:`BB_ENV_PASSTHROUGH_ADDITIONS`. .. note:: From a3ad577439d11474db8f1136c0d2c80abb6c78fe Mon Sep 17 00:00:00 2001 From: "Robert P. J. Day" Date: Tue, 24 Feb 2026 12:23:00 -0500 Subject: [PATCH 15/86] correct small number of "Bitbake" to "BitBake" Use correct spelling of "BitBake" in a few places. Signed-off-by: Robert P. J. Day Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- doc/bitbake-user-manual/bitbake-user-manual-fetching.rst | 2 +- doc/bitbake-user-manual/bitbake-user-manual-intro.rst | 4 ++-- doc/bitbake-user-manual/bitbake-user-manual-metadata.rst | 2 +- doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index 8e1a232cdbc..1dcdc0ffee1 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -402,7 +402,7 @@ Here are some example URLs:: easy to share metadata without removing passwords. SSH keys, ``~/.netrc`` and ``~/.ssh/config`` files can be used as alternatives. -Using tags with the git fetcher may cause surprising behaviour. Bitbake needs to +Using tags with the git fetcher may cause surprising behaviour. BitBake needs to resolve the tag to a specific revision and to do that, it has to connect to and use the upstream repository. This is because the revision the tags point at can change and we've seen cases of this happening in well known public repositories. This can mean diff --git a/doc/bitbake-user-manual/bitbake-user-manual-intro.rst b/doc/bitbake-user-manual/bitbake-user-manual-intro.rst index 6bcab487000..6801323e2fb 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-intro.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-intro.rst @@ -233,7 +233,7 @@ BitBake supports class files installed in three different directories: For details on how BitBake locates class files, see the :ref:`bitbake-user-manual/bitbake-user-manual-metadata:Locating Class Files` -section of the Bitbake User Manual. +section of the BitBake User Manual. Layers ------ @@ -539,7 +539,7 @@ Following is the usage and syntax for BitBake:: make dependency graphs more appealing. .. - Bitbake help output generated with "stty columns 80; bin/bitbake -h" + BitBake help output generated with "stty columns 80; bin/bitbake -h" .. _bitbake-examples: diff --git a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index 18f293f4b7c..f8bdf19a8ff 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -1158,7 +1158,7 @@ Like include files, class files are located using the :term:`BBPATH` variable. The classes can be included in the ``classes-recipe``, ``classes-global`` and ``classes`` directories, as explained in the :ref:`bitbake-user-manual/bitbake-user-manual-intro:Class types` section of the -Bitbake User Manual. Like for the :ref:`include ` and +BitBake User Manual. Like for the :ref:`include ` and :ref:`require ` directives, BitBake stops and inherits the first class that it finds. diff --git a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index fcd93aadc67..06bd536195c 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -1339,7 +1339,7 @@ overview of their function and contents. - When recipes are parsed - then for each parsed recipe. - Bitbake ignores changes to :term:`INHERIT` in individual recipes. + BitBake ignores changes to :term:`INHERIT` in individual recipes. For more information on :term:`INHERIT`, see the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:\`\`inherit\`\` configuration directive`" From 30b88fb066c5600683e4179475639b6582b42221 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Fri, 27 Feb 2026 12:30:15 +0100 Subject: [PATCH 16/86] doc: bitbake-user-manual-metadata: promote inherit_defer directive to subsection Out of all the directives listed in this document, only inherit_defer is documented in a subsubsection rather than in a subsection. Therefore, let's be consistent and make inherit_defer documentation into a subsection. Fixes: eb10df5a9619 ("doc: bitbake-user-manual: document inherit_defer") Signed-off-by: Quentin Schulz Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- doc/bitbake-user-manual/bitbake-user-manual-metadata.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index f8bdf19a8ff..b6002f63b15 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -813,7 +813,7 @@ evaluated at the end of parsing. .. _ref-bitbake-user-manual-metadata-inherit-defer: ``inherit_defer`` Directive -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- The :ref:`inherit_defer ` directive works like the :ref:`inherit From 6b91f4ff3852a5a3c1089d5bb9d4009a2e6fde58 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Fri, 27 Feb 2026 12:30:16 +0100 Subject: [PATCH 17/86] doc: bitbake-user-manual-metadata: list known limitations for shell tasks Javier Tia reported[1] they keep forgetting `[[` conditional isn't supported and scouring our BugZilla, I found[2] another "bug" where arithmetic expansion isn't supported either. Let's add a subsection of the shell tasks with the known limitations. [1] https://lore.kernel.org/bitbake-devel/20250320003358.977774-1-javier.tia@linaro.org/ [2] https://bugzilla.yoctoproject.org/show_bug.cgi?id=11314 Reported-by: Javier Tia Suggested-by: Javier Tia Signed-off-by: Quentin Schulz Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- .../bitbake-user-manual-metadata.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index b6002f63b15..bf517f03033 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -1231,7 +1231,9 @@ When you create these types of functions in your recipe or class files, you need to follow the shell programming rules. The scripts are executed by ``/bin/sh``, which may not be a bash shell but might be something such as ``dash``. You should not use -Bash-specific script (bashisms). +Bash-specific script (bashisms), such as ``[[`` (use ``[`` instead). The BitBake +shell parser also has further +:ref:`limitations `. Overrides and override-style operators like ``:append`` and ``:prepend`` can also be applied to shell functions. Most commonly, this application @@ -1272,6 +1274,20 @@ Running ``do_foo`` prints the following:: You can use the ``bitbake -e recipename`` command to view the final assembled function after all overrides have been applied. +Limitations +^^^^^^^^^^^ + +Shell functions must be parsable by BitBake, which cannot handle all +syntax supported by POSIX shells. In particular: + +#. Arithmetic expansion is not supported. This means you cannot do: + + .. code-block:: shell + + VAR=$(( i + 1 )) + + See https://bugzilla.yoctoproject.org/show_bug.cgi?id=11314. + BitBake-Style Python Functions ------------------------------ From 4c378445969853d6aff4694d937b9af47c7f7300 Mon Sep 17 00:00:00 2001 From: Florian Schmaus Date: Fri, 27 Feb 2026 13:43:36 +0100 Subject: [PATCH 18/86] fetch: Add GIT_CONFIG_GLOBAL to FETCH_EXPORT_VARS Many CI setups rely on the GIT_CONFIG_GLOBAL environment variable to define project-specific gitconfigs, often to inject credentials using git's url..insteadOf configuration. However, BitBake restricts the environment variables passed to the fetch task to a fixed list (FETCH_EXPORT_VARS). Because GIT_CONFIG_GLOBAL is missing from this default list, the variable is stripped from the environment. Add GIT_CONFIG_GLOBAL to FETCH_EXPORT_VARS to natively support this credential injection method (and GIT_CONFIG_GLOBAL in general). This also eliminates confusion for users who expect the variable to propagate after adding it to `BB_ENV_PASSTHROUGH_ADDITIONS`, only to find it has no effect during the fetch task. Signed-off-by: Florian Schmaus Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- lib/bb/fetch2/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py index dcf7adc26ab..aaefd860204 100644 --- a/lib/bb/fetch2/__init__.py +++ b/lib/bb/fetch2/__init__.py @@ -877,6 +877,7 @@ def localpath(url, d): 'FTPS_PROXY', 'ftps_proxy', 'NO_PROXY', 'no_proxy', 'ALL_PROXY', 'all_proxy', + 'GIT_CONFIG_GLOBAL', 'GIT_PROXY_COMMAND', 'GIT_SSH', 'GIT_SSH_COMMAND', From fb1289d0bfa4d563f3ed51857ad0544d948bcd3a Mon Sep 17 00:00:00 2001 From: Chen Qi Date: Sat, 28 Feb 2026 16:24:38 +0800 Subject: [PATCH 19/86] fetch2/git.py: fix update_mirror_links We cannot assume ud has the same fetcher with origud. For example, if we set map git:// to file:// in PREMIRRORS, ud is using local fetcher and origud is using git fetcher. In such case, the ud does have the 'shallow' attribute. And we'll see the following error: Exception: AttributeError: 'FetchData' object has no attribute 'shallow' Looking at the logic of this function, I think it's the origud's shallow that should be checked. So the logic becomes: if origud is using shallow and its full shallow tarball does not exist yet, symlink to ensure it exists. This should make more sense. Signed-off-by: Chen Qi Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- lib/bb/fetch2/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 0fcdb19df11..738174cd104 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -356,7 +356,7 @@ def update_mirror_links(self, ud, origud): super().update_mirror_links(ud, origud) # When using shallow mode, add a symlink to the original fullshallow # path to ensure a valid symlink even in the `PREMIRRORS` case - if ud.shallow and not os.path.exists(origud.fullshallow): + if origud.shallow and not os.path.exists(origud.fullshallow): self.ensure_symlink(ud.localpath, origud.fullshallow) def try_premirror(self, ud, d): From 48efc36b4e03f736e7521d269ced3417522784e9 Mon Sep 17 00:00:00 2001 From: Chen Qi Date: Sat, 28 Feb 2026 16:24:39 +0800 Subject: [PATCH 20/86] lib/bb/tests/fetch.py: add test case to ensure no FetchData attribute error When using git:// -> file:// in PREMIRRORS and we have the same url in SRC_URI twice, we reveal the following error: AttributeError: 'FetchData' object has no attribute 'shallow' Add a test case to ensure future changes don't introduce the error back. Signed-off-by: Chen Qi Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- lib/bb/tests/fetch.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 2a8501dc2ea..74eb7347242 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -2089,6 +2089,29 @@ def test_shallow_mirrors(self): self.fetch_and_unpack() self.assertRevCount(1) + @skipIfNoNetwork() + def test_shallow_mirrors_with_multiple_same_urls(self): + url = "git://git.openembedded.org/bitbake;branch=master;protocol=https;rev=82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40" + + d = self.d.createCopy() + d.delVar('AUTOREV') + d.delVar('SRCREV') + + # prepare premirror + premirror = os.path.join(self.tempdir, "premirror") + os.mkdir(premirror) + d.setVar("DL_DIR", premirror) + fetcher = bb.fetch.Fetch([url], d) + fetcher.download() + + # set PREMIRRORS + d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % premirror) + + # set DL_DIR back and use the same url multiple times to fetch + d.setVar("DL_DIR", self.dldir) + fetcher2 = bb.fetch.Fetch([url, url], d) + fetcher2.download() + def test_shallow_invalid_depth(self): self.add_empty_file('a') self.add_empty_file('b') From 35866d55908009df429870b28cda0d2266074b48 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 4 Mar 2026 09:25:56 -0500 Subject: [PATCH 21/86] runqueue.py: make sure we use bb multiprocessing Otherwise, we see errors like: |ERROR: An uncaught exception occurred in runqueue |Traceback (most recent call last): | File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1663, in execute_runqueue | return self._execute_runqueue() | ~~~~~~~~~~~~~~~~~~~~~~^^ | File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1607, in _execute_runqueue | retval = self.dump_signatures(dumpsigs) | File "/srv/pokybuild/yocto-worker/check-layer/build/layers/bitbake/lib/bb/runqueue.py", line 1718, in dump_signatures | p.start() | ~~~~~~~^^ | File "/usr/lib64/python3.14/multiprocessing/process.py", line 121, in start | self._popen = self._Popen(self) | ~~~~~~~~~~~^^^^^^ | File "/usr/lib64/python3.14/multiprocessing/context.py", line 224, in _Popen | return _default_context.get_context().Process._Popen(process_obj) | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ | File "/usr/lib64/python3.14/multiprocessing/context.py", line 300, in _Popen | return Popen(process_obj) | File "/usr/lib64/python3.14/multiprocessing/popen_forkserver.py", line 35, in __init__ | super().__init__(process_obj) | ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ | File "/usr/lib64/python3.14/multiprocessing/popen_fork.py", line 20, in __init__ | self._launch(process_obj) | ~~~~~~~~~~~~^^^^^^^^^^^^^ | File "/usr/lib64/python3.14/multiprocessing/popen_forkserver.py", line 47, in _launch | reduction.dump(process_obj, buf) | ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ | File "/usr/lib64/python3.14/multiprocessing/reduction.py", line 60, in dump | ForkingPickler(file, protocol).dump(obj) | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^ |_pickle.PicklingError: Can't pickle local object . at 0x7f7925667d70> |when serializing tuple item 0 |when serializing collections.defaultdict reconstructor arguments |when serializing collections.defaultdict object |when serializing dict item 'rundeps' |when serializing bb.cache.CacheData state |when serializing bb.cache.CacheData object |when serializing dict item '' |when serializing dict item 'recipecaches' |when serializing bb.cooker.BBCooker state |when serializing bb.cooker.BBCooker object |when serializing dict item 'cooker' |when serializing bb.runqueue.RunQueue state |when serializing bb.runqueue.RunQueue object |when serializing tuple item 0 |when serializing method reconstructor arguments |when serializing method object |when serializing dict item '_target' |when serializing multiprocessing.context.Process state |when serializing multiprocessing.context.Process object Fixes: [YOCTO #16184] Signed-off-by: Trevor Gamblin Signed-off-by: Richard Purdie --- lib/bb/runqueue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index a880a0d5416..239fa770e05 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -23,11 +23,12 @@ from bb import monitordisk import subprocess import pickle -from multiprocessing import Process import shlex import pprint import time +Process = bb.multiprocessing.Process + bblogger = logging.getLogger("BitBake") logger = logging.getLogger("BitBake.RunQueue") hashequiv_logger = logging.getLogger("BitBake.RunQueue.HashEquiv") From b54600e37073a83be1d0490bb16039df73f09c83 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Tue, 3 Mar 2026 09:24:27 -0700 Subject: [PATCH 22/86] bitbake-worker: Use Lock from bb.multiprocessing bitbake-worker relies on multiprocessing using the "fork" method to pass data to the child process. Starting in Python 3.13 the default changed to "forkserver", so the bb.multiprocessing context needs to be used in order to preserve the "fork" method. Note that bb.multiprocessing is a context, so Lock can't be imported from it; instead a local alias for Lock must be used Signed-off-by: Joshua Watt Signed-off-by: Mathieu Dubois-Briand --- bin/bitbake-worker | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/bitbake-worker b/bin/bitbake-worker index d2b146a6a92..aa14ef1914c 100755 --- a/bin/bitbake-worker +++ b/bin/bitbake-worker @@ -23,9 +23,10 @@ import queue import shlex import subprocess import fcntl -from multiprocessing import Lock from threading import Thread +Lock = bb.multiprocessing.Lock + # Remove when we have a minimum of python 3.10 if not hasattr(fcntl, 'F_SETPIPE_SZ'): fcntl.F_SETPIPE_SZ = 1031 From a01be6f0e9102e25719bad653c864b96537af56c Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Mon, 9 Mar 2026 15:12:36 +0100 Subject: [PATCH 23/86] bitbake-setup: ensure that config_id is always an absolute path If config_id (given on command line in 'init' operation) is provided and interpreted as a relative path to a local file (by checking for the file existence), it should be recorded into configuration instance as an absolute path. Otherwise status/update operations will fail if executed from a different directory that the template file was in, as they won't be able to find the file. Note that the patch context doesn't show where having the absolute path actually matters: further down in the function, config_id is used to set a 'non-interactive-cmdline-options' property in configuration instance dictionary. This property is used by status/update operations. Signed-off-by: Alexander Kanavin Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- bin/bitbake-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 553164b575d..6083bffeeec 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -600,9 +600,10 @@ def obtain_config(top_dir, registry, args, source_overrides, d): config_id = args.config[0] config_parameters = args.config[1:] if os.path.exists(config_id): + config_id = os.path.abspath(config_id) logger.info("Reading configuration from local file\n {}".format(config_id)) upstream_config = {'type':'local', - 'path':os.path.abspath(config_id), + 'path':config_id, 'name':get_config_name(config_id), 'data':json.load(open(config_id)) } From da479ccf5dff85e6c0fc321f4596d4e8a802b47c Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Mon, 9 Mar 2026 15:12:37 +0100 Subject: [PATCH 24/86] bitbake-setup: are_layers_changed(): calculate local_revision only once There's no need to repeatedly do it in the loop, as it doesn't change. Also reposition 'remotes' and 'changed' variables assignments, as this will aid the code readability after the following commit. Signed-off-by: Alexander Kanavin Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- bin/bitbake-setup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 6083bffeeec..86eccf95255 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -742,12 +742,14 @@ def get_diff(file1, file2): def are_layers_changed(layers, layerdir, d): def _is_git_remote_changed(r_remote, repodir): - changed = False rev = r_remote['rev'] branch = r_remote.get('branch', None) - remotes = _get_remotes(r_remote) + rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir))) + local_revision = rev_parse_result[0].strip() + remotes = _get_remotes(r_remote) + changed = False for remote in remotes: type,host,path,user,pswd,params = bb.fetch.decodeurl(remote) fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params)) @@ -756,8 +758,6 @@ def are_layers_changed(layers, layerdir, d): else: fetcher = bb.fetch.FetchData("{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir), d) upstream_revision = fetcher.method.latest_revision(fetcher, d, 'default') - rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir))) - local_revision = rev_parse_result[0].strip() if upstream_revision != local_revision: changed = True logger.info('Layer repository {} checked out into {} updated revision {} from {} to {}'.format(remote, os.path.join(layerdir, repodir), rev, local_revision, upstream_revision)) From ead5e02ad7370cbe58d4da5496f725ceeefdc1a3 Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Mon, 9 Mar 2026 15:12:38 +0100 Subject: [PATCH 25/86] bitbake-setup: correctly determine latest revision in status/update when it is fixed to a commit id The code was relying on git fetcher's latest_revision() method, which works properly if the revision in the json config template is set to a branch or a tag, but does not work if it is set to a fixed revision (the method will return the latest revision on the respective branch, and bitbake-setup will incorrectly determine that the layer needs to be updated to that revision). Add a guard to check if the revision is a fixed commit id, and do a direct revision comparison if so, skipping the fetcher query. [YOCTO #16190] Signed-off-by: Alexander Kanavin Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- bin/bitbake-setup | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 86eccf95255..53dbb622136 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -742,11 +742,18 @@ def get_diff(file1, file2): def are_layers_changed(layers, layerdir, d): def _is_git_remote_changed(r_remote, repodir): + from bb.fetch2.git import sha1_re + rev = r_remote['rev'] branch = r_remote.get('branch', None) rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir))) local_revision = rev_parse_result[0].strip() + if sha1_re.match(rev): + if rev != local_revision: + logger.info('Layer repository checked out into {} is at revision {} but should be at {}'.format(os.path.join(layerdir, repodir),local_revision, rev)) + return True + return False remotes = _get_remotes(r_remote) changed = False From b59bdcf8134e6f5935e8f7b0ed8033cbb745e0ff Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Wed, 11 Mar 2026 10:42:23 +0000 Subject: [PATCH 26/86] tests/fetch: Drop github url from testing This was introduced as a test for the fix in 6ec70d5d2e330b41b932b0a655b838a5f37df01e however the way github works has changed in the decade since and this test is no longer testing what was intended. The test is failing intermittently on our infrastructure thanks to spam and IP block throttling restrictions which lead to occasional failures. At this point it is probably simplest just to drop the url from testing. Signed-off-by: Richard Purdie --- lib/bb/tests/fetch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 74eb7347242..7b8297a7827 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -1579,8 +1579,6 @@ class FetchCheckStatusTest(FetcherTest): "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", - # GitHub releases are hosted on Amazon S3, which doesn't support HEAD - "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" ] @skipIfNoNetwork() From e8ace6e16c78ddbc2cfe985b88b66800d99282b3 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 12 Mar 2026 22:28:41 +0000 Subject: [PATCH 27/86] fetch/wget: Improve connection error handling We see occasional connection errors in wget testing of sstate mirrors. It appears there is a case where http.client.RemoteDisconnected is returned against getrepsonse() which the current code doesn't handle well. Rather than trying to handle very specific error cases, catch any errors and drop the cached connection in those cases in the do_open() code. Similarly, try again in a more complete set of error cases (all of OSError instead of special cases and RemoteDisconnected). [YOCTO #15945] The traceback from the logs is included below for reference. Traceback (most recent call last): File "/srv/pokybuild/yocto-worker/a-full/build/layers/bitbake/lib/bb/fetch2/wget.py", line 415, in checkstatus with opener.open(r, timeout=100) as response: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/urllib/request.py", line 519, in open response = self._open(req, data) ^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/urllib/request.py", line 536, in _open result = self._call_chain(self.handle_open, protocol, protocol + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/urllib/request.py", line 496, in _call_chain result = func(*args) ^^^^^^^^^^^ File "/srv/pokybuild/yocto-worker/a-full/build/layers/bitbake/lib/bb/fetch2/wget.py", line 178, in http_open return self.do_open(HTTPConnectionCache, req) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/srv/pokybuild/yocto-worker/a-full/build/layers/bitbake/lib/bb/fetch2/wget.py", line 249, in do_open r = h.getresponse() ^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/http/client.py", line 1374, in getresponse response.begin() File "/usr/lib/python3.11/http/client.py", line 318, in begin version, status, reason = self._read_status() ^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/http/client.py", line 287, in _read_status raise RemoteDisconnected("Remote end closed connection without" http.client.RemoteDisconnected: Remote end closed connection without response During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/srv/pokybuild/yocto-worker/a-full/build/layers/bitbake/lib/bb/fetch2/wget.py", line 228, in do_open h.request(req.get_method(), req.selector, req.data, headers) File "/usr/lib/python3.11/http/client.py", line 1282, in request self._send_request(method, url, body, headers, encode_chunked) File "/usr/lib/python3.11/http/client.py", line 1328, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/usr/lib/python3.11/http/client.py", line 1277, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/usr/lib/python3.11/http/client.py", line 1037, in _send_output self.send(msg) File "/usr/lib/python3.11/http/client.py", line 998, in send self.sock.sendall(data) OSError: [Errno 9] Bad file descriptor During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/srv/pokybuild/yocto-worker/a-full/build/layers/bitbake/lib/bb/fetch2/wget.py", line 415, in checkstatus with opener.open(r, timeout=100) as response: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/urllib/request.py", line 519, in open response = self._open(req, data) ^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/urllib/request.py", line 536, in _open result = self._call_chain(self.handle_open, protocol, protocol + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/urllib/request.py", line 496, in _call_chain result = func(*args) ^^^^^^^^^^^ File "/srv/pokybuild/yocto-worker/a-full/build/layers/bitbake/lib/bb/fetch2/wget.py", line 178, in http_open return self.do_open(HTTPConnectionCache, req) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/srv/pokybuild/yocto-worker/a-full/build/layers/bitbake/lib/bb/fetch2/wget.py", line 246, in do_open raise urllib.error.URLError(err) Signed-off-by: Richard Purdie --- lib/bb/fetch2/wget.py | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py index 4e3505599b4..ca4959ab5f7 100644 --- a/lib/bb/fetch2/wget.py +++ b/lib/bb/fetch2/wget.py @@ -216,31 +216,20 @@ def do_open(self, http_class, req): try: h.request(req.get_method(), req.selector, req.data, headers) - except socket.error as err: # XXX what error? - # Don't close connection when cache is enabled. - # Instead, try to detect connections that are no longer - # usable (for example, closed unexpectedly) and remove - # them from the cache. - if fetch.connection_cache is None: - h.close() - elif isinstance(err, OSError) and err.errno == errno.EBADF: - # This happens when the server closes the connection despite the Keep-Alive. - # Apparently urllib then uses the file descriptor, expecting it to be - # connected, when in reality the connection is already gone. - # We let the request fail and expect it to be - # tried once more ("try_again" in check_status()), - # with the dead connection removed from the cache. - # If it still fails, we give up, which can happen for bad - # HTTP proxy settings. + r = h.getresponse() + except: + # This can happen when the server closes the connection despite the Keep-Alive. + # Apparently urllib then uses the file descriptor, expecting it to be + # connected, when in reality the connection is already gone. + # We let the request fail and expect it to be + # tried once more ("try_again" in check_status()), + # with the dead connection removed from the cache. + # If it still fails, we give up, which can happen for bad + # HTTP proxy settings. + if fetch.connection_cache: fetch.connection_cache.remove_connection(h.host, h.port) - raise urllib.error.URLError(err) - else: - try: - r = h.getresponse() - except TimeoutError as e: - if fetch.connection_cache: - fetch.connection_cache.remove_connection(h.host, h.port) - raise TimeoutError(e) + h.close() + raise # Pick apart the HTTPResponse object to get the addinfourl # object initialized properly. @@ -404,9 +393,9 @@ def add_basic_auth(login_str, request): with opener.open(r, timeout=100) as response: pass - except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e: + except (urllib.error.URLError, OSError, http.client.RemoteDisconnected) as e: if try_again: - logger.debug2("checkstatus: trying again") + logger.debug2("checkstatus: trying again after exception %s" % str(e)) return self.checkstatus(fetch, ud, d, False) else: # debug for now to avoid spamming the logs in e.g. remote sstate searches From 240ae9a7ddd530d0ecf7ec93fe85293c8a1986aa Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Tue, 17 Mar 2026 11:08:36 +0100 Subject: [PATCH 28/86] doc: move the Git repo check to the conf.py file This check can be done earlier when building the documentation, and does not need any version information. Move it to the beginning of conf.py. Signed-off-by: Antonin Godard Signed-off-by: Richard Purdie --- doc/conf.py | 10 ++++++++++ doc/setversions.py | 9 --------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 9318358731c..40a022eaa00 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -8,10 +8,20 @@ import datetime import os +import subprocess import sys from pathlib import Path +# Test that we are building from a Git repository +try: + subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) +except subprocess.CalledProcessError: + sys.exit("Building bitbake's documentation must be done from its Git repository.\n" + "Clone the repository with the following command:\n" + "git clone https://git.openembedded.org/bitbake ") + sys.path.insert(0, os.path.abspath('.')) import setversions diff --git a/doc/setversions.py b/doc/setversions.py index 9e4138025ac..2ec68ac9945 100755 --- a/doc/setversions.py +++ b/doc/setversions.py @@ -43,15 +43,6 @@ def get_current_version(): ourversion = None - # Test that we are building from a Git repository - try: - subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) - except subprocess.CalledProcessError: - sys.exit("Building bitbake's documentation must be done from its Git repository.\n" - "Clone the repository with the following command:\n" - "git clone https://git.openembedded.org/bitbake ") - # Test tags exist and inform the user to fetch if not try: subprocess.run(["git", "show", f"{LTSSERIES[0]}.0"], From 2f07c60e55996c5f06e1e82c014b9e876c0792cf Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Tue, 17 Mar 2026 11:08:37 +0100 Subject: [PATCH 29/86] doc/setversions.py: simplify the get_current_version() function While this was inspired with yocto-docs' set_versions.py file[1], some of the logic here can be simplified with early returns. As a consequence of these early returns, move the print() call showing the computed version to the conf.py file. Signed-off-by: Antonin Godard Signed-off-by: Richard Purdie --- doc/conf.py | 1 + doc/setversions.py | 90 +++++++++++++++++++++------------------------- 2 files changed, 42 insertions(+), 49 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 40a022eaa00..315fb7d06a7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -26,6 +26,7 @@ import setversions current_version = setversions.get_current_version() +print(f"Version calculated to be {current_version}") # String used in sidebar version = 'Version: ' + current_version diff --git a/doc/setversions.py b/doc/setversions.py index 2ec68ac9945..4130df67695 100755 --- a/doc/setversions.py +++ b/doc/setversions.py @@ -41,8 +41,6 @@ BB_RELEASE_TAG_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") def get_current_version(): - ourversion = None - # Test tags exist and inform the user to fetch if not try: subprocess.run(["git", "show", f"{LTSSERIES[0]}.0"], @@ -50,63 +48,57 @@ def get_current_version(): except subprocess.CalledProcessError: sys.exit("Please run 'git fetch --tags' before building the documentation") - # Try and figure out what we are tags = subprocess.run(["git", "tag", "--points-at", "HEAD"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stdout for t in tags.split(): if re.match(BB_RELEASE_TAG_RE, t): - ourversion = t - break + return t - if ourversion: - # We're a tagged release - components = ourversion.split(".") - else: - # We're floating on a branch - branch = subprocess.run(["git", "branch", "--show-current"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True).stdout.strip() - - if branch == "" or branch not in list(YOCTO_MAPPING.keys()) + ["master", "master-next"]: - # We're not on a known release branch so we have to guess. Compare the - # numbers of commits from each release branch and assume the smallest - # number of commits is the one we're based off - possible_branch = None - branch_count = 0 - for b in itertools.chain(YOCTO_MAPPING.keys(), ["master"]): - result = subprocess.run(["git", "log", "--format=oneline", "HEAD..origin/" + b], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) - if result.returncode == 0: - count = result.stdout.count('\n') - if not possible_branch or count < branch_count: - print("Branch %s has count %s" % (b, count)) - possible_branch = b - branch_count = count - if possible_branch: - branch = possible_branch - else: - branch = "master" - print("Nearest release branch estimated to be %s" % branch) + # We're floating on a branch + branch = subprocess.run(["git", "branch", "--show-current"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout.strip() - if branch == "master": - ourversion = "dev" - elif branch == "master-next": - ourversion = "next" + if branch == "" or branch not in list(YOCTO_MAPPING.keys()) + ["master", "master-next"]: + # We're not on a known release branch so we have to guess. Compare the + # numbers of commits from each release branch and assume the smallest + # number of commits is the one we're based off + possible_branch = None + branch_count = 0 + for b in itertools.chain(YOCTO_MAPPING.keys(), ["master"]): + result = subprocess.run(["git", "log", "--format=oneline", "HEAD..origin/" + b], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if result.returncode == 0: + count = result.stdout.count('\n') + if not possible_branch or count < branch_count: + print("Branch %s has count %s" % (b, count)) + possible_branch = b + branch_count = count + if possible_branch: + branch = possible_branch else: - ourversion = branch - head_commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True).stdout.strip() - branch_commit = subprocess.run(["git", "rev-parse", "--short", branch], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True).stdout.strip() - if head_commit != branch_commit: - ourversion += f" ({head_commit})" + branch = "master" + print("Nearest release branch estimated to be %s" % branch) + + if branch == "master": + return "dev" + + if branch == "master-next": + return "next" + + ourversion = branch + head_commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout.strip() + branch_commit = subprocess.run(["git", "rev-parse", "--short", branch], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout.strip() + if head_commit != branch_commit: + ourversion += f" ({head_commit})" - print("Version calculated to be %s" % ourversion) return ourversion def write_switchers_js(js_in, js_out, current_version): From b55497ab7e29eec398fe52359878a76b185e1af0 Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Tue, 17 Mar 2026 11:08:39 +0100 Subject: [PATCH 30/86] doc: build using remote releases.json Like what was done for yocto-docs[1], build using the remote releases.json file we can fetch from [2]. The file is downloaded if not present, and not re-downloaded for subsequent builds. In case of fetch failure, fallback to default values. [1]: https://git.yoctoproject.org/yocto-docs/commit/?id=5bebe38a808a33fea3deefc21dda39a35d90a7dd [2]: https://dashboard.yoctoproject.org/releases.json Signed-off-by: Antonin Godard Signed-off-by: Richard Purdie --- doc/.gitignore | 1 + doc/Makefile | 2 +- doc/setversions.py | 59 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/doc/.gitignore b/doc/.gitignore index dee9494dcaf..1ee009c2012 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1,2 +1,3 @@ _build/ sphinx-static/switchers.js +releases.json diff --git a/doc/Makefile b/doc/Makefile index 5e1632314c5..752f9b53b3e 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -27,7 +27,7 @@ publish: Makefile html singlehtml sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html clean: - @rm -rf $(BUILDDIR) sphinx-static/switchers.js + @rm -rf $(BUILDDIR) sphinx-static/switchers.js releases.json # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/doc/setversions.py b/doc/setversions.py index 4130df67695..6aa6ec9ee2e 100755 --- a/doc/setversions.py +++ b/doc/setversions.py @@ -14,30 +14,69 @@ # import itertools +import json +import os import re import subprocess import sys +from urllib.request import urlopen, URLError + +# NOTE: the following variables contain default values in case we are not able to fetch +# the releases.json file from https://dashboard.yoctoproject.org/releases.json DEVBRANCH = "2.18" LTSSERIES = ["2.8", "2.0"] ACTIVERELEASES = ["2.16"] + LTSSERIES - YOCTO_MAPPING = { "2.18": "wrynose", "2.16": "whinlatter", - "2.12": "walnascar", - "2.10": "styhead", "2.8": "scarthgap", - "2.6": "nanbield", - "2.4": "mickledore", - "2.2": "langdale", "2.0": "kirkstone", - "1.52": "honister", - "1.50": "hardknott", - "1.48": "gatesgarth", - "1.46": "dunfell", } +RELEASES_FROM_JSON = {} + +# Use the local releases.json file if found, fetch it from the dashboard otherwise +releases_json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "releases.json") +try: + with open(releases_json_path, "r") as f: + RELEASES_FROM_JSON = json.load(f) +except FileNotFoundError: + print("Fetching releases.json from https://dashboard.yoctoproject.org/releases.json...", + file=sys.stderr) + try: + with urlopen("https://dashboard.yoctoproject.org/releases.json") as r, \ + open(releases_json_path, "w") as f: + RELEASES_FROM_JSON = json.load(r) + json.dump(RELEASES_FROM_JSON, f) + except URLError: + print("WARNING: tried to fetch https://dashboard.yoctoproject.org/releases.json " + "but failed, using default values for active releases", file=sys.stderr) + pass + +if RELEASES_FROM_JSON: + ACTIVERELEASES = [] + DEVBRANCH = "" + LTSSERIES = [] + YOCTO_MAPPING = {} + + for release in RELEASES_FROM_JSON: + bb_ver = release["bitbake_version"] + if release["status"] == "Active Development": + DEVBRANCH = bb_ver + if release["series"] == "current": + ACTIVERELEASES.append(bb_ver) + if "LTS until" in release["status"]: + LTSSERIES.append(bb_ver) + if release["bitbake_version"]: + YOCTO_MAPPING[bb_ver] = release["release_codename"] + + ACTIVERELEASES.remove(DEVBRANCH) + +print(f"ACTIVERELEASES calculated to be {ACTIVERELEASES}", file=sys.stderr) +print(f"DEVBRANCH calculated to be {DEVBRANCH}", file=sys.stderr) +print(f"LTSSERIES calculated to be {LTSSERIES}", file=sys.stderr) + BB_RELEASE_TAG_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") def get_current_version(): From 02cebba47b3d2b415c4aa8098a6af53da448f71e Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Tue, 17 Mar 2026 11:08:40 +0100 Subject: [PATCH 31/86] doc: generate the releases.rst file Use the information we gather from releases.json to automatically create the supported and outdated releases document. The older releases do not follow the same linking scheme, so add them statically at the bottom of the file from script. Signed-off-by: Antonin Godard Signed-off-by: Richard Purdie --- doc/.gitignore | 1 + doc/Makefile | 2 +- doc/conf.py | 2 + doc/releases.rst | 199 --------------------------------------------- doc/setversions.py | 191 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+), 200 deletions(-) delete mode 100644 doc/releases.rst diff --git a/doc/.gitignore b/doc/.gitignore index 1ee009c2012..3a570d8c655 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1,3 +1,4 @@ _build/ sphinx-static/switchers.js releases.json +releases.rst diff --git a/doc/Makefile b/doc/Makefile index 752f9b53b3e..2f8e898c4fc 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -27,7 +27,7 @@ publish: Makefile html singlehtml sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html clean: - @rm -rf $(BUILDDIR) sphinx-static/switchers.js releases.json + @rm -rf $(BUILDDIR) sphinx-static/switchers.js releases.json releases.rst # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/doc/conf.py b/doc/conf.py index 315fb7d06a7..9a53e55c938 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -39,6 +39,8 @@ "sphinx-static/switchers.js", current_version) +setversions.write_releases_rst("releases.rst") + # -- Project information ----------------------------------------------------- project = 'Bitbake' diff --git a/doc/releases.rst b/doc/releases.rst deleted file mode 100644 index a0b0ce766ea..00000000000 --- a/doc/releases.rst +++ /dev/null @@ -1,199 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-2.5 - -================================= -BitBake Supported Release Manuals -================================= - -******************************* -Release Series 5.3 (whinlatter) -******************************* - -- :yocto_docs:`BitBake 2.16 User Manual ` - -******************************* -Release Series 5.0 (scarthgap) -******************************* - -- :yocto_docs:`BitBake 2.8 User Manual ` - -****************************** -Release Series 4.0 (kirkstone) -****************************** - -- :yocto_docs:`BitBake 2.0 User Manual ` - -================================ -BitBake Outdated Release Manuals -================================ - -****************************** -Release Series 5.2 (walnascar) -****************************** - -- :yocto_docs:`BitBake 2.12 User Manual ` - -**************************** -Release Series 5.1 (styhead) -**************************** - -- :yocto_docs:`BitBake 2.10 User Manual ` - -******************************* -Release Series 4.3 (nanbield) -******************************* - -- :yocto_docs:`BitBake 2.6 User Manual ` - -******************************* -Release Series 4.2 (mickledore) -******************************* - -- :yocto_docs:`BitBake 2.4 User Manual ` - -***************************** -Release Series 4.1 (langdale) -***************************** - -- :yocto_docs:`BitBake 2.2 User Manual ` - -****************************** -Release Series 3.4 (honister) -****************************** - -- :yocto_docs:`BitBake 1.52 User Manual ` - -****************************** -Release Series 3.3 (hardknott) -****************************** - -- :yocto_docs:`BitBake 1.50 User Manual ` - -******************************* -Release Series 3.2 (gatesgarth) -******************************* - -- :yocto_docs:`BitBake 1.48 User Manual ` - -**************************** -Release Series 3.1 (dunfell) -**************************** - -- :yocto_docs:`BitBake 1.46 User Manual ` -- :yocto_docs:`3.1 BitBake User Manual ` -- :yocto_docs:`3.1.1 BitBake User Manual ` -- :yocto_docs:`3.1.2 BitBake User Manual ` -- :yocto_docs:`3.1.3 BitBake User Manual ` - -************************* -Release Series 3.0 (zeus) -************************* - -- :yocto_docs:`3.0 BitBake User Manual ` -- :yocto_docs:`3.0.1 BitBake User Manual ` -- :yocto_docs:`3.0.2 BitBake User Manual ` -- :yocto_docs:`3.0.3 BitBake User Manual ` -- :yocto_docs:`3.0.4 BitBake User Manual ` - -**************************** -Release Series 2.7 (warrior) -**************************** - -- :yocto_docs:`2.7 BitBake User Manual ` -- :yocto_docs:`2.7.1 BitBake User Manual ` -- :yocto_docs:`2.7.2 BitBake User Manual ` -- :yocto_docs:`2.7.3 BitBake User Manual ` -- :yocto_docs:`2.7.4 BitBake User Manual ` - -************************* -Release Series 2.6 (thud) -************************* - -- :yocto_docs:`2.6 BitBake User Manual ` -- :yocto_docs:`2.6.1 BitBake User Manual ` -- :yocto_docs:`2.6.2 BitBake User Manual ` -- :yocto_docs:`2.6.3 BitBake User Manual ` -- :yocto_docs:`2.6.4 BitBake User Manual ` - -************************* -Release Series 2.5 (sumo) -************************* - -- :yocto_docs:`2.5 Documentation ` -- :yocto_docs:`2.5.1 Documentation ` -- :yocto_docs:`2.5.2 Documentation ` -- :yocto_docs:`2.5.3 Documentation ` - -************************** -Release Series 2.4 (rocko) -************************** - -- :yocto_docs:`2.4 BitBake User Manual ` -- :yocto_docs:`2.4.1 BitBake User Manual ` -- :yocto_docs:`2.4.2 BitBake User Manual ` -- :yocto_docs:`2.4.3 BitBake User Manual ` -- :yocto_docs:`2.4.4 BitBake User Manual ` - -************************* -Release Series 2.3 (pyro) -************************* - -- :yocto_docs:`2.3 BitBake User Manual ` -- :yocto_docs:`2.3.1 BitBake User Manual ` -- :yocto_docs:`2.3.2 BitBake User Manual ` -- :yocto_docs:`2.3.3 BitBake User Manual ` -- :yocto_docs:`2.3.4 BitBake User Manual ` - -************************** -Release Series 2.2 (morty) -************************** - -- :yocto_docs:`2.2 BitBake User Manual ` -- :yocto_docs:`2.2.1 BitBake User Manual ` -- :yocto_docs:`2.2.2 BitBake User Manual ` -- :yocto_docs:`2.2.3 BitBake User Manual ` - -**************************** -Release Series 2.1 (krogoth) -**************************** - -- :yocto_docs:`2.1 BitBake User Manual ` -- :yocto_docs:`2.1.1 BitBake User Manual ` -- :yocto_docs:`2.1.2 BitBake User Manual ` -- :yocto_docs:`2.1.3 BitBake User Manual ` - -*************************** -Release Series 2.0 (jethro) -*************************** - -- :yocto_docs:`1.9 BitBake User Manual ` -- :yocto_docs:`2.0 BitBake User Manual ` -- :yocto_docs:`2.0.1 BitBake User Manual ` -- :yocto_docs:`2.0.2 BitBake User Manual ` -- :yocto_docs:`2.0.3 BitBake User Manual ` - -************************* -Release Series 1.8 (fido) -************************* - -- :yocto_docs:`1.8 BitBake User Manual ` -- :yocto_docs:`1.8.1 BitBake User Manual ` -- :yocto_docs:`1.8.2 BitBake User Manual ` - -************************** -Release Series 1.7 (dizzy) -************************** - -- :yocto_docs:`1.7 BitBake User Manual ` -- :yocto_docs:`1.7.1 BitBake User Manual ` -- :yocto_docs:`1.7.2 BitBake User Manual ` -- :yocto_docs:`1.7.3 BitBake User Manual ` - -************************** -Release Series 1.6 (daisy) -************************** - -- :yocto_docs:`1.6 BitBake User Manual ` -- :yocto_docs:`1.6.1 BitBake User Manual ` -- :yocto_docs:`1.6.2 BitBake User Manual ` -- :yocto_docs:`1.6.3 BitBake User Manual ` - diff --git a/doc/setversions.py b/doc/setversions.py index 6aa6ec9ee2e..008bae59f7f 100755 --- a/doc/setversions.py +++ b/doc/setversions.py @@ -19,6 +19,7 @@ import re import subprocess import sys +import textwrap from urllib.request import urlopen, URLError @@ -152,3 +153,193 @@ def write_switchers_js(js_in, js_out, current_version): else: w.write(line) print("switchers.js generated from switchers.js.in") + +def _release_section(series_version: str, codename: str, bitbake_version: str) -> str: + """ + Helper function to generate a release section, as: + + ******************** + Release Series xxxxx + ******************** + + - + """ + section_length = len(series_version) + len(codename) + 18 + return textwrap.dedent( + f"""\ + {'*' * section_length} + Release Series {series_version} ({codename}) + {'*' * section_length} + + - :yocto_docs:`BitBake {bitbake_version} User Manual ` + + """) + +def write_releases_rst(releases_rst_out: str): + """ + Generates the releases.rst file automatically, based on what is found + in the releases.json file. + """ + with open(releases_rst_out, "w") as f: + f.write(textwrap.dedent("""\ + .. SPDX-License-Identifier: CC-BY-2.5 + + ================================= + BitBake Supported Release Manuals + ================================= + + """)) + + for release in RELEASES_FROM_JSON: + if release["status"] == "Active Development": + continue + + if not release["bitbake_version"]: + continue + + if release["series"] == "current": + f.write(_release_section( + release["series_version"], + release["release_codename"], + release["bitbake_version"])) + + f.write(textwrap.dedent("""\ + ================================ + BitBake Outdated Release Manuals + ================================ + + """)) + + for release in RELEASES_FROM_JSON: + if not release["series"] == "previous": + continue + + if not release["bitbake_version"]: + continue + + f.write(_release_section( + release["series_version"], + release["release_codename"], + release["bitbake_version"])) + + # old legacy links, which cannot be auto-generated + f.write(textwrap.dedent( + """\ + - :yocto_docs:`3.1.2 BitBake User Manual ` + - :yocto_docs:`3.1 BitBake User Manual ` + - :yocto_docs:`3.1.1 BitBake User Manual ` + - :yocto_docs:`3.1.3 BitBake User Manual ` + + ************************* + Release Series 3.0 (Zeus) + ************************* + + - :yocto_docs:`3.0 BitBake User Manual ` + - :yocto_docs:`3.0.1 BitBake User Manual ` + - :yocto_docs:`3.0.2 BitBake User Manual ` + - :yocto_docs:`3.0.3 BitBake User Manual ` + - :yocto_docs:`3.0.4 BitBake User Manual ` + + **************************** + Release Series 2.7 (Warrior) + **************************** + + - :yocto_docs:`2.7 BitBake User Manual ` + - :yocto_docs:`2.7.1 BitBake User Manual ` + - :yocto_docs:`2.7.2 BitBake User Manual ` + - :yocto_docs:`2.7.3 BitBake User Manual ` + - :yocto_docs:`2.7.4 BitBake User Manual ` + + ************************* + Release Series 2.6 (Thud) + ************************* + + - :yocto_docs:`2.6 BitBake User Manual ` + - :yocto_docs:`2.6.1 BitBake User Manual ` + - :yocto_docs:`2.6.2 BitBake User Manual ` + - :yocto_docs:`2.6.3 BitBake User Manual ` + - :yocto_docs:`2.6.4 BitBake User Manual ` + + ************************* + Release Series 2.5 (Sumo) + ************************* + + - :yocto_docs:`2.5 Documentation ` + - :yocto_docs:`2.5.1 Documentation ` + - :yocto_docs:`2.5.2 Documentation ` + - :yocto_docs:`2.5.3 Documentation ` + + ************************** + Release Series 2.4 (Rocko) + ************************** + + - :yocto_docs:`2.4 BitBake User Manual ` + - :yocto_docs:`2.4.1 BitBake User Manual ` + - :yocto_docs:`2.4.2 BitBake User Manual ` + - :yocto_docs:`2.4.3 BitBake User Manual ` + - :yocto_docs:`2.4.4 BitBake User Manual ` + + ************************* + Release Series 2.3 (Pyro) + ************************* + + - :yocto_docs:`2.3 BitBake User Manual ` + - :yocto_docs:`2.3.1 BitBake User Manual ` + - :yocto_docs:`2.3.2 BitBake User Manual ` + - :yocto_docs:`2.3.3 BitBake User Manual ` + - :yocto_docs:`2.3.4 BitBake User Manual ` + + ************************** + Release Series 2.2 (Morty) + ************************** + + - :yocto_docs:`2.2 BitBake User Manual ` + - :yocto_docs:`2.2.1 BitBake User Manual ` + - :yocto_docs:`2.2.2 BitBake User Manual ` + - :yocto_docs:`2.2.3 BitBake User Manual ` + + **************************** + Release Series 2.1 (Krogoth) + **************************** + + - :yocto_docs:`2.1 BitBake User Manual ` + - :yocto_docs:`2.1.1 BitBake User Manual ` + - :yocto_docs:`2.1.2 BitBake User Manual ` + - :yocto_docs:`2.1.3 BitBake User Manual ` + + *************************** + Release Series 2.0 (Jethro) + *************************** + + - :yocto_docs:`1.9 BitBake User Manual ` + - :yocto_docs:`2.0 BitBake User Manual ` + - :yocto_docs:`2.0.1 BitBake User Manual ` + - :yocto_docs:`2.0.2 BitBake User Manual ` + - :yocto_docs:`2.0.3 BitBake User Manual ` + + ************************* + Release Series 1.8 (Fido) + ************************* + + - :yocto_docs:`1.8 BitBake User Manual ` + - :yocto_docs:`1.8.1 BitBake User Manual ` + - :yocto_docs:`1.8.2 BitBake User Manual ` + + ************************** + Release Series 1.7 (Dizzy) + ************************** + + - :yocto_docs:`1.7 BitBake User Manual ` + - :yocto_docs:`1.7.1 BitBake User Manual ` + - :yocto_docs:`1.7.2 BitBake User Manual ` + - :yocto_docs:`1.7.3 BitBake User Manual ` + + ************************** + Release Series 1.6 (Daisy) + ************************** + + - :yocto_docs:`1.6 BitBake User Manual ` + - :yocto_docs:`1.6.1 BitBake User Manual ` + - :yocto_docs:`1.6.2 BitBake User Manual ` + - :yocto_docs:`1.6.3 BitBake User Manual ` + """)) From 76b6edbd3b1f8d60c9e5720c8beae41caec694b7 Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Tue, 17 Mar 2026 11:08:38 +0100 Subject: [PATCH 32/86] doc/setversions.py: fix an f-string without placeholders This f-string does not have any placeholders. Turn it into a regular string. Signed-off-by: Antonin Godard Signed-off-by: Richard Purdie --- doc/setversions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/setversions.py b/doc/setversions.py index 008bae59f7f..82a0e8b191b 100755 --- a/doc/setversions.py +++ b/doc/setversions.py @@ -147,7 +147,7 @@ def write_switchers_js(js_in, js_out, current_version): for line in lines: if "VERSIONS_PLACEHOLDER" in line: if current_version != "dev": - w.write(f" 'dev': 'Unstable (dev)',\n") + w.write(" 'dev': 'Unstable (dev)',\n") for series in ACTIVERELEASES: w.write(f" '{series}': '{series} ({YOCTO_MAPPING[series]})',\n") else: From a631efbbdbb43d3427e82ae5e9f97ee19225c3ec Mon Sep 17 00:00:00 2001 From: Mathieu Dubois-Briand Date: Fri, 13 Mar 2026 16:12:12 +0100 Subject: [PATCH 33/86] taskexp_ncurses: update tests for new spdx tasks Some new tasks have been added to handled SPDX 3.0 [1], changing a bit the test output: some small changes are needed to account for these changes. [1]: https://lore.kernel.org/all/20260220154123.376880-1-JPEWhacker@gmail.com/ Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- lib/bb/ui/taskexp_ncurses.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/bb/ui/taskexp_ncurses.py b/lib/bb/ui/taskexp_ncurses.py index ea94a4987f9..551ad142a2a 100755 --- a/lib/bb/ui/taskexp_ncurses.py +++ b/lib/bb/ui/taskexp_ncurses.py @@ -146,6 +146,9 @@ def debug_frame(nbox_ojb): unit_test = os.environ.get('TASK_EXP_UNIT_TEST') unit_test_cmnds=[ '# Default selected task in primary box', + 'tst_selected=.do_create_recipe_spdx', + '# Move to next entry, more predictable', + 'tst_entry=', # optional injected error 'tst_selected=.do_recipe_qa', '# Default selected task in deps', 'tst_entry=', From e7d5e156275782948d3346f299780389ab263ab6 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 19 Mar 2026 12:07:41 +0000 Subject: [PATCH 34/86] fetch/git: Add an 'update' unpack mode to the fetchers (git only for now) We need a way to allow the git fetcher to update data in place rather than remove and replace. This change adds an unpack_update() function which can be used in place of the usual unpack() call. This will attempt to rebase changes on top of the upstream changes. It will raise an error if any local uncommitted changes are present. The implementation adds a "dldir" origin to the list of origins in the git repo which we can then fetch from and update against. This origin may be of use to users accessing the git repo outside the fetcher too. unpack_update() should never delete existing data in the way unpack() does but can error out in many more different ways due to the number of possible input states. Currently only git is supported. The intention is for this to use used by bitbake-setup instead of unpack. Signed-off-by: Richard Purdie --- lib/bb/fetch2/__init__.py | 13 ++++++++-- lib/bb/fetch2/git.py | 51 +++++++++++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py index aaefd860204..909ae98146e 100644 --- a/lib/bb/fetch2/__init__.py +++ b/lib/bb/fetch2/__init__.py @@ -1634,6 +1634,9 @@ def unpack(self, urldata, rootdir, data): return + def unpack_update(self, urldata, rootdir, data): + raise RuntimeError("No method available for this url type: %s" % urldata.type) + def clean(self, urldata, d): """ Clean any existing full or partial download @@ -1990,7 +1993,7 @@ def checkstatus(self, urls=None): if not ret: raise FetchError("URL doesn't work", u) - def unpack(self, root, urls=None): + def unpack(self, root, urls=None, update=False): """ Unpack urls to root """ @@ -2009,7 +2012,10 @@ def unpack(self, root, urls=None): lf = bb.utils.lockfile(ud.lockfile) unpack_tracer.start_url(u) - ud.method.unpack(ud, root, self.d) + if update: + ud.method.unpack_update(ud, root, self.d) + else: + ud.method.unpack(ud, root, self.d) unpack_tracer.finish_url(u) finally: @@ -2018,6 +2024,9 @@ def unpack(self, root, urls=None): unpack_tracer.complete() + def unpack_update(self, root, urls=None): + self.unpack(root, urls, update=True) + def clean(self, urls=None): """ Clean files that the fetcher gets or places diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 738174cd104..84fecdad07e 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -656,7 +656,10 @@ def clone_shallow_local(self, ud, dest, d): # The url is local ud.clonedir, set it to upstream one runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest) - def unpack(self, ud, destdir, d): + def unpack_update(self, ud, destdir, d): + return self.unpack(ud, destdir, d, update=True) + + def unpack(self, ud, destdir, d, update=False): """ unpack the downloaded src to destdir""" subdir = ud.parm.get("subdir") @@ -680,7 +683,7 @@ def unpack(self, ud, destdir, d): destsuffix = ud.parm.get("destsuffix", def_destsuffix) destdir = ud.destdir = os.path.join(destdir, destsuffix) - if os.path.exists(destdir): + if os.path.exists(destdir) and not update: bb.utils.prunedir(destdir) if not ud.bareclone: ud.unpack_tracer.unpack("git", destdir) @@ -691,11 +694,15 @@ def unpack(self, ud, destdir, d): ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd source_found = False + update_mode = False source_error = [] clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) if clonedir_is_up_to_date: - runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) + if update and os.path.exists(destdir): + update_mode = True + else: + runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) source_found = True else: source_error.append("clone directory not available or not up to date: " + ud.clonedir) @@ -703,8 +710,11 @@ def unpack(self, ud, destdir, d): if not source_found: if ud.shallow: if os.path.exists(ud.fullshallow): - bb.utils.mkdirhier(destdir) - runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir) + if update and os.path.exists(destdir): + update_mode = True + else: + bb.utils.mkdirhier(destdir) + runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir) source_found = True else: source_error.append("shallow clone not available: " + ud.fullshallow) @@ -714,6 +724,32 @@ def unpack(self, ud, destdir, d): if not source_found: raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) + if update_mode: + if ud.shallow: + raise bb.fetch2.UnpackError("Can't update shallow clones checkouts without network access, not supported.", ud.url) + + output = runfetchcmd("%s status --untracked-files=no --porcelain" % (ud.basecmd), d, workdir=destdir) + if output: + raise bb.fetch2.UnpackError("Repository at %s has uncommitted changes, unable to update:\n%s" % (destdir, output), ud.url) + + # Set up remote for the download location if it doesn't exist + try: + runfetchcmd("%s remote get-url dldir" % (ud.basecmd), d, workdir=destdir) + except bb.fetch2.FetchError: + if ud.clonedir: + runfetchcmd("%s remote add dldir file://%s" % (ud.basecmd, ud.clonedir), d, workdir=destdir) + try: + runfetchcmd("%s fetch dldir" % (ud.basecmd), d, workdir=destdir) + runfetchcmd("%s rebase --no-autosquash --no-autostash %s" % (ud.basecmd, ud.revision), d, workdir=destdir) + except bb.fetch2.FetchError as e: + # If rebase failed, abort it + try: + runfetchcmd("%s rebase --abort" % (ud.basecmd), d, workdir=destdir) + except Exception: + pass + raise bb.fetch2.UnpackError("Failed to update checkout in place: %s" % str(e), ud.url) + return True + # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag # matches the revision if 'tag' in ud.parm and sha1_re.match(ud.revision): @@ -729,6 +765,11 @@ def unpack(self, ud, destdir, d): repourl = self._get_repo_url(ud) runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) + if ud.clonedir: + try: + runfetchcmd("%s remote get-url dldir" % (ud.basecmd), d, workdir=destdir) + except bb.fetch2.FetchError: + runfetchcmd("%s remote add dldir file://%s" % (ud.basecmd, ud.clonedir), d, workdir=destdir) if self._contains_lfs(ud, d, destdir): if not need_lfs: From 59b927fd1bc610fdeccac7078de557af1ca89caf Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Fri, 20 Mar 2026 13:00:40 +0100 Subject: [PATCH 35/86] bitbake-setup: use non-destructive 'update' mode of the git fetcher This allows removing the git hook that prohibited making commits in repositories under layers/, significantly improving user experience. bitbake-setup logic is adjusted to account for: - if a git checkout is being replaced with a local symlink, that checkout is moved to a backup location, regardless of whether it has local changes or not The tests are modified so that: - the check for layer backups is run earlier, and check that backups happen when they're supposed to (which should be just once). - the test that previously checked that local commits are rejected is adjusted to check that such a commit is accepted, and layer is updated from remote without creating a backup directory. - a test is added to force a rebase conflict on update, and check that it indeed happens. We can think of better ways to handle rebase conflicts in bitbake-setup later on; right now it simply stops at that point. Signed-off-by: Alexander Kanavin Signed-off-by: Richard Purdie --- bin/bitbake-setup | 40 ++-------------------------------------- lib/bb/tests/setup.py | 42 +++++++++++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 53 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 53dbb622136..46d31d8d1bf 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -191,39 +191,6 @@ def checkout_layers(layers, confdir, layerdir, d): logger.plain("Making a symbolic link {} pointing to {}".format(dst, src)) os.symlink(src, dst) - def _has_local_modifications(r_name, r_path): - fixed_revisions = json.load(open(os.path.join(confdir, "sources-fixed-revisions.json"))) - - if not r_name in fixed_revisions['sources']: - logger.warning("""Source {} is added with path {}. -This path already exists, but it has no entry in a fixed revisions -record. To ensure possible local modifications are not lost, it will -be preserved in a backup directory.""".format(r_name, r_path)) - return True - - rev = fixed_revisions['sources'][r_name]['git-remote']['rev'] - status = bb.process.run('git -C {} status --porcelain'.format(r_path))[0] - if status: - return True - diff = bb.process.run('git -C {} diff {} {}'.format(r_path, get_diff_color_param(), rev))[0] - if diff: - return True - return False - - def _restrict_commits(r_name, r_path): - hook_path = os.path.join(r_path, '.git', 'hooks', 'pre-commit') - restrict_hook = """#!/bin/sh -echo "This repository is managed by bitbake-setup, and making commits is restricted. -If you wish to make local modifications, clone it separately, and re-initialize using -bitbake-setup init -L {} /path/to/repo/checkout" -exit 1 -""".format(r_name) - with open(hook_path, 'w') as f: - f.write(restrict_hook) - import stat - st = os.stat(hook_path) - os.chmod(hook_path, st.st_mode | stat.S_IEXEC) - layers_fixed_revisions = copy.deepcopy(layers) repodirs = [] oesetupbuild = None @@ -242,19 +209,16 @@ exit 1 if os.path.lexists(repodir_path): if os.path.islink(repodir_path): os.remove(repodir_path) - elif _has_local_modifications(r_name, repodir_path): + elif r_local: backup_path = add_unique_timestamp_to_path(repodir_path + '-backup') logger.warning("""Source {} in {} contains local modifications. Renaming to {} to preserve them. For local development work it is recommended to clone the needed layers separately and re-initialize using -L option: bitbake-setup init -L {} /path/to/repo/checkout""".format( r_name, repodir_path, backup_path, r_name)) os.rename(repodir_path, backup_path) - else: - shutil.rmtree(repodir_path) if r_remote: _checkout_git_remote(r_remote, repodir, layers_fixed_revisions) - _restrict_commits(r_name, repodir_path) if r_local: _symlink_local(os.path.expanduser(r_local["path"]), repodir_path) @@ -830,7 +794,7 @@ def do_fetch(fetcher, dir): oldstdout = sys.stdout sys.stdout = f fetcher.download() - fetcher.unpack(dir) + fetcher.unpack_update(dir) sys.stdout = oldstdout def update_registry(registry, cachedir, d): diff --git a/lib/bb/tests/setup.py b/lib/bb/tests/setup.py index e3557d8f3b9..a66f05b36ba 100644 --- a/lib/bb/tests/setup.py +++ b/lib/bb/tests/setup.py @@ -447,16 +447,27 @@ def _check_local_sources(custom_setup_dir): self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path)) self.config_is_unchanged(custom_setup_path) + def _check_layer_backups(layer_path, expected_backups): + files = os.listdir(layer_path) + backups = len([f for f in files if 'backup' in f]) + self.assertEqual(backups, expected_backups, msg = "Expected {} layer backups, got {}, directory listing: {}".format(expected_backups, backups, files)) + # Change the configuration to refer to a local source, then to another local source, then back to a git remote # Run status/update after each change and verify that nothing breaks + # Also check that layer backups happen when expected c = 'gadget' setuppath = self.get_setup_path('test-config-1', c) self.config_is_unchanged(setuppath) + layers_path = os.path.join(setuppath, 'layers') + layer_path = os.path.join(layers_path, 'test-repo') + _check_layer_backups(layers_path, 0) + json_1 = self.add_local_json_config_to_registry('test-config-1.conf.json', self.testrepopath) os.environ['BBPATH'] = os.path.join(setuppath, 'build') out = self.runbbsetup("update --update-bb-conf='yes'") _check_local_sources(setuppath) + _check_layer_backups(layers_path, 1) prev_path = self.testrepopath self.testrepopath = prev_path + "-2" @@ -465,23 +476,14 @@ def _check_local_sources(custom_setup_dir): os.environ['BBPATH'] = os.path.join(setuppath, 'build') out = self.runbbsetup("update --update-bb-conf='yes'") _check_local_sources(setuppath) + _check_layer_backups(layers_path, 1) self.testrepopath = prev_path json_1 = self.add_json_config_to_registry('test-config-1.conf.json', branch, branch) os.environ['BBPATH'] = os.path.join(setuppath, 'build') out = self.runbbsetup("update --update-bb-conf='yes'") self.check_setupdir_files(setuppath, test_file_content) - - # Also check that there are no layer backups up to this point, then make a change that should - # result in a layer backup, and check that it does happen. - def _check_layer_backups(layer_path, expected_backups): - files = os.listdir(layer_path) - backups = len([f for f in files if 'backup' in f]) - self.assertEqual(backups, expected_backups, msg = "Expected {} layer backups, got {}, directory listing: {}".format(expected_backups, backups, files)) - - layers_path = os.path.join(setuppath, 'layers') - layer_path = os.path.join(layers_path, 'test-repo') - _check_layer_backups(layers_path, 0) + _check_layer_backups(layers_path, 1) ## edit a file without making a commit with open(os.path.join(layer_path, 'local-modification'), 'w') as f: @@ -492,16 +494,26 @@ def _check_layer_backups(layer_path, expected_backups): out = self.runbbsetup("update --update-bb-conf='yes'") _check_layer_backups(layers_path, 1) - ## edit a file and try to make a commit; this should be rejected + ## edit a file and make a commit such that no rebase conflicts occur with open(os.path.join(layer_path, 'local-modification'), 'w') as f: f.write('locally-modified-again\n') self.git('add .', cwd=layer_path) - with self.assertRaisesRegex(bb.process.ExecutionError, "making commits is restricted"): - self.git('commit -m "Adding a local modification"', cwd=layer_path) + self.git('commit -m "Adding a local modification"', cwd=layer_path) test_file_content = "modified-again-and-again\n" self.add_file_to_testrepo('test-file', test_file_content) out = self.runbbsetup("update --update-bb-conf='yes'") - _check_layer_backups(layers_path, 2) + _check_layer_backups(layers_path, 1) + + ## edit a file and make a commit in a way that causes a rebase conflict + with open(os.path.join(layer_path, 'test-file'), 'w') as f: + f.write('locally-modified\n') + self.git('add .', cwd=layer_path) + self.git('commit -m "Adding a local modification"', cwd=layer_path) + test_file_content = "remotely-modified\n" + self.add_file_to_testrepo('test-file', test_file_content) + with self.assertRaisesRegex(bb.process.ExecutionError, "Merge conflict in test-file"): + out = self.runbbsetup("update --update-bb-conf='yes'") + _check_layer_backups(layers_path, 1) # check source overrides, local sources provided with symlinks, and custom setup dir name source_override_content = """ From a70c336790a9188aae67975fac6ca13579ad1d3e Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Thu, 29 Jan 2026 13:04:21 +0100 Subject: [PATCH 36/86] bitbake-setup: share sstate by default between builds Nowadays sharing sstate must also include sharing the hash equivalency information and thus, managing a hash equivalency server. There are two ways to do it: - starting/stopping the server outside the bitbake invocations, and guaranteeing that it's available when bitbake is invoked. - using bitbake's built-in start/stop code which launches a server before a build starts and stops it when a build is finished; essentially this is a private server, using a database private to a build directory (by default). I couldn't come up with a good way to do the first option in bitbake-setup: it needs to be invisible to users, they should not have to run special commands and they should not wonder why there is a mysterious background process. It's not impossible to auto-start a shared server, but that will quickly run into synchronization issues: if one server is being started, another should not be started at the same time. If one server is shutting down (e.g. after an inactivity timeout), another starting server should wait until it frees the socket, and block all bitbake invocations on that. Memory resident bitbake does this in lib/bb/server/process.py with a lot of complexity, and I don't think it should be added to the hash server as well. On the other hand, hash equivalency database is sqlite-driven, and sqlite documentation reassures that sharing it between different simultaneous processes is okay: nothing will get lost or corrupted: https://sqlite.org/faq.html#q5 I've confirmed this by running simultaneous builds that way: nothing unusual happened, and sstate was shared as it's supposed to. There's a new setting that turns off this behavior for situations where the server and sstate are managed externally. Signed-off-by: Alexander Kanavin Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- bin/bitbake-setup | 28 ++++++++++++++++++- .../bitbake-user-manual-environment-setup.rst | 19 +++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 46d31d8d1bf..dcad9c16968 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -898,6 +898,31 @@ def create_siteconf(top_dir, non_interactive, settings): os.makedirs(top_dir, exist_ok=True) with open(siteconfpath, 'w') as siteconffile: + sstate_settings = textwrap.dedent( + """ + # + # Where to place shared-state files + # + # BitBake has the capability to accelerate builds based on previously built output. + # This is done using "shared state" files which can be thought of as cache objects + # and this option determines where those files are placed. + # + # You can wipe out TMPDIR leaving this directory intact and the build would regenerate + # from these files if no changes were made to the configuration. If changes were made + # to the configuration, only shared state files where the state was still valid would + # be used (done using checksums). + SSTATE_DIR ?= "{sstate_dir}" + # + # Hash Equivalence database location + # + # Hash equivalence improves reuse of sstate by detecting when a given sstate + # artifact can be reused as equivalent, even if the current task hash doesn't + # match the one that generated the artifact. This variable controls where the + # Hash Equivalence database ("hashserv.db") is stored and can be shared between + # concurrent builds. + BB_HASHSERVE_DB_DIR ?= "${{SSTATE_DIR}}" + """.format(sstate_dir=os.path.join(top_dir, ".sstate-cache")) + ) siteconffile.write( textwrap.dedent( """\ @@ -915,7 +940,7 @@ def create_siteconf(top_dir, non_interactive, settings): """.format( dl_dir=settings["default"]["dl-dir"], ) - ) + ) + (sstate_settings if settings["default"]["common-sstate"] == 'yes' else "") ) @@ -1126,6 +1151,7 @@ def main(): 'top-dir-name':'bitbake-builds', 'registry':get_default_registry(), 'use-full-setup-dir-name':'no', + 'common-sstate':'yes', } global_settings = load_settings(global_settings_path(args)) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst b/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst index f7580c8e0e6..37518fcebb3 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst @@ -516,6 +516,7 @@ A valid settings file would for example be: registry = /path/to/bitbake/default-registry dl-dir = /path/to/bitbake-setup-downloads use-full-setup-dir-name = yes + common-sstate = yes Settings and their values can be listed and modified with the ``bitbake-setup settings`` command. See the :ref:`ref-bbsetup-command-settings` section for @@ -620,6 +621,24 @@ will override the suggestions for the :term:`Setup` directory name made by will make the directory names longer, but fully specific: they will contain all selections made during initialization. +.. _ref-bbsetup-setting-common-sstate: + +``common-sstate`` +----------------- + +When this setting is set to ``yes`` (which is also the default), bitbake-setup will +set up a common sstate directory and common hash equivalency database for all the +:term:`setups ` in a :term:`Top Directory`. This is very beneficial for speeding +up builds as build artefacts will be reused whenever possible between them. + +Set this to ``no`` for advanced use cases, such as placing the sstate directory on a NFS +mount and maintaining a separate hash equivalency server, so that sstate and hash equivalency +data can be shared between several computers. For such use cases the sstate settings need +to be added to a build configuration separately. + +See https://docs.yoctoproject.org/dev-manual/hashequivserver.html for how to share sstate +on the network. + .. _ref-bbsetup-section-config-reference: Configuration Template Files Reference From 112bddd8fc684fbdd71139429251b127739f863b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Sommer?= Date: Wed, 18 Mar 2026 15:16:32 +0100 Subject: [PATCH 37/86] data: Add exception details if build_dependencies catches one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this information, you'll only get messages like this, which don't indicate what might be wrong: WARNING: 
.bb: Exception during build_dependencies for do_patch:00:17 WARNING: 
.bb: Error during finalise of 
.bb ERROR: ParseError in None: Not all recipes parsed, parser thread killed/died? Exiting. ERROR: Parsing halted due to errors, see error messages above Signed-off-by: Jörg Sommer Signed-off-by: Richard Purdie --- lib/bb/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bb/data.py b/lib/bb/data.py index 061e63386f0..5fdcdb04a6a 100644 --- a/lib/bb/data.py +++ b/lib/bb/data.py @@ -367,7 +367,7 @@ def handle_remove(value, deps, removes, d): except bb.parse.SkipRecipe: raise except Exception as e: - bb.warn("Exception during build_dependencies for %s" % key) + bb.warn("Exception during build_dependencies for %s: %s" % (key, repr(e))) raise return frozenset(deps), value #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) From e827db50d8f61f0d04c9bd0753c05aeb1ed0715f Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Tue, 24 Mar 2026 21:37:25 +0000 Subject: [PATCH 38/86] progressbar/knotty: Allow mixing log messages and progress bars If we try and print a log message in the middle of progress bar, the display can be corrupted. Add a clear() method to progress bars allowing them to be removed for the log message, then reprinted using update(). Signed-off-by: Richard Purdie --- lib/bb/ui/knotty.py | 10 ++++++++++ lib/progressbar/progressbar.py | 3 +++ 2 files changed, 13 insertions(+) diff --git a/lib/bb/ui/knotty.py b/lib/bb/ui/knotty.py index e4b7b83061e..82531ef8f31 100644 --- a/lib/bb/ui/knotty.py +++ b/lib/bb/ui/knotty.py @@ -87,6 +87,7 @@ class NonInteractiveProgress(object): def __init__(self, msg, maxval): self.id = msg self.msg = msg + self.currval = 0 self.maxval = maxval self.finished = False @@ -96,6 +97,10 @@ def start(self, update=True): return self def update(self, value): + self.currval = value + pass + + def clear(self): pass def finish(self): @@ -791,7 +796,12 @@ def main(server, eventHandler, params, tf = TerminalFilter): event.msg = taskinfo['title'] + ': ' + event.msg if hasattr(event, 'fn') and event.levelno not in [bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]: event.msg = event.fn + ': ' + event.msg + # Need to remove any progress bar, then add it back after we print this message + if parseprogress: + parseprogress.clear() logging.getLogger(event.name).handle(event) + if parseprogress: + parseprogress.update(parseprogress.currval) continue if isinstance(event, bb.build.TaskFailedSilent): diff --git a/lib/progressbar/progressbar.py b/lib/progressbar/progressbar.py index a8e2dc09c91..1562774ba1a 100644 --- a/lib/progressbar/progressbar.py +++ b/lib/progressbar/progressbar.py @@ -278,6 +278,9 @@ def update(self, value=None): self.last_update_time = now return output + def clear(self): + self.fd.write(" " * self.term_width + '\r') + self.fd.flush() def start(self, update=True): """Starts measuring time, and prints the bar at 0%. From 05a0ca339a6fee75f7979c3ee95fc91b2bd9b92f Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Wed, 25 Mar 2026 12:49:38 +0000 Subject: [PATCH 39/86] toaster: Update fixtures for wrynose release Update the fixtures for toaster to add wrynose. Update the tests to match. Signed-off-by: Richard Purdie --- lib/toaster/orm/fixtures/gen_fixtures.py | 8 +-- lib/toaster/orm/fixtures/oe-core.xml | 49 ++++++------- lib/toaster/orm/fixtures/poky.xml | 72 +++++++++---------- .../functional/test_create_new_project.py | 25 +++++-- 4 files changed, 86 insertions(+), 68 deletions(-) diff --git a/lib/toaster/orm/fixtures/gen_fixtures.py b/lib/toaster/orm/fixtures/gen_fixtures.py index 4b87ba0baab..9488ce852e3 100755 --- a/lib/toaster/orm/fixtures/gen_fixtures.py +++ b/lib/toaster/orm/fixtures/gen_fixtures.py @@ -35,19 +35,19 @@ # [0=Codename, 1=Yocto Project Version, 2=Release Date, 3=Current Version, 4=Support Level, 5=Poky Version, 6=BitBake branch] current_releases = [ # Release slot #1 - ['Scarthgap','5.0','April 2024','5.0.0 (April 2024)','Long Term Support (until April 2028)','','2.8'], + ['Wrynose','6.0','April 2026','6.0 (April 2026)','Long Term Support (until April 2030)','','2.16'], # Release slot #2 'local' ['HEAD','HEAD','','Local Yocto Project','HEAD','','HEAD'], # Release slot #3 'master' ['Master','master','','Yocto Project master','master','','master'], # Release slot #4 - ['Whinlatter','5.3','October 2025','5.3.0 (October 2024)','Support for 7 months (until May 2026)','','2.14'], - ['Walnascar','5.2','April 2025','5.2.0 (April 2025)','Support for 7 months (until October 2025)','','2.12'], + ['Scarthgap','5.0','April 2024','5.0 (April 2024)','Long Term Support (until April 2028)','','2.8'], #['Styhead','5.1','November 2024','5.1.0 (November 2024)','Support for 7 months (until May 2025)','','2.10'], #['Nanbield','4.3','November 2023','4.3.0 (November 2023)','Support for 7 months (until May 2024)','','2.6'], #['Mickledore','4.2','April 2023','4.2.0 (April 2023)','Support for 7 months (until October 2023)','','2.4'], #['Langdale','4.1','October 2022','4.1.2 (January 2023)','Support for 7 months (until May 2023)','','2.2'], - ['Kirkstone','4.0','April 2022','4.0.8 (March 2023)','Stable - Long Term Support (until Apr. 2024)','','2.0'], + ['Kirkstone','4.0','April 2022','4.0 (March 2023)','Stable - Long Term Support (until Apr. 2024)','','2.0'], + ['Whinlatter','5.3','October 2025','5.3 (October 2024)','Support for 7 months (until May 2026)','','2.14'], #['Honister','3.4','October 2021','3.4.2 (February 2022)','Support for 7 months (until May 2022)','26.0','1.52'], #['Hardknott','3.3','April 2021','3.3.5 (March 2022)','Stable - Support for 13 months (until Apr. 2022)','25.0','1.50'], #['Gatesgarth','3.2','Oct 2020','3.2.4 (May 2021)','EOL','24.0','1.48'], diff --git a/lib/toaster/orm/fixtures/oe-core.xml b/lib/toaster/orm/fixtures/oe-core.xml index a83b18844b3..01f55ffda8e 100644 --- a/lib/toaster/orm/fixtures/oe-core.xml +++ b/lib/toaster/orm/fixtures/oe-core.xml @@ -8,9 +8,9 @@ - scarthgap + wrynose git://git.openembedded.org/bitbake - 2.8 + 2.16 HEAD @@ -23,28 +23,28 @@ master - whinlatter + scarthgap git://git.openembedded.org/bitbake - 2.14 + 2.8 - walnascar + kirkstone git://git.openembedded.org/bitbake - 2.12 + 2.0 - kirkstone + whinlatter git://git.openembedded.org/bitbake - 2.0 + 2.14 - scarthgap - Openembedded Scarthgap + wrynose + Openembedded Wrynose 1 - scarthgap - Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=scarthgap\">OpenEmbedded Scarthgap</a> branch. + wrynose + Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=wrynose\">OpenEmbedded Wrynose</a> branch. local @@ -61,26 +61,26 @@ Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/\">OpenEmbedded master</a> branch. - whinlatter - Openembedded Whinlatter + scarthgap + Openembedded Scarthgap 4 - whinlatter - Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=whinlatter\">OpenEmbedded Whinlatter</a> branch. + scarthgap + Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=scarthgap\">OpenEmbedded Scarthgap</a> branch. - walnascar - Openembedded Walnascar - 5 - walnascar - Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=walnascar\">OpenEmbedded Walnascar</a> branch. - - kirkstone Openembedded Kirkstone - 6 + 5 kirkstone Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=kirkstone\">OpenEmbedded Kirkstone</a> branch. + + whinlatter + Openembedded Whinlatter + 6 + whinlatter + Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=whinlatter\">OpenEmbedded Whinlatter</a> branch. + @@ -108,6 +108,7 @@ openembedded-core + openembedded-core diff --git a/lib/toaster/orm/fixtures/poky.xml b/lib/toaster/orm/fixtures/poky.xml index a8a5a7aeb08..3648170bb26 100644 --- a/lib/toaster/orm/fixtures/poky.xml +++ b/lib/toaster/orm/fixtures/poky.xml @@ -8,9 +8,9 @@ - scarthgap + wrynose git://git.openembedded.org/bitbake - 2.8 + 2.16 @@ -26,32 +26,32 @@ - whinlatter + scarthgap git://git.openembedded.org/bitbake - 2.14 + 2.8 - walnascar + kirkstone git://git.openembedded.org/bitbake - 2.12 + 2.0 - kirkstone + whinlatter git://git.openembedded.org/bitbake - 2.0 + 2.14 - scarthgap - Yocto Project 5.0 "Scarthgap" + wrynose + Yocto Project 6.0 "Wrynose" 1 - scarthgap - Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=scarthgap">Yocto Project Scarthgap branch</a>. + wrynose + Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=wrynose">Yocto Project Wrynose branch</a>. local @@ -68,26 +68,26 @@ Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/">Yocto Project Master branch</a>. - whinlatter - Yocto Project 5.3 "Whinlatter" + scarthgap + Yocto Project 5.0 "Scarthgap" 4 - whinlatter - Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=whinlatter">Yocto Project Whinlatter branch</a>. + scarthgap + Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=scarthgap">Yocto Project Scarthgap branch</a>. - walnascar - Yocto Project 5.2 "Walnascar" - 5 - walnascar - Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=walnascar">Yocto Project Walnascar branch</a>. - - kirkstone Yocto Project 4.0 "Kirkstone" - 6 + 5 kirkstone Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=kirkstone">Yocto Project Kirkstone branch</a>. + + whinlatter + Yocto Project 5.3 "Whinlatter" + 6 + whinlatter + Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=whinlatter">Yocto Project Whinlatter branch</a>. + @@ -180,7 +180,7 @@ 1 0 1 - scarthgap + wrynose meta @@ -202,21 +202,21 @@ 1 0 4 - whinlatter + scarthgap meta 1 0 5 - walnascar + kirkstone meta 1 0 6 - kirkstone + whinlatter meta @@ -232,7 +232,7 @@ 2 0 1 - scarthgap + wrynose meta-poky @@ -254,21 +254,21 @@ 2 0 4 - whinlatter + scarthgap meta-poky 2 0 5 - walnascar + kirkstone meta-poky 2 0 6 - kirkstone + whinlatter meta-poky @@ -284,7 +284,7 @@ 3 0 1 - scarthgap + wrynose meta-yocto-bsp @@ -306,21 +306,21 @@ 3 0 4 - whinlatter + scarthgap meta-yocto-bsp 3 0 5 - walnascar + kirkstone meta-yocto-bsp 3 0 6 - kirkstone + whinlatter meta-yocto-bsp diff --git a/lib/toaster/tests/functional/test_create_new_project.py b/lib/toaster/tests/functional/test_create_new_project.py index 51c8c120370..5de3c06f123 100644 --- a/lib/toaster/tests/functional/test_create_new_project.py +++ b/lib/toaster/tests/functional/test_create_new_project.py @@ -31,13 +31,30 @@ def test_create_new_project_master(self): False, ) - def test_create_new_project_scarthgap(self): + def test_create_new_project_wrynose(self): """ Test create new project using: - Project Name: Any string - - Release: Yocto Project 5.0 "Scarthgap" (option value: 1) + - Release: Yocto Project 6.0 "Wrynose" (option value: 1) - Merge Toaster settings: True """ release = '1' + release_title = 'Yocto Project 6.0 "Wrynose"' + project_name = 'projectwrynose' + self.create_new_project( + project_name, + release, + release_title, + True, + ) + + + def test_create_new_project_scarthgap(self): + """ Test create new project using: + - Project Name: Any string + - Release: Yocto Project 5.0 "Scarthgap" (option value: 4) + - Merge Toaster settings: True + """ + release = '4' release_title = 'Yocto Project 5.0 "Scarthgap"' project_name = 'projectscarthgap' self.create_new_project( @@ -50,10 +67,10 @@ def test_create_new_project_scarthgap(self): def test_create_new_project_kirkstone(self): """ Test create new project using: - Project Name: Any string - - Release: Yocto Project 4.0 "Kirkstone" (option value: 6) + - Release: Yocto Project 4.0 "Kirkstone" (option value: 5) - Merge Toaster settings: True """ - release = '6' + release = '5' release_title = 'Yocto Project 4.0 "Kirkstone"' project_name = 'projectkirkstone' self.create_new_project( From 64fda8cb8efaff280d33bf60d93fed6e37244640 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 26 Mar 2026 15:55:15 +0000 Subject: [PATCH 40/86] fetch/git: Update packing commands to match modern git We are using a git command, "git pack-redundant" which is due to be removed from git. There is discussion about bitbake's usage here: https://lore.kernel.org/git/CANYiYbHOczhRoJkOofFNQ6VA3BiyOF=QjffFBDvLn43Ts8B67w@mail.gmail.com/T/#t As mentioned, we should switch to use git repack -adk. We need the k option since we do need to keep unreachable object to retain history. As per the manuals, we still need to run prune-packed to remove loose objects which are now in the pack files. Signed-off-by: Richard Purdie --- lib/bb/fetch2/git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 84fecdad07e..6457463402c 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -475,9 +475,9 @@ def download(self, ud, d): bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir) - runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir) + runfetchcmd("%s repack -adk" % ud.basecmd, d, workdir=ud.clonedir) runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir) - runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir) + runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir) try: os.unlink(ud.fullmirror) except OSError as exc: From 0ac8ea4565b52571e79fc298ebd8c1a3ec889c3d Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Thu, 26 Mar 2026 10:11:14 +0100 Subject: [PATCH 41/86] doc: add warning notes on the disabled NPM fetcher Add warning notes in bitbake-user-manual-fetching.rst and bitbake-user-manual-ref-variables.rst regarding the disabled NPM fetcher. Signed-off-by: Antonin Godard Signed-off-by: Richard Purdie --- .../bitbake-user-manual-fetching.rst | 12 ++++++++++++ .../bitbake-user-manual-ref-variables.rst | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index 1dcdc0ffee1..6af80359125 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -680,6 +680,12 @@ Here is an example URL:: NPM Fetcher (``npm://``) ------------------------ +.. warning:: + + The NPM fetcher is currently disabled due to security concerns. See + `355cd226e0720a9ed7683bb01c8c0a58eee03664 `__ + for more information. + This submodule fetches source code from an `NPM `__ Javascript package registry. @@ -719,6 +725,12 @@ to automatically create a recipe from an NPM URL. NPM shrinkwrap Fetcher (``npmsw://``) ------------------------------------- +.. warning:: + + The NPM fetcher is currently disabled due to security concerns. See + `355cd226e0720a9ed7683bb01c8c0a58eee03664 `__ + for more information. + This submodule fetches source code from an `NPM shrinkwrap `__ description file, which lists the dependencies diff --git a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index 06bd536195c..8d8e8b8b912 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -1738,6 +1738,12 @@ overview of their function and contents. - ``npm://``: Fetches JavaScript modules from a registry. + .. warning:: + + The NPM fetcher is currently disabled due to security concerns. See + `355cd226e0720a9ed7683bb01c8c0a58eee03664 `__ + for more information. + - ``p4://``: Fetches files from a Perforce (``p4``) revision control repository. From ee4fc97b6d382fbc78117c57d1d89da05655b63c Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Thu, 26 Mar 2026 11:57:32 -0700 Subject: [PATCH 42/86] bitbake-setup: Remove extraneous variable from str.format() The invocation of str.format() supplied an extra parameter that was not being referenced in the string. Signed-off-by: Rob Woolley Signed-off-by: Richard Purdie --- bin/bitbake-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index dcad9c16968..212da4f88a3 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -1176,7 +1176,7 @@ def main(): list_configs(all_settings, args) return - logger.info('Bitbake-setup is using {} as top directory.'.format(top_dir, global_settings_path(args))) + logger.info('Bitbake-setup is using {} as top directory.'.format(top_dir)) if args.func == init_config: init_config(top_dir, all_settings, args) From 1fb803e087007f14b4074cafb80306629c556b9a Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Thu, 26 Mar 2026 11:57:35 -0700 Subject: [PATCH 43/86] bitbake-setup: Remove unused stdout variable The ruff lint tool reported that stdout was never used: F841 Local variable `stdout` is assigned to but never used Signed-off-by: Rob Woolley Signed-off-by: Richard Purdie --- bin/bitbake-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 212da4f88a3..595adf3ec88 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -626,7 +626,7 @@ def init_config(top_dir, settings, args): create_siteconf(top_dir, args.non_interactive, settings) d = init_bb_cache(top_dir, settings, args) - stdout = sys.stdout + def handle_task_progress(event, d): rate = event.rate if event.rate else '' progress = event.progress if event.progress > 0 else 0 From 6c293b4a02c9f4d9f6b42ca700eaa45b4b50d77a Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Thu, 26 Mar 2026 11:57:36 -0700 Subject: [PATCH 44/86] bitbake-setup: Replace f-string without placeholders The ruff lint tool reported an f-string that didn't include any placeholders: F541 f-string without any placeholders Signed-off-by: Rob Woolley Signed-off-by: Richard Purdie --- bin/bitbake-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 595adf3ec88..09c997caf5f 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -1040,7 +1040,7 @@ def merge_settings(builtin_settings, global_settings, topdir_settings, cmdline_s return all_settings def sigint_handler(sig, frame, func, top_dir): - logger.plain(f'\nShutting down...') + logger.plain('\nShutting down...') if isinstance(top_dir, str) and os.path.exists(top_dir): if func in [init_config, build_update]: logger.warning(f'{top_dir} may contain an incomplete setup!') From 83665360893f5ee19cd7b141d53bf343b912e99b Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Thu, 26 Mar 2026 11:57:37 -0700 Subject: [PATCH 45/86] bitbake-setup: Remove unused Namespace import The ruff lint tool reported that Namespace was not used: F401 `argparse.Namespace` imported but unused Signed-off-by: Rob Woolley Signed-off-by: Richard Purdie --- bin/bitbake-setup | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 09c997caf5f..e4d60101f74 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -1186,7 +1186,6 @@ def main(): save_bb_cache() else: - from argparse import Namespace parser.print_help() main() From 03283f09f2cbbce4cbe725e044a568e8ce9faf79 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Thu, 26 Mar 2026 11:57:38 -0700 Subject: [PATCH 46/86] bitbake-setup: Remove unused parser_settings_list The ruff lint tool detected an used variable: F841 Local variable `parser_settings_list` is assigned to but never used The list_settings function gets invoked through the parent command's settings_func function. This means that creating this variable is unnecessary. Signed-off-by: Rob Woolley Signed-off-by: Richard Purdie --- bin/bitbake-setup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index e4d60101f74..ec52eb897fd 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -1109,8 +1109,7 @@ def main(): subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file") - parser_settings_list = subparser_settings.add_parser('list', - help="List all settings with their values") + subparser_settings.add_parser('list', help="List all settings with their values") parser_settings_set = subparser_settings.add_parser('set', parents=[parser_settings_arg_global], help="In a Section, set a setting to a certain value") From 02082312c72c43a285e6a48ed03dd694af5e90a4 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Thu, 26 Mar 2026 11:57:39 -0700 Subject: [PATCH 47/86] bitbake-setup: Fix linting error related to membership The pycodestyle E713 recommends that negative tests for membership use `foo not in bar` instead of `not foo in bar`. Signed-off-by: Rob Woolley Signed-off-by: Richard Purdie --- bin/bitbake-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index ec52eb897fd..ddca65bcf0f 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -154,7 +154,7 @@ def add_unique_timestamp_to_path(path): def _get_remotes(r_remote): remotes = [] - if not 'remotes' in r_remote and not 'uri' in r_remote: + if 'remotes' not in r_remote and 'uri' not in r_remote: raise Exception("Expected key(s): 'remotes', 'uri'") if 'remotes' in r_remote: From 700568f1809d48bc1c06009025f89ae8bdca40f7 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Thu, 26 Mar 2026 11:57:40 -0700 Subject: [PATCH 48/86] bitbake-setup: Maintain exception chain Chain the exceptions so that it is easier to trace the exception back to its root cause. This resolves B904 linting issues from flake8-bugbear. Signed-off-by: Rob Woolley Signed-off-by: Richard Purdie --- bin/bitbake-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index ddca65bcf0f..600dc1174ad 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -579,7 +579,7 @@ def obtain_config(top_dir, registry, args, source_overrides, d): json_data = json.load(f) upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json_data} except json.JSONDecodeError as e: - raise Exception ("Invalid JSON from {}. Are you pointing to an HTML page? {}".format(config_id, e)) + raise Exception ("Invalid JSON from {}. Are you pointing to an HTML page? {}".format(config_id, e)) from e else: logger.info("Looking up config {} in configuration registry".format(config_id)) registry_path = update_registry(registry, cache_dir(top_dir), d) From e8b383071e34a8a9015b4c83ccb7bf485a277f30 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Thu, 26 Mar 2026 11:57:42 -0700 Subject: [PATCH 49/86] bitbake-setup: Sort and format imports The ruff linting tool reported 2 problems: I001 Import block is un-sorted or un-formatted E402 Module level import not at top of file An exception is made for sys.path modifications needed to find the modules. Signed-off-by: Rob Woolley Signed-off-by: Richard Purdie --- bin/bitbake-setup | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 600dc1174ad..d4bd536c076 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -4,28 +4,29 @@ # SPDX-License-Identifier: GPL-2.0-only # -import logging -import os -import sys import argparse -import json -import shutil -import time import configparser +import copy import datetime +import functools import glob -import subprocess -import copy -import textwrap +import json +import logging +import os +import shutil import signal -import functools import string +import subprocess +import sys +import textwrap +import time + bindir = os.path.abspath(os.path.dirname(__file__)) sys.path[0:0] = [os.path.join(os.path.dirname(bindir), 'lib')] -import bb.msg -import bb.process +import bb.msg # noqa: E402 +import bb.process # noqa: E402 logger = bb.msg.logger_create('bitbake-setup', sys.stdout) From f8faab22d58432ea0749ab8e80d9b0063a583ba7 Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:49 +0200 Subject: [PATCH 50/86] fetch2: add LocalModificationsError and RebaseError exceptions When unpack_update() detects either of two blocking conditions, it now raises a dedicated exception subclass of UnpackError instead of a generic UnpackError: - LocalModificationsError: for uncommitted changes in the working tree that prevent the update. Includes git status output and instructs the user to commit, stash or discard changes. - RebaseError: for local commits that could not be rebased onto the new upstream revision. Includes the git rebase output and a hint that the 'dldir' remote points to the local download cache and may be used to resolve conflicts manually. Both exceptions take (repodir, url, git_output) as arguments, build the full user-facing message in __init__, and preserve the is-a relationship with UnpackError so existing callers are not broken. The fetch and rebase operations are also split into separate try/except blocks so that a failure to fetch from dldir raises a plain UnpackError while a failed rebase raises RebaseError. Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- lib/bb/fetch2/__init__.py | 17 +++++++++++++++++ lib/bb/fetch2/git.py | 7 +++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py index 909ae98146e..52d5556d368 100644 --- a/lib/bb/fetch2/__init__.py +++ b/lib/bb/fetch2/__init__.py @@ -96,6 +96,23 @@ def __init__(self, message, url): BBFetchException.__init__(self, msg) self.args = (message, url) +class LocalModificationsError(UnpackError): + """Exception raised when a checkout cannot be updated due to local modifications""" + def __init__(self, repodir, url, git_output): + message = ("Repository at %s has uncommitted changes, unable to update:\n%s\n" + "Commit, stash or discard your changes and re-run the update." % (repodir, git_output)) + UnpackError.__init__(self, message, url) + self.args = (repodir, url, git_output) + +class RebaseError(UnpackError): + """Exception raised when a checkout has local commits that could not be rebased onto the new upstream revision""" + def __init__(self, repodir, url, git_output): + message = ("Repository at %s has local commits that could not be rebased onto the new upstream revision:\n%s\n" + "Note: the 'dldir' remote points to the local download cache and may be used to resolve the conflict manually.\n" + "Once resolved, re-run the update." % (repodir, git_output)) + UnpackError.__init__(self, message, url) + self.args = (repodir, url, git_output) + class NoMethodError(BBFetchException): """Exception raised when there is no method to obtain a supplied url or set of urls""" def __init__(self, url): diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 6457463402c..6ed14005b53 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -730,7 +730,7 @@ def unpack(self, ud, destdir, d, update=False): output = runfetchcmd("%s status --untracked-files=no --porcelain" % (ud.basecmd), d, workdir=destdir) if output: - raise bb.fetch2.UnpackError("Repository at %s has uncommitted changes, unable to update:\n%s" % (destdir, output), ud.url) + raise bb.fetch2.LocalModificationsError(destdir, ud.url, output) # Set up remote for the download location if it doesn't exist try: @@ -740,6 +740,9 @@ def unpack(self, ud, destdir, d, update=False): runfetchcmd("%s remote add dldir file://%s" % (ud.basecmd, ud.clonedir), d, workdir=destdir) try: runfetchcmd("%s fetch dldir" % (ud.basecmd), d, workdir=destdir) + except bb.fetch2.FetchError as e: + raise bb.fetch2.UnpackError("Failed to fetch from dldir remote: %s" % str(e), ud.url) + try: runfetchcmd("%s rebase --no-autosquash --no-autostash %s" % (ud.basecmd, ud.revision), d, workdir=destdir) except bb.fetch2.FetchError as e: # If rebase failed, abort it @@ -747,7 +750,7 @@ def unpack(self, ud, destdir, d, update=False): runfetchcmd("%s rebase --abort" % (ud.basecmd), d, workdir=destdir) except Exception: pass - raise bb.fetch2.UnpackError("Failed to update checkout in place: %s" % str(e), ud.url) + raise bb.fetch2.RebaseError(destdir, ud.url, str(e)) return True # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag From 6c1854832aece9eaa2e25b0217434dfd124ba4f8 Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:50 +0200 Subject: [PATCH 51/86] bitbake-selftest: add GitUnpackUpdateTest Add a test class that exercises the new unpack_update() code path in the Git fetcher, ordered from basic building blocks to advanced workflow and error cases: test_unpack_update_full_clone Basic update to a newer upstream revision succeeds and the working tree reflects the new content. test_unpack_update_dldir_remote_setup The "dldir" remote pointing to ud.clonedir is created during the initial unpack and is present for subsequent update calls. test_unpack_update_ff_with_local_changes Full workflow: after a normal unpack the "dldir" remote is verified, a local commit is added, download() brings updated_rev into the clonedir, and unpack_update() fetches from dldir and rebases the local commit fast forward on top. The commit graph (HEAD^ == updated_rev) and both file contents are asserted. test_unpack_update_already_at_target_revision Calling unpack_update() when the checkout is already at SRCREV is a no-op: it succeeds and the working tree is left unchanged. test_unpack_update_with_untracked_file The status check uses --untracked-files=no so untracked files are invisible to it; the update succeeds and the untracked file survives the rebase unchanged. test_unpack_update_with_staged_changes Staged (but not committed) changes are detected by "git status --untracked-files=no --porcelain" and block the update; LocalModificationsError is raised so the caller can fall back to backup + re-fetch. test_unpack_update_with_modified_tracked_file An unstaged modification to a tracked file is detected by "git status --untracked-files=no --porcelain" and blocks the update; LocalModificationsError is raised. test_unpack_update_conflict_raises_rebase_error A local commit that conflicts with the incoming upstream change raises RebaseError; the repository is left in a clean state with no pending rebase (rebase --abort was called). test_unpack_update_untracked_file_overwritten_by_upstream An untracked file that would be overwritten by an incoming upstream commit causes git to refuse the rebase; RebaseError is raised and the repository is not left in a mid-rebase state. Two sub-cases are covered: a top-level file clash and a clash inside a subdirectory (xxx/somefile). test_unpack_update_shallow_clone_fails Shallow clones do not carry enough history; UnpackError is raised. test_unpack_update_stale_dldir_remote When the clonedir has been removed after the initial unpack the dldir remote no longer resolves; UnpackError is raised so the caller can fall back to a full re-fetch. test_fetch_unpack_update_toplevel_api The public Fetch.unpack_update(root) API (used by callers such as bitbake-setup) dispatches correctly end-to-end through to the Git fetcher. Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- lib/bb/tests/fetch.py | 559 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 559 insertions(+) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 7b8297a7827..1d018671a78 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -3793,3 +3793,562 @@ def test_gomodgit_url_host_only(self): self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') + + +class GitUnpackUpdateTest(FetcherTest): + """Test the unpack_update functionality for git fetcher. + + Intended workflow + 1. First-time setup: + 1. download() - clones the upstream repo into DL_DIR/git2/... (clonedir). + 2. unpack() - clones from clonedir into the workspace (S/workdir) and + registers a 'dldir' git remote pointing at + file://DL_DIR/git2/... for later offline use. + + 2. Subsequent updates (what unpack_update is designed for): + 1. The user works in the unpacked source tree. + 2. Upstream advances - SRCREV changes in the recipe. + 3. download() - fetches the new revision into the local clonedir. + 4. unpack_update() - instead of wiping the workspace and re-cloning: + * fetches the new revision from the local 'dldir' remote + * rebases the user's local commits on top of the new SRCREV + * raises LocalModificationsError if uncommitted changes block the + update, RebaseError if local commits cannot be rebased, or a + plain UnpackError for other failures (shallow clone, stale dldir); + in all cases the caller (e.g. bitbake-setup) can fall back to + backup + re-clone. + + Key design constraints: + * unpack_update() never deletes existing data (unlike unpack()). + * Only staged/modified tracked files block the update; untracked files and + committed local work are handled gracefully. + * The 'dldir' remote is intentionally visible to users outside the + fetcher (e.g. for manual 'git log dldir/master'). + * Currently only git is supported. + """ + + def setUp(self): + """Set up a local bare git source repository with two commits on 'master'. + + self.initial_rev - the first commit (testfile.txt: 'initial content') + self.updated_rev - the second commit (testfile.txt: 'updated content') + + SRCREV is initialised to self.initial_rev so individual tests can + advance it to self.updated_rev (or create further commits) as needed. + """ + FetcherTest.setUp(self) + + self.gitdir = os.path.join(self.tempdir, 'gitrepo') + self.srcdir = os.path.join(self.tempdir, 'gitsource') + + self.d.setVar('WORKDIR', self.tempdir) + self.d.setVar('S', self.gitdir) + self.d.delVar('PREMIRRORS') + self.d.delVar('MIRRORS') + + # Create a source git repository + bb.utils.mkdirhier(self.srcdir) + self.git_init(cwd=self.srcdir) + + # Create initial commit + with open(os.path.join(self.srcdir, 'testfile.txt'), 'w') as f: + f.write('initial content\n') + self.git(['add', 'testfile.txt'], cwd=self.srcdir) + self.git(['commit', '-m', 'Initial commit'], cwd=self.srcdir) + self.initial_rev = self.git(['rev-parse', 'HEAD'], cwd=self.srcdir).strip() + + # Create a second commit + with open(os.path.join(self.srcdir, 'testfile.txt'), 'w') as f: + f.write('updated content\n') + self.git(['add', 'testfile.txt'], cwd=self.srcdir) + self.git(['commit', '-m', 'Update commit'], cwd=self.srcdir) + self.updated_rev = self.git(['rev-parse', 'HEAD'], cwd=self.srcdir).strip() + + self.d.setVar('SRCREV', self.initial_rev) + self.d.setVar('SRC_URI', 'git://%s;branch=master;protocol=file' % self.srcdir) + + def test_unpack_update_full_clone(self): + """Test that unpack_update updates an existing checkout in place for a full clone. + + Steps: + 1. Fetch and unpack at self.initial_rev - verify 'initial content'. + 2. Advance SRCREV to self.updated_rev and re-download. + 3. Call unpack_update() instead of unpack() - the existing checkout + must be updated via 'git fetch dldir' + 'git rebase' without + re-cloning the directory. + 4. Verify testfile.txt now contains 'updated content'. + """ + # First fetch at initial revision + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + # Verify initial state + unpack_path = os.path.join(self.unpackdir, 'git') + self.assertTrue(os.path.exists(os.path.join(unpack_path, 'testfile.txt'))) + with open(os.path.join(unpack_path, 'testfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'initial content\n') + + # Update to new revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + # Use unpack_update + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + # Verify updated state + with open(os.path.join(unpack_path, 'testfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'updated content\n') + + def test_unpack_update_dldir_remote_setup(self): + """Test that unpack() adds a 'dldir' git remote pointing at ud.clonedir. + + The 'dldir' remote is used by subsequent unpack_update() calls to fetch + new commits from the local download cache (${DL_DIR}/git2/
) without + requiring network access. After a normal unpack the remote must exist + and its URL must be 'file://'. + """ + # First fetch + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Check that dldir remote exists + remotes = self.git(['remote'], cwd=unpack_path).strip().split('\n') + self.assertIn('dldir', remotes) + + # Verify it points to the clonedir + dldir_url = self.git(['remote', 'get-url', 'dldir'], cwd=unpack_path).strip() + self.assertEqual(dldir_url, 'file://{}'.format(ud.clonedir)) + + def test_unpack_update_ff_with_local_changes(self): + """Test that unpack_update rebases local commits fast forward. + + Full workflow: + 1. Fetch + unpack at initial_rev - verify 'dldir' remote is created + pointing at ud.clonedir. + 2. Add a local commit touching localfile.txt. + 3. Advance SRCREV to updated_rev and call download() - verify that + ud.clonedir (the dldir bare clone) now contains updated_rev. + 4. Call unpack_update() - it fetches updated_rev from dldir into the + working tree and rebases the local commit on top. + 5. Verify the final commit graph: HEAD's parent is updated_rev, and + both testfile.txt ('updated content') and localfile.txt ('local + change') are present. + + Note: git rebase operates the same way regardless of whether HEAD is + detached or on a named branch (e.g. 'master' or a local feature branch), + so this test covers those scenarios implicitly. + """ + # Step 1 - fetch + unpack at initial_rev + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + unpack_path = os.path.join(self.unpackdir, 'git') + + # The normal unpack must have set up the 'dldir' remote pointing at + # ud.clonedir so that subsequent unpack_update() calls work offline. + dldir_url = self.git(['remote', 'get-url', 'dldir'], cwd=unpack_path).strip() + self.assertEqual(dldir_url, 'file://{}'.format(ud.clonedir)) + + # Step 2 - add a local commit that touches a new file + with open(os.path.join(unpack_path, 'localfile.txt'), 'w') as f: + f.write('local change\n') + self.git(['add', 'localfile.txt'], cwd=unpack_path) + self.git(['commit', '-m', 'Local commit'], cwd=unpack_path) + local_commit = self.git(['rev-parse', 'HEAD'], cwd=unpack_path).strip() + + # Step 3 - advance SRCREV and download; clonedir must now contain + # updated_rev so that unpack_update can fetch it without network access. + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + ud = fetcher.ud[uri] + clonedir_refs = self.git(['rev-parse', self.updated_rev], cwd=ud.clonedir).strip() + self.assertEqual(clonedir_refs, self.updated_rev, + "clonedir must contain updated_rev after download()") + + # Step 4 - unpack_update fetches from dldir and rebases + git_fetcher = ud.method + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + # Step 5 - verify the commit graph and working tree content + # HEAD is the rebased local commit; its parent must be updated_rev + head_rev = self.git(['rev-parse', 'HEAD'], cwd=unpack_path).strip() + parent_rev = self.git(['rev-parse', 'HEAD^'], cwd=unpack_path).strip() + self.assertNotEqual(head_rev, local_commit, + "local commit should have a new SHA after rebase") + self.assertEqual(parent_rev, self.updated_rev, + "HEAD's parent must be updated_rev after fast-forward rebase") + + with open(os.path.join(unpack_path, 'testfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'updated content\n') + with open(os.path.join(unpack_path, 'localfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'local change\n') + + def test_unpack_update_already_at_target_revision(self): + """Test that unpack_update is a no-op when the checkout is already at SRCREV. + + Calling unpack_update() without advancing SRCREV must succeed and leave + the working tree unchanged. No rebase should be attempted because the + checkout already points at ud.revision. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + with open(os.path.join(unpack_path, 'testfile.txt')) as f: + self.assertEqual(f.read(), 'initial content\n') + + # Call unpack_update with SRCREV still at initial_rev - no upstream change + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + result = git_fetcher.unpack_update(ud, self.unpackdir, self.d) + self.assertTrue(result) + + # Content must be unchanged + with open(os.path.join(unpack_path, 'testfile.txt')) as f: + self.assertEqual(f.read(), 'initial content\n') + + def test_unpack_update_with_untracked_file(self): + """Test that unpack_update succeeds when the checkout has an untracked file. + + The status check uses '--untracked-files=no', so untracked files are not + detected and do not trigger the fallback path. git rebase also leaves + untracked files untouched, so both the upstream update and the untracked + file must be present after the call. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Create an untracked file (not staged, not committed) + untracked = os.path.join(unpack_path, 'untracked.txt') + with open(untracked, 'w') as f: + f.write('untracked content\n') + + # Update to new upstream revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # --untracked-files=no means the status check passes; rebase preserves the file + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + with open(os.path.join(unpack_path, 'testfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'updated content\n') + + # Untracked file must survive the rebase + self.assertTrue(os.path.exists(untracked)) + with open(untracked, 'r') as f: + self.assertEqual(f.read(), 'untracked content\n') + + def test_unpack_update_with_staged_changes(self): + """Test that unpack_update fails when the checkout has staged (but not committed) changes. + + The rebase is run with --no-autostash so git refuses to rebase over a + dirty index. The caller (bitbake-setup) is expected to catch the + resulting LocalModificationsError and fall back to backup + re-fetch. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Stage a new file without committing it + staged = os.path.join(unpack_path, 'staged.txt') + with open(staged, 'w') as f: + f.write('staged content\n') + self.git(['add', 'staged.txt'], cwd=unpack_path) + + # Update to new upstream revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # Should fail - git rebase refuses to run with a dirty index + with self.assertRaises(bb.fetch2.LocalModificationsError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + def test_unpack_update_with_modified_tracked_file(self): + """Test that unpack_update fails when a tracked file has unstaged modifications. + + 'git status --untracked-files=no --porcelain' reports unstaged modifications + to tracked files (output line ' M filename'), which must block the update so + the caller can fall back to backup + re-fetch rather than silently discarding + work in progress. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Modify a tracked file without staging or committing + with open(os.path.join(unpack_path, 'testfile.txt'), 'w') as f: + f.write('locally modified content\n') + + # Update to new upstream revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # Should fail - unstaged modification to tracked file is detected by + # 'git status --untracked-files=no --porcelain' + with self.assertRaises(bb.fetch2.LocalModificationsError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + def test_unpack_update_conflict_raises_rebase_error(self): + """Test that unpack_update raises RebaseError on a rebase conflict. + + When a local commit modifies the same lines as an incoming upstream commit, + git rebase cannot resolve the conflict automatically. unpack_update must + abort the failed rebase and raise RebaseError so the caller can fall back + to a backup + re-fetch. + """ + # Fetch and unpack at the initial revision + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Make a local commit that edits the same lines as the upcoming upstream commit + with open(os.path.join(unpack_path, 'testfile.txt'), 'w') as f: + f.write('conflicting local content\n') + self.git(['add', 'testfile.txt'], cwd=unpack_path) + self.git(['commit', '-m', 'Local conflicting commit'], cwd=unpack_path) + + # Add a third upstream commit that also edits testfile.txt differently + with open(os.path.join(self.srcdir, 'testfile.txt'), 'w') as f: + f.write('conflicting upstream content\n') + self.git(['add', 'testfile.txt'], cwd=self.srcdir) + self.git(['commit', '-m', 'Upstream conflicting commit'], cwd=self.srcdir) + conflict_rev = self.git(['rev-parse', 'HEAD'], cwd=self.srcdir).strip() + + # Update SRCREV to the new upstream commit + self.d.setVar('SRCREV', conflict_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # unpack_update must fail and clean up (rebase --abort) rather than + # leaving the repo in a mid-rebase state + with self.assertRaises(bb.fetch2.RebaseError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + # Verify the repo is not left in a conflicted / mid-rebase state + rebase_merge = os.path.join(unpack_path, '.git', 'rebase-merge') + rebase_apply = os.path.join(unpack_path, '.git', 'rebase-apply') + self.assertFalse(os.path.exists(rebase_merge), + "rebase-merge dir should not exist after failed unpack_update") + self.assertFalse(os.path.exists(rebase_apply), + "rebase-apply dir should not exist after failed unpack_update") + + def test_unpack_update_untracked_file_overwritten_by_upstream(self): + """Test that unpack_update raises RebaseError when an untracked file would be + overwritten by an incoming upstream commit. + + We skip untracked files in the pre-check (git rebase doesn't touch harmless + untracked files), but git itself refuses to rebase when an untracked file would + be overwritten by the incoming changes. The resulting FetchError must be caught + and re-raised as RebaseError without leaving the repo in a mid-rebase state. + + Two sub-cases are covered: + - top-level untracked file clashing with an incoming upstream file + - untracked file inside a subdirectory (xxx/somefile) clashing with an + upstream commit that adds the same path + """ + def _run_case(upstream_path, local_rel_path, commit_msg): + """ + Add upstream_path to self.srcdir, create local_rel_path as an + untracked file in the checkout, then assert that unpack_update + raises RebaseError and leaves no mid-rebase state, and that the + local file is untouched. + """ + # Fresh fetch + unpack at the current SRCREV + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Upstream adds the file (potentially inside a subdirectory) + full_upstream = os.path.join(self.srcdir, upstream_path) + os.makedirs(os.path.dirname(full_upstream), exist_ok=True) + with open(full_upstream, 'w') as f: + f.write('upstream content\n') + self.git(['add', upstream_path], cwd=self.srcdir) + self.git(['commit', '-m', commit_msg], cwd=self.srcdir) + new_rev = self.git(['rev-parse', 'HEAD'], cwd=self.srcdir).strip() + + # Create the clashing untracked file in the checkout + full_local = os.path.join(unpack_path, local_rel_path) + os.makedirs(os.path.dirname(full_local), exist_ok=True) + with open(full_local, 'w') as f: + f.write('local untracked content\n') + + self.d.setVar('SRCREV', new_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # git rebase refuses because the untracked file would be overwritten + with self.assertRaises(bb.fetch2.RebaseError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + # Repo must not be left in a mid-rebase state + self.assertFalse(os.path.exists(os.path.join(unpack_path, '.git', 'rebase-merge'))) + self.assertFalse(os.path.exists(os.path.join(unpack_path, '.git', 'rebase-apply'))) + + # The local untracked file must be untouched + self.assertTrue(os.path.exists(full_local)) + with open(full_local) as f: + self.assertEqual(f.read(), 'local untracked content\n') + + # Reset unpackdir for the next sub-case + import shutil as _shutil + _shutil.rmtree(self.unpackdir) + os.makedirs(self.unpackdir) + + # Sub-case 1: top-level file clash + _run_case('newfile.txt', 'newfile.txt', + 'Upstream adds newfile.txt') + + # Sub-case 2: file inside a subdirectory (xxx/somefile) + _run_case('xxx/somefile.txt', 'xxx/somefile.txt', + 'Upstream adds xxx/somefile.txt') + + def test_unpack_update_shallow_clone_fails(self): + """Test that unpack_update raises UnpackError for shallow-tarball checkouts. + + Shallow clones lack full history, which makes an in-place rebase impossible + without network access. After fetching with BB_GIT_SHALLOW=1 the clonedir + is deleted so that unpack() is forced to use the shallow tarball. + A subsequent call to unpack_update() must raise UnpackError and the message + must mention 'shallow clone' so callers can distinguish this case. + """ + self.d.setVar('BB_GIT_SHALLOW', '1') + self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') + + # First fetch at initial revision + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + # Remove clonedir to force use of shallow tarball + clonedir = os.path.join(self.dldir, 'git2') + if os.path.exists(clonedir): + shutil.rmtree(clonedir) + + fetcher.unpack(self.unpackdir) + + # Update to new revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + # unpack_update should fail for shallow clones + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + with self.assertRaises(bb.fetch2.UnpackError) as context: + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + self.assertIn("shallow clone", str(context.exception).lower()) + + def test_unpack_update_stale_dldir_remote(self): + """Test that unpack_update raises UnpackError when the dldir remote URL is stale. + + If the clonedir has been removed after the initial unpack (e.g. DL_DIR was + cleaned) the 'dldir' remote URL no longer resolves. The fetch inside + update_mode will fail with a FetchError which must be re-raised as + UnpackError so the caller can fall back to a full re-fetch. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Advance SRCREV to trigger update_mode + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + + # Delete the clonedir and corrupt the dldir remote URL so that + # 'git fetch dldir' fails, simulating a missing or relocated DL_DIR. + shutil.rmtree(ud.clonedir) + self.git(['remote', 'set-url', 'dldir', 'file://' + ud.clonedir], + cwd=unpack_path) + + git_fetcher = ud.method + with self.assertRaises(bb.fetch2.UnpackError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + def test_fetch_unpack_update_toplevel_api(self): + """Test that the top-level Fetch.unpack_update() dispatches to Git.unpack_update(). + + Callers such as bitbake-setup use fetcher.unpack_update(root) rather than + calling the method on the Git fetcher directly. Verify that the public API + works end-to-end: fetch at initial_rev, unpack, advance to updated_rev, + fetch again, then call fetcher.unpack_update(root) and confirm the content + is updated. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + with open(os.path.join(unpack_path, 'testfile.txt')) as f: + self.assertEqual(f.read(), 'initial content\n') + + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + # Use the public Fetch.unpack_update() rather than the method directly + fetcher.unpack_update(self.unpackdir) + + with open(os.path.join(unpack_path, 'testfile.txt')) as f: + self.assertEqual(f.read(), 'updated content\n') From 2d6e21aa9f720ad52d0667994453e8921bd9553b Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:51 +0200 Subject: [PATCH 52/86] tests/fetch: remove unused import, fix trailing whitespace Just cleanup, no functional change. Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- lib/bb/tests/fetch.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 1d018671a78..5b3fc8a4191 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -18,7 +18,6 @@ import signal import tarfile from bb.fetch2 import URI -from bb.fetch2 import FetchMethod import bb import bb.utils from bb.tests.support.httpserver import HTTPService @@ -551,8 +550,8 @@ def test_mirror_of_mirror(self): fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) mirrors = bb.fetch2.mirror_from_string(mirrorvar) uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) - self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', - 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', + self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', + 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) @@ -1390,7 +1389,7 @@ class URLHandle(unittest.TestCase): "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}), 'git://s.o-me_ONE:%s@git.openembedded.org/bitbake;branch=main;protocol=https' % password: ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', password, {'branch': 'main', 'protocol' : 'https'}), } - # we require a pathname to encodeurl but users can still pass such urls to + # we require a pathname to encodeurl but users can still pass such urls to # decodeurl and we need to handle them decodedata = datatable.copy() decodedata.update({ From 85f3ebcce90deb1ed19a4f575b59d14568458c2e Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:52 +0200 Subject: [PATCH 53/86] doc: document the unpack_update() non-destructive git update method Add a new "The Non-Destructive Update (unpack_update)" section to the fetching chapter, describing the in-place update mode introduced in "fetch/git: Add an 'update' unpack mode to the fetchers (git only for now)" e7d5e156275782948d3346f299780389ab263ab6. The section covers: - what unpack_update() does and when to use it vs. unpack() - the three-step internal process: add/update the dldir remote, fetch the new upstream revision, rebase local commits on top - the error conditions: LocalModificationsError (uncommitted changes), RebaseError (conflicting local commits), and the shallow/stale-dldir case - the current limitation that only the Git fetcher supports it Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- .../bitbake-user-manual-fetching.rst | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index 6af80359125..c2747c40124 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -186,6 +186,51 @@ cloning the tree into the final directory. The process is completed using references so that there is only one central copy of the Git metadata needed. +.. _bb-the-unpack-update: + +The Non-Destructive Update (``unpack_update``) +============================================== + +.. note:: + + This is a specialised method intended for tools that manage persistent + layer checkouts on behalf of the user, such as + :ref:`bitbake-setup update `. It is not part of the + normal recipe fetch/unpack flow. + +For Git URLs, an alternative ``unpack_update()`` method is available +that updates an existing checkout *in place* rather than removing and +re-cloning it. This is useful when the target directory may contain +local commits that should be preserved across updates. + +The code to call the non-destructive update looks like the following:: + + rootdir = l.getVar('UNPACKDIR') + fetcher.unpack_update(rootdir) + +``unpack_update()`` performs the following steps: + +1. A ``dldir`` git remote is added (or updated) in the existing + checkout, pointing at the local download cache. This remote may + also be useful for manually resolving conflicts outside the fetcher. +2. The new upstream revision is fetched from the ``dldir`` remote into + the existing repository. +3. Any local commits are rebased on top of the new upstream revision, + preserving local work. + +The method raises an error if: + +- The working tree contains staged or unstaged changes to tracked + files (``LocalModificationsError``). +- Local commits cannot be cleanly rebased onto the new upstream + revision (``RebaseError``). A failed rebase is automatically aborted + before the exception is raised. +- The download cache does not contain a sufficiently recent clone + of the repository, or the checkout is a shallow clone. + +Currently only the Git fetcher supports ``unpack_update()``. All other +fetcher types raise ``RuntimeError`` if it is called. + .. _bb-fetchers: Fetchers From 1bc754c962e7dcdcf7acb0b70a3f8991537c7602 Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:53 +0200 Subject: [PATCH 54/86] bitbake-setup: always restore sys.stdout While working on the bitbake-setup update with a non destructive fetcher, I noticed that if the fetcher raises an exception, sys.stdout is not restored, which can lead to issues in the rest of the code. This change ensures that sys.stdout is always restored, even if an exception occurs during the fetch and unpack process. Showing the full Traceback of the error would be a bit confusing because it happened in code which is not yet ready for review, but the final error was: ValueError: I/O operation on closed file. and this little change seems to be a reasonable improvement to avoid such issues. Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- bin/bitbake-setup | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index d4bd536c076..0a8860f9330 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -794,9 +794,11 @@ def do_fetch(fetcher, dir): with open(fetchlog, 'a') as f: oldstdout = sys.stdout sys.stdout = f - fetcher.download() - fetcher.unpack_update(dir) - sys.stdout = oldstdout + try: + fetcher.download() + fetcher.unpack_update(dir) + finally: + sys.stdout = oldstdout def update_registry(registry, cachedir, d): registrydir = 'configurations' From deefbd24d2ea5c4376a9d06bd9653376bf623f5d Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:54 +0200 Subject: [PATCH 55/86] tests/setup: cleanup imports Add missing top-level imports for os, stat, bb and bb.process and remove the redundant inline 'import os' and 'import stat' that were inside individual methods. Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- lib/bb/tests/setup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/bb/tests/setup.py b/lib/bb/tests/setup.py index a66f05b36ba..d52a8139543 100644 --- a/lib/bb/tests/setup.py +++ b/lib/bb/tests/setup.py @@ -5,9 +5,13 @@ # from bb.tests.fetch import FetcherTest -import json -import hashlib +import bb +import bb.process import glob +import hashlib +import json +import os +import stat from bb.tests.support.httpserver import HTTPService class BitbakeSetupTest(FetcherTest): @@ -208,7 +212,6 @@ def add_file_to_testrepo(self, name, content, script=False): with open(fullname, 'w') as f: f.write(content) if script: - import stat st = os.stat(fullname) os.chmod(fullname, st.st_mode | stat.S_IEXEC) self.git('add {}'.format(name), cwd=self.testrepopath) @@ -279,7 +282,6 @@ def get_setup_path(self, cf, c): def test_setup(self): # unset BBPATH to ensure tests run in isolation from the existing bitbake environment - import os if 'BBPATH' in os.environ: del os.environ['BBPATH'] From f33c5d6d21e55e5d2e939185f8fedab8cceeb4af Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:55 +0200 Subject: [PATCH 56/86] tests/setup: fix dead check_setupdir_files guards Two conditions in check_setupdir_files used wrong key names and so never triggered: - 'oe-fragment' -> 'oe-fragments' (plural): fragment-existence assertions were never reached for any variant that has fragments. - 'bb-environment-passthrough' -> 'bb-env-passthrough-additions': BB_ENV_PASSTHROUGH_ADDITIONS assertions were never reached for the gizmo-env-passthrough variant. Also drop the redundant .keys() call on both guards. Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- lib/bb/tests/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bb/tests/setup.py b/lib/bb/tests/setup.py index d52a8139543..1af3d8b5065 100644 --- a/lib/bb/tests/setup.py +++ b/lib/bb/tests/setup.py @@ -260,11 +260,11 @@ def check_setupdir_files(self, setuppath, test_file_content): ) self.assertIn(filerelative_layer, bblayers) - if 'oe-fragment' in bitbake_config.keys(): + if 'oe-fragments' in bitbake_config: for f in bitbake_config["oe-fragments"]: self.assertTrue(os.path.exists(os.path.join(bb_conf_path, f))) - if 'bb-environment-passthrough' in bitbake_config.keys(): + if 'bb-env-passthrough-additions' in bitbake_config: with open(os.path.join(bb_build_path, 'init-build-env'), 'r') as f: init_build_env = f.read() self.assertTrue('BB_ENV_PASSTHROUGH_ADDITIONS' in init_build_env) From 92fd721941fd17d1febc7205739e9f9ce1bb3aee Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:56 +0200 Subject: [PATCH 57/86] bitbake-setup: generate config files for VSCode This change introduces a function to generate a VSCode workspace file (bitbake.code-workspace). The --init-vscode flag added to bitbake-setup init defaults to True when the code binary is found on PATH, and can be passed explicitly to exercise the feature on machines without code (e.g. when running tests on an autobuilder). Once the workspace file exists, it is updated automatically on every subsequent run. This workspace file is preferred over a project-specific .vscode/settings.json for several reasons: - It allows for a multi-root workspace, which is ideal for a bitbake project structure setup with bitbake-setup. This enables including all layer repositories and the build configuration directory as top-level folders in the explorer. - The workspace file can be located at the top level of the setup, outside of any version-controlled source directory. This avoids cluttering the git repositories with editor-specific configuration. - It provides a centralized place for all VSCode settings related to the project, including those for the bitbake extension, Python language server, and file associations, ensuring a consistent development environment for all users of the project. The Python analysis paths (python.analysis.extraPaths) are configured with absolute paths. This is a workaround for a limitation in the Pylance extension, which does not correctly resolve ${workspaceFolder:...} variables in a multi-root workspace context for import resolution. Using absolute paths ensures that Pylance can find all necessary modules from the various layers. There is room for improvement. Starting a terminal (bitbake or any other) is cumbersome, as VSCode wants to start it for one of the layers rather than the build directory. Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- bin/bitbake-setup | 175 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 168 insertions(+), 7 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 0a8860f9330..30b574bba6a 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -240,7 +240,7 @@ bitbake-setup init -L {} /path/to/repo/checkout""".format( return layers_fixed_revisions -def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf): +def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf, init_vscode=False): def _setup_build_conf(layers, filerelative_layers, build_conf_dir): os.makedirs(build_conf_dir) layers_s = [] @@ -359,6 +359,7 @@ def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_c init_script = os.path.join(bitbake_builddir, "init-build-env") + workspace_file = os.path.join(setupdir, "bitbake.code-workspace") shell = "bash" fragments = bitbake_config.get("oe-fragments", []) + sorted(bitbake_config.get("oe-fragment-choices",{}).values()) if fragments: @@ -370,6 +371,8 @@ def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_c logger.plain('New bitbake configuration from upstream is the same as the current one, no need to update it.') shutil.rmtree(bitbake_confdir) os.rename(backup_bitbake_confdir, bitbake_confdir) + if init_vscode or os.path.exists(workspace_file): + configure_vscode(setupdir, layerdir, bitbake_builddir, init_script) return logger.plain('Upstream bitbake configuration changes were found:') @@ -385,6 +388,8 @@ def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_c logger.plain(f'Leaving the upstream configuration in {upstream_bitbake_confdir}') os.rename(bitbake_confdir, upstream_bitbake_confdir) os.rename(backup_bitbake_confdir, bitbake_confdir) + if init_vscode or os.path.exists(workspace_file): + configure_vscode(setupdir, layerdir, bitbake_builddir, init_script) return logger.plain('Applying upstream bitbake configuration changes') @@ -392,17 +397,21 @@ def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_c fragment_note = "Run 'bitbake-config-build enable-fragment ' to enable additional fragments or replace built-in ones (e.g. machine/ or distro/ to change MACHINE or DISTRO)." + readme_extra = "" + if init_vscode: + readme_extra = "\n\nTo edit the code in VSCode, open the workspace: code {}\n".format(workspace_file) + readme = """{}\n\nAdditional information is in {} and {}\n Source the environment using '. {}' to run builds from the command line.\n {}\n -The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf -""".format( +The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf{}""".format( bitbake_config["description"], os.path.join(bitbake_builddir,'conf/conf-summary.txt'), os.path.join(bitbake_builddir,'conf/conf-notes.txt'), init_script, fragment_note, - bitbake_builddir + bitbake_builddir, + readme_extra ) readme_file = os.path.join(bitbake_builddir, "README") with open(readme_file, 'w') as f: @@ -413,6 +422,11 @@ The bitbake configuration files (local.conf, bblayers.conf and more) can be foun logger.plain("To run builds, source the environment using\n . {}\n".format(init_script)) logger.plain("{}\n".format(fragment_note)) logger.plain("The bitbake configuration files (local.conf, bblayers.conf and more) can be found in\n {}/conf\n".format(bitbake_builddir)) + if init_vscode: + logger.plain("To edit the code in VSCode, open the workspace:\n code {}\n".format(workspace_file)) + + if init_vscode or os.path.exists(workspace_file): + configure_vscode(setupdir, layerdir, bitbake_builddir, init_script) def get_registry_config(registry_path, id): for root, dirs, files in os.walk(registry_path): @@ -428,12 +442,12 @@ def merge_overrides_into_sources(sources, overrides): layers[k] = v return layers -def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt"): +def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt", init_vscode=False): layer_config = merge_overrides_into_sources(config["data"]["sources"], config["source-overrides"]["sources"]) sources_fixed_revisions = checkout_layers(layer_config, confdir, layerdir, d) bitbake_config = config["bitbake-config"] thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None - setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf) + setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf, init_vscode) write_sources_fixed_revisions(confdir, layerdir, sources_fixed_revisions) commit_config(confdir) @@ -622,6 +636,151 @@ def obtain_overrides(args): return overrides +def configure_vscode(setupdir, layerdir, builddir, init_script): + """ + Configure the VSCode environment by creating or updating a workspace file. + + Create or update a bitbake.code-workspace file with folders for the layers and build/conf. + Managed folders are regenerated; user-added folders are kept. Settings are merged, with + managed keys (bitbake.*, python extra paths) always overwritten. + """ + logger.debug("configure_vscode: setupdir={}, layerdir={}, builddir={}, init_script={}".format( + setupdir, layerdir, builddir, init_script)) + + # Get git repository directories + git_repos = [] + if os.path.exists(layerdir): + for entry in os.listdir(layerdir): + entry_path = os.path.join(layerdir, entry) + if os.path.isdir(entry_path) and not os.path.islink(entry_path): + # Check if it's a git repository + if os.path.exists(os.path.join(entry_path, '.git')): + git_repos.append(entry) + logger.debug("configure_vscode: found {} git repos: {}".format(len(git_repos), git_repos)) + + conf_path = os.path.relpath(os.path.join(builddir, "conf"), setupdir) + repo_paths = [os.path.relpath(os.path.join(layerdir, repo), setupdir) for repo in git_repos] + logger.debug("configure_vscode: conf_path={}, repo_paths={}".format(conf_path, repo_paths)) + + # Load existing workspace + workspace_file = os.path.join(setupdir, "bitbake.code-workspace") + workspace = { + "extensions": { + "recommendations": [ + "yocto-project.yocto-bitbake" + ] + } + } + if os.path.exists(workspace_file): + logger.debug("configure_vscode: loading existing workspace file: {}".format(workspace_file)) + try: + with open(workspace_file, 'r') as f: + workspace = json.load(f) + logger.debug("configure_vscode: loaded workspace with {} folders, {} settings".format( + len(workspace.get("folders", [])), len(workspace.get("settings", {})))) + except (json.JSONDecodeError, OSError) as e: + logger.error( + "Unable to read existing workspace file {}: {}. Skipping update.".format( + workspace_file, str(e) + ) + ) + return + else: + logger.debug("configure_vscode: creating new workspace file: {}".format(workspace_file)) + + # Update folders + existing_folders = workspace.get("folders", []) + new_folders = [{"name": "conf", "path": conf_path}] + for rp in repo_paths: + repo_name = os.path.basename(rp) + new_folders.append({"name": repo_name, "path": rp}) + # Keep any user-added folders that are not managed + managed_paths = {f["path"] for f in new_folders} + for f in existing_folders: + if f["path"] not in managed_paths: + new_folders.append(f) + logger.debug("configure_vscode: keeping user-added folder: {}".format(f["path"])) + workspace["folders"] = new_folders + logger.debug("configure_vscode: updated workspace with {} folders".format(len(new_folders))) + + # Build Python extra paths for each layer - only check top level of each repo + extra_paths = [] + subdirs_to_check = ['lib', 'scripts'] + for repo in git_repos: + repo_path_abs = os.path.join(layerdir, repo) + for subdir in subdirs_to_check: + sub_path = os.path.join(repo_path_abs, subdir) + if os.path.isdir(sub_path): + extra_paths.append(sub_path) + + # Update settings + existing_settings = workspace.get("settings", {}) + new_settings = { + "bitbake.disableConfigModification": True, + "bitbake.pathToBitbakeFolder": os.path.join(layerdir, "bitbake"), + "bitbake.pathToBuildFolder": builddir, + "bitbake.pathToEnvScript": init_script, + "bitbake.workingDirectory": builddir, + "files.associations": { + "*.conf": "bitbake", + "*.inc": "bitbake" + }, + "files.exclude": { + "**/.git/**": True + }, + "search.exclude": { + "**/.git/**": True, + "**/logs/**": True + }, + "files.watcherExclude": { + "**/.git/**": True, + "**/logs/**": True + }, + "python.analysis.exclude": [ + "**/.git/**", + "**/logs/**" + ], + "python.autoComplete.extraPaths": extra_paths, + "python.analysis.extraPaths": extra_paths + } + + # Merge settings: add missing, always update bitbake paths and python extra paths + for key, value in new_settings.items(): + if key not in existing_settings: + existing_settings[key] = value + elif key.startswith("bitbake.") or key in [ + "python.autoComplete.extraPaths", + "python.analysis.extraPaths", + ]: + # Always replace - these are managed/machine-generated settings + existing_settings[key] = value + elif key in [ + "files.associations", + "files.exclude", + "search.exclude", + "files.watcherExclude", + "python.analysis.exclude", + ]: + # For dicts and lists, merge new values in without removing user additions + if isinstance(value, dict): + if not isinstance(existing_settings[key], dict): + existing_settings[key] = {} + for k, v in value.items(): + if k not in existing_settings[key]: + existing_settings[key][k] = v + elif isinstance(value, list): + if not isinstance(existing_settings[key], list): + existing_settings[key] = [] + for item in value: + if item not in existing_settings[key]: + existing_settings[key].append(item) + + workspace["settings"] = existing_settings + logger.debug("configure_vscode: merged settings, total {} keys".format(len(existing_settings))) + + with open(workspace_file, 'w') as f: + json.dump(workspace, f, indent=4) + logger.debug("configure_vscode: wrote workspace file: {}".format(workspace_file)) def init_config(top_dir, settings, args): create_siteconf(top_dir, args.non_interactive, settings) @@ -691,7 +850,7 @@ def init_config(top_dir, settings, args): bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d) write_upstream_config(confdir, upstream_config) - update_build(upstream_config, confdir, setupdir, layerdir, d, update_bb_conf="yes") + update_build(upstream_config, confdir, setupdir, layerdir, d, update_bb_conf="yes", init_vscode=args.init_vscode) bb.event.remove("bb.build.TaskProgress", None) @@ -1087,6 +1246,8 @@ def main(): parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.') parser_init.add_argument('-L', '--use-local-source', default=[], action='append', nargs=2, metavar=('SOURCE_NAME', 'PATH'), help='Symlink local source into a build, instead of getting it as prescribed by a configuration (useful for local development).') + parser_init.add_argument('--init-vscode', action=argparse.BooleanOptionalAction, default=bool(shutil.which('code')), + help='Generate VSCode workspace configuration (default: %(default)s)') parser_init.set_defaults(func=init_config) parser_status = subparsers.add_parser('status', help='Check if the setup needs to be synchronized with configuration') From 70ff359d27de7bd83da9e9083b67ed2a63a164d0 Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:57 +0200 Subject: [PATCH 58/86] tests/setup: add test_vscode for VSCode workspace generation Add test_vscode to BitbakeSetupTest covering the --init-vscode option introduced in 'bitbake-setup: generate config files for VSCode': - init --init-vscode creates bitbake.code-workspace with the expected top-level structure (folders, settings, extensions). - Folders list conf and each non-symlink git repo in layers/; all paths are relative. - Bitbake extension settings (pathToBuildFolder, pathToEnvScript, disableConfigModification) are set correctly. - File associations (*.conf, *.inc) and python.analysis.extraPaths / python.autoComplete.extraPaths are populated. - init --no-init-vscode does not create a workspace file. - update after a layer change preserves user-added folders and settings while updating managed ones. - update with a corrupt workspace file logs an error and leaves the file unchanged. Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- lib/bb/tests/setup.py | 93 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/lib/bb/tests/setup.py b/lib/bb/tests/setup.py index 1af3d8b5065..52120546ca0 100644 --- a/lib/bb/tests/setup.py +++ b/lib/bb/tests/setup.py @@ -538,3 +538,96 @@ def _check_layer_backups(layer_path, expected_backups): custom_setup_dir = 'special-setup-dir-with-cmdline-overrides' out = self.runbbsetup("init --non-interactive -L test-repo {} --setup-dir-name {} test-config-1 gadget".format(self.testrepopath, custom_setup_dir)) _check_local_sources(custom_setup_dir) + + def test_vscode(self): + if 'BBPATH' in os.environ: + del os.environ['BBPATH'] + os.chdir(self.tempdir) + + self.runbbsetup("settings set default registry 'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath)) + self.add_file_to_testrepo('test-file', 'initial\n') + self.add_json_config_to_registry('test-config-1.conf.json', 'master', 'master') + + # --init-vscode should create bitbake.code-workspace + self.runbbsetup("init --non-interactive --init-vscode test-config-1 gadget") + setuppath = self.get_setup_path('test-config-1', 'gadget') + workspace_file = os.path.join(setuppath, 'bitbake.code-workspace') + self.assertTrue(os.path.exists(workspace_file), + "bitbake.code-workspace should be created with --init-vscode") + + with open(workspace_file) as f: + workspace = json.load(f) + + # top-level structure + self.assertIn('folders', workspace) + self.assertIn('settings', workspace) + self.assertIn('extensions', workspace) + self.assertIn('yocto-project.yocto-bitbake', + workspace['extensions']['recommendations']) + + # folders: conf dir + test-repo (symlinks like oe-init-build-env-dir are skipped) + folder_names = {f['name'] for f in workspace['folders']} + self.assertIn('conf', folder_names) + self.assertIn('test-repo', folder_names) + + # folder paths must be relative so the workspace is portable + for f in workspace['folders']: + self.assertFalse(os.path.isabs(f['path']), + "Folder path should be relative, got: {}".format(f['path'])) + + # bitbake extension settings + settings = workspace['settings'] + self.assertTrue(settings.get('bitbake.disableConfigModification')) + self.assertEqual(settings['bitbake.pathToBuildFolder'], + os.path.join(setuppath, 'build')) + self.assertEqual(settings['bitbake.pathToEnvScript'], + os.path.join(setuppath, 'build', 'init-build-env')) + + # file associations + self.assertIn('*.conf', settings.get('files.associations', {})) + self.assertIn('*.inc', settings.get('files.associations', {})) + + # python extra paths: test-repo/scripts/ exists and should be listed + extra_paths = settings.get('python.analysis.extraPaths', []) + self.assertTrue(any('scripts' in p for p in extra_paths), + "python.analysis.extraPaths should include the scripts dir") + self.assertEqual(settings.get('python.analysis.extraPaths'), + settings.get('python.autoComplete.extraPaths')) + + # --no-init-vscode should NOT create a workspace file + self.runbbsetup("init --non-interactive --no-init-vscode test-config-1 gadget-notemplate") + notemplate_path = self.get_setup_path('test-config-1', 'gadget-notemplate') + self.assertFalse( + os.path.exists(os.path.join(notemplate_path, 'bitbake.code-workspace')), + "bitbake.code-workspace should not be created with --no-init-vscode") + + # update with --init-vscode after a layer change should preserve + # user-added folders and settings while still rewriting managed ones + workspace['folders'].append({"name": "user-folder", "path": "user/custom"}) + workspace['settings']['my.user.setting'] = 'preserved' + with open(workspace_file, 'w') as f: + json.dump(workspace, f, indent=4) + + self.add_file_to_testrepo('test-file', 'updated\n') + os.environ['BBPATH'] = os.path.join(setuppath, 'build') + self.runbbsetup("update --update-bb-conf='no'") + del os.environ['BBPATH'] + + with open(workspace_file) as f: + updated = json.load(f) + self.assertIn('user/custom', {f['path'] for f in updated['folders']}, + "User-added folder was removed during update") + self.assertIn('my.user.setting', updated['settings'], + "User-added setting was removed during update") + + # update with a corrupt workspace file should log an error and leave it unchanged + self.add_file_to_testrepo('test-file', 'updated-again\n') + with open(workspace_file, 'w') as f: + f.write('{invalid json') + os.environ['BBPATH'] = os.path.join(setuppath, 'build') + self.runbbsetup("update --update-bb-conf='no'") + del os.environ['BBPATH'] + with open(workspace_file) as f: + content = f.read() + self.assertEqual(content, '{invalid json', + "Corrupt workspace file should not be modified") From 2ee3a195bbe1b7458f44a712a271abd9686f90c7 Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:58 +0200 Subject: [PATCH 59/86] bitbake-setup: add --rebase-conflicts-strategy to the update command When unpack_update() raises LocalModificationsError (uncommitted changes) or RebaseError (local commits that could not be rebased), the caller now has a choice of how to handle it, controlled by the new --rebase-conflicts-strategy option on the 'update' subcommand: - abort (default): re-raise the error so the update stops with a clear message that names the affected source and its path. The fetcher has already aborted the rebase and restored the checkout to its previous state. - backup: rename the directory to a timestamped -backup path to preserve local work, then re-clone from upstream via fetcher.unpack(). Both exception types share a single except clause; the exception message already describes the specific failure (uncommitted changes vs. failed rebase with git output), so it is forwarded directly to the warning log. The strategy is threaded from the CLI argument through build_status() and update_build() down to checkout_layers(). Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- bin/bitbake-setup | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 30b574bba6a..adcfe916eb4 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -167,7 +167,7 @@ def _get_remotes(r_remote): return remotes -def checkout_layers(layers, confdir, layerdir, d): +def checkout_layers(layers, confdir, layerdir, d, rebase_conflicts_strategy='abort'): def _checkout_git_remote(r_remote, repodir, layers_fixed_revisions): rev = r_remote['rev'] branch = r_remote.get('branch', None) @@ -183,7 +183,23 @@ def checkout_layers(layers, confdir, layerdir, d): else: src_uri = f"{fetchuri};protocol={prot};rev={rev};nobranch=1;destsuffix={repodir}" fetcher = bb.fetch.Fetch([src_uri], d) - do_fetch(fetcher, layerdir) + repodir_path = os.path.join(layerdir, repodir) + try: + do_fetch(fetcher, layerdir) + except (bb.fetch2.LocalModificationsError, bb.fetch2.RebaseError) as e: + if rebase_conflicts_strategy != 'backup': + e.msg += ("\nUse 'bitbake-setup update --rebase-conflicts-strategy=backup'" + " to automatically back up the directory and re-clone from upstream," + " or use 'bitbake-setup init -L %s /path/to/local/checkout'" + " to work with a local checkout instead." % r_name) + raise + backup_path = add_unique_timestamp_to_path(repodir_path + '-backup') + logger.warning( + "%s\n" + "Renaming %s to %s to preserve your work, then re-cloning from upstream.", + e, repodir_path, backup_path) + os.rename(repodir_path, backup_path) + fetcher.unpack(layerdir) urldata = fetcher.ud[src_uri] revision = urldata.revision layers_fixed_revisions[r_name]['git-remote']['rev'] = revision @@ -442,9 +458,9 @@ def merge_overrides_into_sources(sources, overrides): layers[k] = v return layers -def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt", init_vscode=False): +def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt", init_vscode=False, rebase_conflicts_strategy='abort'): layer_config = merge_overrides_into_sources(config["data"]["sources"], config["source-overrides"]["sources"]) - sources_fixed_revisions = checkout_layers(layer_config, confdir, layerdir, d) + sources_fixed_revisions = checkout_layers(layer_config, confdir, layerdir, d, rebase_conflicts_strategy=rebase_conflicts_strategy) bitbake_config = config["bitbake-config"] thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf, init_vscode) @@ -927,7 +943,7 @@ def build_status(top_dir, settings, args, d, update=False): logger.plain('\nConfiguration in {} has changed:\n{}'.format(setupdir, config_diff)) if update: update_build(new_upstream_config, confdir, setupdir, layerdir, d, - update_bb_conf=args.update_bb_conf) + update_bb_conf=args.update_bb_conf, rebase_conflicts_strategy=args.rebase_conflicts_strategy) else: bb.process.run('git -C {} restore config-upstream.json'.format(confdir)) return @@ -936,7 +952,7 @@ def build_status(top_dir, settings, args, d, update=False): if are_layers_changed(layer_config, layerdir, d): if update: update_build(current_upstream_config, confdir, setupdir, layerdir, - d, update_bb_conf=args.update_bb_conf) + d, update_bb_conf=args.update_bb_conf, rebase_conflicts_strategy=args.rebase_conflicts_strategy) return logger.plain("\nConfiguration in {} has not changed.".format(setupdir)) @@ -1257,6 +1273,10 @@ def main(): parser_update = subparsers.add_parser('update', help='Update a setup to be in sync with configuration') add_setup_dir_arg(parser_update) parser_update.add_argument('--update-bb-conf', choices=['prompt', 'yes', 'no'], default='prompt', help='Update bitbake configuration files (bblayers.conf, local.conf) (default: prompt)') + parser_update.add_argument('--rebase-conflicts-strategy', choices=['abort', 'backup'], default='abort', + help="What to do when a layer repository has local modifications that prevent " + "an in-place update: 'abort' (default) aborts with an error message; " + "'backup' renames the directory to a timestamped backup and re-clones from upstream.") parser_update.set_defaults(func=build_update) parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine') From 097bffb5c10ff39650721d6a169fdb0e7fd722ac Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:32:59 +0200 Subject: [PATCH 60/86] tests/setup: add test_update_rebase_conflicts_strategy Add a dedicated test for the --rebase-conflicts-strategy option introduced in 'bitbake-setup: add --rebase-conflicts-strategy to the update command'. Three scenarios are covered that are not exercised by the existing test_setup: 1. Uncommitted tracked-file change (LocalModificationsError), default 'abort' strategy: update must fail with an error that contains 'has uncommitted changes' and the '--rebase-conflicts-strategy=backup' hint; no backup directory is created. 2. Same uncommitted change, 'backup' strategy: the layer directory is renamed to a timestamped backup, the layer is re-cloned from upstream, and the result is clean (upstream content, no local modifications). 3. Committed local change that conflicts with the next upstream commit (RebaseError): a. Default 'abort' strategy: update must fail with an error that contains 'Merge conflict' and the '--rebase-conflicts-strategy=backup' hint; no backup directory is created. b. 'backup' strategy: instead of the hard rebase-conflict failure, the conflicted directory is backed up and re-cloned successfully. A small _count_layer_backups() helper is added to the class and reused across scenarios. Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- lib/bb/tests/setup.py | 95 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/lib/bb/tests/setup.py b/lib/bb/tests/setup.py index 52120546ca0..638d56d3bb3 100644 --- a/lib/bb/tests/setup.py +++ b/lib/bb/tests/setup.py @@ -631,3 +631,98 @@ def test_vscode(self): content = f.read() self.assertEqual(content, '{invalid json', "Corrupt workspace file should not be modified") + + def _count_layer_backups(self, layers_path): + return len([f for f in os.listdir(layers_path) if 'backup' in f]) + + def test_update_rebase_conflicts_strategy(self): + """Test the --rebase-conflicts-strategy option for the update command. + + Covers three scenarios not exercised by test_setup: + 1. Uncommitted tracked-file change (LocalModificationsError) + default 'abort' + strategy → clean error message containing 'has uncommitted changes' and a + hint at --rebase-conflicts-strategy=backup; no backup directory is created. + 2. Same uncommitted change + 'backup' strategy → directory is renamed to a + timestamped backup and the layer is re-cloned cleanly. + 3. Committed local change that conflicts with an incoming upstream commit + (RebaseError): + a. Default 'abort' strategy → error containing 'Merge conflict' and the + --rebase-conflicts-strategy=backup hint; no backup directory is created. + b. 'backup' strategy → backup + re-clone instead of a hard failure. + """ + if 'BBPATH' in os.environ: + del os.environ['BBPATH'] + os.chdir(self.tempdir) + + self.runbbsetup("settings set default registry 'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath)) + self.add_file_to_testrepo('test-file', 'initial\n') + self.add_json_config_to_registry('test-config-1.conf.json', 'master', 'master') + self.runbbsetup("init --non-interactive test-config-1 gadget") + + setuppath = self.get_setup_path('test-config-1', 'gadget') + layer_path = os.path.join(setuppath, 'layers', 'test-repo') + layers_path = os.path.join(setuppath, 'layers') + + # Scenario 1: uncommitted tracked change, default 'abort' strategy + # Advance upstream so an update is required. + self.add_file_to_testrepo('test-file', 'upstream-v2\n') + # Modify the same tracked file in the layer without committing. + with open(os.path.join(layer_path, 'test-file'), 'w') as f: + f.write('locally-modified\n') + + os.environ['BBPATH'] = os.path.join(setuppath, 'build') + with self.assertRaises(bb.process.ExecutionError) as ctx: + self.runbbsetup("update --update-bb-conf='no'") + self.assertIn('has uncommitted changes', str(ctx.exception)) + self.assertIn('--rebase-conflicts-strategy=backup', str(ctx.exception)) + # No backup directory must have been created. + self.assertEqual(self._count_layer_backups(layers_path), 0, + "abort strategy must not create any backup") + + # Scenario 2: same uncommitted change, 'backup' strategy + out = self.runbbsetup("update --update-bb-conf='no' --rebase-conflicts-strategy=backup") + # One backup directory must now exist. + self.assertEqual(self._count_layer_backups(layers_path), 1, + "backup strategy must create exactly one backup") + # The re-cloned layer must be clean and at the upstream revision. + with open(os.path.join(layer_path, 'test-file')) as f: + self.assertEqual(f.read(), 'upstream-v2\n', + "re-cloned layer must contain the upstream content") + status = self.git('status --porcelain', cwd=layer_path).strip() + self.assertEqual(status, '', + "re-cloned layer must have no local modifications") + del os.environ['BBPATH'] + + # Scenario 3: committed conflicting change, 'backup' strategy + # Re-initialise a fresh setup so we start from a clean state. + self.runbbsetup("init --non-interactive --setup-dir-name rebase-conflict-setup test-config-1 gadget") + conflict_setup = os.path.join(self.tempdir, 'bitbake-builds', 'rebase-conflict-setup') + conflict_layer = os.path.join(conflict_setup, 'layers', 'test-repo') + conflict_layers = os.path.join(conflict_setup, 'layers') + + # Commit a local change that touches the same file as the next upstream commit. + with open(os.path.join(conflict_layer, 'test-file'), 'w') as f: + f.write('conflicting-local\n') + self.git('add test-file', cwd=conflict_layer) + self.git('commit -m "Local conflicting change"', cwd=conflict_layer) + + # Advance upstream with a conflicting edit. + self.add_file_to_testrepo('test-file', 'conflicting-upstream\n') + + os.environ['BBPATH'] = os.path.join(conflict_setup, 'build') + # Default stop strategy must still fail with a conflict error and include + # the --rebase-conflicts-strategy=backup hint (same handler as LocalModificationsError). + with self.assertRaises(bb.process.ExecutionError) as ctx: + self.runbbsetup("update --update-bb-conf='no'") + self.assertIn('Merge conflict in test-file', str(ctx.exception)) + self.assertIn('--rebase-conflicts-strategy=backup', str(ctx.exception)) + self.assertEqual(self._count_layer_backups(conflict_layers), 0) + + # Backup strategy must succeed: backup the conflicted dir and re-clone. + self.runbbsetup("update --update-bb-conf='no' --rebase-conflicts-strategy=backup") + self.assertEqual(self._count_layer_backups(conflict_layers), 1, + "backup strategy must create exactly one backup after a conflict") + with open(os.path.join(conflict_layer, 'test-file')) as f: + self.assertEqual(f.read(), 'conflicting-upstream\n', + "re-cloned layer must contain the upstream content after conflict backup") + del os.environ['BBPATH'] From 9fb6b0011bae34a7c1feb53281e13364d85ddee8 Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:33:00 +0200 Subject: [PATCH 61/86] bitbake-setup: catch unexpected exceptions and show a clean error message Without a top-level exception handler, any unexpected Python exception reaches the user as a raw traceback, which is confusing and exposes implementation details. Add a try/except block around the main dispatch in main() that: - catches all unexpected exceptions and logs them as a single ERROR line, consistent with how other errors are already presented - re-raises when --debug is active, so the full traceback is still available for diagnosis - explicitly re-raises SystemExit and KeyboardInterrupt so sys.exit() calls and Ctrl-C continue to work normally Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- bin/bitbake-setup | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index adcfe916eb4..b02cbc2b1c4 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -1361,13 +1361,21 @@ def main(): logger.info('Bitbake-setup is using {} as top directory.'.format(top_dir)) - if args.func == init_config: - init_config(top_dir, all_settings, args) - else: - d = init_bb_cache(top_dir, all_settings, args) - args.func(top_dir, all_settings, args, d) - - save_bb_cache() + try: + if args.func == init_config: + init_config(top_dir, all_settings, args) + else: + d = init_bb_cache(top_dir, all_settings, args) + args.func(top_dir, all_settings, args, d) + + save_bb_cache() + except (SystemExit, KeyboardInterrupt): + raise + except Exception as e: + if args.debug: + raise + logger.error(str(e)) + sys.exit(1) else: parser.print_help() From 7cf11787ee7942a00ba7e8fa2c590a2563070c38 Mon Sep 17 00:00:00 2001 From: Adrian Freihofer Date: Sun, 29 Mar 2026 20:33:01 +0200 Subject: [PATCH 62/86] doc: document new bitbake-setup init and update features Document the two new options introduced in the setup command: - --init-vscode / --no-init-vscode (66c26e5f0): generates a bitbake.code-workspace file for VS Code in the Setup directory, with layer folders, BitBake extension settings, and file associations. Default is enabled when the 'code' binary is found on PATH. - --rebase-conflicts-strategy (60ce61cca): controls the fallback behaviour when a layer repository cannot be updated in place due to local modifications or conflicting local commits. Documents both the 'abort' (default) and 'backup' values. Also documents the in-place update behaviour of the update command (fetch via dldir remote + rebase), with a cross-reference to the unpack_update() section in the fetching chapter. Also document that the update command is intended to be run from a shell with the BitBake environment sourced, and why --autostash is not used (transparency: uncommitted changes are surfaced rather than silently stashed and re-applied). Add examples for the four distinct update scenarios, using real output from an actual update session: - normal update with local commits rebased and preserved - update blocked by uncommitted changes, with stash + retry steps - rebase conflict with manual resolution via the dldir git remote, including the rebase --continue + re-run sequence - rebase conflict resolved via --rebase-conflicts-strategy=backup, showing the real WARNING output and timestamped backup directory name, with a step-by-step git workflow (cd into the fresh clone, fetch the backup branch by local path, cherry-pick oldest-first) to recover commits from the backup, and cleanup instructions (rm -rf the backup, or remove the VS Code workspace folder via the UI or by re-running bitbake-setup update after deletion) Signed-off-by: Adrian Freihofer Signed-off-by: Richard Purdie --- .../bitbake-user-manual-environment-setup.rst | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst b/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst index 37518fcebb3..c22e19bd665 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst @@ -327,6 +327,16 @@ In addition, the command can take the following arguments: with a ``local`` source in it. See the :ref:`ref-bbsetup-source-overrides` section for more information on source overrides. +- ``--init-vscode`` / ``--no-init-vscode``: generate (or skip generating) a + ``bitbake.code-workspace`` file in the :term:`Setup` directory. The workspace + file configures the `Yocto Project BitBake + `_ + VS Code extension with paths to the build directory and the init script, and + lists the layer directories as workspace folders. Any user-added folders or + settings in an existing workspace file are preserved across updates. + The default is ``true`` when ``code`` (the VS Code binary) is found on + ``PATH``, and ``false`` otherwise. + ``bitbake-setup init`` Examples ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -410,6 +420,30 @@ the latest changes from the :term:`Configuration Template` it was constructed fr The :ref:`ref-bbsetup-command-status` command can be used to show the current status of the :term:`Setup` before updating it. +This command is intended to be run from a shell where the BitBake environment +has been sourced (e.g. after ``source build/init-build-env``), so that +``bitbake-setup`` can automatically identify the current :term:`Setup` without +requiring the ``--setup-dir`` argument. + +When a layer repository already exists in the :term:`Setup` (i.e. it has been +previously checked out by ``bitbake-setup init`` or a prior ``update``), the +update is performed *in place* using the fetcher's +:ref:`unpack_update ` method: the new upstream revision +is fetched into the local download cache and then rebased on top of the +checkout's current HEAD. This means any local commits in the layer directory +are preserved and rebased onto the new upstream revision. If the working tree +contains staged or unstaged changes to tracked files, the update is blocked +until those changes are committed, stashed or discarded. + +.. note:: + + ``bitbake-setup`` performs the rebase to fast-forward local commits onto + the new upstream revision, but intentionally does not go further by using + ``--autostash`` (which would silently stash uncommitted changes before the + rebase and pop them afterwards). Any uncommitted modifications are surfaced + to the user before the update proceeds, so there are no surprises from an + automatic stash/pop cycle. + In addition, the command can take the following arguments: - ``--update-bb-conf``: whether to update the :term:`BitBake Build` @@ -420,10 +454,181 @@ In addition, the command can take the following arguments: - ``yes``: update the configuration files. - ``no``: don't update the configuration files. +- ``--rebase-conflicts-strategy``: what to do when a layer repository has + local modifications or commits that prevent an in-place update. Accepted + values are: + + - ``abort`` (default): stop with an error message describing the problem. + The repository is left in its previous state (the failed rebase is + automatically aborted). The error message includes a hint to re-run with + ``--rebase-conflicts-strategy=backup``. + - ``backup``: rename the conflicting layer directory to a timestamped + ``-backup-`` path (preserving local work), then + re-clone the layer from upstream into a fresh directory. + - ``--setup-dir``: path to the :term:`Setup` to update. Not required if the command is invoked from an initialized BitBake environment that contains :term:`BBPATH`. +``bitbake-setup update`` Examples +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- This example performs a standard update from an initialized BitBake + environment. Local commits in any layer directory are rebased on top of + the new upstream revision and preserved: + + .. code-block:: shell + + $ bitbake-setup update + NOTE: Bitbake-setup is using /path/to/bitbake-builds as top directory. + NOTE: Looking up config poky-master in configuration registry + NOTE: Layer repository https://git.openembedded.org/openembedded-core + checked out into /path/to/bitbake-builds/poky-master/layers/openembedded-core + updated revision master from d383ea3... to b50d6de... + Fetching layer/tool repositories into /path/to/bitbake-builds/poky-master/layers + bitbake + meta-yocto + openembedded-core + +- This example shows what happens when a layer directory contains staged or + unstaged changes to tracked files. The update is blocked with an error: + + .. code-block:: shell + + $ bitbake-setup update + NOTE: Bitbake-setup is using /path/to/bitbake-builds as top directory. + NOTE: Looking up config poky-master in configuration registry + Fetching layer/tool repositories into /path/to/bitbake-builds/poky-master/layers + bitbake + meta-yocto + openembedded-core + ERROR: Unpack failure for URL: + 'git://git.openembedded.org/openembedded-core;protocol=https;rev=master;branch=master;destsuffix=openembedded-core'. + Repository at /path/to/bitbake-builds/poky-master/layers/openembedded-core has uncommitted changes, unable to update: + M meta/recipes-devtools/ccache/ccache_4.13.1.bb + + Commit, stash or discard your changes and re-run the update. + Use 'bitbake-setup update --rebase-conflicts-strategy=backup' + to automatically back up the directory and re-clone from upstream, + or use 'bitbake-setup init -L openembedded-core /path/to/local/checkout' + to work with a local checkout instead. + + Stashing the changes and re-running resolves the issue: + + .. code-block:: shell + + $ git -C layers/openembedded-core stash + $ bitbake-setup update + $ git -C layers/openembedded-core stash pop + +- This example shows what happens when a layer directory contains local + commits that conflict with the incoming upstream changes. The failed rebase + is automatically aborted, and the ``dldir`` remote is left in the repository + for manual resolution: + + .. code-block:: shell + + $ bitbake-setup update + ERROR: Repository at layers/openembedded-core has local commits that could + not be rebased onto the new upstream revision: + ... + Note: the 'dldir' remote points to the local download cache and may be + used to resolve the conflict manually. + Once resolved, re-run the update. + + The conflict can be resolved manually using the ``dldir`` remote that + ``bitbake-setup`` adds to the repository: + + .. code-block:: shell + + $ git -C layers/openembedded-core rebase dldir/master + # fix conflicts in an editor, then stage the resolved files: + $ git -C layers/openembedded-core add meta/recipes-core/base-files/base-files.bb + $ git -C layers/openembedded-core rebase --continue + $ bitbake-setup update + +- When manual conflict resolution is not desired, the + ``--rebase-conflicts-strategy=backup`` option can be used instead. It + preserves the conflicting directory under a timestamped backup path and + re-clones the layer cleanly from upstream: + + .. code-block:: shell + + $ bitbake-setup update --rebase-conflicts-strategy=backup + NOTE: Bitbake-setup is using /path/to/bitbake-builds as top directory. + NOTE: Looking up config poky-master in configuration registry + NOTE: Layer repository https://git.openembedded.org/openembedded-core checked + out into /path/to/bitbake-builds/poky-master/layers/openembedded-core + updated revision master from 2ec283e... to b50d6de... + Fetching layer/tool repositories into /path/to/bitbake-builds/poky-master/layers + bitbake + meta-yocto + openembedded-core + WARNING: Unpack failure for URL: + 'git://git.openembedded.org/openembedded-core;protocol=https;rev=master;branch=master;destsuffix=openembedded-core'. + Repository at /path/to/bitbake-builds/poky-master/layers/openembedded-core + has local commits that could not be rebased onto the new upstream revision: + ... + Note: the 'dldir' remote points to the local download cache and may be used to resolve the conflict manually. + Once resolved, re-run the update. + Renaming /path/to/bitbake-builds/poky-master/layers/openembedded-core to + /path/to/bitbake-builds/poky-master/layers/openembedded-core-backup.20260329160426 + to preserve your work, then re-cloning from upstream. + + The backup directory is a complete git repository. Local commits can be + recovered from it after the update by fetching a branch from the backup + into the fresh clone (git accepts local paths as remote URLs) and then + cherry-picking the desired commits. For example, given a ``my-wip`` branch + with two commits existing in the backup repository and not in the fresh clone, + the following commands can be used to apply these commits on top of the new + upstream revision in the fresh clone: + + .. code-block:: shell + + $ git -C layers/openembedded-core-backup.20260329160426 log --oneline my-wip + a1b2c3d u-boot: fix compilation with newer GCC + 2ec283e base-files: update version + ... + + $ cd layers/openembedded-core + $ git checkout -b my-wip + Switched to a new branch 'my-wip' + $ git fetch ../openembedded-core-backup.20260329160426 my-wip + + $ git cherry-pick 2ec283e + # resolve any conflicts, then stage the resolved files: + $ git add meta/recipes-core/base-files/base-files.bb + $ git cherry-pick --continue + + $ git cherry-pick a1b2c3d + # resolve any conflicts, then stage the resolved files: + $ git add meta/recipes-devtools/u-boot/u-boot_2026.04.bb + $ git cherry-pick --continue + + $ cd ../.. + + The sequence above is: + + #. Inspect the backup's branch history to identify the commits to recover. + #. Change into the fresh clone and create a matching branch. + #. Fetch the backup branch so its objects become available locally. Git + accepts filesystem paths as remote URLs, and from inside + ``layers/openembedded-core/``, ``../`` points to ``layers/``, where the + backup directory sits. + #. Cherry-pick the commits in oldest-first order. + + Once all desired commits have been recovered and verified, the backup + directory can be removed: + + .. code-block:: shell + + $ rm -rf layers/openembedded-core-backup.20260329160426 + + If a VSCode workspace is in use, the backup directory will appear as an + additional workspace folder until it is cleaned up. It can be removed from + the workspace via the VS Code UI by right-clicking the folder and selecting + *Remove Folder from Workspace*. + .. _ref-bbsetup-command-install-buildtools: ``bitbake-setup install-buildtools`` From fc589ec6f61e5b0f936a90eab0a93b89a8994a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Buli=C5=84ski?= Date: Mon, 30 Mar 2026 15:54:48 +0200 Subject: [PATCH 63/86] Prevent loading unnecessary resources from layerindex-web MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When I was using `bitbake-layers layerindex-fetch ...` I noticed it being quite slow. Upon further investigation, I found that the call to `load_layerindex()` inside layerindexlib was called with the deafult `load` argument. The deafault was set to ['layerDependencies', 'recipes', 'machines', 'distros'] where the `layerindex-fetch` effectively requires only ['layerDependencies']. So whenever a `layerindex-fetch` is used, we pull for example 'recipes', which is about 18-20MB in size - it takes time to produce that response by the layerindex-web and ship over the internet, just to be discarded. This change addresses the issue by setting a "sane" default (empty `load` list) and sets a correct, explicit `load` list whenever it's needed (e.g. toaster) by the caller. Lastly, there seems to be an error when the cooker index is loaded in the process with the `load='layerDependencies'`, that's incorrect, as the implementation expects also a list, so that's corrected as well. Signed-off-by: Piotr BuliƄski Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- lib/bblayers/layerindex.py | 4 ++-- lib/layerindexlib/__init__.py | 6 +++--- lib/toaster/orm/management/commands/lsupdates.py | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/bblayers/layerindex.py b/lib/bblayers/layerindex.py index 308a5532d9f..2ba1103a3ee 100644 --- a/lib/bblayers/layerindex.py +++ b/lib/bblayers/layerindex.py @@ -116,7 +116,7 @@ def _construct_url(baseurls, branches): # Load the cooker DB cookerIndex = layerindexlib.LayerIndex(self.tinfoil.config_data) - cookerIndex.load_layerindex('cooker://', load='layerDependencies') + cookerIndex.load_layerindex('cooker://', load=['layerDependencies']) # Fast path, check if we already have what has been requested! (dependencies, invalidnames) = cookerIndex.find_dependencies(names=args.layername, ignores=ignore_layers) @@ -137,7 +137,7 @@ def _construct_url(baseurls, branches): for remoteurl in _construct_url(apiurl, branches): logger.plain("Loading %s..." % remoteurl) - remoteIndex.load_layerindex(remoteurl) + remoteIndex.load_layerindex(remoteurl, load=['layerDependencies']) if remoteIndex.is_empty(): logger.error("Remote layer index %s is empty for branches %s" % (apiurl, branches)) diff --git a/lib/layerindexlib/__init__.py b/lib/layerindexlib/__init__.py index c3265ddaa14..e693e5c1105 100644 --- a/lib/layerindexlib/__init__.py +++ b/lib/layerindexlib/__init__.py @@ -174,15 +174,15 @@ def _fetch_url(self, url, username=None, password=None, debuglevel=0): return res - def load_layerindex(self, indexURI, load=['layerDependencies', 'recipes', 'machines', 'distros'], reload=False): + def load_layerindex(self, indexURI, load=[], reload=False): '''Load the layerindex. indexURI - An index to load. (Use multiple calls to load multiple indexes) reload - If reload is True, then any previously loaded indexes will be forgotten. - load - List of elements to load. Default loads all items. - Note: plugs may ignore this. + load - List of elements to load. By default, an empty list is used to keep things lean. + Callers need to specify a minimal set of elements to load, such as ['layerDependencies'] for dependency resolution. The format of the indexURI: diff --git a/lib/toaster/orm/management/commands/lsupdates.py b/lib/toaster/orm/management/commands/lsupdates.py index 0ee00aa1595..153b26f60a5 100644 --- a/lib/toaster/orm/management/commands/lsupdates.py +++ b/lib/toaster/orm/management/commands/lsupdates.py @@ -105,7 +105,8 @@ def update(self): url_branches = ";branch=%s" % ','.join(allowed_branch_names) else: url_branches = "" - layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches)) + layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches), + load=['layerDependencies', 'recipes', 'machines', 'distros']) http_progress.stop() From a160b4cc0049ff6d76ac7113a760db77789996f1 Mon Sep 17 00:00:00 2001 From: Zhangfei Gao Date: Fri, 20 Mar 2026 09:44:07 +0000 Subject: [PATCH 64/86] bitbake-setup: make DL_DIR in site.conf overridable by env Use '?=' instead of '=' when writing DL_DIR to site.conf, so an exported DL_DIR can be reused across environments. This keeps a default value while allowing environment-based cache configuration in mixed workflows (e.g. container/host). Signed-off-by: Zhangfei Gao Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- bin/bitbake-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index b02cbc2b1c4..535b5aa052f 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -1114,7 +1114,7 @@ def create_siteconf(top_dir, non_interactive, settings): # wiping and rebuilding you can preserve this directory to speed up this part of # subsequent builds. This directory is safe to share between multiple builds on # the same machine too. - DL_DIR = "{dl_dir}" + DL_DIR ?= "{dl_dir}" """.format( dl_dir=settings["default"]["dl-dir"], ) From 92238d267122c7b0e7c9e18632abbe0bbce0645b Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Tue, 24 Mar 2026 10:44:46 +0100 Subject: [PATCH 65/86] bitbake-setup: ensure args.setup_dir is an absolute path This fixes another corner case: 'bitbake-setup update --setup-dir ./relative/path' Normally setup-dir is deduced from bitbake environment, but when it is passed in on the command line as a relative path, and there is something to update, new bblayer.conf is created with relative paths in it, causing breakage. Signed-off-by: Alexander Kanavin Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- bin/bitbake-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 535b5aa052f..c006a059c09 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -922,7 +922,7 @@ def are_layers_changed(layers, layerdir, d): return changed def build_status(top_dir, settings, args, d, update=False): - setupdir = args.setup_dir + setupdir = os.path.abspath(args.setup_dir) confdir = os.path.join(setupdir, "config") layerdir = os.path.join(setupdir, "layers") From 7c42f89fb7d5e9e19434d199415738e0ad79e79c Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Thu, 26 Mar 2026 09:54:57 +0100 Subject: [PATCH 66/86] doc/bitbake-user-manual-fetching: add warning on the Git subpath parameter [YOCTO #15558] Add a warning on the subpath option which has an unexpected behavior with regards to how the HEAD is updated in the checked out repository. Signed-off-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- .../bitbake-user-manual-fetching.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index c2747c40124..1f7583ff307 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -425,6 +425,21 @@ This fetcher supports the following parameters: - *"subpath":* Limits the checkout to a specific subpath of the tree. By default, the whole tree is checked out. + .. warning:: + + When using this option, the value of :term:`SRCREV` may not be reflected + in the checked out repository. + + To achieve a partial checkout of the repository with the ``subpath`` + option, the BitBake Git fetcher makes use of Git "plumbing" commands: `git + read-tree `__ and `git + checkout-index `__. However, + these commands only update the **files** within the repository, and do not + update the current ``HEAD`` to point to the commit specified by + :term:`SRCREV`. Instead, the value of ``HEAD`` will always point to the + tip of the branch specified by the ``branch`` parameter, which may or may + not correspond to :term:`SRCREV`. + - *"destsuffix":* The name of the path in which to place the checkout. By default, the path is ``git/``. From 13ee2e78e22dfab32d10c45f1451890c8c3733c6 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Thu, 26 Mar 2026 20:23:31 +0100 Subject: [PATCH 67/86] doc: bitbake-user-manual-metadata: fix syntax for code-block in addfragments OE_FRAGMENTS was meant as a code-block and thus needs two colons, a newline and an increased indent for it to work, however we were missing the second colon. Let's fix that oversight. Fixes: 3b9d7bea915d ("parse/ast: add support for 'built-in' fragments") Signed-off-by: Quentin Schulz Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- doc/bitbake-user-manual/bitbake-user-manual-metadata.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index bf517f03033..56604e0ddc8 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -1052,7 +1052,7 @@ The variable containing a built-in fragment definitions could look like this:: OE_FRAGMENTS_BUILTIN = "someprefix:SOMEVARIABLE anotherprefix:ANOTHERVARIABLE" and then if 'someprefix/somevalue' is added to the variable that holds the list -of enabled fragments: +of enabled fragments:: OE_FRAGMENTS = "... someprefix/somevalue" From a3b66c4c29cdf0569e3ada1b99b4f2fa35f52a8c Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Thu, 26 Mar 2026 20:23:32 +0100 Subject: [PATCH 68/86] doc: bitbake-user-manual-metadata: fix indentation consistency We've decided code-blocks should be indented by three whitespaces compared to the parent block and this only had two, so let's add the missing whitespace. This is cosmetic in the sources only, it doesn't change the output. Fixes: 3b9d7bea915d ("parse/ast: add support for 'built-in' fragments") Signed-off-by: Quentin Schulz Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- doc/bitbake-user-manual/bitbake-user-manual-metadata.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index 56604e0ddc8..4909b0d757b 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -1054,11 +1054,11 @@ The variable containing a built-in fragment definitions could look like this:: and then if 'someprefix/somevalue' is added to the variable that holds the list of enabled fragments:: - OE_FRAGMENTS = "... someprefix/somevalue" + OE_FRAGMENTS = "... someprefix/somevalue" bitbake will treat that as direct value assignment in its configuration:: - SOMEVARIABLE = "somevalue" + SOMEVARIABLE = "somevalue" Locating Include Files ---------------------- From 7fd4d9d10524ac798ba96057462ea77e6998b0ff Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Thu, 26 Mar 2026 20:23:33 +0100 Subject: [PATCH 69/86] doc: bitbake-user-manual-metadata: fix indentation consistency We've decided bullet lists should be started by a dash followed by two whitespaces and the other lines in the bullet list should be indented at that level, that is with three whitespaces compared to the parent block. This only has two while the rest of the file has three, so let's add the missing whitespace. This is cosmetic in the sources only, it doesn't change the output. Fixes: 6bc65e6402a7 ("documentation: bitbake: add file-checksums to varflags section") Signed-off-by: Quentin Schulz Signed-off-by: Mathieu Dubois-Briand Signed-off-by: Richard Purdie --- .../bitbake-user-manual-metadata.rst | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index 4909b0d757b..40cae6b05be 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -1771,22 +1771,22 @@ functionality of the task: directory listed is used as the current working directory for the task. -- ``[file-checksums]``: Controls the file dependencies for a task. The - baseline file list is the set of files associated with - :term:`SRC_URI`. May be used to set additional dependencies on - files not associated with :term:`SRC_URI`. - - The value set to the list is a file-boolean pair where the first - value is the file name and the second is whether or not it - physically exists on the filesystem. :: - - do_configure[file-checksums] += "${MY_DIRPATH}/my-file.txt:True" - - It is important to record any paths which the task looked at and - which didn't exist. This means that if these do exist at a later - time, the task can be rerun with the new additional files. The - "exists" True or False value after the path allows this to be - handled. +- ``[file-checksums]``: Controls the file dependencies for a task. The + baseline file list is the set of files associated with + :term:`SRC_URI`. May be used to set additional dependencies on + files not associated with :term:`SRC_URI`. + + The value set to the list is a file-boolean pair where the first + value is the file name and the second is whether or not it + physically exists on the filesystem. :: + + do_configure[file-checksums] += "${MY_DIRPATH}/my-file.txt:True" + + It is important to record any paths which the task looked at and + which didn't exist. This means that if these do exist at a later + time, the task can be rerun with the new additional files. The + "exists" True or False value after the path allows this to be + handled. - ``[lockfiles]``: Specifies one or more lockfiles to lock while the task executes. Only one task may hold a lockfile, and any task that From 5d722b5d65e4eef7befe6376983385421e993f86 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Wed, 8 Apr 2026 07:28:41 +0100 Subject: [PATCH 70/86] tests/fetch: Avoid using git protocol in tests Two of the tests were still using git protocol to access git services. For the submodule test, the upstream repo has been updated. In the other case, we need to pass the correct command to the manual git commandline, we can't use a recipe url that previously just happened to work. Signed-off-by: Richard Purdie --- lib/bb/tests/fetch.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 5b3fc8a4191..077f741e1d3 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -1114,7 +1114,7 @@ def test_git_submodule(self): # URL with ssh submodules url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https" # Original URL (comment this if you have ssh access to git.yoctoproject.org) - url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https" + url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=38e61644af90dccd73c03ed3acaed98c8dda9294;branch=master;protocol=https" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -3482,6 +3482,7 @@ def setUp(self): self.reponame = "fstests" self.clonedir = os.path.join(self.tempdir, "git") self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) + self.giturl = "https://git.yoctoproject.org/fstests" self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https;branch=master" self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") self.d.setVar("BB_NO_NETWORK", "0") @@ -3490,7 +3491,7 @@ def setUp(self): def make_git_repo(self): self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" os.makedirs(self.clonedir) - self.git("clone --bare {}".format(self.recipe_url), self.clonedir) + self.git("clone --bare {}".format(self.giturl), self.clonedir) self.git("update-ref HEAD 15413486df1f5a5b5af699b6f3ba5f0984e52a9f", self.gitdir) bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) shutil.rmtree(self.clonedir) From de7c87ae881ae18e7e063c6010643267f93faff3 Mon Sep 17 00:00:00 2001 From: Antonin Godard Date: Mon, 13 Apr 2026 09:58:47 +0200 Subject: [PATCH 71/86] doc: set_versions.py: change strategy for the default page A problem can happen with the current code: if the current stable release is made EOL before the development branch is made the current stable, then we only have LTS releases displayed. Change the logic to _always_ display the latest non-dev branch as the default page, even if EOL. Signed-off-by: Antonin Godard Signed-off-by: Mathieu Dubois-Briand --- doc/setversions.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/setversions.py b/doc/setversions.py index 82a0e8b191b..f1aea0a4225 100755 --- a/doc/setversions.py +++ b/doc/setversions.py @@ -65,14 +65,25 @@ bb_ver = release["bitbake_version"] if release["status"] == "Active Development": DEVBRANCH = bb_ver - if release["series"] == "current": - ACTIVERELEASES.append(bb_ver) if "LTS until" in release["status"]: LTSSERIES.append(bb_ver) if release["bitbake_version"]: YOCTO_MAPPING[bb_ver] = release["release_codename"] - ACTIVERELEASES.remove(DEVBRANCH) + # Find the first non-dev release, which should be displayed as the default + # page on the docs website. + current_branch = "" + for release in RELEASES_FROM_JSON: + if release["status"] != "Active Development": + current_branch = release["bitbake_version"] + break + + if not current_branch: + sys.exit("Unable to find a current release! Exiting...") + + # make the list of releases unique, there can be duplication when the + # current releases is also an LTS + ACTIVERELEASES = list(dict.fromkeys([current_branch] + LTSSERIES)) print(f"ACTIVERELEASES calculated to be {ACTIVERELEASES}", file=sys.stderr) print(f"DEVBRANCH calculated to be {DEVBRANCH}", file=sys.stderr) From 1ffccdeaa7d46bcfcb05f8887d88276a07bfab53 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 13 Apr 2026 09:59:31 +0000 Subject: [PATCH 72/86] runqueue: Fix comment typos Fix comment typos in RunQueueData.prepare Signed-off-by: Yann Dirson Signed-off-by: Mathieu Dubois-Briand --- lib/bb/runqueue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 239fa770e05..a4689e134ad 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -694,7 +694,7 @@ def prepare(self): # To create the actual list of tasks to execute we fix the list of # providers and then resolve the dependencies into task IDs. This # process is repeated for each type of dependency (tdepends, deptask, - # rdeptast, recrdeptask, idepends). + # rdeptask, recrdeptask, idepends). def add_build_dependencies(depids, tasknames, depends, mc): for depname in depids: @@ -842,7 +842,7 @@ def add_mc_dependencies(mc, tid): # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed - # Generating/interating recursive lists of dependencies is painful and potentially slow + # Generating/iterating recursive lists of dependencies is painful and potentially slow # Precompute recursive task dependencies here by: # a) create a temp list of reverse dependencies (revdeps) # b) walk up the ends of the chains (when a given task no longer has dependencies i.e. len(deps) == 0) From 23a91cb156ac71e3a855fd844621fe913deb3b37 Mon Sep 17 00:00:00 2001 From: Mark-Pk Tsai Date: Wed, 15 Apr 2026 15:55:06 +0800 Subject: [PATCH 73/86] fetch2/git: fix HEAD and branch ref when subpath is used When subpath is set the checkout code uses read-tree + checkout-index, but never updates HEAD. It stays on whatever branch the clone defaulted to instead of pointing to the requested revision. Fix by pointing HEAD at the correct revision after checkout-index, and creating the branch ref when branch= is specified. Also remove the warning note in the documentation that described this limitation. Signed-off-by: Mark-Pk Tsai Signed-off-by: Mathieu Dubois-Briand --- .../bitbake-user-manual-fetching.rst | 15 --------------- lib/bb/fetch2/git.py | 6 ++++++ 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index 1f7583ff307..c2747c40124 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -425,21 +425,6 @@ This fetcher supports the following parameters: - *"subpath":* Limits the checkout to a specific subpath of the tree. By default, the whole tree is checked out. - .. warning:: - - When using this option, the value of :term:`SRCREV` may not be reflected - in the checked out repository. - - To achieve a partial checkout of the repository with the ``subpath`` - option, the BitBake Git fetcher makes use of Git "plumbing" commands: `git - read-tree `__ and `git - checkout-index `__. However, - these commands only update the **files** within the repository, and do not - update the current ``HEAD`` to point to the commit specified by - :term:`SRCREV`. Instead, the value of ``HEAD`` will always point to the - tip of the branch specified by the ``branch`` parameter, which may or may - not correspond to :term:`SRCREV`. - - *"destsuffix":* The name of the path in which to place the checkout. By default, the path is ``git/``. diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 6ed14005b53..21f5f28a029 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -787,6 +787,12 @@ def unpack(self, ud, destdir, d, update=False): runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d, workdir=destdir) runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) + runfetchcmd("%s update-ref --no-deref HEAD %s" % (ud.basecmd, ud.revision), + d, workdir=destdir) + if not ud.nobranch: + branchname = ud.branch + runfetchcmd("%s update-ref refs/heads/%s %s" % (ud.basecmd, branchname, + ud.revision), d, workdir=destdir) elif not ud.nobranch: branchname = ud.branch runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ From 3c20c0a4d850d4e4f3aa342d98284a9c7c9df3d9 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 13 Apr 2026 09:59:32 +0000 Subject: [PATCH 74/86] utils: goh1_file: deal with false positives from is_zipfile This function is known https://github.com/python/cpython/issues/72680 for false-positives. With python 3.13.5 there is one with https://vault.almalinux.org/10.0/CRB/x86_64_v2/os/Packages/jdom2-2.0.6.1-8.el10.noarch.rpm The double "is_zipfile = False" is redundant but likely more clear. Signed-off-by: Yann Dirson Signed-off-by: Mathieu Dubois-Briand --- lib/bb/utils.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/bb/utils.py b/lib/bb/utils.py index b2bda08cb7d..7c5c5e8e65d 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -694,14 +694,19 @@ def goh1_file(filename): import zipfile lines = [] + is_zipfile = False if zipfile.is_zipfile(filename): - with zipfile.ZipFile(filename) as archive: - for fn in sorted(archive.namelist()): - method = hashlib.sha256() - method.update(archive.read(fn)) - hash = method.hexdigest() - lines.append("%s %s\n" % (hash, fn)) - else: + try: + with zipfile.ZipFile(filename) as archive: + for fn in sorted(archive.namelist()): + method = hashlib.sha256() + method.update(archive.read(fn)) + hash = method.hexdigest() + lines.append("%s %s\n" % (hash, fn)) + is_zipfile = True + except zipfile.BadZipFile: + is_zipfile = False + if not is_zipfile: hash = _hasher(hashlib.sha256(), filename) lines.append("%s go.mod\n" % hash) method = hashlib.sha256() From 1557c502a00e2519abd77fc5a5781489ad7f0b2f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 13 Apr 2026 09:59:30 +0000 Subject: [PATCH 75/86] taskdata: add_tasks: use sets to dedup from debug output When packages appear multiple times it gets hard to make any sense of the list display. Using sets does the job for us. Signed-off-by: Yann Dirson Signed-off-by: Mathieu Dubois-Briand --- lib/bb/taskdata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/bb/taskdata.py b/lib/bb/taskdata.py index 66545a65af9..b40afc02b71 100644 --- a/lib/bb/taskdata.py +++ b/lib/bb/taskdata.py @@ -138,15 +138,15 @@ def handle_deps(task, dep_name, depends, seen): rdependids = set() rdepends = dataCache.rundeps[fn] rrecs = dataCache.runrecs[fn] - rdependlist = [] - rreclist = [] + rdependlist = set() + rreclist = set() for package in rdepends: for rdepend in rdepends[package]: - rdependlist.append(rdepend) + rdependlist.add(rdepend) rdependids.add(rdepend) for package in rrecs: for rdepend in rrecs[package]: - rreclist.append(rdepend) + rreclist.add(rdepend) rdependids.add(rdepend) if rdependlist: logger.debug2("Added runtime dependencies %s for %s", str(rdependlist), fn) From 6acd484971b137dc96744296982ea3d4da9daf9e Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Tue, 31 Mar 2026 09:06:56 -0700 Subject: [PATCH 76/86] bitbake-setup: Add the conditional script stanza This ensures that the main function is only executed if the module is executed in the top-level environment. Signed-off-by: Rob Woolley Signed-off-by: Richard Purdie --- bin/bitbake-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/bitbake-setup b/bin/bitbake-setup index c006a059c09..220540f7f9c 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -1379,4 +1379,5 @@ def main(): else: parser.print_help() -main() +if __name__ == '__main__': + main() From cfd3af4651cde26dbe32457160334b09b934218f Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Wed, 22 Apr 2026 10:55:48 +0100 Subject: [PATCH 77/86] default-registry: Add wrynose Add the new release Signed-off-by: Richard Purdie --- .../oe-nodistro-wrynose.conf.json | 48 ++++++++++++ .../configurations/poky-wrynose.conf.json | 74 +++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 default-registry/configurations/oe-nodistro-wrynose.conf.json create mode 100644 default-registry/configurations/poky-wrynose.conf.json diff --git a/default-registry/configurations/oe-nodistro-wrynose.conf.json b/default-registry/configurations/oe-nodistro-wrynose.conf.json new file mode 100644 index 00000000000..278ce4dd029 --- /dev/null +++ b/default-registry/configurations/oe-nodistro-wrynose.conf.json @@ -0,0 +1,48 @@ +{ + "description": "OpenEmbedded - 'nodistro' basic configuration, release 6.0 'wrynose'", + "expires": "2030-05-31", + "sources": { + "bitbake": { + "git-remote": { + "uri": "https://git.openembedded.org/bitbake", + "branch": "2.18", + "rev": "2.18" + } + }, + "openembedded-core": { + "git-remote": { + "uri": "https://git.openembedded.org/openembedded-core", + "branch": "wrynose", + "rev": "wrynose" + } + }, + "yocto-docs": { + "git-remote": { + "uri": "https://git.yoctoproject.org/yocto-docs", + "branch": "wrynose", + "rev": "wrynose" + } + } + }, + "bitbake-setup": { + "configurations": [ + { + "name": "nodistro", + "description": "OpenEmbedded 'nodistro'", + "setup-dir-name": "oe-nodistro-wrynose", + "bb-layers": ["openembedded-core/meta"], + "oe-fragments-one-of": { + "machine": { + "description": "Target machines", + "options" : [ + { "name": "machine/qemux86-64", "description": "x86-64 system on QEMU" }, + { "name": "machine/qemuarm64", "description": "ARMv8 system on QEMU" }, + { "name": "machine/qemuriscv64", "description": "RISC-V system on QEMU" } + ] + } + } + } + ] + }, + "version": "1.0" +} diff --git a/default-registry/configurations/poky-wrynose.conf.json b/default-registry/configurations/poky-wrynose.conf.json new file mode 100644 index 00000000000..d2792843773 --- /dev/null +++ b/default-registry/configurations/poky-wrynose.conf.json @@ -0,0 +1,74 @@ +{ + "description": "Poky - The Yocto Project testing distribution configurations and hardware test platforms, release 6.0 'wrynose'", + "expires": "2030-05-31", + "sources": { + "bitbake": { + "git-remote": { + "uri": "https://git.openembedded.org/bitbake", + "branch": "2.18", + "rev": "2.18" + } + }, + "openembedded-core": { + "git-remote": { + "uri": "https://git.openembedded.org/openembedded-core", + "branch": "wrynose", + "rev": "wrynose" + } + }, + "meta-yocto": { + "git-remote": { + "uri": "https://git.yoctoproject.org/meta-yocto", + "branch": "wrynose", + "rev": "wrynose" + } + }, + "yocto-docs": { + "git-remote": { + "uri": "https://git.yoctoproject.org/yocto-docs", + "branch": "wrynose", + "rev": "wrynose" + } + } + }, + "bitbake-setup": { + "configurations": [ + { + "bb-layers": ["openembedded-core/meta","meta-yocto/meta-yocto-bsp","meta-yocto/meta-poky"], + "setup-dir-name": "$distro-wrynose", + "oe-fragments-one-of": { + "machine": { + "description": "Target machines", + "options" : [ + { "name": "machine/qemux86-64", "description": "x86-64 system on QEMU" }, + { "name": "machine/qemuarm64", "description": "ARMv8 system on QEMU" }, + { "name": "machine/qemuriscv64", "description": "RISC-V system on QEMU" }, + { "name": "machine/genericarm64", "description": "Arm64 SystemReady IR/ES platforms" }, + { "name": "machine/genericx86-64", "description": "x86_64 (64-bit) PCs and servers" } + ] + }, + "distro": { + "description": "Target distributions", + "options" : [ + { "name": "distro/poky", "description": "Yocto Project Reference Distro" }, + { "name": "distro/poky-altcfg", "description": "Poky alternative with systemd as init manager" }, + { "name": "distro/poky-tiny", "description": "Poky alternative optimized for size" } + ] + } + }, + "configurations": [ + { + "name": "poky", + "description": "Poky - The Yocto Project testing distribution" + }, + { + "name": "poky-with-sstate", + "description": "Poky - The Yocto Project testing distribution with internet sstate acceleration. Use with caution as it requires a completely robust local network with sufficient bandwidth.", + "oe-fragments": ["core/yocto/sstate-mirror-cdn"] + } + ] + } + ] + }, + "version": "1.0" +} From b508e646b539b7260422b2ffd38168742094ebf3 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 23 Apr 2026 12:08:01 +0100 Subject: [PATCH 78/86] utils: Add filter_string function There are cases where we would like the filter functionality but don't want to read the data from a variable. Add a new function, filter_string to allow this, basically separating filter() into two functions. Signed-off-by: Richard Purdie --- lib/bb/utils.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/bb/utils.py b/lib/bb/utils.py index 7c5c5e8e65d..b04ff6ffc76 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -1336,21 +1336,18 @@ def contains_any(variable, checkvalues, truevalue, falsevalue, d): return truevalue return falsevalue -def filter(variable, checkvalues, d): - """Return all words in the variable that are present in the ``checkvalues``. +def filter_string(val, checkvalues): + """Return all words in the string that are present in the ``checkvalues``. Arguments: - - ``variable``: the variable name. This will be fetched and expanded (using - d.getVar(variable)) and then split into a set(). + - ``val``: the string data to filter after being split into a set(). - ``checkvalues``: if this is a string it is split on whitespace into a set(), otherwise coerced directly into a set(). - ``d``: the data store. Returns a list of string. """ - - val = d.getVar(variable) if not val: return '' val = set(val.split()) @@ -1360,6 +1357,22 @@ def filter(variable, checkvalues, d): checkvalues = set(checkvalues) return ' '.join(sorted(checkvalues & val)) +def filter(variable, checkvalues, d): + """Return all words in the variable that are present in the ``checkvalues``. + + Arguments: + + - ``variable``: the variable name. This will be fetched and expanded (using + d.getVar(variable)) and then split into a set(). + - ``checkvalues``: if this is a string it is split on whitespace into a set(), + otherwise coerced directly into a set(). + - ``d``: the data store. + + Returns a list of string. + """ + + val = d.getVar(variable) + return filter_string(val, checkvalues) def get_referenced_vars(start_expr, d): """ From 33581c84f3a85008239acbd940501a35de48dc91 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 23 Apr 2026 14:31:28 +0100 Subject: [PATCH 79/86] bitbake: Update version to 2.18.0 Signed-off-by: Richard Purdie --- bin/bitbake | 2 +- lib/bb/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/bitbake b/bin/bitbake index a995bd66535..5a820417b2a 100755 --- a/bin/bitbake +++ b/bin/bitbake @@ -27,7 +27,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException bb.utils.check_system_locale() -__version__ = "2.16.0" +__version__ = "2.18.0" if __name__ == "__main__": if __version__ != bb.__version__: diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py index 6765fa6134d..77edfa2bc98 100644 --- a/lib/bb/__init__.py +++ b/lib/bb/__init__.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -__version__ = "2.16.0" +__version__ = "2.18.0" import sys if sys.version_info < (3, 9, 0): From d1f775f266610a79d5fbfe420b2843288c0011fe Mon Sep 17 00:00:00 2001 From: Johan Anderholm Date: Mon, 27 Apr 2026 15:51:23 +0200 Subject: [PATCH 80/86] fetch2/crate: use CDN for fetching crates This avoids the 1 req/sec that the API has. Reference: https://github.com/rust-lang/crates.io/issues/13482 Reference: https://blog.rust-lang.org/2024/03/11/crates-io-download-changes/ Signed-off-by: Johan Anderholm Signed-off-by: Richard Purdie --- lib/bb/fetch2/crate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py index e611736f069..2d30788998a 100644 --- a/lib/bb/fetch2/crate.py +++ b/lib/bb/fetch2/crate.py @@ -68,8 +68,11 @@ def _crate_urldata_init(self, ud, d): # if using upstream just fix it up nicely if host == 'crates.io': host = 'crates.io/api/v1/crates' + cdn_host = 'static.crates.io/crates' + else: + cdn_host = host - ud.url = "https://%s/%s/%s/download" % (host, name, version) + ud.url = "https://%s/%s/%s/download" % (cdn_host, name, version) ud.versionsurl = "https://%s/%s/versions" % (host, name) ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) if 'name' not in ud.parm: From a2dd9be788274d9c7280be5b1be5eb5c3990cf54 Mon Sep 17 00:00:00 2001 From: Ross Burton Date: Wed, 29 Apr 2026 15:44:01 +0100 Subject: [PATCH 81/86] fetch2/crate: use CDN endpoint for version checking if possible If the crate host is crates.io then we can fetch the index for the crate from index.crates.io instead of hitting the API. This is the recommended way to do automated checks and means we don't break the data access policy[1] by not setting an explicit User-Agent. [1] https://crates.io/data-access Signed-off-by: Ross Burton Signed-off-by: Richard Purdie --- lib/bb/fetch2/crate.py | 43 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py index 2d30788998a..b89817ab94d 100644 --- a/lib/bb/fetch2/crate.py +++ b/lib/bb/fetch2/crate.py @@ -45,6 +45,17 @@ def urldata_init(self, ud, d): super(Crate, self).urldata_init(ud, d) + def _generate_index_path(self, name): + # https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files + if len(name) == 1: + return f"1/{name}" + elif len(name) == 2: + return f"2/{name}" + elif len(name) == 3: + return f"3/{name[0]}/{name}" + else: + return f"{name[0:2]}/{name[2:4]}/{name}" + def _crate_urldata_init(self, ud, d): """ Sets up the download for a crate @@ -65,15 +76,15 @@ def _crate_urldata_init(self, ud, d): # host (this is to allow custom crate registries to be specified host = '/'.join(parts[2:-2]) - # if using upstream just fix it up nicely + # If using crates.io use the CDN directly as per https://crates.io/data-access if host == 'crates.io': - host = 'crates.io/api/v1/crates' - cdn_host = 'static.crates.io/crates' + ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version) + ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name) + self.latest_versionstring = self.latest_versionstring_from_index else: - cdn_host = host + ud.url = "https://%s/%s/%s/download" % (host, name, version) + ud.versionsurl = "https://%s/%s/versions" % (host, name) - ud.url = "https://%s/%s/%s/download" % (cdn_host, name, version) - ud.versionsurl = "https://%s/%s/versions" % (host, name) ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) if 'name' not in ud.parm: ud.parm['name'] = '%s-%s' % (name, version) @@ -145,9 +156,29 @@ def _crate_unpack(self, ud, rootdir, d): json.dump(metadata, f) def latest_versionstring(self, ud, d): + """ + Return the latest version available when versionsurl is the [name]/versions URL. + """ from functools import cmp_to_key json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d)) versions = [(0, i["num"], "") for i in json_data["versions"]] versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp)) return (versions[-1][1], "") + + def latest_versionstring_from_index(self, ud, d): + """ + Return the latest version available when versionsurl is a Cargo index + file. + https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files + """ + from functools import cmp_to_key + + versions = [] + response = self._fetch_index(ud.versionsurl, ud, d) + for line in response.splitlines(): + data = json.loads(line) + versions.append((0, data["vers"], "")) + + versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp)) + return (versions[-1][1], "") From a82590d57b17e797575deb27ed29097d8713c9fa Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Wed, 13 May 2026 15:39:34 +0200 Subject: [PATCH 82/86] fetch/git: Fix leaking of temporary directory We create a temporary directory for holding a clone but we never clean it up. Fix this by using a context manager areound the temporary directory. This resolves a buildup of tmp directories in DL_DIR in builds. Signed-off-by: Richard Purdie --- lib/bb/fetch2/git.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 21f5f28a029..17cb38cf396 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -392,14 +392,14 @@ def download(self, ud, d): bb.utils.mkdirhier(ud.clonedir) runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) else: - tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) - runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir) - output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) - if 'mirror' in output: - runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir) - runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd) - runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) + with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir: + runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir) + output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) + if 'mirror' in output: + runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir) + runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir) + fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd) + runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) repourl = self._get_repo_url(ud) needs_clone = False From 0415a00658b59f66bdab22a3558794f238c42973 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Wed, 13 May 2026 15:39:35 +0200 Subject: [PATCH 83/86] fetch/git: Improve temporary directory handling Whilst this code doesn't leak a temporary directory, it is a little unsafe around directory creation/reuse due to the cleanup and retry logic. Restructure the function so it can use tempfile.TemporaryDirectory as a context manager with a context per loop iteration. Behavour shouldn't be changed, the deletion/recreation is just removed. Signed-off-by: Richard Purdie --- lib/bb/fetch2/git.py | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 17cb38cf396..ecf4340b1bc 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -543,27 +543,23 @@ def build_mirror_data(self, ud, d): runfetchcmd("touch %s.done" % ud.fullmirror, d) def clone_shallow_with_tarball(self, ud, d): - ret = False - tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) - shallowclone = os.path.join(tempdir, 'git') - try: - try: - self.clone_shallow_local(ud, shallowclone, d) - except: - logger.warning("Fast shallow clone failed, try to skip fast mode now.") - bb.utils.remove(tempdir, recurse=True) - os.mkdir(tempdir) - ud.shallow_skip_fast = True - self.clone_shallow_local(ud, shallowclone, d) - logger.info("Creating tarball of git repository") - with self.create_atomic(ud.fullshallow) as tfile: - runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) - runfetchcmd("touch %s.done" % ud.fullshallow, d) - ret = True - finally: - bb.utils.remove(tempdir, recurse=True) - - return ret + for fast in [True, False]: + ud.shallow_skip_fast = not fast + with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tempdir: + shallowclone = os.path.join(tempdir, 'git') + try: + self.clone_shallow_local(ud, shallowclone, d) + except: + if not fast: + raise + logger.warning("Fast shallow clone failed, try to skip fast mode now.") + continue + logger.info("Creating tarball of git repository") + with self.create_atomic(ud.fullshallow) as tfile: + runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) + runfetchcmd("touch %s.done" % ud.fullshallow, d) + return True + return False def clone_shallow_local(self, ud, dest, d): """ From 41393883d5e9f745b714e2ccb077c085b3c0f25b Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Wed, 22 Apr 2026 17:19:12 +0200 Subject: [PATCH 84/86] fetch/wget: in upstream version checks, match versioned directories exactly This change is in code that iterates over a html page, picking out links to versioned subdirectories to check further for new versions. Without the ^ and $ guards, there would be sub-string matches. This results in querying the server multiple times for the same directory (for example, perl check where the page contains multiple references to 5.0/something-unrelated) or querying the server for directory that doesn't exist (nasm check, where existence of 3.02rcX directories would result in querying the server for non-existing 3.02). I have confirmed that the overall oe-core version check returns same results as before, with less network churn and nasm version check working properly again. Signed-off-by: Alexander Kanavin Signed-off-by: Richard Purdie (cherry picked from commit 4af5f6a65a63bd92021d5eaa9d9ecc14f7397823) Signed-off-by: Yoann Congal --- lib/bb/fetch2/wget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py index ca4959ab5f7..6ac4306c0cc 100644 --- a/lib/bb/fetch2/wget.py +++ b/lib/bb/fetch2/wget.py @@ -539,7 +539,7 @@ def _check_latest_version_by_dir(self, dirver, package, package_regex, current_v version_dir = ['', '', ''] version = ['', '', ''] - dirver_regex = re.compile(r"(?P\D*)(?P(\d+[\.\-_])*(\d+))") + dirver_regex = re.compile(r"^(?P\D*)(?P(\d+[\.\-_])*(\d+))$") s = dirver_regex.search(dirver) if s: version_dir[1] = s.group('ver') From 1b64da8f2a7d93908ea79eb839291ab7f6febdbc Mon Sep 17 00:00:00 2001 From: Yoann Congal Date: Tue, 19 May 2026 11:06:23 +0200 Subject: [PATCH 85/86] b4-config: add send-prefixes for 2.18/wrynose That might help new users send correct first stable patches. Cc: Quentin Schulz Signed-off-by: Yoann Congal --- .b4-config | 1 + 1 file changed, 1 insertion(+) diff --git a/.b4-config b/.b4-config index 047f0b94a4f..4d9ab7bd0bd 100644 --- a/.b4-config +++ b/.b4-config @@ -2,3 +2,4 @@ send-series-to = bitbake-devel@lists.openembedded.org send-auto-cc-cmd = ./contrib/b4-wrapper-bitbake.py send-auto-cc-cmd prep-pre-flight-checks = disable-needs-checking + send-prefixes = 2.18 From 22021758e66737bcf68dfd2b74adc6a0cb1d42d9 Mon Sep 17 00:00:00 2001 From: Yoann Congal Date: Tue, 19 May 2026 11:08:15 +0200 Subject: [PATCH 86/86] README: Add "2.18" subject-prefix to git-send-email suggestion That might help new users send correct first stable patches. Signed-off-by: Yoann Congal --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index e9f4c858ee1..0701dbbe2aa 100644 --- a/README +++ b/README @@ -24,12 +24,12 @@ for full details on how to submit changes. As a quick guide, patches should be sent to bitbake-devel@lists.openembedded.org The git command to do that would be: - git send-email -M -1 --to bitbake-devel@lists.openembedded.org + git send-email -M -1 --to bitbake-devel@lists.openembedded.org --subject-prefix="2.18][PATCH If you're sending a patch related to the BitBake manual, make sure you copy the Yocto Project documentation mailing list: - git send-email -M -1 --to bitbake-devel@lists.openembedded.org --cc docs@lists.yoctoproject.org + git send-email -M -1 --to bitbake-devel@lists.openembedded.org --cc docs@lists.yoctoproject.org --subject-prefix="2.18][PATCH Mailing list: