diff --git a/.b4-config b/.b4-config index 047f0b94a4f..4d9ab7bd0bd 100644 --- a/.b4-config +++ b/.b4-config @@ -2,3 +2,4 @@ send-series-to = bitbake-devel@lists.openembedded.org send-auto-cc-cmd = ./contrib/b4-wrapper-bitbake.py send-auto-cc-cmd prep-pre-flight-checks = disable-needs-checking + send-prefixes = 2.18 diff --git a/README b/README index e9f4c858ee1..0701dbbe2aa 100644 --- a/README +++ b/README @@ -24,12 +24,12 @@ for full details on how to submit changes. As a quick guide, patches should be sent to bitbake-devel@lists.openembedded.org The git command to do that would be: - git send-email -M -1 --to bitbake-devel@lists.openembedded.org + git send-email -M -1 --to bitbake-devel@lists.openembedded.org --subject-prefix="2.18][PATCH If you're sending a patch related to the BitBake manual, make sure you copy the Yocto Project documentation mailing list: - git send-email -M -1 --to bitbake-devel@lists.openembedded.org --cc docs@lists.yoctoproject.org + git send-email -M -1 --to bitbake-devel@lists.openembedded.org --cc docs@lists.yoctoproject.org --subject-prefix="2.18][PATCH Mailing list: diff --git a/bin/bitbake b/bin/bitbake index a995bd66535..5a820417b2a 100755 --- a/bin/bitbake +++ b/bin/bitbake @@ -27,7 +27,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException bb.utils.check_system_locale() -__version__ = "2.16.0" +__version__ = "2.18.0" if __name__ == "__main__": if __version__ != bb.__version__: diff --git a/bin/bitbake-setup b/bin/bitbake-setup index 927c03a1033..220540f7f9c 100755 --- a/bin/bitbake-setup +++ b/bin/bitbake-setup @@ -4,28 +4,29 @@ # SPDX-License-Identifier: GPL-2.0-only # -import logging -import os -import sys import argparse -import json -import shutil -import time import configparser +import copy import datetime +import functools import glob -import subprocess -import copy -import textwrap +import json +import logging +import os +import shutil import signal -import functools import string +import subprocess +import sys +import textwrap +import time + bindir = os.path.abspath(os.path.dirname(__file__)) sys.path[0:0] = [os.path.join(os.path.dirname(bindir), 'lib')] -import bb.msg -import bb.process +import bb.msg # noqa: E402 +import bb.process # noqa: E402 logger = bb.msg.logger_create('bitbake-setup', sys.stdout) @@ -35,6 +36,43 @@ GLOBAL_ONLY_SETTINGS = ( "top-dir-name", ) +def color_enabled() -> bool: + """ + Our logger has a BBLogFormatter formatter which holds whether color is + enabled or not. Return this value. + """ + return logger.handlers[0].formatter.color_enabled + +def get_diff_color_param() -> str: + return "--color=always" if color_enabled() else "--color=never" + +def print_configs(prompt: str, choices: list[str], descriptions: list[str] = []): + """ + Helper function to print a list of choices and align the output. + Each option name is made bold to stand out, unless color is not enabled in + our logger. + """ + if not prompt.endswith(':'): + prompt += ":" + logger.plain(prompt) + + if not descriptions: + descriptions = ["" for _ in choices] + + # maximum size of all choices, for alignment + cmax = max([len(c) for c in choices]) + 1 + + for n, c in enumerate(choices): + msg = f"{n + 1}. " + if color_enabled(): + # make it bold + msg += "\033[1m" + msg += f"{c:<{cmax}}" + if color_enabled(): + msg += "\033[0m" + msg += f" {descriptions[n]}" + logger.plain(msg) + # If bitbake is from a release tarball or somewhere like pypi where # updates may not be straightforward, prefer to use the git repo as the # default registry @@ -117,7 +155,7 @@ def add_unique_timestamp_to_path(path): def _get_remotes(r_remote): remotes = [] - if not 'remotes' in r_remote and not 'uri' in r_remote: + if 'remotes' not in r_remote and 'uri' not in r_remote: raise Exception("Expected key(s): 'remotes', 'uri'") if 'remotes' in r_remote: @@ -129,7 +167,7 @@ def _get_remotes(r_remote): return remotes -def checkout_layers(layers, confdir, layerdir, d): +def checkout_layers(layers, confdir, layerdir, d, rebase_conflicts_strategy='abort'): def _checkout_git_remote(r_remote, repodir, layers_fixed_revisions): rev = r_remote['rev'] branch = r_remote.get('branch', None) @@ -145,7 +183,23 @@ def checkout_layers(layers, confdir, layerdir, d): else: src_uri = f"{fetchuri};protocol={prot};rev={rev};nobranch=1;destsuffix={repodir}" fetcher = bb.fetch.Fetch([src_uri], d) - do_fetch(fetcher, layerdir) + repodir_path = os.path.join(layerdir, repodir) + try: + do_fetch(fetcher, layerdir) + except (bb.fetch2.LocalModificationsError, bb.fetch2.RebaseError) as e: + if rebase_conflicts_strategy != 'backup': + e.msg += ("\nUse 'bitbake-setup update --rebase-conflicts-strategy=backup'" + " to automatically back up the directory and re-clone from upstream," + " or use 'bitbake-setup init -L %s /path/to/local/checkout'" + " to work with a local checkout instead." % r_name) + raise + backup_path = add_unique_timestamp_to_path(repodir_path + '-backup') + logger.warning( + "%s\n" + "Renaming %s to %s to preserve your work, then re-cloning from upstream.", + e, repodir_path, backup_path) + os.rename(repodir_path, backup_path) + fetcher.unpack(layerdir) urldata = fetcher.ud[src_uri] revision = urldata.revision layers_fixed_revisions[r_name]['git-remote']['rev'] = revision @@ -154,39 +208,6 @@ def checkout_layers(layers, confdir, layerdir, d): logger.plain("Making a symbolic link {} pointing to {}".format(dst, src)) os.symlink(src, dst) - def _has_local_modifications(r_name, r_path): - fixed_revisions = json.load(open(os.path.join(confdir, "sources-fixed-revisions.json"))) - - if not r_name in fixed_revisions['sources']: - logger.warning("""Source {} is added with path {}. -This path already exists, but it has no entry in a fixed revisions -record. To ensure possible local modifications are not lost, it will -be preserved in a backup directory.""".format(r_name, r_path)) - return True - - rev = fixed_revisions['sources'][r_name]['git-remote']['rev'] - status = bb.process.run('git -C {} status --porcelain'.format(r_path))[0] - if status: - return True - diff = bb.process.run('git -C {} diff {}'.format(r_path, rev))[0] - if diff: - return True - return False - - def _restrict_commits(r_name, r_path): - hook_path = os.path.join(r_path, '.git', 'hooks', 'pre-commit') - restrict_hook = """#!/bin/sh -echo "This repository is managed by bitbake-setup, and making commits is restricted. -If you wish to make local modifications, clone it separately, and re-initialize using -bitbake-setup init -L {} /path/to/repo/checkout" -exit 1 -""".format(r_name) - with open(hook_path, 'w') as f: - f.write(restrict_hook) - import stat - st = os.stat(hook_path) - os.chmod(hook_path, st.st_mode | stat.S_IEXEC) - layers_fixed_revisions = copy.deepcopy(layers) repodirs = [] oesetupbuild = None @@ -205,19 +226,16 @@ exit 1 if os.path.lexists(repodir_path): if os.path.islink(repodir_path): os.remove(repodir_path) - elif _has_local_modifications(r_name, repodir_path): + elif r_local: backup_path = add_unique_timestamp_to_path(repodir_path + '-backup') logger.warning("""Source {} in {} contains local modifications. Renaming to {} to preserve them. For local development work it is recommended to clone the needed layers separately and re-initialize using -L option: bitbake-setup init -L {} /path/to/repo/checkout""".format( r_name, repodir_path, backup_path, r_name)) os.rename(repodir_path, backup_path) - else: - shutil.rmtree(repodir_path) if r_remote: _checkout_git_remote(r_remote, repodir, layers_fixed_revisions) - _restrict_commits(r_name, repodir_path) if r_local: _symlink_local(os.path.expanduser(r_local["path"]), repodir_path) @@ -238,7 +256,7 @@ bitbake-setup init -L {} /path/to/repo/checkout""".format( return layers_fixed_revisions -def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf): +def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf, init_vscode=False): def _setup_build_conf(layers, filerelative_layers, build_conf_dir): os.makedirs(build_conf_dir) layers_s = [] @@ -357,6 +375,7 @@ def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_c init_script = os.path.join(bitbake_builddir, "init-build-env") + workspace_file = os.path.join(setupdir, "bitbake.code-workspace") shell = "bash" fragments = bitbake_config.get("oe-fragments", []) + sorted(bitbake_config.get("oe-fragment-choices",{}).values()) if fragments: @@ -368,6 +387,8 @@ def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_c logger.plain('New bitbake configuration from upstream is the same as the current one, no need to update it.') shutil.rmtree(bitbake_confdir) os.rename(backup_bitbake_confdir, bitbake_confdir) + if init_vscode or os.path.exists(workspace_file): + configure_vscode(setupdir, layerdir, bitbake_builddir, init_script) return logger.plain('Upstream bitbake configuration changes were found:') @@ -383,6 +404,8 @@ def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_c logger.plain(f'Leaving the upstream configuration in {upstream_bitbake_confdir}') os.rename(bitbake_confdir, upstream_bitbake_confdir) os.rename(backup_bitbake_confdir, bitbake_confdir) + if init_vscode or os.path.exists(workspace_file): + configure_vscode(setupdir, layerdir, bitbake_builddir, init_script) return logger.plain('Applying upstream bitbake configuration changes') @@ -390,17 +413,21 @@ def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_c fragment_note = "Run 'bitbake-config-build enable-fragment ' to enable additional fragments or replace built-in ones (e.g. machine/ or distro/ to change MACHINE or DISTRO)." + readme_extra = "" + if init_vscode: + readme_extra = "\n\nTo edit the code in VSCode, open the workspace: code {}\n".format(workspace_file) + readme = """{}\n\nAdditional information is in {} and {}\n Source the environment using '. {}' to run builds from the command line.\n {}\n -The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf -""".format( +The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf{}""".format( bitbake_config["description"], os.path.join(bitbake_builddir,'conf/conf-summary.txt'), os.path.join(bitbake_builddir,'conf/conf-notes.txt'), init_script, fragment_note, - bitbake_builddir + bitbake_builddir, + readme_extra ) readme_file = os.path.join(bitbake_builddir, "README") with open(readme_file, 'w') as f: @@ -411,6 +438,11 @@ The bitbake configuration files (local.conf, bblayers.conf and more) can be foun logger.plain("To run builds, source the environment using\n . {}\n".format(init_script)) logger.plain("{}\n".format(fragment_note)) logger.plain("The bitbake configuration files (local.conf, bblayers.conf and more) can be found in\n {}/conf\n".format(bitbake_builddir)) + if init_vscode: + logger.plain("To edit the code in VSCode, open the workspace:\n code {}\n".format(workspace_file)) + + if init_vscode or os.path.exists(workspace_file): + configure_vscode(setupdir, layerdir, bitbake_builddir, init_script) def get_registry_config(registry_path, id): for root, dirs, files in os.walk(registry_path): @@ -426,12 +458,12 @@ def merge_overrides_into_sources(sources, overrides): layers[k] = v return layers -def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt"): +def update_build(config, confdir, setupdir, layerdir, d, update_bb_conf="prompt", init_vscode=False, rebase_conflicts_strategy='abort'): layer_config = merge_overrides_into_sources(config["data"]["sources"], config["source-overrides"]["sources"]) - sources_fixed_revisions = checkout_layers(layer_config, confdir, layerdir, d) + sources_fixed_revisions = checkout_layers(layer_config, confdir, layerdir, d, rebase_conflicts_strategy=rebase_conflicts_strategy) bitbake_config = config["bitbake-config"] thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None - setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf) + setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir, update_bb_conf, init_vscode) write_sources_fixed_revisions(confdir, layerdir, sources_fixed_revisions) commit_config(confdir) @@ -490,36 +522,39 @@ def choose_bitbake_config(configs, parameters, non_interactive): if non_interactive: raise Exception("Unable to choose from bitbake configurations in non-interactive mode: {}".format(configs_dict)) - logger.plain("\nAvailable bitbake configurations:") - for n, config_data in enumerated_configs: - logger.plain("{}. {}\t{}".format(n, config_data["name"], config_data["description"])) + logger.plain("") + print_configs("Available bitbake configurations", + [c["name"] for c in flattened_configs], + [c["description"] for c in flattened_configs]) config_n = int_input([i[0] for i in enumerated_configs], "\nPlease select one of the above bitbake configurations by its number: ") - 1 return flattened_configs[config_n] def choose_config(configs, non_interactive): not_expired_configs = [k for k in sorted(configs.keys()) if not has_expired(configs[k].get("expires", None))] - config_list = list(enumerate(not_expired_configs, 1)) - if len(config_list) == 1: - only_config = config_list[0][1] + if len(not_expired_configs) == 1: + only_config = not_expired_configs[0] logger.plain("\nSelecting the only available configuration {}\n".format(only_config)) return only_config if non_interactive: raise Exception("Unable to choose from configurations in non-interactive mode: {}".format(not_expired_configs)) - logger.plain("\nAvailable Configuration Templates:") - for n, config_name in config_list: - config_data = configs[config_name] - expiry_date = config_data.get("expires", None) - config_desc = config_data["description"] + descs = [] + for c in not_expired_configs: + d = configs[c]["description"] + expiry_date = configs[c].get("expires", None) if expiry_date: - logger.plain("{}. {}\t{} (supported until {})".format(n, config_name, config_desc, expiry_date)) - else: - logger.plain("{}. {}\t{}".format(n, config_name, config_desc)) - config_n = int_input([i[0] for i in config_list], - "\nPlease select one of the above Configuration Templates by its number: ") - 1 - return config_list[config_n][1] + d += f" (supported until {expiry_date})" + descs.append(d) + + logger.plain("") + print_configs("Available Configuration Templates", + [c for c in not_expired_configs], + descs) + config_n = int_input([i[0] for i in list(enumerate(not_expired_configs, 1))], + "\nPlease select one of the above configurations by its number: ") - 1 + return not_expired_configs[config_n] def choose_fragments(possibilities, parameters, non_interactive, skip_selection): choices = {} @@ -545,13 +580,11 @@ def choose_fragments(possibilities, parameters, non_interactive, skip_selection) if non_interactive: raise Exception(f"Unable to choose from options in non-interactive mode: {[o['name'] for o in options]}") - logger.plain("\n" + v["description"] + ":") + logger.plain("") + print_configs(v["description"], + [o['name'] for o in options], + [o['description'] for o in options]) options_enumerated = list(enumerate(options, 1)) - for n,o in options_enumerated: - opt_str = f"{n}. {o['name']}" - if o["description"]: - opt_str += f"\t{o['description']}" - logger.plain(opt_str) option_n = int_input([i[0] for i in options_enumerated], "\nPlease select one of the above options by its number: ") - 1 choices[k] = options_enumerated[option_n][1]["name"] @@ -562,9 +595,10 @@ def obtain_config(top_dir, registry, args, source_overrides, d): config_id = args.config[0] config_parameters = args.config[1:] if os.path.exists(config_id): + config_id = os.path.abspath(config_id) logger.info("Reading configuration from local file\n {}".format(config_id)) upstream_config = {'type':'local', - 'path':os.path.abspath(config_id), + 'path':config_id, 'name':get_config_name(config_id), 'data':json.load(open(config_id)) } @@ -576,7 +610,7 @@ def obtain_config(top_dir, registry, args, source_overrides, d): json_data = json.load(f) upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json_data} except json.JSONDecodeError as e: - raise Exception ("Invalid JSON from {}. Are you pointing to an HTML page? {}".format(config_id, e)) + raise Exception ("Invalid JSON from {}. Are you pointing to an HTML page? {}".format(config_id, e)) from e else: logger.info("Looking up config {} in configuration registry".format(config_id)) registry_path = update_registry(registry, cache_dir(top_dir), d) @@ -618,12 +652,157 @@ def obtain_overrides(args): return overrides +def configure_vscode(setupdir, layerdir, builddir, init_script): + """ + Configure the VSCode environment by creating or updating a workspace file. + + Create or update a bitbake.code-workspace file with folders for the layers and build/conf. + Managed folders are regenerated; user-added folders are kept. Settings are merged, with + managed keys (bitbake.*, python extra paths) always overwritten. + """ + logger.debug("configure_vscode: setupdir={}, layerdir={}, builddir={}, init_script={}".format( + setupdir, layerdir, builddir, init_script)) + + # Get git repository directories + git_repos = [] + if os.path.exists(layerdir): + for entry in os.listdir(layerdir): + entry_path = os.path.join(layerdir, entry) + if os.path.isdir(entry_path) and not os.path.islink(entry_path): + # Check if it's a git repository + if os.path.exists(os.path.join(entry_path, '.git')): + git_repos.append(entry) + logger.debug("configure_vscode: found {} git repos: {}".format(len(git_repos), git_repos)) + + conf_path = os.path.relpath(os.path.join(builddir, "conf"), setupdir) + repo_paths = [os.path.relpath(os.path.join(layerdir, repo), setupdir) for repo in git_repos] + logger.debug("configure_vscode: conf_path={}, repo_paths={}".format(conf_path, repo_paths)) + + # Load existing workspace + workspace_file = os.path.join(setupdir, "bitbake.code-workspace") + workspace = { + "extensions": { + "recommendations": [ + "yocto-project.yocto-bitbake" + ] + } + } + if os.path.exists(workspace_file): + logger.debug("configure_vscode: loading existing workspace file: {}".format(workspace_file)) + try: + with open(workspace_file, 'r') as f: + workspace = json.load(f) + logger.debug("configure_vscode: loaded workspace with {} folders, {} settings".format( + len(workspace.get("folders", [])), len(workspace.get("settings", {})))) + except (json.JSONDecodeError, OSError) as e: + logger.error( + "Unable to read existing workspace file {}: {}. Skipping update.".format( + workspace_file, str(e) + ) + ) + return + else: + logger.debug("configure_vscode: creating new workspace file: {}".format(workspace_file)) + + # Update folders + existing_folders = workspace.get("folders", []) + new_folders = [{"name": "conf", "path": conf_path}] + for rp in repo_paths: + repo_name = os.path.basename(rp) + new_folders.append({"name": repo_name, "path": rp}) + # Keep any user-added folders that are not managed + managed_paths = {f["path"] for f in new_folders} + for f in existing_folders: + if f["path"] not in managed_paths: + new_folders.append(f) + logger.debug("configure_vscode: keeping user-added folder: {}".format(f["path"])) + workspace["folders"] = new_folders + logger.debug("configure_vscode: updated workspace with {} folders".format(len(new_folders))) + + # Build Python extra paths for each layer - only check top level of each repo + extra_paths = [] + subdirs_to_check = ['lib', 'scripts'] + for repo in git_repos: + repo_path_abs = os.path.join(layerdir, repo) + for subdir in subdirs_to_check: + sub_path = os.path.join(repo_path_abs, subdir) + if os.path.isdir(sub_path): + extra_paths.append(sub_path) + + # Update settings + existing_settings = workspace.get("settings", {}) + new_settings = { + "bitbake.disableConfigModification": True, + "bitbake.pathToBitbakeFolder": os.path.join(layerdir, "bitbake"), + "bitbake.pathToBuildFolder": builddir, + "bitbake.pathToEnvScript": init_script, + "bitbake.workingDirectory": builddir, + "files.associations": { + "*.conf": "bitbake", + "*.inc": "bitbake" + }, + "files.exclude": { + "**/.git/**": True + }, + "search.exclude": { + "**/.git/**": True, + "**/logs/**": True + }, + "files.watcherExclude": { + "**/.git/**": True, + "**/logs/**": True + }, + "python.analysis.exclude": [ + "**/.git/**", + "**/logs/**" + ], + "python.autoComplete.extraPaths": extra_paths, + "python.analysis.extraPaths": extra_paths + } + + # Merge settings: add missing, always update bitbake paths and python extra paths + for key, value in new_settings.items(): + if key not in existing_settings: + existing_settings[key] = value + elif key.startswith("bitbake.") or key in [ + "python.autoComplete.extraPaths", + "python.analysis.extraPaths", + ]: + # Always replace - these are managed/machine-generated settings + existing_settings[key] = value + elif key in [ + "files.associations", + "files.exclude", + "search.exclude", + "files.watcherExclude", + "python.analysis.exclude", + ]: + # For dicts and lists, merge new values in without removing user additions + if isinstance(value, dict): + if not isinstance(existing_settings[key], dict): + existing_settings[key] = {} + for k, v in value.items(): + if k not in existing_settings[key]: + existing_settings[key][k] = v + elif isinstance(value, list): + if not isinstance(existing_settings[key], list): + existing_settings[key] = [] + for item in value: + if item not in existing_settings[key]: + existing_settings[key].append(item) + + workspace["settings"] = existing_settings + logger.debug("configure_vscode: merged settings, total {} keys".format(len(existing_settings))) + + with open(workspace_file, 'w') as f: + json.dump(workspace, f, indent=4) + logger.debug("configure_vscode: wrote workspace file: {}".format(workspace_file)) def init_config(top_dir, settings, args): create_siteconf(top_dir, args.non_interactive, settings) d = init_bb_cache(top_dir, settings, args) - stdout = sys.stdout + def handle_task_progress(event, d): rate = event.rate if event.rate else '' progress = event.progress if event.progress > 0 else 0 @@ -687,13 +866,13 @@ def init_config(top_dir, settings, args): bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d) write_upstream_config(confdir, upstream_config) - update_build(upstream_config, confdir, setupdir, layerdir, d, update_bb_conf="yes") + update_build(upstream_config, confdir, setupdir, layerdir, d, update_bb_conf="yes", init_vscode=args.init_vscode) bb.event.remove("bb.build.TaskProgress", None) def get_diff(file1, file2): try: - bb.process.run('diff -uNr {} {}'.format(file1, file2)) + bb.process.run('diff {} -uNr {} {}'.format(get_diff_color_param(), file1, file2)) except bb.process.ExecutionError as e: if e.exitcode == 1: return e.stdout @@ -703,12 +882,21 @@ def get_diff(file1, file2): def are_layers_changed(layers, layerdir, d): def _is_git_remote_changed(r_remote, repodir): - changed = False + from bb.fetch2.git import sha1_re + rev = r_remote['rev'] branch = r_remote.get('branch', None) - remotes = _get_remotes(r_remote) + rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir))) + local_revision = rev_parse_result[0].strip() + if sha1_re.match(rev): + if rev != local_revision: + logger.info('Layer repository checked out into {} is at revision {} but should be at {}'.format(os.path.join(layerdir, repodir),local_revision, rev)) + return True + return False + remotes = _get_remotes(r_remote) + changed = False for remote in remotes: type,host,path,user,pswd,params = bb.fetch.decodeurl(remote) fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params)) @@ -717,8 +905,6 @@ def are_layers_changed(layers, layerdir, d): else: fetcher = bb.fetch.FetchData("{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir), d) upstream_revision = fetcher.method.latest_revision(fetcher, d, 'default') - rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir))) - local_revision = rev_parse_result[0].strip() if upstream_revision != local_revision: changed = True logger.info('Layer repository {} checked out into {} updated revision {} from {} to {}'.format(remote, os.path.join(layerdir, repodir), rev, local_revision, upstream_revision)) @@ -736,7 +922,7 @@ def are_layers_changed(layers, layerdir, d): return changed def build_status(top_dir, settings, args, d, update=False): - setupdir = args.setup_dir + setupdir = os.path.abspath(args.setup_dir) confdir = os.path.join(setupdir, "config") layerdir = os.path.join(setupdir, "layers") @@ -751,13 +937,13 @@ def build_status(top_dir, settings, args, d, update=False): new_upstream_config = obtain_config(top_dir, registry, args, source_overrides, d) write_upstream_config(confdir, new_upstream_config) - config_diff = bb.process.run('git -C {} diff'.format(confdir))[0] + config_diff = bb.process.run('git -C {} diff {}'.format(confdir, get_diff_color_param()))[0] if config_diff: logger.plain('\nConfiguration in {} has changed:\n{}'.format(setupdir, config_diff)) if update: update_build(new_upstream_config, confdir, setupdir, layerdir, d, - update_bb_conf=args.update_bb_conf) + update_bb_conf=args.update_bb_conf, rebase_conflicts_strategy=args.rebase_conflicts_strategy) else: bb.process.run('git -C {} restore config-upstream.json'.format(confdir)) return @@ -766,7 +952,7 @@ def build_status(top_dir, settings, args, d, update=False): if are_layers_changed(layer_config, layerdir, d): if update: update_build(current_upstream_config, confdir, setupdir, layerdir, - d, update_bb_conf=args.update_bb_conf) + d, update_bb_conf=args.update_bb_conf, rebase_conflicts_strategy=args.rebase_conflicts_strategy) return logger.plain("\nConfiguration in {} has not changed.".format(setupdir)) @@ -783,9 +969,11 @@ def do_fetch(fetcher, dir): with open(fetchlog, 'a') as f: oldstdout = sys.stdout sys.stdout = f - fetcher.download() - fetcher.unpack(dir) - sys.stdout = oldstdout + try: + fetcher.download() + fetcher.unpack_update(dir) + finally: + sys.stdout = oldstdout def update_registry(registry, cachedir, d): registrydir = 'configurations' @@ -888,6 +1076,31 @@ def create_siteconf(top_dir, non_interactive, settings): os.makedirs(top_dir, exist_ok=True) with open(siteconfpath, 'w') as siteconffile: + sstate_settings = textwrap.dedent( + """ + # + # Where to place shared-state files + # + # BitBake has the capability to accelerate builds based on previously built output. + # This is done using "shared state" files which can be thought of as cache objects + # and this option determines where those files are placed. + # + # You can wipe out TMPDIR leaving this directory intact and the build would regenerate + # from these files if no changes were made to the configuration. If changes were made + # to the configuration, only shared state files where the state was still valid would + # be used (done using checksums). + SSTATE_DIR ?= "{sstate_dir}" + # + # Hash Equivalence database location + # + # Hash equivalence improves reuse of sstate by detecting when a given sstate + # artifact can be reused as equivalent, even if the current task hash doesn't + # match the one that generated the artifact. This variable controls where the + # Hash Equivalence database ("hashserv.db") is stored and can be shared between + # concurrent builds. + BB_HASHSERVE_DB_DIR ?= "${{SSTATE_DIR}}" + """.format(sstate_dir=os.path.join(top_dir, ".sstate-cache")) + ) siteconffile.write( textwrap.dedent( """\ @@ -901,11 +1114,11 @@ def create_siteconf(top_dir, non_interactive, settings): # wiping and rebuilding you can preserve this directory to speed up this part of # subsequent builds. This directory is safe to share between multiple builds on # the same machine too. - DL_DIR = "{dl_dir}" + DL_DIR ?= "{dl_dir}" """.format( dl_dir=settings["default"]["dl-dir"], ) - ) + ) + (sstate_settings if settings["default"]["common-sstate"] == 'yes' else "") ) @@ -1005,7 +1218,7 @@ def merge_settings(builtin_settings, global_settings, topdir_settings, cmdline_s return all_settings def sigint_handler(sig, frame, func, top_dir): - logger.plain(f'\nShutting down...') + logger.plain('\nShutting down...') if isinstance(top_dir, str) and os.path.exists(top_dir): if func in [init_config, build_update]: logger.warning(f'{top_dir} may contain an incomplete setup!') @@ -1049,6 +1262,8 @@ def main(): parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.') parser_init.add_argument('-L', '--use-local-source', default=[], action='append', nargs=2, metavar=('SOURCE_NAME', 'PATH'), help='Symlink local source into a build, instead of getting it as prescribed by a configuration (useful for local development).') + parser_init.add_argument('--init-vscode', action=argparse.BooleanOptionalAction, default=bool(shutil.which('code')), + help='Generate VSCode workspace configuration (default: %(default)s)') parser_init.set_defaults(func=init_config) parser_status = subparsers.add_parser('status', help='Check if the setup needs to be synchronized with configuration') @@ -1058,6 +1273,10 @@ def main(): parser_update = subparsers.add_parser('update', help='Update a setup to be in sync with configuration') add_setup_dir_arg(parser_update) parser_update.add_argument('--update-bb-conf', choices=['prompt', 'yes', 'no'], default='prompt', help='Update bitbake configuration files (bblayers.conf, local.conf) (default: prompt)') + parser_update.add_argument('--rebase-conflicts-strategy', choices=['abort', 'backup'], default='abort', + help="What to do when a layer repository has local modifications that prevent " + "an in-place update: 'abort' (default) aborts with an error message; " + "'backup' renames the directory to a timestamped backup and re-clones from upstream.") parser_update.set_defaults(func=build_update) parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine') @@ -1074,8 +1293,7 @@ def main(): subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file") - parser_settings_list = subparser_settings.add_parser('list', - help="List all settings with their values") + subparser_settings.add_parser('list', help="List all settings with their values") parser_settings_set = subparser_settings.add_parser('set', parents=[parser_settings_arg_global], help="In a Section, set a setting to a certain value") @@ -1116,6 +1334,7 @@ def main(): 'top-dir-name':'bitbake-builds', 'registry':get_default_registry(), 'use-full-setup-dir-name':'no', + 'common-sstate':'yes', } global_settings = load_settings(global_settings_path(args)) @@ -1140,17 +1359,25 @@ def main(): list_configs(all_settings, args) return - logger.info('Bitbake-setup is using {} as top directory.'.format(top_dir, global_settings_path(args))) - - if args.func == init_config: - init_config(top_dir, all_settings, args) - else: - d = init_bb_cache(top_dir, all_settings, args) - args.func(top_dir, all_settings, args, d) + logger.info('Bitbake-setup is using {} as top directory.'.format(top_dir)) - save_bb_cache() + try: + if args.func == init_config: + init_config(top_dir, all_settings, args) + else: + d = init_bb_cache(top_dir, all_settings, args) + args.func(top_dir, all_settings, args, d) + + save_bb_cache() + except (SystemExit, KeyboardInterrupt): + raise + except Exception as e: + if args.debug: + raise + logger.error(str(e)) + sys.exit(1) else: - from argparse import Namespace parser.print_help() -main() +if __name__ == '__main__': + main() diff --git a/bin/bitbake-worker b/bin/bitbake-worker index d2b146a6a92..aa14ef1914c 100755 --- a/bin/bitbake-worker +++ b/bin/bitbake-worker @@ -23,9 +23,10 @@ import queue import shlex import subprocess import fcntl -from multiprocessing import Lock from threading import Thread +Lock = bb.multiprocessing.Lock + # Remove when we have a minimum of python 3.10 if not hasattr(fcntl, 'F_SETPIPE_SZ'): fcntl.F_SETPIPE_SZ = 1031 diff --git a/default-registry/configurations/oe-nodistro-wrynose.conf.json b/default-registry/configurations/oe-nodistro-wrynose.conf.json new file mode 100644 index 00000000000..278ce4dd029 --- /dev/null +++ b/default-registry/configurations/oe-nodistro-wrynose.conf.json @@ -0,0 +1,48 @@ +{ + "description": "OpenEmbedded - 'nodistro' basic configuration, release 6.0 'wrynose'", + "expires": "2030-05-31", + "sources": { + "bitbake": { + "git-remote": { + "uri": "https://git.openembedded.org/bitbake", + "branch": "2.18", + "rev": "2.18" + } + }, + "openembedded-core": { + "git-remote": { + "uri": "https://git.openembedded.org/openembedded-core", + "branch": "wrynose", + "rev": "wrynose" + } + }, + "yocto-docs": { + "git-remote": { + "uri": "https://git.yoctoproject.org/yocto-docs", + "branch": "wrynose", + "rev": "wrynose" + } + } + }, + "bitbake-setup": { + "configurations": [ + { + "name": "nodistro", + "description": "OpenEmbedded 'nodistro'", + "setup-dir-name": "oe-nodistro-wrynose", + "bb-layers": ["openembedded-core/meta"], + "oe-fragments-one-of": { + "machine": { + "description": "Target machines", + "options" : [ + { "name": "machine/qemux86-64", "description": "x86-64 system on QEMU" }, + { "name": "machine/qemuarm64", "description": "ARMv8 system on QEMU" }, + { "name": "machine/qemuriscv64", "description": "RISC-V system on QEMU" } + ] + } + } + } + ] + }, + "version": "1.0" +} diff --git a/default-registry/configurations/poky-wrynose.conf.json b/default-registry/configurations/poky-wrynose.conf.json new file mode 100644 index 00000000000..d2792843773 --- /dev/null +++ b/default-registry/configurations/poky-wrynose.conf.json @@ -0,0 +1,74 @@ +{ + "description": "Poky - The Yocto Project testing distribution configurations and hardware test platforms, release 6.0 'wrynose'", + "expires": "2030-05-31", + "sources": { + "bitbake": { + "git-remote": { + "uri": "https://git.openembedded.org/bitbake", + "branch": "2.18", + "rev": "2.18" + } + }, + "openembedded-core": { + "git-remote": { + "uri": "https://git.openembedded.org/openembedded-core", + "branch": "wrynose", + "rev": "wrynose" + } + }, + "meta-yocto": { + "git-remote": { + "uri": "https://git.yoctoproject.org/meta-yocto", + "branch": "wrynose", + "rev": "wrynose" + } + }, + "yocto-docs": { + "git-remote": { + "uri": "https://git.yoctoproject.org/yocto-docs", + "branch": "wrynose", + "rev": "wrynose" + } + } + }, + "bitbake-setup": { + "configurations": [ + { + "bb-layers": ["openembedded-core/meta","meta-yocto/meta-yocto-bsp","meta-yocto/meta-poky"], + "setup-dir-name": "$distro-wrynose", + "oe-fragments-one-of": { + "machine": { + "description": "Target machines", + "options" : [ + { "name": "machine/qemux86-64", "description": "x86-64 system on QEMU" }, + { "name": "machine/qemuarm64", "description": "ARMv8 system on QEMU" }, + { "name": "machine/qemuriscv64", "description": "RISC-V system on QEMU" }, + { "name": "machine/genericarm64", "description": "Arm64 SystemReady IR/ES platforms" }, + { "name": "machine/genericx86-64", "description": "x86_64 (64-bit) PCs and servers" } + ] + }, + "distro": { + "description": "Target distributions", + "options" : [ + { "name": "distro/poky", "description": "Yocto Project Reference Distro" }, + { "name": "distro/poky-altcfg", "description": "Poky alternative with systemd as init manager" }, + { "name": "distro/poky-tiny", "description": "Poky alternative optimized for size" } + ] + } + }, + "configurations": [ + { + "name": "poky", + "description": "Poky - The Yocto Project testing distribution" + }, + { + "name": "poky-with-sstate", + "description": "Poky - The Yocto Project testing distribution with internet sstate acceleration. Use with caution as it requires a completely robust local network with sufficient bandwidth.", + "oe-fragments": ["core/yocto/sstate-mirror-cdn"] + } + ] + } + ] + }, + "version": "1.0" +} diff --git a/doc/.gitignore b/doc/.gitignore index 69fa449dd96..3a570d8c655 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -1 +1,4 @@ _build/ +sphinx-static/switchers.js +releases.json +releases.rst diff --git a/doc/Makefile b/doc/Makefile index 996f01b7d5c..2f8e898c4fc 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -27,7 +27,7 @@ publish: Makefile html singlehtml sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html clean: - @rm -rf $(BUILDDIR) + @rm -rf $(BUILDDIR) sphinx-static/switchers.js releases.json releases.rst # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst b/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst index f7580c8e0e6..c22e19bd665 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-environment-setup.rst @@ -327,6 +327,16 @@ In addition, the command can take the following arguments: with a ``local`` source in it. See the :ref:`ref-bbsetup-source-overrides` section for more information on source overrides. +- ``--init-vscode`` / ``--no-init-vscode``: generate (or skip generating) a + ``bitbake.code-workspace`` file in the :term:`Setup` directory. The workspace + file configures the `Yocto Project BitBake + `_ + VS Code extension with paths to the build directory and the init script, and + lists the layer directories as workspace folders. Any user-added folders or + settings in an existing workspace file are preserved across updates. + The default is ``true`` when ``code`` (the VS Code binary) is found on + ``PATH``, and ``false`` otherwise. + ``bitbake-setup init`` Examples ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -410,6 +420,30 @@ the latest changes from the :term:`Configuration Template` it was constructed fr The :ref:`ref-bbsetup-command-status` command can be used to show the current status of the :term:`Setup` before updating it. +This command is intended to be run from a shell where the BitBake environment +has been sourced (e.g. after ``source build/init-build-env``), so that +``bitbake-setup`` can automatically identify the current :term:`Setup` without +requiring the ``--setup-dir`` argument. + +When a layer repository already exists in the :term:`Setup` (i.e. it has been +previously checked out by ``bitbake-setup init`` or a prior ``update``), the +update is performed *in place* using the fetcher's +:ref:`unpack_update ` method: the new upstream revision +is fetched into the local download cache and then rebased on top of the +checkout's current HEAD. This means any local commits in the layer directory +are preserved and rebased onto the new upstream revision. If the working tree +contains staged or unstaged changes to tracked files, the update is blocked +until those changes are committed, stashed or discarded. + +.. note:: + + ``bitbake-setup`` performs the rebase to fast-forward local commits onto + the new upstream revision, but intentionally does not go further by using + ``--autostash`` (which would silently stash uncommitted changes before the + rebase and pop them afterwards). Any uncommitted modifications are surfaced + to the user before the update proceeds, so there are no surprises from an + automatic stash/pop cycle. + In addition, the command can take the following arguments: - ``--update-bb-conf``: whether to update the :term:`BitBake Build` @@ -420,10 +454,181 @@ In addition, the command can take the following arguments: - ``yes``: update the configuration files. - ``no``: don't update the configuration files. +- ``--rebase-conflicts-strategy``: what to do when a layer repository has + local modifications or commits that prevent an in-place update. Accepted + values are: + + - ``abort`` (default): stop with an error message describing the problem. + The repository is left in its previous state (the failed rebase is + automatically aborted). The error message includes a hint to re-run with + ``--rebase-conflicts-strategy=backup``. + - ``backup``: rename the conflicting layer directory to a timestamped + ``-backup-`` path (preserving local work), then + re-clone the layer from upstream into a fresh directory. + - ``--setup-dir``: path to the :term:`Setup` to update. Not required if the command is invoked from an initialized BitBake environment that contains :term:`BBPATH`. +``bitbake-setup update`` Examples +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- This example performs a standard update from an initialized BitBake + environment. Local commits in any layer directory are rebased on top of + the new upstream revision and preserved: + + .. code-block:: shell + + $ bitbake-setup update + NOTE: Bitbake-setup is using /path/to/bitbake-builds as top directory. + NOTE: Looking up config poky-master in configuration registry + NOTE: Layer repository https://git.openembedded.org/openembedded-core + checked out into /path/to/bitbake-builds/poky-master/layers/openembedded-core + updated revision master from d383ea3... to b50d6de... + Fetching layer/tool repositories into /path/to/bitbake-builds/poky-master/layers + bitbake + meta-yocto + openembedded-core + +- This example shows what happens when a layer directory contains staged or + unstaged changes to tracked files. The update is blocked with an error: + + .. code-block:: shell + + $ bitbake-setup update + NOTE: Bitbake-setup is using /path/to/bitbake-builds as top directory. + NOTE: Looking up config poky-master in configuration registry + Fetching layer/tool repositories into /path/to/bitbake-builds/poky-master/layers + bitbake + meta-yocto + openembedded-core + ERROR: Unpack failure for URL: + 'git://git.openembedded.org/openembedded-core;protocol=https;rev=master;branch=master;destsuffix=openembedded-core'. + Repository at /path/to/bitbake-builds/poky-master/layers/openembedded-core has uncommitted changes, unable to update: + M meta/recipes-devtools/ccache/ccache_4.13.1.bb + + Commit, stash or discard your changes and re-run the update. + Use 'bitbake-setup update --rebase-conflicts-strategy=backup' + to automatically back up the directory and re-clone from upstream, + or use 'bitbake-setup init -L openembedded-core /path/to/local/checkout' + to work with a local checkout instead. + + Stashing the changes and re-running resolves the issue: + + .. code-block:: shell + + $ git -C layers/openembedded-core stash + $ bitbake-setup update + $ git -C layers/openembedded-core stash pop + +- This example shows what happens when a layer directory contains local + commits that conflict with the incoming upstream changes. The failed rebase + is automatically aborted, and the ``dldir`` remote is left in the repository + for manual resolution: + + .. code-block:: shell + + $ bitbake-setup update + ERROR: Repository at layers/openembedded-core has local commits that could + not be rebased onto the new upstream revision: + ... + Note: the 'dldir' remote points to the local download cache and may be + used to resolve the conflict manually. + Once resolved, re-run the update. + + The conflict can be resolved manually using the ``dldir`` remote that + ``bitbake-setup`` adds to the repository: + + .. code-block:: shell + + $ git -C layers/openembedded-core rebase dldir/master + # fix conflicts in an editor, then stage the resolved files: + $ git -C layers/openembedded-core add meta/recipes-core/base-files/base-files.bb + $ git -C layers/openembedded-core rebase --continue + $ bitbake-setup update + +- When manual conflict resolution is not desired, the + ``--rebase-conflicts-strategy=backup`` option can be used instead. It + preserves the conflicting directory under a timestamped backup path and + re-clones the layer cleanly from upstream: + + .. code-block:: shell + + $ bitbake-setup update --rebase-conflicts-strategy=backup + NOTE: Bitbake-setup is using /path/to/bitbake-builds as top directory. + NOTE: Looking up config poky-master in configuration registry + NOTE: Layer repository https://git.openembedded.org/openembedded-core checked + out into /path/to/bitbake-builds/poky-master/layers/openembedded-core + updated revision master from 2ec283e... to b50d6de... + Fetching layer/tool repositories into /path/to/bitbake-builds/poky-master/layers + bitbake + meta-yocto + openembedded-core + WARNING: Unpack failure for URL: + 'git://git.openembedded.org/openembedded-core;protocol=https;rev=master;branch=master;destsuffix=openembedded-core'. + Repository at /path/to/bitbake-builds/poky-master/layers/openembedded-core + has local commits that could not be rebased onto the new upstream revision: + ... + Note: the 'dldir' remote points to the local download cache and may be used to resolve the conflict manually. + Once resolved, re-run the update. + Renaming /path/to/bitbake-builds/poky-master/layers/openembedded-core to + /path/to/bitbake-builds/poky-master/layers/openembedded-core-backup.20260329160426 + to preserve your work, then re-cloning from upstream. + + The backup directory is a complete git repository. Local commits can be + recovered from it after the update by fetching a branch from the backup + into the fresh clone (git accepts local paths as remote URLs) and then + cherry-picking the desired commits. For example, given a ``my-wip`` branch + with two commits existing in the backup repository and not in the fresh clone, + the following commands can be used to apply these commits on top of the new + upstream revision in the fresh clone: + + .. code-block:: shell + + $ git -C layers/openembedded-core-backup.20260329160426 log --oneline my-wip + a1b2c3d u-boot: fix compilation with newer GCC + 2ec283e base-files: update version + ... + + $ cd layers/openembedded-core + $ git checkout -b my-wip + Switched to a new branch 'my-wip' + $ git fetch ../openembedded-core-backup.20260329160426 my-wip + + $ git cherry-pick 2ec283e + # resolve any conflicts, then stage the resolved files: + $ git add meta/recipes-core/base-files/base-files.bb + $ git cherry-pick --continue + + $ git cherry-pick a1b2c3d + # resolve any conflicts, then stage the resolved files: + $ git add meta/recipes-devtools/u-boot/u-boot_2026.04.bb + $ git cherry-pick --continue + + $ cd ../.. + + The sequence above is: + + #. Inspect the backup's branch history to identify the commits to recover. + #. Change into the fresh clone and create a matching branch. + #. Fetch the backup branch so its objects become available locally. Git + accepts filesystem paths as remote URLs, and from inside + ``layers/openembedded-core/``, ``../`` points to ``layers/``, where the + backup directory sits. + #. Cherry-pick the commits in oldest-first order. + + Once all desired commits have been recovered and verified, the backup + directory can be removed: + + .. code-block:: shell + + $ rm -rf layers/openembedded-core-backup.20260329160426 + + If a VSCode workspace is in use, the backup directory will appear as an + additional workspace folder until it is cleaned up. It can be removed from + the workspace via the VS Code UI by right-clicking the folder and selecting + *Remove Folder from Workspace*. + .. _ref-bbsetup-command-install-buildtools: ``bitbake-setup install-buildtools`` @@ -516,6 +721,7 @@ A valid settings file would for example be: registry = /path/to/bitbake/default-registry dl-dir = /path/to/bitbake-setup-downloads use-full-setup-dir-name = yes + common-sstate = yes Settings and their values can be listed and modified with the ``bitbake-setup settings`` command. See the :ref:`ref-bbsetup-command-settings` section for @@ -620,6 +826,24 @@ will override the suggestions for the :term:`Setup` directory name made by will make the directory names longer, but fully specific: they will contain all selections made during initialization. +.. _ref-bbsetup-setting-common-sstate: + +``common-sstate`` +----------------- + +When this setting is set to ``yes`` (which is also the default), bitbake-setup will +set up a common sstate directory and common hash equivalency database for all the +:term:`setups ` in a :term:`Top Directory`. This is very beneficial for speeding +up builds as build artefacts will be reused whenever possible between them. + +Set this to ``no`` for advanced use cases, such as placing the sstate directory on a NFS +mount and maintaining a separate hash equivalency server, so that sstate and hash equivalency +data can be shared between several computers. For such use cases the sstate settings need +to be added to a build configuration separately. + +See https://docs.yoctoproject.org/dev-manual/hashequivserver.html for how to share sstate +on the network. + .. _ref-bbsetup-section-config-reference: Configuration Template Files Reference diff --git a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst index 8e1a232cdbc..c2747c40124 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst @@ -186,6 +186,51 @@ cloning the tree into the final directory. The process is completed using references so that there is only one central copy of the Git metadata needed. +.. _bb-the-unpack-update: + +The Non-Destructive Update (``unpack_update``) +============================================== + +.. note:: + + This is a specialised method intended for tools that manage persistent + layer checkouts on behalf of the user, such as + :ref:`bitbake-setup update `. It is not part of the + normal recipe fetch/unpack flow. + +For Git URLs, an alternative ``unpack_update()`` method is available +that updates an existing checkout *in place* rather than removing and +re-cloning it. This is useful when the target directory may contain +local commits that should be preserved across updates. + +The code to call the non-destructive update looks like the following:: + + rootdir = l.getVar('UNPACKDIR') + fetcher.unpack_update(rootdir) + +``unpack_update()`` performs the following steps: + +1. A ``dldir`` git remote is added (or updated) in the existing + checkout, pointing at the local download cache. This remote may + also be useful for manually resolving conflicts outside the fetcher. +2. The new upstream revision is fetched from the ``dldir`` remote into + the existing repository. +3. Any local commits are rebased on top of the new upstream revision, + preserving local work. + +The method raises an error if: + +- The working tree contains staged or unstaged changes to tracked + files (``LocalModificationsError``). +- Local commits cannot be cleanly rebased onto the new upstream + revision (``RebaseError``). A failed rebase is automatically aborted + before the exception is raised. +- The download cache does not contain a sufficiently recent clone + of the repository, or the checkout is a shallow clone. + +Currently only the Git fetcher supports ``unpack_update()``. All other +fetcher types raise ``RuntimeError`` if it is called. + .. _bb-fetchers: Fetchers @@ -402,7 +447,7 @@ Here are some example URLs:: easy to share metadata without removing passwords. SSH keys, ``~/.netrc`` and ``~/.ssh/config`` files can be used as alternatives. -Using tags with the git fetcher may cause surprising behaviour. Bitbake needs to +Using tags with the git fetcher may cause surprising behaviour. BitBake needs to resolve the tag to a specific revision and to do that, it has to connect to and use the upstream repository. This is because the revision the tags point at can change and we've seen cases of this happening in well known public repositories. This can mean @@ -680,6 +725,12 @@ Here is an example URL:: NPM Fetcher (``npm://``) ------------------------ +.. warning:: + + The NPM fetcher is currently disabled due to security concerns. See + `355cd226e0720a9ed7683bb01c8c0a58eee03664 `__ + for more information. + This submodule fetches source code from an `NPM `__ Javascript package registry. @@ -719,6 +770,12 @@ to automatically create a recipe from an NPM URL. NPM shrinkwrap Fetcher (``npmsw://``) ------------------------------------- +.. warning:: + + The NPM fetcher is currently disabled due to security concerns. See + `355cd226e0720a9ed7683bb01c8c0a58eee03664 `__ + for more information. + This submodule fetches source code from an `NPM shrinkwrap `__ description file, which lists the dependencies diff --git a/doc/bitbake-user-manual/bitbake-user-manual-intro.rst b/doc/bitbake-user-manual/bitbake-user-manual-intro.rst index 6bcab487000..6801323e2fb 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-intro.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-intro.rst @@ -233,7 +233,7 @@ BitBake supports class files installed in three different directories: For details on how BitBake locates class files, see the :ref:`bitbake-user-manual/bitbake-user-manual-metadata:Locating Class Files` -section of the Bitbake User Manual. +section of the BitBake User Manual. Layers ------ @@ -539,7 +539,7 @@ Following is the usage and syntax for BitBake:: make dependency graphs more appealing. .. - Bitbake help output generated with "stty columns 80; bin/bitbake -h" + BitBake help output generated with "stty columns 80; bin/bitbake -h" .. _bitbake-examples: diff --git a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst index 18f293f4b7c..40cae6b05be 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst @@ -813,7 +813,7 @@ evaluated at the end of parsing. .. _ref-bitbake-user-manual-metadata-inherit-defer: ``inherit_defer`` Directive -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- The :ref:`inherit_defer ` directive works like the :ref:`inherit @@ -1052,13 +1052,13 @@ The variable containing a built-in fragment definitions could look like this:: OE_FRAGMENTS_BUILTIN = "someprefix:SOMEVARIABLE anotherprefix:ANOTHERVARIABLE" and then if 'someprefix/somevalue' is added to the variable that holds the list -of enabled fragments: +of enabled fragments:: - OE_FRAGMENTS = "... someprefix/somevalue" + OE_FRAGMENTS = "... someprefix/somevalue" bitbake will treat that as direct value assignment in its configuration:: - SOMEVARIABLE = "somevalue" + SOMEVARIABLE = "somevalue" Locating Include Files ---------------------- @@ -1158,7 +1158,7 @@ Like include files, class files are located using the :term:`BBPATH` variable. The classes can be included in the ``classes-recipe``, ``classes-global`` and ``classes`` directories, as explained in the :ref:`bitbake-user-manual/bitbake-user-manual-intro:Class types` section of the -Bitbake User Manual. Like for the :ref:`include ` and +BitBake User Manual. Like for the :ref:`include ` and :ref:`require ` directives, BitBake stops and inherits the first class that it finds. @@ -1231,7 +1231,9 @@ When you create these types of functions in your recipe or class files, you need to follow the shell programming rules. The scripts are executed by ``/bin/sh``, which may not be a bash shell but might be something such as ``dash``. You should not use -Bash-specific script (bashisms). +Bash-specific script (bashisms), such as ``[[`` (use ``[`` instead). The BitBake +shell parser also has further +:ref:`limitations `. Overrides and override-style operators like ``:append`` and ``:prepend`` can also be applied to shell functions. Most commonly, this application @@ -1272,6 +1274,20 @@ Running ``do_foo`` prints the following:: You can use the ``bitbake -e recipename`` command to view the final assembled function after all overrides have been applied. +Limitations +^^^^^^^^^^^ + +Shell functions must be parsable by BitBake, which cannot handle all +syntax supported by POSIX shells. In particular: + +#. Arithmetic expansion is not supported. This means you cannot do: + + .. code-block:: shell + + VAR=$(( i + 1 )) + + See https://bugzilla.yoctoproject.org/show_bug.cgi?id=11314. + BitBake-Style Python Functions ------------------------------ @@ -1755,22 +1771,22 @@ functionality of the task: directory listed is used as the current working directory for the task. -- ``[file-checksums]``: Controls the file dependencies for a task. The - baseline file list is the set of files associated with - :term:`SRC_URI`. May be used to set additional dependencies on - files not associated with :term:`SRC_URI`. +- ``[file-checksums]``: Controls the file dependencies for a task. The + baseline file list is the set of files associated with + :term:`SRC_URI`. May be used to set additional dependencies on + files not associated with :term:`SRC_URI`. - The value set to the list is a file-boolean pair where the first - value is the file name and the second is whether or not it - physically exists on the filesystem. :: + The value set to the list is a file-boolean pair where the first + value is the file name and the second is whether or not it + physically exists on the filesystem. :: - do_configure[file-checksums] += "${MY_DIRPATH}/my-file.txt:True" + do_configure[file-checksums] += "${MY_DIRPATH}/my-file.txt:True" - It is important to record any paths which the task looked at and - which didn't exist. This means that if these do exist at a later - time, the task can be rerun with the new additional files. The - "exists" True or False value after the path allows this to be - handled. + It is important to record any paths which the task looked at and + which didn't exist. This means that if these do exist at a later + time, the task can be rerun with the new additional files. The + "exists" True or False value after the path allows this to be + handled. - ``[lockfiles]``: Specifies one or more lockfiles to lock while the task executes. Only one task may hold a lockfile, and any task that diff --git a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst index d523880dcb8..8d8e8b8b912 100644 --- a/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst +++ b/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst @@ -286,8 +286,8 @@ overview of their function and contents. Specifies the internal list of variables to allow through from the external environment into BitBake's datastore. If the value of this variable is not specified (which is the default), the following - list is used: :term:`BBPATH`, :term:`BB_PRESERVE_ENV`, - :term:`BB_ENV_PASSTHROUGH`, and :term:`BB_ENV_PASSTHROUGH_ADDITIONS`. + list is used: :term:`BBPATH`, :term:`BB_PRESERVE_ENV`, and + :term:`BB_ENV_PASSTHROUGH_ADDITIONS`. .. note:: @@ -1048,28 +1048,101 @@ overview of their function and contents. compiler. Consequently, the syntax follows Python's Regular Expression (re) syntax. The expressions are compared against the full paths to the files. For complete syntax information, see Python's - documentation at http://docs.python.org/3/library/re.html. + documentation at https://docs.python.org/3/library/re.html. The following example uses a complete regular expression to tell - BitBake to ignore all recipe and recipe append files in the - ``meta-ti/recipes-misc/`` directory:: + BitBake to ignore all recipe and recipe append files in + ``recipes-bsp`` directory (recursively) of ``meta-ti-bsp``:: - BBMASK = "meta-ti/recipes-misc/" + BBMASK = "${BBFILE_PATTERN_meta-ti-bsp}/recipes-bsp/" If you want to mask out multiple directories or recipes, you can specify multiple regular expression fragments. This next example masks out multiple directories and individual recipes:: - BBMASK += "/meta-ti/recipes-misc/ meta-ti/recipes-ti/packagegroup/" - BBMASK += "/meta-oe/recipes-support/" - BBMASK += "/meta-foo/.*/openldap" - BBMASK += "opencv.*\.bbappend" - BBMASK += "lzma" + BBMASK += "${BBFILE_PATTERN_meta-ti-bsp}/recipes-graphics/libgal/" + BBMASK += "${BBFILE_PATTERN_openembedded-layer}/recipes-support/" + BBMASK += "${BBFILE_PATTERN_openembedded-layer}/.*/openldap" + BBMASK += "${BBFILE_PATTERN_meta-ti-bsp}/.*/optee.*\.bbappend" + + This masks: + + - everything under the ``recipes-graphics/libgal/`` directory from + ``meta-ti-bsp``, + - everything under the ``recipes-support/`` directory in ``meta-oe``, + - everything under a directory whose name starts with ``openldap``, and + every file with the same naming scheme, in ``meta-oe`` at any directory + depth > 1 (e.g. in ``meta-oe``, ``recipes-foo/openldap-stuff/`` or + ``recipes-bar/baz/openldap_0.1.bb`` but not ``openldap/``), + - every append file whose name starts with ``optee`` in ``meta-ti-bsp`` at + any directory depth > 1 (e.g. ``optee/optee-examples_%.bbappend`` and + ``recipes-security/optee/optee-client_%.bbappend``). .. note:: - When specifying a directory name, use the trailing slash character - to ensure you match just that directory name. + Because these are complete regular expressions, if you want to match a + directory and not a file, you must end the expression with a trailing + slash. That is:: + + BBMASK += "${BBFILE_PATTERN_meta-ti-bsp}/recipes-graphics/libgal/" + + Will match anything under ``recipes-graphics/ligbal/`` directory of + ``meta-ti-bsp``. And:: + + BBMASK += "${BBFILE_PATTERN_meta-ti-bsp}/recipes-graphics/libgal" + + Will match in ``meta-ti-bsp`` any file prefixed with ``libgal`` in + ``recipes-graphics/`` and any directory (recursively; and its + recipes and recipe append files regardless how they are named) prefixed + with ``libgal`` in ``recipes-graphics/``. That is, provided your layers + are available at ``/bitbake-builds/poky-master/layers/``, it'll match:: + + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal.bb + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal_%.bbappend + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal-foo/foo.bb + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal-foo/foo/bz.bbappend + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal/bar.bb + + .. note:: + + Because these are complete regular expressions, failing to start the + pattern with a ``^`` sign (which is usually the first character in + :term:`BBFILE_PATTERN`) means it can match *any* portion of a path. + Take the following as an example:: + + BBMASK = "recipes-graphics/libgal/" + + This will match subdirectories and files in any path containing + ``recipes-graphics/libgal/``, meaning (considering your layers are + available at ``/bitbake-builds/poky-master/layers/``):: + + /bitbake-builds/poky-master/layers/meta-ti/meta-ti-bsp/recipes-graphics/libgal/ + /bitbake-builds/poky-master/layers/my-layer/foo-recipes-graphics/libgal/ + + will be both matched. This may be a more relaxed way of matching + directories, recipes and recipe append files from any third party layer + but is generally discouraged as it may be casting too wide of a net. + + .. note:: + + Because these are complete regular expressions, a leading slash does + not mean the path is absolute. It simply forces the directory to be + named exactly that. Take:: + + BBMASK = "recipes-graphics/libgal/" + + If you happen to have a directory ``foo-recipes-graphics/libgal/``, it + will be matched. + + Leading with a slash:: + + BBMASK = "/recipes-graphics/libgal/" + + makes sure that doesn't happen. However, this doesn't prevent matching + a directory ``recipes-graphics/libgal/`` from another layer. + + Again, we highly recommend using :term:`BBFILE_PATTERN` in the + expressions to specify absolute paths specific to one layer. :term:`BBMULTICONFIG` Enables BitBake to perform multiple configuration builds and lists @@ -1266,7 +1339,7 @@ overview of their function and contents. - When recipes are parsed - then for each parsed recipe. - Bitbake ignores changes to :term:`INHERIT` in individual recipes. + BitBake ignores changes to :term:`INHERIT` in individual recipes. For more information on :term:`INHERIT`, see the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:\`\`inherit\`\` configuration directive`" @@ -1665,6 +1738,12 @@ overview of their function and contents. - ``npm://``: Fetches JavaScript modules from a registry. + .. warning:: + + The NPM fetcher is currently disabled due to security concerns. See + `355cd226e0720a9ed7683bb01c8c0a58eee03664 `__ + for more information. + - ``p4://``: Fetches files from a Perforce (``p4``) revision control repository. diff --git a/doc/conf.py b/doc/conf.py index bce386624e2..9a53e55c938 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -6,20 +6,27 @@ # -- Path setup -------------------------------------------------------------- -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - -import sys import datetime +import os +import subprocess +import sys from pathlib import Path -current_version = "dev" +# Test that we are building from a Git repository +try: + subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) +except subprocess.CalledProcessError: + sys.exit("Building bitbake's documentation must be done from its Git repository.\n" + "Clone the repository with the following command:\n" + "git clone https://git.openembedded.org/bitbake ") + +sys.path.insert(0, os.path.abspath('.')) +import setversions + +current_version = setversions.get_current_version() +print(f"Version calculated to be {current_version}") # String used in sidebar version = 'Version: ' + current_version @@ -28,6 +35,12 @@ # Version seen in documentation_options.js and hence in js switchers code release = current_version +setversions.write_switchers_js("sphinx-static/switchers.js.in", + "sphinx-static/switchers.js", + current_version) + +setversions.write_releases_rst("releases.rst") + # -- Project information ----------------------------------------------------- project = 'Bitbake' diff --git a/doc/releases.rst b/doc/releases.rst deleted file mode 100644 index a0b0ce766ea..00000000000 --- a/doc/releases.rst +++ /dev/null @@ -1,199 +0,0 @@ -.. SPDX-License-Identifier: CC-BY-2.5 - -================================= -BitBake Supported Release Manuals -================================= - -******************************* -Release Series 5.3 (whinlatter) -******************************* - -- :yocto_docs:`BitBake 2.16 User Manual ` - -******************************* -Release Series 5.0 (scarthgap) -******************************* - -- :yocto_docs:`BitBake 2.8 User Manual ` - -****************************** -Release Series 4.0 (kirkstone) -****************************** - -- :yocto_docs:`BitBake 2.0 User Manual ` - -================================ -BitBake Outdated Release Manuals -================================ - -****************************** -Release Series 5.2 (walnascar) -****************************** - -- :yocto_docs:`BitBake 2.12 User Manual ` - -**************************** -Release Series 5.1 (styhead) -**************************** - -- :yocto_docs:`BitBake 2.10 User Manual ` - -******************************* -Release Series 4.3 (nanbield) -******************************* - -- :yocto_docs:`BitBake 2.6 User Manual ` - -******************************* -Release Series 4.2 (mickledore) -******************************* - -- :yocto_docs:`BitBake 2.4 User Manual ` - -***************************** -Release Series 4.1 (langdale) -***************************** - -- :yocto_docs:`BitBake 2.2 User Manual ` - -****************************** -Release Series 3.4 (honister) -****************************** - -- :yocto_docs:`BitBake 1.52 User Manual ` - -****************************** -Release Series 3.3 (hardknott) -****************************** - -- :yocto_docs:`BitBake 1.50 User Manual ` - -******************************* -Release Series 3.2 (gatesgarth) -******************************* - -- :yocto_docs:`BitBake 1.48 User Manual ` - -**************************** -Release Series 3.1 (dunfell) -**************************** - -- :yocto_docs:`BitBake 1.46 User Manual ` -- :yocto_docs:`3.1 BitBake User Manual ` -- :yocto_docs:`3.1.1 BitBake User Manual ` -- :yocto_docs:`3.1.2 BitBake User Manual ` -- :yocto_docs:`3.1.3 BitBake User Manual ` - -************************* -Release Series 3.0 (zeus) -************************* - -- :yocto_docs:`3.0 BitBake User Manual ` -- :yocto_docs:`3.0.1 BitBake User Manual ` -- :yocto_docs:`3.0.2 BitBake User Manual ` -- :yocto_docs:`3.0.3 BitBake User Manual ` -- :yocto_docs:`3.0.4 BitBake User Manual ` - -**************************** -Release Series 2.7 (warrior) -**************************** - -- :yocto_docs:`2.7 BitBake User Manual ` -- :yocto_docs:`2.7.1 BitBake User Manual ` -- :yocto_docs:`2.7.2 BitBake User Manual ` -- :yocto_docs:`2.7.3 BitBake User Manual ` -- :yocto_docs:`2.7.4 BitBake User Manual ` - -************************* -Release Series 2.6 (thud) -************************* - -- :yocto_docs:`2.6 BitBake User Manual ` -- :yocto_docs:`2.6.1 BitBake User Manual ` -- :yocto_docs:`2.6.2 BitBake User Manual ` -- :yocto_docs:`2.6.3 BitBake User Manual ` -- :yocto_docs:`2.6.4 BitBake User Manual ` - -************************* -Release Series 2.5 (sumo) -************************* - -- :yocto_docs:`2.5 Documentation ` -- :yocto_docs:`2.5.1 Documentation ` -- :yocto_docs:`2.5.2 Documentation ` -- :yocto_docs:`2.5.3 Documentation ` - -************************** -Release Series 2.4 (rocko) -************************** - -- :yocto_docs:`2.4 BitBake User Manual ` -- :yocto_docs:`2.4.1 BitBake User Manual ` -- :yocto_docs:`2.4.2 BitBake User Manual ` -- :yocto_docs:`2.4.3 BitBake User Manual ` -- :yocto_docs:`2.4.4 BitBake User Manual ` - -************************* -Release Series 2.3 (pyro) -************************* - -- :yocto_docs:`2.3 BitBake User Manual ` -- :yocto_docs:`2.3.1 BitBake User Manual ` -- :yocto_docs:`2.3.2 BitBake User Manual ` -- :yocto_docs:`2.3.3 BitBake User Manual ` -- :yocto_docs:`2.3.4 BitBake User Manual ` - -************************** -Release Series 2.2 (morty) -************************** - -- :yocto_docs:`2.2 BitBake User Manual ` -- :yocto_docs:`2.2.1 BitBake User Manual ` -- :yocto_docs:`2.2.2 BitBake User Manual ` -- :yocto_docs:`2.2.3 BitBake User Manual ` - -**************************** -Release Series 2.1 (krogoth) -**************************** - -- :yocto_docs:`2.1 BitBake User Manual ` -- :yocto_docs:`2.1.1 BitBake User Manual ` -- :yocto_docs:`2.1.2 BitBake User Manual ` -- :yocto_docs:`2.1.3 BitBake User Manual ` - -*************************** -Release Series 2.0 (jethro) -*************************** - -- :yocto_docs:`1.9 BitBake User Manual ` -- :yocto_docs:`2.0 BitBake User Manual ` -- :yocto_docs:`2.0.1 BitBake User Manual ` -- :yocto_docs:`2.0.2 BitBake User Manual ` -- :yocto_docs:`2.0.3 BitBake User Manual ` - -************************* -Release Series 1.8 (fido) -************************* - -- :yocto_docs:`1.8 BitBake User Manual ` -- :yocto_docs:`1.8.1 BitBake User Manual ` -- :yocto_docs:`1.8.2 BitBake User Manual ` - -************************** -Release Series 1.7 (dizzy) -************************** - -- :yocto_docs:`1.7 BitBake User Manual ` -- :yocto_docs:`1.7.1 BitBake User Manual ` -- :yocto_docs:`1.7.2 BitBake User Manual ` -- :yocto_docs:`1.7.3 BitBake User Manual ` - -************************** -Release Series 1.6 (daisy) -************************** - -- :yocto_docs:`1.6 BitBake User Manual ` -- :yocto_docs:`1.6.1 BitBake User Manual ` -- :yocto_docs:`1.6.2 BitBake User Manual ` -- :yocto_docs:`1.6.3 BitBake User Manual ` - diff --git a/doc/setversions.py b/doc/setversions.py new file mode 100755 index 00000000000..f1aea0a4225 --- /dev/null +++ b/doc/setversions.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +# +# This file defines is used in doc/conf.py to setup the version information for +# the documentation: +# - get_current_version() used in doc/conf.py computes the current version by +# trying to guess the approximate versions we're at using git tags and +# branches from the repository. +# - write_switchers_js() write the switchers.js file used for switching between +# versions of the documentation. +# +# Copyright (c) 2026 Antonin Godard +# +# SPDX-License-Identifier: MIT +# + +import itertools +import json +import os +import re +import subprocess +import sys +import textwrap + +from urllib.request import urlopen, URLError + +# NOTE: the following variables contain default values in case we are not able to fetch +# the releases.json file from https://dashboard.yoctoproject.org/releases.json +DEVBRANCH = "2.18" +LTSSERIES = ["2.8", "2.0"] +ACTIVERELEASES = ["2.16"] + LTSSERIES +YOCTO_MAPPING = { + "2.18": "wrynose", + "2.16": "whinlatter", + "2.8": "scarthgap", + "2.0": "kirkstone", +} + +RELEASES_FROM_JSON = {} + +# Use the local releases.json file if found, fetch it from the dashboard otherwise +releases_json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "releases.json") +try: + with open(releases_json_path, "r") as f: + RELEASES_FROM_JSON = json.load(f) +except FileNotFoundError: + print("Fetching releases.json from https://dashboard.yoctoproject.org/releases.json...", + file=sys.stderr) + try: + with urlopen("https://dashboard.yoctoproject.org/releases.json") as r, \ + open(releases_json_path, "w") as f: + RELEASES_FROM_JSON = json.load(r) + json.dump(RELEASES_FROM_JSON, f) + except URLError: + print("WARNING: tried to fetch https://dashboard.yoctoproject.org/releases.json " + "but failed, using default values for active releases", file=sys.stderr) + pass + +if RELEASES_FROM_JSON: + ACTIVERELEASES = [] + DEVBRANCH = "" + LTSSERIES = [] + YOCTO_MAPPING = {} + + for release in RELEASES_FROM_JSON: + bb_ver = release["bitbake_version"] + if release["status"] == "Active Development": + DEVBRANCH = bb_ver + if "LTS until" in release["status"]: + LTSSERIES.append(bb_ver) + if release["bitbake_version"]: + YOCTO_MAPPING[bb_ver] = release["release_codename"] + + # Find the first non-dev release, which should be displayed as the default + # page on the docs website. + current_branch = "" + for release in RELEASES_FROM_JSON: + if release["status"] != "Active Development": + current_branch = release["bitbake_version"] + break + + if not current_branch: + sys.exit("Unable to find a current release! Exiting...") + + # make the list of releases unique, there can be duplication when the + # current releases is also an LTS + ACTIVERELEASES = list(dict.fromkeys([current_branch] + LTSSERIES)) + +print(f"ACTIVERELEASES calculated to be {ACTIVERELEASES}", file=sys.stderr) +print(f"DEVBRANCH calculated to be {DEVBRANCH}", file=sys.stderr) +print(f"LTSSERIES calculated to be {LTSSERIES}", file=sys.stderr) + +BB_RELEASE_TAG_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") + +def get_current_version(): + # Test tags exist and inform the user to fetch if not + try: + subprocess.run(["git", "show", f"{LTSSERIES[0]}.0"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + except subprocess.CalledProcessError: + sys.exit("Please run 'git fetch --tags' before building the documentation") + + # Try and figure out what we are + tags = subprocess.run(["git", "tag", "--points-at", "HEAD"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout + for t in tags.split(): + if re.match(BB_RELEASE_TAG_RE, t): + return t + + # We're floating on a branch + branch = subprocess.run(["git", "branch", "--show-current"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout.strip() + + if branch == "" or branch not in list(YOCTO_MAPPING.keys()) + ["master", "master-next"]: + # We're not on a known release branch so we have to guess. Compare the + # numbers of commits from each release branch and assume the smallest + # number of commits is the one we're based off + possible_branch = None + branch_count = 0 + for b in itertools.chain(YOCTO_MAPPING.keys(), ["master"]): + result = subprocess.run(["git", "log", "--format=oneline", "HEAD..origin/" + b], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + if result.returncode == 0: + count = result.stdout.count('\n') + if not possible_branch or count < branch_count: + print("Branch %s has count %s" % (b, count)) + possible_branch = b + branch_count = count + if possible_branch: + branch = possible_branch + else: + branch = "master" + print("Nearest release branch estimated to be %s" % branch) + + if branch == "master": + return "dev" + + if branch == "master-next": + return "next" + + ourversion = branch + head_commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout.strip() + branch_commit = subprocess.run(["git", "rev-parse", "--short", branch], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True).stdout.strip() + if head_commit != branch_commit: + ourversion += f" ({head_commit})" + + return ourversion + +def write_switchers_js(js_in, js_out, current_version): + with open(js_in, "r") as r, open(js_out, "w") as w: + lines = r.readlines() + for line in lines: + if "VERSIONS_PLACEHOLDER" in line: + if current_version != "dev": + w.write(" 'dev': 'Unstable (dev)',\n") + for series in ACTIVERELEASES: + w.write(f" '{series}': '{series} ({YOCTO_MAPPING[series]})',\n") + else: + w.write(line) + print("switchers.js generated from switchers.js.in") + +def _release_section(series_version: str, codename: str, bitbake_version: str) -> str: + """ + Helper function to generate a release section, as: + + ******************** + Release Series xxxxx + ******************** + + - + """ + section_length = len(series_version) + len(codename) + 18 + return textwrap.dedent( + f"""\ + {'*' * section_length} + Release Series {series_version} ({codename}) + {'*' * section_length} + + - :yocto_docs:`BitBake {bitbake_version} User Manual ` + + """) + +def write_releases_rst(releases_rst_out: str): + """ + Generates the releases.rst file automatically, based on what is found + in the releases.json file. + """ + with open(releases_rst_out, "w") as f: + f.write(textwrap.dedent("""\ + .. SPDX-License-Identifier: CC-BY-2.5 + + ================================= + BitBake Supported Release Manuals + ================================= + + """)) + + for release in RELEASES_FROM_JSON: + if release["status"] == "Active Development": + continue + + if not release["bitbake_version"]: + continue + + if release["series"] == "current": + f.write(_release_section( + release["series_version"], + release["release_codename"], + release["bitbake_version"])) + + f.write(textwrap.dedent("""\ + ================================ + BitBake Outdated Release Manuals + ================================ + + """)) + + for release in RELEASES_FROM_JSON: + if not release["series"] == "previous": + continue + + if not release["bitbake_version"]: + continue + + f.write(_release_section( + release["series_version"], + release["release_codename"], + release["bitbake_version"])) + + # old legacy links, which cannot be auto-generated + f.write(textwrap.dedent( + """\ + - :yocto_docs:`3.1.2 BitBake User Manual ` + - :yocto_docs:`3.1 BitBake User Manual ` + - :yocto_docs:`3.1.1 BitBake User Manual ` + - :yocto_docs:`3.1.3 BitBake User Manual ` + + ************************* + Release Series 3.0 (Zeus) + ************************* + + - :yocto_docs:`3.0 BitBake User Manual ` + - :yocto_docs:`3.0.1 BitBake User Manual ` + - :yocto_docs:`3.0.2 BitBake User Manual ` + - :yocto_docs:`3.0.3 BitBake User Manual ` + - :yocto_docs:`3.0.4 BitBake User Manual ` + + **************************** + Release Series 2.7 (Warrior) + **************************** + + - :yocto_docs:`2.7 BitBake User Manual ` + - :yocto_docs:`2.7.1 BitBake User Manual ` + - :yocto_docs:`2.7.2 BitBake User Manual ` + - :yocto_docs:`2.7.3 BitBake User Manual ` + - :yocto_docs:`2.7.4 BitBake User Manual ` + + ************************* + Release Series 2.6 (Thud) + ************************* + + - :yocto_docs:`2.6 BitBake User Manual ` + - :yocto_docs:`2.6.1 BitBake User Manual ` + - :yocto_docs:`2.6.2 BitBake User Manual ` + - :yocto_docs:`2.6.3 BitBake User Manual ` + - :yocto_docs:`2.6.4 BitBake User Manual ` + + ************************* + Release Series 2.5 (Sumo) + ************************* + + - :yocto_docs:`2.5 Documentation ` + - :yocto_docs:`2.5.1 Documentation ` + - :yocto_docs:`2.5.2 Documentation ` + - :yocto_docs:`2.5.3 Documentation ` + + ************************** + Release Series 2.4 (Rocko) + ************************** + + - :yocto_docs:`2.4 BitBake User Manual ` + - :yocto_docs:`2.4.1 BitBake User Manual ` + - :yocto_docs:`2.4.2 BitBake User Manual ` + - :yocto_docs:`2.4.3 BitBake User Manual ` + - :yocto_docs:`2.4.4 BitBake User Manual ` + + ************************* + Release Series 2.3 (Pyro) + ************************* + + - :yocto_docs:`2.3 BitBake User Manual ` + - :yocto_docs:`2.3.1 BitBake User Manual ` + - :yocto_docs:`2.3.2 BitBake User Manual ` + - :yocto_docs:`2.3.3 BitBake User Manual ` + - :yocto_docs:`2.3.4 BitBake User Manual ` + + ************************** + Release Series 2.2 (Morty) + ************************** + + - :yocto_docs:`2.2 BitBake User Manual ` + - :yocto_docs:`2.2.1 BitBake User Manual ` + - :yocto_docs:`2.2.2 BitBake User Manual ` + - :yocto_docs:`2.2.3 BitBake User Manual ` + + **************************** + Release Series 2.1 (Krogoth) + **************************** + + - :yocto_docs:`2.1 BitBake User Manual ` + - :yocto_docs:`2.1.1 BitBake User Manual ` + - :yocto_docs:`2.1.2 BitBake User Manual ` + - :yocto_docs:`2.1.3 BitBake User Manual ` + + *************************** + Release Series 2.0 (Jethro) + *************************** + + - :yocto_docs:`1.9 BitBake User Manual ` + - :yocto_docs:`2.0 BitBake User Manual ` + - :yocto_docs:`2.0.1 BitBake User Manual ` + - :yocto_docs:`2.0.2 BitBake User Manual ` + - :yocto_docs:`2.0.3 BitBake User Manual ` + + ************************* + Release Series 1.8 (Fido) + ************************* + + - :yocto_docs:`1.8 BitBake User Manual ` + - :yocto_docs:`1.8.1 BitBake User Manual ` + - :yocto_docs:`1.8.2 BitBake User Manual ` + + ************************** + Release Series 1.7 (Dizzy) + ************************** + + - :yocto_docs:`1.7 BitBake User Manual ` + - :yocto_docs:`1.7.1 BitBake User Manual ` + - :yocto_docs:`1.7.2 BitBake User Manual ` + - :yocto_docs:`1.7.3 BitBake User Manual ` + + ************************** + Release Series 1.6 (Daisy) + ************************** + + - :yocto_docs:`1.6 BitBake User Manual ` + - :yocto_docs:`1.6.1 BitBake User Manual ` + - :yocto_docs:`1.6.2 BitBake User Manual ` + - :yocto_docs:`1.6.3 BitBake User Manual ` + """)) diff --git a/doc/sphinx-static/switchers.js b/doc/sphinx-static/switchers.js deleted file mode 100644 index 32113cfa960..00000000000 --- a/doc/sphinx-static/switchers.js +++ /dev/null @@ -1,233 +0,0 @@ -(function() { - 'use strict'; - - var all_versions = { - 'dev': 'dev (3.2)', - '3.1.2': '3.1.2', - '3.0.3': '3.0.3', - '2.7.4': '2.7.4', - }; - - var all_doctypes = { - 'single': 'Individual Webpages', - 'mega': "All-in-one 'Mega' Manual", - }; - - // Simple version comparision - // Return 1 if a > b - // Return -1 if a < b - // Return 0 if a == b - function ver_compare(a, b) { - if (a == "dev") { - return 1; - } - - if (a === b) { - return 0; - } - - var a_components = a.split("."); - var b_components = b.split("."); - - var len = Math.min(a_components.length, b_components.length); - - // loop while the components are equal - for (var i = 0; i < len; i++) { - // A bigger than B - if (parseInt(a_components[i]) > parseInt(b_components[i])) { - return 1; - } - - // B bigger than A - if (parseInt(a_components[i]) < parseInt(b_components[i])) { - return -1; - } - } - - // If one's a prefix of the other, the longer one is greater. - if (a_components.length > b_components.length) { - return 1; - } - - if (a_components.length < b_components.length) { - return -1; - } - - // Otherwise they are the same. - return 0; - } - - function build_version_select(current_series, current_version) { - var buf = [''); - return buf.join(''); - } - - function build_doctype_select(current_doctype) { - var buf = [''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - var url = urls.shift(); - - // Web browsers won't redirect file:// urls to file urls using ajax but - // its useful for local testing - if (url.startsWith("file://")) { - window.location.href = url; - return; - } - - if (urls.length == 0) { - window.location.href = url; - return; - } - $.ajax({ - url: url, - success: function() { - window.location.href = url; - }, - error: function() { - navigate_to_first_existing(urls); - } - }); - } - - function get_docroot_url() { - var url = window.location.href; - var root = DOCUMENTATION_OPTIONS.URL_ROOT; - - var urlarray = url.split('/'); - // Trim off anything after '/' - urlarray.pop(); - var depth = (root.match(/\.\.\//g) || []).length; - for (var i = 0; i < depth; i++) { - urlarray.pop(); - } - - return urlarray.join('/') + '/'; - } - - function on_version_switch() { - var selected_version = $(this).children('option:selected').attr('value'); - var url = window.location.href; - var current_version = DOCUMENTATION_OPTIONS.VERSION; - var docroot = get_docroot_url() - - var new_versionpath = selected_version + '/'; - if (selected_version == "dev") - new_versionpath = ''; - - // dev versions have no version prefix - if (current_version == "dev") { - var new_url = docroot + new_versionpath + url.replace(docroot, ""); - var fallback_url = docroot + new_versionpath; - } else { - var new_url = url.replace('/' + current_version + '/', '/' + new_versionpath); - var fallback_url = new_url.replace(url.replace(docroot, ""), ""); - } - - console.log(get_docroot_url()) - console.log(url + " to url " + new_url); - console.log(url + " to fallback " + fallback_url); - - if (new_url != url) { - navigate_to_first_existing([ - new_url, - fallback_url, - 'https://www.yoctoproject.org/docs/', - ]); - } - } - - function on_doctype_switch() { - var selected_doctype = $(this).children('option:selected').attr('value'); - var url = window.location.href; - if (selected_doctype == 'mega') { - var docroot = get_docroot_url() - var current_version = DOCUMENTATION_OPTIONS.VERSION; - // Assume manuals before 3.2 are using old docbook mega-manual - if (ver_compare(current_version, "3.2") < 0) { - var new_url = docroot + "mega-manual/mega-manual.html"; - } else { - var new_url = docroot + "singleindex.html"; - } - } else { - var new_url = url.replace("singleindex.html", "index.html") - } - - if (new_url != url) { - navigate_to_first_existing([ - new_url, - 'https://www.yoctoproject.org/docs/', - ]); - } - } - - // Returns the current doctype based upon the url - function doctype_segment_from_url(url) { - if (url.includes("singleindex") || url.includes("mega-manual")) - return "mega"; - return "single"; - } - - $(document).ready(function() { - var release = DOCUMENTATION_OPTIONS.VERSION; - var current_doctype = doctype_segment_from_url(window.location.href); - var current_series = release.substr(0, 3); - var version_select = build_version_select(current_series, release); - - $('.version_switcher_placeholder').html(version_select); - $('.version_switcher_placeholder select').bind('change', on_version_switch); - - var doctype_select = build_doctype_select(current_doctype); - - $('.doctype_switcher_placeholder').html(doctype_select); - $('.doctype_switcher_placeholder select').bind('change', on_doctype_switch); - - if (ver_compare(release, "3.1") < 0) { - $('#outdated-warning').html('Version ' + release + ' of the project is now considered obsolete, please select and use a more recent version'); - $('#outdated-warning').css('padding', '.5em'); - } else if (release != "dev") { - $.each(all_versions, function(version, title) { - var series = version.substr(0, 3); - if (series == current_series && version != release) { - $('#outdated-warning').html('This document is for outdated version ' + release + ', you should select the latest release version in this series, ' + version + '.'); - $('#outdated-warning').css('padding', '.5em'); - } - }); - } - }); -})(); diff --git a/doc/sphinx-static/switchers.js.in b/doc/sphinx-static/switchers.js.in new file mode 100644 index 00000000000..0b209e959da --- /dev/null +++ b/doc/sphinx-static/switchers.js.in @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: MIT +(function () { + "use strict"; + + var all_versions = { + VERSIONS_PLACEHOLDER, + }; + + var all_doctypes = { + single: "Individual Webpages", + mega: "All-in-one 'Mega' Manual", + }; + + // Simple version comparision + // Return 1 if a > b + // Return -1 if a < b + // Return 0 if a == b + function ver_compare(a, b) { + if (a == "dev") { + return 1; + } + + if (a === b) { + return 0; + } + + var a_components = a.split("."); + var b_components = b.split("."); + + var len = Math.min(a_components.length, b_components.length); + + // loop while the components are equal + for (var i = 0; i < len; i++) { + // A bigger than B + if (parseInt(a_components[i]) > parseInt(b_components[i])) { + return 1; + } + + // B bigger than A + if (parseInt(a_components[i]) < parseInt(b_components[i])) { + return -1; + } + } + + // If one's a prefix of the other, the longer one is greater. + if (a_components.length > b_components.length) { + return 1; + } + + if (a_components.length < b_components.length) { + return -1; + } + + // Otherwise they are the same. + return 0; + } + + function build_version_select(current_version) { + var buf = [""); + return buf.join(""); + } + + function build_doctype_select(current_doctype) { + var buf = [""); + return buf.join(""); + } + + function navigate_to_first_existing(urls) { + // Navigate to the first existing URL in urls. + var url = urls.shift(); + + // Web browsers won't redirect file:// urls to file urls using ajax but + // its useful for local testing + if (url.startsWith("file://")) { + window.location.href = url; + return; + } + + if (urls.length == 0) { + window.location.href = url; + return; + } + $.ajax({ + url: url, + success: function () { + window.location.href = url; + }, + error: function () { + navigate_to_first_existing(urls); + }, + }); + } + + function get_docroot_url() { + var url = window.location.href; + // Try to get the variable from documentation_options.js + var root = DOCUMENTATION_OPTIONS.URL_ROOT; + if (root == null) { + // In recent versions of Sphinx, URL_ROOT was removed from + // documentation_options.js, so get it like searchtools.js does. + root = document.documentElement.dataset.content_root; + } + + var urlarray = url.split("/"); + // Trim off anything after '/' + urlarray.pop(); + var depth = (root.match(/\.\.\//g) || []).length; + for (var i = 0; i < depth; i++) { + urlarray.pop(); + } + + return urlarray.join("/") + "/"; + } + + function on_version_switch() { + var selected_version = $(this).children("option:selected").attr("value"); + var url = window.location.href; + var current_version = DOCUMENTATION_OPTIONS.VERSION; + var docroot = get_docroot_url(); + + var new_versionpath = selected_version + "/"; + + // latest tag is also the default page (without version information) + if (docroot.endsWith("dev/")) { + var new_url = url.replace("/dev/", "/" + new_versionpath); + var fallback_url = new_url.replace(url.replace(docroot, ""), ""); + } else if (docroot.endsWith(current_version + "/") == false) { + var new_url = docroot + new_versionpath + url.replace(docroot, ""); + var fallback_url = docroot + new_versionpath; + } else { + var new_url = url.replace( + "/" + current_version + "/", + "/" + new_versionpath, + ); + var fallback_url = new_url.replace(url.replace(docroot, ""), ""); + } + + console.log(url + " to url " + new_url); + console.log(url + " to fallback " + fallback_url); + + if (new_url != url) { + navigate_to_first_existing([ + new_url, + fallback_url, + "https://www.yoctoproject.org/bitbake/", + ]); + } + } + + function on_doctype_switch() { + var selected_doctype = $(this).children("option:selected").attr("value"); + var url = window.location.href; + if (selected_doctype == "mega") { + var docroot = get_docroot_url(); + var current_version = DOCUMENTATION_OPTIONS.VERSION; + var new_url = docroot + "singleindex.html"; + } else { + var new_url = url.replace("singleindex.html", "index.html"); + } + + if (new_url != url) { + navigate_to_first_existing([ + new_url, + "https://www.yoctoproject.org/docs/", + ]); + } + } + + // Returns the current doctype based upon the url + function doctype_segment_from_url(url) { + if (url.includes("singleindex") || url.includes("mega-manual")) + return "mega"; + return "single"; + } + + $(document).ready(function () { + var release = DOCUMENTATION_OPTIONS.VERSION; + var current_doctype = doctype_segment_from_url(window.location.href); + var current_series = release.substr(0, 3); + var version_select = build_version_select(release); + + $(".version_switcher_placeholder").html(version_select); + $(".version_switcher_placeholder select").bind("change", on_version_switch); + + var doctype_select = build_doctype_select(current_doctype); + + $(".doctype_switcher_placeholder").html(doctype_select); + $(".doctype_switcher_placeholder select").bind("change", on_doctype_switch); + + // if release = "X.Y ()", remove the "()" so that only X.Y is compared + release = release.split(" ")[0]; + if (!(["dev", "next"].includes(release)) && !(release in all_versions)) { + $("#outdated-warning").html( + "Version " + release + " of the project is now considered obsolete, please select and use a more recent version", + ); + $("#outdated-warning").css("padding", ".5em"); + } + }); +})(); diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py index 6765fa6134d..77edfa2bc98 100644 --- a/lib/bb/__init__.py +++ b/lib/bb/__init__.py @@ -9,7 +9,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -__version__ = "2.16.0" +__version__ = "2.18.0" import sys if sys.version_info < (3, 9, 0): diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py index 22ac95eac9f..59f808c9694 100644 --- a/lib/bb/cookerdata.py +++ b/lib/bb/cookerdata.py @@ -512,6 +512,8 @@ def _parse_recipe(bb_data, bbfile, appends, mc, layername): bb_data.setVar("__BBMULTICONFIG", mc) bb_data.setVar("FILE_LAYERNAME", layername) + bb_data.setVar("__BB_RECIPE_FILE", bbfile) + bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) bb.parse.cached_mtime_noerror(bbfile_loc) diff --git a/lib/bb/data.py b/lib/bb/data.py index 061e63386f0..5fdcdb04a6a 100644 --- a/lib/bb/data.py +++ b/lib/bb/data.py @@ -367,7 +367,7 @@ def handle_remove(value, deps, removes, d): except bb.parse.SkipRecipe: raise except Exception as e: - bb.warn("Exception during build_dependencies for %s" % key) + bb.warn("Exception during build_dependencies for %s: %s" % (key, repr(e))) raise return frozenset(deps), value #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py index dcf7adc26ab..52d5556d368 100644 --- a/lib/bb/fetch2/__init__.py +++ b/lib/bb/fetch2/__init__.py @@ -96,6 +96,23 @@ def __init__(self, message, url): BBFetchException.__init__(self, msg) self.args = (message, url) +class LocalModificationsError(UnpackError): + """Exception raised when a checkout cannot be updated due to local modifications""" + def __init__(self, repodir, url, git_output): + message = ("Repository at %s has uncommitted changes, unable to update:\n%s\n" + "Commit, stash or discard your changes and re-run the update." % (repodir, git_output)) + UnpackError.__init__(self, message, url) + self.args = (repodir, url, git_output) + +class RebaseError(UnpackError): + """Exception raised when a checkout has local commits that could not be rebased onto the new upstream revision""" + def __init__(self, repodir, url, git_output): + message = ("Repository at %s has local commits that could not be rebased onto the new upstream revision:\n%s\n" + "Note: the 'dldir' remote points to the local download cache and may be used to resolve the conflict manually.\n" + "Once resolved, re-run the update." % (repodir, git_output)) + UnpackError.__init__(self, message, url) + self.args = (repodir, url, git_output) + class NoMethodError(BBFetchException): """Exception raised when there is no method to obtain a supplied url or set of urls""" def __init__(self, url): @@ -877,6 +894,7 @@ def localpath(url, d): 'FTPS_PROXY', 'ftps_proxy', 'NO_PROXY', 'no_proxy', 'ALL_PROXY', 'all_proxy', + 'GIT_CONFIG_GLOBAL', 'GIT_PROXY_COMMAND', 'GIT_SSH', 'GIT_SSH_COMMAND', @@ -1633,6 +1651,9 @@ def unpack(self, urldata, rootdir, data): return + def unpack_update(self, urldata, rootdir, data): + raise RuntimeError("No method available for this url type: %s" % urldata.type) + def clean(self, urldata, d): """ Clean any existing full or partial download @@ -1989,7 +2010,7 @@ def checkstatus(self, urls=None): if not ret: raise FetchError("URL doesn't work", u) - def unpack(self, root, urls=None): + def unpack(self, root, urls=None, update=False): """ Unpack urls to root """ @@ -2008,7 +2029,10 @@ def unpack(self, root, urls=None): lf = bb.utils.lockfile(ud.lockfile) unpack_tracer.start_url(u) - ud.method.unpack(ud, root, self.d) + if update: + ud.method.unpack_update(ud, root, self.d) + else: + ud.method.unpack(ud, root, self.d) unpack_tracer.finish_url(u) finally: @@ -2017,6 +2041,9 @@ def unpack(self, root, urls=None): unpack_tracer.complete() + def unpack_update(self, root, urls=None): + self.unpack(root, urls, update=True) + def clean(self, urls=None): """ Clean files that the fetcher gets or places diff --git a/lib/bb/fetch2/crate.py b/lib/bb/fetch2/crate.py index e611736f069..b89817ab94d 100644 --- a/lib/bb/fetch2/crate.py +++ b/lib/bb/fetch2/crate.py @@ -45,6 +45,17 @@ def urldata_init(self, ud, d): super(Crate, self).urldata_init(ud, d) + def _generate_index_path(self, name): + # https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files + if len(name) == 1: + return f"1/{name}" + elif len(name) == 2: + return f"2/{name}" + elif len(name) == 3: + return f"3/{name[0]}/{name}" + else: + return f"{name[0:2]}/{name[2:4]}/{name}" + def _crate_urldata_init(self, ud, d): """ Sets up the download for a crate @@ -65,12 +76,15 @@ def _crate_urldata_init(self, ud, d): # host (this is to allow custom crate registries to be specified host = '/'.join(parts[2:-2]) - # if using upstream just fix it up nicely + # If using crates.io use the CDN directly as per https://crates.io/data-access if host == 'crates.io': - host = 'crates.io/api/v1/crates' + ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version) + ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name) + self.latest_versionstring = self.latest_versionstring_from_index + else: + ud.url = "https://%s/%s/%s/download" % (host, name, version) + ud.versionsurl = "https://%s/%s/versions" % (host, name) - ud.url = "https://%s/%s/%s/download" % (host, name, version) - ud.versionsurl = "https://%s/%s/versions" % (host, name) ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version) if 'name' not in ud.parm: ud.parm['name'] = '%s-%s' % (name, version) @@ -142,9 +156,29 @@ def _crate_unpack(self, ud, rootdir, d): json.dump(metadata, f) def latest_versionstring(self, ud, d): + """ + Return the latest version available when versionsurl is the [name]/versions URL. + """ from functools import cmp_to_key json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d)) versions = [(0, i["num"], "") for i in json_data["versions"]] versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp)) return (versions[-1][1], "") + + def latest_versionstring_from_index(self, ud, d): + """ + Return the latest version available when versionsurl is a Cargo index + file. + https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files + """ + from functools import cmp_to_key + + versions = [] + response = self._fetch_index(ud.versionsurl, ud, d) + for line in response.splitlines(): + data = json.loads(line) + versions.append((0, data["vers"], "")) + + versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp)) + return (versions[-1][1], "") diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py index 0fcdb19df11..ecf4340b1bc 100644 --- a/lib/bb/fetch2/git.py +++ b/lib/bb/fetch2/git.py @@ -356,7 +356,7 @@ def update_mirror_links(self, ud, origud): super().update_mirror_links(ud, origud) # When using shallow mode, add a symlink to the original fullshallow # path to ensure a valid symlink even in the `PREMIRRORS` case - if ud.shallow and not os.path.exists(origud.fullshallow): + if origud.shallow and not os.path.exists(origud.fullshallow): self.ensure_symlink(ud.localpath, origud.fullshallow) def try_premirror(self, ud, d): @@ -392,14 +392,14 @@ def download(self, ud, d): bb.utils.mkdirhier(ud.clonedir) runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) else: - tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) - runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir) - output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) - if 'mirror' in output: - runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir) - runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd) - runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) + with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir: + runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir) + output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) + if 'mirror' in output: + runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir) + runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir) + fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd) + runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) repourl = self._get_repo_url(ud) needs_clone = False @@ -475,9 +475,9 @@ def download(self, ud, d): bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir) - runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir) + runfetchcmd("%s repack -adk" % ud.basecmd, d, workdir=ud.clonedir) runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir) - runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir) + runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir) try: os.unlink(ud.fullmirror) except OSError as exc: @@ -543,27 +543,23 @@ def build_mirror_data(self, ud, d): runfetchcmd("touch %s.done" % ud.fullmirror, d) def clone_shallow_with_tarball(self, ud, d): - ret = False - tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) - shallowclone = os.path.join(tempdir, 'git') - try: - try: - self.clone_shallow_local(ud, shallowclone, d) - except: - logger.warning("Fast shallow clone failed, try to skip fast mode now.") - bb.utils.remove(tempdir, recurse=True) - os.mkdir(tempdir) - ud.shallow_skip_fast = True - self.clone_shallow_local(ud, shallowclone, d) - logger.info("Creating tarball of git repository") - with self.create_atomic(ud.fullshallow) as tfile: - runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) - runfetchcmd("touch %s.done" % ud.fullshallow, d) - ret = True - finally: - bb.utils.remove(tempdir, recurse=True) - - return ret + for fast in [True, False]: + ud.shallow_skip_fast = not fast + with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tempdir: + shallowclone = os.path.join(tempdir, 'git') + try: + self.clone_shallow_local(ud, shallowclone, d) + except: + if not fast: + raise + logger.warning("Fast shallow clone failed, try to skip fast mode now.") + continue + logger.info("Creating tarball of git repository") + with self.create_atomic(ud.fullshallow) as tfile: + runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) + runfetchcmd("touch %s.done" % ud.fullshallow, d) + return True + return False def clone_shallow_local(self, ud, dest, d): """ @@ -656,7 +652,10 @@ def clone_shallow_local(self, ud, dest, d): # The url is local ud.clonedir, set it to upstream one runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest) - def unpack(self, ud, destdir, d): + def unpack_update(self, ud, destdir, d): + return self.unpack(ud, destdir, d, update=True) + + def unpack(self, ud, destdir, d, update=False): """ unpack the downloaded src to destdir""" subdir = ud.parm.get("subdir") @@ -680,7 +679,7 @@ def unpack(self, ud, destdir, d): destsuffix = ud.parm.get("destsuffix", def_destsuffix) destdir = ud.destdir = os.path.join(destdir, destsuffix) - if os.path.exists(destdir): + if os.path.exists(destdir) and not update: bb.utils.prunedir(destdir) if not ud.bareclone: ud.unpack_tracer.unpack("git", destdir) @@ -691,11 +690,15 @@ def unpack(self, ud, destdir, d): ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd source_found = False + update_mode = False source_error = [] clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) if clonedir_is_up_to_date: - runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) + if update and os.path.exists(destdir): + update_mode = True + else: + runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) source_found = True else: source_error.append("clone directory not available or not up to date: " + ud.clonedir) @@ -703,8 +706,11 @@ def unpack(self, ud, destdir, d): if not source_found: if ud.shallow: if os.path.exists(ud.fullshallow): - bb.utils.mkdirhier(destdir) - runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir) + if update and os.path.exists(destdir): + update_mode = True + else: + bb.utils.mkdirhier(destdir) + runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir) source_found = True else: source_error.append("shallow clone not available: " + ud.fullshallow) @@ -714,6 +720,35 @@ def unpack(self, ud, destdir, d): if not source_found: raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) + if update_mode: + if ud.shallow: + raise bb.fetch2.UnpackError("Can't update shallow clones checkouts without network access, not supported.", ud.url) + + output = runfetchcmd("%s status --untracked-files=no --porcelain" % (ud.basecmd), d, workdir=destdir) + if output: + raise bb.fetch2.LocalModificationsError(destdir, ud.url, output) + + # Set up remote for the download location if it doesn't exist + try: + runfetchcmd("%s remote get-url dldir" % (ud.basecmd), d, workdir=destdir) + except bb.fetch2.FetchError: + if ud.clonedir: + runfetchcmd("%s remote add dldir file://%s" % (ud.basecmd, ud.clonedir), d, workdir=destdir) + try: + runfetchcmd("%s fetch dldir" % (ud.basecmd), d, workdir=destdir) + except bb.fetch2.FetchError as e: + raise bb.fetch2.UnpackError("Failed to fetch from dldir remote: %s" % str(e), ud.url) + try: + runfetchcmd("%s rebase --no-autosquash --no-autostash %s" % (ud.basecmd, ud.revision), d, workdir=destdir) + except bb.fetch2.FetchError as e: + # If rebase failed, abort it + try: + runfetchcmd("%s rebase --abort" % (ud.basecmd), d, workdir=destdir) + except Exception: + pass + raise bb.fetch2.RebaseError(destdir, ud.url, str(e)) + return True + # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag # matches the revision if 'tag' in ud.parm and sha1_re.match(ud.revision): @@ -729,6 +764,11 @@ def unpack(self, ud, destdir, d): repourl = self._get_repo_url(ud) runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) + if ud.clonedir: + try: + runfetchcmd("%s remote get-url dldir" % (ud.basecmd), d, workdir=destdir) + except bb.fetch2.FetchError: + runfetchcmd("%s remote add dldir file://%s" % (ud.basecmd, ud.clonedir), d, workdir=destdir) if self._contains_lfs(ud, d, destdir): if not need_lfs: @@ -743,6 +783,12 @@ def unpack(self, ud, destdir, d): runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d, workdir=destdir) runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) + runfetchcmd("%s update-ref --no-deref HEAD %s" % (ud.basecmd, ud.revision), + d, workdir=destdir) + if not ud.nobranch: + branchname = ud.branch + runfetchcmd("%s update-ref refs/heads/%s %s" % (ud.basecmd, branchname, + ud.revision), d, workdir=destdir) elif not ud.nobranch: branchname = ud.branch runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ diff --git a/lib/bb/fetch2/svn.py b/lib/bb/fetch2/svn.py index 0852108e7d9..a097ffb76b5 100644 --- a/lib/bb/fetch2/svn.py +++ b/lib/bb/fetch2/svn.py @@ -34,7 +34,7 @@ def urldata_init(self, ud, d): if not "module" in ud.parm: raise MissingParameterError('module', ud.url) - ud.basecmd = d.getVar("FETCHCMD_svn") or "/usr/bin/env svn --non-interactive --trust-server-cert" + ud.basecmd = d.getVar("FETCHCMD_svn") or "/usr/bin/env svn --non-interactive" ud.module = ud.parm["module"] diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py index 4e3505599b4..6ac4306c0cc 100644 --- a/lib/bb/fetch2/wget.py +++ b/lib/bb/fetch2/wget.py @@ -216,31 +216,20 @@ def do_open(self, http_class, req): try: h.request(req.get_method(), req.selector, req.data, headers) - except socket.error as err: # XXX what error? - # Don't close connection when cache is enabled. - # Instead, try to detect connections that are no longer - # usable (for example, closed unexpectedly) and remove - # them from the cache. - if fetch.connection_cache is None: - h.close() - elif isinstance(err, OSError) and err.errno == errno.EBADF: - # This happens when the server closes the connection despite the Keep-Alive. - # Apparently urllib then uses the file descriptor, expecting it to be - # connected, when in reality the connection is already gone. - # We let the request fail and expect it to be - # tried once more ("try_again" in check_status()), - # with the dead connection removed from the cache. - # If it still fails, we give up, which can happen for bad - # HTTP proxy settings. + r = h.getresponse() + except: + # This can happen when the server closes the connection despite the Keep-Alive. + # Apparently urllib then uses the file descriptor, expecting it to be + # connected, when in reality the connection is already gone. + # We let the request fail and expect it to be + # tried once more ("try_again" in check_status()), + # with the dead connection removed from the cache. + # If it still fails, we give up, which can happen for bad + # HTTP proxy settings. + if fetch.connection_cache: fetch.connection_cache.remove_connection(h.host, h.port) - raise urllib.error.URLError(err) - else: - try: - r = h.getresponse() - except TimeoutError as e: - if fetch.connection_cache: - fetch.connection_cache.remove_connection(h.host, h.port) - raise TimeoutError(e) + h.close() + raise # Pick apart the HTTPResponse object to get the addinfourl # object initialized properly. @@ -404,9 +393,9 @@ def add_basic_auth(login_str, request): with opener.open(r, timeout=100) as response: pass - except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e: + except (urllib.error.URLError, OSError, http.client.RemoteDisconnected) as e: if try_again: - logger.debug2("checkstatus: trying again") + logger.debug2("checkstatus: trying again after exception %s" % str(e)) return self.checkstatus(fetch, ud, d, False) else: # debug for now to avoid spamming the logs in e.g. remote sstate searches @@ -550,7 +539,7 @@ def _check_latest_version_by_dir(self, dirver, package, package_regex, current_v version_dir = ['', '', ''] version = ['', '', ''] - dirver_regex = re.compile(r"(?P\D*)(?P(\d+[\.\-_])*(\d+))") + dirver_regex = re.compile(r"^(?P\D*)(?P(\d+[\.\-_])*(\d+))$") s = dirver_regex.search(dirver) if s: version_dir[1] = s.group('ver') diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index a880a0d5416..a4689e134ad 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -23,11 +23,12 @@ from bb import monitordisk import subprocess import pickle -from multiprocessing import Process import shlex import pprint import time +Process = bb.multiprocessing.Process + bblogger = logging.getLogger("BitBake") logger = logging.getLogger("BitBake.RunQueue") hashequiv_logger = logging.getLogger("BitBake.RunQueue.HashEquiv") @@ -693,7 +694,7 @@ def prepare(self): # To create the actual list of tasks to execute we fix the list of # providers and then resolve the dependencies into task IDs. This # process is repeated for each type of dependency (tdepends, deptask, - # rdeptast, recrdeptask, idepends). + # rdeptask, recrdeptask, idepends). def add_build_dependencies(depids, tasknames, depends, mc): for depname in depids: @@ -841,7 +842,7 @@ def add_mc_dependencies(mc, tid): # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed - # Generating/interating recursive lists of dependencies is painful and potentially slow + # Generating/iterating recursive lists of dependencies is painful and potentially slow # Precompute recursive task dependencies here by: # a) create a temp list of reverse dependencies (revdeps) # b) walk up the ends of the chains (when a given task no longer has dependencies i.e. len(deps) == 0) diff --git a/lib/bb/taskdata.py b/lib/bb/taskdata.py index 66545a65af9..b40afc02b71 100644 --- a/lib/bb/taskdata.py +++ b/lib/bb/taskdata.py @@ -138,15 +138,15 @@ def handle_deps(task, dep_name, depends, seen): rdependids = set() rdepends = dataCache.rundeps[fn] rrecs = dataCache.runrecs[fn] - rdependlist = [] - rreclist = [] + rdependlist = set() + rreclist = set() for package in rdepends: for rdepend in rdepends[package]: - rdependlist.append(rdepend) + rdependlist.add(rdepend) rdependids.add(rdepend) for package in rrecs: for rdepend in rrecs[package]: - rreclist.append(rdepend) + rreclist.add(rdepend) rdependids.add(rdepend) if rdependlist: logger.debug2("Added runtime dependencies %s for %s", str(rdependlist), fn) diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py index 2a8501dc2ea..077f741e1d3 100644 --- a/lib/bb/tests/fetch.py +++ b/lib/bb/tests/fetch.py @@ -18,7 +18,6 @@ import signal import tarfile from bb.fetch2 import URI -from bb.fetch2 import FetchMethod import bb import bb.utils from bb.tests.support.httpserver import HTTPService @@ -551,8 +550,8 @@ def test_mirror_of_mirror(self): fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) mirrors = bb.fetch2.mirror_from_string(mirrorvar) uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) - self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', - 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', + self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', + 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) @@ -1115,7 +1114,7 @@ def test_git_submodule(self): # URL with ssh submodules url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https" # Original URL (comment this if you have ssh access to git.yoctoproject.org) - url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https" + url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=38e61644af90dccd73c03ed3acaed98c8dda9294;branch=master;protocol=https" fetcher = bb.fetch.Fetch([url], self.d) fetcher.download() # Previous cwd has been deleted @@ -1390,7 +1389,7 @@ class URLHandle(unittest.TestCase): "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}), 'git://s.o-me_ONE:%s@git.openembedded.org/bitbake;branch=main;protocol=https' % password: ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', password, {'branch': 'main', 'protocol' : 'https'}), } - # we require a pathname to encodeurl but users can still pass such urls to + # we require a pathname to encodeurl but users can still pass such urls to # decodeurl and we need to handle them decodedata = datatable.copy() decodedata.update({ @@ -1579,8 +1578,6 @@ class FetchCheckStatusTest(FetcherTest): "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", - # GitHub releases are hosted on Amazon S3, which doesn't support HEAD - "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" ] @skipIfNoNetwork() @@ -2089,6 +2086,29 @@ def test_shallow_mirrors(self): self.fetch_and_unpack() self.assertRevCount(1) + @skipIfNoNetwork() + def test_shallow_mirrors_with_multiple_same_urls(self): + url = "git://git.openembedded.org/bitbake;branch=master;protocol=https;rev=82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40" + + d = self.d.createCopy() + d.delVar('AUTOREV') + d.delVar('SRCREV') + + # prepare premirror + premirror = os.path.join(self.tempdir, "premirror") + os.mkdir(premirror) + d.setVar("DL_DIR", premirror) + fetcher = bb.fetch.Fetch([url], d) + fetcher.download() + + # set PREMIRRORS + d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % premirror) + + # set DL_DIR back and use the same url multiple times to fetch + d.setVar("DL_DIR", self.dldir) + fetcher2 = bb.fetch.Fetch([url, url], d) + fetcher2.download() + def test_shallow_invalid_depth(self): self.add_empty_file('a') self.add_empty_file('b') @@ -3462,6 +3482,7 @@ def setUp(self): self.reponame = "fstests" self.clonedir = os.path.join(self.tempdir, "git") self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) + self.giturl = "https://git.yoctoproject.org/fstests" self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https;branch=master" self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") self.d.setVar("BB_NO_NETWORK", "0") @@ -3470,7 +3491,7 @@ def setUp(self): def make_git_repo(self): self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" os.makedirs(self.clonedir) - self.git("clone --bare {}".format(self.recipe_url), self.clonedir) + self.git("clone --bare {}".format(self.giturl), self.clonedir) self.git("update-ref HEAD 15413486df1f5a5b5af699b6f3ba5f0984e52a9f", self.gitdir) bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) shutil.rmtree(self.clonedir) @@ -3772,3 +3793,562 @@ def test_gomodgit_url_host_only(self): self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') + + +class GitUnpackUpdateTest(FetcherTest): + """Test the unpack_update functionality for git fetcher. + + Intended workflow + 1. First-time setup: + 1. download() - clones the upstream repo into DL_DIR/git2/... (clonedir). + 2. unpack() - clones from clonedir into the workspace (S/workdir) and + registers a 'dldir' git remote pointing at + file://DL_DIR/git2/... for later offline use. + + 2. Subsequent updates (what unpack_update is designed for): + 1. The user works in the unpacked source tree. + 2. Upstream advances - SRCREV changes in the recipe. + 3. download() - fetches the new revision into the local clonedir. + 4. unpack_update() - instead of wiping the workspace and re-cloning: + * fetches the new revision from the local 'dldir' remote + * rebases the user's local commits on top of the new SRCREV + * raises LocalModificationsError if uncommitted changes block the + update, RebaseError if local commits cannot be rebased, or a + plain UnpackError for other failures (shallow clone, stale dldir); + in all cases the caller (e.g. bitbake-setup) can fall back to + backup + re-clone. + + Key design constraints: + * unpack_update() never deletes existing data (unlike unpack()). + * Only staged/modified tracked files block the update; untracked files and + committed local work are handled gracefully. + * The 'dldir' remote is intentionally visible to users outside the + fetcher (e.g. for manual 'git log dldir/master'). + * Currently only git is supported. + """ + + def setUp(self): + """Set up a local bare git source repository with two commits on 'master'. + + self.initial_rev - the first commit (testfile.txt: 'initial content') + self.updated_rev - the second commit (testfile.txt: 'updated content') + + SRCREV is initialised to self.initial_rev so individual tests can + advance it to self.updated_rev (or create further commits) as needed. + """ + FetcherTest.setUp(self) + + self.gitdir = os.path.join(self.tempdir, 'gitrepo') + self.srcdir = os.path.join(self.tempdir, 'gitsource') + + self.d.setVar('WORKDIR', self.tempdir) + self.d.setVar('S', self.gitdir) + self.d.delVar('PREMIRRORS') + self.d.delVar('MIRRORS') + + # Create a source git repository + bb.utils.mkdirhier(self.srcdir) + self.git_init(cwd=self.srcdir) + + # Create initial commit + with open(os.path.join(self.srcdir, 'testfile.txt'), 'w') as f: + f.write('initial content\n') + self.git(['add', 'testfile.txt'], cwd=self.srcdir) + self.git(['commit', '-m', 'Initial commit'], cwd=self.srcdir) + self.initial_rev = self.git(['rev-parse', 'HEAD'], cwd=self.srcdir).strip() + + # Create a second commit + with open(os.path.join(self.srcdir, 'testfile.txt'), 'w') as f: + f.write('updated content\n') + self.git(['add', 'testfile.txt'], cwd=self.srcdir) + self.git(['commit', '-m', 'Update commit'], cwd=self.srcdir) + self.updated_rev = self.git(['rev-parse', 'HEAD'], cwd=self.srcdir).strip() + + self.d.setVar('SRCREV', self.initial_rev) + self.d.setVar('SRC_URI', 'git://%s;branch=master;protocol=file' % self.srcdir) + + def test_unpack_update_full_clone(self): + """Test that unpack_update updates an existing checkout in place for a full clone. + + Steps: + 1. Fetch and unpack at self.initial_rev - verify 'initial content'. + 2. Advance SRCREV to self.updated_rev and re-download. + 3. Call unpack_update() instead of unpack() - the existing checkout + must be updated via 'git fetch dldir' + 'git rebase' without + re-cloning the directory. + 4. Verify testfile.txt now contains 'updated content'. + """ + # First fetch at initial revision + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + # Verify initial state + unpack_path = os.path.join(self.unpackdir, 'git') + self.assertTrue(os.path.exists(os.path.join(unpack_path, 'testfile.txt'))) + with open(os.path.join(unpack_path, 'testfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'initial content\n') + + # Update to new revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + # Use unpack_update + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + # Verify updated state + with open(os.path.join(unpack_path, 'testfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'updated content\n') + + def test_unpack_update_dldir_remote_setup(self): + """Test that unpack() adds a 'dldir' git remote pointing at ud.clonedir. + + The 'dldir' remote is used by subsequent unpack_update() calls to fetch + new commits from the local download cache (${DL_DIR}/git2/…) without + requiring network access. After a normal unpack the remote must exist + and its URL must be 'file://'. + """ + # First fetch + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Check that dldir remote exists + remotes = self.git(['remote'], cwd=unpack_path).strip().split('\n') + self.assertIn('dldir', remotes) + + # Verify it points to the clonedir + dldir_url = self.git(['remote', 'get-url', 'dldir'], cwd=unpack_path).strip() + self.assertEqual(dldir_url, 'file://{}'.format(ud.clonedir)) + + def test_unpack_update_ff_with_local_changes(self): + """Test that unpack_update rebases local commits fast forward. + + Full workflow: + 1. Fetch + unpack at initial_rev - verify 'dldir' remote is created + pointing at ud.clonedir. + 2. Add a local commit touching localfile.txt. + 3. Advance SRCREV to updated_rev and call download() - verify that + ud.clonedir (the dldir bare clone) now contains updated_rev. + 4. Call unpack_update() - it fetches updated_rev from dldir into the + working tree and rebases the local commit on top. + 5. Verify the final commit graph: HEAD's parent is updated_rev, and + both testfile.txt ('updated content') and localfile.txt ('local + change') are present. + + Note: git rebase operates the same way regardless of whether HEAD is + detached or on a named branch (e.g. 'master' or a local feature branch), + so this test covers those scenarios implicitly. + """ + # Step 1 - fetch + unpack at initial_rev + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + unpack_path = os.path.join(self.unpackdir, 'git') + + # The normal unpack must have set up the 'dldir' remote pointing at + # ud.clonedir so that subsequent unpack_update() calls work offline. + dldir_url = self.git(['remote', 'get-url', 'dldir'], cwd=unpack_path).strip() + self.assertEqual(dldir_url, 'file://{}'.format(ud.clonedir)) + + # Step 2 - add a local commit that touches a new file + with open(os.path.join(unpack_path, 'localfile.txt'), 'w') as f: + f.write('local change\n') + self.git(['add', 'localfile.txt'], cwd=unpack_path) + self.git(['commit', '-m', 'Local commit'], cwd=unpack_path) + local_commit = self.git(['rev-parse', 'HEAD'], cwd=unpack_path).strip() + + # Step 3 - advance SRCREV and download; clonedir must now contain + # updated_rev so that unpack_update can fetch it without network access. + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + ud = fetcher.ud[uri] + clonedir_refs = self.git(['rev-parse', self.updated_rev], cwd=ud.clonedir).strip() + self.assertEqual(clonedir_refs, self.updated_rev, + "clonedir must contain updated_rev after download()") + + # Step 4 - unpack_update fetches from dldir and rebases + git_fetcher = ud.method + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + # Step 5 - verify the commit graph and working tree content + # HEAD is the rebased local commit; its parent must be updated_rev + head_rev = self.git(['rev-parse', 'HEAD'], cwd=unpack_path).strip() + parent_rev = self.git(['rev-parse', 'HEAD^'], cwd=unpack_path).strip() + self.assertNotEqual(head_rev, local_commit, + "local commit should have a new SHA after rebase") + self.assertEqual(parent_rev, self.updated_rev, + "HEAD's parent must be updated_rev after fast-forward rebase") + + with open(os.path.join(unpack_path, 'testfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'updated content\n') + with open(os.path.join(unpack_path, 'localfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'local change\n') + + def test_unpack_update_already_at_target_revision(self): + """Test that unpack_update is a no-op when the checkout is already at SRCREV. + + Calling unpack_update() without advancing SRCREV must succeed and leave + the working tree unchanged. No rebase should be attempted because the + checkout already points at ud.revision. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + with open(os.path.join(unpack_path, 'testfile.txt')) as f: + self.assertEqual(f.read(), 'initial content\n') + + # Call unpack_update with SRCREV still at initial_rev - no upstream change + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + result = git_fetcher.unpack_update(ud, self.unpackdir, self.d) + self.assertTrue(result) + + # Content must be unchanged + with open(os.path.join(unpack_path, 'testfile.txt')) as f: + self.assertEqual(f.read(), 'initial content\n') + + def test_unpack_update_with_untracked_file(self): + """Test that unpack_update succeeds when the checkout has an untracked file. + + The status check uses '--untracked-files=no', so untracked files are not + detected and do not trigger the fallback path. git rebase also leaves + untracked files untouched, so both the upstream update and the untracked + file must be present after the call. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Create an untracked file (not staged, not committed) + untracked = os.path.join(unpack_path, 'untracked.txt') + with open(untracked, 'w') as f: + f.write('untracked content\n') + + # Update to new upstream revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # --untracked-files=no means the status check passes; rebase preserves the file + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + with open(os.path.join(unpack_path, 'testfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'updated content\n') + + # Untracked file must survive the rebase + self.assertTrue(os.path.exists(untracked)) + with open(untracked, 'r') as f: + self.assertEqual(f.read(), 'untracked content\n') + + def test_unpack_update_with_staged_changes(self): + """Test that unpack_update fails when the checkout has staged (but not committed) changes. + + The rebase is run with --no-autostash so git refuses to rebase over a + dirty index. The caller (bitbake-setup) is expected to catch the + resulting LocalModificationsError and fall back to backup + re-fetch. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Stage a new file without committing it + staged = os.path.join(unpack_path, 'staged.txt') + with open(staged, 'w') as f: + f.write('staged content\n') + self.git(['add', 'staged.txt'], cwd=unpack_path) + + # Update to new upstream revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # Should fail - git rebase refuses to run with a dirty index + with self.assertRaises(bb.fetch2.LocalModificationsError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + def test_unpack_update_with_modified_tracked_file(self): + """Test that unpack_update fails when a tracked file has unstaged modifications. + + 'git status --untracked-files=no --porcelain' reports unstaged modifications + to tracked files (output line ' M filename'), which must block the update so + the caller can fall back to backup + re-fetch rather than silently discarding + work in progress. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Modify a tracked file without staging or committing + with open(os.path.join(unpack_path, 'testfile.txt'), 'w') as f: + f.write('locally modified content\n') + + # Update to new upstream revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # Should fail - unstaged modification to tracked file is detected by + # 'git status --untracked-files=no --porcelain' + with self.assertRaises(bb.fetch2.LocalModificationsError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + def test_unpack_update_conflict_raises_rebase_error(self): + """Test that unpack_update raises RebaseError on a rebase conflict. + + When a local commit modifies the same lines as an incoming upstream commit, + git rebase cannot resolve the conflict automatically. unpack_update must + abort the failed rebase and raise RebaseError so the caller can fall back + to a backup + re-fetch. + """ + # Fetch and unpack at the initial revision + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Make a local commit that edits the same lines as the upcoming upstream commit + with open(os.path.join(unpack_path, 'testfile.txt'), 'w') as f: + f.write('conflicting local content\n') + self.git(['add', 'testfile.txt'], cwd=unpack_path) + self.git(['commit', '-m', 'Local conflicting commit'], cwd=unpack_path) + + # Add a third upstream commit that also edits testfile.txt differently + with open(os.path.join(self.srcdir, 'testfile.txt'), 'w') as f: + f.write('conflicting upstream content\n') + self.git(['add', 'testfile.txt'], cwd=self.srcdir) + self.git(['commit', '-m', 'Upstream conflicting commit'], cwd=self.srcdir) + conflict_rev = self.git(['rev-parse', 'HEAD'], cwd=self.srcdir).strip() + + # Update SRCREV to the new upstream commit + self.d.setVar('SRCREV', conflict_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # unpack_update must fail and clean up (rebase --abort) rather than + # leaving the repo in a mid-rebase state + with self.assertRaises(bb.fetch2.RebaseError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + # Verify the repo is not left in a conflicted / mid-rebase state + rebase_merge = os.path.join(unpack_path, '.git', 'rebase-merge') + rebase_apply = os.path.join(unpack_path, '.git', 'rebase-apply') + self.assertFalse(os.path.exists(rebase_merge), + "rebase-merge dir should not exist after failed unpack_update") + self.assertFalse(os.path.exists(rebase_apply), + "rebase-apply dir should not exist after failed unpack_update") + + def test_unpack_update_untracked_file_overwritten_by_upstream(self): + """Test that unpack_update raises RebaseError when an untracked file would be + overwritten by an incoming upstream commit. + + We skip untracked files in the pre-check (git rebase doesn't touch harmless + untracked files), but git itself refuses to rebase when an untracked file would + be overwritten by the incoming changes. The resulting FetchError must be caught + and re-raised as RebaseError without leaving the repo in a mid-rebase state. + + Two sub-cases are covered: + - top-level untracked file clashing with an incoming upstream file + - untracked file inside a subdirectory (xxx/somefile) clashing with an + upstream commit that adds the same path + """ + def _run_case(upstream_path, local_rel_path, commit_msg): + """ + Add upstream_path to self.srcdir, create local_rel_path as an + untracked file in the checkout, then assert that unpack_update + raises RebaseError and leaves no mid-rebase state, and that the + local file is untouched. + """ + # Fresh fetch + unpack at the current SRCREV + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Upstream adds the file (potentially inside a subdirectory) + full_upstream = os.path.join(self.srcdir, upstream_path) + os.makedirs(os.path.dirname(full_upstream), exist_ok=True) + with open(full_upstream, 'w') as f: + f.write('upstream content\n') + self.git(['add', upstream_path], cwd=self.srcdir) + self.git(['commit', '-m', commit_msg], cwd=self.srcdir) + new_rev = self.git(['rev-parse', 'HEAD'], cwd=self.srcdir).strip() + + # Create the clashing untracked file in the checkout + full_local = os.path.join(unpack_path, local_rel_path) + os.makedirs(os.path.dirname(full_local), exist_ok=True) + with open(full_local, 'w') as f: + f.write('local untracked content\n') + + self.d.setVar('SRCREV', new_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + # git rebase refuses because the untracked file would be overwritten + with self.assertRaises(bb.fetch2.RebaseError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + # Repo must not be left in a mid-rebase state + self.assertFalse(os.path.exists(os.path.join(unpack_path, '.git', 'rebase-merge'))) + self.assertFalse(os.path.exists(os.path.join(unpack_path, '.git', 'rebase-apply'))) + + # The local untracked file must be untouched + self.assertTrue(os.path.exists(full_local)) + with open(full_local) as f: + self.assertEqual(f.read(), 'local untracked content\n') + + # Reset unpackdir for the next sub-case + import shutil as _shutil + _shutil.rmtree(self.unpackdir) + os.makedirs(self.unpackdir) + + # Sub-case 1: top-level file clash + _run_case('newfile.txt', 'newfile.txt', + 'Upstream adds newfile.txt') + + # Sub-case 2: file inside a subdirectory (xxx/somefile) + _run_case('xxx/somefile.txt', 'xxx/somefile.txt', + 'Upstream adds xxx/somefile.txt') + + def test_unpack_update_shallow_clone_fails(self): + """Test that unpack_update raises UnpackError for shallow-tarball checkouts. + + Shallow clones lack full history, which makes an in-place rebase impossible + without network access. After fetching with BB_GIT_SHALLOW=1 the clonedir + is deleted so that unpack() is forced to use the shallow tarball. + A subsequent call to unpack_update() must raise UnpackError and the message + must mention 'shallow clone' so callers can distinguish this case. + """ + self.d.setVar('BB_GIT_SHALLOW', '1') + self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') + + # First fetch at initial revision + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + # Remove clonedir to force use of shallow tarball + clonedir = os.path.join(self.dldir, 'git2') + if os.path.exists(clonedir): + shutil.rmtree(clonedir) + + fetcher.unpack(self.unpackdir) + + # Update to new revision + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + # unpack_update should fail for shallow clones + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + git_fetcher = ud.method + + with self.assertRaises(bb.fetch2.UnpackError) as context: + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + self.assertIn("shallow clone", str(context.exception).lower()) + + def test_unpack_update_stale_dldir_remote(self): + """Test that unpack_update raises UnpackError when the dldir remote URL is stale. + + If the clonedir has been removed after the initial unpack (e.g. DL_DIR was + cleaned) the 'dldir' remote URL no longer resolves. The fetch inside + update_mode will fail with a FetchError which must be re-raised as + UnpackError so the caller can fall back to a full re-fetch. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + + # Advance SRCREV to trigger update_mode + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + uri = self.d.getVar('SRC_URI') + ud = fetcher.ud[uri] + + # Delete the clonedir and corrupt the dldir remote URL so that + # 'git fetch dldir' fails, simulating a missing or relocated DL_DIR. + shutil.rmtree(ud.clonedir) + self.git(['remote', 'set-url', 'dldir', 'file://' + ud.clonedir], + cwd=unpack_path) + + git_fetcher = ud.method + with self.assertRaises(bb.fetch2.UnpackError): + git_fetcher.unpack_update(ud, self.unpackdir, self.d) + + def test_fetch_unpack_update_toplevel_api(self): + """Test that the top-level Fetch.unpack_update() dispatches to Git.unpack_update(). + + Callers such as bitbake-setup use fetcher.unpack_update(root) rather than + calling the method on the Git fetcher directly. Verify that the public API + works end-to-end: fetch at initial_rev, unpack, advance to updated_rev, + fetch again, then call fetcher.unpack_update(root) and confirm the content + is updated. + """ + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + + unpack_path = os.path.join(self.unpackdir, 'git') + with open(os.path.join(unpack_path, 'testfile.txt')) as f: + self.assertEqual(f.read(), 'initial content\n') + + self.d.setVar('SRCREV', self.updated_rev) + fetcher = bb.fetch2.Fetch([self.d.getVar('SRC_URI')], self.d) + fetcher.download() + + # Use the public Fetch.unpack_update() rather than the method directly + fetcher.unpack_update(self.unpackdir) + + with open(os.path.join(unpack_path, 'testfile.txt')) as f: + self.assertEqual(f.read(), 'updated content\n') diff --git a/lib/bb/tests/parse-tests/classes/base.bbclass b/lib/bb/tests/parse-tests/classes/base.bbclass new file mode 100644 index 00000000000..db8898e12f2 --- /dev/null +++ b/lib/bb/tests/parse-tests/classes/base.bbclass @@ -0,0 +1,5 @@ +# At least one task is required for bitbake to parse +do_fetch() { + : +} +addtask do_fetch diff --git a/lib/bb/tests/parse-tests/classes/recipe-file-class.bbclass b/lib/bb/tests/parse-tests/classes/recipe-file-class.bbclass new file mode 100644 index 00000000000..4682dc6c3d8 --- /dev/null +++ b/lib/bb/tests/parse-tests/classes/recipe-file-class.bbclass @@ -0,0 +1,2 @@ +BBCLASS_RECIPE_FILE := "${@os.path.basename(d.getVar('__BB_RECIPE_FILE'))}" +BBCLASS_FILE := "${@os.path.basename(d.getVar('FILE'))}" diff --git a/lib/bb/tests/parse-tests/conf/bitbake.conf b/lib/bb/tests/parse-tests/conf/bitbake.conf new file mode 100644 index 00000000000..e03625061bb --- /dev/null +++ b/lib/bb/tests/parse-tests/conf/bitbake.conf @@ -0,0 +1,15 @@ +CACHE = "${TOPDIR}/cache" +THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" +COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}" +EXTRA_BBFILES ?= "" +BBFILES = "${COREBASE}/recipes/*.bb ${COREBASE}/recipes/*.bbappend ${EXTRA_BBFILES}" +PROVIDES = "${PN}" +PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" +PF = "${BB_CURRENT_MC}:${PN}" +export PATH +TMPDIR ??= "${TOPDIR}" +STAMP = "${TMPDIR}/stamps/${PN}" +T = "${TMPDIR}/workdir/${PN}/temp" +BB_NUMBER_THREADS = "4" + +BB_BASEHASH_IGNORE_VARS = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK" diff --git a/lib/bb/tests/parse.py b/lib/bb/tests/parse.py index d3867ece988..6ac2137e07a 100644 --- a/lib/bb/tests/parse.py +++ b/lib/bb/tests/parse.py @@ -11,6 +11,8 @@ import logging import bb import os +import subprocess +import textwrap logger = logging.getLogger('BitBake.TestParse') @@ -210,7 +212,7 @@ def test_parse_combinations(self): # # Test based upon a real world data corruption issue. One # data store changing a variable poked through into a different data - # store. This test case replicates that issue where the value 'B' would + # store. This test case replicates that issue where the value 'B' would # become unset/disappear. # def test_parse_classextend_contamination(self): @@ -508,3 +510,131 @@ def test_helper(content, result): test_helper("require some3.conf", " foobar") test_helper("include_all some.conf", " bar foo") test_helper("include_all some3.conf", " foobar") + + def test_file_variables(self): + # Tests the values of FILE and __BB_RECIPE_FILE in different + # combinations of bbappends, includes, and inherits + + def write_file(path, data): + with open(path, "w") as f: + f.write(textwrap.dedent(data)) + + def run_bitbake(cmd, builddir, extraenv={}): + env = os.environ.copy() + env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "parse-tests")) + env["BB_ENV_PASSTHROUGH_ADDITIONS"] = "TOPDIR" + env["TOPDIR"] = builddir + for k, v in extraenv.items(): + env[k] = v + env["BB_ENV_PASSTHROUGH_ADDITIONS"] = env["BB_ENV_PASSTHROUGH_ADDITIONS"] + " " + k + try: + return subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT, universal_newlines=True, cwd=builddir) + except subprocess.CalledProcessError as e: + self.fail("Command %s failed with %s" % (cmd, e.output)) + + with tempfile.TemporaryDirectory(prefix="parserecipes") as recipes, tempfile.TemporaryDirectory(prefix="parsetest") as builddir: + extraenv = { + "EXTRA_BBFILES": f"{recipes}/*.bb {recipes}/*.bbappend", + } + + inc_path = f"{recipes}/recipe-file.inc" + bbappend_path = f"{recipes}/recipe-%.bbappend" + recipe_path = f"{recipes}/recipe-file1.bb" + + # __BB_RECIPE_FILE should always be the name of .bb file, even + # when set in a bbappend. FILE is the name of the bbappend + write_file(recipe_path, "") + write_file(bbappend_path, + """\ + BBAPPEND_RECIPE_FILE := "${@os.path.basename(d.getVar('__BB_RECIPE_FILE'))}" + BBAPPEND_FILE := "${@os.path.basename(d.getVar('FILE'))}" + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBAPPEND_FILE="recipe-%.bbappend"', output) + self.assertIn(f'BBAPPEND_RECIPE_FILE="recipe-file1.bb"', output) + + # __BB_RECIPE_FILE should always be the name of .bb file, even when + # set in an include file. FILE is the name of the include + write_file(recipe_path, + """\ + require recipe-file.inc + """ + ) + write_file(inc_path, + """\ + INC_RECIPE_FILE := "${@os.path.basename(d.getVar('__BB_RECIPE_FILE'))}" + INC_FILE := "${@os.path.basename(d.getVar('FILE'))}" + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('INC_FILE="recipe-file.inc"', output) + self.assertIn(f'INC_RECIPE_FILE="recipe-file1.bb"', output) + + # Test when the include file is included from a bbappend + write_file(recipe_path, "") + write_file(bbappend_path, + """\ + require recipe-file.inc + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('INC_FILE="recipe-file.inc"', output) + self.assertIn(f'INC_RECIPE_FILE="recipe-file1.bb"', output) + + # Test the variables in a bbclass when inherited directly in the + # recipe. Note that FILE still refers to the recipe in a bbclass + write_file(recipe_path, + """\ + inherit recipe-file-class + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBCLASS_FILE="recipe-file1.bb"', output) + self.assertIn(f'BBCLASS_RECIPE_FILE="recipe-file1.bb"', output) + + # Test the variables when the inherit is in a bbappend. In this + # case, FILE is the bbappend + write_file(recipe_path, "") + write_file(bbappend_path, + """\ + inherit recipe-file-class + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBCLASS_FILE="recipe-%.bbappend"', output) + self.assertIn(f'BBCLASS_RECIPE_FILE="recipe-file1.bb"', output) + + # Test the variables when the inherit is in a include. In this + # case, FILE is the include file + write_file(recipe_path, + """\ + require recipe-file.inc + """ + ) + write_file(bbappend_path, "") + write_file(inc_path, + """\ + inherit recipe-file-class + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBCLASS_FILE="recipe-file.inc"', output) + self.assertIn(f'BBCLASS_RECIPE_FILE="recipe-file1.bb"', output) + + # Test the variables when the inherit is in a include included from + # a bbappend. In this case, FILE is the include file + write_file(recipe_path, "") + write_file(bbappend_path, + """\ + require recipe-file.inc + """ + ) + write_file(inc_path, + """\ + inherit recipe-file-class + """ + ) + output = run_bitbake(["bitbake", "-e", "recipe-file1"], builddir, extraenv).splitlines() + self.assertIn('BBCLASS_FILE="recipe-file.inc"', output) + self.assertIn(f'BBCLASS_RECIPE_FILE="recipe-file1.bb"', output) diff --git a/lib/bb/tests/setup.py b/lib/bb/tests/setup.py index e3557d8f3b9..638d56d3bb3 100644 --- a/lib/bb/tests/setup.py +++ b/lib/bb/tests/setup.py @@ -5,9 +5,13 @@ # from bb.tests.fetch import FetcherTest -import json -import hashlib +import bb +import bb.process import glob +import hashlib +import json +import os +import stat from bb.tests.support.httpserver import HTTPService class BitbakeSetupTest(FetcherTest): @@ -208,7 +212,6 @@ def add_file_to_testrepo(self, name, content, script=False): with open(fullname, 'w') as f: f.write(content) if script: - import stat st = os.stat(fullname) os.chmod(fullname, st.st_mode | stat.S_IEXEC) self.git('add {}'.format(name), cwd=self.testrepopath) @@ -257,11 +260,11 @@ def check_setupdir_files(self, setuppath, test_file_content): ) self.assertIn(filerelative_layer, bblayers) - if 'oe-fragment' in bitbake_config.keys(): + if 'oe-fragments' in bitbake_config: for f in bitbake_config["oe-fragments"]: self.assertTrue(os.path.exists(os.path.join(bb_conf_path, f))) - if 'bb-environment-passthrough' in bitbake_config.keys(): + if 'bb-env-passthrough-additions' in bitbake_config: with open(os.path.join(bb_build_path, 'init-build-env'), 'r') as f: init_build_env = f.read() self.assertTrue('BB_ENV_PASSTHROUGH_ADDITIONS' in init_build_env) @@ -279,7 +282,6 @@ def get_setup_path(self, cf, c): def test_setup(self): # unset BBPATH to ensure tests run in isolation from the existing bitbake environment - import os if 'BBPATH' in os.environ: del os.environ['BBPATH'] @@ -447,16 +449,27 @@ def _check_local_sources(custom_setup_dir): self.assertEqual(self.testrepopath, os.path.realpath(custom_layer_path)) self.config_is_unchanged(custom_setup_path) + def _check_layer_backups(layer_path, expected_backups): + files = os.listdir(layer_path) + backups = len([f for f in files if 'backup' in f]) + self.assertEqual(backups, expected_backups, msg = "Expected {} layer backups, got {}, directory listing: {}".format(expected_backups, backups, files)) + # Change the configuration to refer to a local source, then to another local source, then back to a git remote # Run status/update after each change and verify that nothing breaks + # Also check that layer backups happen when expected c = 'gadget' setuppath = self.get_setup_path('test-config-1', c) self.config_is_unchanged(setuppath) + layers_path = os.path.join(setuppath, 'layers') + layer_path = os.path.join(layers_path, 'test-repo') + _check_layer_backups(layers_path, 0) + json_1 = self.add_local_json_config_to_registry('test-config-1.conf.json', self.testrepopath) os.environ['BBPATH'] = os.path.join(setuppath, 'build') out = self.runbbsetup("update --update-bb-conf='yes'") _check_local_sources(setuppath) + _check_layer_backups(layers_path, 1) prev_path = self.testrepopath self.testrepopath = prev_path + "-2" @@ -465,23 +478,14 @@ def _check_local_sources(custom_setup_dir): os.environ['BBPATH'] = os.path.join(setuppath, 'build') out = self.runbbsetup("update --update-bb-conf='yes'") _check_local_sources(setuppath) + _check_layer_backups(layers_path, 1) self.testrepopath = prev_path json_1 = self.add_json_config_to_registry('test-config-1.conf.json', branch, branch) os.environ['BBPATH'] = os.path.join(setuppath, 'build') out = self.runbbsetup("update --update-bb-conf='yes'") self.check_setupdir_files(setuppath, test_file_content) - - # Also check that there are no layer backups up to this point, then make a change that should - # result in a layer backup, and check that it does happen. - def _check_layer_backups(layer_path, expected_backups): - files = os.listdir(layer_path) - backups = len([f for f in files if 'backup' in f]) - self.assertEqual(backups, expected_backups, msg = "Expected {} layer backups, got {}, directory listing: {}".format(expected_backups, backups, files)) - - layers_path = os.path.join(setuppath, 'layers') - layer_path = os.path.join(layers_path, 'test-repo') - _check_layer_backups(layers_path, 0) + _check_layer_backups(layers_path, 1) ## edit a file without making a commit with open(os.path.join(layer_path, 'local-modification'), 'w') as f: @@ -492,16 +496,26 @@ def _check_layer_backups(layer_path, expected_backups): out = self.runbbsetup("update --update-bb-conf='yes'") _check_layer_backups(layers_path, 1) - ## edit a file and try to make a commit; this should be rejected + ## edit a file and make a commit such that no rebase conflicts occur with open(os.path.join(layer_path, 'local-modification'), 'w') as f: f.write('locally-modified-again\n') self.git('add .', cwd=layer_path) - with self.assertRaisesRegex(bb.process.ExecutionError, "making commits is restricted"): - self.git('commit -m "Adding a local modification"', cwd=layer_path) + self.git('commit -m "Adding a local modification"', cwd=layer_path) test_file_content = "modified-again-and-again\n" self.add_file_to_testrepo('test-file', test_file_content) out = self.runbbsetup("update --update-bb-conf='yes'") - _check_layer_backups(layers_path, 2) + _check_layer_backups(layers_path, 1) + + ## edit a file and make a commit in a way that causes a rebase conflict + with open(os.path.join(layer_path, 'test-file'), 'w') as f: + f.write('locally-modified\n') + self.git('add .', cwd=layer_path) + self.git('commit -m "Adding a local modification"', cwd=layer_path) + test_file_content = "remotely-modified\n" + self.add_file_to_testrepo('test-file', test_file_content) + with self.assertRaisesRegex(bb.process.ExecutionError, "Merge conflict in test-file"): + out = self.runbbsetup("update --update-bb-conf='yes'") + _check_layer_backups(layers_path, 1) # check source overrides, local sources provided with symlinks, and custom setup dir name source_override_content = """ @@ -524,3 +538,191 @@ def _check_layer_backups(layer_path, expected_backups): custom_setup_dir = 'special-setup-dir-with-cmdline-overrides' out = self.runbbsetup("init --non-interactive -L test-repo {} --setup-dir-name {} test-config-1 gadget".format(self.testrepopath, custom_setup_dir)) _check_local_sources(custom_setup_dir) + + def test_vscode(self): + if 'BBPATH' in os.environ: + del os.environ['BBPATH'] + os.chdir(self.tempdir) + + self.runbbsetup("settings set default registry 'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath)) + self.add_file_to_testrepo('test-file', 'initial\n') + self.add_json_config_to_registry('test-config-1.conf.json', 'master', 'master') + + # --init-vscode should create bitbake.code-workspace + self.runbbsetup("init --non-interactive --init-vscode test-config-1 gadget") + setuppath = self.get_setup_path('test-config-1', 'gadget') + workspace_file = os.path.join(setuppath, 'bitbake.code-workspace') + self.assertTrue(os.path.exists(workspace_file), + "bitbake.code-workspace should be created with --init-vscode") + + with open(workspace_file) as f: + workspace = json.load(f) + + # top-level structure + self.assertIn('folders', workspace) + self.assertIn('settings', workspace) + self.assertIn('extensions', workspace) + self.assertIn('yocto-project.yocto-bitbake', + workspace['extensions']['recommendations']) + + # folders: conf dir + test-repo (symlinks like oe-init-build-env-dir are skipped) + folder_names = {f['name'] for f in workspace['folders']} + self.assertIn('conf', folder_names) + self.assertIn('test-repo', folder_names) + + # folder paths must be relative so the workspace is portable + for f in workspace['folders']: + self.assertFalse(os.path.isabs(f['path']), + "Folder path should be relative, got: {}".format(f['path'])) + + # bitbake extension settings + settings = workspace['settings'] + self.assertTrue(settings.get('bitbake.disableConfigModification')) + self.assertEqual(settings['bitbake.pathToBuildFolder'], + os.path.join(setuppath, 'build')) + self.assertEqual(settings['bitbake.pathToEnvScript'], + os.path.join(setuppath, 'build', 'init-build-env')) + + # file associations + self.assertIn('*.conf', settings.get('files.associations', {})) + self.assertIn('*.inc', settings.get('files.associations', {})) + + # python extra paths: test-repo/scripts/ exists and should be listed + extra_paths = settings.get('python.analysis.extraPaths', []) + self.assertTrue(any('scripts' in p for p in extra_paths), + "python.analysis.extraPaths should include the scripts dir") + self.assertEqual(settings.get('python.analysis.extraPaths'), + settings.get('python.autoComplete.extraPaths')) + + # --no-init-vscode should NOT create a workspace file + self.runbbsetup("init --non-interactive --no-init-vscode test-config-1 gadget-notemplate") + notemplate_path = self.get_setup_path('test-config-1', 'gadget-notemplate') + self.assertFalse( + os.path.exists(os.path.join(notemplate_path, 'bitbake.code-workspace')), + "bitbake.code-workspace should not be created with --no-init-vscode") + + # update with --init-vscode after a layer change should preserve + # user-added folders and settings while still rewriting managed ones + workspace['folders'].append({"name": "user-folder", "path": "user/custom"}) + workspace['settings']['my.user.setting'] = 'preserved' + with open(workspace_file, 'w') as f: + json.dump(workspace, f, indent=4) + + self.add_file_to_testrepo('test-file', 'updated\n') + os.environ['BBPATH'] = os.path.join(setuppath, 'build') + self.runbbsetup("update --update-bb-conf='no'") + del os.environ['BBPATH'] + + with open(workspace_file) as f: + updated = json.load(f) + self.assertIn('user/custom', {f['path'] for f in updated['folders']}, + "User-added folder was removed during update") + self.assertIn('my.user.setting', updated['settings'], + "User-added setting was removed during update") + + # update with a corrupt workspace file should log an error and leave it unchanged + self.add_file_to_testrepo('test-file', 'updated-again\n') + with open(workspace_file, 'w') as f: + f.write('{invalid json') + os.environ['BBPATH'] = os.path.join(setuppath, 'build') + self.runbbsetup("update --update-bb-conf='no'") + del os.environ['BBPATH'] + with open(workspace_file) as f: + content = f.read() + self.assertEqual(content, '{invalid json', + "Corrupt workspace file should not be modified") + + def _count_layer_backups(self, layers_path): + return len([f for f in os.listdir(layers_path) if 'backup' in f]) + + def test_update_rebase_conflicts_strategy(self): + """Test the --rebase-conflicts-strategy option for the update command. + + Covers three scenarios not exercised by test_setup: + 1. Uncommitted tracked-file change (LocalModificationsError) + default 'abort' + strategy → clean error message containing 'has uncommitted changes' and a + hint at --rebase-conflicts-strategy=backup; no backup directory is created. + 2. Same uncommitted change + 'backup' strategy → directory is renamed to a + timestamped backup and the layer is re-cloned cleanly. + 3. Committed local change that conflicts with an incoming upstream commit + (RebaseError): + a. Default 'abort' strategy → error containing 'Merge conflict' and the + --rebase-conflicts-strategy=backup hint; no backup directory is created. + b. 'backup' strategy → backup + re-clone instead of a hard failure. + """ + if 'BBPATH' in os.environ: + del os.environ['BBPATH'] + os.chdir(self.tempdir) + + self.runbbsetup("settings set default registry 'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath)) + self.add_file_to_testrepo('test-file', 'initial\n') + self.add_json_config_to_registry('test-config-1.conf.json', 'master', 'master') + self.runbbsetup("init --non-interactive test-config-1 gadget") + + setuppath = self.get_setup_path('test-config-1', 'gadget') + layer_path = os.path.join(setuppath, 'layers', 'test-repo') + layers_path = os.path.join(setuppath, 'layers') + + # Scenario 1: uncommitted tracked change, default 'abort' strategy + # Advance upstream so an update is required. + self.add_file_to_testrepo('test-file', 'upstream-v2\n') + # Modify the same tracked file in the layer without committing. + with open(os.path.join(layer_path, 'test-file'), 'w') as f: + f.write('locally-modified\n') + + os.environ['BBPATH'] = os.path.join(setuppath, 'build') + with self.assertRaises(bb.process.ExecutionError) as ctx: + self.runbbsetup("update --update-bb-conf='no'") + self.assertIn('has uncommitted changes', str(ctx.exception)) + self.assertIn('--rebase-conflicts-strategy=backup', str(ctx.exception)) + # No backup directory must have been created. + self.assertEqual(self._count_layer_backups(layers_path), 0, + "abort strategy must not create any backup") + + # Scenario 2: same uncommitted change, 'backup' strategy + out = self.runbbsetup("update --update-bb-conf='no' --rebase-conflicts-strategy=backup") + # One backup directory must now exist. + self.assertEqual(self._count_layer_backups(layers_path), 1, + "backup strategy must create exactly one backup") + # The re-cloned layer must be clean and at the upstream revision. + with open(os.path.join(layer_path, 'test-file')) as f: + self.assertEqual(f.read(), 'upstream-v2\n', + "re-cloned layer must contain the upstream content") + status = self.git('status --porcelain', cwd=layer_path).strip() + self.assertEqual(status, '', + "re-cloned layer must have no local modifications") + del os.environ['BBPATH'] + + # Scenario 3: committed conflicting change, 'backup' strategy + # Re-initialise a fresh setup so we start from a clean state. + self.runbbsetup("init --non-interactive --setup-dir-name rebase-conflict-setup test-config-1 gadget") + conflict_setup = os.path.join(self.tempdir, 'bitbake-builds', 'rebase-conflict-setup') + conflict_layer = os.path.join(conflict_setup, 'layers', 'test-repo') + conflict_layers = os.path.join(conflict_setup, 'layers') + + # Commit a local change that touches the same file as the next upstream commit. + with open(os.path.join(conflict_layer, 'test-file'), 'w') as f: + f.write('conflicting-local\n') + self.git('add test-file', cwd=conflict_layer) + self.git('commit -m "Local conflicting change"', cwd=conflict_layer) + + # Advance upstream with a conflicting edit. + self.add_file_to_testrepo('test-file', 'conflicting-upstream\n') + + os.environ['BBPATH'] = os.path.join(conflict_setup, 'build') + # Default stop strategy must still fail with a conflict error and include + # the --rebase-conflicts-strategy=backup hint (same handler as LocalModificationsError). + with self.assertRaises(bb.process.ExecutionError) as ctx: + self.runbbsetup("update --update-bb-conf='no'") + self.assertIn('Merge conflict in test-file', str(ctx.exception)) + self.assertIn('--rebase-conflicts-strategy=backup', str(ctx.exception)) + self.assertEqual(self._count_layer_backups(conflict_layers), 0) + + # Backup strategy must succeed: backup the conflicted dir and re-clone. + self.runbbsetup("update --update-bb-conf='no' --rebase-conflicts-strategy=backup") + self.assertEqual(self._count_layer_backups(conflict_layers), 1, + "backup strategy must create exactly one backup after a conflict") + with open(os.path.join(conflict_layer, 'test-file')) as f: + self.assertEqual(f.read(), 'conflicting-upstream\n', + "re-cloned layer must contain the upstream content after conflict backup") + del os.environ['BBPATH'] diff --git a/lib/bb/tinfoil.py b/lib/bb/tinfoil.py index d9e985c612c..9c1768942ab 100644 --- a/lib/bb/tinfoil.py +++ b/lib/bb/tinfoil.py @@ -685,7 +685,14 @@ def get_recipe_file(self, pn): if skipreasons: raise bb.providers.NoProvider('%s is unavailable:\n %s' % (pn, ' \n'.join(skipreasons))) else: - raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) + msg = f'Unable to find any recipe file matching "{pn}"' + import difflib + providers = self.get_all_providers() + close_matches = difflib.get_close_matches(pn, providers, cutoff=0.7) + if close_matches: + close_matches = "\n ".join(close_matches) + msg += f'. Close matches:\n {close_matches}' + raise bb.providers.NoProvider(msg) return best[3] def get_file_appends(self, fn, mc=''): diff --git a/lib/bb/ui/knotty.py b/lib/bb/ui/knotty.py index e4b7b83061e..82531ef8f31 100644 --- a/lib/bb/ui/knotty.py +++ b/lib/bb/ui/knotty.py @@ -87,6 +87,7 @@ class NonInteractiveProgress(object): def __init__(self, msg, maxval): self.id = msg self.msg = msg + self.currval = 0 self.maxval = maxval self.finished = False @@ -96,6 +97,10 @@ def start(self, update=True): return self def update(self, value): + self.currval = value + pass + + def clear(self): pass def finish(self): @@ -791,7 +796,12 @@ def main(server, eventHandler, params, tf = TerminalFilter): event.msg = taskinfo['title'] + ': ' + event.msg if hasattr(event, 'fn') and event.levelno not in [bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]: event.msg = event.fn + ': ' + event.msg + # Need to remove any progress bar, then add it back after we print this message + if parseprogress: + parseprogress.clear() logging.getLogger(event.name).handle(event) + if parseprogress: + parseprogress.update(parseprogress.currval) continue if isinstance(event, bb.build.TaskFailedSilent): diff --git a/lib/bb/ui/taskexp_ncurses.py b/lib/bb/ui/taskexp_ncurses.py index ea94a4987f9..551ad142a2a 100755 --- a/lib/bb/ui/taskexp_ncurses.py +++ b/lib/bb/ui/taskexp_ncurses.py @@ -146,6 +146,9 @@ def debug_frame(nbox_ojb): unit_test = os.environ.get('TASK_EXP_UNIT_TEST') unit_test_cmnds=[ '# Default selected task in primary box', + 'tst_selected=.do_create_recipe_spdx', + '# Move to next entry, more predictable', + 'tst_entry=', # optional injected error 'tst_selected=.do_recipe_qa', '# Default selected task in deps', 'tst_entry=', diff --git a/lib/bb/utils.py b/lib/bb/utils.py index 974611bc7e0..b04ff6ffc76 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -694,14 +694,19 @@ def goh1_file(filename): import zipfile lines = [] + is_zipfile = False if zipfile.is_zipfile(filename): - with zipfile.ZipFile(filename) as archive: - for fn in sorted(archive.namelist()): - method = hashlib.sha256() - method.update(archive.read(fn)) - hash = method.hexdigest() - lines.append("%s %s\n" % (hash, fn)) - else: + try: + with zipfile.ZipFile(filename) as archive: + for fn in sorted(archive.namelist()): + method = hashlib.sha256() + method.update(archive.read(fn)) + hash = method.hexdigest() + lines.append("%s %s\n" % (hash, fn)) + is_zipfile = True + except zipfile.BadZipFile: + is_zipfile = False + if not is_zipfile: hash = _hasher(hashlib.sha256(), filename) lines.append("%s go.mod\n" % hash) method = hashlib.sha256() @@ -1331,21 +1336,18 @@ def contains_any(variable, checkvalues, truevalue, falsevalue, d): return truevalue return falsevalue -def filter(variable, checkvalues, d): - """Return all words in the variable that are present in the ``checkvalues``. +def filter_string(val, checkvalues): + """Return all words in the string that are present in the ``checkvalues``. Arguments: - - ``variable``: the variable name. This will be fetched and expanded (using - d.getVar(variable)) and then split into a set(). + - ``val``: the string data to filter after being split into a set(). - ``checkvalues``: if this is a string it is split on whitespace into a set(), otherwise coerced directly into a set(). - ``d``: the data store. Returns a list of string. """ - - val = d.getVar(variable) if not val: return '' val = set(val.split()) @@ -1355,6 +1357,22 @@ def filter(variable, checkvalues, d): checkvalues = set(checkvalues) return ' '.join(sorted(checkvalues & val)) +def filter(variable, checkvalues, d): + """Return all words in the variable that are present in the ``checkvalues``. + + Arguments: + + - ``variable``: the variable name. This will be fetched and expanded (using + d.getVar(variable)) and then split into a set(). + - ``checkvalues``: if this is a string it is split on whitespace into a set(), + otherwise coerced directly into a set(). + - ``d``: the data store. + + Returns a list of string. + """ + + val = d.getVar(variable) + return filter_string(val, checkvalues) def get_referenced_vars(start_expr, d): """ @@ -2269,6 +2287,11 @@ def is_path_on_nfs(path): """ Returns True if ``path`` argument is on a NFS mount. """ + # strip not existing path + if os.path.isabs(path): + while not os.path.exists(path): + path = os.path.dirname(path) + import bb.process fstype = bb.process.run("stat -f -c %T {}".format(path))[0].strip() return fstype == "nfs" diff --git a/lib/bblayers/layerindex.py b/lib/bblayers/layerindex.py index 308a5532d9f..2ba1103a3ee 100644 --- a/lib/bblayers/layerindex.py +++ b/lib/bblayers/layerindex.py @@ -116,7 +116,7 @@ def _construct_url(baseurls, branches): # Load the cooker DB cookerIndex = layerindexlib.LayerIndex(self.tinfoil.config_data) - cookerIndex.load_layerindex('cooker://', load='layerDependencies') + cookerIndex.load_layerindex('cooker://', load=['layerDependencies']) # Fast path, check if we already have what has been requested! (dependencies, invalidnames) = cookerIndex.find_dependencies(names=args.layername, ignores=ignore_layers) @@ -137,7 +137,7 @@ def _construct_url(baseurls, branches): for remoteurl in _construct_url(apiurl, branches): logger.plain("Loading %s..." % remoteurl) - remoteIndex.load_layerindex(remoteurl) + remoteIndex.load_layerindex(remoteurl, load=['layerDependencies']) if remoteIndex.is_empty(): logger.error("Remote layer index %s is empty for branches %s" % (apiurl, branches)) diff --git a/lib/layerindexlib/__init__.py b/lib/layerindexlib/__init__.py index c3265ddaa14..e693e5c1105 100644 --- a/lib/layerindexlib/__init__.py +++ b/lib/layerindexlib/__init__.py @@ -174,15 +174,15 @@ def _fetch_url(self, url, username=None, password=None, debuglevel=0): return res - def load_layerindex(self, indexURI, load=['layerDependencies', 'recipes', 'machines', 'distros'], reload=False): + def load_layerindex(self, indexURI, load=[], reload=False): '''Load the layerindex. indexURI - An index to load. (Use multiple calls to load multiple indexes) reload - If reload is True, then any previously loaded indexes will be forgotten. - load - List of elements to load. Default loads all items. - Note: plugs may ignore this. + load - List of elements to load. By default, an empty list is used to keep things lean. + Callers need to specify a minimal set of elements to load, such as ['layerDependencies'] for dependency resolution. The format of the indexURI: diff --git a/lib/progressbar/progressbar.py b/lib/progressbar/progressbar.py index a8e2dc09c91..1562774ba1a 100644 --- a/lib/progressbar/progressbar.py +++ b/lib/progressbar/progressbar.py @@ -278,6 +278,9 @@ def update(self, value=None): self.last_update_time = now return output + def clear(self): + self.fd.write(" " * self.term_width + '\r') + self.fd.flush() def start(self, update=True): """Starts measuring time, and prints the bar at 0%. diff --git a/lib/toaster/orm/fixtures/gen_fixtures.py b/lib/toaster/orm/fixtures/gen_fixtures.py index 4b87ba0baab..9488ce852e3 100755 --- a/lib/toaster/orm/fixtures/gen_fixtures.py +++ b/lib/toaster/orm/fixtures/gen_fixtures.py @@ -35,19 +35,19 @@ # [0=Codename, 1=Yocto Project Version, 2=Release Date, 3=Current Version, 4=Support Level, 5=Poky Version, 6=BitBake branch] current_releases = [ # Release slot #1 - ['Scarthgap','5.0','April 2024','5.0.0 (April 2024)','Long Term Support (until April 2028)','','2.8'], + ['Wrynose','6.0','April 2026','6.0 (April 2026)','Long Term Support (until April 2030)','','2.16'], # Release slot #2 'local' ['HEAD','HEAD','','Local Yocto Project','HEAD','','HEAD'], # Release slot #3 'master' ['Master','master','','Yocto Project master','master','','master'], # Release slot #4 - ['Whinlatter','5.3','October 2025','5.3.0 (October 2024)','Support for 7 months (until May 2026)','','2.14'], - ['Walnascar','5.2','April 2025','5.2.0 (April 2025)','Support for 7 months (until October 2025)','','2.12'], + ['Scarthgap','5.0','April 2024','5.0 (April 2024)','Long Term Support (until April 2028)','','2.8'], #['Styhead','5.1','November 2024','5.1.0 (November 2024)','Support for 7 months (until May 2025)','','2.10'], #['Nanbield','4.3','November 2023','4.3.0 (November 2023)','Support for 7 months (until May 2024)','','2.6'], #['Mickledore','4.2','April 2023','4.2.0 (April 2023)','Support for 7 months (until October 2023)','','2.4'], #['Langdale','4.1','October 2022','4.1.2 (January 2023)','Support for 7 months (until May 2023)','','2.2'], - ['Kirkstone','4.0','April 2022','4.0.8 (March 2023)','Stable - Long Term Support (until Apr. 2024)','','2.0'], + ['Kirkstone','4.0','April 2022','4.0 (March 2023)','Stable - Long Term Support (until Apr. 2024)','','2.0'], + ['Whinlatter','5.3','October 2025','5.3 (October 2024)','Support for 7 months (until May 2026)','','2.14'], #['Honister','3.4','October 2021','3.4.2 (February 2022)','Support for 7 months (until May 2022)','26.0','1.52'], #['Hardknott','3.3','April 2021','3.3.5 (March 2022)','Stable - Support for 13 months (until Apr. 2022)','25.0','1.50'], #['Gatesgarth','3.2','Oct 2020','3.2.4 (May 2021)','EOL','24.0','1.48'], diff --git a/lib/toaster/orm/fixtures/oe-core.xml b/lib/toaster/orm/fixtures/oe-core.xml index a83b18844b3..01f55ffda8e 100644 --- a/lib/toaster/orm/fixtures/oe-core.xml +++ b/lib/toaster/orm/fixtures/oe-core.xml @@ -8,9 +8,9 @@ - scarthgap + wrynose git://git.openembedded.org/bitbake - 2.8 + 2.16 HEAD @@ -23,28 +23,28 @@ master - whinlatter + scarthgap git://git.openembedded.org/bitbake - 2.14 + 2.8 - walnascar + kirkstone git://git.openembedded.org/bitbake - 2.12 + 2.0 - kirkstone + whinlatter git://git.openembedded.org/bitbake - 2.0 + 2.14 - scarthgap - Openembedded Scarthgap + wrynose + Openembedded Wrynose 1 - scarthgap - Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=scarthgap\">OpenEmbedded Scarthgap</a> branch. + wrynose + Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=wrynose\">OpenEmbedded Wrynose</a> branch. local @@ -61,26 +61,26 @@ Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/\">OpenEmbedded master</a> branch. - whinlatter - Openembedded Whinlatter + scarthgap + Openembedded Scarthgap 4 - whinlatter - Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=whinlatter\">OpenEmbedded Whinlatter</a> branch. + scarthgap + Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=scarthgap\">OpenEmbedded Scarthgap</a> branch. - walnascar - Openembedded Walnascar - 5 - walnascar - Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=walnascar\">OpenEmbedded Walnascar</a> branch. - - kirkstone Openembedded Kirkstone - 6 + 5 kirkstone Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=kirkstone\">OpenEmbedded Kirkstone</a> branch. + + whinlatter + Openembedded Whinlatter + 6 + whinlatter + Toaster will run your builds using the tip of the <a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=whinlatter\">OpenEmbedded Whinlatter</a> branch. + @@ -108,6 +108,7 @@ openembedded-core + openembedded-core diff --git a/lib/toaster/orm/fixtures/poky.xml b/lib/toaster/orm/fixtures/poky.xml index a8a5a7aeb08..3648170bb26 100644 --- a/lib/toaster/orm/fixtures/poky.xml +++ b/lib/toaster/orm/fixtures/poky.xml @@ -8,9 +8,9 @@ - scarthgap + wrynose git://git.openembedded.org/bitbake - 2.8 + 2.16 @@ -26,32 +26,32 @@ - whinlatter + scarthgap git://git.openembedded.org/bitbake - 2.14 + 2.8 - walnascar + kirkstone git://git.openembedded.org/bitbake - 2.12 + 2.0 - kirkstone + whinlatter git://git.openembedded.org/bitbake - 2.0 + 2.14 - scarthgap - Yocto Project 5.0 "Scarthgap" + wrynose + Yocto Project 6.0 "Wrynose" 1 - scarthgap - Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=scarthgap">Yocto Project Scarthgap branch</a>. + wrynose + Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=wrynose">Yocto Project Wrynose branch</a>. local @@ -68,26 +68,26 @@ Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/">Yocto Project Master branch</a>. - whinlatter - Yocto Project 5.3 "Whinlatter" + scarthgap + Yocto Project 5.0 "Scarthgap" 4 - whinlatter - Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=whinlatter">Yocto Project Whinlatter branch</a>. + scarthgap + Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=scarthgap">Yocto Project Scarthgap branch</a>. - walnascar - Yocto Project 5.2 "Walnascar" - 5 - walnascar - Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=walnascar">Yocto Project Walnascar branch</a>. - - kirkstone Yocto Project 4.0 "Kirkstone" - 6 + 5 kirkstone Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=kirkstone">Yocto Project Kirkstone branch</a>. + + whinlatter + Yocto Project 5.3 "Whinlatter" + 6 + whinlatter + Toaster will run your builds using the tip of the <a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=whinlatter">Yocto Project Whinlatter branch</a>. + @@ -180,7 +180,7 @@ 1 0 1 - scarthgap + wrynose meta @@ -202,21 +202,21 @@ 1 0 4 - whinlatter + scarthgap meta 1 0 5 - walnascar + kirkstone meta 1 0 6 - kirkstone + whinlatter meta @@ -232,7 +232,7 @@ 2 0 1 - scarthgap + wrynose meta-poky @@ -254,21 +254,21 @@ 2 0 4 - whinlatter + scarthgap meta-poky 2 0 5 - walnascar + kirkstone meta-poky 2 0 6 - kirkstone + whinlatter meta-poky @@ -284,7 +284,7 @@ 3 0 1 - scarthgap + wrynose meta-yocto-bsp @@ -306,21 +306,21 @@ 3 0 4 - whinlatter + scarthgap meta-yocto-bsp 3 0 5 - walnascar + kirkstone meta-yocto-bsp 3 0 6 - kirkstone + whinlatter meta-yocto-bsp diff --git a/lib/toaster/orm/management/commands/lsupdates.py b/lib/toaster/orm/management/commands/lsupdates.py index 0ee00aa1595..153b26f60a5 100644 --- a/lib/toaster/orm/management/commands/lsupdates.py +++ b/lib/toaster/orm/management/commands/lsupdates.py @@ -105,7 +105,8 @@ def update(self): url_branches = ";branch=%s" % ','.join(allowed_branch_names) else: url_branches = "" - layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches)) + layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches), + load=['layerDependencies', 'recipes', 'machines', 'distros']) http_progress.stop() diff --git a/lib/toaster/tests/functional/test_create_new_project.py b/lib/toaster/tests/functional/test_create_new_project.py index 51c8c120370..5de3c06f123 100644 --- a/lib/toaster/tests/functional/test_create_new_project.py +++ b/lib/toaster/tests/functional/test_create_new_project.py @@ -31,13 +31,30 @@ def test_create_new_project_master(self): False, ) - def test_create_new_project_scarthgap(self): + def test_create_new_project_wrynose(self): """ Test create new project using: - Project Name: Any string - - Release: Yocto Project 5.0 "Scarthgap" (option value: 1) + - Release: Yocto Project 6.0 "Wrynose" (option value: 1) - Merge Toaster settings: True """ release = '1' + release_title = 'Yocto Project 6.0 "Wrynose"' + project_name = 'projectwrynose' + self.create_new_project( + project_name, + release, + release_title, + True, + ) + + + def test_create_new_project_scarthgap(self): + """ Test create new project using: + - Project Name: Any string + - Release: Yocto Project 5.0 "Scarthgap" (option value: 4) + - Merge Toaster settings: True + """ + release = '4' release_title = 'Yocto Project 5.0 "Scarthgap"' project_name = 'projectscarthgap' self.create_new_project( @@ -50,10 +67,10 @@ def test_create_new_project_scarthgap(self): def test_create_new_project_kirkstone(self): """ Test create new project using: - Project Name: Any string - - Release: Yocto Project 4.0 "Kirkstone" (option value: 6) + - Release: Yocto Project 4.0 "Kirkstone" (option value: 5) - Merge Toaster settings: True """ - release = '6' + release = '5' release_title = 'Yocto Project 4.0 "Kirkstone"' project_name = 'projectkirkstone' self.create_new_project(