From 0698a9b38fe2eeb9a2517deef144ae9c6351efd9 Mon Sep 17 00:00:00 2001 From: Delphix Engineering Date: Thu, 2 Oct 2025 05:12:56 +0000 Subject: [PATCH] Version: 25.2-0ubuntu1~24.04.1 -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 Format: 3.0 (quilt) Source: cloud-init Binary: cloud-init Architecture: all Version: 25.2-0ubuntu1~24.04.1 Maintainer: Ubuntu Developers Homepage: https://cloud-init.io/ Standards-Version: 4.5.0 Vcs-Browser: https://github.com/canonical/cloud-init/tree/ubuntu/devel Vcs-Git: https://github.com/canonical/cloud-init -b ubuntu/devel Build-Depends: debhelper-compat (= 13), dh-python, iproute2, po-debconf, python3, python3-configobj, python3-debconf, python3-jinja2, python3-jsonpatch, python3-jsonschema, python3-mock, python3-oauthlib, python3-pytest, python3-pytest-mock, python3-requests, python3-serial, python3-setuptools, python3-yaml, python3-responses, python3-passlib Package-List: cloud-init deb admin optional arch=all Checksums-Sha1: cf3952ef3bc956dc154c72bac6972339f1b8799d 2039157 cloud-init_25.2.orig.tar.gz 3759e93fb86f0d40d874a63800304241fc957d0f 96768 cloud-init_25.2-0ubuntu1~24.04.1.debian.tar.xz Checksums-Sha256: c1e64581ecd1e57a009aeee78f34a014b1e1a1b1d8bdfba1baa0380b0eabffe8 2039157 cloud-init_25.2.orig.tar.gz 22eb23fe1452289d1b02cc6ccf7796512f4a425eb138be2af393d3ce5255e385 96768 cloud-init_25.2-0ubuntu1~24.04.1.debian.tar.xz Files: 723cd38717755e5838bd7cd59117fb11 2039157 cloud-init_25.2.orig.tar.gz 4b605b5a27607962e6f0202fe7e8e4c4 96768 cloud-init_25.2-0ubuntu1~24.04.1.debian.tar.xz -----BEGIN PGP SIGNATURE----- iQGzBAEBCgAdFiEEsyC5UhjFbFFc1+BoZ0j1UGIrpdQFAmicua0ACgkQZ0j1UGIr pdRw8wv+MKPEfCTlVQHSy6fVc8AsRdeM4TRtsrf1H09R2/QJRSguyv0/ybhA8jt8 B12VSLsSyCC4D5u28s2+lJzGISWahsY4DKBXwhi050JaxBaDCulS7Dj6g+9lQoCV RMxjfjYJQBcsURvneGmX6Uc8zqyFHgoLP3r/lS/9dUi9uuRPETYSZz95O40cxs2v CZf413u2AYb6B/0IyW5FD2zXfkJGPfbk7w8wdhAVsxe1YoWRyEThfO2T615rbj4g gi6mZl4QXcSXM+21f7xcBMzmyhk+H1JslRfTkDqI6RtQ0S7jHdd5Lda7MI7uPVXU 2SNSm92Ka//S2uU6CtIyyXhu5OyfcG+yCU3LigBiE7NfsndwLfaOOj1HMg/QlstG 9NMTFmyK+u5H+Bn3GLyIK71aCXW6xKj03zbh0KoQurWi3WBRFWwYr7y/sZnKWwbE vEeVPGrXCMy0ZTAJQEABchOvrroGckuZMRhRwFT5Fcr5K4PCx/NP4X/uHBV6J6Qf BbJLodW/ =vcgF -----END PGP SIGNATURE----- --- .github/PULL_REQUEST_TEMPLATE.md | 1 - .github/workflows/cla.yml | 45 +- .github/workflows/integration.yml | 2 + .github/workflows/packaging-upstream.yml | 2 +- .gitignore | 58 +- .pc/applied-patches | 1 + .../cloudinit/features.py | 10 + .../schemas/schema-cloud-config-v1.json | 263 ++- .../cloudinit/sources/DataSourceNoCloud.py | 6 - .../cloudinit/util.py | 3 + .../tests/unittests/test_util.py | 228 ++- .../cloudinit/features.py | 10 + .../tests/unittests/test_data.py | 915 ---------- .../cloudinit/config/cc_mounts.py | 28 +- .../schemas/schema-cloud-config-v1.json | 263 ++- .../systemd/cloud-init-local.service.tmpl | 5 +- .../systemd/cloud-init-main.service.tmpl | 2 +- .../systemd/cloud-init-network.service.tmpl | 11 +- .../tests/unittests/config/test_cc_mounts.py | 22 +- .../cloudinit/features.py | 152 ++ ChangeLog | 227 +++ cloudinit/analyze/__init__.py | 24 +- cloudinit/analyze/dump.py | 24 +- cloudinit/analyze/show.py | 95 +- cloudinit/cmd/clean.py | 31 +- cloudinit/cmd/devel/hotplug_hook.py | 9 +- cloudinit/cmd/main.py | 59 +- cloudinit/config/cc_ansible.py | 97 +- cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/config/cc_apt_pipelining.py | 2 +- cloudinit/config/cc_bootcmd.py | 7 +- cloudinit/config/cc_byobu.py | 2 +- cloudinit/config/cc_ca_certs.py | 14 +- cloudinit/config/cc_disk_setup.py | 487 ++++- cloudinit/config/cc_install_hotplug.py | 2 +- cloudinit/config/cc_mounts.py | 28 +- cloudinit/config/cc_ntp.py | 6 + .../cc_package_update_upgrade_install.py | 7 +- cloudinit/config/cc_power_state_change.py | 7 +- cloudinit/config/cc_raspberry_pi.py | 216 +++ cloudinit/config/cc_ssh_import_id.py | 2 +- .../schemas/schema-cloud-config-v1.json | 263 ++- .../schemas/schema-network-config-v1.json | 8 + cloudinit/distros/__init__.py | 9 +- cloudinit/distros/raspberry_pi_os.py | 80 + cloudinit/features.py | 10 + cloudinit/net/__init__.py | 17 +- cloudinit/net/activators.py | 37 +- cloudinit/net/dhcp.py | 38 +- cloudinit/net/eni.py | 143 +- cloudinit/net/ephemeral.py | 2 +- cloudinit/net/netplan.py | 54 +- cloudinit/net/network_manager.py | 4 +- cloudinit/net/network_state.py | 1 + cloudinit/net/networkd.py | 336 +++- cloudinit/net/renderers.py | 15 +- cloudinit/signal_handler.py | 42 +- cloudinit/socket.py | 3 + cloudinit/sources/DataSourceAzure.py | 87 +- cloudinit/sources/DataSourceCloudStack.py | 128 +- cloudinit/sources/DataSourceEc2.py | 38 +- cloudinit/sources/DataSourceGCE.py | 6 + cloudinit/sources/DataSourceHetzner.py | 7 + cloudinit/sources/DataSourceMAAS.py | 2 +- cloudinit/sources/DataSourceNoCloud.py | 6 - cloudinit/sources/DataSourceOracle.py | 104 +- cloudinit/sources/DataSourceVMware.py | 212 ++- cloudinit/sources/DataSourceWSL.py | 129 +- cloudinit/sources/__init__.py | 8 + cloudinit/sources/azure/errors.py | 35 +- cloudinit/sources/azure/kvp.py | 13 +- cloudinit/sources/helpers/azure.py | 55 +- cloudinit/sources/helpers/hetzner.py | 2 +- cloudinit/ssh_util.py | 4 + cloudinit/subp.py | 5 +- cloudinit/url_helper.py | 3 +- cloudinit/util.py | 3 + cloudinit/version.py | 2 +- config/cloud.cfg.tmpl | 39 +- conftest.py | 18 +- debian/changelog | 16 + .../deprecation-version-boundary.patch | 4 +- debian/patches/grub-dpkg-support.patch | 2 +- debian/patches/no-nocloud-network.patch | 26 +- .../patches/no-remove-networkd-online.patch | 11 - debian/patches/no-single-process.patch | 40 +- debian/patches/series | 1 + debian/patches/strip-invalid-mtu.patch | 18 + .../cloud-config-ansible-controller.txt | 2 +- doc/examples/cloud-config-ansible-managed.txt | 2 +- doc/examples/cloud-config-ansible-pull.txt | 4 +- doc/examples/cloud-config-datasources.txt | 3 + doc/examples/cloud-config-disk-setup.txt | 10 +- doc/examples/cloud-config-mount-points.txt | 2 +- doc/examples/cloud-config-user-groups.txt | 12 +- doc/examples/cloud-config.txt | 6 +- .../network-config-v1-subnet-routes.yaml | 2 + doc/module-docs/cc_ansible/data.yaml | 6 +- doc/module-docs/cc_ansible/example1.yaml | 4 +- doc/module-docs/cc_ansible/example2.yaml | 4 +- doc/module-docs/cc_mounts/data.yaml | 2 +- doc/module-docs/cc_raspberry_pi/data.yaml | 26 + doc/module-docs/cc_raspberry_pi/example1.yaml | 5 + doc/module-docs/cc_raspberry_pi/example2.yaml | 4 + doc/module-docs/cc_raspberry_pi/example3.yaml | 7 + doc/module-docs/cc_raspberry_pi/example4.yaml | 9 + doc/module-docs/cc_raspberry_pi/example5.yaml | 3 + doc/rtd/conf.py | 23 +- doc/rtd/development/contribute_code.rst | 1 - doc/rtd/development/contribute_docs.rst | 8 - doc/rtd/development/first_PR.rst | 29 +- doc/rtd/development/index.rst | 3 - doc/rtd/development/integration_tests.rst | 17 + doc/rtd/explanation/format.rst | 7 +- doc/rtd/explanation/hardening.rst | 84 + doc/rtd/explanation/index.rst | 2 + doc/rtd/explanation/instancedata.rst | 2 +- doc/rtd/explanation/net-device-info.rst | 42 + doc/rtd/howto/index.rst | 1 + doc/rtd/howto/launch_lxd.rst | 6 +- doc/rtd/howto/launch_qemu.rst | 2 +- doc/rtd/howto/shared/download_image.txt | 2 +- doc/rtd/howto/wait_for_cloud_init.rst | 33 + doc/rtd/index.rst | 4 +- doc/rtd/reference/availability.rst | 24 +- doc/rtd/reference/breaking_changes.rst | 69 +- doc/rtd/reference/cli.rst | 17 +- doc/rtd/reference/datasources/vmware.rst | 73 +- doc/rtd/reference/datasources/wsl.rst | 19 +- doc/rtd/reference/distros.rst | 43 - doc/rtd/reference/examples.rst | 4 +- doc/rtd/reference/index.rst | 1 - doc/rtd/reference/modules.rst | 2 + .../reference/network-config-format-v1.rst | 10 + .../reference/network-config-format-v2.rst | 13 +- doc/rtd/reference/network-config.rst | 8 +- .../yaml_examples/ansible_controller.rst | 2 +- .../yaml_examples/ansible_managed.rst | 3 +- doc/rtd/reference/yaml_examples/mounts.rst | 2 +- .../reference/yaml_examples/user_groups.rst | 4 +- doc/rtd/spelling_word_list.txt | 1 + doc/rtd/templates/module_property.tmpl | 15 +- doc/rtd/tutorial/qemu-script.sh | 4 +- doc/rtd/tutorial/qemu.rst | 6 +- integration-requirements.txt | 4 + packages/redhat/cloud-init.spec.in | 2 +- pyproject.toml | 16 - setup.py | 9 +- setup_utils.py | 7 +- systemd/cloud-init-local.service.tmpl | 5 +- systemd/cloud-init.service.tmpl | 11 +- templates/hosts.openeuler.tmpl | 24 + templates/sources.list.debian.deb822.tmpl | 2 +- tests/data/merge_sources/expected7.yaml | 2 +- tests/data/merge_sources/source7-1.yaml | 3 +- tests/integration_tests/clouds.py | 44 +- tests/integration_tests/cmd/test_analyze.py | 36 + tests/integration_tests/cmd/test_clean.py | 7 +- tests/integration_tests/cmd/test_schema.py | 4 +- tests/integration_tests/conftest.py | 174 +- .../datasources/test_caching.py | 6 + .../datasources/test_lxd_discovery.py | 2 +- .../datasources/test_nocloud.py | 30 +- .../datasources/test_none.py | 16 +- .../datasources/test_oci_networking.py | 47 + tests/integration_tests/instances.py | 67 +- .../integration_tests/integration_settings.py | 4 +- .../integration_tests/modules/test_ansible.py | 59 +- .../modules/test_apt_functionality.py | 116 +- tests/integration_tests/modules/test_cli.py | 8 +- .../modules/test_combined.py | 18 +- .../modules/test_disk_setup.py | 2 +- .../integration_tests/modules/test_hotplug.py | 75 +- .../modules/test_keys_to_console.py | 2 +- .../test_package_update_upgrade_install.py | 3 +- .../modules/test_power_state_change.py | 6 +- .../modules/test_ssh_keysfile.py | 11 +- .../modules/test_users_groups.py | 47 +- tests/integration_tests/reaper.py | 8 +- tests/integration_tests/test_ds_identify.py | 5 +- tests/integration_tests/test_networking.py | 37 +- tests/integration_tests/test_reaper.py | 10 +- .../integration_tests/test_signal_handler.py | 18 + tests/integration_tests/test_upgrade.py | 20 + tests/integration_tests/util.py | 57 +- tests/unittests/analyze/test_boot.py | 123 +- .../unittests/cmd/devel/test_hotplug_hook.py | 49 +- tests/unittests/cmd/devel/test_net_convert.py | 1 - tests/unittests/cmd/test_clean.py | 143 +- tests/unittests/cmd/test_main.py | 17 + .../test_apt_configure_sources_list_v1.py | 8 +- tests/unittests/config/test_apt_source_v3.py | 2 +- tests/unittests/config/test_cc_ansible.py | 426 ++++- .../unittests/config/test_cc_apk_configure.py | 123 +- tests/unittests/config/test_cc_bootcmd.py | 69 +- .../config/test_cc_disable_ec2_metadata.py | 11 +- tests/unittests/config/test_cc_disk_setup.py | 60 +- .../config/test_cc_install_hotplug.py | 2 +- tests/unittests/config/test_cc_keyboard.py | 36 +- tests/unittests/config/test_cc_landscape.py | 14 +- tests/unittests/config/test_cc_mcollective.py | 88 +- tests/unittests/config/test_cc_mounts.py | 22 +- tests/unittests/config/test_cc_puppet.py | 249 ++- .../unittests/config/test_cc_raspberry_pi.py | 215 +++ tests/unittests/config/test_cc_resizefs.py | 276 +-- tests/unittests/config/test_cc_resolv_conf.py | 90 +- .../config/test_cc_rh_subscription.py | 217 +-- tests/unittests/config/test_cc_runcmd.py | 74 +- tests/unittests/config/test_cc_snap.py | 24 +- tests/unittests/config/test_cc_ssh.py | 18 +- .../config/test_cc_ubuntu_drivers.py | 21 +- .../config/test_cc_update_etc_hosts.py | 57 +- .../unittests/config/test_cc_users_groups.py | 91 +- tests/unittests/config/test_cc_wireguard.py | 140 +- tests/unittests/config/test_cc_write_files.py | 172 +- .../config/test_cc_write_files_deferred.py | 38 +- .../unittests/config/test_cc_yum_add_repo.py | 51 +- .../config/test_cc_zypper_add_repo.py | 107 +- tests/unittests/config/test_schema.py | 53 +- tests/unittests/conftest.py | 89 +- tests/unittests/distros/__init__.py | 18 - .../distros/package_management/test_apt.py | 7 +- tests/unittests/distros/test__init__.py | 139 +- tests/unittests/distros/test_aosc.py | 9 +- tests/unittests/distros/test_arch.py | 13 +- tests/unittests/distros/test_azurelinux.py | 13 +- tests/unittests/distros/test_bsd_utils.py | 80 +- tests/unittests/distros/test_create_users.py | 5 +- tests/unittests/distros/test_dragonflybsd.py | 7 +- tests/unittests/distros/test_freebsd.py | 19 +- tests/unittests/distros/test_gentoo.py | 43 +- tests/unittests/distros/test_init.py | 22 +- .../unittests/distros/test_manage_service.py | 20 +- tests/unittests/distros/test_mariner.py | 13 +- tests/unittests/distros/test_netbsd.py | 6 +- tests/unittests/distros/test_netconfig.py | 589 +++--- tests/unittests/distros/test_openbsd.py | 7 +- tests/unittests/distros/test_photon.py | 40 +- .../unittests/distros/test_raspberry_pi_os.py | 95 + tests/unittests/distros/test_sles.py | 9 +- tests/unittests/filters/test_launch_index.py | 33 +- tests/unittests/helpers.py | 296 +-- .../network/10-cloud-init-bond.200.netdev | 8 + .../network/10-cloud-init-bond.200.network | 14 + .../network/10-cloud-init-bond0.netdev | 11 + .../network/10-cloud-init-bond0.network | 10 + .../network/10-cloud-init-eth0.network | 11 + .../network/10-cloud-init-eth1.network | 5 + .../network/10-cloud-init-eth2.network | 16 + .../network/10-cloud-init-vl101.netdev | 6 + .../network/10-cloud-init-vl101.network | 5 + .../network/10-cloud-init-vlan100.netdev | 8 + .../network/10-cloud-init-vlan100.network | 27 + .../net/artifacts/photon_net_config_v2.yaml | 86 + tests/unittests/net/network_configs.py | 358 +++- tests/unittests/net/test_dhcp.py | 8 +- tests/unittests/net/test_init.py | 56 +- tests/unittests/net/test_net_rendering.py | 48 +- tests/unittests/net/test_netplan.py | 52 + tests/unittests/runs/test_merge_run.py | 88 +- tests/unittests/runs/test_simple_run.py | 170 +- tests/unittests/sources/azure/test_errors.py | 35 +- tests/unittests/sources/azure/test_kvp.py | 24 +- tests/unittests/sources/conftest.py | 14 + tests/unittests/sources/helpers/test_ec2.py | 143 +- .../unittests/sources/helpers/test_netlink.py | 129 +- tests/unittests/sources/test_aliyun.py | 367 ++-- tests/unittests/sources/test_altcloud.py | 452 ++--- tests/unittests/sources/test_azure.py | 1597 +++++++++-------- tests/unittests/sources/test_azure_helper.py | 635 +++---- tests/unittests/sources/test_cloudsigma.py | 136 +- tests/unittests/sources/test_cloudstack.py | 620 ++++--- tests/unittests/sources/test_common.py | 1 + tests/unittests/sources/test_configdrive.py | 251 ++- tests/unittests/sources/test_digitalocean.py | 168 +- tests/unittests/sources/test_ec2.py | 73 +- tests/unittests/sources/test_exoscale.py | 197 +- tests/unittests/sources/test_gce.py | 150 +- tests/unittests/sources/test_hetzner.py | 45 +- tests/unittests/sources/test_ibmcloud.py | 176 +- tests/unittests/sources/test_init.py | 589 +++--- tests/unittests/sources/test_lxd.py | 6 +- tests/unittests/sources/test_maas.py | 8 +- tests/unittests/sources/test_openstack.py | 87 +- tests/unittests/sources/test_oracle.py | 91 +- tests/unittests/sources/test_scaleway.py | 126 +- tests/unittests/sources/test_smartos.py | 930 +++++----- tests/unittests/sources/test_vmware.py | 930 +++++----- tests/unittests/sources/test_wsl.py | 108 +- tests/unittests/test__init__.py | 3 +- tests/unittests/test_cli.py | 42 + tests/unittests/test_data.py | 64 +- tests/unittests/test_dmi.py | 146 +- tests/unittests/test_ds_identify.py | 1159 ++++++------ tests/unittests/test_features.py | 6 +- tests/unittests/test_helpers.py | 19 +- tests/unittests/test_log.py | 57 +- tests/unittests/test_merging.py | 20 +- tests/unittests/test_net.py | 117 +- tests/unittests/test_net_activators.py | 29 +- tests/unittests/test_render_template.py | 100 ++ tests/unittests/test_signal_handler.py | 68 +- tests/unittests/test_ssh_util.py | 12 + tests/unittests/test_upgrade.py | 1 + tests/unittests/test_url_helper.py | 9 +- tests/unittests/test_util.py | 228 ++- tests/unittests/util.py | 2 + tools/.github-cla-signers | 234 --- tools/.lp-to-git-user | 37 - tools/check-cla-signers | 15 - tools/ds-identify | 5 - tools/migrate-lp-user-to-github | 309 ---- tools/read-dependencies | 43 +- tools/render-template | 1 + tools/run-container | 23 +- tools/run-lint | 5 +- 316 files changed, 13417 insertions(+), 10255 deletions(-) delete mode 100644 .pc/no-remove-networkd-online.patch/tests/unittests/test_data.py create mode 100644 .pc/strip-invalid-mtu.patch/cloudinit/features.py create mode 100644 cloudinit/config/cc_raspberry_pi.py create mode 100644 cloudinit/distros/raspberry_pi_os.py create mode 100644 debian/patches/strip-invalid-mtu.patch create mode 100644 doc/module-docs/cc_raspberry_pi/data.yaml create mode 100644 doc/module-docs/cc_raspberry_pi/example1.yaml create mode 100644 doc/module-docs/cc_raspberry_pi/example2.yaml create mode 100644 doc/module-docs/cc_raspberry_pi/example3.yaml create mode 100644 doc/module-docs/cc_raspberry_pi/example4.yaml create mode 100644 doc/module-docs/cc_raspberry_pi/example5.yaml create mode 100644 doc/rtd/explanation/hardening.rst create mode 100644 doc/rtd/explanation/net-device-info.rst create mode 100644 doc/rtd/howto/wait_for_cloud_init.rst delete mode 100644 doc/rtd/reference/distros.rst create mode 100644 templates/hosts.openeuler.tmpl create mode 100644 tests/integration_tests/cmd/test_analyze.py create mode 100644 tests/integration_tests/test_signal_handler.py create mode 100644 tests/unittests/config/test_cc_raspberry_pi.py create mode 100644 tests/unittests/distros/test_raspberry_pi_os.py create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond.200.netdev create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond.200.network create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond0.netdev create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond0.network create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth0.network create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth1.network create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth2.network create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vl101.netdev create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vl101.network create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vlan100.netdev create mode 100644 tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vlan100.network create mode 100644 tests/unittests/net/artifacts/photon_net_config_v2.yaml delete mode 100644 tools/.github-cla-signers delete mode 100644 tools/.lp-to-git-user delete mode 100755 tools/check-cla-signers delete mode 100755 tools/migrate-lp-user-to-github diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1f6890f7..d4a1af93 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,7 +4,6 @@ Thank you for submitting a PR to cloud-init! To ease the process of reviewing your PR, do make sure to complete the following checklist **before** submitting a pull request. - [ ] I have signed the CLA: https://ubuntu.com/legal/contributors -- [ ] I have added my Github username to ``tools/.github-cla-signers`` - [ ] I have included a comprehensive commit message using the guide below - [ ] I have added unit tests to cover the new behavior under ``tests/unittests/`` - Test files should map to source files i.e. a source file ``cloudinit/example.py`` should be tested by ``tests/unittests/test_example.py`` diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 65daf51d..66bfb0f1 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -3,47 +3,8 @@ name: CLA Check on: [pull_request] jobs: - cla-validate: - + cla-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Check CLA signing status for ${{ github.event.pull_request.user.login }} - run: | - cat > unsigned-cla.txt < 0) + m_cmdline = mocker.patch(M_PATH + "get_cmdline", return_value=cmdline) + assert util.system_is_snappy() is True + assert m_cmdline.call_count > 0 - @mock.patch(M_PATH + "get_cmdline") - def test_nothing_found_is_not_snappy(self, m_cmdline): + def test_nothing_found_is_not_snappy(self, mocker): """If no positive identification, then not snappy.""" - m_cmdline.return_value = "root=/dev/sda" - self.reRoot() - self.assertFalse(util.system_is_snappy()) - self.assertTrue(m_cmdline.call_count > 0) + m_cmdline = mocker.patch( + M_PATH + "get_cmdline", return_value="root=/dev/sda" + ) + assert util.system_is_snappy() is False + assert m_cmdline.call_count > 0 - @mock.patch(M_PATH + "get_cmdline") - def test_channel_ini_with_snappy_is_snappy(self, m_cmdline): + def test_channel_ini_with_snappy_is_snappy(self, mocker): """A Channel.ini file with 'ubuntu-core' indicates snappy.""" - m_cmdline.return_value = "root=/dev/sda" - root_d = self.tmp_dir() + mocker.patch(M_PATH + "get_cmdline", return_value="root=/dev/sda") content = "\n".join(["[Foo]", "source = 'ubuntu-core'", ""]) - helpers.populate_dir(root_d, {"etc/system-image/channel.ini": content}) - self.reRoot(root_d) - self.assertTrue(util.system_is_snappy()) + helpers.populate_dir("/", {"etc/system-image/channel.ini": content}) + assert util.system_is_snappy() is True - @mock.patch(M_PATH + "get_cmdline") - def test_system_image_config_dir_is_snappy(self, m_cmdline): + def test_system_image_config_dir_is_snappy(self, mocker): """Existence of /etc/system-image/config.d indicates snappy.""" - m_cmdline.return_value = "root=/dev/sda" - root_d = self.tmp_dir() + mocker.patch(M_PATH + "get_cmdline", return_value="root=/dev/sda") helpers.populate_dir( - root_d, {"etc/system-image/config.d/my.file": "_unused"} + "/", {"etc/system-image/config.d/my.file": "_unused"} ) - self.reRoot(root_d) - self.assertTrue(util.system_is_snappy()) + assert util.system_is_snappy() is True class TestLoadShellContent(helpers.TestCase): diff --git a/.pc/no-remove-networkd-online.patch/cloudinit/features.py b/.pc/no-remove-networkd-online.patch/cloudinit/features.py index 7a11434f..8c9c99a4 100644 --- a/.pc/no-remove-networkd-online.patch/cloudinit/features.py +++ b/.pc/no-remove-networkd-online.patch/cloudinit/features.py @@ -98,6 +98,16 @@ to modify the systemd unit files. """ +STRIP_INVALID_MTU = False +""" +If ``STRIP_INVALID_MTU`` is True, then cloud-init will strip invalid MTU +values from rendered v2 netplan configuration. Cloud-init allowed these values +prior to 24.2, so this flag is used to maintain compatibility with +previously generated network configurations. + +(This flag can be removed when Noble is no longer supported.) +""" + DEPRECATION_INFO_BOUNDARY = "24.1" """ DEPRECATION_INFO_BOUNDARY is used by distros to configure at which upstream diff --git a/.pc/no-remove-networkd-online.patch/tests/unittests/test_data.py b/.pc/no-remove-networkd-online.patch/tests/unittests/test_data.py deleted file mode 100644 index 0e26ba51..00000000 --- a/.pc/no-remove-networkd-online.patch/tests/unittests/test_data.py +++ /dev/null @@ -1,915 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""Tests for handling of userdata within cloud init.""" - -import gzip -import logging -import os -from email import encoders -from email.mime.application import MIMEApplication -from email.mime.base import MIMEBase -from email.mime.multipart import MIMEMultipart -from io import BytesIO -from pathlib import Path -from unittest import mock - -import pytest -import responses - -from cloudinit import handlers -from cloudinit import helpers as c_helpers -from cloudinit import safeyaml, stages -from cloudinit import user_data as ud -from cloudinit import util -from cloudinit.config.modules import Modules -from cloudinit.settings import DEFAULT_RUN_DIR, PER_INSTANCE -from tests.unittests import helpers -from tests.unittests.util import FakeDataSource - -MPATH = "cloudinit.stages" - - -def count_messages(root): - am = 0 - for m in root.walk(): - if ud.is_skippable(m): - continue - am += 1 - return am - - -def gzip_text(text): - contents = BytesIO() - f = gzip.GzipFile(fileobj=contents, mode="wb") - f.write(util.encode_text(text)) - f.flush() - f.close() - return contents.getvalue() - - -@pytest.fixture(scope="function") -def init_tmp(request, tmpdir): - ci = stages.Init() - cloud_dir = tmpdir.join("cloud") - cloud_dir.mkdir() - run_dir = tmpdir.join("run") - run_dir.mkdir() - ci._cfg = { - "system_info": { - "default_user": {"name": "ubuntu"}, - "distro": "ubuntu", - "paths": { - "cloud_dir": cloud_dir.strpath, - "run_dir": run_dir.strpath, - }, - } - } - run_dir.join("instance-data-sensitive.json").write("{}") - return ci - - -class TestConsumeUserData: - def test_simple_jsonp(self, init_tmp): - user_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/bar", "value": "qux2" } -] -""" - init_tmp.datasource = FakeDataSource(user_blob) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert len(cc) == 2 - assert cc["baz"] == "qux" - assert cc["bar"] == "qux2" - - @pytest.mark.usefixtures("fake_filesystem") - def test_simple_jsonp_vendor_and_vendor2_and_user(self): - # test that user-data wins over vendor - user_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/bar", "value": "qux2" }, - { "op": "add", "path": "/foobar", "value": "qux3" } -] -""" - vendor_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "quxA" }, - { "op": "add", "path": "/bar", "value": "quxB" }, - { "op": "add", "path": "/foo", "value": "quxC" }, - { "op": "add", "path": "/corge", "value": "quxEE" } -] -""" - vendor2_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/corge", "value": "quxD" }, - { "op": "add", "path": "/grault", "value": "quxFF" }, - { "op": "add", "path": "/foobar", "value": "quxGG" } -] -""" - initer = stages.Init() - initer.datasource = FakeDataSource( - user_blob, vendordata=vendor_blob, vendordata2=vendor2_blob - ) - initer.read_cfg() - initer.initialize() - initer.fetch() - initer.instancify() - with mock.patch( - "cloudinit.util.read_conf_from_cmdline", return_value={} - ): - initer.update() - initer.cloudify().run( - "consume_data", - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(initer) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - cfg = mods.cfg - assert "vendor_data" in cfg - assert "vendor_data2" in cfg - # Confirm that vendordata2 overrides vendordata, and that - # userdata overrides both - assert cfg["baz"] == "qux" - assert cfg["bar"] == "qux2" - assert cfg["foobar"] == "qux3" - assert cfg["foo"] == "quxC" - assert cfg["corge"] == "quxD" - assert cfg["grault"] == "quxFF" - - @pytest.mark.usefixtures("fake_filesystem") - def test_simple_jsonp_no_vendor_consumed(self): - # make sure that vendor data is not consumed - user_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/bar", "value": "qux2" }, - { "op": "add", "path": "/vendor_data", "value": {"enabled": "false"}} -] -""" - vendor_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "quxA" }, - { "op": "add", "path": "/bar", "value": "quxB" }, - { "op": "add", "path": "/foo", "value": "quxC" } -] -""" - initer = stages.Init() - initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) - initer.read_cfg() - initer.initialize() - initer.fetch() - initer.instancify() - initer.update() - initer.cloudify().run( - "consume_data", - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(initer) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - cfg = mods.cfg - assert cfg["baz"] == "qux" - assert cfg["bar"] == "qux2" - assert "foo" not in cfg - - def test_mixed_cloud_config(self, init_tmp): - blob_cc = """ -#cloud-config -a: b -c: d -""" - message_cc = MIMEBase("text", "cloud-config") - message_cc.set_payload(blob_cc) - - blob_jp = """ -#cloud-config-jsonp -[ - { "op": "replace", "path": "/a", "value": "c" }, - { "op": "remove", "path": "/c" } -] -""" - - message_jp = MIMEBase("text", "cloud-config-jsonp") - message_jp.set_payload(blob_jp) - - message = MIMEMultipart() - message.attach(message_cc) - message.attach(message_jp) - - init_tmp.datasource = FakeDataSource(str(message)) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert len(cc) == 1 - assert cc["a"] == "c" - - def test_cloud_config_as_x_shell_script(self, init_tmp): - blob_cc = """ -#cloud-config -a: b -c: d -""" - message_cc = MIMEBase("text", "x-shellscript") - message_cc.set_payload(blob_cc) - - blob_jp = """ -#cloud-config-jsonp -[ - { "op": "replace", "path": "/a", "value": "c" }, - { "op": "remove", "path": "/c" } -] -""" - - message_jp = MIMEBase("text", "cloud-config-jsonp") - message_jp.set_payload(blob_jp) - - message = MIMEMultipart() - message.attach(message_cc) - message.attach(message_jp) - - init_tmp.datasource = FakeDataSource(str(message)) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert len(cc) == 1 - assert cc["a"] == "c" - - @pytest.mark.usefixtures("fake_filesystem") - def test_vendor_user_yaml_cloud_config(self): - vendor_blob = """ -#cloud-config -a: b -name: vendor -run: - - x - - y -""" - - user_blob = """ -#cloud-config -a: c -vendor_data: - enabled: true - prefix: /bin/true -name: user -run: - - z -""" - initer = stages.Init() - initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) - initer.read_cfg() - initer.initialize() - initer.fetch() - initer.instancify() - initer.update() - initer.cloudify().run( - "consume_data", - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(initer) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - cfg = mods.cfg - assert "vendor_data" in cfg - assert cfg["a"] == "c" - assert cfg["name"] == "user" - assert "x" not in cfg["run"] - assert "y" not in cfg["run"] - assert "z" in cfg["run"] - - @pytest.mark.usefixtures("fake_filesystem") - def test_vendordata_script(self): - vendor_blob = """ -#!/bin/bash -echo "test" -""" - vendor2_blob = """ -#!/bin/bash -echo "dynamic test" -""" - - user_blob = """ -#cloud-config -vendor_data: - enabled: true - prefix: /bin/true -""" - initer = stages.Init() - initer.datasource = FakeDataSource( - user_blob, vendordata=vendor_blob, vendordata2=vendor2_blob - ) - initer.read_cfg() - initer.initialize() - initer.fetch() - initer.instancify() - initer.update() - initer.cloudify().run( - "consume_data", - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(initer) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - vendor_script = initer.paths.get_ipath_cur("vendor_scripts") - vendor_script_fns = "%s/part-001" % vendor_script - assert os.path.exists(vendor_script_fns) is True - - def test_merging_cloud_config(self, tmpdir): - blob = """ -#cloud-config -a: b -e: f -run: - - b - - c -""" - message1 = MIMEBase("text", "cloud-config") - message1.set_payload(blob) - - blob2 = """ -#cloud-config -a: e -e: g -run: - - stuff - - morestuff -""" - message2 = MIMEBase("text", "cloud-config") - message2["X-Merge-Type"] = ( - "dict(recurse_array,recurse_str)+list(append)+str(append)" - ) - message2.set_payload(blob2) - - blob3 = """ -#cloud-config -e: - - 1 - - 2 - - 3 -p: 1 -""" - message3 = MIMEBase("text", "cloud-config") - message3.set_payload(blob3) - - messages = [message1, message2, message3] - - paths = c_helpers.Paths( - {"cloud_dir": tmpdir, "run_dir": tmpdir}, ds=FakeDataSource("") - ) - cloud_cfg = handlers.cloud_config.CloudConfigPartHandler(paths) - - cloud_cfg.handle_part( - None, handlers.CONTENT_START, None, None, None, None - ) - for i, m in enumerate(messages): - headers = dict(m) - fn = "part-%s" % (i + 1) - payload = m.get_payload(decode=True) - cloud_cfg.handle_part( - None, headers["Content-Type"], fn, payload, None, headers - ) - cloud_cfg.handle_part( - None, handlers.CONTENT_END, None, None, None, None - ) - contents = util.load_text_file(paths.get_ipath("cloud_config")) - contents = util.load_yaml(contents) - assert contents["run"], ["b", "c", "stuff", "morestuff"] - assert contents["a"] == "be" - assert contents["e"] == [1, 2, 3] - assert contents["p"] == 1 - - def test_unhandled_type_warning(self, init_tmp, caplog): - """Raw text without magic is ignored but shows warning.""" - data = "arbitrary text\n" - init_tmp.datasource = FakeDataSource(data) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert ( - "Unhandled non-multipart (text/x-not-multipart) userdata:" - in caplog.text - ) - mockobj.assert_called_once_with( - init_tmp.paths.get_ipath("cloud_config"), "", 0o600 - ) - - def test_mime_gzip_compressed(self, init_tmp): - """Tests that individual message gzip encoding works.""" - - def gzip_part(text): - return MIMEApplication(gzip_text(text), "gzip") - - base_content1 = """ -#cloud-config -a: 2 -""" - - base_content2 = """ -#cloud-config -b: 3 -c: 4 -""" - - message = MIMEMultipart("test") - message.attach(gzip_part(base_content1)) - message.attach(gzip_part(base_content2)) - init_tmp.datasource = FakeDataSource(str(message)) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - contents = util.load_yaml(contents) - assert isinstance(contents, dict) is True - assert len(contents) == 3 - assert contents["a"] == 2 - assert contents["b"] == 3 - assert contents["c"] == 4 - - def test_mime_text_plain(self, init_tmp, caplog): - """Mime message of type text/plain is ignored but shows warning.""" - message = MIMEBase("text", "plain") - message.set_payload("Just text") - init_tmp.datasource = FakeDataSource(message.as_string().encode()) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert "Unhandled unknown content-type (text/plain)" in caplog.text - mockobj.assert_called_once_with( - init_tmp.paths.get_ipath("cloud_config"), "", 0o600 - ) - - # Since features are intended to be overridden downstream, mock them - # all here so new feature flags don't require a new change to this - # unit test. - @mock.patch.multiple( - "cloudinit.features", - ERROR_ON_USER_DATA_FAILURE=True, - ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES=True, - EXPIRE_APPLIES_TO_HASHED_USERS=False, - NETPLAN_CONFIG_ROOT_READ_ONLY=True, - DEPRECATION_INFO_BOUNDARY="devel", - NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, - APT_DEB822_SOURCE_LIST_FILE=True, - ) - def test_shellscript(self, init_tmp, tmpdir, caplog): - """Raw text starting #!/bin/sh is treated as script.""" - script = "#!/bin/sh\necho hello\n" - init_tmp.datasource = FakeDataSource(script) - - outpath = os.path.join( - init_tmp.paths.get_ipath_cur("scripts"), "part-001" - ) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert caplog.records == [] # No warnings - - mockobj.assert_has_calls( - [ - mock.call(outpath, script, 0o700), - mock.call(init_tmp.paths.get_ipath("cloud_config"), "", 0o600), - ] - ) - expected = { - "features": { - "ERROR_ON_USER_DATA_FAILURE": True, - "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES": True, - "EXPIRE_APPLIES_TO_HASHED_USERS": False, - "NETPLAN_CONFIG_ROOT_READ_ONLY": True, - "DEPRECATION_INFO_BOUNDARY": "devel", - "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, - "APT_DEB822_SOURCE_LIST_FILE": True, - "MANUAL_NETWORK_WAIT": True, - }, - "system_info": { - "default_user": {"name": "ubuntu"}, - "distro": "ubuntu", - "paths": { - "cloud_dir": tmpdir.join("cloud").strpath, - "run_dir": tmpdir.join("run").strpath, - }, - }, - } - - loaded_json = util.load_json( - util.load_text_file( - init_tmp.paths.get_runpath("instance_data_sensitive") - ) - ) - assert expected == loaded_json - - expected["_doc"] = stages.COMBINED_CLOUD_CONFIG_DOC - assert expected == util.load_json( - util.load_text_file( - init_tmp.paths.get_runpath("combined_cloud_config") - ) - ) - - def test_mime_text_x_shellscript(self, init_tmp, caplog): - """Mime message of type text/x-shellscript is treated as script.""" - script = "#!/bin/sh\necho hello\n" - message = MIMEBase("text", "x-shellscript") - message.set_payload(script) - init_tmp.datasource = FakeDataSource(message.as_string()) - - outpath = os.path.join( - init_tmp.paths.get_ipath_cur("scripts"), "part-001" - ) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert caplog.records == [] # No warnings - - mockobj.assert_has_calls( - [ - mock.call(outpath, script, 0o700), - mock.call(init_tmp.paths.get_ipath("cloud_config"), "", 0o600), - ] - ) - - def test_mime_text_plain_shell(self, init_tmp, caplog): - """Mime type text/plain starting #!/bin/sh is treated as script.""" - script = "#!/bin/sh\necho hello\n" - message = MIMEBase("text", "plain") - message.set_payload(script) - init_tmp.datasource = FakeDataSource(message.as_string()) - - outpath = os.path.join( - init_tmp.paths.get_ipath_cur("scripts"), "part-001" - ) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert caplog.records == [] # No warnings - - mockobj.assert_has_calls( - [ - mock.call(outpath, script, 0o700), - mock.call(init_tmp.paths.get_ipath("cloud_config"), "", 0o600), - ] - ) - - def test_mime_application_octet_stream(self, init_tmp, caplog): - """Mime type application/octet-stream is ignored but shows warning.""" - message = MIMEBase("application", "octet-stream") - message.set_payload(b"\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc") - encoders.encode_base64(message) - init_tmp.datasource = FakeDataSource(message.as_string().encode()) - - with mock.patch("cloudinit.util.write_file") as mockobj: - with caplog.at_level(logging.WARNING): - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - assert ( - "Unhandled unknown content-type" - " (application/octet-stream)" in caplog.text - ) - mockobj.assert_called_once_with( - init_tmp.paths.get_ipath("cloud_config"), "", 0o600 - ) - - def test_cloud_config_archive(self, init_tmp): - non_decodable = b"\x11\xc9\xb4gTH\xee\x12" - data = [ - {"content": "#cloud-config\npassword: gocubs\n"}, - {"content": "#cloud-config\nlocale: chicago\n"}, - {"content": non_decodable}, - ] - message = b"#cloud-config-archive\n" + safeyaml.dumps(data).encode() - - init_tmp.datasource = FakeDataSource(message) - - fs = {} - - def fsstore(filename, content, mode=0o0644, omode="wb"): - fs[filename] = content - - # consuming the user-data provided should write 'cloud_config' file - # which will have our yaml in it. - with mock.patch("cloudinit.util.write_file") as mockobj: - mockobj.side_effect = fsstore - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset"): - init_tmp.consume_data() - - cfg = util.load_yaml(fs[init_tmp.paths.get_ipath("cloud_config")]) - assert cfg.get("password") == "gocubs" - assert cfg.get("locale") == "chicago" - - @pytest.mark.usefixtures("fake_filesystem") - @mock.patch("cloudinit.util.read_conf_with_confd") - def test_dont_allow_user_data(self, mock_cfg): - mock_cfg.return_value = {"allow_userdata": False} - - # test that user-data is ignored but vendor-data is kept - user_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/bar", "value": "qux2" } -] -""" - vendor_blob = """ -#cloud-config-jsonp -[ - { "op": "add", "path": "/baz", "value": "quxA" }, - { "op": "add", "path": "/bar", "value": "quxB" }, - { "op": "add", "path": "/foo", "value": "quxC" } -] -""" - init = stages.Init() - init.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) - init.read_cfg() - init.initialize() - init.fetch() - init.instancify() - init.update() - init.cloudify().run( - "consume_data", - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE, - ) - mods = Modules(init) - (_which_ran, _failures) = mods.run_section("cloud_init_modules") - cfg = mods.cfg - assert "vendor_data" in cfg - assert cfg["baz"] == "quxA" - assert cfg["bar"] == "quxB" - assert cfg["foo"] == "quxC" - - -class TestConsumeUserDataHttp: - @responses.activate - @mock.patch("cloudinit.url_helper.time.sleep") - def test_include(self, mock_sleep, init_tmp): - """Test #include.""" - included_url = "http://hostname/path" - included_data = "#cloud-config\nincluded: true\n" - responses.add(responses.GET, included_url, included_data) - - init_tmp.datasource = FakeDataSource("#include\nhttp://hostname/path") - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset") as _reset: - init_tmp.consume_data() - assert _reset.call_count == 1 - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert cc.get("included") is True - - @responses.activate - @mock.patch("cloudinit.url_helper.time.sleep") - def test_include_bad_url(self, mock_sleep, init_tmp): - """Test #include with a bad URL.""" - bad_url = "http://bad/forbidden" - bad_data = "#cloud-config\nbad: true\n" - responses.add(responses.GET, bad_url, bad_data, status=403) - - included_url = "http://hostname/path" - included_data = "#cloud-config\nincluded: true\n" - responses.add(responses.GET, included_url, included_data) - - init_tmp.datasource = FakeDataSource( - "#include\nhttp://bad/forbidden\nhttp://hostname/path" - ) - init_tmp.fetch() - with pytest.raises(Exception, match="403"): - with mock.patch.object(init_tmp, "_reset") as _reset: - init_tmp.consume_data() - assert _reset.call_count == 1 - - with pytest.raises(FileNotFoundError): - util.load_text_file(init_tmp.paths.get_ipath("cloud_config")) - - @responses.activate - @mock.patch("cloudinit.url_helper.time.sleep") - @mock.patch("cloudinit.util.is_container") - @mock.patch( - "cloudinit.user_data.features.ERROR_ON_USER_DATA_FAILURE", False - ) - def test_include_bad_url_no_fail( - self, is_container, mock_sleep, tmpdir, init_tmp, caplog - ): - """Test #include with a bad URL and failure disabled""" - is_container.return_value = True - bad_url = "http://bad/forbidden" - responses.add( - responses.GET, - bad_url, - body="forbidden", - status=403, - ) - - included_url = "http://hostname/path" - included_data = "#cloud-config\nincluded: true\n" - responses.add(responses.GET, included_url, included_data) - - init_tmp.datasource = FakeDataSource( - "#include\nhttp://bad/forbidden\nhttp://hostname/path" - ) - init_tmp.fetch() - with mock.patch.object(init_tmp, "_reset") as _reset: - init_tmp.consume_data() - assert _reset.call_count == 1 - - assert ( - "403 Client Error: Forbidden for url: %s" % bad_url in caplog.text - ) - - cc_contents = util.load_text_file( - init_tmp.paths.get_ipath("cloud_config") - ) - cc = util.load_yaml(cc_contents) - assert cc.get("bad") is None - assert cc.get("included") is True - - -class TestUDProcess(helpers.ResourceUsingTestCase): - def test_bytes_in_userdata(self): - msg = b"#cloud-config\napt_update: True\n" - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) - message = ud_proc.process(msg) - self.assertTrue(count_messages(message) == 1) - - def test_string_in_userdata(self): - msg = "#cloud-config\napt_update: True\n" - - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) - message = ud_proc.process(msg) - self.assertTrue(count_messages(message) == 1) - - def test_compressed_in_userdata(self): - msg = gzip_text("#cloud-config\napt_update: True\n") - - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) - message = ud_proc.process(msg) - self.assertTrue(count_messages(message) == 1) - - -class TestConvertString(helpers.TestCase): - def test_handles_binary_non_utf8_decodable(self): - """Printable unicode (not utf8-decodable) is safely converted.""" - blob = b"#!/bin/bash\necho \xc3\x84\n" - msg = ud.convert_string(blob) - self.assertEqual(blob, msg.get_payload(decode=True)) - - def test_handles_binary_utf8_decodable(self): - blob = b"\x32\x32" - msg = ud.convert_string(blob) - self.assertEqual(blob, msg.get_payload(decode=True)) - - def test_handle_headers(self): - text = "hi mom" - msg = ud.convert_string(text) - self.assertEqual(text, msg.get_payload(decode=False)) - - def test_handle_mime_parts(self): - """Mime parts are properly returned as a mime message.""" - message = MIMEBase("text", "plain") - message.set_payload("Just text") - msg = ud.convert_string(str(message)) - self.assertEqual("Just text", msg.get_payload(decode=False)) - - -class TestFetchBaseConfig: - @pytest.fixture(autouse=True) - def mocks(self, mocker): - mocker.patch(f"{MPATH}.util.read_conf_from_cmdline") - mocker.patch(f"{MPATH}.read_runtime_config") - - def test_only_builtin_gets_builtin(self, mocker): - mocker.patch(f"{MPATH}.read_runtime_config", return_value={}) - mocker.patch(f"{MPATH}.util.read_conf_with_confd") - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - assert util.get_builtin_cfg() == config - - def test_conf_d_overrides_defaults(self, mocker): - builtin = util.get_builtin_cfg() - test_key = sorted(builtin)[0] - test_value = "test" - - mocker.patch( - f"{MPATH}.util.read_conf_with_confd", - return_value={test_key: test_value}, - ) - mocker.patch(f"{MPATH}.read_runtime_config", return_value={}) - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - assert config.get(test_key) == test_value - builtin[test_key] = test_value - assert config == builtin - - def test_confd_with_template(self, mocker, tmp_path: Path): - instance_data_path = tmp_path / "test_confd_with_template.json" - instance_data_path.write_text('{"template_var": "template_value"}') - cfg_path = tmp_path / "test_conf_with_template.cfg" - cfg_path.write_text('## template:jinja\n{"key": "{{template_var}}"}') - - mocker.patch("cloudinit.stages.CLOUD_CONFIG", cfg_path) - mocker.patch(f"{MPATH}.util.get_builtin_cfg", return_value={}) - config = stages.fetch_base_config( - DEFAULT_RUN_DIR, instance_data_file=instance_data_path - ) - assert config == {"key": "template_value"} - - def test_cmdline_overrides_defaults(self, mocker): - builtin = util.get_builtin_cfg() - test_key = sorted(builtin)[0] - test_value = "test" - cmdline = {test_key: test_value} - - mocker.patch(f"{MPATH}.util.read_conf_with_confd") - mocker.patch( - f"{MPATH}.util.read_conf_from_cmdline", - return_value=cmdline, - ) - mocker.patch(f"{MPATH}.read_runtime_config") - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - assert config.get(test_key) == test_value - builtin[test_key] = test_value - assert config == builtin - - def test_cmdline_overrides_confd_runtime_and_defaults(self, mocker): - builtin = {"key1": "value0", "key3": "other2"} - conf_d = {"key1": "value1", "key2": "other1"} - cmdline = {"key3": "other3", "key2": "other2"} - runtime = {"key3": "runtime3"} - - mocker.patch(f"{MPATH}.util.read_conf_with_confd", return_value=conf_d) - mocker.patch(f"{MPATH}.util.get_builtin_cfg", return_value=builtin) - mocker.patch(f"{MPATH}.read_runtime_config", return_value=runtime) - mocker.patch( - f"{MPATH}.util.read_conf_from_cmdline", - return_value=cmdline, - ) - - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - assert config == {"key1": "value1", "key2": "other2", "key3": "other3"} - - def test_order_precedence_is_builtin_system_runtime_cmdline(self, mocker): - builtin = {"key1": "builtin0", "key3": "builtin3"} - conf_d = {"key1": "confd1", "key2": "confd2", "keyconfd1": "kconfd1"} - runtime = {"key1": "runtime1", "key2": "runtime2"} - cmdline = {"key1": "cmdline1"} - - mocker.patch(f"{MPATH}.util.read_conf_with_confd", return_value=conf_d) - mocker.patch(f"{MPATH}.util.get_builtin_cfg", return_value=builtin) - mocker.patch( - f"{MPATH}.util.read_conf_from_cmdline", - return_value=cmdline, - ) - mocker.patch(f"{MPATH}.read_runtime_config", return_value=runtime) - - config = stages.fetch_base_config(DEFAULT_RUN_DIR) - - assert config == { - "key1": "cmdline1", - "key2": "runtime2", - "key3": "builtin3", - "keyconfd1": "kconfd1", - } diff --git a/.pc/no-single-process.patch/cloudinit/config/cc_mounts.py b/.pc/no-single-process.patch/cloudinit/config/cc_mounts.py index bf9d8032..c3998e57 100644 --- a/.pc/no-single-process.patch/cloudinit/config/cc_mounts.py +++ b/.pc/no-single-process.patch/cloudinit/config/cc_mounts.py @@ -514,9 +514,35 @@ def mount_if_needed( subp.subp(["systemctl", "daemon-reload"]) +def cleanup_fstab(ds_remove_entries: list = []) -> None: + if not os.path.exists(FSTAB_PATH): + return + + base_entry = [MNT_COMMENT] + + with open(FSTAB_PATH, "r") as f: + lines = f.readlines() + new_lines = [] + changed = False + for line in lines: + if all(entry in line for entry in [*base_entry, *ds_remove_entries]): + changed = True + continue + new_lines.append(line) + + # rewrite fstab + try: + if changed: + with open(FSTAB_PATH, "w") as f: + f.writelines(new_lines) + LOG.info("Removed resource disk entries from %s", FSTAB_PATH) + except Exception as e: + LOG.warning("Failed to clean resource disk entries from fstab: %s", e) + + def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: """Handle the mounts configuration.""" - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno uses_systemd = cloud.distro.uses_systemd() default_mount_options = ( "defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev" diff --git a/.pc/no-single-process.patch/cloudinit/config/schemas/schema-cloud-config-v1.json b/.pc/no-single-process.patch/cloudinit/config/schemas/schema-cloud-config-v1.json index c09e8fdd..5fe97a5f 100644 --- a/.pc/no-single-process.patch/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/.pc/no-single-process.patch/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -43,6 +43,7 @@ "power-state-change", "power_state_change", "puppet", + "raspberry_pi", "reset-rmc", "reset_rmc", "resizefs", @@ -493,6 +494,108 @@ } } }, + "ansible.pull": { + "oneOf": [ + { + "required": [ + "url", + "playbook_name" + ] + }, + { + "required": [ + "url", + "playbook_names" + ] + } + ], + "type": "object", + "additionalProperties": false, + "properties": { + "accept_host_key": { + "type": "boolean", + "default": false + }, + "clean": { + "type": "boolean", + "default": false + }, + "full": { + "type": "boolean", + "default": false + }, + "diff": { + "type": "boolean", + "default": false + }, + "ssh_common_args": { + "type": "string" + }, + "scp_extra_args": { + "type": "string" + }, + "sftp_extra_args": { + "type": "string" + }, + "private_key": { + "type": "string" + }, + "checkout": { + "type": "string" + }, + "module_path": { + "type": "string" + }, + "timeout": { + "type": "string" + }, + "url": { + "type": "string" + }, + "connection": { + "type": "string" + }, + "vault_id": { + "type": "string" + }, + "vault_password_file": { + "type": "string" + }, + "verify_commit": { + "type": "boolean", + "default": false + }, + "inventory": { + "type": "string" + }, + "module_name": { + "type": "string" + }, + "sleep": { + "type": "string" + }, + "tags": { + "type": "string" + }, + "skip_tags": { + "type": "string" + }, + "playbook_name": { + "deprecated": true, + "deprecated_version": "25.2", + "deprecated_description": "Use **playbook_names** key instead.", + "description": "Single playbook_name to run with ansible-pull", + "type": "string" + }, + "playbook_names": { + "type": "array", + "description": "List of playbook_names to run with ansible-pull", + "items": { + "type": "string" + } + } + } + }, "apt_configure.mirror": { "type": "array", "items": { @@ -866,85 +969,21 @@ "default": "ansible" }, "pull": { - "required": [ - "url", - "playbook_name" - ], - "type": "object", - "additionalProperties": false, - "properties": { - "accept_host_key": { - "type": "boolean", - "default": false - }, - "clean": { - "type": "boolean", - "default": false - }, - "full": { - "type": "boolean", - "default": false - }, - "diff": { - "type": "boolean", - "default": false - }, - "ssh_common_args": { - "type": "string" - }, - "scp_extra_args": { - "type": "string" - }, - "sftp_extra_args": { - "type": "string" - }, - "private_key": { - "type": "string" - }, - "checkout": { - "type": "string" - }, - "module_path": { - "type": "string" - }, - "timeout": { - "type": "string" - }, - "url": { - "type": "string" - }, - "connection": { - "type": "string" - }, - "vault_id": { - "type": "string" - }, - "vault_password_file": { - "type": "string" - }, - "verify_commit": { - "type": "boolean", - "default": false - }, - "inventory": { - "type": "string" - }, - "module_name": { - "type": "string" - }, - "sleep": { - "type": "string" - }, - "tags": { - "type": "string" - }, - "skip_tags": { - "type": "string" + "description": "pull playbooks from a VCS repo and run them on the host", + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/ansible.pull" + } }, - "playbook_name": { - "type": "string" + { + "$ref": "#/$defs/ansible.pull", + "deprecated": true, + "deprecated_version": "25.2", + "deprecated_description": "Expect **ansible.pull** as list of objects instead of a single object." } - } + ] } } } @@ -2029,7 +2068,7 @@ "minItems": 1, "maxItems": 6 }, - "description": "List of lists. Each inner list entry is a list of ``/etc/fstab`` mount declarations of the format: [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno ]. A mount declaration with less than 6 items will get remaining values from **mount_default_fields**. A mount declaration with only `fs_spec` and no `fs_file` mountpoint will be skipped.", + "description": "List of lists. Each inner list entry is a list of ``/etc/fstab`` mount declarations of the format: [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno ]. A mount declaration with less than 6 items will get remaining values from **mount_default_fields**. A mount declaration with only `fs_spec` and no `fs_file` mountpoint will be skipped.", "minItems": 1 }, "mount_default_fields": { @@ -2606,6 +2645,70 @@ } } }, + "cc_raspberry_pi": { + "type": "object", + "properties": { + "rpi": { + "type": "object", + "properties": { + "interfaces": { + "type": "object", + "properties": { + "spi": { + "type": "boolean", + "description": "Enable SPI interface. Default: ``false``.", + "default": false + }, + "i2c": { + "type": "boolean", + "description": "Enable I2C interface. Default: ``false``.", + "default": false + }, + "serial": { + "default": false, + "description": "Enable serial console. Default: ``false``.", + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "object", + "properties": { + "console": { + "type": "boolean", + "description": "Enable serial console. Default: ``false``.", + "default": false + }, + "hardware": { + "type": "boolean", + "description": "Enable UART hardware. Default: ``false``.", + "default": false + } + } + } + ] + }, + "onewire": { + "type": "boolean", + "description": "Enable 1-Wire interface. Default: ``false``.", + "default": false + }, + "remote_gpio": { + "type": "boolean", + "description": "Enable remote GPIO interface. Default: ``false``.", + "default": false + } + } + }, + "enable_rpi_connect": { + "type": "boolean", + "default": false, + "description": "Install and enable Raspberry Pi Connect. Default: ``false``." + } + } + } + } + }, "cc_rsyslog": { "type": "object", "properties": { @@ -3861,6 +3964,9 @@ { "$ref": "#/$defs/cc_puppet" }, + { + "$ref": "#/$defs/cc_raspberry_pi" + }, { "$ref": "#/$defs/cc_resizefs" }, @@ -4011,6 +4117,7 @@ "resize_rootfs": {}, "resolv_conf": {}, "rh_subscription": {}, + "rpi": {}, "rsyslog": {}, "runcmd": {}, "salt_minion": {}, diff --git a/.pc/no-single-process.patch/systemd/cloud-init-local.service.tmpl b/.pc/no-single-process.patch/systemd/cloud-init-local.service.tmpl index b123193a..26a6aee1 100644 --- a/.pc/no-single-process.patch/systemd/cloud-init-local.service.tmpl +++ b/.pc/no-single-process.patch/systemd/cloud-init-local.service.tmpl @@ -2,17 +2,18 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Local Stage (pre-network) -{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "rhel"] %} +{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} Wants=network-pre.target After=hv_kvp_daemon.service +Before=auditd.service Before=network-pre.target Before=shutdown.target {% if variant in ["almalinux", "cloudlinux", "rhel"] %} Before=firewalld.target {% endif %} -{% if variant in ["ubuntu", "unknown", "debian"] %} +{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} Before=sysinit.target {% endif %} Conflicts=shutdown.target diff --git a/.pc/no-single-process.patch/systemd/cloud-init-main.service.tmpl b/.pc/no-single-process.patch/systemd/cloud-init-main.service.tmpl index b80f324f..2ca6220c 100644 --- a/.pc/no-single-process.patch/systemd/cloud-init-main.service.tmpl +++ b/.pc/no-single-process.patch/systemd/cloud-init-main.service.tmpl @@ -8,7 +8,7 @@ # https://www.freedesktop.org/software/systemd/man/latest/systemd-remount-fs.service.html [Unit] Description=Cloud-init: Single Process -{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "rhel"] %} +{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} {% if variant in ["almalinux", "cloudlinux", "rhel"] %} diff --git a/.pc/no-single-process.patch/systemd/cloud-init-network.service.tmpl b/.pc/no-single-process.patch/systemd/cloud-init-network.service.tmpl index bdc7c8f8..61425b4a 100644 --- a/.pc/no-single-process.patch/systemd/cloud-init-network.service.tmpl +++ b/.pc/no-single-process.patch/systemd/cloud-init-network.service.tmpl @@ -2,7 +2,7 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Network Stage -{% if variant not in ["almalinux", "cloudlinux", "photon", "rhel"] %} +{% if variant not in ["almalinux", "cloudlinux", "photon", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} Wants=cloud-init-local.service @@ -12,12 +12,12 @@ After=cloud-init-local.service {% if variant not in ["ubuntu"] %} After=systemd-networkd-wait-online.service {% endif %} -{% if variant in ["ubuntu", "unknown", "debian"] %} +{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} After=networking.service {% endif %} {% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", - "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", - "suse", "TencentOS", "virtuozzo"] %} + "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "raspberry-pi-os", + "rhel", "rocky", "suse", "TencentOS", "virtuozzo"] %} After=NetworkManager.service After=NetworkManager-wait-online.service {% endif %} @@ -28,6 +28,9 @@ After=wicked.service After=dbus.service {% endif %} Before=network-online.target +{% if variant == "raspberry-pi-os" %} +Before=avahi-daemon.service +{% endif %} Before=sshd-keygen.service Before=sshd.service Before=systemd-user-sessions.service diff --git a/.pc/no-single-process.patch/tests/unittests/config/test_cc_mounts.py b/.pc/no-single-process.patch/tests/unittests/config/test_cc_mounts.py index 0e6d8379..549c59cf 100644 --- a/.pc/no-single-process.patch/tests/unittests/config/test_cc_mounts.py +++ b/.pc/no-single-process.patch/tests/unittests/config/test_cc_mounts.py @@ -241,7 +241,7 @@ def test_swap_creation_method_fallocate_on_xfs( m_kernel_version.return_value = (4, 20) m_get_mount_info.return_value = ["", "xfs"] - cc_mounts.handle(None, self.cc, self.mock_cloud, []) + cc_mounts.handle("", self.cc, self.mock_cloud, []) self.m_subp.assert_has_calls( [ mock.call( @@ -260,7 +260,7 @@ def test_swap_creation_method_xfs( m_kernel_version.return_value = (3, 18) m_get_mount_info.return_value = ["", "xfs"] - cc_mounts.handle(None, self.cc, self.mock_cloud, []) + cc_mounts.handle("", self.cc, self.mock_cloud, []) self.m_subp.assert_has_calls( [ mock.call( @@ -286,7 +286,7 @@ def test_swap_creation_method_btrfs( m_kernel_version.return_value = (4, 20) m_get_mount_info.return_value = ["", "btrfs"] - cc_mounts.handle(None, self.cc, self.mock_cloud, []) + cc_mounts.handle("", self.cc, self.mock_cloud, []) self.m_subp.assert_has_calls( [ mock.call(["truncate", "-s", "0", self.swap_path]), @@ -308,7 +308,7 @@ def test_swap_creation_method_ext4( m_kernel_version.return_value = (5, 14) m_get_mount_info.return_value = ["", "ext4"] - cc_mounts.handle(None, self.cc, self.mock_cloud, []) + cc_mounts.handle("", self.cc, self.mock_cloud, []) self.m_subp.assert_has_calls( [ mock.call( @@ -371,7 +371,7 @@ def test_no_fstab(self): "%s\tnone\tswap\tsw,comment=cloudconfig\t0\t0\n" % (self.swap_path,) ) - cc_mounts.handle(None, {}, self.mock_cloud, []) + cc_mounts.handle("", {}, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() assert fstab_expected_content == fstab_new_content @@ -431,7 +431,7 @@ def test_swap_creation_command(self, fstype, expected, mocker): cc = { "swap": {"filename": "/swap.img", "size": "512", "maxsize": "512"} } - cc_mounts.handle(None, cc, self.mock_cloud, []) + cc_mounts.handle("", cc, self.mock_cloud, []) assert self.m_subp.call_args_list == expected + [ mock.call(["mkswap", "/swap.img"]), mock.call(["swapon", "-a"]), @@ -452,7 +452,7 @@ def test_fstab_no_swap_device(self): with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab_original_content) - cc_mounts.handle(None, {}, self.mock_cloud, []) + cc_mounts.handle("", {}, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() @@ -470,7 +470,7 @@ def test_fstab_same_swap_device_already_configured(self): with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab_original_content) - cc_mounts.handle(None, {}, self.mock_cloud, []) + cc_mounts.handle("", {}, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() @@ -491,7 +491,7 @@ def test_fstab_alternate_swap_device_already_configured(self): with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab_original_content) - cc_mounts.handle(None, {}, self.mock_cloud, []) + cc_mounts.handle("", {}, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() @@ -510,7 +510,7 @@ def test_no_change_fstab_sets_needs_mount_all(self): cc = {"mounts": [["/dev/vdb", "/mnt", "auto", "defaults,noexec"]]} with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab_original_content) - cc_mounts.handle(None, cc, self.mock_cloud, []) + cc_mounts.handle("", cc, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() assert fstab_original_content == fstab_new_content.strip() @@ -555,7 +555,7 @@ def test_fstab_mounts_combinations(self): ["/dev/sda3", "/mnt4", "btrfs"], ] } - cc_mounts.handle(None, cfg, self.mock_cloud, []) + cc_mounts.handle("", cfg, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() diff --git a/.pc/strip-invalid-mtu.patch/cloudinit/features.py b/.pc/strip-invalid-mtu.patch/cloudinit/features.py new file mode 100644 index 00000000..63b35fcd --- /dev/null +++ b/.pc/strip-invalid-mtu.patch/cloudinit/features.py @@ -0,0 +1,152 @@ +# This file is part of cloud-init. See LICENSE file for license information. +""" +Feature flags are used as a way to easily toggle configuration +**at build time**. They are provided to accommodate feature deprecation and +downstream configuration changes. + +Currently used upstream values for feature flags are set in +``cloudinit/features.py``. Overrides to these values should be +patched directly (e.g., via quilt patch) by downstreams. + +Each flag should include a short comment regarding the reason for +the flag and intended lifetime. + +Tests are required for new feature flags, and tests must verify +all valid states of a flag, not just the default state. +""" +import re +import sys +from typing import Dict + +ERROR_ON_USER_DATA_FAILURE = True +""" +If there is a failure in obtaining user data (i.e., #include or +decompress fails) and ``ERROR_ON_USER_DATA_FAILURE`` is ``False``, +cloud-init will log a warning and proceed. If it is ``True``, +cloud-init will instead raise an exception. + +As of 20.3, ``ERROR_ON_USER_DATA_FAILURE`` is ``True``. + +(This flag can be removed after Focal is no longer supported.) +""" + + +ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES = False +""" +When configuring apt mirrors, if +``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``True`` cloud-init +will detect that a datasource's ``availability_zone`` property looks +like an EC2 availability zone and set the ``ec2_region`` variable when +generating mirror URLs; this can lead to incorrect mirrors being +configured in clouds whose AZs follow EC2's naming pattern. + +As of 20.3, ``ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES`` is ``False`` +so we no longer include ``ec2_region`` in mirror determination on +non-AWS cloud platforms. + +If the old behavior is desired, users can provide the appropriate +mirrors via :py:mod:`apt: ` +directives in cloud-config. +""" + + +EXPIRE_APPLIES_TO_HASHED_USERS = True +""" +If ``EXPIRE_APPLIES_TO_HASHED_USERS`` is True, then when expire is set true +in cc_set_passwords, hashed passwords will be expired. Previous to 22.3, +only non-hashed passwords were expired. + +(This flag can be removed after Jammy is no longer supported.) +""" + +NETPLAN_CONFIG_ROOT_READ_ONLY = True +""" +If ``NETPLAN_CONFIG_ROOT_READ_ONLY`` is True, then netplan configuration will +be written as a single root read-only file /etc/netplan/50-cloud-init.yaml. +This prevents wifi passwords in network v2 configuration from being +world-readable. Prior to 23.1, netplan configuration is world-readable. + +(This flag can be removed after Jammy is no longer supported.) +""" + + +NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH = True +""" +Append a forward slash '/' if NoCloud seedurl does not end with either +a querystring or forward slash. Prior to 23.1, nocloud seedurl would be used +unaltered, appending meta-data, user-data and vendor-data to without URL path +separators. + +(This flag can be removed when Jammy is no longer supported.) +""" + +APT_DEB822_SOURCE_LIST_FILE = True +""" +On Debian and Ubuntu systems, cc_apt_configure will write a deb822 compatible +/etc/apt/sources.list.d/(debian|ubuntu).sources file. When set False, continue +to write /etc/apt/sources.list directly. +""" + +MANUAL_NETWORK_WAIT = False +""" +On Ubuntu systems, cloud-init-network.service will start immediately after +cloud-init-local.service and manually wait for network online when necessary. +If False, rely on systemd ordering to ensure network is available before +starting cloud-init-network.service. + +Note that in addition to this flag, downstream patches are also likely needed +to modify the systemd unit files. +""" + +STRIP_INVALID_MTU = False +""" +If ``STRIP_INVALID_MTU`` is True, then cloud-init will strip invalid MTU +values from rendered v2 netplan configuration. Cloud-init allowed these values +prior to 24.2, so this flag is used to maintain compatibility with +previously generated network configurations. + +(This flag can be removed when Noble is no longer supported.) +""" + +DEPRECATION_INFO_BOUNDARY = "24.1" +""" +DEPRECATION_INFO_BOUNDARY is used by distros to configure at which upstream +version to start logging deprecations at a level higher than INFO. + +The default value "devel" tells cloud-init to log all deprecations higher +than INFO. This value may be overriden by downstreams in order to maintain +stable behavior across releases. + +Jsonschema key deprecations and inline logger deprecations include a +deprecated_version key. When the variable below is set to a version, +cloud-init will use that version as a demarcation point. Deprecations which +are added after this version will be logged as at an INFO level. Deprecations +which predate this version will be logged at the higher DEPRECATED level. +Downstreams that want stable log behavior may set the variable below to the +first version released in their stable distro. By doing this, they can expect +that newly added deprecations will be logged at INFO level. The implication of +the different log levels is that logs at DEPRECATED level result in a return +code of 2 from `cloud-init status`. + +This may may also be used in some limited cases where new error messages may be +logged which increase the risk of regression in stable downstreams where the +error was previously unreported yet downstream users expected stable behavior +across new cloud-init releases. + +format: + + :: = | + ::= "devel" + ::= "." ["." ] + +where , , and are positive integers +""" + + +def get_features() -> Dict[str, bool]: + """Return a dict of applicable features/overrides and their values.""" + return { + k: getattr(sys.modules["cloudinit.features"], k) + for k in sys.modules["cloudinit.features"].__dict__.keys() + if re.match(r"^[_A-Z0-9]+$", k) + } diff --git a/ChangeLog b/ChangeLog index 9440238e..a9f62d02 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,230 @@ +25.2 + - fix: Ensure 822 template renders correctly on Debian (#6381) (GH: 6380) + - test: support systemctl try-reload-or-restart messaging alternatives + (#6377) + - test: increase 2nd disk size for disk setup tests (#6376) + - feat(clean): Add a new clean option to clean fstab entries (#6348) + [Ani Sinha] + - test: handle TMPDIR != "/tmp" (#6356) [Dan Bungert] + - fix: add openeuler hosts template to avoid cloud-init service failures + (#6328) [xqs] + - test: increase timeout on test_clean_package_install (#6362) + - test: relax checks on authorized keys (#6361) + - fix: Correct v2 ENI route and dns rendering (#6331) + - test: drop citestcase from i sources (#6346) + - test: drop citestcase from g and h sources (#6344) + - test: drop citestcase from c and d sources (#6336) + - test: drop citestcase from al* sources tests (#6318) + - disk_setup: Optionally use 'sfdisk' for GPT [Vitaly Kuznetsov] (GH: 5797) + - disk_setup: Fully support full GPT partition GUIDs [Vitaly Kuznetsov] + - fix(ENI): render keys with hyphens (#6333) (GH: 5234) + - fix: fixed defs types in cloudinit analyze dump (#6343) [abdulganiyy] + - feat(networkd): support vlan and bond rendering (#6324) + [Shreenidhi Shedi] + - fix(analyze-boot): use monotonic clock for containers #6322 + [Mostafa Abdelwahab] (GH: 5773) + - clean: do not attempt to find datasource when cache has been cleaned + (#6325) [Ani Sinha] + - test: drop citestcase from azure_helper tests (#6335) + - chore: remove unused target arg from Activator.available + - chore: add typing to cloudinit.net.eni + - feat(azure): Implement the clean callback for DataSourceAzure (#6321) + [Amy Chen] + - feat(oracle): set keep_configuration to true for iscsi instances [a-dubs] + - feat(network): add keep_configuration setting to v1 [a-dubs] + - doc: Document how to wait for cloud-init + - doc: Document changed systemd order in 24.4 + - fixed defs types in cloudinit analyze (#6308) [abdulganiyy] + - ci: downgrade LXD to fix integration tests (#6340) + - test: drop citestcase from azure source test (#6320) + - fix: catch and log exception during pip upgrade in ansible pull (#6301) + [Mostafa Abdelwahab] (GH: 6074) + - test: drop citestcase from some config tests + - fix: GCE datasource query issue (#3398) (#6279) + [ludovictual-system-u] (GH: 3398) + - test: drop citestcase from distro tests (#6317) + - test: drop citestcase from rest of config tests (#6314) + - ci: update patch checking to use current supported releases (#6315) + - test: drop FilesystemMockingTestCase (#6294) (GH: 5760) + - test: ensure image setup only runs once, even for xdist + - test: turn reaper into a fixture + - test: refactor obtaining session args into new function + - test: simplify passing of lxd_setup callback + - test: drop citestcase from analyze tests (#6312) + - test: drop FilesystemMockingTestCase in netconfig (#6290) + - test: expose tests.unittests.ditros._get_distro (#6290) + - doc: detailed examples of openstack config alternatives for non-x86 + images (#6303) + - test: drop FilesystemMockingTestCase in smartos (#6291) + - test: drop some FilesystemMockingTestCases (#6284) + - docs: clarify what #include can include (#6309) + - doc(ssdlc): Add hardening page (#6297) + - docs: Document network device table and limitations (#6187) + [Bryan Fraschetti] + - fix: allow downstreams to strip invalid MTU (#6246) (GH: 6239) + - test: drop FilesystemMockingTestCase from cc tests (#6282) + - docs: use correct jinja content-type (#6296) (GH: 6027) + - test: drop ResponsesTestCase (#6277) + - test: Consolidate use of paths fixture (#6289) + - test: pytestify test_vmware.py (#6288) [Mostafa Abdelwahab] + - feat(ca_certs): Add CentOS support (#6287) [Kees Bakker] + - feat(wsl): Reuse metadata as Landscape installation_request_id (#6200) + [Carlos Nihelton] + - test: drop / pytestify ResourceUsingTestCase (#6276) + - docs(mounts): Correct fs-freq reference to fs_freq (#6262) + [ibrahim-mojalled] (GH: 6210) + - test: pytestify ds_identify tests (#6274) + - test: pytestify t/u/test_log.py (#6275) + - feat(hetzner): integrate private networks metadata (#6224) + [Thomas Boerger] + - test: pytestify cloudinit.s.helpers.netlink tests (#6273) + - Release 25.1.4 + - Release 25.1.3 + - fix: strict disable in ds-identify on no datasources found (LP: #2069607) + - test: add integration test to install cloud-init from clean state (#6260) + - docs: provide example3 for PAM and ssh_pwauth behavior (#27) + - fix: Make hotplug socket writable only by root (#25) + - fix: Don't attempt to identify non-x86 OpenStack instances (LP: #2069607) + - doc: update discourse release schedule link (#6263) + - doc: update discourse link + - chore: log URL when retrying 503s (#6264) + - test: add option to keep instance on error (#6241) + - eni.py: ensure that a "dns" entry is not rendered in interfaces file + (#6253) [dermotbradley] + - feat(azure): improve handling for reading VM ID (#6199) [Chris Patterson] + - fix: remove unnecessary 'Wants' from cloud-init-main.service (#6255) + - test: use latest version of LXD in integration tests (#6249) + - docs: include missing --seed flags in clean CLI (#6244) [Faizan Alam] + - ec2: Improve metadata retrieval by iterating all interfaces (#6233) + [yukariatlas] (GH: 6232) + - feat: Add Raspberry Pi OS support (#5827) [Paul] + - feat(azure): add interface to dhcp_log_func (#6238) [Chris Patterson] + - fix: fix untyped-defs on tests/unittests and cloudinit/sources (#6230) + [Ritvikj23] + - test: pre-fetch instance id when logging from reaper (#6234) + - feat: Add subnet metric support for netplan (#6222) [Artsiom] + - test: remove FilesystemMockingTestCase from test_cc_apk_configure.py + (#6226) + - fix: make 'cloud-init --all-stages' work interactively (#6211) + - test: remove FilesystemMockingTestCase from test_util.py (#6220) + - test: remove FilesystemMockingTestCase from test_dmi.py (#6219) + - docs: merge "Availability" and "Supported distros" pages (#6217) + [Jacob C. Chin] + - fix: ansible-pull multiple playbooks on older ansible ver < 2.12 (#6218) + - docs: remove monospace formatting from SSH section heading (#6215) + [Chad Dougherty] + - fix: no traceback on command line missing subcommand (#6214) + [Robert Schweikert] + - feat: Change ansible pull module type from dict to list (#6010) + [Amirhossein Shaerpour] + - test(apt): add questing versiong for hello pkg (#6213) + - docs: Use Noble for examples instead of Jammy (#6209) [Aarni Koskela] + - feat(ca_certs): add rocky to rhel distro overrides (#6208) + [Lukas Friedhoff] + - docs: ensure proper 'sudo' representation (#6196) (GH: 6195) + - fix(net): ignore udevadm failures when enumerating nics (#6185) + [Chris Patterson] + - Release 25.1.2 (#6197) + - revert "chore: Deprecate partially supported system config (#5515)" + (LP: #2100232) + - fix: copr builds of CentOS9 require CRB and baseurl in centos.repo + (#6192) + - chore: add fedora package build support for run-container and read-deps + (#6174) + - test: drop fixed xfail tests + - fix: simplify MAAS check logic + - feat(azure): allow unspecified user name (#6177) [Chris Patterson] + - fix: stop reporting error if cloud-init receives signal (#6159) + (GH: 6151) + - Fix: Add Ephemeral Network for CloudStackLocal DS (#6144) + [Bryan Fraschetti] (GH: 6143) + - docs: clarify examples for network addresses/gateway (#6186) + [Dan Bungert] + - fix: setup.py doesn't match AmazonLinux CPE 2.2 releases (#6173) + - fix: ensure MAAS datasource retries on failure (#6167) (LP: #2106671) + - cloud.cfg.tmpl: do not enable cc_reset_rmc for Alpine Linux (#6170) + [dermotbradley] + - tests: ibm avoid schema validation for DataSourceNone on ibm (#6168) + - tests: ibm fix apt and ds-id testing (#6168) + - test: fix integration test on new lxd versions (#6164) + - tests: ibm correct logged message. Invalid schema: vendor-data (#6163) + - tests: ibm expect invalid vendor-data in stderr (#6160) + - tests: ibm expect invalid vendor-data in stderr (#6158) + - tests: fix ibm expected warnings on invalid vendor-data schema (#6157) + - fix: drop udev remove action in hotplug (#6152) [yukariatlas] + - chore: remove reference to refresh_rmc_and_interface module (#6156) + [Ani Sinha] + - chore: reorder iface filters & log on inherited MAC (#6140) + [Wesley Hershberger] + - fix(azure): update ReportableErrorUnhandledException (#6133) + [Ksenija Stanojevic] + - chore: make auditd wait for cloud-init-local.service (#6138) + [Robert Schweikert] + - chore: allow custom pkg-config binary path (#6118) + [Alexandre Burgoni] (GH: 6099) + - fix: rename "reload-or-try-restart" to "try-reload-or-restart" (#6142) + [sxt1001] + - chore: make lint interpreter configurable (#6121) [Robert Schweikert] + - fix: ensure system sshd_config is not overwritten (#6105) [Sludge] + - fix(oracle): properly detect ipv6 only for private ULA addresses (#6123) + [Alec Warren] + - feat(oracle): downgrade warning log about falling back to imds (#6134) + [Alec Warren] + - Release 25.1.1 (#6120) + - chore: remove remaining references to .github-cla-signers (#6116) + - fix(cli): wrong usage output when invalid arg in subcommand (#6115) + [Dillon] (GH: 4609) + - fix(Azure): don't reraise FileNotFoundError during ephemeral setup + (#6113) + - fix(azure): handle unexpected exceptions during obtain_lease() (#6092) + [Ksenija Stanojevic] + - feat: add callback for datasources to clean config changes (#6100) + [Ani Sinha] + - chore: Fix untyped-defs on tests/unittests/config (#6104) [Vlad Apostol] + - chore: switch to has-signed-canonical-cla GH action (#6109) + - Allow to set mac_address for VLAN subinterface (#6081) + [jumpojoy] (GH: 5364) + - change retry sleep for wireserver (#6107) [Ksenija Stanojevic] + - test: pytestify cc_chef tests, add migration test + - chef + - chef: migrate files in old config directories for backups and cache + - fix: correct the path for Chef's backups (#5994) + - test: replace version check with has_netplanlib() (#6106) + - feat(vmware): Support network events (#6063) [Andrew Kutz] + - test: correctly mock fallback nic in openstack tests (#6101) + - fix: Remove erroneous EC2 reference from 503 warning (#6077) + - test: update keys_to_console timeout (#6087) + - test: move to has_netplanlib() in test_networking.py (#6089) + - fix: NM reload and bring up individual network conns (#6073) [Ani Sinha] + - ci(oracle): fix issue installing cloud-init on custom image creation + (#6084) [Alec Warren] + - test: decouple netplan integrations from libnetplan SRU (#6085) + - test: warning on users/groups test is version specific (#6078) + - test: fix errors in custom datasource networking (#6076) + - test: ensure software-properties-common properly removed (#6080) + - test: remove script death check from test_signal_handler.py (#6079) + - test: ensure NoCloud networking works on plucky (#6072) + - fix: stop warning on dual-stack request failure (#6044) + - test: Add integration test for signal handling warnings/errors (#6037) + - feat(azure): format ReportableErrorImdsInvalidMetadata without repr + (#6052) [Ksenija Stanojevic] + - docs: v1.cloud_name section typo (#6070) [Jack Bernhardt] + - fix: install_method: pip cannot find ansible-pull command path (#6021) + [Hasan Aliyev] (GH: 5720) + - fix: Fix DataSourceAliYun exception_cb signature (#6068) (GH: 6066) + - fix: Update OauthUrlHelper to use readurl exception_cb signature + (GH: 6065) + - test: add OauthUrlHelper tests + - test: Remove CiTestCase from test_url_helper.py + - test: pytestify test_url_helper.py + - test: check for correct logrotate config (#6060) + - test: don't fail test if ppa has no uploads (#6059) + - test: make users/groups warning test release aware (#6056) + - fix: when get_session_cloud() fails, _SESSION_CLOUD isn't set (#6051) + - chore: Sort .gitignore + - chore: Add new entries to gitignore and glob more + - fix: track more removed modules (#6043) + 25.1.4 - fix: disable cloud-init when non-x86 environments have no DMI-data and no strict datasources detected (LP: #2069607) (CVE-2024-6174) diff --git a/cloudinit/analyze/__init__.py b/cloudinit/analyze/__init__.py index 24dfa936..60ff8daa 100644 --- a/cloudinit/analyze/__init__.py +++ b/cloudinit/analyze/__init__.py @@ -6,13 +6,15 @@ import re import sys from datetime import datetime, timezone -from typing import IO +from typing import IO, Dict, List, Optional, Tuple, Union from cloudinit.analyze import dump, show from cloudinit.atomic_helper import json_dumps -def get_parser(parser=None): +def get_parser( + parser: Optional[argparse.ArgumentParser] = None, +) -> argparse.ArgumentParser: if not parser: parser = argparse.ArgumentParser( prog="cloudinit-analyze", @@ -113,7 +115,7 @@ def get_parser(parser=None): return parser -def analyze_boot(name, args): +def analyze_boot(name: str, args: argparse.Namespace) -> int: """Report a list of how long different boot operations took. For Example: @@ -138,10 +140,10 @@ def analyze_boot(name, args): e for e in _get_events(infh) if e["name"] == "init-local" - and "starting search" in e["description"] + and "starting search" in str(e["description"]) ][-1] - ci_start = datetime.fromtimestamp( - last_init_local["timestamp"], timezone.utc + ci_start: Union[datetime, str] = datetime.fromtimestamp( + float(last_init_local["timestamp"]), timezone.utc ) except IndexError: ci_start = "Could not find init-local log-line in cloud-init.log" @@ -200,7 +202,7 @@ def analyze_boot(name, args): return status_code -def analyze_blame(name, args): +def analyze_blame(name, args: argparse.Namespace) -> None: """Report a list of records sorted by largest time delta. For example: @@ -227,7 +229,7 @@ def analyze_blame(name, args): clean_io(infh, outfh) -def analyze_show(name, args): +def analyze_show(name, args: argparse.Namespace) -> None: """Generate output records using the 'standard' format to printing events. Example output follows: @@ -264,14 +266,14 @@ def analyze_show(name, args): clean_io(infh, outfh) -def analyze_dump(name, args): +def analyze_dump(name, args: argparse.Namespace) -> None: """Dump cloud-init events in json format""" infh, outfh = configure_io(args) outfh.write(json_dumps(_get_events(infh)) + "\n") clean_io(infh, outfh) -def _get_events(infile): +def _get_events(infile: IO) -> List[Dict[str, Union[str, float]]]: rawdata = None events, rawdata = show.load_events_infile(infile) if not events: @@ -279,7 +281,7 @@ def _get_events(infile): return events -def configure_io(args): +def configure_io(args: argparse.Namespace) -> Tuple[IO, IO]: """Common parsing and setup of input/output files""" if args.infile == "-": infh = sys.stdin diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py index 55d149c4..dd1b9c32 100644 --- a/cloudinit/analyze/dump.py +++ b/cloudinit/analyze/dump.py @@ -3,10 +3,11 @@ import calendar import sys from datetime import datetime, timezone +from typing import IO, Any, Dict, List, Optional, TextIO, Tuple from cloudinit import atomic_helper, subp, util -stage_to_description = { +stage_to_description: Dict[str, str] = { "finished": "finished running cloud-init", "init-local": "starting search for local datasources", "init-network": "searching for network datasources", @@ -27,7 +28,7 @@ DEFAULT_FMT = "%b %d %H:%M:%S %Y" -def parse_timestamp(timestampstr): +def parse_timestamp(timestampstr: str) -> float: # default syslog time does not include the current year months = [calendar.month_abbr[m] for m in range(1, 13)] if timestampstr.split()[0] in months: @@ -54,7 +55,7 @@ def parse_timestamp(timestampstr): return float(timestamp) -def has_gnu_date(): +def has_gnu_date() -> bool: """GNU date includes a string containing the word GNU in it in help output. Posix date does not. Use this to indicate on Linux systems without GNU date that the extended parsing is not @@ -63,7 +64,7 @@ def has_gnu_date(): return "GNU" in subp.subp(["date", "--help"]).stdout -def parse_timestamp_from_date(timestampstr): +def parse_timestamp_from_date(timestampstr: str) -> float: if not util.is_Linux() and subp.which("gdate"): date = "gdate" elif has_gnu_date(): @@ -77,7 +78,7 @@ def parse_timestamp_from_date(timestampstr): ) -def parse_ci_logline(line): +def parse_ci_logline(line: str) -> Optional[Dict[str, Any]]: # Stage Starts: # Cloud-init v. 0.7.7 running 'init-local' at \ # Fri, 02 Sep 2016 19:28:07 +0000. Up 1.0 seconds. @@ -163,7 +164,10 @@ def parse_ci_logline(line): return event -def dump_events(cisource=None, rawdata=None): +def dump_events( + cisource: Optional[IO[str]] = None, + rawdata: Optional[str] = None, +) -> Tuple[List[Dict[str, Any]], List[str]]: events = [] event = None CI_EVENT_MATCHES = ["start:", "finish:", "Cloud-init v."] @@ -171,9 +175,9 @@ def dump_events(cisource=None, rawdata=None): if not any([cisource, rawdata]): raise ValueError("Either cisource or rawdata parameters are required") - if rawdata: + if rawdata is not None: data = rawdata.splitlines() - else: + elif cisource is not None: data = cisource.readlines() for line in data: @@ -189,9 +193,9 @@ def dump_events(cisource=None, rawdata=None): return events, data -def main(): +def main() -> str: if len(sys.argv) > 1: - cisource = open(sys.argv[1]) + cisource: TextIO = open(sys.argv[1]) else: cisource = sys.stdin diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index e9a8c7a1..b4491bca 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -121,11 +121,17 @@ class SystemctlReader: """ def __init__(self, property, parameter=None): - self.epoch = None + self.stdout = None self.args = [subp.which("systemctl"), "show"] if parameter: self.args.append(parameter) - self.args.extend(["-p", property]) + # --timestamp=utc is needed for native date strings. Othwerise, + # the datetime will be returned in the local timezone (which would be + # a problem for strptime used later on in this method) + # This option does not affect monotonic properties (values as + # microsecond int) + self.args.extend(["-p", property, "--timestamp=us+utc"]) + # Don't want the init of our object to break. Instead of throwing # an exception, set an error code that gets checked when data is # requested from the object @@ -142,12 +148,12 @@ def subp(self): value, err = subp.subp(self.args, capture=True) if err: return err - self.epoch = value + self.stdout = value return None except Exception as systemctl_fail: return systemctl_fail - def parse_epoch_as_float(self): + def convert_val_to_float(self): """ If subp call succeeded, return the timestamp from subp as a float. @@ -162,10 +168,27 @@ def parse_epoch_as_float(self): "returning error code ({})".format(self.failure) ) # Output from systemctl show has the format Property=Value. - # For example, UserspaceMonotonic=1929304 - timestamp = self.epoch.split("=")[1] - # Timestamps reported by systemctl are in microseconds, converting - return float(timestamp) / 1000000 + val = self.stdout.split("=")[1].strip() + + if val.isnumeric(): + # Float Timestamps reported by systemctl are in + # microseconds, converting to seconds + # For example, UserspaceMonotonic=1929304 + timestamp = float(val) / 1000000 + else: + # The format in this case is always "%a %Y-%m-%d %H:%M:%S %Z" + # For example, UserspaceTimestamp=Wed 2025-07-30 05:14:32 UTC + + # strptime returns a naive datetime so we need to explictly + # set the timezone of this datetime + # at the timezone of the parsed string (utc) + timestamp = ( + datetime.datetime.strptime(val, "%a %Y-%m-%d %H:%M:%S.%f %Z") + .replace(tzinfo=datetime.timezone.utc) + .timestamp() + ) + + return timestamp def dist_check_timestamp(): @@ -226,24 +249,40 @@ def gather_timestamps_using_systemd(): Gather timestamps that corresponds to kernel begin initialization, kernel finish initialization. and cloud-init systemd unit activation - :return: the three timestamps + :return: the three timesread_propertystamps """ - kernel_start = float(time.time()) - float(util.uptime()) try: - delta_k_end = SystemctlReader( - "UserspaceTimestampMonotonic" - ).parse_epoch_as_float() - delta_ci_s = SystemctlReader( - "InactiveExitTimestampMonotonic", "cloud-init-local" - ).parse_epoch_as_float() - base_time = kernel_start - status = SUCCESS_CODE - # lxc based containers do not set their monotonic zero point to be when - # the container starts, instead keep using host boot as zero point + # The use of the monotonic timestamps is needed in cloud-init-related + # dates to account for the 2-second delay when cloud-init sets up NTP if util.is_container(): - status = CONTAINER_CODE - kernel_end = base_time + delta_k_end - cloudinit_sysd = base_time + delta_ci_s + # lxc based containers do not set their monotonic zero point to be + # when the container starts, + # instead keep using host boot as zero point + kernel_start = SystemctlReader( + "UserspaceTimestamp" + ).convert_val_to_float() + monotonic_offset = SystemctlReader( + "UserspaceTimestampMonotonic" + ).convert_val_to_float() + else: + kernel_start = SystemctlReader( + "KernelTimestamp" + ).convert_val_to_float() + monotonic_offset = SystemctlReader( + "KernelTimestampMonotonic" + ).convert_val_to_float() + kernel_end = ( + SystemctlReader( + "UserspaceTimestampMonotonic" + ).convert_val_to_float() + - monotonic_offset + ) + cloudinit_sysd = ( + SystemctlReader( + "InactiveExitTimestampMonotonic", "cloud-init-local" + ).convert_val_to_float() + - monotonic_offset + ) except Exception as e: # Except ALL exceptions as Systemctl reader can throw many different @@ -251,7 +290,15 @@ def gather_timestamps_using_systemd(): # obtained print(e) return TIMESTAMP_UNKNOWN - return status, kernel_start, kernel_end, cloudinit_sysd + + status = CONTAINER_CODE if util.is_container() else SUCCESS_CODE + + return ( + status, + kernel_start, + kernel_start + kernel_end, + kernel_start + cloudinit_sysd, + ) def generate_records( diff --git a/cloudinit/cmd/clean.py b/cloudinit/cmd/clean.py index 1d61c11d..1ad32905 100755 --- a/cloudinit/cmd/clean.py +++ b/cloudinit/cmd/clean.py @@ -8,10 +8,12 @@ import argparse import glob +import logging import os import sys -from cloudinit import settings +from cloudinit import settings, sources +from cloudinit.config import cc_mounts from cloudinit.distros import uses_systemd from cloudinit.log import log_util from cloudinit.net.netplan import CLOUDINIT_NETPLAN_FILE @@ -25,6 +27,7 @@ write_file, ) +LOG = logging.getLogger(__name__) ETC_MACHINE_ID = "/etc/machine-id" GEN_NET_CONFIG_FILES = [ CLOUDINIT_NETPLAN_FILE, @@ -96,6 +99,8 @@ def get_parser(parser=None): "all", "ssh_config", "network", + "datasource", + "fstab", ], default=[], nargs="+", @@ -115,7 +120,7 @@ def remove_artifacts(init, remove_logs, remove_seed=False, remove_config=None): @param: remove_seed: Boolean. Set True to also delete seed subdir in paths.cloud_dir. @param: remove_config: List of strings. - Can be any of: all, network, ssh_config. + Can be any of: all, network, ssh_config, datasource, fstab. @returns: 0 on success, 1 otherwise. """ init.read_cfg() @@ -131,9 +136,31 @@ def remove_artifacts(init, remove_logs, remove_seed=False, remove_config=None): ): for conf in GEN_SSH_CONFIG_FILES: del_file(conf) + if remove_config and set(remove_config).intersection(["all", "fstab"]): + cc_mounts.cleanup_fstab() + + clean_datasource = remove_config and set(remove_config).intersection( + ["all", "datasource"] + ) if not os.path.isdir(init.paths.cloud_dir): + log_util.multi_log( + "Artifacts already cleaned.", + log=LOG, + log_level=logging.INFO, + ) return 0 # Artifacts dir already cleaned + + if clean_datasource: + try: + init.fetch().clean() + except sources.DataSourceNotFoundException: + log_util.multi_log( + "No datasource found, nothing cleaned.", + log=LOG, + log_level=logging.INFO, + ) + seed_path = os.path.join(init.paths.cloud_dir, "seed") for path in glob.glob("%s/*" % init.paths.cloud_dir): if path == seed_path and not remove_seed: diff --git a/cloudinit/cmd/devel/hotplug_hook.py b/cloudinit/cmd/devel/hotplug_hook.py index 168387d4..a7bcc2ff 100755 --- a/cloudinit/cmd/devel/hotplug_hook.py +++ b/cloudinit/cmd/devel/hotplug_hook.py @@ -68,7 +68,7 @@ def get_parser(parser=None): "--udevaction", required=True, help="Specify action to take.", - choices=["add", "remove"], + choices=["add"], ) subparsers.add_parser( @@ -103,8 +103,6 @@ def detect_hotplugged_device(self): detect_presence = None if self.action == "add": detect_presence = True - elif self.action == "remove": - detect_presence = False else: raise ValueError("Unknown action: %s" % self.action) @@ -146,11 +144,6 @@ def apply(self): raise RuntimeError( "Failed to bring up device: {}".format(self.devpath) ) - elif self.action == "remove": - if not activator.bring_down_interface(interface_name): - raise RuntimeError( - "Failed to bring down device: {}".format(self.devpath) - ) @property def config(self): diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 207b97cf..d8d1d80c 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -79,6 +79,38 @@ LOG = logging.getLogger(__name__) +class SubcommandAwareArgumentParser(argparse.ArgumentParser): + def parse_args(self, args=None, namespace=None): + """Override parse_args to store raw arguments for error handling.""" + self._raw_args = args + return super().parse_args(args, namespace) + + def error(self, message): + """Override error method to show subcommand usage if applicable.""" + print(f"error: {message}\n", file=sys.stderr) + + # Scan for the first valid subcommand + + if not hasattr(self, "_raw_args"): + self._raw_args = sys.argv[1:] + subcommand = None + if self._raw_args: + for arg in self._raw_args: + if arg in self._subparsers._group_actions[0].choices: + subcommand = arg + break + # Check if the subcommand exists and show its help + + if subcommand: + subparser = self._subparsers._group_actions[0].choices[subcommand] + subparser.print_help( + file=sys.stderr + ) # Print subcommand help to stderr + else: + self.print_help(file=sys.stderr) + sys.exit(2) + + # Used for when a logger may not be active # and we still want to print exceptions... def print_exc(msg=""): @@ -1035,7 +1067,7 @@ def main(sysv_args=None): loggers.configure_root_logger() if not sysv_args: sysv_args = sys.argv - parser = argparse.ArgumentParser(prog=sysv_args.pop(0)) + parser = SubcommandAwareArgumentParser(prog=sysv_args.pop(0)) # Top level args parser.add_argument( @@ -1269,7 +1301,7 @@ def main(sysv_args=None): args = parser.parse_args(args=sysv_args) setattr(args, "skip_log_setup", False) if not args.all_stages: - return sub_main(args) + return sub_main(args, parser) return all_stages(parser) @@ -1289,7 +1321,7 @@ def all_stages(parser): args = parser.parse_args(args=["init", "--local"]) args.skip_log_setup = False # run local stage - sync.systemd_exit_code = sub_main(args) + sync.systemd_exit_code = sub_main(args, parser) # wait for cloud-init-network.service to start with sync("network"): @@ -1297,7 +1329,7 @@ def all_stages(parser): args = parser.parse_args(args=["init"]) args.skip_log_setup = True # run init stage - sync.systemd_exit_code = sub_main(args) + sync.systemd_exit_code = sub_main(args, parser) # wait for cloud-config.service to start with sync("config"): @@ -1305,7 +1337,7 @@ def all_stages(parser): args = parser.parse_args(args=["modules", "--mode=config"]) args.skip_log_setup = True # run config stage - sync.systemd_exit_code = sub_main(args) + sync.systemd_exit_code = sub_main(args, parser) # wait for cloud-final.service to start with sync("final"): @@ -1313,7 +1345,7 @@ def all_stages(parser): args = parser.parse_args(args=["modules", "--mode=final"]) args.skip_log_setup = True # run final stage - sync.systemd_exit_code = sub_main(args) + sync.systemd_exit_code = sub_main(args, parser) # signal completion to cloud-init-main.service if sync.experienced_any_error: @@ -1332,10 +1364,19 @@ def all_stages(parser): socket.sd_notify("STOPPING=1") -def sub_main(args): +def sub_main(args, parser): - # Subparsers.required = True and each subparser sets action=(name, functor) - (name, functor) = args.action + try: + # Subparsers.required = True + # and each subparser sets action=(name, functor) + (name, functor) = args.action + except AttributeError: + parser.print_usage() + sys.stderr.write( + "\nNo Subcommand specified. Please specify a subcommand in" + " addition to the option" + ) + sys.exit(1) # Setup basic logging for cloud-init: # - for cloud-init stages if --debug diff --git a/cloudinit/config/cc_ansible.py b/cloudinit/config/cc_ansible.py index 870c8823..66c6f425 100644 --- a/cloudinit/config/cc_ansible.py +++ b/cloudinit/config/cc_ansible.py @@ -7,9 +7,9 @@ import sys import sysconfig from copy import deepcopy -from typing import Optional +from typing import List, Optional -from cloudinit import lifecycle, signal_handler, subp +from cloudinit import lifecycle, subp from cloudinit.cloud import Cloud from cloudinit.config import Config from cloudinit.config.schema import MetaSchema @@ -63,8 +63,7 @@ def do_as(self, command: list, **kwargs): return self.distro.do_as(command, self.run_user, **kwargs) def subp(self, command, **kwargs): - with signal_handler.suspend_crash(): - return subp.subp(command, update_env=self.env, **kwargs) + return subp.subp(command, update_env=self.env, **kwargs) @abc.abstractmethod def is_installed(self): @@ -127,8 +126,27 @@ def install(self, pkg_name: str): if self.run_user: cmd.append("--user") - self.do_as([*cmd, "--upgrade", "pip"]) + self.__upgrade_pip(cmd) + LOG.info("Installing the %s package", pkg_name) self.do_as([*cmd, pkg_name]) + LOG.info("Installed the %s package", pkg_name) + + def __upgrade_pip(self, cmd: list): + """Try to upgrade pip then not raise an exception + if the pip upgrade fails""" + LOG.info("Upgrading pip") + try: + self.do_as([*cmd, "--upgrade", "pip"]) + except subp.ProcessExecutionError as e: + LOG.warning( + ( + "Failed at upgrading pip. This is usually not critical" + "so the script will skip this step.\n%s" + ), + e, + ) + else: + LOG.info("Upgraded pip") def is_installed(self) -> bool: cmd = [sys.executable, "-m", "pip", "list"] @@ -184,7 +202,10 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: ansible_galaxy(galaxy_cfg, ansible) if pull_cfg: - run_ansible_pull(ansible, deepcopy(pull_cfg)) + if isinstance(pull_cfg, dict): + pull_cfg = [pull_cfg] + for cfg in pull_cfg: + run_ansible_pull(ansible, deepcopy(cfg)) if setup_controller: ansible_controller(setup_controller, ansible) @@ -198,10 +219,35 @@ def validate_config(cfg: dict): for key in required_keys: if not get_cfg_by_path(cfg, key): raise ValueError(f"Missing required key '{key}' from {cfg}") - if cfg.get("pull"): - for key in "pull/url", "pull/playbook_name": - if not get_cfg_by_path(cfg, key): - raise ValueError(f"Missing required key '{key}' from {cfg}") + pull_cfg = cfg.get("pull") + if pull_cfg: + if isinstance(pull_cfg, dict): + pull_cfg = [pull_cfg] + elif not isinstance(pull_cfg, list): + raise ValueError( + "Invalid value ansible.pull. Expected either dict of list of" + f" dicts but found {pull_cfg}" + ) + for p_cfg in pull_cfg: + if not isinstance(p_cfg, dict): + raise ValueError( + "Invalid value of ansible.pull. Expected dict but found" + f" {p_cfg}" + ) + if not get_cfg_by_path(p_cfg, "url"): + raise ValueError(f"Missing required key 'url' from {p_cfg}") + has_playbook = get_cfg_by_path(p_cfg, "playbook_name") + has_playbooks = get_cfg_by_path(p_cfg, "playbook_names") + if not any([has_playbook, has_playbooks]): + raise ValueError( + f"Missing required key 'playbook_names' from {p_cfg}" + ) + elif all([has_playbooks, has_playbook]): + raise ValueError( + "Key 'ansible.pull.playbook_name' and" + " 'ansible.pull.playbook_names' are mutually exclusive." + f" Please use 'playbook_names' in {p_cfg}" + ) controller_cfg = cfg.get("setup_controller") if controller_cfg: @@ -228,8 +274,10 @@ def filter_args(cfg: dict) -> dict: def run_ansible_pull(pull: AnsiblePull, cfg: dict): - playbook_name: str = cfg.pop("playbook_name") - + playbook_name: str = cfg.pop("playbook_name", None) + playbook_names: List[str] = cfg.pop("playbook_names", None) + if playbook_name: + playbook_names = [playbook_name] v = pull.get_version() if not v: LOG.warning("Cannot parse ansible version") @@ -240,15 +288,22 @@ def run_ansible_pull(pull: AnsiblePull, cfg: dict): f"Ansible version {v.major}.{v.minor}.{v.patch}" "doesn't support --diff flag, exiting." ) - stdout = pull.pull( - *[ - f"--{key}={value}" if value is not True else f"--{key}" - for key, value in filter_args(cfg).items() - ], - playbook_name, - ) - if stdout: - sys.stdout.write(f"{stdout}") + pull_args = [ + f"--{key}={value}" if value is not True else f"--{key}" + for key, value in filter_args(cfg).items() + ] + if v and v >= lifecycle.Version(2, 12, 0): + # Multiple playbook support was resolved in 2.12. + # https://github.com/ansible/ansible/pull/73172 + stdout = pull.pull(*pull_args, *playbook_names) + if stdout: + sys.stdout.write(f"{stdout}") + else: + # Older ansible must pull separate playbooks + for playbook in playbook_names: + stdout = pull.pull(*pull_args, playbook) + if stdout: + sys.stdout.write(f"{stdout}") def ansible_galaxy(cfg: dict, ansible: AnsiblePull): diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 853e5296..d22fde88 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -41,7 +41,7 @@ meta: MetaSchema = { "id": "cc_apt_configure", - "distros": ["ubuntu", "debian"], + "distros": ["ubuntu", "debian", "raspberry-pi-os"], "frequency": PER_INSTANCE, "activate_by_schema_keys": [], } diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index eacecd17..1fb9933b 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -28,7 +28,7 @@ meta: MetaSchema = { "id": "cc_apt_pipelining", - "distros": ["ubuntu", "debian"], + "distros": ["ubuntu", "debian", "raspberry-pi-os"], "frequency": PER_INSTANCE, "activate_by_schema_keys": ["apt_pipelining"], } diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 4e4c113a..b21728f7 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -11,7 +11,7 @@ import logging -from cloudinit import signal_handler, subp, temp_utils, util +from cloudinit import subp, temp_utils, util from cloudinit.cloud import Cloud from cloudinit.config import Config from cloudinit.config.schema import MetaSchema @@ -50,10 +50,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: try: iid = cloud.get_instance_id() env = {"INSTANCE_ID": str(iid)} if iid else {} - with signal_handler.suspend_crash(): - subp.subp( - ["/bin/sh", tmpf.name], update_env=env, capture=False - ) + subp.subp(["/bin/sh", tmpf.name], update_env=env, capture=False) except Exception: util.logexc(LOG, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 86cc4bda..5e9a9a27 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -21,7 +21,7 @@ meta: MetaSchema = { "id": "cc_byobu", - "distros": ["ubuntu", "debian"], + "distros": ["ubuntu", "debian", "raspberry-pi-os"], "frequency": PER_INSTANCE, "activate_by_schema_keys": [], } diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 2e56e1c9..4b89977f 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -72,17 +72,21 @@ for distro in ( "almalinux", + "centos", "cloudlinux", + "rocky", ): DISTRO_OVERRIDES[distro] = DISTRO_OVERRIDES["rhel"] distros = [ "almalinux", "aosc", + "centos", "cloudlinux", "alpine", "debian", "fedora", + "raspberry-pi-os", "rhel", "opensuse", "opensuse-microos", @@ -157,10 +161,16 @@ def disable_default_ca_certs(distro_name, distro_cfg): """ if distro_name in ["rhel", "photon"]: remove_default_ca_certs(distro_cfg) - elif distro_name in ["alpine", "aosc", "debian", "ubuntu"]: + elif distro_name in [ + "alpine", + "aosc", + "debian", + "raspberry-pi-os", + "ubuntu", + ]: disable_system_ca_certs(distro_cfg) - if distro_name in ["debian", "ubuntu"]: + if distro_name in ["debian", "raspberry-pi-os", "ubuntu"]: debconf_sel = ( "ca-certificates ca-certificates/trust_new_crts " + "select no" ) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 0f3bf12d..058d27ea 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -7,6 +7,7 @@ """Disk Setup: Configure partitions and filesystems.""" +import json import logging import os import shlex @@ -396,7 +397,378 @@ def check_partition_mbr_layout(device, layout): return found_layout -def check_partition_gpt_layout(device, layout): +# gdisk uses its own ids, convert them to standard GPT partition GUIDs. +# From gdisk sources: +# grep " AddType" parttypes.cc | +# sed -e 's,AddType(,,' -e 's,"\,.*$,"\,,' -e 's,0x,",' -e 's,\,,":,' | +# tr "[a-z]" "[A-Z]" +sgdisk_to_gpt_id = { + "0000": "00000000-0000-0000-0000-000000000000", + "0100": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "0400": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "0600": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "0700": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "0701": "558D43C5-A1AC-43C0-AAC8-D1472B2923D1", + "0702": "90B6FF38-B98F-4358-A21F-48F35B4A8AD3", + "0B00": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "0C00": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "0C01": "E3C9E316-0B5C-4DB8-817D-F92DF00215AE", + "0E00": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "1100": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "1400": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "1600": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "1700": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "1B00": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "1C00": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "1E00": "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7", + "2700": "DE94BBA4-06D1-4D40-A16A-BFD50179D6AC", + "3000": "7412F7D5-A156-4B13-81DC-867174929325", + "3001": "D4E6E2CD-4469-46F3-B5CB-1BFF57AFC149", + "3900": "C91818F9-8025-47AF-89D2-F030D7000C2C", + "4100": "9E1A2D38-C612-4316-AA26-8B49521E5A8B", + "4200": "AF9B60A0-1431-4F62-BC68-3311714A69AD", + "4201": "5808C8AA-7E8F-42E0-85D2-E1E90434CFB3", + "4202": "E75CAF8F-F680-4CEE-AFA3-B001E56EFC2D", + "7501": "37AFFC90-EF7D-4E96-91C3-2D7AE055B174", + "7F00": "FE3A2A5D-4F32-41A7-B725-ACCC3285A309", + "7F01": "3CB8E202-3B7E-47DD-8A3C-7FF2A13CFCEC", + "7F02": "2E0A753D-9E48-43B0-8337-B15192CB1B5E", + "7F03": "CAB6E88E-ABF3-4102-A07A-D4BB9BE3C1D3", + "7F04": "09845860-705F-4BB5-B16C-8A8A099CAF52", + "7F05": "3F0F8318-F146-4E6B-8222-C28C8F02E0D5", + "8200": "0657FD6D-A4AB-43C4-84E5-0933C84B4F4F", + "8300": "0FC63DAF-8483-4772-8E79-3D69D8477DE4", + "8301": "8DA63339-0007-60C0-C436-083AC8230908", + "8302": "933AC7E1-2EB4-4F13-B844-0E14E2AEF915", + "8303": "44479540-F297-41B2-9AF7-D131D5F0458A", + "8304": "4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709", + "8305": "B921B045-1DF0-41C3-AF44-4C6F280D3FAE", + "8306": "3B8F8425-20E0-4F3B-907F-1A25A76F98E8", + "8307": "69DAD710-2CE4-4E3C-B16C-21A1D49ABED3", + "8308": "7FFEC5C9-2D00-49B7-8941-3EA10A5586B7", + "8309": "CA7D7CCB-63ED-4C53-861C-1742536059CC", + "830A": "993D8D3D-F80E-4225-855A-9DAF8ED7EA97", + "830B": "D13C5D3B-B5D1-422A-B29F-9454FDC89D76", + "830C": "2C7357ED-EBD2-46D9-AEC1-23D437EC2BF5", + "830D": "7386CDF2-203C-47A9-A498-F2ECCE45A2D6", + "830E": "DF3300CE-D69F-4C92-978C-9BFB0F38D820", + "830F": "86ED10D5-B607-45BB-8957-D350F23D0571", + "8310": "4D21B016-B534-45C2-A9FB-5C16E091FD2D", + "8311": "7EC6F557-3BC5-4ACA-B293-16EF5DF639D1", + "8312": "773F91EF-66D4-49B5-BD83-D683BF40AD16", + "8313": "75250D76-8CC6-458E-BD66-BD47CC81A812", + "8314": "8484680C-9521-48C6-9C11-B0720656F69E", + "8315": "7D0359A3-02B3-4F0A-865C-654403E70625", + "8316": "B0E01050-EE5F-4390-949A-9101B17104E9", + "8317": "4301D2A6-4E3B-4B2A-BB94-9E0B2C4225EA", + "8318": "8F461B0D-14EE-4E81-9AA9-049B6FB97ABD", + "8319": "77FF5F63-E7B6-4633-ACF4-1565B864C0E6", + "831A": "C215D751-7BCD-4649-BE90-6627490A4C05", + "831B": "6E11A4E7-FBCA-4DED-B9E9-E1A512BB664E", + "831C": "6A491E03-3BE7-4545-8E38-83320E0EA880", + "831D": "6523F8AE-3EB1-4E2A-A05A-18B695AE656F", + "831E": "D27F46ED-2919-4CB8-BD25-9531F3C16534", + "831F": "77055800-792C-4F94-B39A-98C91B762BB6", + "8320": "E9434544-6E2C-47CC-BAE2-12D6DEAFB44C", + "8321": "D113AF76-80EF-41B4-BDB6-0CFF4D3D4A25", + "8322": "37C58C8A-D913-4156-A25F-48B1B64E07F0", + "8323": "700BDA43-7A34-4507-B179-EEB93D7A7CA3", + "8324": "1AACDB3B-5444-4138-BD9E-E5C2239B2346", + "8325": "1DE3F1EF-FA98-47B5-8DCD-4A860A654D78", + "8326": "912ADE1D-A839-4913-8964-A10EEE08FBD2", + "8327": "C31C45E6-3F39-412E-80FB-4809C4980599", + "8328": "60D5A7FE-8E7D-435C-B714-3DD8162144E1", + "8329": "72EC70A6-CF74-40E6-BD49-4BDA08E8F224", + "832A": "08A7ACEA-624C-4A20-91E8-6E0FA67D23F9", + "832B": "5EEAD9A9-FE09-4A1E-A1D7-520D00531306", + "832C": "C50CDD70-3862-4CC3-90E1-809A8C93EE2C", + "832D": "E18CF08C-33EC-4C0D-8246-C6C6FB3DA024", + "832E": "7978A683-6316-4922-BBEE-38BFF5A2FECC", + "832F": "E611C702-575C-4CBE-9A46-434FA0BF7E3F", + "8330": "773B2ABC-2A99-4398-8BF5-03BAAC40D02B", + "8331": "57E13958-7331-4365-8E6E-35EEEE17C61B", + "8332": "0F4868E9-9952-4706-979F-3ED3A473E947", + "8333": "C97C1F32-BA06-40B4-9F22-236061B08AA8", + "8334": "DC4A4480-6917-4262-A4EC-DB9384949F25", + "8335": "7D14FEC5-CC71-415D-9D6C-06BF0B3C3EAF", + "8336": "2C9739E2-F068-46B3-9FD0-01C5A9AFBCCA", + "8337": "15BB03AF-77E7-4D4A-B12B-C0D084F7491C", + "8338": "B933FB22-5C3F-4F91-AF90-E2BB0FA50702", + "8339": "BEAEC34B-8442-439B-A40B-984381ED097D", + "833A": "CD0F869B-D0FB-4CA0-B141-9EA87CC78D66", + "833B": "8A4F5770-50AA-4ED3-874A-99B710DB6FEA", + "833C": "55497029-C7C1-44CC-AA39-815ED1558630", + "833D": "FC56D9E9-E6E5-4C06-BE32-E74407CE09A5", + "833E": "24B2D975-0F97-4521-AFA1-CD531E421B8D", + "833F": "F3393B22-E9AF-4613-A948-9D3BFBD0C535", + "8340": "7A430799-F711-4C7E-8E5B-1D685BD48607", + "8341": "579536F8-6A33-4055-A95A-DF2D5E2C42A8", + "8342": "D7D150D2-2A04-4A33-8F12-16651205FF7B", + "8343": "16B417F8-3E06-4F57-8DD2-9B5232F41AA6", + "8344": "D212A430-FBC5-49F9-A983-A7FEEF2B8D0E", + "8345": "906BD944-4589-4AAE-A4E4-DD983917446A", + "8346": "9225A9A3-3C19-4D89-B4F6-EEFF88F17631", + "8347": "98CFE649-1588-46DC-B2F0-ADD147424925", + "8348": "AE0253BE-1167-4007-AC68-43926C14C5DE", + "8349": "B6ED5582-440B-4209-B8DA-5FF7C419EA3D", + "834A": "7AC63B47-B25C-463B-8DF8-B4A94E6C90E1", + "834B": "B325BFBE-C7BE-4AB8-8357-139E652D2F6B", + "834C": "966061EC-28E4-4B2E-B4A5-1F0A825A1D84", + "834D": "8CCE0D25-C0D0-4A44-BD87-46331BF1DF67", + "834E": "FCA0598C-D880-4591-8C16-4EDA05C7347C", + "834F": "F46B2C26-59AE-48F0-9106-C50ED47F673D", + "8350": "6E5A1BC8-D223-49B7-BCA8-37A5FCCEB996", + "8351": "81CF9D90-7458-4DF4-8DCF-C8A3A404F09B", + "8352": "46B98D8D-B55C-4E8F-AAB3-37FCA7F80752", + "8353": "3C3D61FE-B5F3-414D-BB71-8739A694A4EF", + "8354": "5843D618-EC37-48D7-9F12-CEA8E08768B2", + "8355": "EE2B9983-21E8-4153-86D9-B6901A54D1CE", + "8356": "BDB528A5-A259-475F-A87D-DA53FA736A07", + "8357": "DF765D00-270E-49E5-BC75-F47BB2118B09", + "8358": "CB1EE4E3-8CD0-4136-A0A4-AA61A32E8730", + "8359": "8F1056BE-9B05-47C4-81D6-BE53128E5B54", + "835A": "B663C618-E7BC-4D6D-90AA-11B756BB1797", + "835B": "31741CC4-1A2A-4111-A581-E00B447D2D06", + "835C": "2FB4BF56-07FA-42DA-8132-6B139F2026AE", + "835D": "D46495B7-A053-414F-80F7-700C99921EF8", + "835E": "143A70BA-CBD3-4F06-919F-6C05683A78BC", + "835F": "42B0455F-EB11-491D-98D3-56145BA9D037", + "8360": "6DB69DE6-29F4-4758-A7A5-962190F00CE3", + "8361": "E98B36EE-32BA-4882-9B12-0CE14655F46A", + "8362": "5AFB67EB-ECC8-4F85-AE8E-AC1E7C50E7D0", + "8363": "BBA210A2-9C5D-45EE-9E87-FF2CCBD002D0", + "8364": "43CE94D4-0F3D-4999-8250-B9DEAFD98E6E", + "8365": "C919CC1F-4456-4EFF-918C-F75E94525CA5", + "8366": "904E58EF-5C65-4A31-9C57-6AF5FC7C5DE7", + "8367": "15DE6170-65D3-431C-916E-B0DCD8393F25", + "8368": "D4A236E7-E873-4C07-BF1D-BF6CF7F1C3C6", + "8369": "F5E2C20C-45B2-4FFA-BCE9-2A60737E1AAF", + "836A": "1B31B5AA-ADD9-463A-B2ED-BD467FC857E7", + "836B": "3A112A75-8729-4380-B4CF-764D79934448", + "836C": "EFE0F087-EA8D-4469-821A-4C2A96A8386A", + "836D": "3482388E-4254-435A-A241-766A065F9960", + "836E": "C80187A5-73A3-491A-901A-017C3FA953E9", + "836F": "B3671439-97B0-4A53-90F7-2D5A8F3AD47B", + "8370": "41092B05-9FC8-4523-994F-2DEF0408B176", + "8371": "5996FC05-109C-48DE-808B-23FA0830B676", + "8372": "5C6E1C76-076A-457A-A0FE-F3B4CD21CE6E", + "8373": "94F9A9A1-9971-427A-A400-50CB297F0F35", + "8374": "D7FF812F-37D1-4902-A810-D76BA57B975A", + "8375": "C23CE4FF-44BD-4B00-B2D4-B41B3419E02A", + "8376": "8DE58BC2-2A43-460D-B14E-A76E4A17B47F", + "8377": "B024F315-D330-444C-8461-44BBDE524E99", + "8378": "97AE158D-F216-497B-8057-F7F905770F54", + "8379": "05816CE2-DD40-4AC6-A61D-37D32DC1BA7D", + "837A": "3E23CA0B-A4BC-4B4E-8087-5AB6A26AA8A9", + "837B": "F2C2C7EE-ADCC-4351-B5C6-EE9816B66E16", + "837C": "450DD7D1-3224-45EC-9CF2-A43A346D71EE", + "837D": "C8BFBD1E-268E-4521-8BBA-BF314C399557", + "837E": "0B888863-D7F8-4D9E-9766-239FCE4D58AF", + "837F": "7007891D-D371-4A80-86A4-5CB875B9302E", + "8380": "C3836A13-3137-45BA-B583-B16C50FE5EB4", + "8381": "D2F9000A-7A18-453F-B5CD-4D32F77A7B32", + "8382": "17440E4F-A8D0-467F-A46E-3912AE6EF2C5", + "8383": "3F324816-667B-46AE-86EE-9B0C0C6C11B4", + "8384": "4EDE75E2-6CCC-4CC8-B9C7-70334B087510", + "8385": "E7BB33FB-06CF-4E81-8273-E543B413E2E2", + "8386": "974A71C0-DE41-43C3-BE5D-5C5CCD1AD2C0", + "8400": "D3BFE2DE-3DAF-11DF-BA40-E3A556D89593", + "8401": "7C5222BD-8F5D-4087-9C00-BF9843C7B58C", + "8500": "5DFBF5F4-2848-4BAC-AA5E-0D9A20B745A6", + "8501": "3884DD41-8582-4404-B9A8-E9B84F2DF50E", + "8502": "C95DC21A-DF0E-4340-8D7B-26CBFA9A03E0", + "8503": "BE9067B9-EA49-4F15-B4F6-F36F8C9E1818", + "8E00": "E6D6D379-F507-44C2-A23C-238F2A3DF928", + "A000": "2568845D-2332-4675-BC39-8FA5A4748D15", + "A001": "114EAFFE-1552-4022-B26E-9B053604CF84", + "A002": "49A4D17F-93A3-45C1-A0DE-F50B2EBE2599", + "A003": "4177C722-9E92-4AAB-8644-43502BFD5506", + "A004": "EF32A33B-A409-486C-9141-9FFB711F6266", + "A005": "20AC26BE-20B7-11E3-84C5-6CFDB94711E9", + "A006": "38F428E6-D326-425D-9140-6E0EA133647C", + "A007": "A893EF21-E428-470A-9E55-0668FD91A2D9", + "A008": "DC76DDA9-5AC1-491C-AF42-A82591580C0D", + "A009": "EBC597D0-2053-4B15-8B64-E0AAC75F4DB1", + "A00A": "8F68CC74-C5E5-48DA-BE91-A0C8C15E9C80", + "A00B": "767941D0-2085-11E3-AD3B-6CFDB94711E9", + "A00C": "AC6D7924-EB71-4DF8-B48D-E267B27148FF", + "A00D": "C5A0AEEC-13EA-11E5-A1B1-001E67CA0C3C", + "A00E": "BD59408B-4514-490D-BF12-9878D963F378", + "A00F": "9FDAA6EF-4B3F-40D2-BA8D-BFF16BFB887B", + "A010": "19A710A2-B3CA-11E4-B026-10604B889DCF", + "A011": "193D1EA4-B3CA-11E4-B075-10604B889DCF", + "A012": "DEA0BA2C-CBDD-4805-B4F9-F428251C3E98", + "A013": "8C6B52AD-8A9E-4398-AD09-AE916E53AE2D", + "A014": "05E044DF-92F1-4325-B69E-374A82E97D6E", + "A015": "400FFDCD-22E0-47E7-9A23-F16ED9382388", + "A016": "A053AA7F-40B8-4B1C-BA08-2F68AC71A4F4", + "A017": "E1A6A689-0C8D-4CC6-B4E8-55A4320FBD8A", + "A018": "098DF793-D712-413D-9D4E-89D711772228", + "A019": "D4E0D938-B7FA-48C1-9D21-BC5ED5C4B203", + "A01A": "20A0C19C-286A-42FA-9CE7-F64C3226A794", + "A01B": "A19F205F-CCD8-4B6D-8F1E-2D9BC24CFFB1", + "A01C": "66C9B323-F7FC-48B6-BF96-6F32E335A428", + "A01D": "303E6AC3-AF15-4C54-9E9B-D9A8FBECF401", + "A01E": "C00EEF24-7709-43D6-9799-DD2B411E7A3C", + "A01F": "82ACC91F-357C-4A68-9C8F-689E1B1A23A1", + "A020": "E2802D54-0545-E8A1-A1E8-C7A3E245ACD4", + "A021": "65ADDCF4-0C5C-4D9A-AC2D-D90B5CBFCD03", + "A022": "E6E98DA2-E22A-4D12-AB33-169E7DEAA507", + "A023": "ED9E8101-05FA-46B7-82AA-8D58770D200B", + "A024": "11406F35-1173-4869-807B-27DF71802812", + "A025": "9D72D4E4-9958-42DA-AC26-BEA7A90B0434", + "A026": "6C95E238-E343-4BA8-B489-8681ED22AD0B", + "A027": "EBBEADAF-22C9-E33B-8F5D-0E81686A68CB", + "A028": "0A288B1F-22C9-E33B-8F5D-0E81686A68CB", + "A029": "57B90A16-22C9-E33B-8F5D-0E81686A68CB", + "A02A": "638FF8E2-22C9-E33B-8F5D-0E81686A68CB", + "A02B": "2013373E-1AC4-4131-BFD8-B6A7AC638772", + "A02C": "2C86E742-745E-4FDD-BFD8-B6A7AC638772", + "A02D": "DE7D4029-0F5B-41C8-AE7E-F6C023A02B33", + "A02E": "323EF595-AF7A-4AFA-8060-97BE72841BB9", + "A02F": "45864011-CF89-46E6-A445-85262E065604", + "A030": "8ED8AE95-597F-4C8A-A5BD-A7FF8E4DFAA9", + "A031": "DF24E5ED-8C96-4B86-B00B-79667DC6DE11", + "A032": "7C29D3AD-78B9-452E-9DEB-D098D542F092", + "A033": "379D107E-229E-499D-AD4F-61F5BCF87BD4", + "A034": "0DEA65E5-A676-4CDF-823C-77568B577ED5", + "A035": "4627AE27-CFEF-48A1-88FE-99C3509ADE26", + "A036": "20117F86-E985-4357-B9EE-374BC1D8487D", + "A037": "86A7CB80-84E1-408C-99AB-694F1A410FC7", + "A038": "97D7B011-54DA-4835-B3C4-917AD6E73D74", + "A039": "5594C694-C871-4B5F-90B1-690A6F68E0F7", + "A03A": "1B81E7E6-F50D-419B-A739-2AEEF8DA3335", + "A03B": "98523EC6-90FE-4C67-B50A-0FC59ED6F56D", + "A03C": "2644BCC0-F36A-4792-9533-1738BED53EE3", + "A03D": "DD7C91E9-38C9-45C5-8A12-4A80F7E14057", + "A03E": "7696D5B6-43FD-4664-A228-C563C4A1E8CC", + "A03F": "0D802D54-058D-4A20-AD2D-C7A362CEACD4", + "A040": "10A0C19C-516A-5444-5CE3-664C3226A794", + "A200": "734E5AFE-F61A-11E6-BC64-92361F002671", + "A500": "516E7CB4-6ECF-11D6-8FF8-00022D09712B", + "A501": "83BD6B9D-7F41-11DC-BE0B-001560B84F0F", + "A502": "516E7CB5-6ECF-11D6-8FF8-00022D09712B", + "A503": "516E7CB6-6ECF-11D6-8FF8-00022D09712B", + "A504": "516E7CBA-6ECF-11D6-8FF8-00022D09712B", + "A505": "516E7CB8-6ECF-11D6-8FF8-00022D09712B", + "A506": "74BA7DD9-A689-11E1-BD04-00E081286ACF", + "A580": "85D5E45A-237C-11E1-B4B3-E89A8F7FC3A7", + "A581": "85D5E45E-237C-11E1-B4B3-E89A8F7FC3A7", + "A582": "85D5E45B-237C-11E1-B4B3-E89A8F7FC3A7", + "A583": "0394EF8B-237E-11E1-B4B3-E89A8F7FC3A7", + "A584": "85D5E45D-237C-11E1-B4B3-E89A8F7FC3A7", + "A585": "85D5E45C-237C-11E1-B4B3-E89A8F7FC3A7", + "A600": "824CC7A0-36A8-11E3-890A-952519AD3F61", + "A800": "55465300-0000-11AA-AA11-00306543ECAC", + "A900": "516E7CB4-6ECF-11D6-8FF8-00022D09712B", + "A901": "49F48D32-B10E-11DC-B99B-0019D1879648", + "A902": "49F48D5A-B10E-11DC-B99B-0019D1879648", + "A903": "49F48D82-B10E-11DC-B99B-0019D1879648", + "A904": "2DB519C4-B10F-11DC-B99B-0019D1879648", + "A905": "2DB519EC-B10F-11DC-B99B-0019D1879648", + "A906": "49F48DAA-B10E-11DC-B99B-0019D1879648", + "AB00": "426F6F74-0000-11AA-AA11-00306543ECAC", + "AF00": "48465300-0000-11AA-AA11-00306543ECAC", + "AF01": "52414944-0000-11AA-AA11-00306543ECAC", + "AF02": "52414944-5F4F-11AA-AA11-00306543ECAC", + "AF03": "4C616265-6C00-11AA-AA11-00306543ECAC", + "AF04": "5265636F-7665-11AA-AA11-00306543ECAC", + "AF05": "53746F72-6167-11AA-AA11-00306543ECAC", + "AF06": "B6FA30DA-92D2-4A9A-96F1-871EC6486200", + "AF07": "2E313465-19B9-463F-8126-8A7993773801", + "AF08": "FA709C7E-65B1-4593-BFD5-E71D61DE9B02", + "AF09": "BBBA6DF5-F46F-4A89-8F59-8765B2727503", + "AF0A": "7C3457EF-0000-11AA-AA11-00306543ECAC", + "AF0B": "69646961-6700-11AA-AA11-00306543ECAC", + "AF0C": "52637672-7900-11AA-AA11-00306543ECAC", + "B000": "3DE21764-95BD-54BD-A5C3-4ABE786F38A8", + "B300": "CEF5A9AD-73BC-4601-89F3-CDEEEEE321A1", + "BB00": "4778ED65-BF42-45FA-9C5B-287A1DC4AAB1", + "BC00": "0311FC50-01CA-4725-AD77-9ADBB20ACE98", + "BE00": "6A82CB45-1DD2-11B2-99A6-080020736631", + "BF00": "6A85CF4D-1DD2-11B2-99A6-080020736631", + "BF01": "6A898CC3-1DD2-11B2-99A6-080020736631", + "BF02": "6A87C46F-1DD2-11B2-99A6-080020736631", + "BF03": "6A8B642B-1DD2-11B2-99A6-080020736631", + "BF04": "6A8EF2E9-1DD2-11B2-99A6-080020736631", + "BF05": "6A90BA39-1DD2-11B2-99A6-080020736631", + "BF06": "6A9283A5-1DD2-11B2-99A6-080020736631", + "BF07": "6A945A3B-1DD2-11B2-99A6-080020736631", + "BF08": "6A9630D1-1DD2-11B2-99A6-080020736631", + "BF09": "6A980767-1DD2-11B2-99A6-080020736631", + "BF0A": "6A96237F-1DD2-11B2-99A6-080020736631", + "BF0B": "6A8D2AC7-1DD2-11B2-99A6-080020736631", + "C001": "75894C1E-3AEB-11D3-B7C1-7B03A0000000", + "C002": "E2A1E728-32E3-11D6-A682-7B03A0000000", + "E100": "7412F7D5-A156-4B13-81DC-867174929325", + "E101": "D4E6E2CD-4469-46F3-B5CB-1BFF57AFC149", + "E900": "8C8F8EFF-AC95-4770-814A-21994F2DBC8F", + "EA00": "BC13C2FF-59E6-4262-A352-B275FD6F7172", + "EB00": "42465331-3BA3-10F1-802A-4861696B7521", + "ED00": "F4019732-066E-4E12-8273-346C5641494F", + "ED01": "BFBFAFE7-A34F-448A-9A5B-6213EB736C22", + "EF00": "C12A7328-F81F-11D2-BA4B-00A0C93EC93B", + "EF01": "024DEE41-33E7-11D3-9D69-0008C781F39F", + "EF02": "21686148-6449-6E6F-744E-656564454649", + "F100": "FE8A2634-5E2E-46BA-99E3-3A192091A350", + "F101": "D9FD4535-106C-4CEC-8D37-DFC020CA87CB", + "F102": "A409E16B-78AA-4ACC-995C-302352621A41", + "F103": "F95D940E-CABA-4578-9B93-BB6C90F29D3E", + "F104": "10B8DBAA-D2BF-42A9-98C6-A7C5DB3701E7", + "F105": "49FD7CB8-DF15-4E73-B9D9-992070127F0F", + "F106": "421A8BFC-85D9-4D85-ACDA-B64EEC0133E9", + "F107": "9B37FFF6-2E58-466A-983A-F7926D0B04E0", + "F108": "C12A7328-F81F-11D2-BA4B-00A0C93EC93B", + "F109": "606B000B-B7C7-4653-A7D5-B737332C899D", + "F10A": "08185F0C-892D-428A-A789-DBEEC8F55E6A", + "F10B": "48435546-4953-2041-494E-5354414C4C52", + "F10C": "2967380E-134C-4CBB-B6DA-17E7CE1CA45D", + "F10D": "41D0E340-57E3-954E-8C1E-17ECAC44CFF5", + "F10E": "DE30CC86-1F4A-4A31-93C4-66F147D33E05", + "F10F": "23CC04DF-C278-4CE7-8471-897D1A4BCDF7", + "F110": "A0E5CF57-2DEF-46BE-A80C-A2067C37CD49", + "F111": "4E5E989E-4C86-11E8-A15B-480FCF35F8E6", + "F112": "5A3A90BE-4C86-11E8-A15B-480FCF35F8E6", + "F113": "5ECE94FE-4C86-11E8-A15B-480FCF35F8E6", + "F114": "8B94D043-30BE-4871-9DFA-D69556E8C1F3", + "F115": "A13B4D9A-EC5F-11E8-97D8-6C3BE52705BF", + "F116": "A288ABF2-EC5F-11E8-97D8-6C3BE52705BF", + "F117": "6A2460C3-CD11-4E8B-80A8-12CCE268ED0A", + "F118": "1D75395D-F2C6-476B-A8B7-45CC1C97B476", + "F119": "900B0FC5-90CD-4D4F-84F9-9F8ED579DB88", + "F11A": "B2B2E8D1-7C10-4EBC-A2D0-4614568260AD", + "F800": "4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D", + "F801": "4FBD7E29-9D25-41B8-AFD0-5EC00CEFF05D", + "F802": "45B0969E-9B03-4F30-B4C6-B4B80CEFF106", + "F803": "45B0969E-9B03-4F30-B4C6-5EC00CEFF106", + "F804": "89C57F98-2FE5-4DC0-89C1-F3AD0CEFF2BE", + "F805": "89C57F98-2FE5-4DC0-89C1-5EC00CEFF2BE", + "F806": "CAFECAFE-9B03-4F30-B4C6-B4B80CEFF106", + "F807": "30CD0809-C2B2-499C-8879-2D6B78529876", + "F808": "5CE17FCE-4087-4169-B7FF-056CC58473F9", + "F809": "FB3AABF9-D25F-47CC-BF5E-721D1816496B", + "F80A": "4FBD7E29-8AE0-4982-BF9D-5A8D867AF560", + "F80B": "45B0969E-8AE0-4982-BF9D-5A8D867AF560", + "F80C": "CAFECAFE-8AE0-4982-BF9D-5A8D867AF560", + "F80D": "7F4A666A-16F3-47A2-8445-152EF4D03F6C", + "F80E": "EC6D6385-E346-45DC-BE91-DA2A7C8B3261", + "F80F": "01B41E1B-002A-453C-9F17-88793989FF8F", + "F810": "CAFECAFE-9B03-4F30-B4C6-5EC00CEFF106", + "F811": "93B0052D-02D9-4D8A-A43B-33A3EE4DFBC3", + "F812": "306E8683-4FE2-4330-B7C0-00A917C16966", + "F813": "45B0969E-9B03-4F30-B4C6-35865CEFF106", + "F814": "CAFECAFE-9B03-4F30-B4C6-35865CEFF106", + "F815": "166418DA-C469-4022-ADF4-B30AFD37F176", + "F816": "86A32090-3647-40B9-BBBD-38D8C573AA86", + "F817": "4FBD7E29-9D25-41B8-AFD0-35865CEFF05D", + "FB00": "AA31E02A-400F-11DB-9590-000C2911D1B8", + "FB01": "9198EFFC-31C0-11DB-8F78-000C2911D1B8", + "FC00": "9D275380-40AD-11DB-BF97-000C2911D1B8", + "FD00": "A19D880F-05FC-4D3B-A006-743F0F84911E", +} + + +def check_partition_gpt_layout_sgdisk(device, layout): prt_cmd = ["sgdisk", "-p", device] try: out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) @@ -426,15 +798,76 @@ def check_partition_gpt_layout(device, layout): if line.strip().startswith("Number"): break - codes = [line.strip().split()[5] for line in out_lines] - cleaned = [] + return [line.strip().split()[5] for line in out_lines] + + +def check_partition_gpt_layout_sfdisk(device, layout): + # Use sfdisk's JSON output for reliability + prt_cmd = ["sfdisk", "-l", "-J", device] + try: + out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) + ptable = json.loads(out)["partitiontable"] + if "partitions" in ptable: + partitions = ptable["partitions"] + else: + partitions = [] + + except Exception as e: + raise RuntimeError( + "Error running partition command on %s\n%s" % (device, e) + ) from e + + return [p["type"] for p in partitions] + + +def check_partition_gpt_layout(device, layout): + if subp.which("sgdisk"): + return check_partition_gpt_layout_sgdisk(device, layout) + return check_partition_gpt_layout_sfdisk(device, layout) + + +def partition_type_matches(found_type, expected_type): + """ + Check if the observed partition type matches the expectation which + can either be a two digit legacy code, a four digit 'sgdisk' type or + a GPT UUID. + """ + found_type = str(found_type).upper() + if len(found_type) not in [2, 4, 36]: + raise RuntimeError("Unknown partition type found: %s" % found_type) + + expected_type = str(expected_type).upper() + if len(expected_type) not in [2, 4, 36]: + raise RuntimeError("Unknown partition type specified: %s" % found_type) + + # Promote 2-digit codes to 4-digit + if len(found_type) == 2: + found_type += "00" + if len(expected_type) == 2: + expected_type += "00" + + # Check if four digit codes match + if len(found_type) == len(expected_type): + return found_type == expected_type + + # Promote four digit codes to GPT partition GUIDs + if len(found_type) == 4: + if found_type in sgdisk_to_gpt_id: + found_type = sgdisk_to_gpt_id[found_type] + else: + raise RuntimeError( + "Cannot find GPT GUID for found type %s" % found_type + ) + if len(expected_type) == 4: + if expected_type in sgdisk_to_gpt_id: + expected_type = sgdisk_to_gpt_id[expected_type] + else: + raise RuntimeError( + "Cannot find GPT GUID for expected type %s" % found_type + ) - # user would expect a code '83' to be Linux, but sgdisk outputs 8300. - for code in codes: - if len(code) == 4 and code.endswith("00"): - code = code[0:2] - cleaned.append(code) - return cleaned + # Both codes are GPT UUIDs now + return found_type == expected_type def check_partition_layout(table_type, device, layout): @@ -476,7 +909,7 @@ def check_partition_layout(table_type, device, layout): "Layout types=%s. Found types=%s", layout_types, found_layout ) for itype, ftype in zip(layout_types, found_layout): - if itype is not None and str(ftype) != str(itype): + if itype is not None and not partition_type_matches(itype, ftype): return False return True @@ -651,7 +1084,7 @@ def exec_mkpart_mbr(device, layout): read_parttbl(device) -def exec_mkpart_gpt(device, layout): +def exec_mkpart_gpt_sgdisk(device, layout): try: subp.subp(["sgdisk", "-Z", device]) for index, (partition_type, (start, end)) in enumerate(layout): @@ -675,6 +1108,38 @@ def exec_mkpart_gpt(device, layout): LOG.warning("Failed to partition device %s", device) raise + +def exec_mkpart_gpt_sfdisk(device, layout): + cmd = "" + # Promote partition types to GPT partition GUIDs + for partition_type, (start, end) in layout: + partition_type = str(partition_type).ljust(4, "0") + if len(partition_type) == 4 and partition_type in sgdisk_to_gpt_id: + partition_type = sgdisk_to_gpt_id[partition_type] + if len(partition_type) != 36: + if partition_type != "None": + LOG.warning( + "Unknown GPT partition type %s, using Linux", + partition_type, + ) + partition_type = "0FC63DAF-8483-4772-8E79-3D69D8477DE4" + if str(end) != "0": + cmd += ",%s,%s\n" % (end, partition_type) + else: + cmd += ",,%s\n" % partition_type + try: + subp.subp(["sfdisk", "-X", "gpt", "--force", device], data="%s" % cmd) + except Exception: + LOG.warning("Failed to partition device %s", device) + raise + + +def exec_mkpart_gpt(device, layout): + if subp.which("sgdisk"): + exec_mkpart_gpt_sgdisk(device, layout) + else: + exec_mkpart_gpt_sfdisk(device, layout) + read_parttbl(device) diff --git a/cloudinit/config/cc_install_hotplug.py b/cloudinit/config/cc_install_hotplug.py index 883c0c17..8cd19dbf 100644 --- a/cloudinit/config/cc_install_hotplug.py +++ b/cloudinit/config/cc_install_hotplug.py @@ -27,7 +27,7 @@ HOTPLUG_UDEV_PATH = "/etc/udev/rules.d/90-cloud-init-hook-hotplug.rules" HOTPLUG_UDEV_RULES_TEMPLATE = """\ # Installed by cloud-init due to network hotplug userdata -ACTION!="add|remove", GOTO="cloudinit_end"{extra_rules} +ACTION!="add", GOTO="cloudinit_end"{extra_rules} LABEL="cloudinit_hook" SUBSYSTEM=="net", RUN+="{libexecdir}/hook-hotplug" LABEL="cloudinit_end" diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index a5d10362..9d2de69c 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -514,9 +514,35 @@ def mount_if_needed( subp.subp(["systemctl", "daemon-reload"]) +def cleanup_fstab(ds_remove_entries: list = []) -> None: + if not os.path.exists(FSTAB_PATH): + return + + base_entry = [MNT_COMMENT] + + with open(FSTAB_PATH, "r") as f: + lines = f.readlines() + new_lines = [] + changed = False + for line in lines: + if all(entry in line for entry in [*base_entry, *ds_remove_entries]): + changed = True + continue + new_lines.append(line) + + # rewrite fstab + try: + if changed: + with open(FSTAB_PATH, "w") as f: + f.writelines(new_lines) + LOG.info("Removed resource disk entries from %s", FSTAB_PATH) + except Exception as e: + LOG.warning("Failed to clean resource disk entries from fstab: %s", e) + + def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: """Handle the mounts configuration.""" - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno uses_systemd = cloud.distro.uses_systemd() default_mount_options = ( "defaults,nofail,x-systemd.after=cloud-init.service,_netdev" diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 0501a89a..a8f1defb 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -45,6 +45,7 @@ "opensuse-tumbleweed", "opensuse-leap", "photon", + "raspberry-pi-os", "rhel", "rocky", "sle_hpc", @@ -211,6 +212,11 @@ "confpath": "/etc/systemd/timesyncd.conf", }, }, + "raspberry-pi-os": { + "chrony": { + "confpath": "/etc/chrony/chrony.conf", + }, + }, "rhel": { "ntp": { "service_name": "ntpd", diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py index 0dd4f89f..b729b5b4 100644 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -10,7 +10,7 @@ import os import time -from cloudinit import signal_handler, subp, util +from cloudinit import subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config from cloudinit.config.schema import MetaSchema @@ -48,10 +48,7 @@ def _fire_reboot( wait_attempts: int = 6, initial_sleep: int = 1, backoff: int = 2 ): """Run a reboot command and panic if it doesn't happen fast enough.""" - # systemd will kill cloud-init with a signal - # this is expected so don't behave as if this is a failure state - with signal_handler.suspend_crash(): - subp.subp(REBOOT_CMD) + subp.subp(REBOOT_CMD) start = time.monotonic() wait_time = initial_sleep for _i in range(wait_attempts): diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 4bd9f8cc..c1eccc59 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -13,7 +13,7 @@ import subprocess import time -from cloudinit import signal_handler, subp, util +from cloudinit import subp, util from cloudinit.cloud import Cloud from cloudinit.config import Config from cloudinit.config.schema import MetaSchema @@ -220,7 +220,4 @@ def fatal(msg): except Exception as e: fatal("Unexpected Exception when checking condition: %s" % e) - # systemd could kill this process with a signal before it exits - # this is expected, so don't crash - with signal_handler.suspend_crash(): - func(*args) + func(*args) diff --git a/cloudinit/config/cc_raspberry_pi.py b/cloudinit/config/cc_raspberry_pi.py new file mode 100644 index 00000000..a80a8f08 --- /dev/null +++ b/cloudinit/config/cc_raspberry_pi.py @@ -0,0 +1,216 @@ +# Copyright (C) 2024-2025, Raspberry Pi Ltd. +# +# Author: Paul Oberosler +# +# This file is part of cloud-init. See LICENSE file for license information. + +import logging +from typing import Union + +from cloudinit import subp +from cloudinit.cloud import Cloud +from cloudinit.config import Config +from cloudinit.config.schema import MetaSchema +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) +RPI_BASE_KEY = "rpi" +RPI_INTERFACES_KEY = "interfaces" +ENABLE_RPI_CONNECT_KEY = "enable_rpi_connect" +SUPPORTED_INTERFACES = { + "spi": "do_spi", + "i2c": "do_i2c", + "serial": "do_serial", + "onewire": "do_onewire", + "remote_gpio": "do_rgpio", +} +RASPI_CONFIG_SERIAL_CONS_FN = "do_serial_cons" +RASPI_CONFIG_SERIAL_HW_FN = "do_serial_hw" + +meta: MetaSchema = { + "id": "cc_raspberry_pi", + "distros": ["raspberry-pi-os"], + "frequency": PER_INSTANCE, + "activate_by_schema_keys": [RPI_BASE_KEY], +} + + +def configure_rpi_connect(enable: bool) -> None: + LOG.debug("Configuring rpi-connect: %s", enable) + + num = 0 if enable else 1 + + try: + subp.subp(["/usr/bin/raspi-config", "do_rpi_connect", str(num)]) + except subp.ProcessExecutionError as e: + LOG.error("Failed to configure rpi-connect: %s", e) + + +def is_pifive() -> bool: + try: + subp.subp(["/usr/bin/raspi-config", "nonint", "is_pifive"]) + return True + except subp.ProcessExecutionError: + return False + + +def configure_serial_interface( + cfg: Union[dict, bool], instCfg: Config, cloud: Cloud +) -> None: + def get_bool_field(cfg_dict: dict, name: str, default=False): + val = cfg_dict.get(name, default) + if not isinstance(val, bool): + LOG.warning( + "Invalid value for %s.serial.%s: %s", + RPI_INTERFACES_KEY, + name, + val, + ) + return default + return val + + enable_console = False + enable_hw = False + + if isinstance(cfg, dict): + enable_console = get_bool_field(cfg, "console") + enable_hw = get_bool_field(cfg, "hardware") + + elif isinstance(cfg, bool): + # default to enabling console as if < pi5 + # this will also enable the hardware + enable_console = cfg + + if not is_pifive() and enable_console: + # only pi5 has 2 usable UARTs + # on other models, enabling the console + # will also block the other UART + enable_hw = True + + try: + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + RASPI_CONFIG_SERIAL_CONS_FN, + str(0 if enable_console else 1), + ] + ) + + try: + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + RASPI_CONFIG_SERIAL_HW_FN, + str(0 if enable_hw else 1), + ] + ) + except subp.ProcessExecutionError as e: + LOG.error("Failed to configure serial hardware: %s", e) + + # Reboot to apply changes + cmd = cloud.distro.shutdown_command( + mode="reboot", + delay="now", + message="Rebooting to apply serial console changes", + ) + subp.subp(cmd) + except subp.ProcessExecutionError as e: + LOG.error("Failed to configure serial console: %s", e) + + +def configure_interface(iface: str, enable: bool) -> None: + assert ( + iface in SUPPORTED_INTERFACES.keys() and iface != "serial" + ), f"Unsupported interface: {iface}" + + try: + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + SUPPORTED_INTERFACES[iface], + str(0 if enable else 1), + ] + ) + except subp.ProcessExecutionError as e: + LOG.error("Failed to configure %s: %s", iface, e) + + +def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: + if RPI_BASE_KEY not in cfg: + return + elif not isinstance(cfg[RPI_BASE_KEY], dict): + LOG.warning( + "Invalid value for %s: %s", + RPI_BASE_KEY, + cfg[RPI_BASE_KEY], + ) + return + elif not cfg[RPI_BASE_KEY]: + LOG.debug("Empty value for %s. Skipping...", RPI_BASE_KEY) + return + + for key in cfg[RPI_BASE_KEY]: + if key == ENABLE_RPI_CONNECT_KEY: + enable = cfg[RPI_BASE_KEY][key] + + if isinstance(enable, bool): + configure_rpi_connect(enable) + else: + LOG.warning( + "Invalid value for %s: %s", ENABLE_RPI_CONNECT_KEY, enable + ) + continue + elif key == RPI_INTERFACES_KEY: + if not isinstance(cfg[RPI_BASE_KEY][key], dict): + LOG.warning( + "Invalid value for %s: %s", + RPI_BASE_KEY, + cfg[RPI_BASE_KEY][key], + ) + return + elif not cfg[RPI_BASE_KEY][key]: + LOG.debug("Empty value for %s. Skipping...", key) + return + + subkeys = list(cfg[RPI_BASE_KEY][key].keys()) + # Move " serial" to the end if it exists + if "serial" in subkeys: + subkeys.append(subkeys.pop(subkeys.index("serial"))) + + # check for supported ARM interfaces + for subkey in subkeys: + if subkey not in SUPPORTED_INTERFACES.keys(): + LOG.warning( + "Invalid key for %s: %s", RPI_INTERFACES_KEY, subkey + ) + continue + + enable = cfg[RPI_BASE_KEY][key][subkey] + + if subkey == "serial": + if not isinstance(enable, (dict, bool)): + LOG.warning( + "Invalid value for %s.%s: %s", + RPI_INTERFACES_KEY, + subkey, + enable, + ) + else: + configure_serial_interface(enable, cfg, cloud) + continue + + if isinstance(enable, bool): + configure_interface(subkey, enable) + else: + LOG.warning( + "Invalid value for %s.%s: %s", + RPI_INTERFACES_KEY, + subkey, + enable, + ) + else: + LOG.warning("Unsupported key: %s", key) + continue diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 3e3bf056..7d988322 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -23,7 +23,7 @@ meta: MetaSchema = { "id": "cc_ssh_import_id", - "distros": ["alpine", "cos", "debian", "ubuntu"], + "distros": ["alpine", "cos", "debian", "raspberry-pi-os", "ubuntu"], "frequency": PER_INSTANCE, "activate_by_schema_keys": [], } diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index c6458853..3e051048 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -43,6 +43,7 @@ "power-state-change", "power_state_change", "puppet", + "raspberry_pi", "reset-rmc", "reset_rmc", "resizefs", @@ -493,6 +494,108 @@ } } }, + "ansible.pull": { + "oneOf": [ + { + "required": [ + "url", + "playbook_name" + ] + }, + { + "required": [ + "url", + "playbook_names" + ] + } + ], + "type": "object", + "additionalProperties": false, + "properties": { + "accept_host_key": { + "type": "boolean", + "default": false + }, + "clean": { + "type": "boolean", + "default": false + }, + "full": { + "type": "boolean", + "default": false + }, + "diff": { + "type": "boolean", + "default": false + }, + "ssh_common_args": { + "type": "string" + }, + "scp_extra_args": { + "type": "string" + }, + "sftp_extra_args": { + "type": "string" + }, + "private_key": { + "type": "string" + }, + "checkout": { + "type": "string" + }, + "module_path": { + "type": "string" + }, + "timeout": { + "type": "string" + }, + "url": { + "type": "string" + }, + "connection": { + "type": "string" + }, + "vault_id": { + "type": "string" + }, + "vault_password_file": { + "type": "string" + }, + "verify_commit": { + "type": "boolean", + "default": false + }, + "inventory": { + "type": "string" + }, + "module_name": { + "type": "string" + }, + "sleep": { + "type": "string" + }, + "tags": { + "type": "string" + }, + "skip_tags": { + "type": "string" + }, + "playbook_name": { + "deprecated": true, + "deprecated_version": "25.2", + "deprecated_description": "Use **playbook_names** key instead.", + "description": "Single playbook_name to run with ansible-pull", + "type": "string" + }, + "playbook_names": { + "type": "array", + "description": "List of playbook_names to run with ansible-pull", + "items": { + "type": "string" + } + } + } + }, "apt_configure.mirror": { "type": "array", "items": { @@ -866,85 +969,21 @@ "default": "ansible" }, "pull": { - "required": [ - "url", - "playbook_name" - ], - "type": "object", - "additionalProperties": false, - "properties": { - "accept_host_key": { - "type": "boolean", - "default": false - }, - "clean": { - "type": "boolean", - "default": false - }, - "full": { - "type": "boolean", - "default": false - }, - "diff": { - "type": "boolean", - "default": false - }, - "ssh_common_args": { - "type": "string" - }, - "scp_extra_args": { - "type": "string" - }, - "sftp_extra_args": { - "type": "string" - }, - "private_key": { - "type": "string" - }, - "checkout": { - "type": "string" - }, - "module_path": { - "type": "string" - }, - "timeout": { - "type": "string" - }, - "url": { - "type": "string" - }, - "connection": { - "type": "string" - }, - "vault_id": { - "type": "string" - }, - "vault_password_file": { - "type": "string" - }, - "verify_commit": { - "type": "boolean", - "default": false - }, - "inventory": { - "type": "string" - }, - "module_name": { - "type": "string" - }, - "sleep": { - "type": "string" - }, - "tags": { - "type": "string" - }, - "skip_tags": { - "type": "string" + "description": "pull playbooks from a VCS repo and run them on the host", + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/ansible.pull" + } }, - "playbook_name": { - "type": "string" + { + "$ref": "#/$defs/ansible.pull", + "deprecated": true, + "deprecated_version": "25.2", + "deprecated_description": "Expect **ansible.pull** as list of objects instead of a single object." } - } + ] } } } @@ -2029,7 +2068,7 @@ "minItems": 1, "maxItems": 6 }, - "description": "List of lists. Each inner list entry is a list of ``/etc/fstab`` mount declarations of the format: [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno ]. A mount declaration with less than 6 items will get remaining values from **mount_default_fields**. A mount declaration with only `fs_spec` and no `fs_file` mountpoint will be skipped.", + "description": "List of lists. Each inner list entry is a list of ``/etc/fstab`` mount declarations of the format: [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno ]. A mount declaration with less than 6 items will get remaining values from **mount_default_fields**. A mount declaration with only `fs_spec` and no `fs_file` mountpoint will be skipped.", "minItems": 1 }, "mount_default_fields": { @@ -2606,6 +2645,70 @@ } } }, + "cc_raspberry_pi": { + "type": "object", + "properties": { + "rpi": { + "type": "object", + "properties": { + "interfaces": { + "type": "object", + "properties": { + "spi": { + "type": "boolean", + "description": "Enable SPI interface. Default: ``false``.", + "default": false + }, + "i2c": { + "type": "boolean", + "description": "Enable I2C interface. Default: ``false``.", + "default": false + }, + "serial": { + "default": false, + "description": "Enable serial console. Default: ``false``.", + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "object", + "properties": { + "console": { + "type": "boolean", + "description": "Enable serial console. Default: ``false``.", + "default": false + }, + "hardware": { + "type": "boolean", + "description": "Enable UART hardware. Default: ``false``.", + "default": false + } + } + } + ] + }, + "onewire": { + "type": "boolean", + "description": "Enable 1-Wire interface. Default: ``false``.", + "default": false + }, + "remote_gpio": { + "type": "boolean", + "description": "Enable remote GPIO interface. Default: ``false``.", + "default": false + } + } + }, + "enable_rpi_connect": { + "type": "boolean", + "default": false, + "description": "Install and enable Raspberry Pi Connect. Default: ``false``." + } + } + } + } + }, "cc_rsyslog": { "type": "object", "properties": { @@ -3861,6 +3964,9 @@ { "$ref": "#/$defs/cc_puppet" }, + { + "$ref": "#/$defs/cc_raspberry_pi" + }, { "$ref": "#/$defs/cc_resizefs" }, @@ -4011,6 +4117,7 @@ "resize_rootfs": {}, "resolv_conf": {}, "rh_subscription": {}, + "rpi": {}, "rsyslog": {}, "runcmd": {}, "salt_minion": {}, diff --git a/cloudinit/config/schemas/schema-network-config-v1.json b/cloudinit/config/schemas/schema-network-config-v1.json index 99e8c68b..92ded462 100644 --- a/cloudinit/config/schemas/schema-network-config-v1.json +++ b/cloudinit/config/schemas/schema-network-config-v1.json @@ -40,6 +40,10 @@ "accept-ra": { "type": "boolean", "description": "Whether to accept IPv6 Router Advertisements (RA) on this interface. If unset, it will not be rendered" + }, + "keep_configuration": { + "type": "boolean", + "description": "Designate the connection as 'critical to the system', meaning that special care will be taken not to release the assigned IP when the daemon is restarted. (only recognized by Netplan renderer)." } } }, @@ -545,6 +549,10 @@ "ipv6": { "type": "boolean", "description": "Indicate if the subnet is IPv6. If not specified, it will be inferred from the subnet type or address. This is exists for compatibility with OpenStack's ``network_data.json`` when rendered through sysconfig." + }, + "metric": { + "type": "integer", + "description": "Specify metric cost for interface and routes of this subnet. " } } }, diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index b0b18ab1..eb58095c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -64,7 +64,7 @@ "alpine": ["alpine"], "aosc": ["aosc"], "arch": ["arch"], - "debian": ["debian", "ubuntu"], + "debian": ["debian", "ubuntu", "raspberry-pi-os"], "freebsd": ["freebsd", "dragonfly"], "gentoo": ["gentoo", "cos"], "netbsd": ["netbsd"], @@ -1365,7 +1365,7 @@ def manage_service( "disable": ["disable", service], "restart": ["restart", service], "reload": ["reload-or-restart", service], - "try-reload": ["reload-or-try-restart", service], + "try-reload": ["try-reload-or-restart", service], "status": ["status", service], } else: @@ -1768,6 +1768,11 @@ def _get_arch_package_mirror_info(package_mirrors, arch): def fetch(name: str) -> Type[Distro]: locs, looked_locs = importer.find_module(name, ["", __name__], ["Distro"]) + if not locs: + # Some distros may have a `-` in the name but an `_` in the module + locs, _ = importer.find_module( + name.replace("-", "_"), ["", __name__], ["Distro"] + ) if not locs: raise ImportError( "No distribution found for distro %s (searched %s)" diff --git a/cloudinit/distros/raspberry_pi_os.py b/cloudinit/distros/raspberry_pi_os.py new file mode 100644 index 00000000..f1bd111a --- /dev/null +++ b/cloudinit/distros/raspberry_pi_os.py @@ -0,0 +1,80 @@ +# Copyright (C) 2024-2025 Raspberry Pi Ltd. All rights reserved. +# +# Author: Paul Oberosler +# +# This file is part of cloud-init. See LICENSE file for license information. + +import logging + +from cloudinit import subp +from cloudinit.distros import debian + +LOG = logging.getLogger(__name__) + + +class Distro(debian.Distro): + def set_keymap(self, layout: str, model: str, variant: str, options: str): + """Currently Raspberry Pi OS sys-mods only supports + setting the layout""" + + subp.subp( + [ + "/usr/lib/raspberrypi-sys-mods/imager_custom", + "set_keymap", + layout, + ] + ) + + def apply_locale(self, locale, out_fn=None, keyname="LANG"): + try: + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + "do_change_locale", + f"{locale}", + ] + ) + except subp.ProcessExecutionError: + if not locale.endswith(".UTF-8"): + LOG.info("Trying to set locale %s.UTF-8", locale) + subp.subp( + [ + "/usr/bin/raspi-config", + "nonint", + "do_change_locale", + f"{locale}.UTF-8", + ] + ) + else: + LOG.error("Failed to set locale %s") + + def add_user(self, name, **kwargs) -> bool: + """ + Add a user to the system using standard GNU tools + + This should be overridden on distros where useradd is not desirable or + not available. + + Returns False if user already exists, otherwise True. + """ + result = super().add_user(name, **kwargs) + + if not result: + return result + + try: + subp.subp( + [ + "/usr/bin/rename-user", + "-f", + "-s", + ], + update_env={"SUDO_USER": name}, + ) + + except subp.ProcessExecutionError as e: + LOG.error("Failed to setup user: %s", e) + return False + + return True diff --git a/cloudinit/features.py b/cloudinit/features.py index fa83a48d..52a82f09 100644 --- a/cloudinit/features.py +++ b/cloudinit/features.py @@ -98,6 +98,16 @@ to modify the systemd unit files. """ +STRIP_INVALID_MTU = True +""" +If ``STRIP_INVALID_MTU`` is True, then cloud-init will strip invalid MTU +values from rendered v2 netplan configuration. Cloud-init allowed these values +prior to 24.2, so this flag is used to maintain compatibility with +previously generated network configurations. + +(This flag can be removed when Noble is no longer supported.) +""" + DEPRECATION_INFO_BOUNDARY = "24.1" """ DEPRECATION_INFO_BOUNDARY is used by distros to configure at which upstream diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index f71dd6b6..1983e1da 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -486,7 +486,17 @@ def find_candidate_nics_on_linux() -> List[str]: "Found unstable nic names: %s; calling udevadm settle", unstable, ) - util.udevadm_settle() + try: + util.udevadm_settle() + except subp.ProcessExecutionError as error: + LOG.warning( + "udevadm failed to settle: " + "cmd=%r stderr=%r stdout=%r exit_code=%s", + error.cmd, + error.stderr, + error.stdout, + error.exit_code, + ) # sort into interfaces with carrier, interfaces which could have carrier, # and ignore interfaces that are definitely disconnected @@ -1044,8 +1054,6 @@ def get_interfaces( # 16 somewhat arbitrarily chosen. Normally a mac is 6 '00:' tokens. zero_mac = ":".join(("00",) * 16) for name in devs: - if filter_without_own_mac and not interface_has_own_mac(name): - continue if is_bridge(name): filtered_logger("Ignoring bridge interface: %s", name) continue @@ -1054,6 +1062,9 @@ def get_interfaces( if is_bond(name): filtered_logger("Ignoring bond interface: %s", name) continue + if filter_without_own_mac and not interface_has_own_mac(name): + filtered_logger("Ignoring interface with inherited MAC: %s", name) + continue if ( filter_slave_if_master_not_bridge_bond_openvswitch and get_master(name) is not None diff --git a/cloudinit/net/activators.py b/cloudinit/net/activators.py index 94212894..5200c428 100644 --- a/cloudinit/net/activators.py +++ b/cloudinit/net/activators.py @@ -2,7 +2,7 @@ import logging from abc import ABC, abstractmethod from functools import partial -from typing import Callable, Dict, Iterable, List, Optional, Type, Union +from typing import Callable, Dict, Iterable, List, Optional, Type from cloudinit import subp, util from cloudinit.net import eni, netplan, network_manager, networkd @@ -45,7 +45,7 @@ def _alter_interface_callable( class NetworkActivator(ABC): @staticmethod @abstractmethod - def available(target: Optional[str] = None) -> bool: + def available() -> bool: """Return True if activator is available, otherwise return False.""" raise NotImplementedError() @@ -97,9 +97,9 @@ class IfUpDownActivator(NetworkActivator): # E.g., NetworkManager has a ifupdown plugin that requires the name # of a specific connection. @staticmethod - def available(target: Optional[str] = None) -> bool: + def available() -> bool: """Return true if ifupdown can be used on this system.""" - return eni.available(target=target) + return eni.available() @staticmethod def bring_up_interface(device_name: str) -> bool: @@ -122,11 +122,11 @@ def bring_down_interface(device_name: str) -> bool: class IfConfigActivator(NetworkActivator): @staticmethod - def available(target=None) -> bool: + def available() -> bool: """Return true if ifconfig can be used on this system.""" expected = "ifconfig" search = ["/sbin"] - return bool(subp.which(expected, search=search, target=target)) + return bool(subp.which(expected, search=search)) @staticmethod def bring_up_interface(device_name: str) -> bool: @@ -149,9 +149,9 @@ def bring_down_interface(device_name: str) -> bool: class NetworkManagerActivator(NetworkActivator): @staticmethod - def available(target=None) -> bool: + def available() -> bool: """Return true if NetworkManager can be used on this system.""" - return network_manager.available(target=target) + return network_manager.available() @staticmethod def bring_up_interface(device_name: str) -> bool: @@ -215,9 +215,9 @@ class NetplanActivator(NetworkActivator): NETPLAN_CMD = ["netplan", "apply"] @staticmethod - def available(target=None) -> bool: + def available() -> bool: """Return true if netplan can be used on this system.""" - return netplan.available(target=target) + return netplan.available() @staticmethod def bring_up_interface(device_name: str) -> bool: @@ -283,9 +283,9 @@ def wait_for_network() -> None: class NetworkdActivator(NetworkActivator): @staticmethod - def available(target=None) -> bool: + def available() -> bool: """Return true if ifupdown can be used on this system.""" - return networkd.available(target=target) + return networkd.available() @staticmethod def bring_up_interface(device_name: str) -> bool: @@ -334,9 +334,7 @@ def wait_for_network() -> None: } -def search_activator( - priority: List[str], target: Union[str, None] -) -> Optional[Type[NetworkActivator]]: +def search_activator(priority: List[str]) -> Optional[Type[NetworkActivator]]: """Returns the first available activator from the priority list or None.""" unknown = [i for i in priority if i not in DEFAULT_PRIORITY] if unknown: @@ -348,22 +346,21 @@ def search_activator( ( activator_cls for activator_cls in activator_classes - if activator_cls.available(target) + if activator_cls.available() ), None, ) def select_activator( - priority: Optional[List[str]] = None, target: Optional[str] = None + priority: Optional[List[str]] = None, ) -> Type[NetworkActivator]: if priority is None: priority = DEFAULT_PRIORITY - selected = search_activator(priority, target) + selected = search_activator(priority) if not selected: - tmsg = f" in target={target}" if target and target != "/" else "" raise NoActivatorException( - f"No available network activators found{tmsg}. " + f"No available network activators found. " f"Searched through list: {priority}" ) LOG.debug( diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index e67c6304..a301c76b 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -83,7 +83,9 @@ class NoDHCPLeaseMissingDhclientError(NoDHCPLeaseError): def maybe_perform_dhcp_discovery( - distro, nic=None, dhcp_log_func=None + distro, + nic=None, + dhcp_log_func: Optional[Callable[[str, str, str], None]] = None, ) -> Dict[str, Any]: """Perform dhcp discovery if nic valid and dhclient command exists. @@ -91,8 +93,8 @@ def maybe_perform_dhcp_discovery( skip dhcp_discovery and return an empty dict. @param nic: Name of the network interface we want to run dhclient on. - @param dhcp_log_func: A callable accepting the dhclient output and error - streams. + @param dhcp_log_func: Callable accepting the interface and client's + stdout, stderr streams. @return: A list of dicts representing dhcp options for each lease obtained from the dhclient discovery if run, otherwise an empty list is returned. @@ -203,7 +205,7 @@ def parse_static_routes(routes: str) -> List[Tuple[str, str]]: def dhcp_discovery( self, interface: str, - dhcp_log_func: Optional[Callable] = None, + dhcp_log_func: Optional[Callable[[str, str, str], None]] = None, distro=None, ) -> Dict[str, Any]: """Run dhcp client on the interface without scripts or filesystem @@ -211,8 +213,8 @@ def dhcp_discovery( @param interface: Name of the network interface on which to send a dhcp request - @param dhcp_log_func: A callable accepting the client output and - error streams. + @param dhcp_log_func: Callable accepting the interface and client's + stdout, stderr streams. @param distro: a distro object for network interface manipulation @return: dict of lease options representing the most recent dhcp lease parsed from the dhclient.lease file @@ -294,15 +296,15 @@ def get_newest_lease(self, interface: str) -> Dict[str, Any]: def dhcp_discovery( self, interface: str, - dhcp_log_func: Optional[Callable] = None, + dhcp_log_func: Optional[Callable[[str, str, str], None]] = None, distro=None, ) -> Dict[str, Any]: """Run dhclient on the interface without scripts/filesystem artifacts. @param interface: Name of the network interface on which to send a dhcp request - @param dhcp_log_func: A callable accepting the dhclient output and - error streams. + @param dhcp_log_func: Callable accepting the interface and client's + stdout, stderr streams. @param distro: a distro object for network interface manipulation @return: dict of lease options representing the most recent dhcp lease parsed from the dhclient.lease file @@ -420,7 +422,7 @@ def dhcp_discovery( 0.01 * 1000, ) if dhcp_log_func is not None: - dhcp_log_func(out, err) + dhcp_log_func(interface, out, err) lease = self.get_newest_lease(interface) if lease: return lease @@ -616,15 +618,15 @@ class Dhcpcd(DhcpClient): def dhcp_discovery( self, interface: str, - dhcp_log_func: Optional[Callable] = None, + dhcp_log_func: Optional[Callable[[str, str, str], None]] = None, distro=None, ) -> Dict[str, Any]: """Run dhcpcd on the interface without scripts/filesystem artifacts. @param interface: Name of the network interface on which to send a dhcp request - @param dhcp_log_func: A callable accepting the client output and - error streams. + @param dhcp_log_func: Callable accepting the interface and client's + stdout, stderr streams. @param distro: a distro object for network interface manipulation @return: dict of lease options representing the most recent dhcp lease parsed from the dhclient.lease file @@ -668,7 +670,7 @@ def dhcp_discovery( timeout=self.timeout, ) if dhcp_log_func is not None: - dhcp_log_func(out, err) + dhcp_log_func(interface, out, err) lease = self.get_newest_lease(interface) # Attempt cleanup and leave breadcrumbs if it fails, but return # the lease regardless of failure to clean up dhcpcd. @@ -919,14 +921,14 @@ def __init__(self): def dhcp_discovery( self, interface: str, - dhcp_log_func: Optional[Callable] = None, + dhcp_log_func: Optional[Callable[[str, str, str], None]] = None, distro=None, ) -> Dict[str, Any]: """Run udhcpc on the interface without scripts or filesystem artifacts. @param interface: Name of the network interface on which to run udhcpc. - @param dhcp_log_func: A callable accepting the udhcpc output and - error streams. + @param dhcp_log_func: Callable accepting the interface and client's + stdout, stderr streams. @return: A list of dicts of representing the dhcp leases parsed from the udhcpc lease file. """ @@ -984,7 +986,7 @@ def dhcp_discovery( raise NoDHCPLeaseError from error if dhcp_log_func is not None: - dhcp_log_func(out, err) + dhcp_log_func(interface, out, err) return self.get_newest_lease(interface) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 2eda92e7..a580ff0d 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -1,15 +1,24 @@ # This file is part of cloud-init. See LICENSE file for license information. import copy +import functools import glob import logging import os import re from contextlib import suppress -from typing import Optional +from typing import Any, Dict, List, Optional from cloudinit import performance, subp, util -from cloudinit.net import ParserError, renderer, subnet_is_ipv6 +from cloudinit.net import ( + ParserError, + is_ipv4_address, + is_ipv4_network, + is_ipv6_address, + is_ipv6_network, + renderer, + subnet_is_ipv6, +) from cloudinit.net.network_state import NetworkState LOG = logging.getLogger(__name__) @@ -62,7 +71,7 @@ # TODO: switch valid_map based on mode inet/inet6 -def _iface_add_subnet(iface, subnet): +def _iface_add_subnet(iface: dict, subnet: dict, is_ipv6: bool) -> List[str]: content = [] valid_map = [ "address", @@ -83,7 +92,21 @@ def _iface_add_subnet(iface, subnet): value = "%s/%s" % (subnet["address"], subnet["prefix"]) if value and key in valid_map: if isinstance(value, list): + if key == "dns_nameservers": + value = list( + filter( + functools.partial( + has_same_ip_version, is_ipv6=is_ipv6 + ), + value, + ) + ) value = " ".join(value) + else: + if key == "dns_nameservers" and not has_same_ip_version( + value, is_ipv6 + ): + continue if "_" in key: key = key.replace("_", "-") content.append(" {0} {1}".format(key, value)) @@ -92,7 +115,9 @@ def _iface_add_subnet(iface, subnet): # TODO: switch to valid_map for attrs -def _iface_add_attrs(iface, index, ipv4_subnet_mtu): +def _iface_add_attrs( + iface: dict, index: int, ipv4_subnet_mtu: Optional[str] +) -> List[str]: # If the index is non-zero, this is an alias interface. Alias interfaces # represent additional interface addresses, and should not have additional # attributes. (extra attributes here are almost always either incorrect, @@ -104,6 +129,7 @@ def _iface_add_attrs(iface, index, ipv4_subnet_mtu): ignore_map = [ "control", "device_id", + "dns", "driver", "index", "inet", @@ -126,6 +152,9 @@ def _iface_add_attrs(iface, index, ipv4_subnet_mtu): ignore_map.append("mac_address") for key, value in iface.items(): + key_write = renames.get(key, key) + if "_" in key_write: + key_write = key_write.replace("_", "-") # convert bool to string for eni if isinstance(value, bool): value = "on" if iface[key] else "off" @@ -143,16 +172,18 @@ def _iface_add_attrs(iface, index, ipv4_subnet_mtu): continue if key in multiline_keys: for v in value: - content.append(" {0} {1}".format(renames.get(key, key), v)) + content.append(" {0} {1}".format(key_write, v)) continue if isinstance(value, list): value = " ".join(value) - content.append(" {0} {1}".format(renames.get(key, key), value)) + content.append(" {0} {1}".format(key_write, value)) return sorted(content) -def _iface_start_entry(iface, index, render_hwaddress=False): +def _iface_start_entry( + iface: dict, index, render_hwaddress: bool = False +) -> List[str]: fullname = iface["name"] control = iface["control"] @@ -176,7 +207,9 @@ def _iface_start_entry(iface, index, render_hwaddress=False): return lines -def _parse_deb_config_data(ifaces, contents, src_dir, src_path): +def _parse_deb_config_data( + ifaces: dict, contents: str, src_dir: str, src_path: str +) -> None: """Parses the file contents, placing result into ifaces. '_source_path' is added to every dictionary entry to define which file @@ -309,14 +342,14 @@ def _parse_deb_config_data(ifaces, contents, src_dir, src_path): @performance.timed("Converting eni data") -def convert_eni_data(eni_data): +def convert_eni_data(eni_data: str) -> dict: """Return a network config representation of what is in eni_data""" - ifaces = {} - _parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None) + ifaces: dict = {} + _parse_deb_config_data(ifaces, eni_data, src_dir="None", src_path="None") return _ifaces_to_net_config_data(ifaces) -def _ifaces_to_net_config_data(ifaces): +def _ifaces_to_net_config_data(ifaces: dict) -> dict: """Return network config that represents the ifaces data provided. ifaces = _parse_deb_config_data(...) config = ifaces_to_net_config_data(ifaces) @@ -357,10 +390,16 @@ def _ifaces_to_net_config_data(ifaces): return {"version": 1, "config": [devs[d] for d in sorted(devs)]} +def has_same_ip_version(addr_or_net: str, is_ipv6: bool) -> bool: + if not is_ipv6: + return is_ipv4_address(addr_or_net) or is_ipv4_network(addr_or_net) + return is_ipv6_address(addr_or_net) or is_ipv6_network(addr_or_net) + + class Renderer(renderer.Renderer): """Renders network information in a /etc/network/interfaces format.""" - def __init__(self, config=None): + def __init__(self, config: Optional[dict] = None): if not config: config = {} self.eni_path = config.get("eni_path", "etc/network/interfaces") @@ -369,7 +408,7 @@ def __init__(self, config=None): "netrules_path", "etc/udev/rules.d/70-persistent-net.rules" ) - def _render_route(self, route, indent=""): + def _render_route(self, route: dict, indent: str = "") -> List[str]: """When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces some confusing output for users manually using ifup/ifdown[1]. To @@ -407,7 +446,7 @@ def _render_route(self, route, indent=""): route_line += "%s %s %s" % (default_gw, mapping[k], route[k]) elif k in route: if k == "network": - if ":" in route[k]: + if is_ipv6_address(route[k]): route_line += " -A inet6" elif route.get("prefix") == 32: route_line += " -host" @@ -421,13 +460,15 @@ def _render_route(self, route, indent=""): content.append(down + route_line + or_true) return content - def _render_iface(self, iface, render_hwaddress=False): + def _render_iface( + self, iface: dict, render_hwaddress: bool = False + ) -> List[List[str]]: iface = copy.deepcopy(iface) # Remove irrelevant keys with suppress(KeyError): iface.pop("config_id") - sections = [] + sections: List[List[str]] = [] subnets = iface.get("subnets", {}) accept_ra = iface.pop("accept-ra", None) ethernet_wol = iface.pop("wakeonlan", None) @@ -435,6 +476,8 @@ def _render_iface(self, iface, render_hwaddress=False): # Specify WOL setting 'g' for using "Magic Packet" iface["ethernet-wol"] = "g" if subnets: + dns = None + routes6 = [] for index, subnet in enumerate(subnets): ipv4_subnet_mtu = None iface["index"] = index @@ -442,8 +485,10 @@ def _render_iface(self, iface, render_hwaddress=False): iface["control"] = subnet.get("control", "auto") subnet_inet = "inet" if subnet_is_ipv6(subnet): + is_ipv6 = True subnet_inet += "6" else: + is_ipv6 = False ipv4_subnet_mtu = subnet.get("mtu") iface["inet"] = subnet_inet if ( @@ -480,16 +525,60 @@ def _render_iface(self, iface, render_hwaddress=False): ]: iface["control"] = "alias" + # v1 config has the dns info in the first non-dhcp route, + # replicate dns to others to be more correct + dns_present = ( + "dns_search" in subnet or "dns_nameservers" in subnet + ) + if dns is None and dns_present: + dns = dict( + (k, subnet.get(k)) + for k in ("dns_search", "dns_nameservers") + ) + if dns is not None and not dns_present: + subnet = {**subnet, **dns} + lines = list( _iface_start_entry( iface, index, render_hwaddress=render_hwaddress ) - + _iface_add_subnet(iface, subnet) + + _iface_add_subnet(iface, subnet, is_ipv6) + _iface_add_attrs(iface, index, ipv4_subnet_mtu) ) for route in subnet.get("routes", []): + ipv6_network = is_ipv6_network(route.get("network", "")) + if ipv6_network and not is_ipv6: + routes6.append(route) + continue lines.extend(self._render_route(route, indent=" ")) + if routes6 and is_ipv6: + for route in routes6: + lines.extend(self._render_route(route, indent=" ")) + routes6.clear() + + sections.append(lines) + + if routes6: + # no ipv6 subnet found create a static one to add remaining + # routes: + iface = { + "name": iface["name"], + "control": iface["control"], + "mode": "static", + "inet": "inet6", + } + subnet = {"type": "static", "routes": routes6} + if dns is not None: + subnet = {**subnet, **dns} + lines = list( + _iface_start_entry( + iface, -1, render_hwaddress=render_hwaddress + ) + + _iface_add_subnet(iface, subnet, True) + ) + for route in subnet["routes"]: + lines.extend(self._render_route(route, indent=" ")) sections.append(lines) else: # ifenslave docs say to auto the slave devices @@ -503,12 +592,14 @@ def _render_iface(self, iface, render_hwaddress=False): sections.append(lines) return sections - def _render_interfaces(self, network_state, render_hwaddress=False): + def _render_interfaces( + self, network_state: NetworkState, render_hwaddress: bool = False + ) -> str: """Given state, emit etc/network/interfaces content.""" # handle 'lo' specifically as we need to insert the global dns entries # there (as that is the only interface that will be always up). - lo = { + lo: Dict[str, Any] = { "name": "lo", "type": "physical", "inet": "inet", @@ -520,11 +611,11 @@ def _render_interfaces(self, network_state, render_hwaddress=False): nameservers = network_state.dns_nameservers if nameservers: - lo["subnets"][0]["dns_nameservers"] = " ".join(nameservers) + lo["subnets"][0]["dns_nameservers"] = nameservers searchdomains = network_state.dns_searchdomains if searchdomains: - lo["subnets"][0]["dns_search"] = " ".join(searchdomains) + lo["subnets"][0]["dns_search"] = searchdomains # Apply a sort order to ensure that we write out the physical # interfaces first; this is critical for bonding @@ -559,7 +650,7 @@ def render_network_state( self, network_state: NetworkState, templates: Optional[dict] = None, - target=None, + target: Optional[str] = None, ) -> None: fpeni = subp.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) @@ -576,13 +667,13 @@ def render_network_state( ) -def available(target=None): +def available() -> bool: expected = ["ifquery", "ifup", "ifdown"] search = ["/sbin", "/usr/sbin"] for p in expected: - if not subp.which(p, search=search, target=target): + if not subp.which(p, search=search): return False - eni = subp.target_path(target, "etc/network/interfaces") + eni = "/etc/network/interfaces" if not os.path.isfile(eni): return False diff --git a/cloudinit/net/ephemeral.py b/cloudinit/net/ephemeral.py index 937c77f5..08e6086e 100644 --- a/cloudinit/net/ephemeral.py +++ b/cloudinit/net/ephemeral.py @@ -283,7 +283,7 @@ def __init__( distro, iface=None, connectivity_urls_data: Optional[List[Dict[str, Any]]] = None, - dhcp_log_func=None, + dhcp_log_func: Optional[Callable[[str, str, str], None]] = None, ): self.iface = iface self._ephipv4: Optional[EphemeralIPv4Network] = None diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 50af1769..ae90db4b 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -4,6 +4,7 @@ import io import logging import os +import re import textwrap from tempfile import SpooledTemporaryFile from typing import Callable, List, Optional @@ -107,12 +108,24 @@ def _listify(obj, token=" "): subnets = [] for subnet in subnets: sn_type = subnet.get("type") + sn_metric = subnet.get("metric") if sn_type.startswith("dhcp"): if sn_type == "dhcp": sn_type += "4" entry.update({sn_type: True}) + if sn_metric is not None: + # Add metric to DHCP override if specified in the subnet + dhcp_override_key = f"{sn_type}-overrides" + dhcp_override = entry.get(dhcp_override_key, {}) + dhcp_override["route-metric"] = sn_metric + entry.update({dhcp_override_key: dhcp_override}) elif sn_type in IPV6_DYNAMIC_TYPES: entry.update({"dhcp6": True}) + if sn_metric is not None: + # Add metric to DHCP6 override if specified in the subnet``` + dhcp_override = entry.get("dhcp6-overrides", {}) + dhcp_override["route-metric"] = sn_metric + entry.update({"dhcp6-overrides": dhcp_override}) elif sn_type in ["static", "static6"]: addr = "%s" % subnet.get("address") if "prefix" in subnet: @@ -133,6 +146,9 @@ def _listify(obj, token=" "): addr, ) new_route["on-link"] = True + if sn_metric is not None: + # Add metric to default route if specified in the subnet + new_route["metric"] = sn_metric routes.append(new_route) if "dns_nameservers" in subnet: nameservers += _listify(subnet.get("dns_nameservers", [])) @@ -149,8 +165,12 @@ def _listify(obj, token=" "): "via": route.get("gateway"), "to": to_net, } - if "metric" in route: - new_route.update({"metric": route.get("metric", 100)}) + # Priority for metric: 1. route's metric, 2. subnet's metric + route_metric = route.get("metric") + if route_metric is not None: + new_route["metric"] = route_metric + elif sn_metric is not None: + new_route["metric"] = sn_metric routes.append(new_route) addresses.append(addr) @@ -246,6 +266,7 @@ def netplan_api_write_yaml_file(net_config_content: str) -> bool: CLOUDINIT_NETPLAN_FILE, ) return False + net_config_content = _maybe_strip_invalid_mtu(net_config_content) try: with SpooledTemporaryFile(mode="w") as f: f.write(net_config_content) @@ -275,6 +296,29 @@ def netplan_api_write_yaml_file(net_config_content: str) -> bool: return True +def _maybe_strip_invalid_mtu(net_config_content: str): + """Strip invalid MTU from the netplan config. + + This is a fix for https://github.com/canonical/cloud-init/issues/6239 + A 0 MTU value is NOT valid, but cloud-init accepted it prior to 24.2, + so rejecting it after c465de8 is a breaking change for existing releases. + """ + if features.STRIP_INVALID_MTU: + # Using regex here is admittedly not great, but this is post-processing + # of the netplan config, and we'd have to be dealing with some very + # gnarly yaml to get a multiline mtu: 0 entry. The alternative is + # another round trip of yaml parsing, which is more expensive. Unless + # there's a demonstrated need for proper yaml parsing, the added + # complexity does not seem worth it. + net_config_content = re.sub( + r"^\s*mtu:\s*0\s*\n", + "", + net_config_content, + flags=re.MULTILINE, + ) + return net_config_content + + def has_netplan_config_changed(cfg_file: str, content: str) -> bool: """Return True when new netplan config has changed vs previous.""" if not os.path.exists(cfg_file): @@ -437,6 +481,8 @@ def _render_content(self, network_state: NetworkState) -> str: "set-name": ifname, "match": ifcfg.get("match", None), } + if "keep_configuration" in ifcfg: + eth["critical"] = ifcfg["keep_configuration"] if eth["match"] is None: macaddr = ifcfg.get("mac_address", None) if macaddr is not None: @@ -563,10 +609,10 @@ def _render_section(name, section): return "".join(content) -def available(target=None): +def available(): expected = ["netplan"] search = ["/usr/sbin", "/sbin"] for p in expected: - if not subp.which(p, search=search, target=target): + if not subp.which(p, search=search): return False return True diff --git a/cloudinit/net/network_manager.py b/cloudinit/net/network_manager.py index cc907ce2..d4c705f2 100644 --- a/cloudinit/net/network_manager.py +++ b/cloudinit/net/network_manager.py @@ -646,12 +646,12 @@ def cloud_init_nm_conf_filename(target=None): return f"{target_con_dir}/conf.d/{conf_file}" -def available(target=None): +def available(): # TODO: Move `uses_systemd` to a more appropriate location # It is imported here to avoid circular import from cloudinit.distros import uses_systemd - nmcli_present = subp.which("nmcli", target=target) + nmcli_present = subp.which("nmcli") service_active = True if uses_systemd(): try: diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 2bdd2b61..9fd51e42 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -428,6 +428,7 @@ def handle_physical(self, command): "accept-ra": accept_ra, "wakeonlan": wakeonlan, "optional": optional, + "keep_configuration": command.get("keep_configuration"), } ) iface_key = command.get("config_id", command.get("name")) diff --git a/cloudinit/net/networkd.py b/cloudinit/net/networkd.py index 345935d6..8672e56e 100644 --- a/cloudinit/net/networkd.py +++ b/cloudinit/net/networkd.py @@ -1,12 +1,11 @@ -# Copyright (C) 2021-2022 VMware Inc. +# Copyright (C) 2021-2025 VMware by Broadcom. # # Author: Shreenidhi Shedi # # This file is part of cloud-init. See LICENSE file for license information. import logging -from collections import OrderedDict -from typing import Optional +from typing import Any, Dict, List, Optional from cloudinit import subp, util from cloudinit.net import renderer, should_add_gateway_onlink_flag @@ -15,25 +14,64 @@ LOG = logging.getLogger(__name__) +def normalize(data: Dict[str, List[Any]]) -> Dict[str, List[Any]]: + """ + Normalize a dictionary of lists. + - Assumes top-level keys map to lists. + - Each list and any nested dicts/lists will be recursively normalized. + """ + normalized = {} + for key, value in data.items(): + normalized[key] = _normalize_value(value) + return normalized + + +def _normalize_value(data: Any) -> Any: + """ + Recursively normalize a dictionary or list: + - Dicts: keys sorted, values normalized + - Lists: items normalized and sorted (if comparable) + """ + if isinstance(data, dict): + normalized = {} + for key in sorted(data): + normalized[key] = _normalize_value(data[key]) + return normalized + elif isinstance(data, list): + normalized_items = [] + for item in data: + if isinstance(item, (dict, list)): + normalized_item = _normalize_value(item) + else: + normalized_item = item + normalized_items.append(normalized_item) + try: + return sorted(normalized_items) + except TypeError: + return normalized_items + + return data + + class CfgParser: def __init__(self): - self.conf_dict = OrderedDict( - { - "Match": [], - "Link": [], - "Network": [], - "DHCPv4": [], - "DHCPv6": [], - "Address": [], - "Route": {}, - } - ) + self.conf_dict = { + "Match": [], + "Link": [], + "Network": [], + "DHCPv4": [], + "DHCPv6": [], + "Address": [], + "Route": {}, + "NetDev": [], + "VLAN": [], + "Bond": [], + } def update_section(self, sec, key, val): for k in self.conf_dict.keys(): if k == sec: - self.conf_dict[k].append(key + "=" + str(val)) - # remove duplicates from list + self.conf_dict[k].append(f"{key}={val}") self.conf_dict[k] = list(dict.fromkeys(self.conf_dict[k])) self.conf_dict[k].sort() @@ -46,7 +84,7 @@ def update_route_section(self, sec, rid, key, val): if k == sec: if rid not in self.conf_dict[k]: self.conf_dict[k][rid] = [] - self.conf_dict[k][rid].append(key + "=" + str(val)) + self.conf_dict[k][rid].append(f"{key}={val}") # remove duplicates from list self.conf_dict[k][rid] = list( dict.fromkeys(self.conf_dict[k][rid]) @@ -55,24 +93,25 @@ def update_route_section(self, sec, rid, key, val): def get_final_conf(self): contents = "" + + self.conf_dict = normalize(self.conf_dict) + for k, v in sorted(self.conf_dict.items()): if not v: continue if k == "Address": for e in sorted(v): - contents += "[" + k + "]\n" - contents += e + "\n" - contents += "\n" + contents += f"[{k}]\n{e}\n\n" elif k == "Route": for n in sorted(v): - contents += "[" + k + "]\n" + contents += f"[{k}]\n" for e in sorted(v[n]): - contents += e + "\n" + contents += f"{e}\n" contents += "\n" else: - contents += "[" + k + "]\n" + contents += f"[{k}]\n" for e in sorted(v): - contents += e + "\n" + contents += f"{e}\n" contents += "\n" return contents @@ -101,9 +140,11 @@ def generate_match_section(self, iface, cfg: CfgParser): match_dict = { "name": "Name", "driver": "Driver", - "mac_address": "MACAddress", } + if iface["type"] == "physical": + match_dict["mac_address"] = "MACAddress" + if not iface: return @@ -119,9 +160,12 @@ def generate_link_section(self, iface, cfg: CfgParser): if not iface: return - if "mtu" in iface and iface["mtu"]: + if iface.get("mtu"): cfg.update_section(sec, "MTUBytes", iface["mtu"]) + if iface["type"] != "physical" and iface.get("mac_address"): + cfg.update_section(sec, "MACAddress", iface["mac_address"]) + if "optional" in iface and iface["optional"]: cfg.update_section(sec, "RequiredForOnline", "no") @@ -140,7 +184,7 @@ def parse_routes(self, rid, conf, cfg: CfgParser): # prefix is derived using netmask by network_state prefix = "" if "prefix" in conf: - prefix = "/" + str(conf["prefix"]) + prefix = f"/{conf['prefix']}" for k, v in conf.items(): if k not in route_cfg_map: @@ -155,7 +199,7 @@ def parse_subnets(self, iface, cfg: CfgParser): rid = 0 for e in iface.get("subnets", []): t = e["type"] - if t == "dhcp4" or t == "dhcp": + if t in {"dhcp4", "dhcp"}: if dhcp == "no": dhcp = "ipv4" elif dhcp == "ipv6": @@ -174,7 +218,7 @@ def parse_subnets(self, iface, cfg: CfgParser): if "address" in e: addr = e["address"] if "prefix" in e: - addr += "/" + str(e["prefix"]) + addr += f"/{e['prefix']}" subnet_cfg_map = { "address": "Address", "gateway": "Gateway", @@ -201,13 +245,16 @@ def parse_subnets(self, iface, cfg: CfgParser): "Route", f"a{rid}", "GatewayOnLink", "yes" ) rid = rid + 1 - elif k == "dns_nameservers" or k == "dns_search": + elif k in {"dns_nameservers", "dns_search"}: cfg.update_section(sec, subnet_cfg_map[k], " ".join(v)) cfg.update_section(sec, "DHCP", dhcp) - if isinstance(iface.get("accept-ra", ""), bool): - cfg.update_section(sec, "IPv6AcceptRA", iface["accept-ra"]) + if isinstance(iface.get("accept-ra"), bool): + val = "no" + if iface["accept-ra"]: + val = "yes" + cfg.update_section(sec, "IPv6AcceptRA", val) return dhcp @@ -275,12 +322,14 @@ def parse_dhcp_overrides(self, cfg: CfgParser, device, dhcp, version): if v in dhcp_overrides: cfg.update_section(f"DHCPv{version}", k, dhcp_overrides[v]) - def create_network_file(self, link, conf, nwk_dir): + def create_network_file(self, link, conf, nwk_dir, ext=".network"): net_fn_owner = "systemd-network" LOG.debug("Setting Networking Config for %s", link) + net_fn = f"{nwk_dir}10-cloud-init-{link}{ext}" - net_fn = nwk_dir + "10-cloud-init-" + link + ".network" + if conf.endswith("\n\n"): + conf = conf[:-1] util.write_file(net_fn, conf) util.chownbyname(net_fn, net_fn_owner, net_fn_owner) @@ -296,16 +345,65 @@ def render_network_state( util.ensure_dir(network_dir) - ret_dict = self._render_content(network_state) - for k, v in ret_dict.items(): + network = self._render_content(network_state) + vlan_netdev = network.pop("vlan_netdev", {}) + bond_netdev = network.pop("bond_netdev", {}) + + for k, v in network.items(): self.create_network_file(k, v, network_dir) - def _render_content(self, ns: NetworkState) -> dict: + for k, v in vlan_netdev.items(): + self.create_network_file(k, v, network_dir, ext=".netdev") + + for k, v in bond_netdev.items(): + self.create_network_file(k, v, network_dir, ext=".netdev") + + def _render_content(self, ns: NetworkState): ret_dict = {} + vlan_link = {} + bond_link = {} + + if "vlans" in ns.config: + vlan_dict = self.render_vlans(ns) + vlan_netdev = vlan_dict["vlan_netdev"] + vlan_link = vlan_dict["vlan_link"] + ret_dict["vlan_netdev"] = vlan_netdev + + if "bonds" in ns.config: + bond_dict = self.render_bonds(ns) + bond_netdev = bond_dict["bond_netdev"] + bond_link = bond_dict["bond_link"] + ret_dict["bond_netdev"] = bond_netdev + for iface in ns.iter_interfaces(): cfg = CfgParser() + iface_name = iface["name"] + + vlan_link_name = vlan_link.get(iface_name) + if vlan_link_name: + cfg.update_section("Network", "VLAN", vlan_link_name) + + # TODO: revisit this once network state renders macaddress + # properly for vlan config + if not iface["mac_address"] and vlan_link.get("macaddress"): + mac = vlan_link["macaddress"].get(iface_name) + if mac: + iface["mac_address"] = mac + + bond_link_name = bond_link.get(iface_name) + if bond_link_name: + cfg.update_section("Network", "Bond", bond_link_name) + + # TODO: revisit this once network state renders macaddress + # properly for bond config + if not iface["mac_address"] and bond_link.get("macaddress"): + mac = bond_link["macaddress"].get(iface_name) + if mac: + iface["mac_address"] = mac + link = self.generate_match_section(iface, cfg) + self.generate_link_section(iface, cfg) dhcp = self.parse_subnets(iface, cfg) self.parse_dns(iface, cfg, ns) @@ -358,11 +456,173 @@ def _render_content(self, ns: NetworkState) -> dict: return ret_dict + def render_vlans(self, ns: NetworkState) -> dict: + vlan_link_info: Dict[str, Any] = {} + vlan_ndev_configs = {} + vlan_link_info["macaddress"] = {} + + vlans = ns.config.get("vlans", {}) + for vlan_name, vlan_cfg in vlans.items(): + vlan_id = vlan_cfg.get("id") + parent = vlan_cfg.get("link") + + if vlan_id is None or parent is None: + LOG.warning( + "Skipping VLAN %s - missing 'id' or 'link'", vlan_name + ) + continue + + vlan_link_info[parent] = vlan_name + + # -------- .netdev for VLAN -------- + cfg = CfgParser() + cfg.update_section("NetDev", "Name", vlan_name) + cfg.update_section("NetDev", "Kind", "vlan") + + val = vlan_cfg.get("mtu") + if val: + cfg.update_section("NetDev", "MTUBytes", val) + + val = vlan_cfg.get("macaddress") + if val: + val = val.lower() + cfg.update_section("NetDev", "MACAddress", val) + vlan_link_info["macaddress"][vlan_name] = val + + cfg.update_section("VLAN", "Id", vlan_id) + vlan_ndev_configs[vlan_name] = cfg.get_final_conf() + + ret_dict = { + "vlan_netdev": vlan_ndev_configs, + "vlan_link": vlan_link_info, + } + return ret_dict + + def render_bonds(self, ns: NetworkState) -> dict: + bond_link_info: Dict[str, Any] = {} + bond_ndev_configs = {} + section = "Bond" + + bond_link_info["macaddress"] = {} + + bonds = ns.config.get("bonds", {}) + for bond_name, bond_cfg in bonds.items(): + interfaces = bond_cfg.get("interfaces") + if not interfaces: + LOG.warning( + "Skipping bond %s - missing 'interfaces'", bond_name + ) + continue + + bond_link_info.update({iface: bond_name for iface in interfaces}) + + # -------- .netdev for Bond -------- + cfg = CfgParser() + cfg.update_section("NetDev", "Name", bond_name) + cfg.update_section("NetDev", "Kind", "bond") + + val = bond_cfg.get("mtu") + if val: + cfg.update_section("NetDev", "MTUBytes", val) + + val = bond_cfg.get("macaddress") + if val: + val = val.lower() + cfg.update_section("NetDev", "MACAddress", val) + bond_link_info["macaddress"][bond_name] = val + + # Optional bond parameters + params = bond_cfg.get("parameters", {}) + + if "mode" in params: + cfg.update_section(section, "Mode", params["mode"]) + + if "mii-monitor-interval" in params: + cfg.update_section( + section, + "MIIMonitorSec", + f"{params['mii-monitor-interval']}ms", + ) + + if "updelay" in params: + cfg.update_section( + section, "UpDelaySec", f"{params['updelay']}ms" + ) + + if "downdelay" in params: + cfg.update_section( + section, "DownDelaySec", f"{params['downdelay']}ms" + ) + + if "arp-interval" in params: + cfg.update_section( + section, "ARPIntervalSec", f"{params['arp-interval']}ms" + ) + + if "arp-ip-target" in params: + targets = params["arp-ip-target"] + if isinstance(targets, str): + targets = [targets] + ip_list = " ".join(targets) + cfg.update_section(section, "ARPIPTargets", ip_list) + + if "arp-validate" in params: + cfg.update_section( + section, "ARPValidate", params["arp-validate"] + ) + + if "arp-all-targets" in params: + cfg.update_section( + section, "ARPAllTargets", params["arp-all-targets"] + ) + + if "primary-reselect" in params: + cfg.update_section( + section, + "PrimaryReselectPolicy", + params["primary-reselect"], + ) + + if "lacp-rate" in params: + cfg.update_section( + section, "LACPTransmitRate", params["lacp-rate"] + ) + + if "transmit-hash-policy" in params: + cfg.update_section( + section, + "TransmitHashPolicy", + params["transmit-hash-policy"], + ) + + if "ad-select" in params: + cfg.update_section(section, "AdSelect", params["ad-select"]) + + if "min-links" in params: + cfg.update_section( + section, "MinLinks", str(params["min-links"]) + ) + + if "all-slaves-active" in params: + cfg.update_section( + section, + "AllSlavesActive", + str(params["all-slaves-active"]).lower(), + ) + + bond_ndev_configs[bond_name] = cfg.get_final_conf() + + ret_dict = { + "bond_netdev": bond_ndev_configs, + "bond_link": bond_link_info, + } + return ret_dict + def available(target=None): expected = ["ip", "systemctl"] search = ["/usr/sbin", "/bin"] for p in expected: - if not subp.which(p, search=search, target=target): + if not subp.which(p, search=search): return False return True diff --git a/cloudinit/net/renderers.py b/cloudinit/net/renderers.py index e201bfe4..ab7b7d8b 100644 --- a/cloudinit/net/renderers.py +++ b/cloudinit/net/renderers.py @@ -39,7 +39,7 @@ def search( - priority=None, target=None, first=False + priority=None, first=False ) -> List[Tuple[str, Type[renderer.Renderer]]]: if priority is None: priority = DEFAULT_PRIORITY @@ -55,7 +55,7 @@ def search( found = [] for name in priority: render_mod = available[name] - if render_mod.available(target): + if render_mod.available(): cur = (name, render_mod.Renderer) if first: return [cur] @@ -64,16 +64,13 @@ def search( return found -def select(priority=None, target=None) -> Tuple[str, Type[renderer.Renderer]]: - found = search(priority, target=target, first=True) +def select(priority=None) -> Tuple[str, Type[renderer.Renderer]]: + found = search(priority, first=True) if not found: if priority is None: priority = DEFAULT_PRIORITY - tmsg = "" - if target and target != "/": - tmsg = " in target=%s" % target raise RendererNotFoundError( - "No available network renderers found%s. Searched through list: %s" - % (tmsg, priority) + "No available network renderers found. Searched through list: %s" + % priority ) return found[0] diff --git a/cloudinit/signal_handler.py b/cloudinit/signal_handler.py index 51a7610c..405e9055 100644 --- a/cloudinit/signal_handler.py +++ b/cloudinit/signal_handler.py @@ -5,15 +5,13 @@ # Author: Joshua Harlow # # This file is part of cloud-init. See LICENSE file for license information. -import contextlib import inspect import logging import signal import sys -import threading import types from io import StringIO -from typing import Callable, Dict, Final, NamedTuple, Union +from typing import Callable, Dict, Final, Union from cloudinit import version as vr from cloudinit.log import log_util @@ -29,17 +27,6 @@ } -class ExitBehavior(NamedTuple): - exit_code: int - log_level: int - - -SIGNAL_EXIT_BEHAVIOR_CRASH: Final = ExitBehavior(1, logging.ERROR) -SIGNAL_EXIT_BEHAVIOR_QUIET: Final = ExitBehavior(0, logging.INFO) -_SIGNAL_EXIT_BEHAVIOR = SIGNAL_EXIT_BEHAVIOR_CRASH -_SUSPEND_WRITE_LOCK = threading.RLock() - - def inspect_handler(sig: Union[int, Callable, None]) -> None: """inspect_handler() logs signal handler state""" if callable(sig): @@ -80,9 +67,12 @@ def _handle_exit(signum, frame): contents = StringIO(SIG_MESSAGE.format(vr.version_string(), name)) _pprint_frame(frame, 1, BACK_FRAME_TRACE_DEPTH, contents) log_util.multi_log( - contents.getvalue(), log=LOG, log_level=_SIGNAL_EXIT_BEHAVIOR.log_level + f"Received signal {name} resulting in exit. Cause:\n" + + contents.getvalue(), + log=LOG, + log_level=logging.INFO, ) - sys.exit(_SIGNAL_EXIT_BEHAVIOR.exit_code) + sys.exit(0) def attach_handlers(): @@ -92,23 +82,3 @@ def attach_handlers(): inspect_handler(signal.signal(signum, _handle_exit)) sigs_attached += len(SIGNALS) return sigs_attached - - -@contextlib.contextmanager -def suspend_crash(): - """suspend_crash() allows signals to be received without exiting 1 - - This allow signal handling without a crash where it is expected. The - call stack is still printed if signal is received during this context, but - the return code is 0 and no traceback is printed. - - Threadsafe. - """ - global _SIGNAL_EXIT_BEHAVIOR - - # If multiple threads simultaneously were to modify this - # global state, this function would not behave as expected. - with _SUSPEND_WRITE_LOCK: - _SIGNAL_EXIT_BEHAVIOR = SIGNAL_EXIT_BEHAVIOR_QUIET - yield - _SIGNAL_EXIT_BEHAVIOR = SIGNAL_EXIT_BEHAVIOR_CRASH diff --git a/cloudinit/socket.py b/cloudinit/socket.py index 8acabcd0..98c82886 100644 --- a/cloudinit/socket.py +++ b/cloudinit/socket.py @@ -135,6 +135,9 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): """Notify the socket that this stage is complete.""" + if os.isatty(sys.stdin.fileno()): + # See corresponding log for __enter__() + return message = f"Completed socket interaction for boot stage {self.stage}" if exc_type: # handle exception thrown in context diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index d0746dd3..80ac60b7 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -20,6 +20,7 @@ import requests from cloudinit import net, performance, sources, ssh_util, subp, util +from cloudinit.config import cc_mounts from cloudinit.event import EventScope, EventType from cloudinit.net import device_driver from cloudinit.net.dhcp import ( @@ -332,24 +333,28 @@ def __init__(self, sys_cfg, distro, paths): self._iso_dev = None self._network_config = None self._ephemeral_dhcp_ctx: Optional[EphemeralDHCPv4] = None - self._route_configured_for_imds = False - self._route_configured_for_wireserver = False - self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT self._reported_ready_marker_file = os.path.join( paths.cloud_dir, "data", "reported_ready" ) + self._route_configured_for_imds = False + self._route_configured_for_wireserver = False + self._system_uuid = None + self._vm_id = None + self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT def _unpickle(self, ci_pkl_version: int) -> None: super()._unpickle(ci_pkl_version) self._ephemeral_dhcp_ctx = None self._iso_dev = None - self._route_configured_for_imds = False - self._route_configured_for_wireserver = False - self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT self._reported_ready_marker_file = os.path.join( self.paths.cloud_dir, "data", "reported_ready" ) + self._route_configured_for_imds = False + self._route_configured_for_wireserver = False + self._system_uuid = None + self._vm_id = None + self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT def __str__(self): root = sources.DataSource.__str__(self) @@ -620,6 +625,13 @@ def crawl_metadata(self): @raise: InvalidMetaDataException when the expected metadata service is unavailable, broken or disabled. """ + self._query_vm_id() + report_diagnostic_event( + "Azure VM ID: %s System UUID: %s" + % (self._vm_id, self._system_uuid), + logger_func=LOG.info, + ) + crawled_data = {} # azure removes/ejects the cdrom containing the ovf-env.xml # file on reboot. So, in order to successfully reboot we @@ -773,9 +785,9 @@ def crawl_metadata(self): if self.seed == "IMDS" and not crawled_data["files"]: try: contents = build_minimal_ovf( - username=imds_username, # pyright: ignore - hostname=imds_hostname, # pyright: ignore - disableSshPwd=imds_disable_password, # pyright: ignore + username=imds_username, + hostname=imds_hostname, + disable_ssh_password_auth=imds_disable_password, ) crawled_data["files"] = {"ovf-env.xml": contents} except Exception as e: @@ -1037,22 +1049,46 @@ def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) + def _query_vm_id(self): + """Query the system UUID and VM IDs, if needed. + + They are initialized to None, check only if they are unset. + + Raise as reportable error on failure. + """ + if not self._system_uuid: + try: + self._system_uuid = identity.query_system_uuid() + except RuntimeError as error: + raise errors.ReportableErrorVmIdentification(exception=error) + + if not self._vm_id: + try: + self._vm_id = identity.convert_system_uuid_to_vm_id( + self._system_uuid + ) + except ValueError as error: + raise errors.ReportableErrorVmIdentification( + exception=error, system_uuid=self._system_uuid + ) + def _iid(self, previous=None): + self._query_vm_id() + prev_iid_path = os.path.join( self.paths.get_cpath("data"), "instance-id" ) - system_uuid = identity.query_system_uuid() if os.path.exists(prev_iid_path): previous = util.load_text_file(prev_iid_path).strip() - swapped_id = identity.byte_swap_system_uuid(system_uuid) + swapped_id = identity.byte_swap_system_uuid(self._system_uuid) # Older kernels than 4.15 will have UPPERCASE product_uuid. # We don't want Azure to react to an UPPER/lower difference as # a new instance id as it rewrites SSH host keys. # LP: #1835584 - if previous.lower() in [system_uuid, swapped_id]: + if previous.lower() in [self._system_uuid, swapped_id]: return previous - return system_uuid + return self._system_uuid @azure_ds_telemetry_reporter def _wait_for_nic_detach(self, nl_sock): @@ -1353,12 +1389,13 @@ def _report_failure( @param host_only: Only report to host (error may be recoverable). @return: The success status of sending the failure signal. """ + encoded_report = error.as_encoded_report(vm_id=self._vm_id) report_diagnostic_event( - f"Azure datasource failure occurred: {error.as_encoded_report()}", + f"Azure datasource failure occurred: {encoded_report}", logger_func=LOG.error, ) report_dmesg_to_kvp() - reported = kvp.report_failure_to_host(error) + reported = kvp.report_via_kvp(encoded_report) if host_only: return reported @@ -1370,7 +1407,8 @@ def _report_failure( logger_func=LOG.debug, ) report_failure_to_fabric( - endpoint=self._wireserver_endpoint, error=error + endpoint=self._wireserver_endpoint, + encoded_report=encoded_report, ) self._negotiated = True return True @@ -1393,7 +1431,8 @@ def _report_failure( # Reporting failure will fail, but it will emit telemetry. pass report_failure_to_fabric( - endpoint=self._wireserver_endpoint, error=error + endpoint=self._wireserver_endpoint, + encoded_report=encoded_report, ) self._negotiated = True return True @@ -1418,7 +1457,7 @@ def _report_ready( :returns: List of SSH keys, if requested. """ report_dmesg_to_kvp() - kvp.report_success_to_host() + kvp.report_success_to_host(vm_id=self._vm_id) try: data = get_metadata_from_fabric( @@ -1631,6 +1670,18 @@ def validate_imds_network_metadata(self, imds_md: dict) -> bool: return False + def _cleanup_resourcedisk_fstab(self): + """ + Remove resource disk entries from fstab, which are configured + by cloud-init i.e. lines containing "/dev/disk/cloud/azure_resource" + and cloudconfig comment. + """ + cc_mounts.cleanup_fstab([RESOURCE_DISK_PATH]) + + def clean(self): + # Azure-specific cleanup logic for "cloud-init clean -c datasource" + self._cleanup_resourcedisk_fstab() + def _username_from_imds(imds_data): try: diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 61bf94f5..80dcfb13 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -19,14 +19,18 @@ from socket import gaierror, getaddrinfo, inet_ntoa from struct import pack -from cloudinit import sources, subp +from cloudinit import dmi, net, performance, sources, subp from cloudinit import url_helper as uhelp from cloudinit import util from cloudinit.net import dhcp +from cloudinit.net.dhcp import NoDHCPLeaseError +from cloudinit.net.ephemeral import EphemeralIPNetwork from cloudinit.sources.helpers import ec2 LOG = logging.getLogger(__name__) +CLOUD_STACK_DMI_NAME = "CloudStack" + class CloudStackPasswordServerClient: """ @@ -64,6 +68,7 @@ def _do_request(self, domu_request): ) return output.strip() + @performance.timed("Getting password", log_mode="always") def get_password(self): password = self._do_request("send_my_password") if password in ["", "saved_password"]: @@ -75,7 +80,7 @@ def get_password(self): class DataSourceCloudStack(sources.DataSource): - + perform_dhcp_setup = False dsname = "CloudStack" # Setup read_url parameters per get_url_params. @@ -83,17 +88,11 @@ class DataSourceCloudStack(sources.DataSource): url_timeout = 50 def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) + super().__init__(sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, "cs") # Cloudstack has its metadata/userdata URLs located at # http:///latest/ self.api_ver = "latest" - - self.distro = distro - self.vr_addr = get_vr_address(self.distro) - if not self.vr_addr: - raise RuntimeError("No virtual router found!") - self.metadata_address = f"http://{self.vr_addr}/" self.cfg = {} def _get_domainname(self): @@ -200,6 +199,11 @@ def wait_for_metadata_service(self): def get_config_obj(self): return self.cfg + @staticmethod + def ds_detect() -> bool: + """Check if running on this datasource""" + return is_platform_viable() + def _get_data(self): seed_ret = {} if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): @@ -207,46 +211,63 @@ def _get_data(self): self.metadata = seed_ret["meta-data"] LOG.debug("Using seeded cloudstack data from: %s", self.seed_dir) return True + if self.perform_dhcp_setup: + primary_nic = net.find_fallback_nic() + LOG.debug("Attempting DHCP on: %s", primary_nic) + network_context = EphemeralIPNetwork(self.distro, primary_nic) + else: + network_context = util.nullcontext() try: - if not self.wait_for_metadata_service(): - return False - start_time = time.monotonic() - self.userdata_raw = ec2.get_instance_userdata( - self.api_ver, self.metadata_address - ) - self.metadata = ec2.get_instance_metadata( - self.api_ver, self.metadata_address - ) - LOG.debug( - "Crawl of metadata service took %s seconds", - int(time.monotonic() - start_time), - ) - password_client = CloudStackPasswordServerClient(self.vr_addr) - try: - set_password = password_client.get_password() - except Exception: - util.logexc( - LOG, - "Failed to fetch password from virtual router %s", - self.vr_addr, + with network_context: + vr_addr = get_vr_address(self.distro) + # If vr_addr is a dict, we have the DHCP lease + self.vr_addr = ( + vr_addr.get("dhcp-server-identifier") + if isinstance(vr_addr, dict) + else vr_addr ) - else: - if set_password: - self.cfg = { - "ssh_pwauth": True, - "password": set_password, - "chpasswd": { - "expire": False, - }, - } - return True + if not self.vr_addr: + raise RuntimeError("No virtual router found!") + self.metadata_address = f"http://{self.vr_addr}/" + if not self.wait_for_metadata_service(): + return False + + return self._crawl_metadata() + except NoDHCPLeaseError: + LOG.warning("Unable to obtain a DHCP lease on %s", primary_nic) + return False + except Exception as e: + LOG.warning("Failed fetching metadata service: %s", str(e)) + return False + + @performance.timed("Crawling metadata", log_mode="always") + def _crawl_metadata(self): + self.userdata_raw = ec2.get_instance_userdata( + self.api_ver, self.metadata_address + ) + self.metadata = ec2.get_instance_metadata( + self.api_ver, self.metadata_address + ) + + password_client = CloudStackPasswordServerClient(self.vr_addr) + try: + set_password = password_client.get_password() except Exception: util.logexc( LOG, - "Failed fetching from metadata service %s", - self.metadata_address, + "Failed to fetch password from virtual router %s", + self.vr_addr, ) - return False + else: + if set_password: + self.cfg = { + "ssh_pwauth": True, + "password": set_password, + "chpasswd": { + "expire": False, + }, + } + return True def get_instance_id(self): return self.metadata["instance-id"] @@ -256,6 +277,18 @@ def availability_zone(self): return self.metadata["availability-zone"] +class DataSourceCloudStackLocal(DataSourceCloudStack): + """Run in init-local using a dhcp discovery prior to metadata crawl. + + In init-local, no network is available. This subclass sets up minimal + networking with dhclient on a viable nic so that it can talk to the + metadata service. If the metadata service provides network configuration + then render the network configuration for that instance based on metadata. + """ + + perform_dhcp_setup = True # Get metadata network config if present + + def get_data_server(): # Returns the metadataserver from dns try: @@ -328,8 +361,17 @@ def get_vr_address(distro): return get_default_gateway() +def is_platform_viable() -> bool: + product_name = dmi.read_dmi_data("system-product-name") + if not product_name: + LOG.debug("system-product-name not available in dmi data") + return False + return product_name.startswith(CLOUD_STACK_DMI_NAME) + + # Used to match classes to dependencies datasources = [ + (DataSourceCloudStackLocal, (sources.DEP_FILESYSTEM,)), (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index e4a9c074..16772618 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -154,24 +154,36 @@ def _get_data(self): if util.is_FreeBSD(): LOG.debug("FreeBSD doesn't support running dhclient with -sf") return False - try: - with EphemeralIPNetwork( - self.distro, - self.distro.fallback_interface, - ipv4=True, - ipv6=True, - ) as netw: - self._crawled_metadata = self.crawl_metadata() + candidate_nics = net.find_candidate_nics() + LOG.debug("Looking for the primary NIC in: %s", candidate_nics) + if len(candidate_nics) < 1: + LOG.error("The instance must have at least one eligible NIC") + return False + for candidate_nic in candidate_nics: + try: + with EphemeralIPNetwork( + self.distro, + candidate_nic, + ipv4=True, + ipv6=True, + ) as netw: + self._crawled_metadata = self.crawl_metadata() + if self._crawled_metadata: + self.distro.fallback_interface = candidate_nic + LOG.debug("Set fallback NIC: %s.", candidate_nic) + LOG.debug( + "Crawled metadata service%s", + f" {netw.state_msg}" if netw.state_msg else "", + ) + break + except NoDHCPLeaseError: LOG.debug( - "Crawled metadata service%s", - f" {netw.state_msg}" if netw.state_msg else "", + "Unable to obtain a DHCP lease for %s", candidate_nic ) - - except NoDHCPLeaseError: - return False else: self._crawled_metadata = self.crawl_metadata() if not self._crawled_metadata: + LOG.error("Unable to get metadata") return False self.metadata = self._crawled_metadata.get("meta-data", None) self.userdata_raw = self._crawled_metadata.get("user-data", None) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 8b5ddb5f..00a137df 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -318,6 +318,12 @@ def read_md(address=None, url_params=None, platform_check=True): LOG.warning("unknown user-data-encoding: %s, ignoring", encoding) ret["user-data"] = ud + # Update md with parsed instance-data + md["instance-data"] = instance_data + + # Update md with parsed project-data + md["project-data"] = project_data + ret["meta-data"] = md ret["success"] = True diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 7b919f66..53a950f8 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -78,6 +78,12 @@ def _get_data(self): sec_between=self.wait_retry, retries=self.retries, ) + pn = hc_helper.read_metadata( + self.metadata_address + "/private-networks", + timeout=self.timeout, + sec_between=self.wait_retry, + retries=self.retries, + ) except NoDHCPLeaseError as e: LOG.error("Bailing, DHCP Exception: %s", e) raise @@ -99,6 +105,7 @@ def _get_data(self): self.metadata["local-hostname"] = md["hostname"] self.metadata["network-config"] = md.get("network-config", None) self.metadata["public-keys"] = md.get("public-keys", None) + self.metadata["private-networks"] = pn self.vendordata_raw = md.get("vendor_data", None) # instance-id and serial from SMBIOS should be identical diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 1ad4a98c..781d7a62 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -267,7 +267,7 @@ def check_seed_contents(content, seed): Either return a (userdata, metadata, vendordata) tuple or Raise MAASSeedDirMalformed or MAASSeedDirNone """ - ret = {} + ret: dict = {} missing = [] for spath, dpath, _binary, optional in DS_FIELDS: if spath not in content: diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 22a365de..8e9d1ae6 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -130,12 +130,6 @@ def _pp2d_callback(mp, data): label = self.ds_cfg.get("fs_label", "cidata") if label is not None: - if label.lower() != "cidata": - lifecycle.deprecate( - deprecated="Custom fs_label keys", - deprecated_version="24.3", - extra_message="This key isn't supported by ds-identify.", - ) for dev in self._get_devices(label): try: LOG.debug("Attempting to use data from %s", dev) diff --git a/cloudinit/sources/DataSourceOracle.py b/cloudinit/sources/DataSourceOracle.py index 988fc7d2..aa62aa2d 100644 --- a/cloudinit/sources/DataSourceOracle.py +++ b/cloudinit/sources/DataSourceOracle.py @@ -293,11 +293,17 @@ def network_config(self): return self._network_config set_primary = False - # this is v1 if self._is_iscsi_root(): self._network_config = self._get_iscsi_config() + logging.debug( + "Instance is using iSCSI root, setting primary NIC as critical" + ) + # This is necessary for Oracle baremetal instances in case they are + # running on an IPv6-only network. Without this, they become + # unreachable/unrecoverable after a shutdown. + self._network_config["config"][0]["keep_configuration"] = True if not self._has_network_config(): - LOG.warning( + LOG.debug( "Could not obtain network configuration from initramfs. " "Falling back to IMDS." ) @@ -364,7 +370,7 @@ def _add_network_config_from_opc_imds(self, set_primary: bool = False): is_primary = set_primary and index == 0 mac_address = vnic_dict["macAddr"].lower() is_ipv6_only = vnic_dict.get( - "ipv6SubnetCidrBlock", False + "ipv6VirtualRouterIp", False ) and not vnic_dict.get("privateIp", False) if mac_address not in interfaces_by_mac: LOG.warning( @@ -380,65 +386,41 @@ def _add_network_config_from_opc_imds(self, set_primary: bool = False): else: network = ipaddress.ip_network(vnic_dict["subnetCidrBlock"]) - if self._network_config["version"] == 1: - if is_primary: - if is_ipv6_only: - subnets = [{"type": "dhcp6"}] - else: - subnets = [{"type": "dhcp"}] + if is_primary: + if is_ipv6_only: + subnets = [{"type": "dhcp6"}] else: - subnets = [] - if vnic_dict.get("privateIp"): - subnets.append( - { - "type": "static", - "address": ( - f"{vnic_dict['privateIp']}/" - f"{network.prefixlen}" - ), - } - ) - if vnic_dict.get("ipv6Addresses"): - subnets.append( - { - "type": "static", - "address": ( - f"{vnic_dict['ipv6Addresses'][0]}/" - f"{network.prefixlen}" - ), - } - ) - interface_config = { - "name": name, - "type": "physical", - "mac_address": mac_address, - "mtu": MTU, - "subnets": subnets, - } - self._network_config["config"].append(interface_config) - elif self._network_config["version"] == 2: - # Why does this elif exist??? - # Are there plans to switch to v2? - interface_config = { - "mtu": MTU, - "match": {"macaddress": mac_address}, - } - self._network_config["ethernets"][name] = interface_config - - interface_config["dhcp6"] = is_primary and is_ipv6_only - interface_config["dhcp4"] = is_primary and not is_ipv6_only - if not is_primary: - interface_config["addresses"] = [] - if vnic_dict.get("privateIp"): - interface_config["addresses"].append( - f"{vnic_dict['privateIp']}/{network.prefixlen}" - ) - if vnic_dict.get("ipv6Addresses"): - interface_config["addresses"].append( - f"{vnic_dict['ipv6Addresses'][0]}/" - f"{network.prefixlen}" - ) - self._network_config["ethernets"][name] = interface_config + subnets = [{"type": "dhcp"}] + else: + subnets = [] + if vnic_dict.get("privateIp"): + subnets.append( + { + "type": "static", + "address": ( + f"{vnic_dict['privateIp']}/" + f"{network.prefixlen}" + ), + } + ) + if vnic_dict.get("ipv6Addresses"): + subnets.append( + { + "type": "static", + "address": ( + f"{vnic_dict['ipv6Addresses'][0]}/" + f"{network.prefixlen}" + ), + } + ) + interface_config = { + "name": name, + "type": "physical", + "mac_address": mac_address, + "mtu": MTU, + "subnets": subnets, + } + self._network_config["config"].append(interface_config) class DataSourceOracleNet(DataSourceOracle): diff --git a/cloudinit/sources/DataSourceVMware.py b/cloudinit/sources/DataSourceVMware.py index d69f3673..05a41360 100644 --- a/cloudinit/sources/DataSourceVMware.py +++ b/cloudinit/sources/DataSourceVMware.py @@ -1,10 +1,9 @@ # Cloud-Init DataSource for VMware # -# Copyright (c) 2018-2023 VMware, Inc. All Rights Reserved. +# Copyright (c) 2018-2025 Broadcom. All Rights Reserved. # -# Authors: Anish Swaminathan -# Andrew Kutz -# Pengpeng Sun +# Authors: Andrew Kutz +# Pengpeng Sun # # This file is part of cloud-init. See LICENSE file for license information. @@ -20,14 +19,18 @@ import collections import copy +import functools import ipaddress import json import logging import os import socket +import sys import time +from typing import Dict, Set from cloudinit import atomic_helper, dmi, net, netinfo, sources, util +from cloudinit.event import EventScope, EventType, userdata_to_events from cloudinit.log import loggers from cloudinit.sources.helpers.vmware.imc import guestcust_util from cloudinit.subp import ProcessExecutionError, subp, which @@ -53,6 +56,74 @@ WAIT_ON_NETWORK_IPV4 = "ipv4" WAIT_ON_NETWORK_IPV6 = "ipv6" +SUPPORTED_UPDATE_EVENTS_GUEST_INFO_KEY = "cloudinit.updates.supported" +ENABLED_UPDATE_EVENTS_GUEST_INFO_KEY = "cloudinit.updates.enabled" + +# Support network reconfiguration for the following events: +# TODO(akutz) Add support for METADATA_CHANGE and USER_REQUEST when +# those events are supported by Cloud-Init. +SUPPORTED_UPDATE_EVENTS = { + EventScope.NETWORK: { + EventType.BOOT, + EventType.BOOT_NEW_INSTANCE, + EventType.HOTPLUG, + } +} + +# By default this datasource only configures the network when +# booting a new instance or when a NIC is added or removed. If +# a control plane like VM Service wishes the network to be +# reconfigured on each boot, it should send in vendordata that +# overrides the default update events. +DEFAULT_UPDATE_EVENTS = { + EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.HOTPLUG, + } +} + +# Only trigger hook-hotplug on NICs with VMware drivers. Avoid +# triggering the hook on Docker virtual NICs, etc. +# +# Please note, there is no way for this datasource to work +# out-of-the-box with SR-IOV NICs since their driver IDs are not known +# ahead of time. An SR-IOV NIC's driver is specific to the physical +# NIC. Users that want to support hotplug events for SR-IOV NICs will +# need to set "cloud-init.network-drivers" to a list of strings that +# includes the name of the driver used by the SR-IOV NIC. +DEFAULT_NETWORK_DRIVERS = [ + "e1000", + "e1000e", + "vlance", + "vmxnet2", + "vmxnet3", + "vrdma", +] + +_EXTRA_HOTPLUG_UDEV_RULES_FORMAT = """ +ENV{}=="{}", GOTO="cloudinit_hook" +GOTO="cloudinit_end" +""" + + +def cache(function): + """ + cache is a wrapper around functools.cache that no-ops when Python + is <3.9, the version in which functools.cache was introduced. + """ + if sys.version_info[:2] < (3, 9): + return function + else: + return functools.cache(function) + + +def get_extra_hotplug_udev_rules(driver_ids): + if driver_ids: + return _EXTRA_HOTPLUG_UDEV_RULES_FORMAT.format( + "{ID_NET_DRIVER}", "|".join(driver_ids) + ) + return None + class DataSourceVMware(sources.DataSource): """ @@ -93,6 +164,14 @@ class DataSourceVMware(sources.DataSource): dsname = "VMware" + supported_update_events = copy.deepcopy(SUPPORTED_UPDATE_EVENTS) + + default_update_events = copy.deepcopy(DEFAULT_UPDATE_EVENTS) + + extra_hotplug_udev_rules = get_extra_hotplug_udev_rules( + DEFAULT_NETWORK_DRIVERS + ) + def __init__(self, sys_cfg, distro, paths, ud_proc=None): sources.DataSource.__init__(self, sys_cfg, distro, paths, ud_proc) @@ -139,6 +218,10 @@ def _unpickle(self, ci_pkl_version: int) -> None: (DATA_ACCESS_METHOD_IMC, self.get_imc_data_fn, True), ], ) + if not hasattr(self, "extra_hotplug_udev_rules"): + self.extra_hotplug_udev_rules = get_extra_hotplug_udev_rules( + DEFAULT_NETWORK_DRIVERS + ) def __str__(self): root = sources.DataSource.__str__(self) @@ -223,12 +306,32 @@ def setup(self, is_new_instance): # etc. self.metadata = util.mergemanydict([self.metadata, host_info]) + # Set the extra udev rules based on the configured network + # drivers from the metadata or based on the default drivers. + self.init_extra_hotplug_udev_rules() + # Persist the instance data for versions of cloud-init that support # doing so. This occurs here rather than in the get_data call in # order to ensure that the network interfaces are up and can be # persisted with the metadata. self.persist_instance_data() + def activate(self, cfg, is_new_instance): + """activate(cfg, is_new_instance) + + This is called before the init_modules will be called but after + the user-data and vendor-data have been fully processed. + + The cfg is fully up to date config, it contains a merged view of + system config, datasource config, user config, vendor config. + It should be used rather than the sys_cfg passed to __init__. + + is_new_instance is a boolean indicating if this is a new instance. + """ + + # Reflect the update events into guestinfo. + self.advertise_update_events(cfg) + def _get_subplatform(self): get_key_name_fn = None if self.data_access_method == DATA_ACCESS_METHOD_ENVVAR: @@ -262,6 +365,44 @@ def network_config(self): } return self.metadata["network"]["config"] + def advertise_update_events(self, cfg): + default_events: Dict[EventScope, Set[EventType]] = copy.deepcopy( + self.default_update_events + ) + user_events: Dict[EventScope, Set[EventType]] = userdata_to_events( + cfg.get("updates", {}) + ) + enabled_events = util.mergemanydict( + [ + copy.deepcopy(user_events), + copy.deepcopy(default_events), + ] + ) + return advertise_update_events( + self.supported_update_events, + enabled_events, + self.rpctool, + self.rpctool_fn, + ) + + def init_extra_hotplug_udev_rules(self): + rules = get_extra_hotplug_udev_rules(self.get_network_drivers()) + if rules: + self.extra_hotplug_udev_rules = rules + + @cache + def get_network_drivers(self): + network_drivers = DEFAULT_NETWORK_DRIVERS + + if self.metadata and "network-drivers" in self.metadata: + network_drivers = self.metadata["network-drivers"] + + # Trim any leading/trailing whitespace. + network_drivers = [s.strip(" ") for s in network_drivers] + + LOG.info("got network drivers %s", network_drivers) + return network_drivers + def get_instance_id(self): # Pull the instance ID out of the metadata if present. Otherwise # read the file /sys/class/dmi/id/product_uuid for the instance ID. @@ -527,6 +668,69 @@ def advertise_local_ip_addrs(host_info, rpctool, rpctool_fn): LOG.info("advertised local ipv6 address %s in guestinfo", local_ipv6) +def advertise_update_events( + supported_update_events, enabled_update_events, rpctool, rpctool_fn +): + """ + advertise_update_events publishes the types of supported and + enabled events to guestinfo. + + The string returned from this method adheres to the following + format: + + SCOPE=TYPE[;TYPE][,SCOPE=TYPE[;TYPE]] + + For example: + + * network=boot;hotplug + * network=boot-new-instance + + The only supported event scope at the moment is network, but there + may be more scopes in the future, and this format will support them. + """ + if not rpctool or not rpctool_fn: + return None, None + + def get_events_string(events): + event_scopes_and_types_list = [] + for event_scope, event_types in events.items(): + event_types_list = sorted( + str(event_type) for event_type in event_types + ) + event_scopes_and_types_list.append( + "{}={}".format(event_scope, ";".join(event_types_list)) + ) + return ",".join(event_scopes_and_types_list) + + supported_events_string = get_events_string(supported_update_events) + if supported_events_string: + guestinfo_set_value( + SUPPORTED_UPDATE_EVENTS_GUEST_INFO_KEY, + supported_events_string, + rpctool, + rpctool_fn, + ) + LOG.info( + "advertised supported update events in guestinfo: %s", + supported_events_string, + ) + + enabled_events_string = get_events_string(enabled_update_events) + if enabled_events_string: + guestinfo_set_value( + ENABLED_UPDATE_EVENTS_GUEST_INFO_KEY, + enabled_events_string, + rpctool, + rpctool_fn, + ) + LOG.info( + "advertised enabled update events in guestinfo: %s", + enabled_events_string, + ) + + return supported_events_string, enabled_events_string + + def handle_returned_guestinfo_val(key, val): """ handle_returned_guestinfo_val returns the provided value if it is diff --git a/cloudinit/sources/DataSourceWSL.py b/cloudinit/sources/DataSourceWSL.py index f99ecb5c..e51db4ee 100644 --- a/cloudinit/sources/DataSourceWSL.py +++ b/cloudinit/sources/DataSourceWSL.py @@ -25,6 +25,9 @@ DEFAULT_INSTANCE_ID = "iid-datasource-wsl" LANDSCAPE_DATA_FILE = "%s.user-data" AGENT_DATA_FILE = "agent.yaml" +LANDSCAPE_CFG_KEY = "landscape" +LANDSCAPE_CLIENT_CFG_KEY = "client" +LANDSCAPE_INSTALLATION_REQ_ID = "installation_request_id" def instance_name() -> str: @@ -148,7 +151,10 @@ class ConfigData: """Models a piece of configuration data as a dict if possible, while retaining its raw representation alongside its file path""" - def __init__(self, path: PurePath): + def is_cloud_config(self) -> bool: + return self.config_dict is not None + + def __init__(self, path: PurePath, instance_id: str): self.raw: str = util.load_text_file(path) self.path: PurePath = path @@ -157,35 +163,73 @@ def __init__(self, path: PurePath): if "text/cloud-config" == type_from_starts_with(self.raw): self.config_dict = util.load_yaml(self.raw) - def is_cloud_config(self) -> bool: - return self.config_dict is not None + if ( + self.config_dict # Valid non-empty config + and instance_id + and instance_id != DEFAULT_INSTANCE_ID # custom instance-id + and self.config_dict.get(LANDSCAPE_CFG_KEY, {}).get( + LANDSCAPE_CLIENT_CFG_KEY + ) + is not None # Landscape client config exists + ): + # Let's set the installation_request_id to the metadata.instance-id + # if not already set, and update the raw config if we modified it: + req_id = self.config_dict[LANDSCAPE_CFG_KEY][ + LANDSCAPE_CLIENT_CFG_KEY + ].setdefault(LANDSCAPE_INSTALLATION_REQ_ID, instance_id) + if req_id == instance_id: + self.raw = ( + "#cloud-config\n%s" % yaml.dump(self.config_dict).strip() + ) -def load_instance_metadata( - cloudinitdir: Optional[PurePath], instance_name: str -) -> dict: +def _load_metadata(metadata_path: PurePath) -> Optional[dict]: """ - Returns the relevant metadata loaded from cloudinit dir based on the - instance name + Returns the relevant metadata loaded from the supplied metadata_path. """ - metadata = {"instance-id": DEFAULT_INSTANCE_ID} - if cloudinitdir is None: - return metadata - metadata_path = os.path.join( - cloudinitdir.as_posix(), "%s.meta-data" % instance_name - ) - + metadata = None try: - metadata = util.load_yaml(util.load_text_file(metadata_path)) + metadata = util.load_yaml( + util.load_text_file(metadata_path), default={} + ) except FileNotFoundError: LOG.debug( - "No instance metadata found at %s. Using default instance-id.", + "No instance metadata found at %s.", metadata_path, ) + + return metadata + + +def load_instance_metadata(user_home: PurePath, instance_name: str) -> dict: + """ + Returns the relevant metadata loaded from either the Ubuntu Pro dir + or the user cloud-int dir based on the instance name + """ + metadata = {"instance-id": DEFAULT_INSTANCE_ID} + pro_dir = cloud_init_data_dir(user_home / ".ubuntupro") + user_dir = cloud_init_data_dir(user_home) + found_at = "" + for dir in [pro_dir, user_dir]: + if dir is None: + continue + path = dir / ("%s.meta-data" % instance_name) + dt = _load_metadata(path) + if dt is not None: + metadata = dt + found_at = path.as_posix() + break + + if not found_at: + LOG.debug( + "Unable to find meta-data file candidates." + " Using default instance-id" + ) + if not metadata or "instance-id" not in metadata: # Parsed metadata file invalid msg = ( - f" Metadata at {metadata_path} does not contain instance-id key." + f"Metadata at {found_at} does not contain instance-id key." f" Instead received: {metadata}" ) LOG.error(msg) @@ -195,14 +239,14 @@ def load_instance_metadata( def load_ubuntu_pro_data( - user_home: PurePath, + user_home: PurePath, instance_id: str ) -> Tuple[Optional[ConfigData], Optional[ConfigData]]: """ Read .ubuntupro user-data if present and return a tuple of agent and landscape user-data. """ - pro_dir = os.path.join(user_home, ".ubuntupro/.cloud-init") - if not os.path.isdir(pro_dir): + pro_dir = cloud_init_data_dir(user_home / ".ubuntupro") + if pro_dir is None: return None, None landscape_path = PurePath( @@ -216,12 +260,12 @@ def load_ubuntu_pro_data( landscape_path, cloud_init_data_dir(user_home), ) - landscape_data = ConfigData(landscape_path) + landscape_data = ConfigData(landscape_path, instance_id) agent_path = PurePath(os.path.join(pro_dir, AGENT_DATA_FILE)) agent_data = None if os.path.isfile(agent_path): - agent_data = ConfigData(agent_path) + agent_data = ConfigData(agent_path, instance_id) return agent_data, landscape_data @@ -311,6 +355,27 @@ def merge_agent_landscape_data( ) +def landscape_supports_field(field: str): + """Checks if the landscape-config binary supports the intended + configuration field, returning False if not or if attempting + to run that binary fails. + """ + try: + flag = "--" + field.replace("_", "-") + # landscape-config is the command that understand the config fields, + # not landscape-client itself. + out, _ = subp.subp(["landscape-config", "--help"]) + return flag in out + + except subp.ProcessExecutionError as err: + LOG.warning( + "Unable to verify if landscape-client supports %s: %s", + field, + err.reason, + ) + return False + + class DataSourceWSL(sources.DataSource): dsname = "WSL" @@ -356,8 +421,8 @@ def check_instance_id(self, sys_cfg) -> bool: return False try: - data_dir = cloud_init_data_dir(find_home()) - metadata = load_instance_metadata(data_dir, instance_name()) + home = find_home() + metadata = load_instance_metadata(home, instance_name()) return current == metadata.get("instance-id") except (IOError, ValueError) as err: @@ -389,20 +454,26 @@ def _get_data(self) -> bool: # Load any metadata try: self.metadata = load_instance_metadata( - seed_dir, self.instance_name + user_home, self.instance_name ) except (ValueError, IOError) as err: LOG.error("Unable to load metadata: %s", str(err)) return False - # # Load Ubuntu Pro configs only on Ubuntu distros + iid = self.metadata["instance-id"] + # Load Ubuntu Pro configs only on Ubuntu distros if self.distro.name == "ubuntu": - agent_data, user_data = load_ubuntu_pro_data(user_home) + if not landscape_supports_field(LANDSCAPE_INSTALLATION_REQ_ID): + # Prevents reusing metadata.instance-id as + # landscape.installation_request_id if the + # current version of landscape_client doesn't support it. + iid = "" + agent_data, user_data = load_ubuntu_pro_data(user_home, iid) # Load regular user configs try: if user_data is None and seed_dir is not None: - user_data = ConfigData(self.find_user_data_file(seed_dir)) + user_data = ConfigData(self.find_user_data_file(seed_dir), "") except (ValueError, IOError) as err: log = LOG.info if agent_data else LOG.error diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 38533fe5..b42ec3fa 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -1021,6 +1021,14 @@ def activate(self, cfg, is_new_instance): """ return + def clean(self): + """ + Called when the user issues 'cloud-init clean -c datasource' + command. + Individual data sources should implement their own cleanup handler. + """ + return + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/sources/azure/errors.py b/cloudinit/sources/azure/errors.py index a7ed043e..62c12840 100644 --- a/cloudinit/sources/azure/errors.py +++ b/cloudinit/sources/azure/errors.py @@ -14,7 +14,6 @@ import requests from cloudinit import subp, version -from cloudinit.sources.azure import identity from cloudinit.url_helper import UrlError LOG = logging.getLogger(__name__) @@ -38,10 +37,7 @@ def encode_report( class ReportableError(Exception): def __init__( - self, - reason: str, - *, - supporting_data: Optional[Dict[str, Any]] = None, + self, reason: str, *, supporting_data: Optional[Dict[str, Any]] = None ) -> None: self.agent = f"Cloud-Init/{version.version_string()}" self.documentation_url = "https://aka.ms/linuxprovisioningerror" @@ -54,13 +50,10 @@ def __init__( self.timestamp = datetime.now(timezone.utc) - try: - self.vm_id = identity.query_vm_id() - except Exception as id_error: - self.vm_id = f"failed to read vm id: {id_error!r}" - def as_encoded_report( self, + *, + vm_id: Optional[str], ) -> str: data = [ "result=error", @@ -69,7 +62,7 @@ def as_encoded_report( ] data += [f"{k}={v}" for k, v in self.supporting_data.items()] data += [ - f"vm_id={self.vm_id}", + f"vm_id={vm_id}", f"timestamp={self.timestamp.isoformat()}", f"documentation_url={self.documentation_url}", ] @@ -156,7 +149,8 @@ def __init__(self, *, key: str, value: Any) -> None: super().__init__(f"invalid IMDS metadata for key={key}") self.supporting_data["key"] = key - self.supporting_data["value"] = repr(value) + self.supporting_data["value"] = value + self.supporting_data["type"] = type(value).__name__ class ReportableErrorImdsMetadataParsingException(ReportableError): @@ -191,7 +185,12 @@ def __init__(self, exception: Exception) -> None: type(exception), exception, exception.__traceback__ ) ) - trace_base64 = base64.b64encode(trace.encode("utf-8")).decode("utf-8") + trace_lines = trace.split("\n") + reversed_trace_lines = trace_lines[::-1] + reversed_trace = "\n".join(reversed_trace_lines) + trace_base64 = base64.b64encode(reversed_trace.encode("utf-8")).decode( + "utf-8" + ) self.supporting_data["exception"] = repr(exception) self.supporting_data["traceback_base64"] = trace_base64 @@ -209,3 +208,13 @@ def __init__(self, exception: subp.ProcessExecutionError) -> None: self.supporting_data["exit_code"] = exception.exit_code self.supporting_data["stdout"] = exception.stdout self.supporting_data["stderr"] = exception.stderr + + +class ReportableErrorVmIdentification(ReportableError): + def __init__( + self, *, exception: Exception, system_uuid: Optional[str] = None + ) -> None: + super().__init__("failure to identify Azure VM ID") + + self.supporting_data["exception"] = repr(exception) + self.supporting_data["system_uuid"] = system_uuid diff --git a/cloudinit/sources/azure/kvp.py b/cloudinit/sources/azure/kvp.py index 903b8122..c11d793e 100644 --- a/cloudinit/sources/azure/kvp.py +++ b/cloudinit/sources/azure/kvp.py @@ -8,7 +8,7 @@ from cloudinit import version from cloudinit.reporting import handlers, instantiated_handler_registry -from cloudinit.sources.azure import errors, identity +from cloudinit.sources.azure import errors LOG = logging.getLogger(__name__) @@ -35,16 +35,7 @@ def report_via_kvp(report: str) -> bool: return True -def report_failure_to_host(error: errors.ReportableError) -> bool: - return report_via_kvp(error.as_encoded_report()) - - -def report_success_to_host() -> bool: - try: - vm_id = identity.query_vm_id() - except Exception as id_error: - vm_id = f"failed to read vm id: {id_error!r}" - +def report_success_to_host(*, vm_id: Optional[str]) -> bool: report = errors.encode_report( [ "result=success", diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 3a200ee2..84f8f10e 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -232,7 +232,7 @@ def http_with_retries( *, headers: dict, data: Optional[bytes] = None, - retry_sleep: int = 5, + retry_sleep: int = 1, timeout_minutes: int = 20, ) -> url_helper.UrlResponse: """Readurl wrapper for querying wireserver. @@ -282,10 +282,29 @@ def http_with_retries( def build_minimal_ovf( - username: str, hostname: str, disableSshPwd: str + *, + username: Optional[str], + hostname: Optional[str], + disable_ssh_password_auth: Optional[bool], ) -> bytes: - OVF_ENV_TEMPLATE = textwrap.dedent( - """\ + if username: + ns_username = f"{username}" + else: + ns_username = "" + + if disable_ssh_password_auth is None: + ns_disable_ssh_password_auth = "" + else: + ns_disable_ssh_password_auth = ( + "" + f"{str(disable_ssh_password_auth).lower()}" + "" + ) + + ns_hostname = f"{hostname}" + + return textwrap.dedent( + f"""\ @@ -294,10 +313,9 @@ def build_minimal_ovf( LinuxProvisioningConfiguration - {username} - {disableSshPwd} - - {hostname} + {ns_username} + {ns_disable_ssh_password_auth} + {ns_hostname} @@ -308,11 +326,7 @@ def build_minimal_ovf( """ - ) - ret = OVF_ENV_TEMPLATE.format( - username=username, hostname=hostname, disableSshPwd=disableSshPwd - ) - return ret.encode("utf-8") + ).encode("utf-8") class AzureEndpointHttpClient: @@ -945,21 +959,22 @@ def get_metadata_from_fabric( @azure_ds_telemetry_reporter -def report_failure_to_fabric(endpoint: str, error: "errors.ReportableError"): +def report_failure_to_fabric(endpoint: str, *, encoded_report: str): shim = WALinuxAgentShim(endpoint=endpoint) - description = error.as_encoded_report() try: - shim.register_with_azure_and_report_failure(description=description) + shim.register_with_azure_and_report_failure(description=encoded_report) finally: shim.clean_up() -def dhcp_log_cb(out, err): +def dhcp_log_cb(interface: str, out: str, err: str) -> None: report_diagnostic_event( - "dhclient output stream: %s" % out, logger_func=LOG.debug + f"dhcp client stdout for interface={interface}: {out}", + logger_func=LOG.debug, ) report_diagnostic_event( - "dhclient error stream: %s" % err, logger_func=LOG.debug + f"dhcp client stderr for interface={interface}: {err}", + logger_func=LOG.debug, ) @@ -1102,7 +1117,7 @@ def _parse_linux_configuration_set_section(self, root): required=False, ) self.username = self._parse_property( - config_set, "UserName", required=True + config_set, "UserName", required=False ) self.password = self._parse_property( config_set, "UserPassword", required=False diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py index a1fd92ff..50fbcb04 100644 --- a/cloudinit/sources/helpers/hetzner.py +++ b/cloudinit/sources/helpers/hetzner.py @@ -12,7 +12,7 @@ def read_metadata(url, timeout=2, sec_between=2, retries=30): ) if not response.ok(): raise RuntimeError("unable to read metadata at %s" % url) - return util.load_yaml(response.contents.decode()) + return util.load_yaml(response.contents.decode(), allowed=(dict, list)) def read_userdata(url, timeout=2, sec_between=2, retries=30): diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index d18111a8..c52776c9 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -544,6 +544,10 @@ def parse_ssh_config_map(fname): def _includes_dconf(fname: str) -> bool: + # Handle cases where sshd_config is handled in /usr/etc/ssh/sshd_config + # so /etc/ssh/sshd_config.d/ exists but /etc/ssh/sshd_config doesn't + if not os.path.exists(fname) and os.path.exists(f"{fname}.d"): + return True if not os.path.isfile(fname): return False for line in util.load_text_file(fname).splitlines(): diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 07148f17..eebf009d 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -9,7 +9,7 @@ from io import TextIOWrapper from typing import List, Optional, Union -from cloudinit import performance, signal_handler +from cloudinit import performance LOG = logging.getLogger(__name__) @@ -368,8 +368,7 @@ def runparts(dirp, skip_no_exist=True, exe_prefix=None): if is_exe(exe_path): attempted.append(exe_path) try: - with signal_handler.suspend_crash(): - subp(prefix + [exe_path], capture=False) + subp(prefix + [exe_path], capture=False) except ProcessExecutionError as e: LOG.debug(e) failed.append(exe_name) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index ee7f5956..9734bdbd 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -433,7 +433,8 @@ def _handle_error( if error.code and error.code == 503: LOG.warning( "Endpoint returned a 503 error. " - "HTTP endpoint is overloaded. Retrying." + "HTTP endpoint is overloaded. Retrying URL (%s).", + error.url, ) if error.headers: return _get_retry_after(error.headers.get("Retry-After", "1")) diff --git a/cloudinit/util.py b/cloudinit/util.py index 2f1247c7..cab6e0fb 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -550,6 +550,8 @@ def get_linux_distro(): os_release_rhel = False if os.path.exists("/etc/os-release"): os_release = load_shell_content(load_text_file("/etc/os-release")) + if os.path.exists("/etc/rpi-issue"): + os_release["ID"] = "raspberry-pi-os" if not os_release: os_release_rhel = True os_release = _parse_redhat_release() @@ -625,6 +627,7 @@ def _get_variant(info): "opencloudos", "openmandriva", "photon", + "raspberry-pi-os", "rhel", "rocky", "suse", diff --git a/cloudinit/version.py b/cloudinit/version.py index b0124088..f6e85159 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -4,7 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -__VERSION__ = "25.1.4" +__VERSION__ = "25.2" _PACKAGED_VERSION = "@@PACKAGED_VERSION@@" FEATURES = [ diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 82a7e0fe..dea7f031 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -10,6 +10,7 @@ "mariner": "MarinerOS", "rhel": "Cloud User", "netbsd": "NetBSD", "openbsd": "openBSD", "openmandriva": "OpenMandriva admin", "photon": "PhotonOS", + "raspberry-pi-os": "Raspberry Pi OS", "ubuntu": "Ubuntu", "unknown": "Ubuntu"}) %} {% set groups = ({"alpine": "adm, wheel", "aosc": "wheel", "arch": "wheel, users", "azurelinux": "wheel", @@ -17,6 +18,7 @@ "gentoo": "users, wheel", "mariner": "wheel", "photon": "wheel", "openmandriva": "wheel, users, systemd-journal", + "raspberry-pi-os": "adm, dialout, cdrom, audio, users, sudo, video, games, plugdev, input, gpio, spi, i2c, netdev, render, lpadmin", "suse": "cdrom, users", "ubuntu": "adm, cdrom, dip, lxd, sudo", "unknown": "adm, cdrom, dip, lxd, sudo"}) %} @@ -24,7 +26,8 @@ "freebsd": "/bin/tcsh", "netbsd": "/bin/sh", "openbsd": "/bin/ksh"}) %} {% set usernames = ({"amazon": "ec2-user", "centos": "cloud-user", - "openmandriva": "omv", "rhel": "cloud-user", + "openmandriva": "omv", "raspberry-pi-os": "pi", + "rhel": "cloud-user", "unknown": "ubuntu"}) %} {% if is_bsd %} syslog_fix_perms: root:wheel @@ -137,6 +140,9 @@ cloud_init_modules: - users_groups - ssh - set_passwords +{% if variant == "raspberry-pi-os" %} + - raspberry_pi +{% endif %} # The modules that run in the 'config' stage cloud_config_modules: @@ -207,7 +213,9 @@ cloud_final_modules: {% if variant not in ["azurelinux"] %} - mcollective - salt_minion +{% if variant not in ["alpine"] %} - reset_rmc +{% endif %} {% endif %} - scripts_vendor - scripts_per_once @@ -227,8 +235,8 @@ system_info: # This will affect which distro class gets used {% if variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "debian", "fedora", "freebsd", "gentoo", "mariner", "netbsd", "openbsd", - "OpenCloudOS", "openeuler", "openmandriva", "photon", "suse", - "TencentOS", "ubuntu"] or is_rhel %} + "OpenCloudOS", "openeuler", "openmandriva", "photon", + "raspberry-pi-os", "suse", "TencentOS", "ubuntu"] or is_rhel %} distro: {{ variant }} {% elif variant == "dragonfly" %} distro: dragonflybsd @@ -245,8 +253,8 @@ system_info: {% endif %} {% if variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "debian", "fedora", "gentoo", "mariner", "OpenCloudOS", "openeuler", - "openmandriva", "photon", "suse", "TencentOS", "ubuntu", - "unknown"] + "openmandriva", "photon", "raspberry-pi-os", + "suse", "TencentOS", "ubuntu", "unknown"] or is_bsd or is_rhel %} lock_passwd: True {% endif %} @@ -305,6 +313,10 @@ system_info: {% elif variant == "openmandriva" %} network: renderers: ['network-manager', 'networkd'] +{% elif variant == "raspberry-pi-os" %} + network: + renderers: ['netplan', 'network-manager'] + activators: ['netplan', 'network-manager'] {% elif variant in ["ubuntu", "unknown"] %} {# SRU_BLOCKER: do not ship network renderers on Xenial, Bionic or Eoan #} network: @@ -324,6 +336,8 @@ system_info: {% if variant in ["debian", "ubuntu", "unknown"] %} # Automatically discover the best ntp_client ntp_client: auto +{% elif variant == "raspberry-pi-os" %} + ntp_client: 'systemd-timesyncd' {% endif %} {% if variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "debian", "fedora", "gentoo", "mariner", "OpenCloudOS", "openeuler", @@ -344,6 +358,19 @@ system_info: failsafe: primary: https://deb.debian.org/debian security: https://deb.debian.org/debian-security +{% elif variant == "raspberry-pi-os" %} + package_mirrors: + - arches: [arm64] + failsafe: + primary: + - https://deb.debian.org/debian + - http://archive.raspberrypi.com/debian/ + security: https://deb.debian.org/debian-security + - arches: [armhf] + failsafe: + primary: + - http://raspbian.raspberrypi.com/raspbian/ + - http://archive.raspberrypi.com/debian/ {% elif variant in ["ubuntu", "unknown"] %} package_mirrors: - arches: [i386, amd64] @@ -371,7 +398,7 @@ system_info: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports {% endif %} -{% if variant in ["debian", "ubuntu", "unknown"] %} +{% if variant in ["debian", "raspberry-pi-os", "ubuntu", "unknown"] %} ssh_svcname: ssh {% elif variant in ["alpine", "amazon", "aosc", "arch", "azurelinux", "fedora", "gentoo", "mariner", "OpenCloudOS", "openeuler", diff --git a/conftest.py b/conftest.py index 36320f23..5d7aa563 100644 --- a/conftest.py +++ b/conftest.py @@ -19,7 +19,7 @@ import pytest -from cloudinit import helpers, subp, util +from cloudinit import subp, util @pytest.fixture(autouse=True, scope="function") @@ -197,22 +197,6 @@ def mocked_responses(): yield rsps -@pytest.fixture -def paths(tmpdir): - """ - Return a helpers.Paths object configured to use a tmpdir. - - (This uses the builtin tmpdir fixture.) - """ - dirs = { - "cloud_dir": tmpdir.mkdir("cloud_dir").strpath, - "docs_dir": tmpdir.mkdir("docs_dir").strpath, - "run_dir": tmpdir.mkdir("run_dir").strpath, - "templates_dir": tmpdir.mkdir("templates_dir").strpath, - } - return helpers.Paths(dirs) - - @pytest.fixture(autouse=True, scope="session") def monkeypatch_system_info(): def my_system_info(): diff --git a/debian/changelog b/debian/changelog index ce30a3e6..b6a8a433 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,19 @@ +cloud-init (25.2-0ubuntu1~24.04.1) noble; urgency=medium + + * add d/p/strip-invalid-mtu.patch + - Provides backwards compatibility for an otherwise invalid + MTU in a netplan config. (GH-6239) + * d/cloud-init.templates: + - Move VMware before OVF. See GH-4030 + - Enable CloudCIX by default + * refresh patches: + - d/p/no-single-process.patch + * Upstream snapshot based on 25.2. (LP: #2120495). + List of changes from upstream can be found at + https://raw.githubusercontent.com/canonical/cloud-init/25.2/ChangeLog + + -- James Falcon Tue, 12 Aug 2025 16:19:32 -0500 + cloud-init (25.1.4-0ubuntu0~24.04.1) noble-security; urgency=medium * Upstream snapshot based on 25.1.4. diff --git a/debian/patches/deprecation-version-boundary.patch b/debian/patches/deprecation-version-boundary.patch index 6ffb28ad..0fe59309 100644 --- a/debian/patches/deprecation-version-boundary.patch +++ b/debian/patches/deprecation-version-boundary.patch @@ -9,8 +9,8 @@ Last-Update: 2024-06-28 This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/cloudinit/features.py +++ b/cloudinit/features.py -@@ -98,7 +98,7 @@ Note that in addition to this flag, down - to modify the systemd unit files. +@@ -108,7 +108,7 @@ previously generated network configurati + (This flag can be removed when Noble is no longer supported.) """ -DEPRECATION_INFO_BOUNDARY = "devel" diff --git a/debian/patches/grub-dpkg-support.patch b/debian/patches/grub-dpkg-support.patch index e62b9f66..0bfa06e9 100644 --- a/debian/patches/grub-dpkg-support.patch +++ b/debian/patches/grub-dpkg-support.patch @@ -28,7 +28,7 @@ This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ return --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json -@@ -1616,8 +1616,8 @@ +@@ -1655,8 +1655,8 @@ "properties": { "enabled": { "type": "boolean", diff --git a/debian/patches/no-nocloud-network.patch b/debian/patches/no-nocloud-network.patch index 1577e7e6..bd2d80a7 100644 --- a/debian/patches/no-nocloud-network.patch +++ b/debian/patches/no-nocloud-network.patch @@ -7,7 +7,7 @@ Last-Update: 2024-08-02 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py -@@ -190,7 +190,7 @@ class DataSourceNoCloud(sources.DataSour +@@ -184,7 +184,7 @@ class DataSourceNoCloud(sources.DataSour # This could throw errors, but the user told us to do it # so if errors are raised, let them raise @@ -16,7 +16,7 @@ Last-Update: 2024-08-02 LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed -@@ -199,7 +199,6 @@ class DataSourceNoCloud(sources.DataSour +@@ -193,7 +193,6 @@ class DataSourceNoCloud(sources.DataSour ) mydata["user-data"] = ud mydata["vendor-data"] = vd @@ -26,7 +26,7 @@ Last-Update: 2024-08-02 # Now that we have exhausted any other places merge in the defaults --- a/cloudinit/util.py +++ b/cloudinit/util.py -@@ -1012,7 +1012,6 @@ def read_seeded(base="", ext="", timeout +@@ -1015,7 +1015,6 @@ def read_seeded(base="", ext="", timeout ud_url = base.replace("%s", "user-data" + ext) vd_url = base.replace("%s", "vendor-data" + ext) md_url = base.replace("%s", "meta-data" + ext) @@ -34,7 +34,7 @@ Last-Update: 2024-08-02 else: if features.NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH: if base[-1] != "/" and parse.urlparse(base).query == "": -@@ -1021,17 +1020,7 @@ def read_seeded(base="", ext="", timeout +@@ -1024,17 +1023,7 @@ def read_seeded(base="", ext="", timeout ud_url = "%s%s%s" % (base, "user-data", ext) vd_url = "%s%s%s" % (base, "vendor-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) @@ -54,7 +54,7 @@ Last-Update: 2024-08-02 ) --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py -@@ -2498,7 +2498,7 @@ class TestReadOptionalSeed: +@@ -2489,7 +2489,7 @@ class TestReadOptionalSeed: { "meta-data": {"md": "val"}, "user-data": b"ud", @@ -63,7 +63,7 @@ Last-Update: 2024-08-02 "vendor-data": None, }, True, -@@ -2553,7 +2553,7 @@ class TestReadSeeded: +@@ -2544,7 +2544,7 @@ class TestReadSeeded: assert found_md == {"key1": "val1"} assert found_ud == ud assert found_vd == vd @@ -72,7 +72,7 @@ Last-Update: 2024-08-02 @pytest.mark.parametrize( "base, feature_flag, req_urls", -@@ -2562,7 +2562,6 @@ class TestReadSeeded: +@@ -2553,7 +2553,6 @@ class TestReadSeeded: "http://10.0.0.1/%s?qs=1", True, [ @@ -80,7 +80,7 @@ Last-Update: 2024-08-02 "http://10.0.0.1/meta-data?qs=1", "http://10.0.0.1/user-data?qs=1", "http://10.0.0.1/vendor-data?qs=1", -@@ -2573,7 +2572,6 @@ class TestReadSeeded: +@@ -2564,7 +2563,6 @@ class TestReadSeeded: "https://10.0.0.1:8008/", True, [ @@ -88,7 +88,7 @@ Last-Update: 2024-08-02 "https://10.0.0.1:8008/meta-data", "https://10.0.0.1:8008/user-data", "https://10.0.0.1:8008/vendor-data", -@@ -2584,7 +2582,6 @@ class TestReadSeeded: +@@ -2575,7 +2573,6 @@ class TestReadSeeded: "https://10.0.0.1:8008", True, [ @@ -96,7 +96,7 @@ Last-Update: 2024-08-02 "https://10.0.0.1:8008/meta-data", "https://10.0.0.1:8008/user-data", "https://10.0.0.1:8008/vendor-data", -@@ -2595,7 +2592,6 @@ class TestReadSeeded: +@@ -2586,7 +2583,6 @@ class TestReadSeeded: "https://10.0.0.1:8008", False, [ @@ -104,7 +104,7 @@ Last-Update: 2024-08-02 "https://10.0.0.1:8008meta-data", "https://10.0.0.1:8008user-data", "https://10.0.0.1:8008vendor-data", -@@ -2606,7 +2602,6 @@ class TestReadSeeded: +@@ -2597,7 +2593,6 @@ class TestReadSeeded: "https://10.0.0.1:8008?qs=", True, [ @@ -112,7 +112,7 @@ Last-Update: 2024-08-02 "https://10.0.0.1:8008?qs=meta-data", "https://10.0.0.1:8008?qs=user-data", "https://10.0.0.1:8008?qs=vendor-data", -@@ -2645,7 +2640,7 @@ class TestReadSeeded: +@@ -2636,7 +2631,7 @@ class TestReadSeeded: # user-data, vendor-data read raw. It could be scripts or other format assert found_ud == "/user-data: 1" assert found_vd == "/vendor-data: 1" @@ -121,7 +121,7 @@ Last-Update: 2024-08-02 assert [ mock.call(req_url, timeout=5, retries=10) for req_url in req_urls ] == m_read.call_args_list -@@ -2675,7 +2670,7 @@ class TestReadSeededWithoutVendorData(he +@@ -2666,7 +2661,7 @@ class TestReadSeededWithoutVendorData(he self.assertEqual(found_md, {"key1": "val1"}) self.assertEqual(found_ud, ud) self.assertEqual(found_vd, vd) diff --git a/debian/patches/no-remove-networkd-online.patch b/debian/patches/no-remove-networkd-online.patch index eeb76a5b..f79092cf 100644 --- a/debian/patches/no-remove-networkd-online.patch +++ b/debian/patches/no-remove-networkd-online.patch @@ -16,14 +16,3 @@ Last-Update: 2025-01-16 """ On Ubuntu systems, cloud-init-network.service will start immediately after cloud-init-local.service and manually wait for network online when necessary. ---- a/tests/unittests/test_data.py -+++ b/tests/unittests/test_data.py -@@ -516,7 +516,7 @@ c: 4 - "DEPRECATION_INFO_BOUNDARY": "devel", - "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, - "APT_DEB822_SOURCE_LIST_FILE": True, -- "MANUAL_NETWORK_WAIT": True, -+ "MANUAL_NETWORK_WAIT": False, - }, - "system_info": { - "default_user": {"name": "ubuntu"}, diff --git a/debian/patches/no-single-process.patch b/debian/patches/no-single-process.patch index 46c2f2c2..d2f72fb3 100644 --- a/debian/patches/no-single-process.patch +++ b/debian/patches/no-single-process.patch @@ -19,8 +19,8 @@ Last-Update: 2024-08-02 stdout = query_systemctl( --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py -@@ -519,7 +519,7 @@ def handle(name: str, cfg: Config, cloud - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno +@@ -545,7 +545,7 @@ def handle(name: str, cfg: Config, cloud + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno uses_systemd = cloud.distro.uses_systemd() default_mount_options = ( - "defaults,nofail,x-systemd.after=cloud-init-network.service,_netdev" @@ -30,7 +30,7 @@ Last-Update: 2024-08-02 ) --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json -@@ -2034,12 +2034,12 @@ +@@ -2073,12 +2073,12 @@ }, "mount_default_fields": { "type": "array", @@ -101,10 +101,10 @@ Last-Update: 2024-08-02 Wants=network-pre.target After=hv_kvp_daemon.service +After=systemd-remount-fs.service + Before=auditd.service Before=network-pre.target Before=shutdown.target - {% if variant in ["almalinux", "cloudlinux", "rhel"] %} -@@ -16,6 +17,7 @@ Before=firewalld.target +@@ -17,6 +18,7 @@ Before=firewalld.target Before=sysinit.target {% endif %} Conflicts=shutdown.target @@ -112,7 +112,7 @@ Last-Update: 2024-08-02 ConditionPathExists=!/etc/cloud/cloud-init.disabled ConditionKernelCommandLine=!cloud-init=disabled ConditionEnvironment=!KERNEL_CMDLINE=cloud-init=disabled -@@ -25,14 +27,7 @@ Type=oneshot +@@ -26,14 +28,7 @@ Type=oneshot {% if variant in ["almalinux", "cloudlinux", "rhel"] %} ExecStartPre=/sbin/restorecon /run/cloud-init {% endif %} @@ -141,7 +141,7 @@ Last-Update: 2024-08-02 -# https://www.freedesktop.org/software/systemd/man/latest/systemd-remount-fs.service.html -[Unit] -Description=Cloud-init: Single Process --{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "rhel"] %} +-{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} -DefaultDependencies=no -{% endif %} -{% if variant in ["almalinux", "cloudlinux", "rhel"] %} @@ -175,12 +175,12 @@ Last-Update: 2024-08-02 -WantedBy=cloud-init.target --- a/systemd/cloud-init-network.service.tmpl +++ /dev/null -@@ -1,64 +0,0 @@ +@@ -1,67 +0,0 @@ -## template:jinja -[Unit] -# https://docs.cloud-init.io/en/latest/explanation/boot.html -Description=Cloud-init: Network Stage --{% if variant not in ["almalinux", "cloudlinux", "photon", "rhel"] %} +-{% if variant not in ["almalinux", "cloudlinux", "photon", "raspberry-pi-os", "rhel"] %} -DefaultDependencies=no -{% endif %} -Wants=cloud-init-local.service @@ -190,12 +190,12 @@ Last-Update: 2024-08-02 -{% if variant not in ["ubuntu"] %} -After=systemd-networkd-wait-online.service -{% endif %} --{% if variant in ["ubuntu", "unknown", "debian"] %} +-{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} -After=networking.service -{% endif %} -{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", -- "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", -- "suse", "TencentOS", "virtuozzo"] %} +- "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "raspberry-pi-os", +- "rhel", "rocky", "suse", "TencentOS", "virtuozzo"] %} -After=NetworkManager.service -After=NetworkManager-wait-online.service -{% endif %} @@ -206,6 +206,9 @@ Last-Update: 2024-08-02 -After=dbus.service -{% endif %} -Before=network-online.target +-{% if variant == "raspberry-pi-os" %} +-Before=avahi-daemon.service +-{% endif %} -Before=sshd-keygen.service -Before=sshd.service -Before=systemd-user-sessions.service @@ -242,12 +245,12 @@ Last-Update: 2024-08-02 -WantedBy=cloud-init.target --- /dev/null +++ b/systemd/cloud-init.service.tmpl -@@ -0,0 +1,56 @@ +@@ -0,0 +1,59 @@ +## template:jinja +[Unit] +# https://docs.cloud-init.io/en/latest/explanation/boot.html +Description=Cloud-init: Network Stage -+{% if variant not in ["almalinux", "cloudlinux", "photon", "rhel"] %} ++{% if variant not in ["almalinux", "cloudlinux", "photon", "raspberry-pi-os", "rhel"] %} +DefaultDependencies=no +{% endif %} +Wants=cloud-init-local.service @@ -255,12 +258,12 @@ Last-Update: 2024-08-02 +Wants=sshd.service +After=cloud-init-local.service +After=systemd-networkd-wait-online.service -+{% if variant in ["ubuntu", "unknown", "debian"] %} ++{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} +After=networking.service +{% endif %} +{% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", -+ "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", -+ "suse", "TencentOS", "virtuozzo"] %} ++ "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "raspberry-pi-os", ++ "rhel", "rocky", "suse", "TencentOS", "virtuozzo"] %} + +After=NetworkManager.service +After=NetworkManager-wait-online.service @@ -272,6 +275,9 @@ Last-Update: 2024-08-02 +After=dbus.service +{% endif %} +Before=network-online.target ++{% if variant == "raspberry-pi-os" %} ++Before=avahi-daemon.service ++{% endif %} +Before=sshd-keygen.service +Before=sshd.service +Before=systemd-user-sessions.service diff --git a/debian/patches/series b/debian/patches/series index 8c0008e5..6de088d4 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -3,3 +3,4 @@ no-single-process.patch no-nocloud-network.patch grub-dpkg-support.patch no-remove-networkd-online.patch +strip-invalid-mtu.patch diff --git a/debian/patches/strip-invalid-mtu.patch b/debian/patches/strip-invalid-mtu.patch new file mode 100644 index 00000000..85150d21 --- /dev/null +++ b/debian/patches/strip-invalid-mtu.patch @@ -0,0 +1,18 @@ +Description: Strip invalid MTU values from rendered v2 + netplan configure. This maintains backwards compatibility + as these values are invalid netplan, but were allowed prior to 24.2 +Author: James Falcon +Bug: https://github.com/canonical/cloud-init/issues/6239 +Last-Update: 2025-07-09 + +--- a/cloudinit/features.py ++++ b/cloudinit/features.py +@@ -98,7 +98,7 @@ Note that in addition to this flag, down + to modify the systemd unit files. + """ + +-STRIP_INVALID_MTU = False ++STRIP_INVALID_MTU = True + """ + If ``STRIP_INVALID_MTU`` is True, then cloud-init will strip invalid MTU + values from rendered v2 netplan configuration. Cloud-init allowed these values diff --git a/doc/examples/cloud-config-ansible-controller.txt b/doc/examples/cloud-config-ansible-controller.txt index da2f58f0..3a982dae 100644 --- a/doc/examples/cloud-config-ansible-controller.txt +++ b/doc/examples/cloud-config-ansible-controller.txt @@ -19,7 +19,7 @@ users: gecos: Ansible User shell: /bin/bash groups: users,admin,wheel,lxd - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" # Initialize lxd using cloud-init. # -------------------------------- diff --git a/doc/examples/cloud-config-ansible-managed.txt b/doc/examples/cloud-config-ansible-managed.txt index b6508d21..00696e05 100644 --- a/doc/examples/cloud-config-ansible-managed.txt +++ b/doc/examples/cloud-config-ansible-managed.txt @@ -13,7 +13,7 @@ users: - name: ansible gecos: Ansible User groups: users,admin,wheel - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" shell: /bin/bash lock_passwd: true ssh_authorized_keys: diff --git a/doc/examples/cloud-config-ansible-pull.txt b/doc/examples/cloud-config-ansible-pull.txt index 73985772..61677fea 100644 --- a/doc/examples/cloud-config-ansible-pull.txt +++ b/doc/examples/cloud-config-ansible-pull.txt @@ -10,5 +10,5 @@ packages: ansible: install_method: pip pull: - url: "https://github.com/holmanb/vmboot.git" - playbook_name: ubuntu.yml + - url: "https://github.com/holmanb/vmboot.git" + playbook_names: [ubuntu.yml] diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index 2922181d..ce75d8e7 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -34,6 +34,9 @@ datasource: # seedfrom: http://my.example.com/i-abcde/ seedfrom: None + # fs_label: the label on filesystems to be searched for NoCloud source + fs_label: cidata + # these are optional, but allow you to basically provide a datasource # right here user-data: | diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index 3c8fc36c..cbdf0733 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -82,11 +82,11 @@ disk_setup: table_type: 'mbr' layout: true /dev/xvdh: - table_type: 'mbr' + table_type: 'gpt' layout: - 33 - [33, 82] - - 33 + - [33, '44479540-F297-41B2-9AF7-D131D5F0458A'] overwrite: True # The format is a list of dicts of dicts. The first value is the name of the @@ -95,7 +95,7 @@ disk_setup: # The general format is: # disk_setup: # : -# table_type: 'mbr' +# table_type: 'mbr'|'gpt' # layout: # overwrite: # @@ -126,7 +126,9 @@ disk_setup: # [, [, is the _percentage_ of the disk to use, while -# is the numerical value of the partition type. +# is either the numerical value of the partition type +# (two digit code from fdisk, four digit code from gdisk) or a full +# GPT partition GUID. # # The following setups two partitions, with the first # partition having a swap label, taking 1/3 of the disk space diff --git a/doc/examples/cloud-config-mount-points.txt b/doc/examples/cloud-config-mount-points.txt index 2f45fd4d..a77cdf54 100644 --- a/doc/examples/cloud-config-mount-points.txt +++ b/doc/examples/cloud-config-mount-points.txt @@ -3,7 +3,7 @@ # set up mount points # 'mounts' contains a list of lists # the inner list are entries for an /etc/fstab line -# ie : [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno ] +# ie : [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno ] # # default: # mounts: diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index 2cafef88..741e8d93 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -27,7 +27,7 @@ users: passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo gecos: Bar B. Foo - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" groups: users, admin ssh_import_id: - lp:falcojr @@ -44,7 +44,6 @@ users: inactive: '5' system: true - name: fizzbuzz - sudo: false shell: /bin/bash ssh_authorized_keys: - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSL7uWGj8cgWyIOaspgKdVy0cKJ+UTjfv7jBOjG2H/GN8bJVXy72XAvnhM0dUM+CCs8FOf0YlPX+Frvz2hKInrmRhZVwRSL129PasD12MlI3l44u6IwS1o/W86Q+tkQYEljtqDOo0a+cOsaZkvUNzUyEXUwz/lmYa6G4hMKZH4NBj7nbAAF96wsMCoyNwbWryBnDYUr6wMbjRR1J9Pw7Xh7WRC73wy4Va2YuOgbD3V/5ZrFPLbWZW/7TFXVrql04QVbyei4aiFR5n//GvoqwQDNe58LmbzX/xvxyKJYdny2zXmdAhMxbrpFQsfpkJ9E/H5w0yOdSvnWbUoG5xNGoOB csmith@fringe @@ -103,17 +102,14 @@ users: # strings or False to explicitly deny sudo usage. Examples: # # Allow a user unrestricted sudo access. -# sudo: ALL=(ALL) NOPASSWD:ALL +# sudo: "ALL=(ALL) NOPASSWD:ALL" # or # sudo: ["ALL=(ALL) NOPASSWD:ALL"] # # Adding multiple sudo rule strings. # sudo: -# - ALL=(ALL) NOPASSWD:/bin/mysql -# - ALL=(ALL) ALL -# -# Prevent sudo access for a user. -# sudo: False +# - "ALL=(ALL) NOPASSWD:/bin/mysql" +# - "ALL=(ALL) ALL" # # Note: Please double check your syntax and make sure it is valid. # cloud-init does not parse/check the syntax of the sudo diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 88fb8393..92c3395b 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -22,7 +22,7 @@ packages: # set up mount points # 'mounts' contains a list of lists # the inner list are entries for an /etc/fstab line -# ie : [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno ] +# ie : [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno ] # # default: # mounts: @@ -411,10 +411,10 @@ ssh_pwauth: True # syslog being taken down while cloud-init is running. # # delay: form accepted by shutdown. default is 'now'. other format -# accepted is '+m' (m in minutes) +# accepted is an integer for the number of minutes to delay # mode: required. must be one of 'poweroff', 'halt', 'reboot' # message: provided as the message argument to 'shutdown'. default is none. power_state: - delay: '+30' + delay: 30 mode: poweroff message: Bye Bye diff --git a/doc/examples/network-config-v1-subnet-routes.yaml b/doc/examples/network-config-v1-subnet-routes.yaml index 3e2db6ac..480d1a1d 100644 --- a/doc/examples/network-config-v1-subnet-routes.yaml +++ b/doc/examples/network-config-v1-subnet-routes.yaml @@ -6,7 +6,9 @@ network: mac_address: '00:11:22:33:44:55' subnets: - type: dhcp + metric: 200 - type: static + metric: 100 address: 10.184.225.122 netmask: 255.255.255.252 routes: diff --git a/doc/module-docs/cc_ansible/data.yaml b/doc/module-docs/cc_ansible/data.yaml index ca5ac3bf..b296f8b6 100644 --- a/doc/module-docs/cc_ansible/data.yaml +++ b/doc/module-docs/cc_ansible/data.yaml @@ -9,8 +9,10 @@ cc_ansible: - comment: | Example 1: file: cc_ansible/example1.yaml - - comment: | - Example 2: + - comment: > + Example 2: Multiple ansible-pull URLs can be provided by providing a list + of pull objects. Additionally, multiple playbooks can be provided as a + space-separated playbook_name value. file: cc_ansible/example2.yaml name: Ansible title: Configure Ansible for instance diff --git a/doc/module-docs/cc_ansible/example1.yaml b/doc/module-docs/cc_ansible/example1.yaml index 9fd5e520..85161021 100644 --- a/doc/module-docs/cc_ansible/example1.yaml +++ b/doc/module-docs/cc_ansible/example1.yaml @@ -3,5 +3,5 @@ ansible: package_name: ansible-core install_method: distro pull: - url: https://github.com/holmanb/vmboot.git - playbook_name: ubuntu.yml + - url: https://github.com/holmanb/vmboot.git + playbook_names: [ubuntu.yml] diff --git a/doc/module-docs/cc_ansible/example2.yaml b/doc/module-docs/cc_ansible/example2.yaml index 2adb491d..d54ded7a 100644 --- a/doc/module-docs/cc_ansible/example2.yaml +++ b/doc/module-docs/cc_ansible/example2.yaml @@ -3,5 +3,5 @@ ansible: package_name: ansible-core install_method: pip pull: - url: https://github.com/holmanb/vmboot.git - playbook_name: ubuntu.yml + - url: https://github.com/holmanb/vmboot.git + playbook_names: [ubuntu.yml, watermark.yml] diff --git a/doc/module-docs/cc_mounts/data.yaml b/doc/module-docs/cc_mounts/data.yaml index 18193f06..7548bdda 100644 --- a/doc/module-docs/cc_mounts/data.yaml +++ b/doc/module-docs/cc_mounts/data.yaml @@ -3,7 +3,7 @@ cc_mounts: This module can add or remove mount points from ``/etc/fstab`` as well as configure swap. The ``mounts`` config key takes a list of ``fstab`` entries to add. Each entry is specified as a list of ``[ fs_spec, fs_file, - fs_vfstype, fs_mntops, fs-freq, fs_passno ]``. + fs_vfstype, fs_mntops, fs_freq, fs_passno ]``. For more information on these options, consult the manual for ``/etc/fstab``. When specifying the ``fs_spec``, if the device name starts diff --git a/doc/module-docs/cc_raspberry_pi/data.yaml b/doc/module-docs/cc_raspberry_pi/data.yaml new file mode 100644 index 00000000..b7882bc1 --- /dev/null +++ b/doc/module-docs/cc_raspberry_pi/data.yaml @@ -0,0 +1,26 @@ +cc_raspberry_pi: + description: | + This module handles ARM interface configuration for Raspberry Pi. + + It also handles Raspberry Pi Connect installation and enablement. + Raspberry Pi Connect service will be installed and enabled to auto start on boot. + + This only works on Raspberry Pi OS (bookworm and later). + examples: + - comment: > + This example will enable the SPI and I2C interfaces on Raspberry Pi. + file: cc_raspberry_pi/example1.yaml + - comment: > + This example will enable the serial interface on Raspberry Pi. + file: cc_raspberry_pi/example2.yaml + - comment: > + This example will enable the serial interface on Raspberry Pi 5 and disable the UART hardware while enabling the console. + file: cc_raspberry_pi/example3.yaml + - comment: > + This example will enable ssh and the UART hardware without binding it to the console. + file: cc_raspberry_pi/example4.yaml + - comment: > + This example will enable the Raspberry Pi Connect service. + file: cc_raspberry_pi/example5.yaml + name: Raspberry Pi Configuration + title: Configure Raspberry Pi ARM interfaces and enable Raspberry Pi Connect diff --git a/doc/module-docs/cc_raspberry_pi/example1.yaml b/doc/module-docs/cc_raspberry_pi/example1.yaml new file mode 100644 index 00000000..0fe2ac83 --- /dev/null +++ b/doc/module-docs/cc_raspberry_pi/example1.yaml @@ -0,0 +1,5 @@ +#cloud-config +rpi: + interfaces: + spi: true + i2c: true diff --git a/doc/module-docs/cc_raspberry_pi/example2.yaml b/doc/module-docs/cc_raspberry_pi/example2.yaml new file mode 100644 index 00000000..83ef6390 --- /dev/null +++ b/doc/module-docs/cc_raspberry_pi/example2.yaml @@ -0,0 +1,4 @@ +#cloud-config +rpi: + interfaces: + serial: true diff --git a/doc/module-docs/cc_raspberry_pi/example3.yaml b/doc/module-docs/cc_raspberry_pi/example3.yaml new file mode 100644 index 00000000..eb74656a --- /dev/null +++ b/doc/module-docs/cc_raspberry_pi/example3.yaml @@ -0,0 +1,7 @@ +#cloud-config +rpi: + interfaces: + serial: + # Pi 5 only | disabling hardware while enabling console + console: true + hardware: false diff --git a/doc/module-docs/cc_raspberry_pi/example4.yaml b/doc/module-docs/cc_raspberry_pi/example4.yaml new file mode 100644 index 00000000..c600a276 --- /dev/null +++ b/doc/module-docs/cc_raspberry_pi/example4.yaml @@ -0,0 +1,9 @@ +#cloud-config +rpi: + interfaces: + ssh: true + # works on all Pi models + # only enables the UART hardware without binding it to the console + serial: + console: false + hardware: true diff --git a/doc/module-docs/cc_raspberry_pi/example5.yaml b/doc/module-docs/cc_raspberry_pi/example5.yaml new file mode 100644 index 00000000..3d5354ab --- /dev/null +++ b/doc/module-docs/cc_raspberry_pi/example5.yaml @@ -0,0 +1,3 @@ +#cloud-config +rpi: + enable_rpi_connect: true diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index 53b4b8c1..212ca832 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -1,6 +1,5 @@ import datetime import glob -import itertools import os import sys @@ -240,12 +239,12 @@ def render_property_template(prop_name, prop_cfg, prefix=""): description = f" {prop_cfg['description']}" else: description = "" - description += get_deprecated_str(prop_name, prop_cfg) - description += get_changed_str(prop_name, prop_cfg) jinja_vars = { "prefix": prefix, "name": prop_name, "description": description, + "deprecated": get_deprecated_str(prop_name, prop_cfg), + "changed": get_changed_str(prop_name, prop_cfg), "types": get_types_str(prop_cfg), "prop_cfg": prop_cfg, } @@ -270,14 +269,13 @@ def flatten_schema_refs(src_cfg: dict, defs: dict): if "$ref" in sub_schema: reference = sub_schema.pop("$ref").replace("#/$defs/", "") sub_schema.update(defs[reference]) - for sub_schema in itertools.chain( - src_cfg.get("oneOf", []), - src_cfg.get("anyOf", []), - src_cfg.get("allOf", []), - ): - if "$ref" in sub_schema: - reference = sub_schema.pop("$ref").replace("#/$defs/", "") - sub_schema.update(defs[reference]) + for key in ("anyOf", "oneOf", "allOf"): + if key in src_cfg: + for sub_schema in src_cfg[key]: + flatten_schema_refs(sub_schema, defs) + # if "$ref" in sub_schema: + # reference = sub_schema.pop("$ref").replace("#/$defs/", "") + # sub_schema.update(defs[reference]) def flatten_schema_all_of(src_cfg: dict): @@ -295,6 +293,9 @@ def flatten_schema_all_of(src_cfg: dict): def render_nested_properties(prop_cfg, defs, prefix): prop_str = "" + if "oneOf" in prop_cfg: + for alt_schema in prop_cfg["oneOf"]: + prop_str += render_nested_properties(alt_schema, defs, prefix) prop_types = set(["properties", "patternProperties"]) flatten_schema_refs(prop_cfg, defs) if "items" in prop_cfg: diff --git a/doc/rtd/development/contribute_code.rst b/doc/rtd/development/contribute_code.rst index 46b6755c..bd4942db 100644 --- a/doc/rtd/development/contribute_code.rst +++ b/doc/rtd/development/contribute_code.rst @@ -73,7 +73,6 @@ the codebase is encouraged. .. LINKS: .. include:: ../links.txt .. _quickstart documentation: https://docs.github.com/en/get-started/quickstart -.. _tools/.github-cla-signers: https://github.com/canonical/cloud-init/blob/main/tools/.github-cla-signers .. _repository: https://github.com/canonical/cloud-init .. _contributor-agreement-canonical: https://launchpad.net/%7Econtributor-agreement-canonical/+members .. _PR #344: https://github.com/canonical/cloud-init/pull/344 diff --git a/doc/rtd/development/contribute_docs.rst b/doc/rtd/development/contribute_docs.rst index f57d7146..810da3a9 100644 --- a/doc/rtd/development/contribute_docs.rst +++ b/doc/rtd/development/contribute_docs.rst @@ -52,13 +52,6 @@ write any content. In your first PR ================= -You will need to add your GitHub username (alphabetically) to the in-repository -list that we use to track :ref:`CLA signatures `: -`tools/.github-cla-signers`_. - -Please include this in the same PR alongside your first contribution. Do -not create a separate PR to add your name to the CLA signatures. - If you need some help with your contribution, you can contact us on our `IRC channel `_. If you have already submitted a work-in-progress PR, you can also ask for guidance from our technical author by `tagging s-makin`_ as a @@ -68,5 +61,4 @@ reviewer. .. include:: ../links.txt .. _cloud-init GitHub repository: https://github.com/canonical/cloud-init/tree/main/doc/rtd .. _Read the Docs: https://readthedocs.com/ -.. _tools/.github-cla-signers: https://github.com/canonical/cloud-init/blob/main/tools/.github-cla-signers .. _tagging s-makin: https://github.com/s-makin diff --git a/doc/rtd/development/first_PR.rst b/doc/rtd/development/first_PR.rst index b76028d5..0aa7bf5c 100644 --- a/doc/rtd/development/first_PR.rst +++ b/doc/rtd/development/first_PR.rst @@ -14,37 +14,14 @@ Sign the CLA ============ To contribute to cloud-init, you must first sign the Canonical -`contributor license agreement `_ (CLA). - -If you have already signed it as an individual, your Launchpad username will be -listed in the `contributor-agreement-canonical`_ group. Unfortunately there is -no easy way to check if the organisation or company you are working for has -signed it. - -When you sign: - -* ensure that you fill in the GitHub username field, -* when prompted for a 'Project contact' or 'Canonical Project Manager', enter - 'James Falcon'. - -If your company has signed the CLA for you, please contact us to help in -verifying which Launchpad/GitHub accounts are associated with the company. +`contributor license agreement `_ (CLA). A check is run against +every pull request to ensure that the CLA has been signed. For any questions or help with the process, email `James Falcon `_ with the subject: "Cloud-init CLA". You can also contact user ``falcojr`` in the #cloud-init channel on the `Libera IRC network `_. -Add your name to the CLA signers list -===================================== - -As part of your first PR to cloud-init, you should also add your GitHub -username (alphabetically) to the in-repository list that we use to track CLA -signatures: `tools/.github-cla-signers`_. - -`PR #344`_ and `PR #345`_ are good examples of what this should look like in -your pull request, though please do not use a separate PR for this step. - Create a sandbox environment ============================ @@ -75,7 +52,5 @@ cloud-init :ref:`Code Review Process`, so you can understand how your changes will end up in cloud-init's codebase. .. include:: ../links.txt -.. _tools/.github-cla-signers: https://github.com/canonical/cloud-init/blob/main/tools/.github-cla-signers -.. _contributor-agreement-canonical: https://launchpad.net/%7Econtributor-agreement-canonical/+members .. _PR #344: https://github.com/canonical/cloud-init/pull/344 .. _PR #345: https://github.com/canonical/cloud-init/pull/345 diff --git a/doc/rtd/development/index.rst b/doc/rtd/development/index.rst index 73a90247..18dab526 100644 --- a/doc/rtd/development/index.rst +++ b/doc/rtd/development/index.rst @@ -57,8 +57,6 @@ Pull request checklist Before any pull request can be accepted, remember to do the following: -* Make sure your GitHub username is added (alphabetically) to the in-repository - list that we use to track CLA signatures: `tools/.github-cla-signers`_. * Add or update any :ref:`unit tests` accordingly. * Add or update any :ref:`integration_tests` (if applicable). * Format code (using ``black`` and ``isort``) with ``tox -e do_format``. @@ -79,7 +77,6 @@ Debugging and reporting .. LINKS: .. include:: ../links.txt .. _quickstart documentation: https://docs.github.com/en/get-started/quickstart -.. _tools/.github-cla-signers: https://github.com/canonical/cloud-init/blob/main/tools/.github-cla-signers .. _repository: https://github.com/canonical/cloud-init .. _contributor-agreement-canonical: https://launchpad.net/%7Econtributor-agreement-canonical/+members .. _PR #344: https://github.com/canonical/cloud-init/pull/344 diff --git a/doc/rtd/development/integration_tests.rst b/doc/rtd/development/integration_tests.rst index 17010c05..ee60ae0a 100644 --- a/doc/rtd/development/integration_tests.rst +++ b/doc/rtd/development/integration_tests.rst @@ -95,6 +95,23 @@ to ``True``. KEEP_INSTANCE = True +To keep the instance only if the test was unsuccessful, set ``KEEP_INSTANCE`` +variable to ``ON_ERROR``. + +.. tab-set:: + + .. tab-item:: Inline environment variable + + .. code-block:: bash + + CLOUD_INIT_KEEP_INSTANCE=ON_ERROR tox -e integration_tests + + .. tab-item:: user_settings.py file + + .. code-block:: python + + KEEP_INSTANCE = "ON_ERROR" + Use in-place cloud-init source code ------------------------------------- diff --git a/doc/rtd/explanation/format.rst b/doc/rtd/explanation/format.rst index f999a9ec..a1fd83dd 100644 --- a/doc/rtd/explanation/format.rst +++ b/doc/rtd/explanation/format.rst @@ -157,9 +157,8 @@ Explanation ----------- An include file contains a list of URLs, one per line. Each of the URLs will -be read and their content can be any kind of user-data format, both base -config and meta config. If an error occurs reading a file the remaining files -will not be read. +be read and their content can be any kind of user-data format. If an error +occurs reading a file the remaining files will not be read. .. _user_data_formats-jinja: @@ -388,7 +387,7 @@ as binary data and so may be processed automatically. +--------------------+-----------------------------+-------------------------+ |Cloud config archive|#cloud-config-archive |text/cloud-config-archive| +--------------------+-----------------------------+-------------------------+ -|Jinja template |## template: jinja |text/jinja | +|Jinja template |## template: jinja |text/jinja2 | +--------------------+-----------------------------+-------------------------+ |Include file |#include |text/x-include-url | +--------------------+-----------------------------+-------------------------+ diff --git a/doc/rtd/explanation/hardening.rst b/doc/rtd/explanation/hardening.rst new file mode 100644 index 00000000..06c9be3b --- /dev/null +++ b/doc/rtd/explanation/hardening.rst @@ -0,0 +1,84 @@ +Security Hardening +****************** + +Cloud-init's use case is automating cloud instance initialization, with support +across distributions and platforms. There are a myriad of ways to imrpove the +security posture of a cloud-init configured machine. + +Follow the security hardening guidelines provided by the OSes and cloud +platforms that your cloud-init configuration is targeting. + +Many cloud platforms provide SSH public keys in metadata which setup the +default user with the appropriate configured means of access using +SSH public/private key pairs. + + +Updated packages +================ + +To ensure the available security fixes are applied to you VMs images upon +launch, it is recommended by `Ubuntu security team guidelines`_ to update +the packages + +.. note:: + + Ubuntu cloud images are configured by default to enable unattended-upgrades, + thus this is resolved this issue when the update gets triggered. One can + still apply this recommendation to cloud that gap and update the packages + on first boot. + +.. code-block:: yaml + + #cloud-config + package_update: true + package_upgrade: true + + +No plain text passwords +======================= + +Most of the harmful security exposure comes when custom user-data presented +as ``#cloud-config`` or run scripts by the end-user at VM launch time which +provides credentials in the form of clear passwords or credentials encoded in +URLs for services. + + +It is recommended not to include plain-text passwords or credentials in any +``runcmd``, ``bootcmd``, or user-data scripts (e.g., ``#!/bin/bash``), as this +configuration user-data may be accessible to others on the local network +depending on the cloud platform's instance metadata service (IMDS). +Instead, retrieve credentials for service endpoints from a secure +vault or configuration management service such as Puppet, Chef, Ansible, +or SaltStack. + +While creating users with the +:ref:`Users and Groups module `, do not use the +``user.plain_text_passwd`` key with its associated value as plain text. +``hashed_passwd`` is a more secure alternative. + +Avoid plain text passwords with the +:ref:`Set Passwords `. + +Alternatives to user passwords +------------------------------ + +We recommend using the :ref:`SSH module ` with ``ssh_import_id`` or +``ssh_authorized_keys`` to import public SSH keys. + + + +More info on `managing SSH-keys for openssh-server`_. + +SSH Host keys +============= + +Cloud-init publishes the SSH host public keys generated to the serial console +which can be validated prior to any SSH client connection to the launched VM. + +It provides assurance that you are connecting to the virtual machine you +intended to launch, and not being intercepted by a man-in-the-middle (MITM) +attack. + + +.. _Ubuntu security team guidelines: https://documentation.ubuntu.com/server/explanation/security/security_suggestions/#keep-your-system-up-to-date +.. _managing SSH-keys for openssh-server: https://documentation.ubuntu.com/server/how-to/security/openssh-server/#ssh-keys diff --git a/doc/rtd/explanation/index.rst b/doc/rtd/explanation/index.rst index 9e2b82a7..5465a838 100644 --- a/doc/rtd/explanation/index.rst +++ b/doc/rtd/explanation/index.rst @@ -20,8 +20,10 @@ knowledge and become better at using and configuring ``cloud-init``. instancedata.rst vendordata.rst security.rst + hardening.rst analyze.rst kernel-command-line.rst failure_states.rst exported_errors.rst return_codes.rst + net-device-info.rst diff --git a/doc/rtd/explanation/instancedata.rst b/doc/rtd/explanation/instancedata.rst index 9b6ede9c..6d2b61c2 100644 --- a/doc/rtd/explanation/instancedata.rst +++ b/doc/rtd/explanation/instancedata.rst @@ -278,7 +278,7 @@ on. This is different than the 'platform' item. For example, the cloud name of Amazon Web Services is 'aws', while the platform is 'ec2'. If determining a specific name is not possible or provided in -:file:`meta-data`, then this filed may contain the same content as 'platform'. +:file:`meta-data`, then this field may contain the same content as 'platform'. Example output: diff --git a/doc/rtd/explanation/net-device-info.rst b/doc/rtd/explanation/net-device-info.rst new file mode 100644 index 00000000..6e0eb6cc --- /dev/null +++ b/doc/rtd/explanation/net-device-info.rst @@ -0,0 +1,42 @@ +.. _net-device-info: + +Net Device Info +=============== + +During boot, cloud-init prints a table to the serial console that summarizes +the information about detected network devices and their state at that +moment in time. A sample table appears as follows: + +.. spelling:word-list:: + :ignore-words: + Hw + fe + +.. csv-table:: + :align: center + :header-rows: 1 + + Device, Up, Address, Mask, Scope, Hw-Address + eth0, True, 10.70.162.47, 255.255.255.0, global, 00:16:3e:1d:76:9b + eth0, True, fe80::216:3eff:fe1d:769b/64, ., link, 00:16:3e:1d:76:9b + lo, True, 127.0.0.1, 255.0.0.0, host, . + lo, True, ::1/128, ., host, . + +This table may be useful to analyze scenarios where an instance fails +to boot. The table is not printed in the final stage and it does not +necessarily represent the network’s final state. + +Why does the table show interfaces as down? +---------------------------------------------- + +If the table shows interfaces as down, it means that they were not +activated by the system before the table was printed. There are a few +potential reasons this can occur. Some explanations include: the order +in which the distro starts system services results in network devices +being brought up after the table is printed, the supplied network +configuration either implicitly or explicitly does not bring up the +interfaces + +The recommended troubleshooting advice is to inspect the network from +the shell and confirm whether or not it matches your expected setup +and follow the :ref:`debugging instructions`. diff --git a/doc/rtd/howto/index.rst b/doc/rtd/howto/index.rst index d3bc2552..d03e3909 100644 --- a/doc/rtd/howto/index.rst +++ b/doc/rtd/howto/index.rst @@ -19,6 +19,7 @@ How do I...? :maxdepth: 1 Launch cloud-init with... + Wait for cloud-init Re-run cloud-init Change how often a module runs Validate my user-data diff --git a/doc/rtd/howto/launch_lxd.rst b/doc/rtd/howto/launch_lxd.rst index c3f76f22..236e0be7 100644 --- a/doc/rtd/howto/launch_lxd.rst +++ b/doc/rtd/howto/launch_lxd.rst @@ -29,7 +29,7 @@ we just created: .. code-block:: shell-session - $ lxc init ubuntu-daily:jammy test-container + $ lxc init ubuntu-daily:noble test-container $ lxc config set test-container user.user-data - < user-data.yaml $ lxc start test-container @@ -37,7 +37,7 @@ To avoid the extra commands this can also be done at launch: .. code-block:: shell-session - $ lxc launch ubuntu-daily:jammy test-container --config=user.user-data="$(cat user-data.yaml)" + $ lxc launch ubuntu-daily:noble test-container --config=user.user-data="$(cat user-data.yaml)" Finally, a profile can be set up with the specific data if you need to launch this multiple times: @@ -46,7 +46,7 @@ launch this multiple times: $ lxc profile create dev-user-data $ lxc profile set dev-user-data user.user-data - < cloud-init-config.yaml - $ lxc launch ubuntu-daily:jammy test-container -p default -p dev-user-data + $ lxc launch ubuntu-daily:noble test-container -p default -p dev-user-data LXD configuration types ----------------------- diff --git a/doc/rtd/howto/launch_qemu.rst b/doc/rtd/howto/launch_qemu.rst index 49441828..ed4cd90f 100644 --- a/doc/rtd/howto/launch_qemu.rst +++ b/doc/rtd/howto/launch_qemu.rst @@ -49,7 +49,7 @@ Boot the cloud image with our configuration, ``seed.img``, to QEMU: .. code-block:: shell-session $ qemu-system-x86_64 -m 1024 -net nic -net user \ - -drive file=jammy-server-cloudimg-amd64.img,index=0,format=qcow2,media=disk \ + -drive file=noble-server-cloudimg-amd64.img,index=0,format=qcow2,media=disk \ -drive file=seed.img,index=1,media=cdrom \ -machine accel=kvm:tcg diff --git a/doc/rtd/howto/shared/download_image.txt b/doc/rtd/howto/shared/download_image.txt index 56fad70f..bdc680a9 100644 --- a/doc/rtd/howto/shared/download_image.txt +++ b/doc/rtd/howto/shared/download_image.txt @@ -2,4 +2,4 @@ Download an Ubuntu image to run: .. code-block:: shell-session - wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img + wget https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img diff --git a/doc/rtd/howto/wait_for_cloud_init.rst b/doc/rtd/howto/wait_for_cloud_init.rst new file mode 100644 index 00000000..929ab138 --- /dev/null +++ b/doc/rtd/howto/wait_for_cloud_init.rst @@ -0,0 +1,33 @@ +.. _wait_for_cloud_init: + +How to wait for cloud-init +************************** + +It is useful to be able to wait until cloud-init has completed running prior +to doing some other task. + +CLI +=== + +Cloud-init's command ``cloud-init status --wait`` will exit once cloud-init has +completed. + +SystemD +======= + +Systems using systemd may be configured to start a service after cloud-init +completes. This may be accomplished by including +``After=cloud-init.target multi-user.target`` in the unit file. For example: + +.. code-block:: + + [Unit] + Description=Example service + After=cloud-final.service multi-user.target + + [Service] + Type=oneshot + ExecStart=sh -c 'echo "Howdy partner 🤠"' + + [Install] + WantedBy=multi-user.target diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index 061e429b..76af68cf 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -104,8 +104,8 @@ projects, contributions, suggestions, fixes and constructive feedback. .. LINKS .. include:: links.txt -.. _the cloud-init Discourse forum: https://discourse.ubuntu.com/c/server/cloud-init/ +.. _the cloud-init Discourse forum: https://discourse.ubuntu.com/c/project/cloud-init/54 .. _cloud-init mailing list: https://launchpad.net/~cloud-init .. _mailing list archive: https://lists.launchpad.net/cloud-init/ -.. _Release schedule: https://discourse.ubuntu.com/t/cloud-init-release-schedule/32244 +.. _Release schedule: https://discourse.ubuntu.com/t/2025-cloud-init-release-schedule/55534 .. _Report bugs on GitHub Issues: https://github.com/canonical/cloud-init/issues diff --git a/doc/rtd/reference/availability.rst b/doc/rtd/reference/availability.rst index f26417d7..5f6b983c 100644 --- a/doc/rtd/reference/availability.rst +++ b/doc/rtd/reference/availability.rst @@ -17,19 +17,41 @@ Distributions ``Cloud-init`` has support across all major Linux distributions, FreeBSD, NetBSD, OpenBSD and DragonFlyBSD: +- AlmaLinux - Alpine Linux +- AOSC OS +- Amazon Linux 2023 - Arch Linux +- CentOS +- CloudLinux +- Container-Optimized OS - Debian - DragonFlyBSD +- EuroLinux - Fedora - FreeBSD - Gentoo Linux +- MarinerOS +- MIRACLE LINUX - NetBSD - OpenBSD +- openEuler +- OpenCloudOS +- OpenMandriva - Photon OS -- RHEL/CentOS/AlmaLinux/Rocky Linux/EuroLinux +- Raspberry Pi OS +- Red Hat Enterprise Linux (RHEL) +- Rocky Linux - SLES/openSUSE +- TencentOS - Ubuntu +- Virtuozzo + +.. note:: + + While BSD variants are not typically referred to as "distributions", + ``cloud-init`` has an abstraction to account for operating system differences, + which can be found in the `cloudinit/distros/ `_ directory. Clouds ====== diff --git a/doc/rtd/reference/breaking_changes.rst b/doc/rtd/reference/breaking_changes.rst index 7f85c9df..1f5d613c 100644 --- a/doc/rtd/reference/breaking_changes.rst +++ b/doc/rtd/reference/breaking_changes.rst @@ -11,7 +11,7 @@ releases. many operating system vendors patch out breaking changes in cloud-init to ensure consistent behavior on their platform. -25.1.3 +25.1.4 ====== Strict datasource identity before network @@ -37,28 +37,59 @@ The most likely affected cloud platforms are AltCloud, Ec2 and OpenStack for non-x86 architectures where DMI data is not exposed by the kernel. If your non-x86 architecture or images no longer detect the proper datasource, -any of the following steps can ensure proper detection of cloud-init config: +cloud-init will remain disabled and perform no configuration operations during +boot. -- Provide kernel commandline containing ``ds=`` - which forces ds-identify to discover a specific datasource. -- Image creators: provide a config file part such as - :file:`/etc/cloud/cloud.cfg.d/*.cfg` containing the - case-sensitive ``datasource_list: [ ]`` to force cloud-init - to use a specific datasource without performing discovery. +Any of the following alternatives can ensure proper enablement of cloud-init +in non-x86 images without DMI-data: -For example, to force OpenStack discovery in cloud-init any of the following -approaches work: +- When launching VMs with the openstack command line client, provide + ``--config-drive true``: -- OpenStack: `attach a ConfigDrive`_ as an alternative config source -- Kernel command line containing ``ds=openstack`` -- Custom images provide :file:`/etc/cloud/cloud.cfg.d/91-set-datasource.cfg` - containing: +.. code-block:: shell-session + + $ openstack server create ... --config-drive true + +- On the openstack image command line, modify specific image metadata to + require config drive for the image: + +.. code-block:: shell-session + + $ openstack image set --property img_config_drive=mandatory + + +- OpenStack image creators can place a config file in the image at + :file:`/etc/cloud/cloud.cfg.d/91_openstack.cfg` to force + cloud-init to use OpenStack without DMI-based discovery. The file must + contain a single datasource as follows: .. code-block:: yaml datasource_list: [ OpenStack ] +- Charmed OpenStack Admins using glance-simplestreams-sync can default all + syncronized images to use config_drive: + +.. code-block:: shell-session + + $ juju config glance-simplestreams-sync custom-properties="img_config_drive=mandatory" + + +- OpenStack Nova admins can globally configure Nova to provide config drives + to all images by default in :file:`/etc/nova/nova.conf`: + +.. code-block:: toml + + [DEFAULT] + force_config_drive = true + +- Alternatively, providing + :ref:`kernel command line arguments` to a + virtual machine containing ``ds=openstack`` will force ds-identify to use the + specific datasource. + + 25.1 ==== @@ -70,7 +101,15 @@ anything that was installed to ``/lib`` is now installed to ``/usr/lib``. This shouldn't affect any systemd-based distributions as they have all transitioned to the ``/usr`` merge. However, this could affect older stable releases, non-systemd and non-Linux distributions. See -`this commit `_ +`commit 054734921 `_ +for more details. + +24.4 +==== + +Cloud-init's `cloud-final.service` order was standardized. This caused a +change to the systemd boot order on some distributions. See +`commit 245f94674 `_ for more details. 24.3 diff --git a/doc/rtd/reference/cli.rst b/doc/rtd/reference/cli.rst index c540df3d..dff40b50 100644 --- a/doc/rtd/reference/cli.rst +++ b/doc/rtd/reference/cli.rst @@ -86,10 +86,19 @@ re-run all stages as it did on first boot. remove the file. Best practice when cloning a golden image, to ensure the next boot of that image auto-generates a unique machine ID. `More details on machine-id`_. -* :command:`--configs [all | ssh_config | network ]`: Optionally remove all - ``cloud-init`` generated config files. Argument `ssh_config` cleans - config files for ssh daemon. Argument `network` removes all generated - config files for network. `all` removes config files of all types. +* :command:`--configs [all | ssh_config | network | datasource | fstab ]`: + Optionally remove all ``cloud-init`` generated config files. Argument + `ssh_config` cleans config files for ssh daemon. Argument `network` removes + all generated config files for network. Argument `datasource` removes files + and/or configs written by current datasource. It includes `fstab` entries + that have been only configured by this datasource leaving aside other entries + configured by cloud-init. Argument `fstab` removes all entries that have + been configured by cloud-init including those that are configured by various + datasources. `all` removes config files of all types. +* :command:`--seed`: Remove the cloud-init seed directory + (e.g., :file:`/var/lib/cloud/seed/`) + which stores instance metadata used initializing a datasource. + Useful when regenerating metadata from a new or updated seed source. .. note:: diff --git a/doc/rtd/reference/datasources/vmware.rst b/doc/rtd/reference/datasources/vmware.rst index b7c3edf0..8cae5e01 100644 --- a/doc/rtd/reference/datasources/vmware.rst +++ b/doc/rtd/reference/datasources/vmware.rst @@ -325,6 +325,76 @@ If either of the above values are true, then the datasource will sleep for a second, check the network status, and repeat until one or both addresses from the specified families are available. +Update Event support +-------------------- + +The VMware datasource supports the following types of update events: + +* Network -- ``boot``, ``boot-new-instance``, and ``hotplug`` + +This means the guest will reconfigure networking from the network +configuration provided via guestinfo, IMC, etc. each time the guest +boots or even when a new network interface is added. + +It is possible to override the data source's default set of configured +update events by specifying which events to use via user data. +For example, the following snippet from user data would disable the +`hotplug` event: + + .. code-block:: yaml + + #cloud-config + updates: + network: + when: ["boot", "boot-new-instance"] + +Determining the supported and enabled update events +--------------------------------------------------- + +This datasource also advertises the scope and type of the supported +and enabled events. + +The ``guestinfo`` key ``guestinfo.cloudinit.updates.supported`` +contains a list of the supported scopes and types that adheres to the +format ``SCOPE=TYPE[;TYPE][,SCOPE=TYPE[;TYPE]]``, for example: + +* ``network=boot;hotplug`` +* ``network=boot-new-instance`` + +The value is based on the events supported by the datasource, whether +or not the event is enabled. To inspect which events are enabled, use +``guestinfo.cloudinit.updates.enabled``. + +This allows a consumer to determine if different versions of the +datasource have different supported event types, regardless of which +events are enabled. + +Network drivers and the hotplug update event +-------------------------------------------- + +By default, this datasource only responds to hotplug events if the +driver is one of the following: + +* ``e1000`` +* ``e1000e`` +* ``vlance`` +* ``vmxnet2`` +* ``vmxnet3`` +* ``vrdma`` + +This prevents responding unintentionally to interfaces created by +Docker or other programs. However, it is also possible to override this +list by setting ``metadata.network-drivers`` to a list of drivers: + + .. code-block:: yaml + + network-drivers: + - vmxnet2 + - vmxnet3 + +The above snippet means only NICs that use either the ``vmxnet2`` or +``vmxnet3`` drivers will respond to hotplug events. + Walkthrough of GuestInfo keys transport ======================================= @@ -356,7 +426,7 @@ this datasource using the GuestInfo keys transport: - default - name: akutz primary_group: akutz - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" groups: sudo, wheel lock_passwd: true ssh_authorized_keys: @@ -485,4 +555,3 @@ the meta-data key ``network``. Valid encodings are ``base64`` and .. _VMware vSphere Product Documentation: https://docs.vmware.com/en/VMware-vSphere/8.0/vsphere-vm-administration/GUID-EB5F090E-723C-4470-B640-50B35D1EC016.html#GUID-9A5093A5-C54F-4502-941B-3F9C0F573A39__GUID-40C60643-A2EB-4B05-8927-B51AF7A6CC5E .. _property: https://vdc-repo.vmware.com/vmwb-repository/dcr-public/723e7f8b-4f21-448b-a830-5f22fd931b01/5a8257bd-7f41-4423-9a73-03307535bd42/doc/vim.vm.ConfigInfo.html .. _govc: https://github.com/vmware/govmomi/blob/master/govc - diff --git a/doc/rtd/reference/datasources/wsl.rst b/doc/rtd/reference/datasources/wsl.rst index 54200548..bed96420 100644 --- a/doc/rtd/reference/datasources/wsl.rst +++ b/doc/rtd/reference/datasources/wsl.rst @@ -135,14 +135,15 @@ the case to the best of our knowledge at the time of this writing. Most of what ``meta-data`` is intended for is not applicable under WSL, such as setting a hostname. Yet, the knowledge of ``meta-data.instance-id`` is vital for cloud-init. So, this datasource provides a default value but also supports -optionally sourcing meta-data from a per-instance specific configuration file: -``%USERPROFILE%\.cloud-init\.meta-data``. If that file exists, it -is a YAML-formatted file minimally providing a value for instance ID -such as: ``instance-id: x-y-z``. Advanced users looking to share -snapshots or relaunch a snapshot where cloud-init is re-triggered, must run -``sudo cloud-init clean --logs`` on the instance before snapshot/export, or -create the appropriate ``.meta-data`` file containing ``instance-id: -some-new-instance-id``. +optionally sourcing meta-data from a per-instance specific configuration file +located either at ``%USERPROFILE%\.cloud-init\.meta-data`` or +``%USERPROFILE%\.ubuntupro\.cloud-init\.meta-data``. When both +files exist, only the second will be loaded. The file must be a YAML-formatted +file minimally providing a value for instance ID such as: ``instance-id: +x-y-z``. Advanced users looking to share snapshots or relaunch a snapshot where +cloud-init is re-triggered, must run ``sudo cloud-init clean --logs`` on the +instance before snapshot/export, or create the appropriate ``.meta-data`` file +containing ``instance-id: some-new-instance-id``. Unsupported or restricted modules and features =============================================== @@ -204,7 +205,7 @@ include file. - name: j gecos: Agent J groups: users,sudo,netdev,audio - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" shell: /bin/bash lock_passwd: true diff --git a/doc/rtd/reference/distros.rst b/doc/rtd/reference/distros.rst deleted file mode 100644 index d54cb889..00000000 --- a/doc/rtd/reference/distros.rst +++ /dev/null @@ -1,43 +0,0 @@ -Supported distros -================= - -Cloud-init has support for multiple different operating systems. -Currently support includes various different distributions within the -Unix family of operating systems. See the complete list below. - -* AlmaLinux -* Alpine Linux -* AOSC OS -* Arch Linux -* CentOS -* CloudLinux -* Container-Optimized OS -* Debian -* DragonFlyBSD -* EuroLinux -* Fedora -* FreeBSD -* Gentoo -* MarinerOS -* MIRACLE LINUX -* NetBSD -* OpenBSD -* openEuler -* OpenCloudOS -* OpenMandriva -* PhotonOS -* Red Hat Enterprise Linux -* Rocky -* SLES/openSUSE -* TencentOS -* Ubuntu -* Virtuozzo - -If you would like to add support for another distributions, start by -taking a look at another distro module in ``cloudinit/distros/``. - -.. note:: - - While BSD variants are not typically referred to as "distributions", - cloud-init has an abstraction to account for operating system differences, which - should be contained in `cloudinit/distros/ `_. diff --git a/doc/rtd/reference/examples.rst b/doc/rtd/reference/examples.rst index c88eed3c..26f47c87 100644 --- a/doc/rtd/reference/examples.rst +++ b/doc/rtd/reference/examples.rst @@ -117,8 +117,8 @@ Adjust mount points mounted :language: yaml :linenos: -``Configure instance's SSH keys`` -================================= +Configure instance's SSH keys +============================= .. literalinclude:: ../../examples/cloud-config-ssh-keys.txt :language: yaml diff --git a/doc/rtd/reference/index.rst b/doc/rtd/reference/index.rst index cb6fa173..4a5e7f26 100644 --- a/doc/rtd/reference/index.rst +++ b/doc/rtd/reference/index.rst @@ -18,7 +18,6 @@ matrices and so on. faq.rst merging.rst datasources.rst - distros.rst network-config.rst base_config_reference.rst datasource_dsname_map.rst diff --git a/doc/rtd/reference/modules.rst b/doc/rtd/reference/modules.rst index c5cdef16..5ce7134d 100644 --- a/doc/rtd/reference/modules.rst +++ b/doc/rtd/reference/modules.rst @@ -74,6 +74,8 @@ Modules :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_puppet/data.yaml :template: modules.tmpl +.. datatemplate:yaml:: ../../module-docs/cc_raspberry_pi/data.yaml + :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_resizefs/data.yaml :template: modules.tmpl .. datatemplate:yaml:: ../../module-docs/cc_resolv_conf/data.yaml diff --git a/doc/rtd/reference/network-config-format-v1.rst b/doc/rtd/reference/network-config-format-v1.rst index 1a1bf58a..5186075b 100644 --- a/doc/rtd/reference/network-config-format-v1.rst +++ b/doc/rtd/reference/network-config-format-v1.rst @@ -95,6 +95,15 @@ Physical example .. literalinclude:: ../../examples/network-config-v1-physical-3-nic.yaml :language: yaml +``keep_configuration: `` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Designate the connection as 'critical to the system', meaning that special care +will be taken not to release the assigned IP when the daemon is restarted. + +.. note:: + This is only recognized by Netplan renderer. + Bond ---- @@ -310,6 +319,7 @@ Valid keys for ``subnets`` include the following: - ``dns_nameservers``: Specify a list of IPv4 DNS server IPs. - ``dns_search``: Specify a list of DNS search paths. - ``routes``: Specify a list of routes for a given interface. +- ``metric``: Integer which sets the metric cost of routes within this subnet. Subnet types are one of the following: diff --git a/doc/rtd/reference/network-config-format-v2.rst b/doc/rtd/reference/network-config-format-v2.rst index 3bb0b0d4..c90e3b62 100644 --- a/doc/rtd/reference/network-config-format-v2.rst +++ b/doc/rtd/reference/network-config-format-v2.rst @@ -244,7 +244,9 @@ through DHCP or RA. Each sequence entry is in CIDR notation, i.e., of the form ``addr/prefixlen``. ``addr`` is an IPv4 or IPv6 address as recognized by ``inet_pton(3)`` and ``prefixlen`` the number of bits of the subnet. -Example: ``addresses: [192.168.14.2/24, 2001:1::1/64]`` +Example: :: + + addresses: [192.168.14.2/24, 2001:1::1/64] ``gateway4: or gateway6: <(scalar)>`` ------------------------------------- @@ -254,8 +256,13 @@ Set default gateway for IPv4/6, for manual address configuration. This requires setting ``addresses`` too. Gateway IPs must be in a form recognized by ``inet_pton(3)`` -Example for IPv4: ``gateway4: 172.16.0.1`` -Example for IPv6: ``gateway6: 2001:4::1`` +Example for IPv4: :: + + gateway4: 172.16.0.1 + +Example for IPv6: :: + + gateway6: 2001:4::1 ``mtu: `` ------------------------ diff --git a/doc/rtd/reference/network-config.rst b/doc/rtd/reference/network-config.rst index 46d4d977..5bd99c9a 100644 --- a/doc/rtd/reference/network-config.rst +++ b/doc/rtd/reference/network-config.rst @@ -182,13 +182,13 @@ supports a wide range of networking setups. Configuration is typically stored in :file:`/etc/NetworkManager`. It is the default for a number of Linux distributions; notably Fedora, -CentOS/RHEL, and their derivatives. +CentOS/RHEL, Raspberry Pi OS, and their derivatives. ENI --- :file:`/etc/network/interfaces` or ``ENI`` is supported by the ``ifupdown`` -package found in Alpine Linux, Debian and Ubuntu. +package found in Alpine Linux, Debian, Raspberry Pi OS and Ubuntu. Netplan ------- @@ -270,7 +270,7 @@ Example output: .. code-block:: usage: /usr/bin/cloud-init devel net-convert [-h] -p PATH -k {eni,network_data.json,yaml,azure-imds,vmware-imc} -d PATH -D - {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openEuler} + {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openEuler,raspberry-pi-os} [-m name,mac] [--debug] -O {eni,netplan,networkd,sysconfig,network-manager} options: @@ -281,7 +281,7 @@ Example output: The format of the given network config -d PATH, --directory PATH directory to place output in - -D {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openeuler}, --distro {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openEuler} + -D {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openeuler}, --distro {alpine,arch,azurelinux,debian,ubuntu,freebsd,dragonfly,gentoo,cos,netbsd,openbsd,almalinux,amazon,centos,cloudlinux,eurolinux,fedora,mariner,miraclelinux,openmandriva,photon,rhel,rocky,virtuozzo,opensuse,sles,openEuler,raspberry-pi-os} -m name,mac, --mac name,mac interface name to mac mapping --debug enable debug logging to stderr. diff --git a/doc/rtd/reference/yaml_examples/ansible_controller.rst b/doc/rtd/reference/yaml_examples/ansible_controller.rst index b7da819f..45c9d857 100644 --- a/doc/rtd/reference/yaml_examples/ansible_controller.rst +++ b/doc/rtd/reference/yaml_examples/ansible_controller.rst @@ -28,7 +28,7 @@ For a full list of keys, refer to the `Ansible module`_ schema. gecos: Ansible User shell: /bin/bash groups: users,admin,wheel,lxd - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" # Initialize LXD using cloud-init # ------------------------------- diff --git a/doc/rtd/reference/yaml_examples/ansible_managed.rst b/doc/rtd/reference/yaml_examples/ansible_managed.rst index 83eff726..be1ba7c3 100644 --- a/doc/rtd/reference/yaml_examples/ansible_managed.rst +++ b/doc/rtd/reference/yaml_examples/ansible_managed.rst @@ -20,7 +20,7 @@ schema. - name: ansible gecos: Ansible User groups: users,admin,wheel - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" shell: /bin/bash lock_passwd: true ssh_authorized_keys: @@ -66,4 +66,3 @@ schema. # Ok11rJYIe7+e9B0lhku0AFwGyqlWQmS/MhIpnjHIk5tP4heHGSmzKQWJDbTskNWd6aq1G7 # 6HWfDpX4HgoM8AAAALaG9sbWFuYkBhcmM= # -----END OPENSSH PRIVATE KEY----- - diff --git a/doc/rtd/reference/yaml_examples/mounts.rst b/doc/rtd/reference/yaml_examples/mounts.rst index 4c9534bc..42871ca6 100644 --- a/doc/rtd/reference/yaml_examples/mounts.rst +++ b/doc/rtd/reference/yaml_examples/mounts.rst @@ -44,7 +44,7 @@ contains entries for an ``/etc/fstab`` line, e.g.: .. code-block:: yaml - [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno ] + [ fs_spec, fs_file, fs_vfstype, fs_mntops, fs_freq, fs_passno ] With defaults: diff --git a/doc/rtd/reference/yaml_examples/user_groups.rst b/doc/rtd/reference/yaml_examples/user_groups.rst index 1a23dd1b..c1b2b74c 100644 --- a/doc/rtd/reference/yaml_examples/user_groups.rst +++ b/doc/rtd/reference/yaml_examples/user_groups.rst @@ -64,7 +64,7 @@ exceptions and can be applied to already-existing users: passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo gecos: Bar B. Foo - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" groups: users, admin ssh_import_id: - lp:falcojr @@ -77,7 +77,6 @@ exceptions and can be applied to already-existing users: inactive: '5' system: true - name: fizzbuzz - sudo: false shell: /bin/bash ssh_authorized_keys: - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSL7uWGj8cgWsp... csmith@fringe @@ -136,4 +135,3 @@ supplemental config options. This config will make the default user .. literalinclude:: ../../../module-docs/cc_users_groups/example7.yaml :language: yaml :linenos: - diff --git a/doc/rtd/spelling_word_list.txt b/doc/rtd/spelling_word_list.txt index 4da0b890..7cbf2973 100644 --- a/doc/rtd/spelling_word_list.txt +++ b/doc/rtd/spelling_word_list.txt @@ -190,6 +190,7 @@ setup-keymap shellscript shortid sigonly +simplestreams sk sle sles diff --git a/doc/rtd/templates/module_property.tmpl b/doc/rtd/templates/module_property.tmpl index 895a8956..50875a96 100644 --- a/doc/rtd/templates/module_property.tmpl +++ b/doc/rtd/templates/module_property.tmpl @@ -1,12 +1,23 @@ -{% macro print_prop(name, types, description, prefix) -%} +{% macro print_prop(name, types, description, deprecated, changed, prefix) -%} {% set descr_suffix = description.splitlines()[0]|d('') -%} {% set descr_lines = description.splitlines()[1:]|d([]) -%} +{% set deprecated_lines = deprecated.splitlines()|d([]) -%} +{% set changed_lines = changed.splitlines()|d([]) -%} {{prefix}}* **{{name}}:** ({{types}}){{ descr_suffix }} {% for line in descr_lines -%} {{prefix ~ ' '}}{{ line }} {% endfor -%} +{% for line in deprecated_lines -%} +{{prefix ~ ' '}}{{ line }} +{% endfor -%} +{% for line in changed_lines -%} +{{prefix ~ ' '}}{{ line }} +{% endfor -%} {%- endmacro -%} {% set ns = namespace(is_obj_type=false) -%} +{% for alt_schema in prop_cfg.get("oneOf", []) -%} + {% if ('properties' in alt_schema or 'patternProperties' in alt_schema) %}{% set ns.is_obj_type = true -%}{% endif -%} +{% endfor -%} {% if ('properties' in prop_cfg or 'patternProperties' in prop_cfg) %}{% set ns.is_obj_type = true -%}{% endif -%} {% for key, val in prop_cfg.get('items', {}).items() -%} {% if key in ('properties', 'patternProperties') -%}{% set ns.is_obj_type = true -%}{% endif -%} @@ -19,4 +30,4 @@ {% if ns.is_obj_type -%} {% set description = description ~ " Each object in **" ~ name ~ "** list supports the following keys:" -%} {% endif -%} -{{ print_prop(name, types, description, prefix ) -}} +{{ print_prop(name, types, description, deprecated, changed, prefix ) -}} diff --git a/doc/rtd/tutorial/qemu-script.sh b/doc/rtd/tutorial/qemu-script.sh index 3430eced..5a64b836 100755 --- a/doc/rtd/tutorial/qemu-script.sh +++ b/doc/rtd/tutorial/qemu-script.sh @@ -1,7 +1,7 @@ #!/bin/bash TEMP_DIR=temp -IMAGE_URL="https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" +IMAGE_URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" # setup mkdir "$TEMP_DIR" && cd "$TEMP_DIR" || { @@ -36,7 +36,7 @@ qemu-system-x86_64 \ -cpu host \ -m 512 \ -nographic \ - -hda jammy-server-cloudimg-amd64.img \ + -hda noble-server-cloudimg-amd64.img \ -smbios type=1,serial=ds='nocloud;s=http://10.0.2.2:8000/' echo -e "\nTo reuse the image and config files, start the python webserver and " diff --git a/doc/rtd/tutorial/qemu.rst b/doc/rtd/tutorial/qemu.rst index 22032dd6..ee8d627b 100644 --- a/doc/rtd/tutorial/qemu.rst +++ b/doc/rtd/tutorial/qemu.rst @@ -60,7 +60,7 @@ server image using :command:`wget`: .. code-block:: bash - $ wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img + $ wget https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img .. note:: This example uses emulated CPU instructions on non-x86 hosts, so it may be @@ -189,7 +189,7 @@ take a few moments to complete. -machine accel=kvm:tcg \ -m 512 \ -nographic \ - -hda jammy-server-cloudimg-amd64.img \ + -hda noble-server-cloudimg-amd64.img \ -smbios type=1,serial=ds='nocloud;s=http://10.0.2.2:8000/' .. note:: @@ -201,7 +201,7 @@ line. Many things may be configured: memory size, graphical output, networking information, hard drives and more. Let us examine the final two lines of our previous command. The first of them, -:command:`-hda jammy-server-cloudimg-amd64.img`, tells QEMU to use the cloud +:command:`-hda noble-server-cloudimg-amd64.img`, tells QEMU to use the cloud image as a virtual hard drive. This will cause the virtual machine to boot Ubuntu, which already has cloud-init installed. diff --git a/integration-requirements.txt b/integration-requirements.txt index c65dac98..0a18907f 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -9,6 +9,10 @@ pycloudlib>=1!10.0.2,<1!11 # resulting in an unmet dependency issue: # https://github.com/pytest-dev/pytest/issues/11104 pytest!=7.3.2 +pytest-timeout + +# Even when xdist is not actively used, we have fixtures that require it +pytest-xdist packaging passlib diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 672cd426..a09e87d2 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -170,7 +170,7 @@ fi %if "%{init_system}" == "systemd" /usr/lib/systemd/system-generators/cloud-init-generator -%{_sysconfdir}/systemd/system/sshd-keygen@.service.d/disable-sshd-keygen-if-cloud-init-active.conf +/usr/lib/systemd/system/sshd-keygen@.service.d/disable-sshd-keygen-if-cloud-init-active.conf %{_unitdir}/cloud-* %else %attr(0755, root, root) %{_initddir}/cloud-config diff --git a/pyproject.toml b/pyproject.toml index 3ae24bfc..14315477 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,10 +42,7 @@ no_implicit_optional = true # See GH-5445 [[tool.mypy.overrides]] module = [ - "cloudinit.analyze", - "cloudinit.analyze.dump", "cloudinit.analyze.show", - "cloudinit.cmd.devel.hotplug_hook", "cloudinit.cmd.devel.make_mime", "cloudinit.cmd.devel.net_convert", "cloudinit.cmd.main", @@ -66,7 +63,6 @@ module = [ "cloudinit.distros.ug_util", "cloudinit.helpers", "cloudinit.net.cmdline", - "cloudinit.net.eni", "cloudinit.net.ephemeral", "cloudinit.net.freebsd", "cloudinit.net.netbsd", @@ -85,7 +81,6 @@ module = [ "cloudinit.sources.DataSourceExoscale", "cloudinit.sources.DataSourceGCE", "cloudinit.sources.DataSourceHetzner", - "cloudinit.sources.DataSourceMAAS", "cloudinit.sources.DataSourceNoCloud", "cloudinit.sources.DataSourceOVF", "cloudinit.sources.DataSourceOpenNebula", @@ -110,11 +105,8 @@ module = [ "cloudinit.user_data", "tests.integration_tests.instances", "tests.unittests.analyze.test_show", - "tests.unittests.config.test_apt_configure_sources_list_v1", "tests.unittests.config.test_apt_configure_sources_list_v3", "tests.unittests.config.test_apt_source_v1", - "tests.unittests.config.test_apt_source_v3", - "tests.unittests.config.test_cc_ansible", "tests.unittests.config.test_cc_apk_configure", "tests.unittests.config.test_cc_apt_pipelining", "tests.unittests.config.test_cc_bootcmd", @@ -126,19 +118,13 @@ module = [ "tests.unittests.config.test_cc_grub_dpkg", "tests.unittests.config.test_cc_install_hotplug", "tests.unittests.config.test_cc_keys_to_console", - "tests.unittests.config.test_cc_landscape", - "tests.unittests.config.test_cc_locale", "tests.unittests.config.test_cc_mcollective", - "tests.unittests.config.test_cc_mounts", "tests.unittests.config.test_cc_phone_home", "tests.unittests.config.test_cc_puppet", "tests.unittests.config.test_cc_resizefs", "tests.unittests.config.test_cc_resolv_conf", "tests.unittests.config.test_cc_rh_subscription", - "tests.unittests.config.test_cc_runcmd", - "tests.unittests.config.test_cc_ssh", "tests.unittests.config.test_cc_ubuntu_autoinstall", - "tests.unittests.config.test_cc_ubuntu_drivers", "tests.unittests.config.test_cc_update_etc_hosts", "tests.unittests.config.test_cc_users_groups", "tests.unittests.config.test_cc_wireguard", @@ -146,7 +132,6 @@ module = [ "tests.unittests.config.test_cc_zypper_add_repo", "tests.unittests.config.test_modules", "tests.unittests.config.test_schema", - "tests.unittests.conftest", "tests.unittests.distros.test_alpine", "tests.unittests.distros.test_hosts", "tests.unittests.distros.test_ifconfig", @@ -175,7 +160,6 @@ module = [ "tests.unittests.sources.test_exoscale", "tests.unittests.sources.test_gce", "tests.unittests.sources.test_init", - "tests.unittests.sources.test_lxd", "tests.unittests.sources.test_nocloud", "tests.unittests.sources.test_opennebula", "tests.unittests.sources.test_openstack", diff --git a/setup.py b/setup.py index 30f735ae..fd9692e5 100644 --- a/setup.py +++ b/setup.py @@ -187,14 +187,7 @@ def render_tmpl(template, mode=None, is_yaml=False): elif os.path.isfile("/etc/system-release-cpe"): with open("/etc/system-release-cpe") as f: cpe_data = f.read().rstrip().split(":") - - if cpe_data[1] == "\o": # noqa: W605 - # URI formatted CPE - inc = 0 - else: - # String formatted CPE - inc = 1 - (cpe_vendor, cpe_product, cpe_version) = cpe_data[2 + inc : 5 + inc] + (cpe_vendor, cpe_product, cpe_version) = cpe_data[3:6] if cpe_vendor == "amazon": USR_LIB_EXEC = "usr/libexec" diff --git a/setup_utils.py b/setup_utils.py index e0476c07..476bbd9a 100644 --- a/setup_utils.py +++ b/setup_utils.py @@ -13,6 +13,11 @@ def is_generator(p: str) -> bool: def pkg_config_read(library: str, var: str) -> str: + pkg_config = "pkg-config" + + if os.getenv("PKG_CONFIG"): + pkg_config = os.getenv("PKG_CONFIG") + fallbacks = { "systemd": { "systemdsystemconfdir": "/etc/systemd/system", @@ -23,7 +28,7 @@ def pkg_config_read(library: str, var: str) -> str: "udevdir": "/usr/lib/udev", }, } - cmd = ["pkg-config", f"--variable={var}", library] + cmd = [pkg_config, f"--variable={var}", library] try: path = subprocess.check_output(cmd).decode("utf-8") # nosec B603 path = path.strip() diff --git a/systemd/cloud-init-local.service.tmpl b/systemd/cloud-init-local.service.tmpl index 62decbb1..862664d1 100644 --- a/systemd/cloud-init-local.service.tmpl +++ b/systemd/cloud-init-local.service.tmpl @@ -2,18 +2,19 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Local Stage (pre-network) -{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "rhel"] %} +{% if variant in ["almalinux", "cloudlinux", "ubuntu", "unknown", "debian", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} Wants=network-pre.target After=hv_kvp_daemon.service After=systemd-remount-fs.service +Before=auditd.service Before=network-pre.target Before=shutdown.target {% if variant in ["almalinux", "cloudlinux", "rhel"] %} Before=firewalld.target {% endif %} -{% if variant in ["ubuntu", "unknown", "debian"] %} +{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} Before=sysinit.target {% endif %} Conflicts=shutdown.target diff --git a/systemd/cloud-init.service.tmpl b/systemd/cloud-init.service.tmpl index 93954431..e61b4ac5 100644 --- a/systemd/cloud-init.service.tmpl +++ b/systemd/cloud-init.service.tmpl @@ -2,7 +2,7 @@ [Unit] # https://docs.cloud-init.io/en/latest/explanation/boot.html Description=Cloud-init: Network Stage -{% if variant not in ["almalinux", "cloudlinux", "photon", "rhel"] %} +{% if variant not in ["almalinux", "cloudlinux", "photon", "raspberry-pi-os", "rhel"] %} DefaultDependencies=no {% endif %} Wants=cloud-init-local.service @@ -10,12 +10,12 @@ Wants=sshd-keygen.service Wants=sshd.service After=cloud-init-local.service After=systemd-networkd-wait-online.service -{% if variant in ["ubuntu", "unknown", "debian"] %} +{% if variant in ["ubuntu", "unknown", "debian", "raspberry-pi-os"] %} After=networking.service {% endif %} {% if variant in ["almalinux", "centos", "cloudlinux", "eurolinux", "fedora", - "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "rhel", "rocky", - "suse", "TencentOS", "virtuozzo"] %} + "miraclelinux", "openeuler", "OpenCloudOS", "openmandriva", "raspberry-pi-os", + "rhel", "rocky", "suse", "TencentOS", "virtuozzo"] %} After=NetworkManager.service After=NetworkManager-wait-online.service @@ -27,6 +27,9 @@ After=wicked.service After=dbus.service {% endif %} Before=network-online.target +{% if variant == "raspberry-pi-os" %} +Before=avahi-daemon.service +{% endif %} Before=sshd-keygen.service Before=sshd.service Before=systemd-user-sessions.service diff --git a/templates/hosts.openeuler.tmpl b/templates/hosts.openeuler.tmpl new file mode 100644 index 00000000..ed41ad3d --- /dev/null +++ b/templates/hosts.openeuler.tmpl @@ -0,0 +1,24 @@ +## template:jinja +{# +This file /etc/cloud/templates/hosts.openeuler.tmpl is only utilized +if enabled in cloud-config. Specifically, in order to enable it +you need to add the following to config: + manage_etc_hosts: True +-#} +# Your system has configured 'manage_etc_hosts' as True. +# As a result, if you wish for changes to this file to persist +# then you will need to either +# a.) make changes to the master file in /etc/cloud/templates/hosts.openeuler.tmpl +# b.) change or remove the value of 'manage_etc_hosts' in +# /etc/cloud/cloud.cfg or cloud-config from user-data +# +# The following lines are desirable for IPv4 capable hosts +127.0.0.1 {{fqdn}} {{hostname}} +127.0.0.1 localhost.localdomain localhost +127.0.0.1 localhost4.localdomain4 localhost4 + +# The following lines are desirable for IPv6 capable hosts +::1 {{fqdn}} {{hostname}} +::1 localhost.localdomain localhost +::1 localhost6.localdomain6 localhost6 + diff --git a/templates/sources.list.debian.deb822.tmpl b/templates/sources.list.debian.deb822.tmpl index bb286e66..08032634 100644 --- a/templates/sources.list.debian.deb822.tmpl +++ b/templates/sources.list.debian.deb822.tmpl @@ -29,6 +29,6 @@ Signed-By: {{primary_key | default('/usr/share/keyrings/debian-archive-keyring.g ## Major bug fix updates produced after the final release of the distribution. Types: deb deb-src URIs: {{security}} -Suites: {{codename}}{% if codename in ('buster', 'stretch') %}/updates{% else %}-security{% endif %} +Suites: {{codename}}{% if codename in ('buster', 'stretch') %}/updates{% else %}-security{% endif +%} Components: main Signed-By: {{security_key | default(primary_key, true) | default('/usr/share/keyrings/debian-archive-keyring.gpg', true)}} diff --git a/tests/data/merge_sources/expected7.yaml b/tests/data/merge_sources/expected7.yaml index 8186d13a..a7aad093 100644 --- a/tests/data/merge_sources/expected7.yaml +++ b/tests/data/merge_sources/expected7.yaml @@ -13,7 +13,7 @@ users: passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo gecos: Bar B. Foo - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" groups: users, admin ssh_import_id: None lock-passwd: true diff --git a/tests/data/merge_sources/source7-1.yaml b/tests/data/merge_sources/source7-1.yaml index ec93079f..347e81ae 100644 --- a/tests/data/merge_sources/source7-1.yaml +++ b/tests/data/merge_sources/source7-1.yaml @@ -13,7 +13,7 @@ users: passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo gecos: Bar B. Foo - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" groups: users, admin ssh_import_id: None lock-passwd: true @@ -24,4 +24,3 @@ users: gecos: Magic Cloud App Daemon User inactive: '5' system: true - diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index dead2c1f..a93a2d2d 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -7,7 +7,7 @@ import string from abc import ABC, abstractmethod from copy import deepcopy -from typing import Optional, Type +from typing import Callable, Optional from uuid import UUID from pycloudlib import ( @@ -21,15 +21,17 @@ Openstack, Qemu, ) -from pycloudlib.cloud import ImageType +from pycloudlib.cloud import BaseCloud, ImageType from pycloudlib.ec2.instance import EC2Instance -from pycloudlib.lxd.cloud import BaseCloud, _BaseLXD -from pycloudlib.lxd.instance import BaseInstance, LXDInstance +from pycloudlib.instance import BaseInstance +from pycloudlib.lxd.cloud import _BaseLXD +from pycloudlib.lxd.instance import LXDInstance import cloudinit from cloudinit.subp import ProcessExecutionError, subp from tests.integration_tests import integration_settings from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.reaper import Reaper from tests.integration_tests.releases import CURRENT_RELEASE from tests.integration_tests.util import emit_dots_on_travis @@ -58,14 +60,17 @@ class IntegrationCloud(ABC): def __init__( self, + reaper: Reaper, image_type: ImageType = ImageType.GENERIC, settings=integration_settings, ): + self.reaper = reaper self._image_type = image_type self.settings = settings self.cloud_instance = self._get_cloud_instance() self.initial_image_id = self._get_initial_image() self.snapshot_id: Optional[str] = None + self.has_failed_test = False @property def image_id(self): @@ -139,7 +144,7 @@ def launch( launch_kwargs = {**default_launch_kwargs, **launch_kwargs} display_launch_kwargs = deepcopy(launch_kwargs) if display_launch_kwargs.get("user_data") is not None: - if "token" in display_launch_kwargs.get("user_data"): + if "token" in display_launch_kwargs["user_data"]: display_launch_kwargs["user_data"] = re.sub( r"token: .*", "token: REDACTED", launch_kwargs["user_data"] ) @@ -173,7 +178,14 @@ def get_instance( return IntegrationInstance(self, cloud_instance, settings) def destroy(self): - if self.settings.KEEP_IMAGE or self.settings.KEEP_INSTANCE: + if ( + self.settings.KEEP_IMAGE + or self.settings.KEEP_INSTANCE is True + or ( + self.settings.KEEP_INSTANCE == "ON_ERROR" + and self.has_failed_test + ) + ): log.info( "NOT cleaning cloud instance because KEEP_IMAGE or " "KEEP_INSTANCE is True" @@ -278,7 +290,6 @@ def _get_cloud_instance(self) -> OCI: class _LxdIntegrationCloud(IntegrationCloud): - pycloudlib_instance_cls: Type[_BaseLXD] instance_tag: str cloud_instance: _BaseLXD @@ -287,7 +298,7 @@ def _get_initial_image(self, **kwargs) -> str: image_type=self._image_type, **kwargs ) - def _get_or_set_profile_list(self, release): + def _get_or_set_profile_list(self, release) -> Optional[list]: return None @staticmethod @@ -324,7 +335,12 @@ def _mount_source(instance: LXDInstance): subp(command.split()) def _perform_launch( - self, *, launch_kwargs, wait=True, **kwargs + self, + *, + launch_kwargs, + wait=True, + lxd_setup: Optional[Callable] = None, + **kwargs, ) -> LXDInstance: instance_kwargs = deepcopy(launch_kwargs) instance_kwargs["inst_type"] = instance_kwargs.pop( @@ -351,9 +367,9 @@ def _perform_launch( ) if self.settings.CLOUD_INIT_SOURCE == "IN_PLACE": self._mount_source(pycloudlib_instance) - if "lxd_setup" in kwargs: + if lxd_setup is not None: log.info("Running callback specified by 'lxd_setup' mark") - kwargs["lxd_setup"](pycloudlib_instance) + lxd_setup(pycloudlib_instance) pycloudlib_instance.start(wait=wait) return pycloudlib_instance @@ -361,22 +377,20 @@ def _perform_launch( class LxdContainerCloud(_LxdIntegrationCloud): datasource = "lxd_container" cloud_instance: LXDContainer - pycloudlib_instance_cls = LXDContainer instance_tag = "lxd-container-integration-test" def _get_cloud_instance(self) -> LXDContainer: - return self.pycloudlib_instance_cls(tag=self.instance_tag) + return LXDContainer(tag=self.instance_tag) class LxdVmCloud(_LxdIntegrationCloud): datasource = "lxd_vm" cloud_instance: LXDVirtualMachine - pycloudlib_instance_cls = LXDVirtualMachine instance_tag = "lxd-vm-integration-test" _profile_list: list = [] def _get_cloud_instance(self) -> LXDVirtualMachine: - return self.pycloudlib_instance_cls(tag=self.instance_tag) + return LXDVirtualMachine(tag=self.instance_tag) def _get_or_set_profile_list(self, release) -> list: if self._profile_list: diff --git a/tests/integration_tests/cmd/test_analyze.py b/tests/integration_tests/cmd/test_analyze.py new file mode 100644 index 00000000..ceee47e4 --- /dev/null +++ b/tests/integration_tests/cmd/test_analyze.py @@ -0,0 +1,36 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""Tests for `cloud-init analyze`""" +import pytest + +from cloudinit.distros import uses_systemd +from tests.integration_tests.integration_settings import PLATFORM +from tests.integration_tests.util import get_datetime_from_string + + +class TestAnalyzeCommand: + @pytest.mark.skipif(not uses_systemd(), reason="Relies on systemd output") + @pytest.mark.skipif( + PLATFORM != "lxd_container", + reason="Testing lxdcontainer-specific behavior", + ) + def test_analyze_boot_ordered_timestamps(self, module_client): + """ + Confirm that analyze boot is working correctly in lxd containers + and that the correct zero-point is used for the monotonic clock used + to determine when cloud-init was activated by systemd + """ + assert module_client.execute("cloud-init status --wait --long").ok + result = module_client.execute("cloud-init analyze boot") + assert result.stderr == "container" + + container_start_time = get_datetime_from_string( + result.stdout, "^\\s*Container started at: (.+?)$" + ) + cloudinit_activation_time = get_datetime_from_string( + result.stdout, "^\\s*Cloud-init activated by systemd at: (.+?)$" + ) + cloudinit_start_time = get_datetime_from_string( + result.stdout, "^\\s*Cloud-init start: (.+?)$" + ) + assert container_start_time < cloudinit_activation_time + assert cloudinit_activation_time < cloudinit_start_time diff --git a/tests/integration_tests/cmd/test_clean.py b/tests/integration_tests/cmd/test_clean.py index a27600d3..c3d760e7 100644 --- a/tests/integration_tests/cmd/test_clean.py +++ b/tests/integration_tests/cmd/test_clean.py @@ -4,7 +4,7 @@ import pytest from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.releases import IS_UBUNTU +from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, PLUCKY USER_DATA = """\ #cloud-config @@ -39,7 +39,10 @@ class TestCleanCommand: def test_clean_rotated_logs(self, client: IntegrationInstance): """Clean with log params alters expected files without error""" assert client.execute("cloud-init status --wait --long").ok - assert client.execute("logrotate /etc/logrotate.d/cloud-init").ok + package = ( + "cloud-init" if CURRENT_RELEASE < PLUCKY else "cloud-init-base" + ) + assert client.execute(f"logrotate /etc/logrotate.d/{package}").ok log_paths = ( "/var/log/cloud-init.log", "/var/log/cloud-init.log.1.gz", diff --git a/tests/integration_tests/cmd/test_schema.py b/tests/integration_tests/cmd/test_schema.py index 27a0ef11..ebcc6fc7 100644 --- a/tests/integration_tests/cmd/test_schema.py +++ b/tests/integration_tests/cmd/test_schema.py @@ -6,9 +6,9 @@ from cloudinit import lifecycle from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.releases import CURRENT_RELEASE, JAMMY from tests.integration_tests.util import ( get_feature_flag_value, + has_netplanlib, verify_clean_boot, verify_clean_log, ) @@ -103,7 +103,7 @@ def test_network_config_schema_validation( "annotate": NET_V1_ANNOTATED, }, } - if CURRENT_RELEASE >= JAMMY: + if has_netplanlib(class_client): # Support for netplan API available content_responses[NET_CFG_V2] = { "out": "Valid schema /root/net.yaml" diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index cba33601..194cda58 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import datetime +import fcntl import functools import logging import os @@ -15,7 +16,6 @@ from pycloudlib.cloud import ImageType from pycloudlib.lxd.instance import LXDInstance -import tests.integration_tests.reaper as reaper from tests.integration_tests import integration_settings from tests.integration_tests.clouds import ( AzureCloud, @@ -34,6 +34,7 @@ CloudInitSource, IntegrationInstance, ) +from tests.integration_tests.reaper import Reaper log = logging.getLogger("integration_testing") log.addHandler(logging.StreamHandler(sys.stdout)) @@ -80,21 +81,61 @@ def disable_subp_usage(request): pass -_SESSION_CLOUD: IntegrationCloud -REAPER: reaper._Reaper +def setup_image_or_die(cloud: IntegrationCloud): + try: + setup_image(cloud) + except Exception as e: + if cloud.snapshot_id: + # if a snapshot id was set, then snapshot succeeded, teardown + cloud.delete_snapshot() + cloud.destroy() + pytest.exit( + f"{type(e).__name__} in session setup: {str(e)}", returncode=2 + ) -@pytest.fixture(scope="session") -def session_cloud() -> Generator[IntegrationCloud, None, None]: - """a shared session is created in pytest_sessionstart() +def setup_image_once( + worker_id: str, + cloud: IntegrationCloud, + tmp_path_factory: pytest.TempPathFactory, +) -> None: + """Setup image to be used for (almost) all tests. - yield this shared session + Since pytest-xdist runs tests in parallel, we need to ensure that + the image setup is only done once, and not multiple times in parallel. """ - global _SESSION_CLOUD - yield _SESSION_CLOUD + if worker_id == "master": + # We're running single-threaded, so no synchronization needed + setup_image_or_die(cloud) + return + + # We're an xdist worker, so we need to synchronize. + # Whoever gets the lock first will do the setup, writing the + # image id into the image path. The other workers will + # wait for file access, read the image id from image path and use it. + image_path = Path( + tmp_path_factory.getbasetemp().parent, + f"session_image_id_{worker_id}", + ) + lock_path = image_path.with_suffix(".lock") + with open(lock_path, "w") as lock_file: + # Use a lock to ensure only one worker does the setup + fcntl.lockf(lock_file, fcntl.LOCK_EX) + try: + if not image_path.exists(): + setup_image_or_die(cloud) + image_path.write_text(cloud.image_id) + else: + cloud.snapshot_id = image_path.read_text().strip() + finally: + fcntl.lockf(lock_file, fcntl.LOCK_UN) -def get_session_cloud() -> IntegrationCloud: + +@pytest.fixture(scope="session") +def session_cloud( + worker_id, reaper: Reaper, tmp_path_factory +) -> Generator[IntegrationCloud, None, None]: """get_session_cloud() creates a session from configuration""" if integration_settings.PLATFORM not in platforms.keys(): raise ValueError( @@ -109,9 +150,25 @@ def get_session_cloud() -> IntegrationCloud: f"{integration_settings.OS_IMAGE_TYPE} is an invalid OS_IMAGE_TYPE" f" specified in settings. Must be one of {image_types}" ) - cloud = platforms[integration_settings.PLATFORM](image_type=image_type) + + cloud: IntegrationCloud = platforms[integration_settings.PLATFORM]( + reaper=reaper, image_type=image_type + ) cloud.emit_settings_to_log() - return cloud + + setup_image_once(worker_id, cloud, tmp_path_factory) + + yield cloud + log.info("Tearing down session cloud") + try: + cloud.delete_snapshot() + except Exception as e: + log.warning( + "Could not delete snapshot. Leaked snapshot id %s: %s", + cloud.snapshot_id, + e, + ) + cloud.destroy() def get_validated_source( @@ -285,15 +342,7 @@ def _collect_artifacts( _collect_profile(instance, log_dir) -@contextmanager -def _client( - request, fixture_utils, session_cloud: IntegrationCloud -) -> Iterator[IntegrationInstance]: - """Fixture implementation for the client fixtures. - - Launch the dynamic IntegrationClient instance using any provided - userdata, yield to the test, then cleanup - """ +def get_session_args(request, fixture_utils, session_cloud: IntegrationCloud): getter = functools.partial( fixture_utils.closest_marker_first_arg_or, request, default=None ) @@ -315,16 +364,29 @@ def _client( if not isinstance(session_cloud, _LxdIntegrationCloud): pytest.skip("lxd_use_exec requires LXD") launch_kwargs["execute_via_ssh"] = False - local_launch_kwargs = {} if lxd_setup is not None: if not isinstance(session_cloud, _LxdIntegrationCloud): pytest.skip("lxd_setup requires LXD") - local_launch_kwargs["lxd_setup"] = lxd_setup + return user_data, launch_kwargs, lxd_setup, lxd_use_exec + + +@contextmanager +def _client( + request, fixture_utils, session_cloud: IntegrationCloud +) -> Iterator[IntegrationInstance]: + """Fixture implementation for the client fixtures. + + Launch the dynamic IntegrationClient instance using any provided + userdata, yield to the test, then cleanup + """ + user_data, launch_kwargs, lxd_setup, lxd_use_exec = get_session_args( + request, fixture_utils, session_cloud + ) with session_cloud.launch( user_data=user_data, launch_kwargs=launch_kwargs, - **local_launch_kwargs, + lxd_setup=lxd_setup, ) as instance: if lxd_use_exec is not None and isinstance( instance.instance, LXDInstance @@ -337,6 +399,10 @@ def _client( yield instance test_failed = request.session.testsfailed - previous_failures > 0 _collect_artifacts(instance, request.node.nodeid, test_failed) + instance.test_failed = test_failed + if test_failed: + session_cloud.has_failed_test = True + # conflicting requirements: # - pytest thinks that it can cleanup loggers after tests run # - pycloudlib thinks that at garbage collection is a good place to @@ -475,61 +541,29 @@ def _generate_profile_report() -> None: log.info(command, "final.stats") -# https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_sessionstart -def pytest_sessionstart(session) -> None: - """do session setup""" - global _SESSION_CLOUD - global REAPER - log.info("starting session") +@pytest.fixture(scope="session") +def reaper(): + """Fixture to provide a reaper instance for cleaning up instances.""" + reaper_instance = Reaper() + reaper_instance.start() + + yield reaper_instance + try: - _SESSION_CLOUD = get_session_cloud() - setup_image(_SESSION_CLOUD) - REAPER = reaper._Reaper() - REAPER.start() + reaper_instance.stop() except Exception as e: - if _SESSION_CLOUD: - # if a _SESSION_CLOUD was allocated, clean it up - if _SESSION_CLOUD.snapshot_id: - # if a snapshot id was set, then snapshot succeeded, teardown - _SESSION_CLOUD.delete_snapshot() - _SESSION_CLOUD.destroy() - pytest.exit( - f"{type(e).__name__} in session setup: {str(e)}", returncode=2 + log.warning( + "Could not tear down instance reaper thread: %s(%s)", + type(e).__name__, + e, ) - log.info("started session") def pytest_sessionfinish(session, exitstatus) -> None: - """do session teardown""" - global REAPER - log.info("finishing session") try: if integration_settings.INCLUDE_COVERAGE: _generate_coverage_report() elif integration_settings.INCLUDE_PROFILE: _generate_profile_report() except Exception as e: - log.warning("Could not generate report during teardown: %s", e) - try: - _SESSION_CLOUD.delete_snapshot() - except Exception as e: - log.warning( - "Could not delete snapshot. Leaked snapshot id %s: %s", - _SESSION_CLOUD.snapshot_id, - e, - ) - try: - REAPER.stop() - except Exception as e: - log.warning( - "Could not tear down instance reaper thread: %s(%s)", - type(e).__name__, - e, - ) - try: - _SESSION_CLOUD.destroy() - except Exception as e: - log.warning( - "Could not destroy session cloud: %s(%s)", type(e).__name__, e - ) - log.info("finish session") + log.warning("Could not generate report during session finish: %s", e) diff --git a/tests/integration_tests/datasources/test_caching.py b/tests/integration_tests/datasources/test_caching.py index 467585fa..16e2f003 100644 --- a/tests/integration_tests/datasources/test_caching.py +++ b/tests/integration_tests/datasources/test_caching.py @@ -16,6 +16,12 @@ def setup_custom_datasource(client: IntegrationInstance, datasource_name: str): "/usr/lib/python3/dist-packages/cisources/" f"DataSource{datasource_name}.py", ) + # Since our custom datasource isn't handling networking, disable + # cloud-init networking to avoid wait-online timeouts and errors + client.write_to_file( + "/etc/cloud/cloud.cfg.d/99-disable-networking.cfg", + "network: {config: disabled}", + ) def verify_no_cache_boot(client: IntegrationInstance): diff --git a/tests/integration_tests/datasources/test_lxd_discovery.py b/tests/integration_tests/datasources/test_lxd_discovery.py index 43a4ca78..c468eb43 100644 --- a/tests/integration_tests/datasources/test_lxd_discovery.py +++ b/tests/integration_tests/datasources/test_lxd_discovery.py @@ -119,7 +119,7 @@ def test_lxd_datasource_discovery(client: IntegrationInstance): assert {"public-keys": v1["public_ssh_keys"][0]} == ( yaml.safe_load(ds_cfg["config"]["user.meta-data"]) ) - assert "#cloud-config\ninstance-id" in ds_cfg["meta-data"] + assert "instance-id" in ds_cfg["meta-data"] # Some series no longer provide nocloud-net seed files (LP: #1958460) if lxd_has_nocloud(client): diff --git a/tests/integration_tests/datasources/test_nocloud.py b/tests/integration_tests/datasources/test_nocloud.py index cf3e1db4..494e3921 100644 --- a/tests/integration_tests/datasources/test_nocloud.py +++ b/tests/integration_tests/datasources/test_nocloud.py @@ -24,6 +24,18 @@ - touch /var/tmp/seeded_vendordata_test_file """ +# The fallback network config doesn't work on LXD, leading to +# systemd-network-wait-online timing out and/or cloud-init raising +# an error about wait-online. +# This gives us a NoCloud default to work around these issues +NETWORK_CONFIG = """\ +network: + version: 2 + ethernets: + eth0: + dhcp4: true +""" + LXD_METADATA_NOCLOUD_SEED = """\ /var/lib/cloud/seed/nocloud-net/meta-data: @@ -46,6 +58,12 @@ default: | #cloud-config {} + /var/lib/cloud/seed/nocloud-net/network-config: + when: + - create + - copy + create_only: false + template: netcfg.tpl """ @@ -64,6 +82,13 @@ def setup_nocloud(instance: LXDInstance): ["lxc", "config", "template", "edit", instance.name, "emptycfg.tpl"], data="#cloud-config\n{}\n", ) + subp( + ["lxc", "config", "template", "create", instance.name, "netcfg.tpl"], + ) + subp( + ["lxc", "config", "template", "edit", instance.name, "netcfg.tpl"], + data=NETWORK_CONFIG, + ) subp( ["lxc", "config", "metadata", "edit", instance.name], data=f"{lxd_image_metadata.stdout}{LXD_METADATA_NOCLOUD_SEED}", @@ -87,8 +112,11 @@ def test_nocloud_seedfrom_vendordata(client: IntegrationInstance): "mkdir {seed_dir} && " "touch {seed_dir}/user-data && " "touch {seed_dir}/meta-data && " + "echo '{net}' > {seed_dir}/network-config && " "echo 'seedfrom: {seed_dir}/' > " - "/var/lib/cloud/seed/nocloud-net/meta-data".format(seed_dir=seed_dir) + "/var/lib/cloud/seed/nocloud-net/meta-data".format( + seed_dir=seed_dir, net=NETWORK_CONFIG + ) ) assert result.return_code == 0 diff --git a/tests/integration_tests/datasources/test_none.py b/tests/integration_tests/datasources/test_none.py index a9273957..93bf27c5 100644 --- a/tests/integration_tests/datasources/test_none.py +++ b/tests/integration_tests/datasources/test_none.py @@ -52,7 +52,17 @@ def test_datasource_none_discovery(client: IntegrationInstance): ignore_warnings = [ "Falling back to a hard restart of systemd-networkd.service", ] - log = client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log) - verify_clean_boot(client, ignore_warnings=ignore_warnings) + if client.settings.PLATFORM != "ibm": + verify_schema = True + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + else: + verify_schema = False + ignore_warnings.append( + "Unable to disable SSH logins for vpcuser given ssh_redirect_user:" + " ubuntu. No cloud public-keys present." + ) + verify_clean_boot( + client, ignore_warnings=ignore_warnings, verify_schema=verify_schema + ) assert client.execute("test -f /var/tmp/success-with-datasource-none").ok diff --git a/tests/integration_tests/datasources/test_oci_networking.py b/tests/integration_tests/datasources/test_oci_networking.py index 9b807927..f50d6b96 100644 --- a/tests/integration_tests/datasources/test_oci_networking.py +++ b/tests/integration_tests/datasources/test_oci_networking.py @@ -157,3 +157,50 @@ def test_oci_networking_system_cfg(client: IntegrationInstance, tmpdir): netplan_cfg = yaml.safe_load(netplan_yaml) expected_netplan_cfg = yaml.safe_load(SYSTEM_CFG) assert expected_netplan_cfg == netplan_cfg + + +@pytest.mark.skipif(PLATFORM != "oci", reason="Test is OCI specific") +def test_oci_keep_configuration_networking_config( + session_cloud: IntegrationCloud, +): + """ + Test to ensure the keep_configuration is applied on Oracle ISCSI instances. + + This test launches a Baremetal OCI instance so that ISCSI is used, and + checks that the primary systemd network configuration file contains the + 'KeepConfiguration=true' directive, which indicates that the network + configuration is preserved as expected. + + Assertions: + - At least one netplan file exists under '/run/systemd/network'. + - The primary systemd network configuration file includes the + 'KeepConfiguration=true' directive. + - The netplan configuration includes the 'critical: true' directive. + """ + with session_cloud.launch( + launch_kwargs={ + "instance_type": "BM.Optimized3.36", + }, + ) as client: + r = client.execute("ls /run/systemd/network/10-netplan-*.network") + assert r.ok, ( + "No netplan files found under /run/systemd/network. We are looking" + " for netplan files here to check that the underlying " + "'KeepConfiguration=true' directive is actually being applied to " + "the systemd network configuration." + ) + primary_systemd_file: str = r.stdout.strip().splitlines()[0] + systemd_config = client.read_from_file(primary_systemd_file) + assert ( + "KeepConfiguration=true" in systemd_config + or "CriticalConnection=true" in systemd_config + ), ( + f"Neither 'KeepConfiguration=true' nor 'CriticalConnection=true' " + f"found in '{primary_systemd_file}':\n{primary_systemd_file}" + ) + netplan_config = client.read_from_file( + "/etc/netplan/50-cloud-init.yaml", + ) + assert ( + "critical: true" in netplan_config + ), "critical: true not found in netplan config" diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index d745219e..8268a775 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -2,6 +2,7 @@ import logging import os import re +import time import uuid from enum import Enum from pathlib import Path @@ -14,7 +15,7 @@ from pycloudlib.result import Result from tests.helpers import cloud_init_project_dir -from tests.integration_tests import conftest, integration_settings +from tests.integration_tests import integration_settings from tests.integration_tests.decorators import retry from tests.integration_tests.util import ASSETS_DIR @@ -67,6 +68,7 @@ def __init__( self.cloud = cloud self.instance = instance self.settings = settings + self.test_failed = False self._ip = "" def destroy(self): @@ -161,7 +163,7 @@ def install_coverage(self): # Update and install coverage from pip # We use pip because the versions between distros are incompatible - self._apt_update() + self.update_package_cache() self.execute("apt-get install -qy python3-pip") self.execute(f"pip3 install coverage=={coverage_version}") self.push_file( @@ -182,7 +184,11 @@ def install_new_cloud_init( source: CloudInitSource, clean=True, pkg: str = integration_settings.CLOUD_INIT_PKG, + update=True, ): + if update: + log.info("Updating package cache") + self.update_package_cache() if source == CloudInitSource.DEB_PACKAGE: self.install_deb() elif source == CloudInitSource.PPA: @@ -209,7 +215,6 @@ def install_proposed_image(self, pkg: str): '$(lsb_release -sc)-proposed main" >> ' "/etc/apt/sources.list.d/proposed.list" ).ok - self._apt_update() assert self.execute( f"apt-get install -qy {pkg} -t=$(lsb_release -sc)-proposed" ).ok @@ -218,7 +223,6 @@ def install_ppa(self, pkg: str): log.info("Installing %s from PPA", pkg) if self.execute("which add-apt-repository").failed: log.info("Installing missing software-properties-common package") - self._apt_update() assert self.execute( "apt install -qy software-properties-common" ).ok @@ -244,15 +248,33 @@ def install_ppa(self, pkg: str): "/etc/apt/preferences.d/cloud-init-integration-testing", preferences, ) - assert self.execute( - "add-apt-repository {} -y".format(self.settings.CLOUD_INIT_SOURCE) - ).ok - # PIN this PPA as priority for cloud-init installs regardless of ver - r = self.execute( - f"DEBIAN_FRONTEND=noninteractive" - f" apt-get install -qy {pkg} --allow-downgrades" - ) - assert r.ok, r.stderr + # wait up to 5 minutes for lock to be released + for _ in range(60): + r = self.execute( + "add-apt-repository {} -y".format( + self.settings.CLOUD_INIT_SOURCE + ) + ) + if not r.ok and "Could not get lock" in r.stderr: + log.info("Waiting for lock to be released") + time.sleep(5) + continue + assert r.ok, r.stderr + # PIN this PPA as priority for cloud-init installs + r = self.execute( + f"DEBIAN_FRONTEND=noninteractive" + f" apt-get install -qy {pkg} --allow-downgrades" + ) + if not r.ok and "Could not get lock" in r.stderr: + log.info("Waiting for lock to be released") + time.sleep(5) + continue + assert r.ok, r.stderr + break + else: + raise RuntimeError( + "Failed to install cloud-init from PPA after 5 minutes", + ) @retry(tries=30, delay=1) def install_deb(self): @@ -264,9 +286,6 @@ def install_deb(self): local_path=integration_settings.CLOUD_INIT_SOURCE, remote_path=remote_path, ) - # Update APT cache so all package data is recent to avoid inability - # to install missing dependency errors due to stale cache. - self.execute("apt update") # Use apt install instead of dpkg -i to pull in any changed pkg deps apt_result = self.execute( f"apt install -qy {remote_path} --allow-downgrades" @@ -278,13 +297,11 @@ def install_deb(self): ) @retry(tries=30, delay=1) - def upgrade_cloud_init(self, pkg: str): - log.info("Upgrading %s to latest version in archive", pkg) - self._apt_update() + def upgrade_cloud_init(self, pkg: str, update=True): assert self.execute(f"apt-get install -qy {pkg}").ok - def _apt_update(self): - """Run an apt update. + def update_package_cache(self): + """Update the package cache using apt. `cloud-init single` allows us to ensure apt update is only run once for this instance. It could be done with an lru_cache too, but @@ -329,7 +346,9 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): - if not self.settings.KEEP_INSTANCE: - conftest.REAPER.reap(self) - else: + if self.settings.KEEP_INSTANCE is True or ( + self.settings.KEEP_INSTANCE == "ON_ERROR" and self.test_failed + ): log.info("Keeping Instance, public ip: %s", self.ip()) + else: + self.cloud.reaper.reap(self) diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index 300b4690..cb4bae3a 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -8,7 +8,9 @@ # LAUNCH SETTINGS ################################################################## -# Keep instance (mostly for debugging) when test is finished +# Keep instance (mostly for debugging) when test is finished. +# Can be True, False, or "ON_ERROR" to keep the instance only if the test +# fails. KEEP_INSTANCE = False # Keep snapshot image (mostly for debugging) when test is finished KEEP_IMAGE = False diff --git a/tests/integration_tests/modules/test_ansible.py b/tests/integration_tests/modules/test_ansible.py index bf74cba2..3f7fabb2 100644 --- a/tests/integration_tests/modules/test_ansible.py +++ b/tests/integration_tests/modules/test_ansible.py @@ -1,5 +1,8 @@ +import re + import pytest +from cloudinit.lifecycle import Version from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL @@ -7,6 +10,8 @@ push_and_enable_systemd_unit, verify_clean_boot, verify_clean_log, + verify_ordered_items_in_text, + wait_for_cloud_init, ) # This works by setting up a local repository and web server @@ -37,6 +42,18 @@ - python3-pip roles: - apt + - path: /root/playbooks/watermark.yml + content: | + --- + - hosts: 127.0.0.1 + connection: local + become: true + tasks: + - name: create a watermark file to assert multiple playbooks run + copy: + content: | + cloud-init was here + dest: /MULTIPLAYBOOK_RUN - path: /root/playbooks/roles/apt/tasks/main.yml content: | --- @@ -94,20 +111,21 @@ package_name: {package} galaxy: actions: - - ["ansible-galaxy", "collection", "install", "community.grafana"] + - ["ansible-galaxy", "collection", "install", "community.elastic"] pull: - url: "http://0.0.0.0:8000/" - playbook_name: ubuntu.yml - full: true + - url: "http://0.0.0.0:8000/" + playbook_names: [ubuntu.yml, watermark.yml] + full: true """ -SETUP_REPO = f"cd {REPO_D} &&\ -git config --global user.name auto &&\ -git config --global user.email autom@tic.io &&\ -git config --global init.defaultBranch main &&\ -git init {REPO_D} &&\ -git add {REPO_D}/roles/apt/tasks/main.yml {REPO_D}/ubuntu.yml &&\ -git commit -m auto &&\ +SETUP_REPO = f"cd {REPO_D} && \ +git config --global user.name auto && \ +git config --global user.email autom@tic.io && \ +git config --global init.defaultBranch main && \ +git init {REPO_D} && \ +git add {REPO_D}/roles/apt/tasks/main.yml {REPO_D}/ubuntu.yml && \ +git add {REPO_D}/watermark.yml && \ +git commit -m auto && \ (cd {REPO_D}/.git; git update-server-info)" ANSIBLE_CONTROL = """\ @@ -132,7 +150,7 @@ gecos: Ansible User shell: /bin/bash groups: users,admin,wheel,lxd - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" # Initialize lxd using cloud-init. # -------------------------------- @@ -266,19 +284,31 @@ def _test_ansible_pull_from_local_server(my_client): assert not setup.return_code my_client.execute("cloud-init clean --logs") my_client.restart() + wait_for_cloud_init(my_client) log = my_client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) verify_clean_boot(my_client) output_log = my_client.read_from_file("/var/log/cloud-init-output.log") - assert "ok=3" in output_log + result = my_client.execute("ansible-pull --version").splitlines()[0] + ansible_match = re.search(r"[\d\/.]+", result) + assert ansible_match, f"Unable to parse ansible-pull version {result}" + ansible_version = Version.from_str(ansible_match.group(0)) + if ansible_version >= Version(2, 12, 0): + verify_ansible_ok_logs = ["ok=5"] + else: + # Older ansible-pull versions have to run playbooks separately + verify_ansible_ok_logs = ["ok=3", "ok=2"] + verify_ordered_items_in_text(verify_ansible_ok_logs, output_log) assert "SUCCESS: config-ansible ran successfully" in log # binary location is dependent on install-type, check the filepath # to ensure that the installed collection directory exists output = my_client.execute( - "ls /root/.ansible/collections/ansible_collections/community/grafana" + "ls /root/.ansible/collections/ansible_collections/community/elastic" ) assert not output.stderr.strip() and output.ok + playbook2_content = my_client.read_from_file("/MULTIPLAYBOOK_RUN") + assert "cloud-init was here" == playbook2_content # temporarily disable this test on jenkins until firewall rules are in place @@ -304,7 +334,6 @@ def test_ansible_pull_pip(client: IntegrationInstance): @pytest.mark.user_data( USER_DATA + INSTALL_METHOD.format(package="ansible", method="distro") ) -@pytest.mark.skip("This test is currently broken and needs to be fixed") def test_ansible_pull_distro(client): push_and_enable_systemd_unit(client, "repo_server.service", REPO_SERVER) push_and_enable_systemd_unit(client, "repo_waiter.service", REPO_WAITER) diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index 61336ad5..dd068d04 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -18,7 +18,6 @@ from tests.integration_tests.util import ( get_feature_flag_value, verify_clean_boot, - verify_clean_log, ) logger = logging.getLogger(__name__) @@ -309,6 +308,8 @@ def test_sources_append(self, class_client: IntegrationInstance): - default """ DEFAULT_DATA = _DEFAULT_DATA.format(uri="") +# IBM sets sources_list in vendordata. Unset it +IBM_DEFAULT_DATA = DEFAULT_DATA + "\n sources_list: ''\n" @pytest.mark.skipif(not IS_UBUNTU, reason="Apt usage") @@ -330,13 +331,17 @@ def test_primary_on_openstack(self, class_client: IntegrationInstance): sources_list = class_client.read_from_file(src_file) assert "{}.clouds.archive.ubuntu.com".format(zone) in sources_list + @pytest.mark.skipif( + PLATFORM == "ibm", + reason="IBM has apt.sources_list in vendor-data overriding defaults", + ) def test_security(self, class_client: IntegrationInstance): """Test apt default security sources.""" series = CURRENT_RELEASE.series feature_deb822 = is_true( get_feature_flag_value(class_client, "APT_DEB822_SOURCE_LIST_FILE") ) - if class_client.settings.PLATFORM == "azure": + if PLATFORM == "azure": sec_url = "http://azure.archive.ubuntu.com/ubuntu/" else: sec_url = "http://security.ubuntu.com/ubuntu" @@ -376,21 +381,58 @@ def test_disabled_apt_sources(self, class_client: IntegrationInstance): ) +@pytest.mark.skipif(not IS_UBUNTU, reason="Apt usage") +@pytest.mark.skipif( + PLATFORM != "ibm", + reason="Overrides IBM-specific apt.sources_list in vendor-data", +) +@pytest.mark.user_data(IBM_DEFAULT_DATA) +def test_security_ibm(client: IntegrationInstance): + """Test IBM APT default security sources.""" + series = CURRENT_RELEASE.series + feature_deb822 = is_true( + get_feature_flag_value(client, "APT_DEB822_SOURCE_LIST_FILE") + ) + sec_url = "http://mirrors.adn.networklayer.com/ubuntu" + if feature_deb822: + expected_cfg = dedent( + f"""\ + Types: deb + URIs: {sec_url} + Suites: {series}-security + """ + ) + sources_list = client.read_from_file(DEB822_SOURCES_FILE) + assert expected_cfg in sources_list + else: + sources_list = client.read_from_file(ORIG_SOURCES_FILE) + # 3 lines from main, universe, and multiverse + sec_deb_line = f"deb {sec_url} {series}-security" + sec_src_deb_line = sec_deb_line.replace("deb ", "# deb-src ") + assert 3 == sources_list.count(sec_deb_line) + assert 3 == sources_list.count(sec_src_deb_line) + + DEFAULT_DATA_WITH_URI = _DEFAULT_DATA.format( uri='uri: "http://something.random.invalid/ubuntu"' ) -@pytest.mark.user_data(DEFAULT_DATA_WITH_URI) -def test_default_primary_with_uri(client: IntegrationInstance): +@pytest.mark.skipif(not IS_UBUNTU, reason="Apt usage") +def test_default_primary_with_uri(session_cloud: IntegrationCloud): """Test apt default primary sources.""" - feature_deb822 = is_true( - get_feature_flag_value(client, "APT_DEB822_SOURCE_LIST_FILE") - ) - src_file = DEB822_SOURCES_FILE if feature_deb822 else ORIG_SOURCES_FILE - sources_list = client.read_from_file(src_file) - assert "archive.ubuntu.com" not in sources_list - assert "something.random.invalid" in sources_list + userdata = DEFAULT_DATA_WITH_URI + if PLATFORM == "ibm": + # IBM provides apt.sources_list in vendordata. Unset it + userdata += "\n sources_list: ''\n" + with session_cloud.launch(user_data=userdata) as client: + feature_deb822 = is_true( + get_feature_flag_value(client, "APT_DEB822_SOURCE_LIST_FILE") + ) + src_file = DEB822_SOURCES_FILE if feature_deb822 else ORIG_SOURCES_FILE + sources_list = client.read_from_file(src_file) + assert "archive.ubuntu.com" not in sources_list + assert "something.random.invalid" in sources_list DISABLED_DATA = """\ @@ -480,6 +522,7 @@ def test_apt_proxy(client: IntegrationInstance): #cloud-config runcmd: - DEBIAN_FRONTEND=noninteractive apt-get remove gpg -y + - DEBIAN_FRONTEND=noninteractive apt-get remove software-properties-common -y """ @@ -505,21 +548,21 @@ def test_install_missing_deps(session_cloud: IntegrationCloud): """ Test the installation of missing dependencies using apt on an Ubuntu system. This test is divided into two stages: - Stage 1 (Remove 'gpg' package): - - Launch an instance with user-data that removes the 'gpg' package. + Stage 1 (Remove existing packages): + - Launch an instance with user-data that removes the 'gpg' and + 'software-properties-common' packages. - If on Oracle Cloud, add a command to the user-data to disable the oracle-cloud-agent snap to prevent it from interfering with apt. - Verify that the cloud-init log is clean and the boot process is clean. - - Verify that 'gpg' is actually uninstalled using dpkg. + - Verify the packages are actually uninstalled using dpkg. - Create a snapshot of the instance after 'gpg' has been removed. - If KEEP_INSTANCE is False, destroy the instance after snapshotting. - Stage 2 (re-install 'gpg' package with user-data): + Stage 2 (re-install packages with user-data): - Launch a new instance from the snapshot created in Stage 1 with user-data that installs any missing recommended dependencies. - Verify that the cloud-init log is clean and the boot process is clean. - - Check the cloud-init log to ensure that 'gpg' and its dependencies are - installed successfully. - - Double check that 'gpg' is actually installed using dpkg. + - Check the cloud-init log to ensure that 'gpg' and + 'software-properties-common' are installed successfully. """ # Two stage install: First stage: remove gpg noninteractively from image instance1 = session_cloud.launch( @@ -527,11 +570,14 @@ def test_install_missing_deps(session_cloud: IntegrationCloud): ) # look for r"un gpg" using regex ('un' means uninstalled) - dpkg_output = instance1.execute("dpkg -l gpg") - assert re.search(r"un\s+gpg", dpkg_output.stdout), ( - "gpg package is still installed. it should have been removed by " - "the user-data." - ) + for package in ["gpg", "software-properties-common"]: + dpkg_output = instance1.execute(f"dpkg -l {package}") + assert re.search( + r"[ur][nc]\s+{}".format(package), dpkg_output.stdout + ), ( + f"{package} package is still installed. it should have been " + "removed by the user-data." + ) snapshot_id = instance1.snapshot() if not KEEP_INSTANCE: @@ -548,10 +594,30 @@ def test_install_missing_deps(session_cloud: IntegrationCloud): launch_kwargs={"image_id": snapshot_id}, ) as minimal_client: log = minimal_client.read_from_file("/var/log/cloud-init.log") - verify_clean_log(log) - verify_clean_boot(minimal_client) assert re.search(RE_GPG_SW_PROPERTIES_INSTALLED, log) gpg_installed = re.search( r"ii\s+gpg", minimal_client.execute("dpkg -l gpg").stdout ) + software_properties_common_installed = re.search( + r"ii\s+software-properties-common", + minimal_client.execute( + "dpkg -l software-properties-common" + ).stdout, + ) assert gpg_installed is not None, "gpg package is not installed." + assert ( + software_properties_common_installed is not None + ), "software-properties-common package is not installed." + + # It's a little weird that we're ignoring apt errors when testing apt, + # but we've already verified that we install the missing dependencies. + # To ensure `software-properties-common` is installed, we need to + # specify a ppa in our user data, and `apt update` can fail if no ppa + # has been uploaded for the release being tested. This isn't uncommon + # for the devel release and newer releases in general. + # Ignoring apt update errors seems preferrable to playing whack-a-mole + # with ppas that may or may not be available. + verify_clean_boot( + minimal_client, + ignore_errors=["Failed to update package using apt"], + ) diff --git a/tests/integration_tests/modules/test_cli.py b/tests/integration_tests/modules/test_cli.py index 5106907a..65448dda 100644 --- a/tests/integration_tests/modules/test_cli.py +++ b/tests/integration_tests/modules/test_cli.py @@ -36,7 +36,7 @@ - name: newsuper gecos: Big Stuff groups: users, admin - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" hashed-password: asdfasdf shell: /bin/bash lock_passwd: true @@ -51,7 +51,11 @@ def test_schema_status(self, class_client: IntegrationInstance): PR #575 """ result = class_client.execute("cloud-init schema --system") - assert result.ok + if PLATFORM == "ibm": + assert "Invalid schema: vendor-data" in result.stderr + assert not result.ok + else: + assert result.ok assert "Valid schema user-data" in result.stdout.strip() result = class_client.execute("cloud-init status --long") assert 0 == result.return_code, ( diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index c3458a5a..9d216a2a 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -26,10 +26,11 @@ OS_IMAGE_TYPE, PLATFORM, ) -from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, JAMMY +from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, NOBLE from tests.integration_tests.util import ( get_feature_flag_value, get_inactive_modules, + has_netplanlib, lxd_has_nocloud, network_wait_logged, verify_clean_boot, @@ -99,13 +100,13 @@ def test_netplan_permissions(self, class_client: IntegrationInstance): Test that netplan config file is generated with proper permissions """ log = class_client.read_from_file("/var/log/cloud-init.log") - if CURRENT_RELEASE < JAMMY: + if has_netplanlib(class_client): + assert "Rendered netplan config using netplan python API" in log + else: assert ( "No netplan python module. Fallback to write" " /etc/netplan/50-cloud-init.yaml" in log ) - else: - assert "Rendered netplan config using netplan python API" in log file_perms = class_client.execute( "stat -c %a /etc/netplan/50-cloud-init.yaml" ) @@ -263,9 +264,13 @@ def test_no_problems(self, class_client: IntegrationInstance): # Some minimal images may not have an installed rsyslog package # Test user-data doesn't provide install_rsyslog: true so expect # warnings when not installed. + if CURRENT_RELEASE < NOBLE: + operation_name = "reload-or-try-restart" + else: + operation_name = "try-reload-or-restart" require_warnings.append( - "Failed to reload-or-try-restart rsyslog.service:" - " Unit rsyslog.service not found." + f"Failed to {operation_name} rsyslog.service: Unit" + " rsyslog.service not found." ) # Set ignore_deprecations=True as test_deprecated_message covers this verify_clean_boot( @@ -309,6 +314,7 @@ def test_correct_datasource_detected( "azure": "DataSourceAzure [seed=/dev/sr0]", "ec2": "DataSourceEc2Local", "gce": "DataSourceGCELocal", + "ibm": "DataSourceNoCloud [seed=/dev/vdb]", "oci": "DataSourceOracle", "openstack": "DataSourceOpenStackLocal [net,ver=2]", "qemu": "DataSourceNoCloud [seed=/dev/vda][dsmode=net]", diff --git a/tests/integration_tests/modules/test_disk_setup.py b/tests/integration_tests/modules/test_disk_setup.py index 71a19eba..ce7b4cc8 100644 --- a/tests/integration_tests/modules/test_disk_setup.py +++ b/tests/integration_tests/modules/test_disk_setup.py @@ -29,7 +29,7 @@ def setup_and_mount_lxd_disk(instance: LXDInstance): @pytest.fixture def create_disk(): - subp("dd if=/dev/zero of={} bs=64k count=40".format(DISK_PATH).split()) + subp("dd if=/dev/zero of={} bs=64k count=100".format(DISK_PATH).split()) yield os.remove(DISK_PATH) diff --git a/tests/integration_tests/modules/test_hotplug.py b/tests/integration_tests/modules/test_hotplug.py index 4c531491..2fa3aa2f 100644 --- a/tests/integration_tests/modules/test_hotplug.py +++ b/tests/integration_tests/modules/test_hotplug.py @@ -102,7 +102,7 @@ def _get_ip_addr(client, *, _retries: int = 0): reason="Openstack network metadata support was added in focal.", ) @pytest.mark.user_data(USER_DATA) -def test_hotplug_add_remove(client: IntegrationInstance): +def test_hotplug_add(client: IntegrationInstance): ips_before = _get_ip_addr(client) log = client.read_from_file("/var/log/cloud-init.log") assert "Exiting hotplug handler" not in log @@ -125,17 +125,6 @@ def test_hotplug_add_remove(client: IntegrationInstance): config = yaml.safe_load(netplan_cfg) assert new_addition.interface in config["network"]["ethernets"] - # Remove new NIC - client.instance.remove_network_interface(added_ip) - _wait_till_hotplug_complete(client, expected_runs=2) - ips_after_remove = _get_ip_addr(client) - assert len(ips_after_remove) == len(ips_before) - assert added_ip not in [ip.ip4 for ip in ips_after_remove] - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - assert new_addition.interface not in config["network"]["ethernets"] - assert "enabled" == client.execute( "cloud-init devel hotplug-hook -s net query" ) @@ -202,17 +191,6 @@ def test_hotplug_enable_cmd_ec2(client: IntegrationInstance): config = yaml.safe_load(netplan_cfg) assert new_addition.interface in config["network"]["ethernets"] - # Remove new NIC - client.instance.remove_network_interface(added_ip) - _wait_till_hotplug_complete(client, expected_runs=5) - ips_after_remove = _get_ip_addr(client) - assert len(ips_after_remove) == len(ips_before) - assert added_ip not in [ip.ip4 for ip in ips_after_remove] - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - assert new_addition.interface not in config["network"]["ethernets"] - @pytest.mark.skipif( PLATFORM != "openstack", @@ -287,23 +265,6 @@ def test_multi_nic_hotplug(client: IntegrationInstance): for pub_ip in public_ips: subp("nc -w 10 -zv " + pub_ip + " 22", shell=True) - # Remove new NIC - client.instance.remove_network_interface(secondary_priv_ip) - _wait_till_hotplug_complete(client, expected_runs=2) - - public_ips = client.instance.public_ips - assert len(public_ips) == 1 - # SSH over primary NIC works - subp("nc -w 10 -zv " + public_ips[0] + " 22", shell=True) - - ips_after_remove = _get_ip_addr(client) - assert len(ips_after_remove) == len(ips_before) - assert secondary_priv_ip not in [ip.ip4 for ip in ips_after_remove] - - netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") - config = yaml.safe_load(netplan_cfg) - assert new_addition.interface not in config["network"]["ethernets"] - log_content = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) verify_clean_boot(client) @@ -377,36 +338,9 @@ def test_multi_nic_hotplug_vpc(session_cloud: IntegrationCloud): for route in ip_route_show.splitlines(): assert "metric" in route, "Expected metric to be in the route" - # Remove new NIC - client.instance.remove_network_interface(secondary_priv_ip4) - _wait_till_hotplug_complete(client, expected_runs=2) - - # ping to primary NIC works - retries = 32 - error = "" - for i in range(retries): - if bastion.execute(f"ping -c1 -w5 {primary_priv_ip4}").ok: - break - LOG.info("Failed to ping %s on try #%s", primary_priv_ip4, i + 1) - else: - error = ( - f"Failed to ping {primary_priv_ip4} after {retries} retries" - ) - - for i in range(retries): - if bastion.execute(f"ping -c1 -w5 {primary_priv_ip6}").ok: - break - LOG.info("Failed to ping %s on try #%s", primary_priv_ip6, i + 1) - else: - error = ( - f"Failed to ping {primary_priv_ip6} after {retries} retries" - ) - log_content = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) verify_clean_boot(client) - if error: - raise Exception(error) @pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") @@ -546,7 +480,7 @@ def _customize_environment(client: IntegrationInstance): @pytest.mark.user_data(USER_DATA) def test_nics_before_config_trigger_hotplug(client: IntegrationInstance): """ - Test that NICs added/removed after the Network boot stage but before + Test that NICs added after the Network boot stage but before the rest boot stages do trigger cloud-init-hotplugd. Note: Do not test first boot, as cc_install_hotplug runs at @@ -566,7 +500,7 @@ def test_nics_before_config_trigger_hotplug(client: IntegrationInstance): assert_systemctl_status_code(client, "cloud-config.service", 3) assert_systemctl_status_code(client, "cloud-final.service", 3) - added_ip_0 = client.instance.add_network_interface() + client.instance.add_network_interface() assert_systemctl_status_code(client, "cloud-config.service", 3) assert_systemctl_status_code(client, "cloud-final.service", 3) @@ -585,6 +519,3 @@ def test_nics_before_config_trigger_hotplug(client: IntegrationInstance): wait_for_cloud_init(client) _wait_till_hotplug_complete(client, expected_runs=1) - - client.instance.remove_network_interface(added_ip_0) - _wait_till_hotplug_complete(client, expected_runs=2) diff --git a/tests/integration_tests/modules/test_keys_to_console.py b/tests/integration_tests/modules/test_keys_to_console.py index e9b7c400..b8e74f57 100644 --- a/tests/integration_tests/modules/test_keys_to_console.py +++ b/tests/integration_tests/modules/test_keys_to_console.py @@ -58,7 +58,7 @@ def test_excluded_keys(self, class_client, key_type): # retry decorator here because it can take some time to be reflected # in syslog - @retry(tries=30, delay=1) + @retry(tries=60, delay=1) @pytest.mark.parametrize("key_type", ["ED25519", "RSA"]) def test_included_keys(self, class_client, key_type): assert "({})".format(key_type) in get_syslog_or_console(class_client) diff --git a/tests/integration_tests/modules/test_package_update_upgrade_install.py b/tests/integration_tests/modules/test_package_update_upgrade_install.py index aca0f1da..b69853cb 100644 --- a/tests/integration_tests/modules/test_package_update_upgrade_install.py +++ b/tests/integration_tests/modules/test_package_update_upgrade_install.py @@ -122,6 +122,7 @@ def test_snap_refresh_not_called_when_refresh_hold_forever( HELLO_VERSIONS_BY_RELEASE = { + "questing": "2.10-5", "plucky": "2.10-3build2", "oracular": "2.10-3build2", "noble": "2.10-3build1", @@ -141,7 +142,7 @@ def test_snap_refresh_not_called_when_refresh_hold_forever( @pytest.mark.skipif(not IS_UBUNTU, reason="Uses Apt") def test_versioned_packages_are_installed(session_cloud: IntegrationCloud): pkg_version = HELLO_VERSIONS_BY_RELEASE.get( - CURRENT_RELEASE.series, "2.10-3build1" + CURRENT_RELEASE.series, "2.10-5" ) with session_cloud.launch( user_data=VERSIONED_USER_DATA.format(pkg_version=pkg_version) diff --git a/tests/integration_tests/modules/test_power_state_change.py b/tests/integration_tests/modules/test_power_state_change.py index 4fd483fa..99298852 100644 --- a/tests/integration_tests/modules/test_power_state_change.py +++ b/tests/integration_tests/modules/test_power_state_change.py @@ -12,7 +12,10 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import IS_UBUNTU -from tests.integration_tests.util import verify_ordered_items_in_text +from tests.integration_tests.util import ( + verify_clean_boot, + verify_ordered_items_in_text, +) USER_DATA = """\ #cloud-config @@ -83,6 +86,7 @@ def test_poweroff( instance.instance.start(wait=True) log = instance.read_from_file("/var/log/cloud-init.log") assert _can_connect(instance) + verify_clean_boot(instance) lines_to_check = [ "Running module power_state_change", expected, diff --git a/tests/integration_tests/modules/test_ssh_keysfile.py b/tests/integration_tests/modules/test_ssh_keysfile.py index bf7ce0e6..fb88e79b 100644 --- a/tests/integration_tests/modules/test_ssh_keysfile.py +++ b/tests/integration_tests/modules/test_ssh_keysfile.py @@ -40,15 +40,8 @@ def common_verify(client, expected_keys): # Ensure key is in the key file contents = client.read_from_file(filename) if user in ["ubuntu", "root"]: - lines = contents.split("\n") - if user == "root": - # Our personal public key gets added by pycloudlib in - # addition to the default `ssh_authorized_keys` - assert len(lines) == 2 - else: - # Clouds will insert the keys we've added to our accounts - # or for our launches - assert len(lines) >= 2 + # Our personal public key(s) from pycloudlib or otherwise provided + # at launch may be added to the default and/or root user assert keys.public_key.strip() in contents else: assert contents.strip() == keys.public_key.strip() diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py index 809d988f..55813220 100644 --- a/tests/integration_tests/modules/test_users_groups.py +++ b/tests/integration_tests/modules/test_users_groups.py @@ -10,7 +10,12 @@ import pytest from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, JAMMY +from tests.integration_tests.releases import ( + CURRENT_RELEASE, + IS_UBUNTU, + JAMMY, + NOBLE, +) from tests.integration_tests.util import verify_clean_boot USER_DATA = """\ @@ -33,7 +38,7 @@ AHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo gecos: Bar B. Foo - sudo: ALL=(ALL) NOPASSWD:ALL + sudo: "ALL=(ALL) NOPASSWD:ALL" groups: [cloud-users, secret] lock_passwd: true - name: nopassworduser @@ -110,14 +115,20 @@ def test_users_groups(self, regex, getent_args, class_client): ) ) - def test_user_root_in_secret(self, class_client): - """Test root user is in 'secret' group.""" + def test_initial_warnings(self, class_client): + """Check for initial warnings.""" + warnings = ( + [NEW_USER_EMPTY_PASSWD_WARNING.format(username="nopassworduser")] + if CURRENT_RELEASE > NOBLE + else [] + ) verify_clean_boot( class_client, - require_warnings=[ - NEW_USER_EMPTY_PASSWD_WARNING.format(username="nopassworduser") - ], + require_warnings=warnings, ) + + def test_user_root_in_secret(self, class_client): + """Test root user is in 'secret' group.""" output = class_client.execute("groups root").stdout _, groups_str = output.split(":", maxsplit=1) groups = groups_str.split() @@ -125,28 +136,26 @@ def test_user_root_in_secret(self, class_client): def test_nopassword_unlock_warnings(self, class_client): """Verify warnings for empty passwords for new and existing users.""" - verify_clean_boot( - class_client, - require_warnings=[ - NEW_USER_EMPTY_PASSWD_WARNING.format(username="nopassworduser") - ], - ) - # Fake admin clearing and unlocking and empty unlocked password foobar # This will generate additional warnings about not unlocking passwords # for pre-existing users which have an existing empty password class_client.execute("passwd -d foobar") class_client.instance.clean() class_client.restart() - verify_clean_boot( - class_client, - ignore_warnings=True, # ignore warnings about existing groups - require_warnings=[ + warnings = ( + [ EXISTING_USER_EMPTY_PASSWD_WARNING.format( username="nopassworduser" ), EXISTING_USER_EMPTY_PASSWD_WARNING.format(username="foobar"), - ], + ] + if CURRENT_RELEASE > NOBLE + else [] + ) + verify_clean_boot( + class_client, + ignore_warnings=True, # ignore warnings about existing groups + require_warnings=warnings, ) diff --git a/tests/integration_tests/reaper.py b/tests/integration_tests/reaper.py index 308dbbfe..545ceefb 100644 --- a/tests/integration_tests/reaper.py +++ b/tests/integration_tests/reaper.py @@ -20,7 +20,7 @@ LOG = logging.getLogger() -class _Reaper: +class Reaper: def __init__(self, timeout: float = 30.0): # self.timeout sets the amount of time to sleep before retrying self.timeout = timeout @@ -132,6 +132,7 @@ def _do_reap(self) -> bool: # first destroy all newly reaped instances while not self.reaped_instances.empty(): instance = self.reaped_instances.get_nowait() + instance_id = instance.instance.id success = self._destroy(instance) if not success: LOG.warning( @@ -141,7 +142,7 @@ def _do_reap(self) -> bool: # failure to delete, add to the ledger new_undead_instances.append(instance) else: - LOG.info("Reaper: destroyed %s", instance.instance.id) + LOG.info("Reaper: destroyed %s", instance_id) # every instance has tried at least once and the reaper has been # instructed to tear down - so do it @@ -175,9 +176,10 @@ def _do_reap(self) -> bool: if self.exit_reaper.is_set() and self.reaped_instances.empty(): # don't retry instances if the exit_reaper Event is set break + instance_id = instance.instance.id if self._destroy(instance): self.undead_ledger.remove(instance) - LOG.info("Reaper: destroyed %s (undead)", instance.instance.id) + LOG.info("Reaper: destroyed %s (undead)", instance_id) self._update_undead_ledger(new_undead_instances) return False diff --git a/tests/integration_tests/test_ds_identify.py b/tests/integration_tests/test_ds_identify.py index 8d5a3131..6816df99 100644 --- a/tests/integration_tests/test_ds_identify.py +++ b/tests/integration_tests/test_ds_identify.py @@ -10,11 +10,12 @@ DATASOURCE_LIST_FILE = "/etc/cloud/cloud.cfg.d/90_dpkg.cfg" MAP_PLATFORM_TO_DATASOURCE = { + "ec2": "aws", + "ibm": "nocloud", "lxd_container": "lxd", "lxd_vm": "lxd", - "qemu": "nocloud", - "ec2": "aws", "oci": "oracle", + "qemu": "nocloud", } diff --git a/tests/integration_tests/test_networking.py b/tests/integration_tests/test_networking.py index 4f4a6816..bf010bb8 100644 --- a/tests/integration_tests/test_networking.py +++ b/tests/integration_tests/test_networking.py @@ -17,7 +17,11 @@ JAMMY, NOBLE, ) -from tests.integration_tests.util import verify_clean_boot, verify_clean_log +from tests.integration_tests.util import ( + has_netplanlib, + verify_clean_boot, + verify_clean_log, +) # Older Ubuntu series didn't read cloud-init.* config keys LXD_NETWORK_CONFIG_KEY = ( @@ -66,13 +70,13 @@ def test_skip(self, client: IntegrationInstance): client.execute( "mv /var/log/cloud-init.log /var/log/cloud-init.log.bak" ) - if CURRENT_RELEASE < JAMMY: + if has_netplanlib(client): + assert "Rendered netplan config using netplan python API" in log + else: assert ( "No netplan python module. Fallback to write" " /etc/netplan/50-cloud-init.yaml" in log ) - else: - assert "Rendered netplan config using netplan python API" in log netplan = yaml.safe_load( client.execute("cat /etc/netplan/50-cloud-init.yaml") ) @@ -196,10 +200,10 @@ def test_netplan_rendering(net_config, session_cloud: IntegrationCloud): } with session_cloud.launch(launch_kwargs=launch_kwargs) as client: result = client.execute("cat /etc/netplan/50-cloud-init.yaml") - if CURRENT_RELEASE < JAMMY: - assert result.stdout.startswith(EXPECTED_NETPLAN_HEADER) - else: + if has_netplanlib(client): assert EXPECTED_NETPLAN_HEADER not in result.stdout + else: + assert result.stdout.startswith(EXPECTED_NETPLAN_HEADER) assert expected == yaml.safe_load(result.stdout) @@ -285,16 +289,7 @@ def test_invalid_network_v2_netplan(session_cloud: IntegrationCloud): "config_dict": config_dict, } ) as client: - # Netplan python API only available on JAMMY and later - if CURRENT_RELEASE < JAMMY: - assert ( - "Skipping netplan schema validation. No netplan API available" - ) in client.read_from_file("/var/log/cloud-init.log") - assert ( - "Skipping network-config schema validation for version: 2." - " No netplan API available." - ) in client.execute("cloud-init schema --system") - else: + if has_netplanlib(client): assert ( "network-config failed schema validation! You may run " "'sudo cloud-init schema --system' to check the details." @@ -307,6 +302,14 @@ def test_invalid_network_v2_netplan(session_cloud: IntegrationCloud): "# E1: Invalid netplan schema. Error in network definition:" " invalid boolean value 'badval" ) in client.execute("cloud-init schema --system --annotate") + else: + assert ( + "Skipping netplan schema validation. No netplan API available" + ) in client.read_from_file("/var/log/cloud-init.log") + assert ( + "Skipping network-config schema validation for version: 2." + " No netplan API available." + ) in client.execute("cloud-init schema --system") @pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") diff --git a/tests/integration_tests/test_reaper.py b/tests/integration_tests/test_reaper.py index d34b2a19..6c463c68 100644 --- a/tests/integration_tests/test_reaper.py +++ b/tests/integration_tests/test_reaper.py @@ -39,7 +39,7 @@ def test_start_stop(self): """basic setup teardown""" instance = MockInstance(0) - r = reaper._Reaper() + r = reaper.Reaper() # start / stop r.start() r.stop() @@ -57,7 +57,7 @@ def test_basic_reap(self): """basic setup teardown""" i_1 = MockInstance(0) - r = reaper._Reaper() + r = reaper.Reaper() r.start() r.reap(i_1) r.stop() @@ -68,7 +68,7 @@ def test_unreaped_instance(self): i_1 = MockInstance(64) i_2 = MockInstance(64) - r = reaper._Reaper() + r = reaper.Reaper() r.start() r.reap(i_1) r.reap(i_2) @@ -94,7 +94,7 @@ def test_stubborn_reap(self): ] # forcibly disallow sleeping, to avoid wasted time during tests - r = reaper._Reaper(timeout=0.0) + r = reaper.Reaper(timeout=0.0) r.start() for i in instances: r.reap(i) @@ -130,7 +130,7 @@ def test_start_stop_multiple(self): """ num = 64 instances = [] - r = reaper._Reaper() + r = reaper.Reaper() r.start() for _ in range(num): i = MockInstance(0) diff --git a/tests/integration_tests/test_signal_handler.py b/tests/integration_tests/test_signal_handler.py new file mode 100644 index 00000000..ba667ce4 --- /dev/null +++ b/tests/integration_tests/test_signal_handler.py @@ -0,0 +1,18 @@ +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import verify_clean_boot + +USER_DATA = """\ +#cloud-config +runcmd: + - pkill cloud-init +""" + + +@pytest.mark.user_data(USER_DATA) +def test_no_warnings(client: IntegrationInstance): + """Test that the signal handler does not log errors when suppressed.""" + verify_clean_boot(client) + log = client.read_from_file("/var/log/cloud-init.log") + assert "Received signal 15 resulting in exit" in log diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py index 6da955a0..b76dcfed 100644 --- a/tests/integration_tests/test_upgrade.py +++ b/tests/integration_tests/test_upgrade.py @@ -199,3 +199,23 @@ def test_subsequent_boot_of_upgraded_package(session_cloud: IntegrationCloud): verify_clean_log(log) assert instance.execute("cloud-init status --wait --long").ok verify_clean_boot(instance) + + +@pytest.mark.timeout(600) # A failure here can leave us hanging +def test_clean_package_install(session_cloud: IntegrationCloud): + """Test that the package install works after purge of old package.""" + source = get_validated_source(session_cloud) + if not source.installs_new_version(): + pytest.skip(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) + + launch_kwargs = {"image_id": session_cloud.initial_image_id} + + with session_cloud.launch(launch_kwargs=launch_kwargs) as instance: + # Do the update before uninstalling cloud-init because we + # use cloud-init to do the package update. + instance.update_package_cache() + assert instance.execute("apt --yes remove --purge cloud-init").ok + instance.install_new_cloud_init(source, clean=False, update=False) + instance.restart() + assert instance.execute("cloud-init status --wait --long").ok + verify_clean_boot(instance) diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index 0aabbca9..6e86b677 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -6,6 +6,7 @@ import time from collections import namedtuple from contextlib import contextmanager +from datetime import datetime from functools import lru_cache from itertools import chain from pathlib import Path @@ -85,6 +86,7 @@ def verify_clean_boot( require_deprecations: Optional[list] = None, require_warnings: Optional[list] = None, require_errors: Optional[list] = None, + verify_schema: Optional[bool] = True, ): """Raise exception if the client experienced unexpected conditions. @@ -112,6 +114,7 @@ def verify_clean_boot( require :param require_warnings: list of expected warning messages to require :param require_errors: list of expected error messages to require + :param verify_schema: bool set True to validate cloud-init schema --system """ def append_or_create_list( @@ -185,6 +188,7 @@ def append_or_create_list( require_deprecations=require_deprecations, require_warnings=require_warnings, require_errors=require_errors, + verify_schema=verify_schema, ) @@ -197,6 +201,7 @@ def _verify_clean_boot( require_deprecations: Optional[list] = None, require_warnings: Optional[list] = None, require_errors: Optional[list] = None, + verify_schema: Optional[bool] = True, ): ignore_deprecations = ignore_deprecations or [] ignore_errors = ignore_errors or [] @@ -386,11 +391,22 @@ def _verify_clean_boot( f"Expected rc={rc}, received rc={out.return_code}\nstdout: " f"{out.stdout}\nstderr: {out.stderr}" ) + if not verify_schema: + return schema = instance.execute("cloud-init schema --system --annotate") - assert schema.ok, ( - f"Schema validation failed\nstdout:{schema.stdout}" - f"\nstderr:\n{schema.stderr}" - ) + if "ibm" == PLATFORM: + # IBM provides invalid vendor-data resulting in schema errors + assert "Invalid schema: vendor-data" in schema.stderr + assert not schema.ok, ( + f"Expected IBM schema validation errors due to vendor-data, did " + f"IBM images resolve this?\nstdout: {schema.stdout}\n" + f"stderr:\n{schema.stderr}" + ) + else: + assert schema.ok, ( + f"Schema validation failed\nstdout:{schema.stdout}" + f"\nstderr:\n{schema.stderr}" + ) def verify_clean_log(log: str, ignore_deprecations: bool = True): @@ -590,6 +606,11 @@ def get_feature_flag_value(client: "IntegrationInstance", key): return value +def has_netplanlib(client: "IntegrationInstance") -> bool: + """Return True if netplan python3 pkg is installed on the instance.""" + return client.execute("dpkg-query -W python3-netplan").ok + + def override_kernel_command_line(ds_str: str, instance: "IntegrationInstance"): """set the kernel command line and reboot, return after boot done @@ -643,3 +664,31 @@ def network_wait_logged(log: str) -> bool: "Running command " "['systemctl', 'start', 'systemd-networkd-wait-online.service']" ) in log + + +def get_datetime_from_string( + str, regex, datetime_strformat="%Y-%m-%d %H:%M:%S.%f%z" +): + """ + Extract datetime from a given line in a string + """ + matched = re.search(regex, str, re.M) + assert matched, ( + f"Unable to find the datetime using the regex {regex}", + f"inside the string {str}", + ) + + try: + converted_datetime = datetime.strptime( + matched.group(1), datetime_strformat + ) + except ValueError: + pytest.fail( + " ".join( + ( + f"Unable to parse the datetime {matched.group(1)}", + f"using the format {datetime_strformat}", + ) + ) + ) + return converted_datetime diff --git a/tests/unittests/analyze/test_boot.py b/tests/unittests/analyze/test_boot.py index 1e1ffcf4..18911c96 100644 --- a/tests/unittests/analyze/test_boot.py +++ b/tests/unittests/analyze/test_boot.py @@ -6,25 +6,29 @@ from cloudinit.analyze.show import ( CONTAINER_CODE, FAIL_CODE, + SUCCESS_CODE, SystemctlReader, dist_check_timestamp, + gather_timestamps_using_systemd, ) -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import mock err_code = (FAIL_CODE, -1, -1, -1) -class TestDistroChecker(CiTestCase): - def test_blank_distro(self): - self.assertEqual(err_code, dist_check_timestamp()) +class TestDistroChecker: + @mock.patch("cloudinit.subp.subp") + def test_blank_distro(self, m_subp): + assert err_code == dist_check_timestamp() + @mock.patch("cloudinit.subp.subp") @mock.patch("cloudinit.util.is_FreeBSD", return_value=True) - def test_freebsd_gentoo_cant_find(self, m_is_FreeBSD): - self.assertEqual(err_code, dist_check_timestamp()) + def test_freebsd_gentoo_cant_find(self, m_is_FreeBSD, m_subp): + err_code == dist_check_timestamp() @mock.patch("cloudinit.subp.subp", return_value=(0, 1)) def test_subp_fails(self, m_subp): - self.assertEqual(err_code, dist_check_timestamp()) + assert err_code == dist_check_timestamp() class TestSystemCtlReader: @@ -35,28 +39,36 @@ def test_systemctl_invalid(self, mocker): ) reader = SystemctlReader("dont", "care") with pytest.raises(RuntimeError): - reader.parse_epoch_as_float() + reader.convert_val_to_float() @mock.patch("cloudinit.subp.subp", return_value=("U=1000000", None)) def test_systemctl_works_correctly_threshold(self, m_subp): reader = SystemctlReader("dummyProperty", "dummyParameter") - assert 1.0 == reader.parse_epoch_as_float() - thresh = 1.0 - reader.parse_epoch_as_float() + assert 1.0 == reader.convert_val_to_float() + thresh = 1.0 - reader.convert_val_to_float() assert thresh < 1e-6 assert thresh > (-1 * 1e-6) @mock.patch("cloudinit.subp.subp", return_value=("U=0", None)) def test_systemctl_succeed_zero(self, m_subp): reader = SystemctlReader("dummyProperty", "dummyParameter") - assert 0.0 == reader.parse_epoch_as_float() + assert 0.0 == reader.convert_val_to_float() + + @mock.patch( + "cloudinit.subp.subp", + return_value=("U=Fri 1970-01-02 00:01:15.123 UTC", None), + ) + def test_systemctl_succeed_human_readable_date(self, m_subp): + reader = SystemctlReader("dummyProperty", "dummyParameter") + assert 86475.123 == reader.convert_val_to_float() @mock.patch("cloudinit.subp.subp", return_value=("U=1", None)) def test_systemctl_succeed_distinct(self, m_subp): reader = SystemctlReader("dummyProperty", "dummyParameter") - val1 = reader.parse_epoch_as_float() + val1 = reader.convert_val_to_float() m_subp.return_value = ("U=2", None) reader2 = SystemctlReader("dummyProperty", "dummyParameter") - val2 = reader2.parse_epoch_as_float() + val2 = reader2.convert_val_to_float() assert val1 != val2 @pytest.mark.parametrize( @@ -75,7 +87,7 @@ def test_systemctl_epoch_not_error(self, m_subp, return_value, exception): m_subp.return_value = return_value reader = SystemctlReader("dummyProperty", "dummyParameter") with pytest.raises(exception): - reader.parse_epoch_as_float() + reader.convert_val_to_float() class TestAnalyzeBoot: @@ -179,3 +191,86 @@ def test_container_ci_log_line(self, m_is_container, m_subp, m_get, m_g): self.remove_dummy_file(path, log_path) assert CONTAINER_CODE == finish_code + + @mock.patch("cloudinit.analyze.show.SystemctlReader") + @pytest.mark.parametrize( + ( + "is_container_returnvalue", + "expected_first_2_constructors_call_list", + "m_returns_of_convert_call", + "expected_return", + ), + ( + ( + True, + [ + mock.call("UserspaceTimestamp"), + mock.call("UserspaceTimestampMonotonic"), + ], + [1500, 7, 7, 18], + (CONTAINER_CODE, 1500, 1500, 1511), + ), + ( + False, + [ + mock.call("KernelTimestamp"), + mock.call("KernelTimestampMonotonic"), + ], + [1000, 1, 3, 8], + (SUCCESS_CODE, 1000, 1002, 1007), + ), + ), + ) + def test_gather_timestamps_using_systemd( + self, + mock_SystemctlReader, + is_container_returnvalue, + expected_first_2_constructors_call_list, + m_returns_of_convert_call, + expected_return, + mocker, + ): + """ + Testing the behavior based on whether or not the + instance is a container + """ + mocker.patch( + "cloudinit.util.is_container", + return_value=is_container_returnvalue, + ) + + # Mocking the return values of the 4 calls to convert_val_to_float + # in gather_timestamps_using_systemd + mock_SystemctlReader_instances = [mock.Mock() for _ in range(4)] + for instance, value in zip( + mock_SystemctlReader_instances, m_returns_of_convert_call + ): + instance.convert_val_to_float.return_value = value + mock_SystemctlReader.side_effect = mock_SystemctlReader_instances + + assert expected_return == gather_timestamps_using_systemd() + + # Verifying that the 4 constructor calls of SystemctlReader in + # gather_timestamps_using_systemd were with the expected arguments + assert ( + mock_SystemctlReader.call_args_list[:2] + == expected_first_2_constructors_call_list + ) + assert mock_SystemctlReader.call_args_list[2:] == [ + mock.call("UserspaceTimestampMonotonic"), + mock.call("InactiveExitTimestampMonotonic", "cloud-init-local"), + ] + + @mock.patch( + "cloudinit.analyze.show.SystemctlReader", + side_effect=Exception("ARandomError"), + ) + @mock.patch("cloudinit.subp.subp") + def test_gather_timestamps_using_systemd_with_SystemctlReader_exception( + self, m_subp, systemctlReader_mock + ): + """ + Confirm the function returns the error code when SystemctlReader + raises an exception + """ + assert gather_timestamps_using_systemd() == err_code diff --git a/tests/unittests/cmd/devel/test_hotplug_hook.py b/tests/unittests/cmd/devel/test_hotplug_hook.py index 3a1a2f1d..dc84b195 100644 --- a/tests/unittests/cmd/devel/test_hotplug_hook.py +++ b/tests/unittests/cmd/devel/test_hotplug_hook.py @@ -119,22 +119,6 @@ def test_succcessful_add(self, mocks): mocks.m_activator.bring_down_interface.assert_not_called() init._write_to_cache.assert_called_once_with() - def test_successful_remove(self, mocks): - init = mocks.m_init - mocks.m_network_state.iter_interfaces.return_value = [{}] - handle_hotplug( - hotplug_init=init, - devpath="/dev/fake", - udevaction="remove", - subsystem="net", - ) - init.datasource.update_metadata_if_supported.assert_called_once_with( - [EventType.HOTPLUG] - ) - mocks.m_activator.bring_down_interface.assert_called_once_with("fake") - mocks.m_activator.bring_up_interface.assert_not_called() - init._write_to_cache.assert_called_once_with() - @mock.patch( "cloudinit.cmd.devel.hotplug_hook.NetHandler.detect_hotplugged_device" ) @@ -158,7 +142,7 @@ def test_update_event_disabled(self, mocks, caplog): handle_hotplug( hotplug_init=init, devpath="/dev/fake", - udevaction="remove", + udevaction="add", subsystem="net", ) assert "hotplug not enabled for event of type" in caplog.text @@ -177,7 +161,7 @@ def test_update_metadata_failed(self, mocks): handle_hotplug( hotplug_init=mocks.m_init, devpath="/dev/fake", - udevaction="remove", + udevaction="add", subsystem="net", ) @@ -194,22 +178,6 @@ def test_detect_hotplugged_device_not_detected_on_add(self, mocks): subsystem="net", ) - def test_detect_hotplugged_device_detected_on_remove(self, mocks): - mocks.m_network_state.iter_interfaces.return_value = [ - { - "mac_address": FAKE_MAC, - } - ] - with pytest.raises( - RuntimeError, match="Failed to detect .* in updated metadata" - ): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - udevaction="remove", - subsystem="net", - ) - def test_apply_failed_on_add(self, mocks): mocks.m_network_state.iter_interfaces.return_value = [ { @@ -227,19 +195,6 @@ def test_apply_failed_on_add(self, mocks): subsystem="net", ) - def test_apply_failed_on_remove(self, mocks): - mocks.m_network_state.iter_interfaces.return_value = [{}] - mocks.m_activator.bring_down_interface.return_value = False - with pytest.raises( - RuntimeError, match="Failed to bring down device: /dev/fake" - ): - handle_hotplug( - hotplug_init=mocks.m_init, - devpath="/dev/fake", - udevaction="remove", - subsystem="net", - ) - def test_retry(self, mocks): with pytest.raises(RuntimeError): handle_hotplug( diff --git a/tests/unittests/cmd/devel/test_net_convert.py b/tests/unittests/cmd/devel/test_net_convert.py index 39e04922..9328150d 100644 --- a/tests/unittests/cmd/devel/test_net_convert.py +++ b/tests/unittests/cmd/devel/test_net_convert.py @@ -54,7 +54,6 @@ [Network] DHCP=ipv4 - """ SAMPLE_SYSCONFIG_CONTENT = """\ diff --git a/tests/unittests/cmd/test_clean.py b/tests/unittests/cmd/test_clean.py index e0fa14fd..2f2b6c04 100644 --- a/tests/unittests/cmd/test_clean.py +++ b/tests/unittests/cmd/test_clean.py @@ -7,9 +7,11 @@ import cloudinit.settings from cloudinit.cmd import clean +from cloudinit.config import cc_mounts from cloudinit.distros import Distro +from cloudinit.sources import DataSource from cloudinit.stages import Init -from cloudinit.util import ensure_dir, sym_link, write_file +from cloudinit.util import del_file, ensure_dir, sym_link, write_file from tests.unittests.helpers import mock, wrap_and_call MyPaths = namedtuple("MyPaths", "cloud_dir") @@ -197,6 +199,145 @@ def test_keep_net_conf(self, clean_paths, init_class): assert conf_path.exists() is True, f"file {conf_path} removed!" assert 0 == retcode + @pytest.mark.usefixtures("fake_filesystem") + def test_clean_fstab(self, clean_paths, init_class): + """remove_config removed added entries in fstab when + `cloud-init clean -c fstab` is used. + """ + fstab_original_content = ( + "UUID=abc123 / ext4 defaults 0 0\n" + "/workspace /mnt " + "auto defaults,nofail,x-systemd.after=" + "cloud-init.service,_netdev,comment=cloudconfig 0 2\n" + ) + fstab_expected_content = "UUID=abc123 / ext4 defaults 0 0\n" + + etc_path = "/etc" + if not os.path.exists(etc_path): + os.makedirs(etc_path) + fstab_path = cc_mounts.FSTAB_PATH + with open(fstab_path, "w") as fd: + fd.write(fstab_original_content) + + clean.remove_artifacts( + init_class, + remove_logs=False, + remove_config=["fstab"], + ) + + with open(fstab_path, "r") as fd: + fstab_new_content = fd.read() + assert fstab_expected_content == fstab_new_content + + @pytest.mark.usefixtures("fake_filesystem") + def test_clean_fstab_for_all(self, clean_paths, init_class): + """remove_config removed added entries in fstab when + `cloud-init clean -c all` is used. + """ + fstab_original_content = ( + "UUID=abc123 / ext4 defaults 0 0\n" + "/workspace /mnt " + "auto defaults,nofail,x-systemd.after=" + "cloud-init.service,_netdev,comment=cloudconfig 0 2\n" + ) + fstab_expected_content = "UUID=abc123 / ext4 defaults 0 0\n" + TEST_GEN_SSH_CONFIG_FILES = [ + clean_paths.tmpdir.join(conf_file) + for conf_file in clean.GEN_SSH_CONFIG_FILES + ] + + etc_path = "/etc" + if not os.path.exists(etc_path): + os.makedirs(etc_path) + fstab_path = cc_mounts.FSTAB_PATH + with open(fstab_path, "w") as fd: + fd.write(fstab_original_content) + + with mock.patch( + "cloudinit.cmd.clean.GEN_SSH_CONFIG_FILES", + TEST_GEN_SSH_CONFIG_FILES, + ): + + clean.remove_artifacts( + init_class, + remove_logs=False, + remove_config=["all"], + ) + + with open(fstab_path, "r") as fd: + fstab_new_content = fd.read() + assert fstab_expected_content == fstab_new_content + + def test_clean_datasource_conf_without_cache( + self, clean_paths, init_class + ): + """remove_config does not remove datasource files when cache + is not present. + """ + ds_conf = clean_paths.tmpdir.join("/var/run/ds") + assert ds_conf.exists() is False, f"Unexpected {ds_conf}" + ensure_dir(os.path.dirname(ds_conf)) + ds_conf.write("#generated by generic DataSource\nfoobar\n") + assert ds_conf.exists() is True, "{ds_conf} not written!" + + assert ( + clean_paths.cloud_dir.exists() is False + ), "unexpected cloud_dir present!" + + def ds_clean(): + print("deleting {ds_conf}") + del_file(ds_conf) + + ds = mock.Mock(spec=DataSource) + ds.clean = ds_clean + + def ds_fetch(): + return ds + + init_class.fetch = ds_fetch + retcode = clean.remove_artifacts( + init_class, + remove_logs=False, + remove_config=["datasource"], + ) + assert ds_conf.exists() is True, f"file {ds_conf} was removed!" + assert 0 == retcode + + def test_clean_datasource_conf_with_cache(self, clean_paths, init_class): + """remove_config removes datasource files when cache + is present. + """ + ensure_dir(clean_paths.cloud_dir) + ds_conf = clean_paths.tmpdir.join("/var/run/ds") + assert ds_conf.exists() is False, f"Unexpected {ds_conf}" + ensure_dir(os.path.dirname(ds_conf)) + ds_conf.write("#generated by generic DataSource\nfoobar\n") + assert ds_conf.exists() is True, "{ds_conf} not written!" + + cache = clean_paths.cloud_dir.join("instance") + assert cache.exists() is False, f"unexpected {cache} present!" + ensure_dir(cache) + assert cache.exists() is True, "{cache} not created!" + + def ds_clean(): + print("deleting {ds_conf}") + del_file(ds_conf) + + ds = mock.Mock(spec=DataSource) + ds.clean = ds_clean + + def ds_fetch(): + return ds + + init_class.fetch = ds_fetch + retcode = clean.remove_artifacts( + init_class, + remove_logs=False, + remove_config=["datasource"], + ) + assert ds_conf.exists() is False, f"Unexpected file {ds_conf}" + assert 0 == retcode + @pytest.mark.allow_all_subp def test_remove_artifacts_runparts_clean_d(self, clean_paths, init_class): """remove_artifacts performs runparts on CLEAN_RUNPARTS_DIR""" diff --git a/tests/unittests/cmd/test_main.py b/tests/unittests/cmd/test_main.py index ab5a73bd..bcf93bec 100644 --- a/tests/unittests/cmd/test_main.py +++ b/tests/unittests/cmd/test_main.py @@ -189,6 +189,23 @@ def test_main_sys_argv( main.main() m_clean_get_parser.assert_called_once() + @mock.patch("cloudinit.cmd.clean.get_parser") + @mock.patch("cloudinit.cmd.clean.handle_clean_args") + @mock.patch("cloudinit.log.loggers.configure_root_logger") + def test_main_sys_argv_missing_subcommand( + self, + _m_configure_root_logger, + _m_handle_clean_args, + m_clean_get_parser, + capsys, + ): + with mock.patch("sys.argv", ["cloudinit", "--debug"]): + with pytest.raises(SystemExit): + main.main() + stderr = capsys.readouterr().err + assert "No Subcommand specified." in stderr + m_clean_get_parser.assert_not_called() + @pytest.mark.parametrize( "ds,userdata,expected", [ diff --git a/tests/unittests/config/test_apt_configure_sources_list_v1.py b/tests/unittests/config/test_apt_configure_sources_list_v1.py index 83e0d6bb..87b07dfc 100644 --- a/tests/unittests/config/test_apt_configure_sources_list_v1.py +++ b/tests/unittests/config/test_apt_configure_sources_list_v1.py @@ -129,7 +129,7 @@ def apt_source_list(self, distro, mirror, tmpdir, mirrorcheck=None): tmpl_file = f"/etc/cloud/templates/sources.list.{distro}.tmpl" util.write_file(tmpl_file, EXAMPLE_TMPL) - cc_apt_configure.handle("test", cfg, mycloud, None) + cc_apt_configure.handle("test", cfg, mycloud, []) sources_file = tmpdir.join("/etc/apt/sources.list") assert expected == sources_file.read() assert 0o644 == stat.S_IMODE(sources_file.stat().mode) @@ -152,7 +152,7 @@ def test_apt_v1_source_list_by_distro(self, distro, mirror, tmpdir): mycloud = get_cloud(distro) tmpl_file = f"/etc/cloud/templates/sources.list.{distro}.deb822.tmpl" util.write_file(tmpl_file, EXAMPLE_TMPL_DEB822) - cc_apt_configure.handle("test", {"apt_mirror": mirror}, mycloud, None) + cc_apt_configure.handle("test", {"apt_mirror": mirror}, mycloud, []) sources_file = tmpdir.join(f"/etc/apt/sources.list.d/{distro}.sources") assert ( EXPECTED_CONVERTED_CONTENT_DEB822.replace( @@ -202,7 +202,7 @@ def test_apt_v1_srcl_distro_mirrorfail( util, "is_resolvable", side_effect=self.myresolve ) cc_apt_configure.handle( - "test", {"apt_mirror_search": mirrorlist}, mycloud, None + "test", {"apt_mirror_search": mirrorlist}, mycloud, [] ) sources_file = tmpdir.join(f"/etc/apt/sources.list.d/{distro}.sources") assert ( @@ -268,7 +268,7 @@ def test_apt_v1_srcl_custom( util.write_file(tmpl_file, tmpl_content) # the second mock restores the original subp - cc_apt_configure.handle("notimportant", cfg, mycloud, None) + cc_apt_configure.handle("notimportant", cfg, mycloud, []) sources_file = tmpdir.join(apt_file) assert expected == sources_file.read() assert 0o644 == stat.S_IMODE(sources_file.stat().mode) diff --git a/tests/unittests/config/test_apt_source_v3.py b/tests/unittests/config/test_apt_source_v3.py index 9bdfa954..2e766d77 100644 --- a/tests/unittests/config/test_apt_source_v3.py +++ b/tests/unittests/config/test_apt_source_v3.py @@ -962,7 +962,7 @@ def test_apt_v3_disable_suites(self): deb http://ubuntu.com/ubuntu/ xenial-proposed main""" # disable nothing - disabled = [] + disabled: list = [] expect = """deb http://ubuntu.com//ubuntu xenial main deb http://ubuntu.com//ubuntu xenial-updates main deb http://ubuntu.com//ubuntu xenial-security main diff --git a/tests/unittests/config/test_cc_ansible.py b/tests/unittests/config/test_cc_ansible.py index 37aa40fd..3f3353b2 100644 --- a/tests/unittests/config/test_cc_ansible.py +++ b/tests/unittests/config/test_cc_ansible.py @@ -1,7 +1,9 @@ import os import re +from contextlib import nullcontext from copy import deepcopy from textwrap import dedent +from typing import Any, Dict from unittest import mock from unittest.mock import MagicMock @@ -14,7 +16,11 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import skipUnlessJsonSchema +from cloudinit.subp import ProcessExecutionError +from tests.unittests.helpers import ( + skipUnlessJsonSchema, + skipUnlessJsonSchemaVersionGreaterThan, +) from tests.unittests.util import get_cloud try: @@ -99,7 +105,7 @@ }, } -CFG_FULL_PULL = { +CFG_FULL_PULL_DICT: Dict[str, Any] = { "ansible": { "install_method": "distro", "package_name": "ansible-core", @@ -109,7 +115,7 @@ }, "pull": { "url": "https://github/holmanb/vmboot", - "playbook_name": "arch.yml", + "playbook_names": ["arch.yml"], "accept_host_key": True, "clean": True, "full": True, @@ -133,46 +139,107 @@ } } -CFG_MINIMAL = { +CFG_FULL_PULL_LIST: Dict[str, Any] = { + "ansible": { + "install_method": "distro", + "package_name": "ansible-core", + "ansible_config": "/etc/ansible/ansible.cfg", + "galaxy": { + "actions": [["ansible-galaxy", "install", "debops.apt"]], + }, + "pull": [ + { + "url": "https://github/holmanb/vmboot", + "playbook_names": ["arch.yml"], + "accept_host_key": True, + "clean": True, + "full": True, + "diff": False, + "ssh_common_args": "-y", + "scp_extra_args": "-l", + "sftp_extra_args": "-f", + "checkout": "tree", + "module_path": "~/.ansible/plugins/modules:" + "/usr/share/ansible/plugins/modules", + "timeout": "10", + "vault_id": "me", + "connection": "smart", + "vault_password_file": "/path/to/file", + "module_name": "git", + "sleep": "1", + "tags": "cumulus", + "skip_tags": "cisco", + "private_key": "{nope}", + } + ], + } +} + +CFG_MINIMAL_DICT = { "ansible": { "install_method": "pip", "package_name": "ansible", "run_user": "ansible", "pull": { "url": "https://github/holmanb/vmboot", - "playbook_name": "ubuntu.yml", + "playbook_names": ["ubuntu.yml"], }, } } +CFG_MINIMAL_LIST = { + "ansible": { + "install_method": "pip", + "package_name": "ansible", + "run_user": "ansible", + "pull": [ + { + "url": "https://github/holmanb/vmboot", + "playbook_names": ["ubuntu.yml"], + } + ], + } +} + class TestSchema: + @mark.parametrize( ("config", "error_msg"), ( param( - CFG_MINIMAL, - None, - id="essentials", + CFG_MINIMAL_DICT, + "Expect **ansible.pull** as list of objects", + id="essentials_dict", + ), + param( + CFG_FULL_PULL_DICT, + "Expect **ansible.pull** as list of objects", + id="all-pull-keys-dict", ), + ), + ) + @skipUnlessJsonSchemaVersionGreaterThan(version=(3, 2, 0)) + def test_schema_validation_deprecations(self, config, error_msg): + with raises(SchemaValidationError, match=re.escape(error_msg)): + validate_cloudconfig_schema(config, get_schema(), strict=True) + + @mark.parametrize( + ("config", "error_msg"), + ( param( { "ansible": { "install_method": "distro", "pull": { "url": "https://github/holmanb/vmboot", - "playbook_name": "centos.yml", + "playbook_names": ["centos.yml"], "dance": "bossa nova", }, } }, "Additional properties are not allowed ", - id="additional-properties", - ), - param( - CFG_FULL_PULL, - None, - id="all-pull-keys", + id="additional-properties-dict", ), param( CFG_CTRL, @@ -185,24 +252,24 @@ class TestSchema: "install_method": "true", "pull": { "url": "https://github/holmanb/vmboot", - "playbook_name": "debian.yml", + "playbook_names": ["debian.yml"], }, } }, "'true' is not one of ['distro', 'pip']", - id="install-type", + id="install-type-dict", ), param( { "ansible": { "install_method": "pip", "pull": { - "playbook_name": "fedora.yml", + "playbook_names": ["fedora.yml"], }, } }, "'url' is a required property", - id="require-url", + id="require-url-dict", ), param( { @@ -214,7 +281,81 @@ class TestSchema: } }, "'playbook_name' is a required property", - id="require-url", + id="require-url-dict", + ), + param( + CFG_MINIMAL_LIST, + None, + id="essentials_list", + ), + param( + { + "ansible": { + "install_method": "distro", + "pull": [ + { + "url": "https://github/holmanb/vmboot", + "playbook_names": ["centos.yml"], + "dance": "bossa nova", + } + ], + } + }, + "Additional properties are not allowed ", + id="additional-properties-list", + ), + param( + CFG_FULL_PULL_LIST, + None, + id="all-pull-keys-list", + ), + param( + CFG_CTRL, + None, + id="ctrl-keys", + ), + param( + { + "ansible": { + "install_method": "true", + "pull": [ + { + "url": "https://github/holmanb/vmboot", + "playbook_names": ["debian.yml"], + } + ], + } + }, + "'true' is not one of ['distro', 'pip']", + id="install-type-list", + ), + param( + { + "ansible": { + "install_method": "pip", + "pull": [ + { + "playbook_names": ["fedora.yml"], + } + ], + } + }, + "'url' is a required property", + id="require-url-list", + ), + param( + { + "ansible": { + "install_method": "pip", + "pull": [ + { + "url": "gophers://encrypted-gophers/", + } + ], + } + }, + "'playbook_name' is a required property", + id="require-url-list", ), ), ) @@ -230,12 +371,13 @@ def test_schema_validation(self, config, error_msg): class TestAnsible: def test_filter_args(self): """only diff should be removed""" - out = cc_ansible.filter_args( - CFG_FULL_PULL.get("ansible", {}).get("pull", {}) + src_cfg = deepcopy( + CFG_FULL_PULL_DICT.get("ansible", {}).get("pull", {}) ) - assert out == { + src_cfg.pop("playbook_names") + out_dict = cc_ansible.filter_args(src_cfg) + assert out_dict == { "url": "https://github/holmanb/vmboot", - "playbook-name": "arch.yml", "accept-host-key": True, "clean": True, "full": True, @@ -257,17 +399,17 @@ def test_filter_args(self): } @mark.parametrize( - ("cfg", "exception"), + ("cfg_dict", "exception"), ( - (CFG_FULL_PULL, None), - (CFG_MINIMAL, None), + (CFG_FULL_PULL_DICT, None), + (CFG_MINIMAL_DICT, None), ( { "ansible": { "package_name": "ansible-core", "install_method": "distro", "pull": { - "playbook_name": "ubuntu.yml", + "playbook_names": ["ubuntu.yml"], }, } }, @@ -286,7 +428,7 @@ def test_filter_args(self): ), ), ) - def test_required_keys(self, cfg, exception, mocker): + def test_required_keys_dict(self, cfg_dict, exception, mocker): mocker.patch(M_PATH + "subp.subp", return_value=("", "")) mocker.patch(M_PATH + "subp.which", return_value=True) mocker.patch(M_PATH + "AnsiblePull.check_deps") @@ -300,12 +442,79 @@ def test_required_keys(self, cfg, exception, mocker): ) if exception: with raises(exception): - cc_ansible.handle("", cfg, get_cloud(), None) + cc_ansible.handle("", cfg_dict, get_cloud(), []) else: cloud = get_cloud(mocked_distro=True) cloud.distro.pip_package_name = "python3-pip" - install = cfg["ansible"]["install_method"] - cc_ansible.handle("", cfg, cloud, None) + install = cfg_dict["ansible"]["install_method"] + cc_ansible.handle("", cfg_dict, cloud, []) + if install == "distro": + cloud.distro.install_packages.assert_called_once() + cloud.distro.install_packages.assert_called_with( + ["ansible-core"] + ) + elif install == "pip": + if HAS_PIP: + assert 0 == cloud.distro.install_packages.call_count + else: + cloud.distro.install_packages.assert_called_with( + ["python3-pip"] + ) + + @mark.parametrize( + ("cfg_list", "exception"), + ( + (CFG_FULL_PULL_LIST, None), + (CFG_MINIMAL_LIST, None), + ( + { + "ansible": { + "package_name": "ansible-core", + "install_method": "distro", + "pull": [ + { + "playbook_names": ["ubuntu.yml"], + } + ], + } + }, + ValueError, + ), + ( + { + "ansible": { + "install_method": "pip", + "pull": [ + { + "url": "https://github/holmanb/vmboot", + } + ], + } + }, + ValueError, + ), + ), + ) + def test_required_keys_list(self, cfg_list, exception, mocker): + mocker.patch(M_PATH + "subp.subp", return_value=("", "")) + mocker.patch(M_PATH + "subp.which", return_value=True) + mocker.patch(M_PATH + "AnsiblePull.check_deps") + mocker.patch( + M_PATH + "AnsiblePull.get_version", + return_value=cc_ansible.lifecycle.Version(2, 7, 1), + ) + mocker.patch( + M_PATH + "AnsiblePullDistro.is_installed", + return_value=False, + ) + if exception: + with raises(exception): + cc_ansible.handle("", cfg_list, get_cloud(), []) + else: + cloud = get_cloud(mocked_distro=True) + cloud.distro.pip_package_name = "python3-pip" + install = cfg_list["ansible"]["install_method"] + cc_ansible.handle("", cfg_list, cloud, []) if install == "distro": cloud.distro.install_packages.assert_called_once() cloud.distro.install_packages.assert_called_with( @@ -343,16 +552,75 @@ def test_pip_bootstrap(self, m_which, m_subp): cc_ansible.AnsiblePullPip(distro, "ansible").install("") distro.install_packages.assert_called_once() + @mark.parametrize( + "mocked_doas_kwargs,expected_result,expected_present_logs,expected_absent_logs", + [ + ( + {"return_value": ("", "")}, + nullcontext(), + ["Upgraded pip"], + ["Failed at upgrading pip"], + ), + # When do_as raises a ProcessExecutionError, __upgrade_pip will + # catch it then return without raising it + ( + {"side_effect": ProcessExecutionError()}, + nullcontext(), + ["Failed at upgrading pip"], + ["Upgraded pip"], + ), + # The below use case is to ensure that the function will raise the + # error if the error is not ProcessExecutionError (i.e. something + # went really sideways with do_as and not simply a non-zero exit + # by the executed command) + ( + {"side_effect": ZeroDivisionError()}, + raises(ZeroDivisionError), + [], + ["Failed at upgrading pip", "Upgraded pip"], + ), + ], + ) + def test_upgrade_pip( + self, + mocked_doas_kwargs, + expected_result, + expected_present_logs, + expected_absent_logs, + caplog, + ): + """assert __upgrade_pip will succeed + or will catch an exception and not fail""" + distro = get_cloud(mocked_distro=True).distro + pip = cc_ansible.AnsiblePullPip(distro, "ansible") + base_cmd = ["echo", "abc"] + with mock.patch.object( + pip, "do_as", **mocked_doas_kwargs + ) as mocked_doas: + with expected_result as e: + assert ( + getattr(pip, "_AnsiblePullPip__upgrade_pip")(base_cmd) == e + ) + + mocked_doas.assert_called_once() + mocked_doas.assert_called_with([*base_cmd, "--upgrade", "pip"]) + + for expected_present_log in expected_present_logs: + assert expected_present_log in caplog.text + + for expected_absent_log in expected_absent_logs: + assert expected_absent_log not in caplog.text + @mock.patch(M_PATH + "subp.which", return_value=True) - @mock.patch(M_PATH + "subp.subp", return_value=("stdout", "stderr")) + @mock.patch(M_PATH + "subp.subp", return_value=(distro_version, "stderr")) @mock.patch( - "cloudinit.distros.subp.subp", return_value=("stdout", "stderr") + "cloudinit.distros.subp.subp", return_value=(distro_version, "stderr") ) @mark.parametrize( - ("cfg", "expected"), + ("cfg_dict", "expected"), ( ( - CFG_FULL_PULL, + CFG_FULL_PULL_DICT, [ "ansible-pull", "--url=https://github/holmanb/vmboot", @@ -378,7 +646,7 @@ def test_pip_bootstrap(self, m_which, m_subp): ], ), ( - CFG_MINIMAL, + CFG_MINIMAL_DICT, [ "ansible-pull", "--url=https://github/holmanb/vmboot", @@ -387,18 +655,87 @@ def test_pip_bootstrap(self, m_which, m_subp): ), ), ) - def test_ansible_pull(self, m_subp1, m_subp2, m_which, cfg, expected): + def test_ansible_pull_dict( + self, m_subp1, m_subp2, m_which, cfg_dict, expected + ): """verify expected ansible invocation from userdata config""" - pull_type = cfg["ansible"]["install_method"] + pull_type = cfg_dict["ansible"]["install_method"] distro = get_cloud().distro + distro.do_as = MagicMock(return_value=(distro_version, "")) + ansible_pull = ( cc_ansible.AnsiblePullPip(distro, "ansible") if pull_type == "pip" else cc_ansible.AnsiblePullDistro(distro, "") ) cc_ansible.run_ansible_pull( - ansible_pull, deepcopy(cfg["ansible"]["pull"]) + ansible_pull, deepcopy(cfg_dict["ansible"]["pull"]) + ) + + if pull_type != "pip": + assert m_subp2.call_args[0][0] == expected + assert m_subp2.call_args[1]["update_env"].get( + "HOME" + ) == os.environ.get("HOME", "/root") + + @mock.patch(M_PATH + "subp.which", return_value=True) + @mock.patch(M_PATH + "subp.subp", return_value=(distro_version, "")) + @mock.patch( + "cloudinit.distros.subp.subp", return_value=(distro_version, "") + ) + @mark.parametrize( + ("cfg_list", "expected"), + ( + ( + CFG_FULL_PULL_LIST, + [ + "ansible-pull", + "--url=https://github/holmanb/vmboot", + "--accept-host-key", + "--clean", + "--full", + "--ssh-common-args=-y", + "--scp-extra-args=-l", + "--sftp-extra-args=-f", + "--checkout=tree", + "--module-path=~/.ansible/plugins/modules" + ":/usr/share/ansible/plugins/modules", + "--timeout=10", + "--vault-id=me", + "--connection=smart", + "--vault-password-file=/path/to/file", + "--module-name=git", + "--sleep=1", + "--tags=cumulus", + "--skip-tags=cisco", + "--private-key={nope}", + "arch.yml", + ], + ), + ( + CFG_MINIMAL_LIST, + [ + "ansible-pull", + "--url=https://github/holmanb/vmboot", + "ubuntu.yml", + ], + ), + ), + ) + def test_ansible_pull_list( + self, m_subp1, m_subp2, m_which, cfg_list, expected + ): + """verify expected ansible invocation from userdata config""" + pull_type = cfg_list["ansible"]["install_method"] + distro = get_cloud().distro + distro.do_as = MagicMock(return_value=(distro_version, "")) + ansible_pull = ( + cc_ansible.AnsiblePullPip(distro, "ansible") + if pull_type == "pip" + else cc_ansible.AnsiblePullDistro(distro, "") ) + for cfg in cfg_list["ansible"]["pull"]: + cc_ansible.run_ansible_pull(ansible_pull, deepcopy(cfg)) if pull_type != "pip": assert m_subp2.call_args[0][0] == expected @@ -409,7 +746,7 @@ def test_ansible_pull(self, m_subp1, m_subp2, m_which, cfg, expected): @mock.patch(M_PATH + "validate_config") def test_do_not_run(self, m_validate): """verify that if ansible key not included, don't do anything""" - cc_ansible.handle("", {}, get_cloud(), None) # pyright: ignore + cc_ansible.handle("", {}, get_cloud(), []) assert not m_validate.called @mock.patch( @@ -432,10 +769,13 @@ def test_parse_version_pip(self, m_subp): expected = lifecycle.Version(2, 13, 2) assert received == expected - @mock.patch(M_PATH + "subp.subp", return_value=("stdout", "stderr")) + @mock.patch( + M_PATH + "subp.subp", + side_effect=[("out", "err"), (distro_version, ""), ("out", "err")], + ) @mock.patch(M_PATH + "subp.which", return_value=True) def test_ansible_env_var(self, m_which, m_subp): - cc_ansible.handle("", CFG_FULL_PULL, get_cloud(), []) + cc_ansible.handle("", CFG_FULL_PULL_DICT, get_cloud(), []) # python 3.8 required for Mock.call_args.kwargs dict attribute if isinstance(m_subp.call_args.kwargs, dict): diff --git a/tests/unittests/config/test_cc_apk_configure.py b/tests/unittests/config/test_cc_apk_configure.py index 6d09c573..db78687e 100644 --- a/tests/unittests/config/test_cc_apk_configure.py +++ b/tests/unittests/config/test_cc_apk_configure.py @@ -4,102 +4,58 @@ Test creation of repositories file """ -import os import re import textwrap import pytest -from cloudinit import cloud, helpers, util +from cloudinit import util from cloudinit.config import cc_apk_configure from cloudinit.config.schema import ( SchemaValidationError, get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import ( - SCHEMA_EMPTY_ERROR, - FilesystemMockingTestCase, - mock, - skipUnlessJsonSchema, -) +from tests.unittests.helpers import SCHEMA_EMPTY_ERROR, skipUnlessJsonSchema +from tests.unittests.util import get_cloud REPO_FILE = "/etc/apk/repositories" DEFAULT_MIRROR_URL = "https://alpine.global.ssl.fastly.net/alpine" CC_APK = "cloudinit.config.cc_apk_configure" -class TestNoConfig(FilesystemMockingTestCase): - def setUp(self): - super(TestNoConfig, self).setUp() - self.add_patch(CC_APK + "._write_repositories_file", "m_write_repos") - self.name = "apk_configure" - self.cloud_init = None - self.args = [] +class TestApkConfigure: + @pytest.fixture(autouse=True) + def setup(self, mocker, tmpdir): + mocker.patch("cloudinit.temp_utils._ROOT_TMPDIR", str(tmpdir)) + yield - def test_no_config(self): + @pytest.mark.parametrize( + "config", + ( + ({}), + ({"apk_repos": {}}), + ({"apk_repos": {"alpine_repo": []}}), + ), + ) + def test_no_config(self, config, mocker): """ Test that nothing is done if no apk_configure configuration is provided. """ - config = util.get_builtin_cfg() - - cc_apk_configure.handle(self.name, config, self.cloud_init, self.args) - - self.assertEqual(0, self.m_write_repos.call_count) - - -class TestConfig(FilesystemMockingTestCase): - def setUp(self): - super().setUp() - self.new_root = self.tmp_dir() - self.new_root = self.reRoot(root=self.new_root) - for dirname in ["tmp", "etc/apk"]: - util.ensure_dir(os.path.join(self.new_root, dirname)) - self.paths = helpers.Paths({"templates_dir": self.new_root}) - self.name = "apk_configure" - self.cloud = cloud.Cloud(None, self.paths, None, None, None) - self.args = [] - self.mock = mock.patch( - "cloudinit.temp_utils.get_tmp_ancestor", lambda *_: self.new_root - ) - self.mock.start() - - def tearDown(self): - self.mock.stop() - super().tearDown() - - @mock.patch(CC_APK + "._write_repositories_file") - def test_no_repo_settings(self, m_write_repos): - """ - Test that nothing is written if the 'alpine-repo' key - is not present. - """ - config = {"apk_repos": {}} - - cc_apk_configure.handle(self.name, config, self.cloud, self.args) - - self.assertEqual(0, m_write_repos.call_count) - - @mock.patch(CC_APK + "._write_repositories_file") - def test_empty_repo_settings(self, m_write_repos): - """ - Test that nothing is written if 'alpine_repo' list is empty. - """ - config = {"apk_repos": {"alpine_repo": []}} - - cc_apk_configure.handle(self.name, config, self.cloud, self.args) + m_write_repos = mocker.patch(CC_APK + "._write_repositories_file") - self.assertEqual(0, m_write_repos.call_count) + cc_apk_configure.handle("", config, get_cloud(), []) + assert m_write_repos.call_count == 0 - def test_only_main_repo(self): + def test_only_main_repo(self, fake_filesystem): """ Test when only details of main repo is written to file. """ alpine_version = "v3.12" config = {"apk_repos": {"alpine_repo": {"version": alpine_version}}} - cc_apk_configure.handle(self.name, config, self.cloud, self.args) + cc_apk_configure.handle("", config, get_cloud(), []) expected_content = textwrap.dedent( """\ @@ -116,9 +72,9 @@ def test_only_main_repo(self): ) ) - self.assertEqual(expected_content, util.load_text_file(REPO_FILE)) + assert util.load_text_file(REPO_FILE) == expected_content - def test_main_and_community_repos(self): + def test_main_and_community_repos(self, fake_filesystem): """ Test when only details of main and community repos are written to file. @@ -133,7 +89,7 @@ def test_main_and_community_repos(self): } } - cc_apk_configure.handle(self.name, config, self.cloud, self.args) + cc_apk_configure.handle("", config, get_cloud(), []) expected_content = textwrap.dedent( """\ @@ -151,9 +107,9 @@ def test_main_and_community_repos(self): ) ) - self.assertEqual(expected_content, util.load_text_file(REPO_FILE)) + assert util.load_text_file(REPO_FILE) == expected_content - def test_main_community_testing_repos(self): + def test_main_community_testing_repos(self, fake_filesystem): """ Test when details of main, community and testing repos are written to file. @@ -169,7 +125,7 @@ def test_main_community_testing_repos(self): } } - cc_apk_configure.handle(self.name, config, self.cloud, self.args) + cc_apk_configure.handle("", config, get_cloud(), []) expected_content = textwrap.dedent( """\ @@ -191,9 +147,9 @@ def test_main_community_testing_repos(self): ) ) - self.assertEqual(expected_content, util.load_text_file(REPO_FILE)) + assert util.load_text_file(REPO_FILE) == expected_content - def test_edge_main_community_testing_repos(self): + def test_edge_main_community_testing_repos(self, fake_filesystem): """ Test when details of main, community and testing repos for Edge version of Alpine are written to file. @@ -209,7 +165,7 @@ def test_edge_main_community_testing_repos(self): } } - cc_apk_configure.handle(self.name, config, self.cloud, self.args) + cc_apk_configure.handle("", config, get_cloud(), []) expected_content = textwrap.dedent( """\ @@ -228,9 +184,9 @@ def test_edge_main_community_testing_repos(self): ) ) - self.assertEqual(expected_content, util.load_text_file(REPO_FILE)) + assert util.load_text_file(REPO_FILE) == expected_content - def test_main_community_testing_local_repos(self): + def test_main_community_testing_local_repos(self, fake_filesystem): """ Test when details of main, community, testing and local repos are written to file. @@ -248,7 +204,7 @@ def test_main_community_testing_local_repos(self): } } - cc_apk_configure.handle(self.name, config, self.cloud, self.args) + cc_apk_configure.handle("", config, get_cloud(), []) expected_content = textwrap.dedent( """\ @@ -275,9 +231,9 @@ def test_main_community_testing_local_repos(self): ) ) - self.assertEqual(expected_content, util.load_text_file(REPO_FILE)) + assert util.load_text_file(REPO_FILE) == expected_content - def test_edge_main_community_testing_local_repos(self): + def test_edge_main_community_testing_local_repos(self, fake_filesystem): """ Test when details of main, community, testing and local repos for Edge version of Alpine are written to file. @@ -295,7 +251,7 @@ def test_edge_main_community_testing_local_repos(self): } } - cc_apk_configure.handle(self.name, config, self.cloud, self.args) + cc_apk_configure.handle("", config, get_cloud(), []) expected_content = textwrap.dedent( """\ @@ -319,7 +275,7 @@ def test_edge_main_community_testing_local_repos(self): ) ) - self.assertEqual(expected_content, util.load_text_file(REPO_FILE)) + assert util.load_text_file(REPO_FILE) == expected_content class TestApkConfigureSchema: @@ -347,8 +303,7 @@ class TestApkConfigureSchema: # Invalid schemas ( {"apk_repos": {"alpine_repo": {"version": False}}}, - "apk_repos.alpine_repo.version: False is not of type" - " 'string'", + "apk_repos.alpine_repo.version: False is not of type 'string'", ), ( { diff --git a/tests/unittests/config/test_cc_bootcmd.py b/tests/unittests/config/test_cc_bootcmd.py index 29292405..46ca401e 100644 --- a/tests/unittests/config/test_cc_bootcmd.py +++ b/tests/unittests/config/test_cc_bootcmd.py @@ -13,7 +13,6 @@ ) from tests.unittests.helpers import ( SCHEMA_EMPTY_ERROR, - CiTestCase, mock, skipUnlessJsonSchema, ) @@ -35,72 +34,62 @@ def __exit__(self, exc_type, exc_value, traceback): util.del_file(self.handle.name) -class TestBootcmd(CiTestCase): - - with_logs = True +@pytest.mark.usefixtures("fake_filesystem") +class TestBootcmd: _etmpfile_path = ( "cloudinit.config.cc_bootcmd.temp_utils.ExtendedTemporaryFile" ) - def setUp(self): - super(TestBootcmd, self).setUp() - self.subp = subp.subp - self.new_root = self.tmp_dir() - - def test_handler_skip_if_no_bootcmd(self): + def test_handler_skip_if_no_bootcmd(self, caplog): """When the provided config doesn't contain bootcmd, skip it.""" cfg = {} mycloud = get_cloud() handle("notimportant", cfg, mycloud, None) - self.assertIn( - "Skipping module named notimportant, no 'bootcmd' key", - self.logs.getvalue(), + assert ( + "Skipping module named notimportant, no 'bootcmd' key" + in caplog.text ) - def test_handler_invalid_command_set(self): + def test_handler_invalid_command_set(self, caplog): """Commands which can't be converted to shell will raise errors.""" invalid_config = {"bootcmd": 1} cc = get_cloud() - with self.assertRaises(TypeError) as context_manager: + with pytest.raises( + TypeError, + match="Input to shellify was type 'int'. Expected list or tuple.", + ): handle("cc_bootcmd", invalid_config, cc, []) - self.assertIn("Failed to shellify bootcmd", self.logs.getvalue()) - self.assertEqual( - "Input to shellify was type 'int'. Expected list or tuple.", - str(context_manager.exception), - ) + assert "Failed to shellify bootcmd" in caplog.text invalid_config = { "bootcmd": ["ls /", 20, ["wget", "http://stuff/blah"], {"a": "n"}] } cc = get_cloud() - with self.assertRaises(TypeError) as context_manager: + with pytest.raises( + TypeError, + match="Unable to shellify type 'int'. Expected list, string, " + "tuple. Got: 20", + ): handle("cc_bootcmd", invalid_config, cc, []) - logs = self.logs.getvalue() - self.assertIn("Failed to shellify", logs) - self.assertEqual( - "Unable to shellify type 'int'. Expected list, string, tuple. " - "Got: 20", - str(context_manager.exception), - ) + assert "Failed to shellify" in caplog.text - def test_handler_runs_bootcmd_script_with_error(self): + @pytest.mark.allow_subp_for("/bin/sh") + def test_handler_runs_bootcmd_script_with_error(self, caplog): """When a valid script generates an error, that error is raised.""" cc = get_cloud() valid_config = {"bootcmd": ["exit 1"]} # Script with error with mock.patch(self._etmpfile_path, FakeExtendedTempFile): - with self.allow_subp(["/bin/sh"]): - with self.assertRaises(subp.ProcessExecutionError) as ctxt: - handle("does-not-matter", valid_config, cc, []) - self.assertIn( - "Unexpected error while running command.\nCommand: ['/bin/sh',", - str(ctxt.exception), - ) - self.assertIn( - "Failed to run bootcmd module does-not-matter", - self.logs.getvalue(), - ) + with pytest.raises( + subp.ProcessExecutionError, + match=( + r"Unexpected error while running command.\n" + r"Command: \['/bin/sh'," + ), + ): + handle("does-not-matter", valid_config, cc, []) + assert "Failed to run bootcmd module does-not-matter" in caplog.text @skipUnlessJsonSchema() diff --git a/tests/unittests/config/test_cc_disable_ec2_metadata.py b/tests/unittests/config/test_cc_disable_ec2_metadata.py index e67b07c2..c2ec97c0 100644 --- a/tests/unittests/config/test_cc_disable_ec2_metadata.py +++ b/tests/unittests/config/test_cc_disable_ec2_metadata.py @@ -11,12 +11,12 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from tests.unittests.helpers import mock, skipUnlessJsonSchema DISABLE_CFG = {"disable_ec2_metadata": "true"} -class TestEC2MetadataRoute(CiTestCase): +class TestEC2MetadataRoute: @mock.patch("cloudinit.config.cc_disable_ec2_metadata.subp.which") @mock.patch("cloudinit.config.cc_disable_ec2_metadata.subp.subp") def test_disable_ifconfig(self, m_subp, m_which): @@ -45,9 +45,10 @@ def test_disable_no_tool(self, m_subp, m_which): """Log error when neither route nor ip commands are available""" m_which.return_value = None # Find neither ifconfig nor ip ec2_meta.handle("foo", DISABLE_CFG, None, None) - self.assertEqual( - [mock.call("ip"), mock.call("ifconfig")], m_which.call_args_list - ) + assert [ + mock.call("ip"), + mock.call("ifconfig"), + ] == m_which.call_args_list m_subp.assert_not_called() diff --git a/tests/unittests/config/test_cc_disk_setup.py b/tests/unittests/config/test_cc_disk_setup.py index 734f5e43..b81a3a97 100644 --- a/tests/unittests/config/test_cc_disk_setup.py +++ b/tests/unittests/config/test_cc_disk_setup.py @@ -3,6 +3,7 @@ import random import tempfile +from contextlib import ExitStack import pytest @@ -12,12 +13,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import ( - CiTestCase, - ExitStack, - mock, - skipUnlessJsonSchema, -) +from tests.unittests.helpers import mock, skipUnlessJsonSchema class TestIsDiskUsed: @@ -108,6 +104,40 @@ def test_thirds_with_different_partition_type(self): ) == cc_disk_setup.get_partition_mbr_layout(disk_size, [33, [66, 82]]) +class TestCheckPartitionLayout: + @mock.patch( + "cloudinit.config.cc_disk_setup.check_partition_mbr_layout", + return_value=["83"], + ) + def test_simple_mbr(self, *args): + assert cc_disk_setup.check_partition_layout("mbr", "/dev/xvdb1", True) + assert cc_disk_setup.check_partition_layout( + "mbr", "/dev/xvdb1", [(100, 83)] + ) + + @mock.patch( + "cloudinit.config.cc_disk_setup.check_partition_gpt_layout", + return_value=["8300"], + ) + def test_simple1_gpt(self, *args): + assert cc_disk_setup.check_partition_layout( + "gpt", "/dev/xvdb1", [(100, 83)] + ) + assert cc_disk_setup.check_partition_layout( + "gpt", "/dev/xvdb1", [(100, 8300)] + ) + assert ( + cc_disk_setup.check_partition_layout( + "gpt", "/dev/xvdb1", [(100, 8301)] + ) + is False + ) + Linux_GUID = "0FC63DAF-8483-4772-8E79-3D69D8477DE4" + assert cc_disk_setup.check_partition_layout( + "gpt", "/dev/xvdb1", [(100, Linux_GUID)] + ) + + class TestUpdateFsSetupDevices: def test_regression_1634678(self): # Cf. https://bugs.launchpad.net/cloud-init/+bug/1634678 @@ -198,10 +228,16 @@ def test_purge_disk_ptable(self, *args): ) @mock.patch("cloudinit.config.cc_disk_setup.device_type", return_value=None) @mock.patch("cloudinit.config.cc_disk_setup.subp.subp", return_value=("", "")) -class TestMkfsCommandHandling(CiTestCase): - with_logs = True - - def test_with_cmd(self, subp, *args): +class TestMkfsCommandHandling: + + def test_with_cmd( + self, + subp, + m_device_type, + m_find_device, + m_assert_and_settle_device, + caplog, + ): """mkfs honors cmd and logs warnings when extra_opts or overwrite are provided.""" cc_disk_setup.mkfs( @@ -218,12 +254,12 @@ def test_with_cmd(self, subp, *args): assert ( "extra_opts " "ignored because cmd was specified: mkfs -t ext4 -L with_cmd " - "/dev/xdb1" in self.logs.getvalue() + "/dev/xdb1" in caplog.text ) assert ( "overwrite " "ignored because cmd was specified: mkfs -t ext4 -L with_cmd " - "/dev/xdb1" in self.logs.getvalue() + "/dev/xdb1" in caplog.text ) subp.assert_called_once_with( diff --git a/tests/unittests/config/test_cc_install_hotplug.py b/tests/unittests/config/test_cc_install_hotplug.py index e6676715..d34d95b0 100644 --- a/tests/unittests/config/test_cc_install_hotplug.py +++ b/tests/unittests/config/test_cc_install_hotplug.py @@ -146,7 +146,7 @@ def test_rules_installed_on_ec2(self, mocks): udev_rules = """\ # Installed by cloud-init due to network hotplug userdata -ACTION!="add|remove", GOTO="cloudinit_end" +ACTION!="add", GOTO="cloudinit_end" ENV{ID_NET_DRIVER}=="vif|ena|ixgbevf", GOTO="cloudinit_hook" GOTO="cloudinit_end" diff --git a/tests/unittests/config/test_cc_keyboard.py b/tests/unittests/config/test_cc_keyboard.py index 7606b350..d0731a54 100644 --- a/tests/unittests/config/test_cc_keyboard.py +++ b/tests/unittests/config/test_cc_keyboard.py @@ -14,11 +14,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import ( - FilesystemMockingTestCase, - populate_dir, - skipUnlessJsonSchema, -) +from tests.unittests.helpers import populate_dir, skipUnlessJsonSchema from tests.unittests.util import get_cloud @@ -82,17 +78,11 @@ def test_schema_validation(self, config, error_msg): validate_cloudconfig_schema(config, schema, strict=True) -class TestKeyboard(FilesystemMockingTestCase): - with_logs = True - - def setUp(self): - super(TestKeyboard, self).setUp() - self.root_d = self.tmp_dir() - self.root_d = self.reRoot() - +@pytest.mark.usefixtures("fake_filesystem") +class TestKeyboard: @mock.patch("cloudinit.distros.Distro.uses_systemd") @mock.patch("cloudinit.distros.subp.subp") - def test_systemd_linux_cmd(self, m_subp, m_uses_systemd, *args): + def test_systemd_linux_cmd(self, m_subp, m_uses_systemd): """Non-Debian systems run localectl""" cfg = {"keyboard": {"layout": "us", "variant": "us"}} layout = "us" @@ -132,7 +122,7 @@ def test_debian_linux_cmd(self, m_subp, m_write_file): ) @mock.patch("cloudinit.distros.subp.subp") - def test_alpine_linux_cmd(self, m_subp, *args): + def test_alpine_linux_cmd(self, m_subp, tmpdir): """Alpine Linux runs setup-keymap""" cfg = {"keyboard": {"layout": "us", "variant": "us"}} layout = "us" @@ -142,14 +132,15 @@ def test_alpine_linux_cmd(self, m_subp, *args): # Create a dummy directory and file for keymap keymap_dir = "/usr/share/bkeymaps/%s" % "us" keymap_file = "%s/%s.bmap.gz" % (keymap_dir, "us") - os.makedirs("%s%s" % (self.root_d, keymap_dir)) - populate_dir(self.root_d, {keymap_file: "# Test\n"}) + + os.makedirs(tmpdir.join(keymap_dir)) + populate_dir(str(tmpdir), {keymap_file: "# Test\n"}) cc_keyboard.handle("cc_keyboard", cfg, cloud, []) m_subp.assert_called_once_with(["setup-keymap", layout, variant]) @mock.patch("cloudinit.distros.subp.subp") - def test_alpine_linux_ignore_model(self, m_subp): + def test_alpine_linux_ignore_model(self, m_subp, caplog, tmpdir): """Alpine Linux ignores model setting""" cfg = { "keyboard": { @@ -164,14 +155,11 @@ def test_alpine_linux_ignore_model(self, m_subp): keymap_dir = "/usr/share/bkeymaps/%s" % "us" keymap_file = "%s/%s.bmap.gz" % (keymap_dir, "us") - os.makedirs("%s%s" % (self.root_d, keymap_dir)) - populate_dir(self.root_d, {keymap_file: "# Test\n"}) + os.makedirs(tmpdir.join(keymap_dir)) + populate_dir(str(tmpdir), {keymap_file: "# Test\n"}) cc_keyboard.handle("cc_keyboard", cfg, cloud, []) - assert ( - "Keyboard model is ignored for Alpine Linux." - in self.logs.getvalue() - ) + assert "Keyboard model is ignored for Alpine Linux." in caplog.text m_subp.assert_called_once_with( [ "setup-keymap", diff --git a/tests/unittests/config/test_cc_landscape.py b/tests/unittests/config/test_cc_landscape.py index bc79da7c..47a89b98 100644 --- a/tests/unittests/config/test_cc_landscape.py +++ b/tests/unittests/config/test_cc_landscape.py @@ -24,8 +24,8 @@ def test_skip_empty_landscape_cloudconfig(self, m_subp): """Empty landscape cloud-config section does no work.""" mycloud = get_cloud() mycloud.distro = mock.MagicMock() - cfg = {"landscape": {}} - cc_landscape.handle("notimportant", cfg, mycloud, None) + cfg: dict[str, dict] = {"landscape": {}} + cc_landscape.handle("notimportant", cfg, mycloud, []) assert mycloud.distro.install_packages.called is False def test_handler_error_on_invalid_landscape_type(self, m_subp): @@ -33,7 +33,7 @@ def test_handler_error_on_invalid_landscape_type(self, m_subp): mycloud = get_cloud("ubuntu") cfg = {"landscape": "wrongtype"} with pytest.raises(RuntimeError) as exc: - cc_landscape.handle("notimportant", cfg, mycloud, None) + cc_landscape.handle("notimportant", cfg, mycloud, []) assert "'landscape' key existed in config, but not a dict" in str( exc.value ) @@ -42,7 +42,7 @@ def test_handler_restarts_landscape_client(self, m_subp, tmpdir): """handler restarts landscape-client after install.""" mycloud = get_cloud("ubuntu") mycloud.distro = mock.MagicMock() - cfg = {"landscape": {"client": {}}} + cfg: dict[str, dict[str, dict]] = {"landscape": {"client": {}}} wrap_and_call( "cloudinit.config.cc_landscape", { @@ -137,7 +137,7 @@ def test_handler_writes_merged_client_config_file_with_defaults( client_fn.write("[client]\ncomputer_title = My PC\n") mycloud = get_cloud("ubuntu") mycloud.distro = mock.MagicMock() - cfg = {"landscape": {"client": {}}} + cfg: dict[str, dict[str, dict]] = {"landscape": {"client": {}}} expected_calls = [ mock.call( ["landscape-config", "--silent", "--is-registered"], rcs=[5] @@ -231,7 +231,7 @@ def test_handler_client_failed_registering(self, m_merge_together, m_subp): "Stdout: Could not register client\nStderr: -" ) with pytest.raises(RuntimeError, match=match): - cc_landscape.handle("notimportant", cfg, mycloud, None) + cc_landscape.handle("notimportant", cfg, mycloud, []) @mock.patch(f"{MPATH}.merge_together") def test_handler_client_is_already_registered( @@ -244,7 +244,7 @@ def test_handler_client_is_already_registered( m_subp.side_effect = subp.ProcessExecutionError( "Client already registered to Landscape", exit_code=0 ) - cc_landscape.handle("notimportant", cfg, mycloud, None) + cc_landscape.handle("notimportant", cfg, mycloud, []) assert "Client already registered to Landscape" in caplog.text diff --git a/tests/unittests/config/test_cc_mcollective.py b/tests/unittests/config/test_cc_mcollective.py index 522b7858..bd164018 100644 --- a/tests/unittests/config/test_cc_mcollective.py +++ b/tests/unittests/config/test_cc_mcollective.py @@ -1,8 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging import os -import shutil -import tempfile from io import BytesIO import configobj @@ -46,22 +44,23 @@ """ -class TestConfig(t_help.FilesystemMockingTestCase): - def setUp(self): - super(TestConfig, self).setUp() - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - # "./": make os.path.join behave correctly with abs path as second arg - self.server_cfg = os.path.join( - self.tmp, "./" + cc_mcollective.SERVER_CFG - ) - self.pubcert_file = os.path.join( - self.tmp, "./" + cc_mcollective.PUBCERT_FILE - ) - self.pricert_file = os.path.join( - self.tmp, self.tmp, "./" + cc_mcollective.PRICERT_FILE - ) +@pytest.fixture +def server_cfg(tmp_path): + return str(tmp_path / cc_mcollective.SERVER_CFG) + + +@pytest.fixture +def pricert_file(tmp_path): + return str(tmp_path / cc_mcollective.PRICERT_FILE) + + +@pytest.fixture +def pubcert_file(tmp_path): + return str(tmp_path / cc_mcollective.PUBCERT_FILE) + +@pytest.mark.usefixtures("fake_filesystem") +class TestConfig: def test_basic_config(self): cfg = { "mcollective": { @@ -84,30 +83,27 @@ def test_basic_config(self): } expected = cfg["mcollective"]["conf"] - self.patchUtils(self.tmp) cc_mcollective.configure(cfg["mcollective"]["conf"]) contents = util.load_binary_file(cc_mcollective.SERVER_CFG) contents = configobj.ConfigObj(BytesIO(contents)) - self.assertEqual(expected, dict(contents)) + assert expected == dict(contents) - def test_existing_config_is_saved(self): + def test_existing_config_is_saved(self, server_cfg): cfg = {"loglevel": "warn"} - util.write_file(self.server_cfg, STOCK_CONFIG) - cc_mcollective.configure(config=cfg, server_cfg=self.server_cfg) - self.assertTrue(os.path.exists(self.server_cfg)) - self.assertTrue(os.path.exists(self.server_cfg + ".old")) - self.assertEqual( - util.load_text_file(self.server_cfg + ".old"), STOCK_CONFIG - ) + util.write_file(server_cfg, STOCK_CONFIG) + cc_mcollective.configure(config=cfg, server_cfg=server_cfg) + assert os.path.exists(server_cfg) + assert os.path.exists(server_cfg + ".old") + assert util.load_text_file(server_cfg + ".old") == STOCK_CONFIG - def test_existing_updated(self): + def test_existing_updated(self, server_cfg): cfg = {"loglevel": "warn"} - util.write_file(self.server_cfg, STOCK_CONFIG) - cc_mcollective.configure(config=cfg, server_cfg=self.server_cfg) - cfgobj = configobj.ConfigObj(self.server_cfg) - self.assertEqual(cfg["loglevel"], cfgobj["loglevel"]) + util.write_file(server_cfg, STOCK_CONFIG) + cc_mcollective.configure(config=cfg, server_cfg=server_cfg) + cfgobj = configobj.ConfigObj(server_cfg) + assert cfg["loglevel"] == cfgobj["loglevel"] - def test_certificats_written(self): + def test_certificats_written(self, pricert_file, pubcert_file, server_cfg): # check public-cert and private-cert keys in config get written cfg = { "loglevel": "debug", @@ -117,30 +113,26 @@ def test_certificats_written(self): cc_mcollective.configure( config=cfg, - server_cfg=self.server_cfg, - pricert_file=self.pricert_file, - pubcert_file=self.pubcert_file, + server_cfg=server_cfg, + pricert_file=pricert_file, + pubcert_file=pubcert_file, ) - found = configobj.ConfigObj(self.server_cfg) + found = configobj.ConfigObj(server_cfg) # make sure these didnt get written in - self.assertFalse("public-cert" in found) - self.assertFalse("private-cert" in found) + assert "public-cert" not in found + assert "private-cert" not in found # these need updating to the specified paths - self.assertEqual(found["plugin.ssl_server_public"], self.pubcert_file) - self.assertEqual(found["plugin.ssl_server_private"], self.pricert_file) + assert found["plugin.ssl_server_public"] == pubcert_file + assert found["plugin.ssl_server_private"] == pricert_file # and the security provider should be ssl - self.assertEqual(found["securityprovider"], "ssl") + assert found["securityprovider"] == "ssl" - self.assertEqual( - util.load_text_file(self.pricert_file), cfg["private-cert"] - ) - self.assertEqual( - util.load_text_file(self.pubcert_file), cfg["public-cert"] - ) + assert util.load_text_file(pricert_file) == cfg["private-cert"] + assert util.load_text_file(pubcert_file) == cfg["public-cert"] class TestHandler(t_help.TestCase): diff --git a/tests/unittests/config/test_cc_mounts.py b/tests/unittests/config/test_cc_mounts.py index f710d8a4..7481a473 100644 --- a/tests/unittests/config/test_cc_mounts.py +++ b/tests/unittests/config/test_cc_mounts.py @@ -241,7 +241,7 @@ def test_swap_creation_method_fallocate_on_xfs( m_kernel_version.return_value = (4, 20) m_get_mount_info.return_value = ["", "xfs"] - cc_mounts.handle(None, self.cc, self.mock_cloud, []) + cc_mounts.handle("", self.cc, self.mock_cloud, []) self.m_subp.assert_has_calls( [ mock.call( @@ -260,7 +260,7 @@ def test_swap_creation_method_xfs( m_kernel_version.return_value = (3, 18) m_get_mount_info.return_value = ["", "xfs"] - cc_mounts.handle(None, self.cc, self.mock_cloud, []) + cc_mounts.handle("", self.cc, self.mock_cloud, []) self.m_subp.assert_has_calls( [ mock.call( @@ -286,7 +286,7 @@ def test_swap_creation_method_btrfs( m_kernel_version.return_value = (4, 20) m_get_mount_info.return_value = ["", "btrfs"] - cc_mounts.handle(None, self.cc, self.mock_cloud, []) + cc_mounts.handle("", self.cc, self.mock_cloud, []) self.m_subp.assert_has_calls( [ mock.call(["truncate", "-s", "0", self.swap_path]), @@ -308,7 +308,7 @@ def test_swap_creation_method_ext4( m_kernel_version.return_value = (5, 14) m_get_mount_info.return_value = ["", "ext4"] - cc_mounts.handle(None, self.cc, self.mock_cloud, []) + cc_mounts.handle("", self.cc, self.mock_cloud, []) self.m_subp.assert_has_calls( [ mock.call( @@ -371,7 +371,7 @@ def test_no_fstab(self): "%s\tnone\tswap\tsw,comment=cloudconfig\t0\t0\n" % (self.swap_path,) ) - cc_mounts.handle(None, {}, self.mock_cloud, []) + cc_mounts.handle("", {}, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() assert fstab_expected_content == fstab_new_content @@ -431,7 +431,7 @@ def test_swap_creation_command(self, fstype, expected, mocker): cc = { "swap": {"filename": "/swap.img", "size": "512", "maxsize": "512"} } - cc_mounts.handle(None, cc, self.mock_cloud, []) + cc_mounts.handle("", cc, self.mock_cloud, []) assert self.m_subp.call_args_list == expected + [ mock.call(["mkswap", "/swap.img"]), mock.call(["swapon", "-a"]), @@ -452,7 +452,7 @@ def test_fstab_no_swap_device(self): with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab_original_content) - cc_mounts.handle(None, {}, self.mock_cloud, []) + cc_mounts.handle("", {}, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() @@ -470,7 +470,7 @@ def test_fstab_same_swap_device_already_configured(self): with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab_original_content) - cc_mounts.handle(None, {}, self.mock_cloud, []) + cc_mounts.handle("", {}, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() @@ -491,7 +491,7 @@ def test_fstab_alternate_swap_device_already_configured(self): with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab_original_content) - cc_mounts.handle(None, {}, self.mock_cloud, []) + cc_mounts.handle("", {}, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() @@ -510,7 +510,7 @@ def test_no_change_fstab_sets_needs_mount_all(self): cc = {"mounts": [["/dev/vdb", "/mnt", "auto", "defaults,noexec"]]} with open(cc_mounts.FSTAB_PATH, "w") as fd: fd.write(fstab_original_content) - cc_mounts.handle(None, cc, self.mock_cloud, []) + cc_mounts.handle("", cc, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() assert fstab_original_content == fstab_new_content.strip() @@ -555,7 +555,7 @@ def test_fstab_mounts_combinations(self): ["/dev/sda3", "/mnt4", "btrfs"], ] } - cc_mounts.handle(None, cfg, self.mock_cloud, []) + cc_mounts.handle("", cfg, self.mock_cloud, []) with open(cc_mounts.FSTAB_PATH, "r") as fd: fstab_new_content = fd.read() diff --git a/tests/unittests/config/test_cc_puppet.py b/tests/unittests/config/test_cc_puppet.py index 47f27f93..22911542 100644 --- a/tests/unittests/config/test_cc_puppet.py +++ b/tests/unittests/config/test_cc_puppet.py @@ -13,7 +13,7 @@ ) from cloudinit.distros import PackageInstallerError from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from tests.unittests.helpers import mock, skipUnlessJsonSchema from tests.unittests.util import get_cloud @@ -25,16 +25,12 @@ def fake_tempdir(mocker, tmpdir): @mock.patch("cloudinit.config.cc_puppet.subp.subp") -class TestManagePuppetServices(CiTestCase): - def setUp(self): - super(TestManagePuppetServices, self).setUp() - self.cloud = get_cloud() - +class TestManagePuppetServices: def test_wb_manage_puppet_services_enables_puppet_systemctl( self, m_subp, ): - cc_puppet._manage_puppet_services(self.cloud, "enable") + cc_puppet._manage_puppet_services(get_cloud(), "enable") expected_calls = [ mock.call( ["systemctl", "enable", "puppet-agent.service"], @@ -42,13 +38,13 @@ def test_wb_manage_puppet_services_enables_puppet_systemctl( rcs=None, ) ] - self.assertIn(expected_calls, m_subp.call_args_list) + assert expected_calls in m_subp.call_args_list def test_wb_manage_puppet_services_starts_puppet_systemctl( self, m_subp, ): - cc_puppet._manage_puppet_services(self.cloud, "start") + cc_puppet._manage_puppet_services(get_cloud(), "start") expected_calls = [ mock.call( ["systemctl", "start", "puppet-agent.service"], @@ -56,11 +52,11 @@ def test_wb_manage_puppet_services_starts_puppet_systemctl( rcs=None, ) ] - self.assertIn(expected_calls, m_subp.call_args_list) + assert expected_calls in m_subp.call_args_list def test_enable_fallback_on_failure(self, m_subp): m_subp.side_effect = (ProcessExecutionError, 0) - cc_puppet._manage_puppet_services(self.cloud, "enable") + cc_puppet._manage_puppet_services(get_cloud(), "enable") expected_calls = [ mock.call( ["systemctl", "enable", "puppet-agent.service"], @@ -73,64 +69,60 @@ def test_enable_fallback_on_failure(self, m_subp): rcs=None, ), ] - self.assertEqual(expected_calls, m_subp.call_args_list) + assert expected_calls == m_subp.call_args_list +@pytest.mark.usefixtures("fake_filesystem") @mock.patch("cloudinit.config.cc_puppet._manage_puppet_services") -class TestPuppetHandle(CiTestCase): - with_logs = True - - def setUp(self): - super(TestPuppetHandle, self).setUp() - self.new_root = self.tmp_dir() - self.conf = self.tmp_path("puppet.conf") - self.csr_attributes_path = self.tmp_path("csr_attributes.yaml") - self.cloud = get_cloud() +class TestPuppetHandle: + CONF = "puppet.conf" + CSR_ATTRIBUTES_PATH = "csr_attributes.yaml" - def test_skips_missing_puppet_key_in_cloudconfig(self, m_man_puppet): + def test_skips_missing_puppet_key_in_cloudconfig( + self, m_man_puppet, caplog + ): """Cloud-config containing no 'puppet' key is skipped.""" cfg = {} - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertIn("no 'puppet' configuration found", self.logs.getvalue()) - self.assertEqual(0, m_man_puppet.call_count) + cc_puppet.handle("notimportant", cfg, get_cloud(), None) + assert "no 'puppet' configuration found" in caplog.text + assert 0 == m_man_puppet.call_count @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_config_starts_puppet_service(self, m_subp, m_man_puppet): """Cloud-config 'puppet' configuration starts puppet.""" + cloud = get_cloud() cfg = {"puppet": {"install": False}} - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual(2, m_man_puppet.call_count) + cc_puppet.handle("notimportant", cfg, cloud, None) + assert 2 == m_man_puppet.call_count expected_calls = [ - mock.call(self.cloud, "enable"), - mock.call(self.cloud, "start"), + mock.call(cloud, "enable"), + mock.call(cloud, "start"), ] - self.assertEqual(expected_calls, m_man_puppet.call_args_list) + assert expected_calls == m_man_puppet.call_args_list @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_empty_puppet_config_installs_puppet(self, m_subp, m_man_puppet): """Cloud-config empty 'puppet' configuration installs latest puppet.""" - - self.cloud.distro = mock.MagicMock() + cloud = get_cloud() + cloud.distro = mock.MagicMock() cfg = {"puppet": {}} - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual( - [mock.call(["puppet-agent"])], - self.cloud.distro.install_packages.call_args_list, - ) + cc_puppet.handle("notimportant", cfg, cloud, None) + assert [ + mock.call(["puppet-agent"]) + ] == cloud.distro.install_packages.call_args_list @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_config_installs_puppet_on_true(self, m_subp, _): """Cloud-config with 'puppet' key installs when 'install' is True.""" - - self.cloud.distro = mock.MagicMock() + cloud = get_cloud() + cloud.distro = mock.MagicMock() cfg = {"puppet": {"install": True}} - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertIn( - [mock.call(["puppet-agent"])], - self.cloud.distro.install_packages.call_args_list, - ) + cc_puppet.handle("notimportant", cfg, cloud, None) + assert [ + mock.call(["puppet-agent"]) + ] in cloud.distro.install_packages.call_args_list @mock.patch("cloudinit.config.cc_puppet.install_puppet_aio", autospec=True) @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) @@ -138,9 +130,10 @@ def test_puppet_config_installs_puppet_aio(self, m_subp, m_aio, _): """Cloud-config with 'puppet' key installs when 'install_type' is 'aio'.""" distro = mock.MagicMock() - self.cloud.distro = distro + cloud = get_cloud() + cloud.distro = distro cfg = {"puppet": {"install": True, "install_type": "aio"}} - cc_puppet.handle("notimportant", cfg, self.cloud, None) + cc_puppet.handle("notimportant", cfg, cloud, None) m_aio.assert_called_with( distro, cc_puppet.AIO_INSTALL_URL, None, None, True ) @@ -153,7 +146,8 @@ def test_puppet_config_installs_puppet_aio_with_version( """Cloud-config with 'puppet' key installs when 'install_type' is 'aio' and 'version' is specified.""" distro = mock.MagicMock() - self.cloud.distro = distro + cloud = get_cloud() + cloud.distro = distro cfg = { "puppet": { "install": True, @@ -161,7 +155,7 @@ def test_puppet_config_installs_puppet_aio_with_version( "install_type": "aio", } } - cc_puppet.handle("notimportant", cfg, self.cloud, None) + cc_puppet.handle("notimportant", cfg, cloud, None) m_aio.assert_called_with( distro, cc_puppet.AIO_INSTALL_URL, "6.24.0", None, True ) @@ -174,7 +168,8 @@ def test_puppet_config_installs_puppet_aio_with_collection( """Cloud-config with 'puppet' key installs when 'install_type' is 'aio' and 'collection' is specified.""" distro = mock.MagicMock() - self.cloud.distro = distro + cloud = get_cloud() + cloud.distro = distro cfg = { "puppet": { "install": True, @@ -182,7 +177,7 @@ def test_puppet_config_installs_puppet_aio_with_collection( "install_type": "aio", } } - cc_puppet.handle("notimportant", cfg, self.cloud, None) + cc_puppet.handle("notimportant", cfg, cloud, None) m_aio.assert_called_with( distro, cc_puppet.AIO_INSTALL_URL, None, "puppet6", True ) @@ -195,7 +190,8 @@ def test_puppet_config_installs_puppet_aio_with_custom_url( """Cloud-config with 'puppet' key installs when 'install_type' is 'aio' and 'aio_install_url' is specified.""" distro = mock.MagicMock() - self.cloud.distro = distro + cloud = get_cloud() + cloud.distro = distro cfg = { "puppet": { "install": True, @@ -203,7 +199,7 @@ def test_puppet_config_installs_puppet_aio_with_custom_url( "install_type": "aio", } } - cc_puppet.handle("notimportant", cfg, self.cloud, None) + cc_puppet.handle("notimportant", cfg, cloud, None) m_aio.assert_called_with( distro, "http://test.url/path/to/script.sh", None, None, True ) @@ -216,7 +212,8 @@ def test_puppet_config_installs_puppet_aio_without_cleanup( """Cloud-config with 'puppet' key installs when 'install_type' is 'aio' and no cleanup.""" distro = mock.MagicMock() - self.cloud.distro = distro + cloud = get_cloud() + cloud.distro = distro cfg = { "puppet": { "install": True, @@ -224,7 +221,7 @@ def test_puppet_config_installs_puppet_aio_without_cleanup( "install_type": "aio", } } - cc_puppet.handle("notimportant", cfg, self.cloud, None) + cc_puppet.handle("notimportant", cfg, cloud, None) m_aio.assert_called_with( distro, cc_puppet.AIO_INSTALL_URL, None, None, False ) @@ -232,14 +229,13 @@ def test_puppet_config_installs_puppet_aio_without_cleanup( @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_config_installs_puppet_version(self, m_subp, _): """Cloud-config 'puppet' configuration can specify a version.""" - - self.cloud.distro = mock.MagicMock() + cloud = get_cloud() + cloud.distro = mock.MagicMock() cfg = {"puppet": {"version": "3.8"}} - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual( - [mock.call([["puppet-agent", "3.8"]])], - self.cloud.distro.install_packages.call_args_list, - ) + cc_puppet.handle("notimportant", cfg, cloud, None) + assert [ + mock.call([["puppet-agent", "3.8"]]) + ] == cloud.distro.install_packages.call_args_list @mock.patch("cloudinit.config.cc_puppet.get_config_value") @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) @@ -249,7 +245,7 @@ def test_puppet_config_updates_puppet_conf( """When 'conf' is provided update values in PUPPET_CONF_PATH.""" def _fake_get_config_value(puppet_bin, setting): - return self.conf + return self.CONF m_default.side_effect = _fake_get_config_value @@ -258,12 +254,13 @@ def _fake_get_config_value(puppet_bin, setting): "conf": {"agent": {"server": "puppetserver.example.org"}} } } - util.write_file(self.conf, "[agent]\nserver = origpuppet\nother = 3") - self.cloud.distro = mock.MagicMock() - cc_puppet.handle("notimportant", cfg, self.cloud, None) - content = util.load_text_file(self.conf) + util.write_file(self.CONF, "[agent]\nserver = origpuppet\nother = 3") + cloud = get_cloud() + cloud.distro = mock.MagicMock() + cc_puppet.handle("notimportant", cfg, cloud, None) + content = util.load_text_file(self.CONF) expected = "[agent]\nserver = puppetserver.example.org\nother = 3\n\n" - self.assertEqual(expected, content) + assert expected == content @mock.patch("cloudinit.config.cc_puppet.get_config_value") @mock.patch("cloudinit.config.cc_puppet.subp.subp") @@ -274,11 +271,11 @@ def test_puppet_writes_csr_attributes_file( creates file in PUPPET_CSR_ATTRIBUTES_PATH.""" def _fake_get_config_value(puppet_bin, setting): - return self.csr_attributes_path + return self.CSR_ATTRIBUTES_PATH m_default.side_effect = _fake_get_config_value - self.cloud.distro = mock.MagicMock() + get_cloud().distro = mock.MagicMock() cfg = { "puppet": { "csr_attributes": { @@ -297,8 +294,8 @@ def _fake_get_config_value(puppet_bin, setting): } } } - cc_puppet.handle("notimportant", cfg, self.cloud, None) - content = util.load_text_file(self.csr_attributes_path) + cc_puppet.handle("notimportant", cfg, get_cloud(), None) + content = util.load_text_file(self.CSR_ATTRIBUTES_PATH) expected = textwrap.dedent( """\ custom_attributes: @@ -309,49 +306,47 @@ def _fake_get_config_value(puppet_bin, setting): pp_uuid: ED803750-E3C7-44F5-BB08-41A04433FE2E """ ) - self.assertEqual(expected, content) + assert expected == content @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_runs_puppet_if_requested(self, m_subp, m_man_puppet): """Run puppet with default args if 'exec' is set to True.""" - + cloud = get_cloud() cfg = {"puppet": {"exec": True}} - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual(2, m_man_puppet.call_count) + cc_puppet.handle("notimportant", cfg, cloud, None) + assert 2 == m_man_puppet.call_count expected_calls = [ - mock.call(self.cloud, "enable"), - mock.call(self.cloud, "start"), + mock.call(cloud, "enable"), + mock.call(cloud, "start"), ] - self.assertEqual(expected_calls, m_man_puppet.call_args_list) - self.assertIn( - [mock.call(["puppet", "agent", "--test"], capture=False)], - m_subp.call_args_list, - ) + assert expected_calls == m_man_puppet.call_args_list + assert [ + mock.call(["puppet", "agent", "--test"], capture=False) + ] in m_subp.call_args_list @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_starts_puppetd(self, m_subp, m_man_puppet): """Run puppet with default args if 'exec' is set to True.""" - + cloud = get_cloud() cfg = {"puppet": {}} - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual(2, m_man_puppet.call_count) + cc_puppet.handle("notimportant", cfg, cloud, None) + assert 2 == m_man_puppet.call_count expected_calls = [ - mock.call(self.cloud, "enable"), - mock.call(self.cloud, "start"), + mock.call(cloud, "enable"), + mock.call(cloud, "start"), ] - self.assertEqual(expected_calls, m_man_puppet.call_args_list) + assert expected_calls == m_man_puppet.call_args_list @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_skips_puppetd(self, m_subp, m_man_puppet): """Run puppet with default args if 'exec' is set to True.""" cfg = {"puppet": {"start_service": False}} - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual(0, m_man_puppet.call_count) - self.assertNotIn( - [mock.call(["systemctl", "start", "puppet-agent"], capture=False)], - m_subp.call_args_list, - ) + cc_puppet.handle("notimportant", cfg, get_cloud(), None) + assert 0 == m_man_puppet.call_count + assert [ + mock.call(["systemctl", "start", "puppet-agent"], capture=False) + ] not in m_subp.call_args_list @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_runs_puppet_with_args_list_if_requested( @@ -365,17 +360,14 @@ def test_puppet_runs_puppet_with_args_list_if_requested( "exec_args": ["--onetime", "--detailed-exitcodes"], } } - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual(2, m_man_puppet.call_count) - self.assertIn( - [ - mock.call( - ["puppet", "agent", "--onetime", "--detailed-exitcodes"], - capture=False, - ) - ], - m_subp.call_args_list, - ) + cc_puppet.handle("notimportant", cfg, get_cloud(), None) + assert 2 == m_man_puppet.call_count + assert [ + mock.call( + ["puppet", "agent", "--onetime", "--detailed-exitcodes"], + capture=False, + ) + ] in m_subp.call_args_list @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_runs_puppet_with_args_string_if_requested( @@ -389,17 +381,14 @@ def test_puppet_runs_puppet_with_args_string_if_requested( "exec_args": "--onetime --detailed-exitcodes", } } - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual(2, m_man_puppet.call_count) - self.assertIn( - [ - mock.call( - ["puppet", "agent", "--onetime", "--detailed-exitcodes"], - capture=False, - ) - ], - m_subp.call_args_list, - ) + cc_puppet.handle("notimportant", cfg, get_cloud(), None) + assert 2 == m_man_puppet.call_count + assert [ + mock.call( + ["puppet", "agent", "--onetime", "--detailed-exitcodes"], + capture=False, + ) + ] in m_subp.call_args_list @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_falls_back_to_older_name(self, m_subp, m_man_puppet): @@ -410,12 +399,13 @@ def test_puppet_falls_back_to_older_name(self, m_subp, m_man_puppet): # puppet-agent not installed, but puppet is install_pkg.side_effect = (PackageInstallerError, 0) - cc_puppet.handle("notimportant", cfg, self.cloud, None) + cloud = get_cloud() + cc_puppet.handle("notimportant", cfg, cloud, None) expected_calls = [ - mock.call(self.cloud, "enable"), - mock.call(self.cloud, "start"), + mock.call(cloud, "enable"), + mock.call(cloud, "start"), ] - self.assertEqual(expected_calls, m_man_puppet.call_args_list) + assert expected_calls == m_man_puppet.call_args_list @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_with_conf_package_name_fails(self, m_subp, m_man_puppet): @@ -426,22 +416,17 @@ def test_puppet_with_conf_package_name_fails(self, m_subp, m_man_puppet): # puppet-agent not installed, but puppet is install_pkg.side_effect = (ProcessExecutionError, 0) with pytest.raises(ProcessExecutionError): - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual(0, m_man_puppet.call_count) - self.assertNotIn( - [ - mock.call( - ["systemctl", "start", "puppet-agent"], capture=True - ) - ], - m_subp.call_args_list, - ) + cc_puppet.handle("notimportant", cfg, get_cloud(), None) + assert 0 == m_man_puppet.call_count + assert [ + mock.call(["systemctl", "start", "puppet-agent"], capture=True) + ] not in m_subp.call_args_list @mock.patch("cloudinit.config.cc_puppet.subp.subp", return_value=("", "")) def test_puppet_with_conf_package_name_success(self, m_subp, m_man_puppet): cfg = {"puppet": {"package_name": "puppet"}} - cc_puppet.handle("notimportant", cfg, self.cloud, None) - self.assertEqual(2, m_man_puppet.call_count) + cc_puppet.handle("notimportant", cfg, get_cloud(), None) + assert 2 == m_man_puppet.call_count URL_MOCK = mock.Mock() diff --git a/tests/unittests/config/test_cc_raspberry_pi.py b/tests/unittests/config/test_cc_raspberry_pi.py new file mode 100644 index 00000000..c0bfc859 --- /dev/null +++ b/tests/unittests/config/test_cc_raspberry_pi.py @@ -0,0 +1,215 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import pytest + +import cloudinit.config.cc_raspberry_pi as cc_rpi +from cloudinit.config.cc_raspberry_pi import ( + ENABLE_RPI_CONNECT_KEY, + RPI_BASE_KEY, + RPI_INTERFACES_KEY, +) +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from cloudinit.subp import ProcessExecutionError +from tests.unittests.helpers import mock, skipUnlessJsonSchema +from tests.unittests.util import get_cloud + +M_PATH = "cloudinit.config.cc_raspberry_pi." + + +class TestHandleRaspberryPi: + @mock.patch(M_PATH + "configure_rpi_connect") + def test_handle_rpi_connect_enabled(self, m_connect): + cloud = get_cloud("raspberry_pi_os") + cfg = {RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: True}} + cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) + m_connect.assert_called_once_with(True) + + @mock.patch(M_PATH + "configure_interface") + def test_handle_configure_interface_i2c(self, m_iface): + cloud = get_cloud("raspberry_pi_os") + cfg = {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"i2c": True}}} + cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) + m_iface.assert_called_once_with("i2c", True) + + @mock.patch(M_PATH + "configure_serial_interface") + @mock.patch(M_PATH + "is_pifive", return_value=True) + def test_handle_configure_serial_interface_dict(self, m_ispi5, m_serial): + cloud = get_cloud("raspberry_pi_os") + serial_value = { + "console": True, + "hardware": True, + } + cfg = {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"serial": serial_value}}} + cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) + m_serial.assert_called_once_with(serial_value, cfg, cloud) + + @mock.patch(M_PATH + "configure_serial_interface") + @mock.patch(M_PATH + "is_pifive", return_value=True) + def test_handle_configure_serial_interface_bool(self, m_ispi5, m_serial): + cloud = get_cloud("raspberry_pi_os") + cfg = {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"serial": True}}} + cc_rpi.handle("cc_raspberry_pi", cfg, cloud, []) + m_serial.assert_called_once_with(True, cfg, cloud) + + +class TestRaspberryPiMethods: + @mock.patch("cloudinit.subp.subp") + def test_configure_rpi_connect_enable(self, m_subp): + cc_rpi.configure_rpi_connect(True) + m_subp.assert_called_once_with( + ["/usr/bin/raspi-config", "do_rpi_connect", "0"] + ) + + @mock.patch( + "cloudinit.subp.subp", + side_effect=ProcessExecutionError("1", [], "fail"), + ) + def test_configure_rpi_connect_failure(self, m_subp): + cc_rpi.configure_rpi_connect(False) # Should log error but not raise + + @mock.patch("cloudinit.subp.subp", return_value=("ok", "")) + def test_is_pifive_true(self, m_subp): + assert cc_rpi.is_pifive() is True + + @mock.patch( + "cloudinit.subp.subp", + side_effect=ProcessExecutionError("1", [], "fail"), + ) + def test_is_pifive_false(self, m_subp): + assert cc_rpi.is_pifive() is False + + @mock.patch("cloudinit.subp.subp") + def test_configure_interface_valid(self, m_subp): + cc_rpi.configure_interface("i2c", True) + m_subp.assert_called_once_with( + ["/usr/bin/raspi-config", "nonint", "do_i2c", "0"] + ) + + def test_configure_interface_invalid(self): + with pytest.raises(AssertionError): + cc_rpi.configure_interface("invalid_iface", True) + + @mock.patch("cloudinit.subp.subp") + @mock.patch(M_PATH + "is_pifive", return_value=True) + def test_configure_serial_interface_dict_config(self, m_ispi5, m_subp): + cloud = get_cloud("raspberry_pi_os") + cfg = {"console": True, "hardware": False} + + # Simulate is_pifive returning True to prevent enable_hw override + with mock.patch.object( + cloud.distro, "shutdown_command", return_value=["reboot"] + ): + cc_rpi.configure_serial_interface(cfg, {}, cloud) + + expected_calls = [ + mock.call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_CONS_FN, + "0", + ] + ), + mock.call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_HW_FN, + "1", + ] + ), + mock.call(["reboot"]), + ] + m_subp.assert_has_calls(expected_calls, any_order=False) + + @mock.patch("cloudinit.subp.subp") + @mock.patch(M_PATH + "is_pifive", return_value=False) + def test_configure_serial_interface_boolean_config_non_pi5( + self, m_ispi5, m_subp + ): + cloud = get_cloud("raspberry_pi_os") + + with mock.patch.object( + cloud.distro, + "shutdown_command", + return_value=["shutdown", "-r", "now"], + ): + cc_rpi.configure_serial_interface(True, {}, cloud) + + expected_calls = [ + mock.call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_CONS_FN, + "0", + ] + ), + mock.call( + [ + "/usr/bin/raspi-config", + "nonint", + cc_rpi.RASPI_CONFIG_SERIAL_HW_FN, + "0", + ] + ), + mock.call(["shutdown", "-r", "now"]), + ] + m_subp.assert_has_calls(expected_calls, any_order=False) + + +@skipUnlessJsonSchema() +class TestRaspberryPiSchema: + @pytest.mark.parametrize( + "config, error_msg", + [ + ( + { + RPI_BASE_KEY: { + RPI_INTERFACES_KEY: {"spi": True, "i2c": False} + } + }, + None, + ), + ( + {RPI_BASE_KEY: {RPI_INTERFACES_KEY: {"spi": "true"}}}, + f"{RPI_BASE_KEY}.{RPI_INTERFACES_KEY}.spi: 'true'" + " is not of type 'boolean'", + ), + ( + { + RPI_BASE_KEY: { + RPI_INTERFACES_KEY: { + "serial": {"console": True, "hardware": False} + } + } + }, + None, + ), + ( + { + RPI_BASE_KEY: { + RPI_INTERFACES_KEY: {"serial": {"console": 123}} + } + }, + f"{RPI_BASE_KEY}.{RPI_INTERFACES_KEY}.serial.console: " + "123 is not of type 'boolean'", + ), + ({RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: True}}, None), + ( + {RPI_BASE_KEY: {ENABLE_RPI_CONNECT_KEY: "true"}}, + f"{RPI_BASE_KEY}.{ENABLE_RPI_CONNECT_KEY}: 'true'" + " is not of type 'boolean'", + ), + ], + ) + def test_schema_validation(self, config, error_msg): + if error_msg is None: + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_cc_resizefs.py b/tests/unittests/config/test_cc_resizefs.py index 5a3057f4..612f76f7 100644 --- a/tests/unittests/config/test_cc_resizefs.py +++ b/tests/unittests/config/test_cc_resizefs.py @@ -24,7 +24,6 @@ ) from cloudinit.subp import ProcessExecutionError, SubpResult from tests.unittests.helpers import ( - CiTestCase, mock, readResource, skipUnlessJsonSchema, @@ -36,9 +35,7 @@ M_PATH = "cloudinit.config.cc_resizefs." -class TestResizefs(CiTestCase): - with_logs = True - +class TestResizefs: def setUp(self): super(TestResizefs, self).setUp() self.name = "resizefs" @@ -55,7 +52,7 @@ def test_skip_ufs_resize(self, m_subp): exception = ProcessExecutionError(stderr=err, exit_code=1) m_subp.side_effect = exception res = can_skip_resize(fs_type, resize_what, devpth) - self.assertTrue(res) + assert res @mock.patch("cloudinit.subp.subp") def test_cannot_skip_ufs_resize(self, m_subp): @@ -68,7 +65,7 @@ def test_cannot_skip_ufs_resize(self, m_subp): "leaving 364KB unused\n", ) res = can_skip_resize(fs_type, resize_what, devpth) - self.assertFalse(res) + assert not res @mock.patch("cloudinit.subp.subp") def test_cannot_skip_ufs_growfs_exception(self, m_subp): @@ -78,39 +75,41 @@ def test_cannot_skip_ufs_growfs_exception(self, m_subp): err = "growfs: /dev/da0p2 is not clean - run fsck.\n" exception = ProcessExecutionError(stderr=err, exit_code=1) m_subp.side_effect = exception - with self.assertRaises(ProcessExecutionError): + with pytest.raises(ProcessExecutionError): can_skip_resize(fs_type, resize_what, devpth) def test_can_skip_resize_ext(self): - self.assertFalse(can_skip_resize("ext", "/", "/dev/sda1")) + assert not can_skip_resize("ext", "/", "/dev/sda1") - def test_handle_noops_on_disabled(self): + def test_handle_noops_on_disabled(self, caplog): """The handle function logs when the configuration disables resize.""" cfg = {"resize_rootfs": False} handle("cc_resizefs", cfg, cloud=None, args=[]) - self.assertIn( - "DEBUG: Skipping module named cc_resizefs, resizing disabled\n", - self.logs.getvalue(), - ) + assert ( + mock.ANY, + logging.DEBUG, + "Skipping module named cc_resizefs, resizing disabled", + ) in caplog.record_tuples @mock.patch("cloudinit.config.cc_resizefs.util.get_mount_info") @mock.patch("cloudinit.config.cc_resizefs.LOG") - def test_handle_warns_on_unknown_mount_info(self, m_log, m_get_mount_info): + def test_handle_warns_on_unknown_mount_info( + self, m_log, m_get_mount_info, caplog + ): """handle warns when get_mount_info sees unknown filesystem for /.""" m_get_mount_info.return_value = None cfg = {"resize_rootfs": True} handle("cc_resizefs", cfg, cloud=None, args=[]) - logs = self.logs.getvalue() - self.assertNotIn( - "WARNING: Invalid cloud-config provided:\nresize_rootfs:", logs - ) - self.assertEqual( - ("Could not determine filesystem type of %s", "/"), - m_log.warning.call_args[0], - ) - self.assertEqual( - [mock.call("/", m_log)], m_get_mount_info.call_args_list + logs = caplog.text + assert ( + "WARNING: Invalid cloud-config provided:\nresize_rootfs:" + not in logs ) + assert ( + "Could not determine filesystem type of %s", + "/", + ) == m_log.warning.call_args[0] + assert [mock.call("/", m_log)] == m_get_mount_info.call_args_list @mock.patch("cloudinit.config.cc_resizefs.LOG") def test_handle_warns_on_undiscoverable_root_path_in_command_line( @@ -121,8 +120,8 @@ def test_handle_warns_on_undiscoverable_root_path_in_command_line( exists_mock_path = "cloudinit.config.cc_resizefs.os.path.exists" def fake_mount_info(path, log): - self.assertEqual("/", path) - self.assertEqual(m_log, log) + assert "/" == path + assert m_log == log return ("/dev/root", "ext4", "/") with mock.patch(exists_mock_path) as m_exists: @@ -140,45 +139,39 @@ def fake_mount_info(path, log): cloud=None, args=[], ) - self.assertIn( - "Unable to find device '/dev/root'", m_log.warning.call_args[0] + assert ( + "Unable to find device '/dev/root'" in m_log.warning.call_args[0] ) def test_resize_zfs_cmd_return(self): zpool = "zroot" devpth = "gpt/system" - self.assertEqual( - ("zpool", "online", "-e", zpool, devpth), - _resize_zfs(zpool, devpth), + assert ("zpool", "online", "-e", zpool, devpth) == _resize_zfs( + zpool, devpth ) def test_resize_xfs_cmd_return(self): mount_point = "/mnt/test" devpth = "/dev/sda1" - self.assertEqual( - ("xfs_growfs", mount_point), _resize_xfs(mount_point, devpth) - ) + assert ("xfs_growfs", mount_point) == _resize_xfs(mount_point, devpth) def test_resize_ext_cmd_return(self): mount_point = "/" devpth = "/dev/sdb1" - self.assertEqual( - ("resize2fs", devpth), _resize_ext(mount_point, devpth) - ) + assert ("resize2fs", devpth) == _resize_ext(mount_point, devpth) def test_resize_ufs_cmd_return(self): mount_point = "/" devpth = "/dev/sda2" - self.assertEqual( - ("growfs", "-y", mount_point), _resize_ufs(mount_point, devpth) + assert ("growfs", "-y", mount_point) == _resize_ufs( + mount_point, devpth ) def test_resize_bcachefs_cmd_return(self): mount_point = "/" devpth = "/dev/sdf3" - self.assertEqual( - ("bcachefs", "device", "resize", devpth), - _resize_bcachefs(mount_point, devpth), + assert ("bcachefs", "device", "resize", devpth) == _resize_bcachefs( + mount_point, devpth ) @mock.patch("cloudinit.util.is_container", return_value=False) @@ -203,7 +196,7 @@ def test_handle_zfs_root( handle("cc_resizefs", cfg, cloud=None, args=[]) ret = dresize.call_args[0] - self.assertEqual((("zpool", "online", "-e", "vmzroot", disk),), ret) + assert (("zpool", "online", "-e", "vmzroot", disk),) == ret @mock.patch("cloudinit.util.is_container", return_value=False) @mock.patch("cloudinit.util.get_mount_info") @@ -235,13 +228,12 @@ def fake_stat(devpath): with mock.patch("cloudinit.config.cc_resizefs.os.stat") as m_stat: m_stat.side_effect = fake_stat handle("cc_resizefs", cfg, cloud=None, args=[]) - self.assertEqual( - (("zpool", "online", "-e", "zroot", "/dev/" + disk),), - dresize.call_args[0], - ) + assert ( + ("zpool", "online", "-e", "zroot", "/dev/" + disk), + ) == dresize.call_args[0] -class TestRootDevFromCmdline(CiTestCase): +class TestRootDevFromCmdline: def test_rootdev_from_cmdline_with_no_root(self): """Return None from rootdev_from_cmdline when root is not present.""" invalid_cases = [ @@ -250,40 +242,32 @@ def test_rootdev_from_cmdline_with_no_root(self): "", ] for case in invalid_cases: - self.assertIsNone(util.rootdev_from_cmdline(case)) + assert util.rootdev_from_cmdline(case) is None def test_rootdev_from_cmdline_with_root_startswith_dev(self): """Return the cmdline root when the path starts with /dev.""" - self.assertEqual( - "/dev/this", util.rootdev_from_cmdline("asdf root=/dev/this") - ) + assert "/dev/this" == util.rootdev_from_cmdline("asdf root=/dev/this") def test_rootdev_from_cmdline_with_root_without_dev_prefix(self): """Add /dev prefix to cmdline root when the path lacks the prefix.""" - self.assertEqual( - "/dev/this", util.rootdev_from_cmdline("asdf root=this") - ) + assert "/dev/this" == util.rootdev_from_cmdline("asdf root=this") def test_rootdev_from_cmdline_with_root_with_label(self): """When cmdline root contains a LABEL, our root is disk/by-label.""" - self.assertEqual( - "/dev/disk/by-label/unique", - util.rootdev_from_cmdline("asdf root=LABEL=unique"), + assert "/dev/disk/by-label/unique" == util.rootdev_from_cmdline( + "asdf root=LABEL=unique" ) def test_rootdev_from_cmdline_with_root_with_uuid(self): """When cmdline root contains a UUID, our root is disk/by-uuid.""" - self.assertEqual( - "/dev/disk/by-uuid/adsfdsaf-adsf", - util.rootdev_from_cmdline("asdf root=UUID=adsfdsaf-adsf"), + assert "/dev/disk/by-uuid/adsfdsaf-adsf" == util.rootdev_from_cmdline( + "asdf root=UUID=adsfdsaf-adsf" ) -class TestMaybeGetDevicePathAsWritableBlock(CiTestCase): - - with_logs = True - - def test_maybe_get_writable_device_path_none_on_overlayroot(self): +@pytest.mark.usefixtures("fake_filesystem") +class TestMaybeGetDevicePathAsWritableBlock: + def test_maybe_get_writable_device_path_none_on_overlayroot(self, caplog): """When devpath is overlayroot (on MAAS), is_dev_writable is False.""" info = "does not matter" devpath = wrap_and_call( @@ -293,19 +277,18 @@ def test_maybe_get_writable_device_path_none_on_overlayroot(self): "overlayroot", info, ) - self.assertIsNone(devpath) - self.assertIn( - "Not attempting to resize devpath 'overlayroot'", - self.logs.getvalue(), - ) + assert devpath is None + assert "Not attempting to resize devpath 'overlayroot'" in caplog.text - def test_maybe_get_writable_device_path_warns_missing_cmdline_root(self): + def test_maybe_get_writable_device_path_warns_missing_cmdline_root( + self, caplog + ): """When root does not exist isn't in the cmdline, log warning.""" info = "does not matter" def fake_mount_info(path, log): - self.assertEqual("/", path) - self.assertEqual(LOG, log) + assert "/" == path + assert LOG == log return ("/dev/root", "ext4", "/") exists_mock_path = "cloudinit.config.cc_resizefs.os.path.exists" @@ -322,11 +305,14 @@ def fake_mount_info(path, log): "/dev/root", info, ) - self.assertIsNone(devpath) - logs = self.logs.getvalue() - self.assertIn("WARNING: Unable to find device '/dev/root'", logs) - - def test_maybe_get_writable_device_path_does_not_exist(self): + assert devpath is None + assert ( + mock.ANY, + logging.WARNING, + "Unable to find device '/dev/root'", + ) in caplog.record_tuples + + def test_maybe_get_writable_device_path_does_not_exist(self, caplog): """When devpath does not exist, a warning is logged.""" info = "dev=/dev/I/dont/exist mnt_point=/ path=/dev/none" devpath = wrap_and_call( @@ -336,14 +322,17 @@ def test_maybe_get_writable_device_path_does_not_exist(self): "/dev/I/dont/exist", info, ) - self.assertIsNone(devpath) - self.assertIn( - "WARNING: Device '/dev/I/dont/exist' did not exist." - " cannot resize: %s" % info, - self.logs.getvalue(), - ) + assert devpath is None + assert ( + mock.ANY, + logging.WARNING, + "Device '/dev/I/dont/exist' did not exist. cannot resize: %s" + % info, + ) in caplog.record_tuples - def test_maybe_get_writable_device_path_does_not_exist_in_container(self): + def test_maybe_get_writable_device_path_does_not_exist_in_container( + self, caplog + ): """When devpath does not exist in a container, log a debug message.""" info = "dev=/dev/I/dont/exist mnt_point=/ path=/dev/none" devpath = wrap_and_call( @@ -353,17 +342,18 @@ def test_maybe_get_writable_device_path_does_not_exist_in_container(self): "/dev/I/dont/exist", info, ) - self.assertIsNone(devpath) - self.assertIn( - "DEBUG: Device '/dev/I/dont/exist' did not exist in container." - " cannot resize: %s" % info, - self.logs.getvalue(), - ) + assert devpath is None + assert ( + mock.ANY, + logging.DEBUG, + "Device '/dev/I/dont/exist' did not exist in container. cannot" + " resize: %s" % info, + ) in caplog.record_tuples def test_maybe_get_writable_device_path_raises_oserror(self): """When unexpected OSError is raises by os.stat it is reraised.""" info = "dev=/dev/I/dont/exist mnt_point=/ path=/dev/none" - with self.assertRaises(OSError) as context_manager: + with pytest.raises(OSError, match="Something unexpected"): wrap_and_call( "cloudinit.config.cc_resizefs", { @@ -376,13 +366,10 @@ def test_maybe_get_writable_device_path_raises_oserror(self): "/dev/I/dont/exist", info, ) - self.assertEqual( - "Something unexpected", str(context_manager.exception) - ) - def test_maybe_get_writable_device_path_non_block(self): + def test_maybe_get_writable_device_path_non_block(self, caplog): """When device is not a block device, emit warning return False.""" - fake_devpath = self.tmp_path("dev/readwrite") + fake_devpath = "dev/readwrite" util.write_file(fake_devpath, "", mode=0o600) # read-write info = "dev=/dev/root mnt_point=/ path={0}".format(fake_devpath) @@ -393,17 +380,20 @@ def test_maybe_get_writable_device_path_non_block(self): fake_devpath, info, ) - self.assertIsNone(devpath) - self.assertIn( - "WARNING: device '{0}' not a block device. cannot resize".format( - fake_devpath + assert devpath is None + assert ( + mock.ANY, + logging.WARNING, + "device '{0}' not a block device. cannot resize: {1}".format( + fake_devpath, info ), - self.logs.getvalue(), - ) + ) in caplog.record_tuples - def test_maybe_get_writable_device_path_non_block_on_container(self): + def test_maybe_get_writable_device_path_non_block_on_container( + self, caplog + ): """When device is non-block device in container, emit debug log.""" - fake_devpath = self.tmp_path("dev/readwrite") + fake_devpath = "dev/readwrite" util.write_file(fake_devpath, "", mode=0o600) # read-write info = "dev=/dev/root mnt_point=/ path={0}".format(fake_devpath) @@ -414,16 +404,19 @@ def test_maybe_get_writable_device_path_non_block_on_container(self): fake_devpath, info, ) - self.assertIsNone(devpath) - self.assertIn( - "DEBUG: device '{0}' not a block device in container." - " cannot resize".format(fake_devpath), - self.logs.getvalue(), - ) + assert devpath is None + assert ( + mock.ANY, + logging.DEBUG, + "device '{0}' not a block device in container. cannot resize:" + " {1}".format(fake_devpath, info), + ) in caplog.record_tuples - def test_maybe_get_writable_device_path_returns_command_line_root(self): + def test_maybe_get_writable_device_path_returns_command_line_root( + self, caplog + ): """When root device is UUID in kernel command_line, update devpath.""" - # XXX Long-term we want to use FilesystemMocking test to avoid + # XXX Long-term we want to use fake_filesystem test to avoid # touching os.stat. FakeStat = namedtuple( "FakeStat", ["st_mode", "st_size", "st_mtime"] @@ -443,12 +436,13 @@ def test_maybe_get_writable_device_path_returns_command_line_root(self): "/dev/root", info, ) - self.assertEqual("/dev/disk/by-uuid/my-uuid", devpath) - self.assertIn( - "DEBUG: Converted /dev/root to '/dev/disk/by-uuid/my-uuid'" - " per kernel cmdline", - self.logs.getvalue(), - ) + assert "/dev/disk/by-uuid/my-uuid" == devpath + assert ( + mock.ANY, + logging.DEBUG, + "Converted /dev/root to '/dev/disk/by-uuid/my-uuid' per kernel" + " cmdline", + ) in caplog.record_tuples @mock.patch("cloudinit.util.mount_is_read_write") @mock.patch("cloudinit.config.cc_resizefs.os.path.isdir") @@ -458,10 +452,13 @@ def test_resize_btrfs_mount_is_ro(self, m_subp, m_is_dir, m_is_rw): m_is_rw.return_value = False m_is_dir.return_value = True m_subp.return_value = SubpResult("btrfs-progs v4.19 \n", "") - self.assertEqual( - ("btrfs", "filesystem", "resize", "max", "//.snapshots"), - _resize_btrfs("/", "/dev/sda1"), - ) + assert ( + "btrfs", + "filesystem", + "resize", + "max", + "//.snapshots", + ) == _resize_btrfs("/", "/dev/sda1") @mock.patch("cloudinit.util.mount_is_read_write") @mock.patch("cloudinit.config.cc_resizefs.os.path.isdir") @@ -471,9 +468,8 @@ def test_resize_btrfs_mount_is_rw(self, m_subp, m_is_dir, m_is_rw): m_is_rw.return_value = True m_is_dir.return_value = True m_subp.return_value = SubpResult("btrfs-progs v4.19 \n", "") - self.assertEqual( - ("btrfs", "filesystem", "resize", "max", "/"), - _resize_btrfs("/", "/dev/sda1"), + assert ("btrfs", "filesystem", "resize", "max", "/") == _resize_btrfs( + "/", "/dev/sda1" ) @mock.patch("cloudinit.util.mount_is_read_write") @@ -486,10 +482,14 @@ def test_resize_btrfs_mount_is_rw_has_queue( m_is_rw.return_value = True m_is_dir.return_value = True m_subp.return_value = SubpResult("btrfs-progs v5.10 \n", "") - self.assertEqual( - ("btrfs", "filesystem", "resize", "--enqueue", "max", "/"), - _resize_btrfs("/", "/dev/sda1"), - ) + assert ( + "btrfs", + "filesystem", + "resize", + "--enqueue", + "max", + "/", + ) == _resize_btrfs("/", "/dev/sda1") @mock.patch("cloudinit.util.mount_is_read_write") @mock.patch("cloudinit.config.cc_resizefs.os.path.isdir") @@ -503,10 +503,14 @@ def test_resize_btrfs_version(self, m_subp, m_is_dir, m_is_rw): "+UDEV +FSVERITY +ZONED CRYPTO=libgcrypt", "", ) - self.assertEqual( - ("btrfs", "filesystem", "resize", "--enqueue", "max", "/"), - _resize_btrfs("/", "/dev/sda1"), - ) + assert ( + "btrfs", + "filesystem", + "resize", + "--enqueue", + "max", + "/", + ) == _resize_btrfs("/", "/dev/sda1") @mock.patch("cloudinit.util.is_container", return_value=True) @mock.patch("cloudinit.util.is_FreeBSD") @@ -516,7 +520,7 @@ def test_maybe_get_writable_device_path_zfs_freebsd( freebsd.return_value = True info = "dev=gpt/system mnt_point=/ path=/" devpth = maybe_get_writable_device_path("gpt/system", info) - self.assertEqual("gpt/system", devpth) + assert "gpt/system" == devpth class TestResizefsSchema: diff --git a/tests/unittests/config/test_cc_resolv_conf.py b/tests/unittests/config/test_cc_resolv_conf.py index 58ca647b..6bf67dda 100644 --- a/tests/unittests/config/test_cc_resolv_conf.py +++ b/tests/unittests/config/test_cc_resolv_conf.py @@ -1,15 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging -import os -import shutil -import tempfile from copy import deepcopy from unittest import mock import pytest -from cloudinit import cloud, distros, helpers, util +from cloudinit import cloud, helpers from cloudinit.config import cc_resolv_conf from cloudinit.config.cc_resolv_conf import generate_resolv_conf from cloudinit.config.schema import ( @@ -18,10 +15,7 @@ validate_cloudconfig_schema, ) from tests.helpers import cloud_init_project_dir -from tests.unittests.helpers import ( - FilesystemMockingTestCase, - skipUnlessJsonSchema, -) +from tests.unittests.helpers import skipUnlessJsonSchema from tests.unittests.util import MockDistro LOG = logging.getLogger(__name__) @@ -33,89 +27,77 @@ #\n\n""" -class TestResolvConf(FilesystemMockingTestCase): - with_logs = True +@pytest.mark.usefixtures("fake_filesystem") +class TestResolvConf: cfg = {"manage_resolv_conf": True, "resolv_conf": {}} - def setUp(self): - super(TestResolvConf, self).setUp() - self.tmp = tempfile.mkdtemp() - util.ensure_dir(os.path.join(self.tmp, "data")) - self.addCleanup(shutil.rmtree, self.tmp) - - def _fetch_distro(self, kind, conf=None): - cls = distros.fetch(kind) - paths = helpers.Paths({"cloud_dir": self.tmp}) - conf = {} if conf is None else conf - return cls(kind, conf, paths) - - def call_resolv_conf_handler(self, distro_name, conf, cc=None): - if not cc: - ds = None - distro = self._fetch_distro(distro_name, conf) - paths = helpers.Paths({"cloud_dir": self.tmp}) - cc = cloud.Cloud(ds, paths, {}, distro, None) + def call_resolv_conf_handler(self, distro, conf, paths): + ds = None + cc = cloud.Cloud(ds, paths, {}, distro, None) cc_resolv_conf.handle("cc_resolv_conf", conf, cc, []) @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") - def test_resolv_conf_systemd_resolved(self, m_render_to_file): - self.call_resolv_conf_handler("photon", self.cfg) + def test_resolv_conf_systemd_resolved( + self, m_render_to_file, Distro, paths + ): + dist = Distro("photon", self.cfg) + self.call_resolv_conf_handler(dist, self.cfg, paths) assert [ mock.call(mock.ANY, "/etc/systemd/resolved.conf", mock.ANY) ] == m_render_to_file.call_args_list @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") - def test_resolv_conf_no_param(self, m_render_to_file): + def test_resolv_conf_no_param(self, m_render_to_file, caplog, paths): tmp = deepcopy(self.cfg) - self.logs.truncate(0) tmp.pop("resolv_conf") - self.call_resolv_conf_handler("photon", tmp) + self.call_resolv_conf_handler("photon", tmp, paths) - self.assertIn( - "manage_resolv_conf True but no parameters provided", - self.logs.getvalue(), + assert ( + "manage_resolv_conf True but no parameters provided" in caplog.text ) assert [ mock.call(mock.ANY, "/etc/systemd/resolved.conf", mock.ANY) ] not in m_render_to_file.call_args_list @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") - def test_resolv_conf_manage_resolv_conf_false(self, m_render_to_file): + def test_resolv_conf_manage_resolv_conf_false( + self, m_render_to_file, caplog, Distro, paths + ): tmp = deepcopy(self.cfg) - self.logs.truncate(0) tmp["manage_resolv_conf"] = False - self.call_resolv_conf_handler("photon", tmp) - self.assertIn( - "'manage_resolv_conf' present but set to False", - self.logs.getvalue(), - ) + dist = Distro("photon", self.cfg) + self.call_resolv_conf_handler(dist, tmp, paths) + assert "'manage_resolv_conf' present but set to False" in caplog.text assert [ mock.call(mock.ANY, "/etc/systemd/resolved.conf", mock.ANY) ] not in m_render_to_file.call_args_list @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") - def test_resolv_conf_etc_resolv_conf(self, m_render_to_file): - self.call_resolv_conf_handler("rhel", self.cfg) + def test_resolv_conf_etc_resolv_conf( + self, m_render_to_file, Distro, paths + ): + dist = Distro("rhel", self.cfg) + self.call_resolv_conf_handler(dist, self.cfg, paths) assert [ mock.call(mock.ANY, "/etc/resolv.conf", mock.ANY) ] == m_render_to_file.call_args_list @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") - def test_resolv_conf_invalid_resolve_conf_fn(self, m_render_to_file): + def test_resolv_conf_invalid_resolve_conf_fn( + self, m_render_to_file, caplog, Distro, tmp_path + ): ds = None - distro = self._fetch_distro("rhel", self.cfg) - paths = helpers.Paths({"cloud_dir": self.tmp}) - cc = cloud.Cloud(ds, paths, {}, distro, None) + dist = Distro("rhel", self.cfg) + paths = helpers.Paths({"cloud_dir": str(tmp_path)}) + cc = cloud.Cloud(ds, paths, {}, dist, None) cc.distro.resolve_conf_fn = "bla" - self.logs.truncate(0) - self.call_resolv_conf_handler("rhel", self.cfg, cc) + cc_resolv_conf.handle("rhel", self.cfg, cc, []) - self.assertIn( - "No template found, not rendering resolve configs", - self.logs.getvalue(), + assert ( + "No template found, not rendering resolve configs" in caplog.text ) assert [ diff --git a/tests/unittests/config/test_cc_rh_subscription.py b/tests/unittests/config/test_cc_rh_subscription.py index d811d16a..0f467617 100644 --- a/tests/unittests/config/test_cc_rh_subscription.py +++ b/tests/unittests/config/test_cc_rh_subscription.py @@ -14,52 +14,45 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from tests.unittests.helpers import mock, skipUnlessJsonSchema SUBMGR = cc_rh_subscription.SubscriptionManager SUB_MAN_CLI = "cloudinit.config.cc_rh_subscription._sub_man_cli" +NAME = "cc_rh_subscription" @mock.patch(SUB_MAN_CLI) -class GoodTests(CiTestCase): - with_logs = True +class TestHappyPath: - def setUp(self): - super(GoodTests, self).setUp() - self.name = "cc_rh_subscription" - self.cloud_init = None - self.log = logging.getLogger("good_tests") - self.args = [] - self.handle = cc_rh_subscription.handle - - self.config = { - "rh_subscription": { - "username": "scooby@do.com", - "password": "scooby-snacks", - } + CONFIG = { + "rh_subscription": { + "username": "scooby@do.com", + "password": "scooby-snacks", } - self.config_full = { - "rh_subscription": { - "username": "scooby@do.com", - "password": "scooby-snacks", - "auto-attach": True, - "service-level": "self-support", - "add-pool": ["pool1", "pool2", "pool3"], - "enable-repo": ["repo1", "repo2", "repo3"], - "disable-repo": ["repo4", "repo5"], - } + } + + CONFIG_FULL = { + "rh_subscription": { + "username": "scooby@do.com", + "password": "scooby-snacks", + "auto-attach": True, + "service-level": "self-support", + "add-pool": ["pool1", "pool2", "pool3"], + "enable-repo": ["repo1", "repo2", "repo3"], + "disable-repo": ["repo4", "repo5"], } + } - def test_already_registered(self, m_sman_cli): + def test_already_registered(self, m_sman_cli, caplog): """ Emulates a system that is already registered. Ensure it gets a non-ProcessExecution error from is_registered() """ - self.handle(self.name, self.config, self.cloud_init, self.args) - self.assertEqual(m_sman_cli.call_count, 1) - self.assertIn("System is already registered", self.logs.getvalue()) + cc_rh_subscription.handle(NAME, self.CONFIG, None, []) + assert m_sman_cli.call_count == 1 + assert "System is already registered" in caplog.text - def test_simple_registration(self, m_sman_cli): + def test_simple_registration(self, m_sman_cli, caplog): """ Simple registration with username and password """ @@ -68,9 +61,9 @@ def test_simple_registration(self, m_sman_cli): " 12345678-abde-abcde-1234-1234567890abc" ) m_sman_cli.side_effect = [subp.ProcessExecutionError, (reg, "bar")] - self.handle(self.name, self.config, self.cloud_init, self.args) - self.assertIn(mock.call(["identity"]), m_sman_cli.call_args_list) - self.assertIn( + cc_rh_subscription.handle(NAME, self.CONFIG, None, []) + assert mock.call(["identity"]) in m_sman_cli.call_args_list + assert ( mock.call( [ "register", @@ -78,30 +71,27 @@ def test_simple_registration(self, m_sman_cli): "--password=scooby-snacks", ], logstring_val=True, - ), - m_sman_cli.call_args_list, - ) - self.assertIn( - "rh_subscription plugin completed successfully", - self.logs.getvalue(), + ) + in m_sman_cli.call_args_list ) - self.assertEqual(m_sman_cli.call_count, 2) + assert "rh_subscription plugin completed successfully" in caplog.text + assert m_sman_cli.call_count == 2 @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_getRepos") def test_update_repos_disable_with_none(self, m_get_repos, m_sman_cli): - cfg = copy.deepcopy(self.config) + cfg = copy.deepcopy(self.CONFIG) m_get_repos.return_value = ([], ["repo1"]) cfg["rh_subscription"].update( {"enable-repo": ["repo1"], "disable-repo": None} ) mysm = cc_rh_subscription.SubscriptionManager(cfg) - self.assertEqual(True, mysm.update_repos()) + assert True is mysm.update_repos() m_get_repos.assert_called_with() - self.assertEqual( - m_sman_cli.call_args_list, [mock.call(["repos", "--enable=repo1"])] - ) + assert m_sman_cli.call_args_list == [ + mock.call(["repos", "--enable=repo1"]) + ] - def test_full_registration(self, m_sman_cli): + def test_full_registration(self, m_sman_cli, caplog): """ Registration with auto-attach, service-level, adding pools, and enabling and disabling yum repos @@ -127,38 +117,30 @@ def test_full_registration(self, m_sman_cli): ("Repo ID: repo2\nRepo ID: repo3\nRepo ID: repo4", ""), ("", ""), ] - self.handle(self.name, self.config_full, self.cloud_init, self.args) - self.assertEqual(m_sman_cli.call_count, 9) + cc_rh_subscription.handle(NAME, self.CONFIG_FULL, None, []) + assert m_sman_cli.call_count == 9 for call in call_lists: - self.assertIn(mock.call(call), m_sman_cli.call_args_list) - self.assertIn( - "rh_subscription plugin completed successfully", - self.logs.getvalue(), - ) + assert mock.call(call) in m_sman_cli.call_args_list + assert "rh_subscription plugin completed successfully" in caplog.text @mock.patch(SUB_MAN_CLI) -class TestBadInput(CiTestCase): - with_logs = True - name = "cc_rh_subscription" - cloud_init = None - log = logging.getLogger("bad_tests") - args: list = [] +class TestBadInput: SM = cc_rh_subscription.SubscriptionManager - reg = ( + REG = ( "The system has been registered with ID:" " 12345678-abde-abcde-1234-1234567890abc" ) - config_no_password = {"rh_subscription": {"username": "scooby@do.com"}} + CONFIG_NO_PASSWORD = {"rh_subscription": {"username": "scooby@do.com"}} - config_no_key = { + CONFIG_NO_KEY = { "rh_subscription": { "activation-key": "1234abcde", } } - config_service = { + CONFIG_SERVICE = { "rh_subscription": { "username": "scooby@do.com", "password": "scooby-snacks", @@ -166,21 +148,21 @@ class TestBadInput(CiTestCase): } } - config_badpool = { + CONFIG_BADPOOL = { "rh_subscription": { "username": "scooby@do.com", "password": "scooby-snacks", "add-pool": "not_a_list", } } - config_badrepo = { + CONFIG_BADREPO = { "rh_subscription": { "username": "scooby@do.com", "password": "scooby-snacks", "enable-repo": "not_a_list", } } - config_badkey = { + CONFIG_BADKEY = { "rh_subscription": { "activation-key": "abcdef1234", "fookey": "bar", @@ -188,125 +170,112 @@ class TestBadInput(CiTestCase): } } - def setUp(self): - super(TestBadInput, self).setUp() - self.handle = cc_rh_subscription.handle - - def assert_logged_warnings(self, warnings): - logs = self.logs.getvalue() - missing = [w for w in warnings if "WARNING: " + w not in logs] - self.assertEqual([], missing, "Missing expected warnings.") + def assert_logged_warnings(self, warnings, caplog): + missing = [ + w + for w in warnings + if (mock.ANY, logging.WARNING, w) not in caplog.record_tuples + ] + assert [] == missing, "Missing expected warnings." def test_no_password(self, m_sman_cli): """Attempt to register without the password key/value.""" m_sman_cli.side_effect = [ subp.ProcessExecutionError, - (self.reg, "bar"), + (self.REG, "bar"), ] - self.handle( - self.name, - self.config_no_password, - self.cloud_init, - self.args, - ) - self.assertEqual(m_sman_cli.call_count, 0) + cc_rh_subscription.handle(NAME, self.CONFIG_NO_PASSWORD, None, []) + assert m_sman_cli.call_count == 0 - def test_no_org(self, m_sman_cli): + def test_no_org(self, m_sman_cli, caplog): """Attempt to register without the org key/value.""" m_sman_cli.side_effect = [subp.ProcessExecutionError] - self.handle(self.name, self.config_no_key, self.cloud_init, self.args) + cc_rh_subscription.handle(NAME, self.CONFIG_NO_KEY, None, []) m_sman_cli.assert_called_with(["identity"]) - self.assertEqual(m_sman_cli.call_count, 1) + assert m_sman_cli.call_count == 1 self.assert_logged_warnings( ( "Unable to register system due to incomplete information.", "Use either activationkey and org *or* userid and password", "Registration failed or did not run completely", "rh_subscription plugin did not complete successfully", - ) + ), + caplog, ) - def test_service_level_without_auto(self, m_sman_cli): + def test_service_level_without_auto(self, m_sman_cli, caplog): """Attempt to register using service-level without auto-attach key.""" m_sman_cli.side_effect = [ subp.ProcessExecutionError, - (self.reg, "bar"), + (self.REG, "bar"), ] - self.handle( - self.name, - self.config_service, - self.cloud_init, - self.args, - ) - self.assertEqual(m_sman_cli.call_count, 1) + cc_rh_subscription.handle(NAME, self.CONFIG_SERVICE, None, []) + assert m_sman_cli.call_count == 1 self.assert_logged_warnings( ( - "The service-level key must be used in conjunction with ", + "The service-level key must be used in conjunction with the" + " auto-attach key. Please re-run with auto-attach: True", "rh_subscription plugin did not complete successfully", - ) + ), + caplog, ) - def test_pool_not_a_list(self, m_sman_cli): + def test_pool_not_a_list(self, m_sman_cli, caplog): """ Register with pools that are not in the format of a list """ m_sman_cli.side_effect = [ subp.ProcessExecutionError, - (self.reg, "bar"), + (self.REG, "bar"), ] - self.handle( - self.name, - self.config_badpool, - self.cloud_init, - self.args, - ) - self.assertEqual(m_sman_cli.call_count, 2) + cc_rh_subscription.handle(NAME, self.CONFIG_BADPOOL, None, []) + assert m_sman_cli.call_count == 2 self.assert_logged_warnings( ( "Pools must in the format of a list", "rh_subscription plugin did not complete successfully", - ) + ), + caplog, ) - def test_repo_not_a_list(self, m_sman_cli): + def test_repo_not_a_list(self, m_sman_cli, caplog): """ Register with repos that are not in the format of a list """ m_sman_cli.side_effect = [ subp.ProcessExecutionError, - (self.reg, "bar"), + (self.REG, "bar"), ] - self.handle( - self.name, - self.config_badrepo, - self.cloud_init, - self.args, - ) - self.assertEqual(m_sman_cli.call_count, 2) + cc_rh_subscription.handle(NAME, self.CONFIG_BADREPO, None, []) + assert m_sman_cli.call_count == 2 self.assert_logged_warnings( ( "Repo IDs must in the format of a list.", "Unable to add or remove repos", "rh_subscription plugin did not complete successfully", - ) + ), + caplog, ) - def test_bad_key_value(self, m_sman_cli): + def test_bad_key_value(self, m_sman_cli, caplog): """ Attempt to register with a key that we don't know """ m_sman_cli.side_effect = [ subp.ProcessExecutionError, - (self.reg, "bar"), + (self.REG, "bar"), ] - self.handle(self.name, self.config_badkey, self.cloud_init, self.args) - self.assertEqual(m_sman_cli.call_count, 1) + cc_rh_subscription.handle(NAME, self.CONFIG_BADKEY, None, []) + assert m_sman_cli.call_count == 1 self.assert_logged_warnings( ( "fookey is not a valid key for rh_subscription. Valid keys" - " are:", + " are: org, activation-key, username, password, disable-repo," + " enable-repo, add-pool, rhsm-baseurl, server-hostname," + " auto-attach, service-level", "rh_subscription plugin did not complete successfully", - ) + ), + caplog, ) diff --git a/tests/unittests/config/test_cc_runcmd.py b/tests/unittests/config/test_cc_runcmd.py index c99e3dfa..bdf394a5 100644 --- a/tests/unittests/config/test_cc_runcmd.py +++ b/tests/unittests/config/test_cc_runcmd.py @@ -2,85 +2,73 @@ import logging import os import stat +from typing import Any, Dict from unittest.mock import patch import pytest -from cloudinit import helpers, subp, util +from cloudinit import helpers, util from cloudinit.config.cc_runcmd import handle from cloudinit.config.schema import ( SchemaValidationError, get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import ( - SCHEMA_EMPTY_ERROR, - FilesystemMockingTestCase, - skipUnlessJsonSchema, -) +from tests.unittests.helpers import SCHEMA_EMPTY_ERROR, skipUnlessJsonSchema from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) -class TestRuncmd(FilesystemMockingTestCase): +@pytest.fixture +def cloud(tmp_path): + paths = helpers.Paths({"scripts": str(tmp_path)}) + return get_cloud(paths=paths) - with_logs = True - def setUp(self): - super(TestRuncmd, self).setUp() - self.subp = subp.subp - self.new_root = self.tmp_dir() - self.patchUtils(self.new_root) - self.paths = helpers.Paths({"scripts": self.new_root}) +@pytest.mark.usefixtures("fake_filesystem") +class TestRuncmd: + # self.paths = helpers.Paths({"scripts": self.new_root}) - def test_handler_skip_if_no_runcmd(self): + def test_handler_skip_if_no_runcmd(self, caplog, cloud): """When the provided config doesn't contain runcmd, skip it.""" - cfg = {} - mycloud = get_cloud(paths=self.paths) - handle("notimportant", cfg, mycloud, None) - self.assertIn( - "Skipping module named notimportant, no 'runcmd' key", - self.logs.getvalue(), + cfg: Dict[str, Any] = {} + handle("notimportant", cfg, cloud, []) + assert ( + "Skipping module named notimportant, no 'runcmd' key" + in caplog.text ) + @pytest.mark.allow_subp_for("/bin/sh") @patch("cloudinit.util.shellify") - def test_runcmd_shellify_fails(self, cls): + def test_runcmd_shellify_fails(self, cls, cloud): """When shellify fails throw exception""" cls.side_effect = TypeError("patched shellify") valid_config = {"runcmd": ["echo 42"]} - cc = get_cloud(paths=self.paths) - with self.assertRaises(TypeError) as cm: - with self.allow_subp(["/bin/sh"]): - handle("cc_runcmd", valid_config, cc, None) - self.assertIn("Failed to shellify", str(cm.exception)) + with pytest.raises(TypeError, match="Failed to shellify"): + handle("cc_runcmd", valid_config, cloud, []) - def test_handler_invalid_command_set(self): + def test_handler_invalid_command_set(self, cloud): """Commands which can't be converted to shell will raise errors.""" invalid_config = {"runcmd": 1} - cc = get_cloud(paths=self.paths) - with self.assertRaises(TypeError) as cm: - handle("cc_runcmd", invalid_config, cc, []) - self.assertIn( - "Failed to shellify 1 into file" + with pytest.raises( + TypeError, + match="Failed to shellify 1 into file" " /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd", - str(cm.exception), - ) + ): + handle("cc_runcmd", invalid_config, cloud, []) - def test_handler_write_valid_runcmd_schema_to_file(self): + def test_handler_write_valid_runcmd_schema_to_file(self, cloud, tmp_path): """Valid runcmd schema is written to a runcmd shell script.""" valid_config = {"runcmd": [["ls", "/"]]} - cc = get_cloud(paths=self.paths) - handle("cc_runcmd", valid_config, cc, []) + handle("cc_runcmd", valid_config, cloud, []) runcmd_file = os.path.join( - self.new_root, + tmp_path, "var/lib/cloud/instances/iid-datasource-none/scripts/runcmd", ) - self.assertEqual( - "#!/bin/sh\n'ls' '/'\n", util.load_text_file(runcmd_file) - ) + assert "#!/bin/sh\n'ls' '/'\n" == util.load_text_file(runcmd_file) file_stat = os.stat(runcmd_file) - self.assertEqual(0o700, stat.S_IMODE(file_stat.st_mode)) + assert 0o700 == stat.S_IMODE(file_stat.st_mode) @skipUnlessJsonSchema() diff --git a/tests/unittests/config/test_cc_snap.py b/tests/unittests/config/test_cc_snap.py index c5026783..e170541d 100644 --- a/tests/unittests/config/test_cc_snap.py +++ b/tests/unittests/config/test_cc_snap.py @@ -14,7 +14,6 @@ ) from tests.unittests.helpers import ( SCHEMA_EMPTY_ERROR, - CiTestCase, mock, skipUnlessJsonSchema, ) @@ -171,29 +170,22 @@ def test_add_assertions_adds_assertions_as_dict( ) -class TestRunCommands(CiTestCase): - with_logs = True - allowed_subp = [CiTestCase.SUBP_SHELL_TRUE] - - def setUp(self): - super(TestRunCommands, self).setUp() - self.tmp = self.tmp_dir() - +@pytest.mark.usefixtures("fake_filesystem") +class TestRunCommands: @mock.patch("cloudinit.config.cc_snap.subp.subp") - def test_run_commands_on_empty_list(self, m_subp): + def test_run_commands_on_empty_list(self, m_subp, caplog): """When provided with an empty list, run_commands does nothing.""" run_commands([]) - self.assertEqual("", self.logs.getvalue()) + assert "" == caplog.text m_subp.assert_not_called() def test_run_commands_on_non_list_or_dict(self): """When provided an invalid type, run_commands raises an error.""" - with self.assertRaises(TypeError) as context_manager: + with pytest.raises( + TypeError, + match="commands parameter was not a list or dict: I'm Not Valid", + ): run_commands(commands="I'm Not Valid") - self.assertEqual( - "commands parameter was not a list or dict: I'm Not Valid", - str(context_manager.exception), - ) @pytest.mark.allow_all_subp diff --git a/tests/unittests/config/test_cc_ssh.py b/tests/unittests/config/test_cc_ssh.py index a49fbf01..3a5098f8 100644 --- a/tests/unittests/config/test_cc_ssh.py +++ b/tests/unittests/config/test_cc_ssh.py @@ -2,7 +2,7 @@ import logging import os.path -from typing import Optional +from typing import Any, Dict, Optional from unittest import mock import pytest @@ -109,7 +109,7 @@ def test_handle_no_cfg( ): """Test handle with no config ignores generating existing keyfiles.""" m_fips.return_value = fips_enabled - cfg = {} + cfg: Dict[str, Any] = {} keys = ["key1"] m_glob.return_value = [] # Return no matching keys to prevent removal # Mock os.path.exits to True to short-circuit the key writing logic @@ -117,7 +117,7 @@ def test_handle_no_cfg( m_nug.return_value = ([], {}) cc_ssh.PUBLISH_HOST_KEYS = False cloud = get_cloud(distro="ubuntu", metadata={"public-keys": keys}) - cc_ssh.handle("name", cfg, cloud, None) + cc_ssh.handle("name", cfg, cloud, []) options = ssh_util.DISABLE_USER_OPTS.replace("$USER", "NONE") options = options.replace("$DISABLE_USER", "root") m_glob.assert_called_once_with("/etc/ssh/ssh_host_*key*") @@ -158,7 +158,7 @@ def test_dont_allow_public_ssh_keys( m_path_exists.return_value = True m_nug.return_value = ({user: {"default": user}}, {}) cloud = get_cloud(distro="ubuntu", metadata={"public-keys": keys}) - cc_ssh.handle("name", cfg, cloud, None) + cc_ssh.handle("name", cfg, cloud, []) options = ssh_util.DISABLE_USER_OPTS.replace("$USER", user) options = options.replace("$DISABLE_USER", "root") @@ -211,7 +211,7 @@ def test_handle_default_root( cloud = get_cloud(distro="ubuntu", metadata={"public-keys": keys}) if mock_get_public_ssh_keys: cloud.get_public_ssh_keys = mock.Mock(return_value=keys) - cc_ssh.handle("name", cfg, cloud, None) + cc_ssh.handle("name", cfg, cloud, []) if empty_opts: options = "" @@ -299,7 +299,7 @@ def test_handle_publish_hostkeys( with mock.patch.object( cloud.datasource, "publish_host_keys", mock.Mock() ): - cc_ssh.handle("name", cfg, cloud, None) + cc_ssh.handle("name", cfg, cloud, []) assert ( expected_calls == cloud.datasource.publish_host_keys.call_args_list @@ -372,7 +372,7 @@ def test_handle_ssh_keys_in_cfg( else: sshd_conf_fname = "/etc/ssh/sshd_config" - cfg = {"ssh_keys": {}} + cfg: Dict[str, Any] = {"ssh_keys": {}} expected_calls = [] cert_content = "" @@ -427,7 +427,7 @@ def test_handle_ssh_keys_in_cfg( with mock.patch( MODPATH + "ssh_util.parse_ssh_config", return_value=[] ): - cc_ssh.handle("name", cfg, get_cloud(distro="ubuntu"), None) + cc_ssh.handle("name", cfg, get_cloud(distro="ubuntu"), []) # Check that all expected output has been done. for call_ in expected_calls: @@ -474,7 +474,7 @@ def test_handle_invalid_ssh_keys_are_skipped( with mock.patch( MODPATH + "ssh_util.parse_ssh_config", return_value=[] ): - cc_ssh.handle("name", cfg, get_cloud("ubuntu"), None) + cc_ssh.handle("name", cfg, get_cloud("ubuntu"), []) assert [] == m_write_file.call_args_list expected_log_msgs = [ f'Skipping {reason} ssh_keys entry: "{key_type}_private"', diff --git a/tests/unittests/config/test_cc_ubuntu_drivers.py b/tests/unittests/config/test_cc_ubuntu_drivers.py index 6a4503de..02854585 100644 --- a/tests/unittests/config/test_cc_ubuntu_drivers.py +++ b/tests/unittests/config/test_cc_ubuntu_drivers.py @@ -4,6 +4,7 @@ import logging import os import re +from typing import Any, Dict import pytest @@ -88,7 +89,7 @@ def test_happy_path_taken( debconf_file = tdir.join("nvidia.template") m_tmp.return_value = tdir myCloud = mock.MagicMock() - drivers.handle("ubuntu_drivers", new_config, myCloud, None) + drivers.handle("ubuntu_drivers", new_config, myCloud, []) assert [ mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list @@ -122,7 +123,7 @@ def test_handle_raises_error_if_no_drivers_found( ) with pytest.raises(Exception): - drivers.handle("ubuntu_drivers", cfg_accepted, myCloud, None) + drivers.handle("ubuntu_drivers", cfg_accepted, myCloud, []) assert [ mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list @@ -185,7 +186,7 @@ def test_handle_inert( ): """Helper to reduce repetition when testing negative cases""" myCloud = mock.MagicMock() - drivers.handle("ubuntu_drivers", config, myCloud, None) + drivers.handle("ubuntu_drivers", config, myCloud, []) assert 0 == myCloud.distro.install_packages.call_count assert 0 == m_subp.call_count @@ -196,7 +197,7 @@ def test_handle_no_drivers_does_nothing( ): """If no 'drivers' key in the config, nothing should be done.""" myCloud = mock.MagicMock() - drivers.handle("ubuntu_drivers", {"foo": "bzr"}, myCloud, None) + drivers.handle("ubuntu_drivers", {"foo": "bzr"}, myCloud, []) assert "Skipping module named" in m_log.debug.call_args_list[0][0][0] assert 0 == m_install_drivers.call_count @@ -267,7 +268,7 @@ def test_install_drivers_handles_old_ubuntu_drivers_gracefully( ) with pytest.raises(Exception): - drivers.handle("ubuntu_drivers", cfg_accepted, myCloud, None) + drivers.handle("ubuntu_drivers", cfg_accepted, myCloud, []) assert [ mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list @@ -304,7 +305,7 @@ def test_debconf_not_installed_does_nothing( "drivers": {"nvidia": {"license-accepted": True, "version": None}} } with pytest.raises(AttributeError): - drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, None) + drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, []) assert ( 0 == m_debconf.DebconfCommunicator.__enter__().command.call_count ) @@ -333,7 +334,7 @@ def test_version_none_uses_latest( version_none_cfg = { "drivers": {"nvidia": {"license-accepted": True, "version": None}} } - drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, None) + drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, []) assert [ mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list @@ -358,8 +359,8 @@ def test_no_cfg_drivers_does_nothing( ): m_tmp.return_value = tmpdir myCloud = mock.MagicMock() - version_none_cfg = {} - drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, None) + version_none_cfg: Dict[Any, Any] = {} + drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, []) assert 0 == m_install_drivers.call_count assert ( mock.call( @@ -384,7 +385,7 @@ def test_has_not_debconf_does_nothing( m_tmp.return_value = tmpdir myCloud = mock.MagicMock() version_none_cfg = {"drivers": {"nvidia": {"license-accepted": True}}} - drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, None) + drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, []) assert 0 == m_install_drivers.call_count assert ( mock.call( diff --git a/tests/unittests/config/test_cc_update_etc_hosts.py b/tests/unittests/config/test_cc_update_etc_hosts.py index 6ede9954..ebf6f599 100644 --- a/tests/unittests/config/test_cc_update_etc_hosts.py +++ b/tests/unittests/config/test_cc_update_etc_hosts.py @@ -14,66 +14,75 @@ get_schema, validate_cloudconfig_schema, ) +from tests.helpers import cloud_init_project_dir from tests.unittests import helpers as t_help LOG = logging.getLogger(__name__) -class TestHostsFile(t_help.FilesystemMockingTestCase): - def setUp(self): - super(TestHostsFile, self).setUp() - self.tmp = self.tmp_dir() +@pytest.fixture(autouse=True) +def with_templates(tmp_path, fake_filesystem_hook): + shutil.copytree( + str(cloud_init_project_dir("templates")), + str(tmp_path / "templates"), + dirs_exist_ok=True, + ) + +@pytest.mark.usefixtures("fake_filesystem") +class TestHostsFile: def _fetch_distro(self, kind): cls = distros.fetch(kind) paths = helpers.Paths({}) return cls(kind, {}, paths) - def test_write_etc_hosts_suse_localhost(self): + def test_write_etc_hosts_suse_localhost(self, tmp_path): cfg = { "manage_etc_hosts": "localhost", "hostname": "cloud-init.test.us", } - os.makedirs("%s/etc/" % self.tmp) + os.makedirs(tmp_path / "etc/") hosts_content = "192.168.1.1 blah.blah.us blah\n" - fout = open("%s/etc/hosts" % self.tmp, "w") + etc_hosts = str(tmp_path / "etc/hosts") + fout = open(etc_hosts, "w") fout.write(hosts_content) fout.close() distro = self._fetch_distro("sles") - distro.hosts_fn = "%s/etc/hosts" % self.tmp + distro.hosts_fn = etc_hosts paths = helpers.Paths({}) ds = None cc = cloud.Cloud(ds, paths, {}, distro, None) - self.patchUtils(self.tmp) cc_update_etc_hosts.handle("test", cfg, cc, []) - contents = util.load_text_file("%s/etc/hosts" % self.tmp) - if "127.0.1.1\tcloud-init.test.us\tcloud-init" not in contents: - self.assertIsNone("No entry for 127.0.1.1 in etc/hosts") - if "192.168.1.1\tblah.blah.us\tblah" not in contents: - self.assertIsNone("Default etc/hosts content modified") + contents = util.load_text_file(etc_hosts) + assert ( + "127.0.1.1\tcloud-init.test.us\tcloud-init" in contents + ), "No entry for 127.0.1.1 in etc/hosts" + assert ( + "192.168.1.1\tblah.blah.us\tblah" in contents + ), "Default etc/hosts content modified" @t_help.skipUnlessJinja() - def test_write_etc_hosts_suse_template(self): + def test_write_etc_hosts_suse_template(self, tmp_path): cfg = { "manage_etc_hosts": "template", "hostname": "cloud-init.test.us", } shutil.copytree( - t_help.cloud_init_project_dir("templates"), - "%s/etc/cloud/templates" % self.tmp, + tmp_path / "templates", str(tmp_path / "etc/cloud/templates") ) distro = self._fetch_distro("sles") paths = helpers.Paths({}) - paths.template_tpl = "%s" % self.tmp + "/etc/cloud/templates/%s.tmpl" + paths.template_tpl = str(tmp_path / "etc/cloud/templates/%s.tmpl") ds = None cc = cloud.Cloud(ds, paths, {}, distro, None) - self.patchUtils(self.tmp) cc_update_etc_hosts.handle("test", cfg, cc, []) - contents = util.load_text_file("%s/etc/hosts" % self.tmp) - if "127.0.1.1 cloud-init.test.us cloud-init" not in contents: - self.assertIsNone("No entry for 127.0.1.1 in etc/hosts") - if "::1 cloud-init.test.us cloud-init" not in contents: - self.assertIsNone("No entry for 127.0.0.1 in etc/hosts") + contents = util.load_text_file(tmp_path / "etc/hosts") + assert ( + "127.0.1.1 cloud-init.test.us cloud-init" in contents + ), "No entry for 127.0.1.1 in etc/hosts" + assert ( + "::1 cloud-init.test.us cloud-init" in contents + ), "No entry for 127.0.0.1 in etc/hosts" class TestUpdateEtcHosts: diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index ec2b1833..3f8f799e 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -1,4 +1,5 @@ # This file is part of cloud-init. See LICENSE file for license information. +import logging import re import pytest @@ -10,22 +11,21 @@ validate_cloudconfig_schema, ) from tests.unittests.helpers import ( - CiTestCase, + assert_count_equal, does_not_raise, mock, skipUnlessJsonSchema, ) +from tests.unittests.util import get_cloud MODPATH = "cloudinit.config.cc_users_groups" @mock.patch("cloudinit.distros.ubuntu.Distro.create_group") @mock.patch("cloudinit.distros.ubuntu.Distro.create_user") -class TestHandleUsersGroups(CiTestCase): +class TestHandleUsersGroups: """Test cc_users_groups handling of config.""" - with_logs = True - def test_handle_no_cfg_creates_no_users_or_groups(self, m_user, m_group): """Test handle with no config will not create users or groups.""" cfg = {} # merged cloud-config @@ -39,9 +39,7 @@ def test_handle_no_cfg_creates_no_users_or_groups(self, m_user, m_group): } } metadata = {} - cloud = self.tmp_cloud( - distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata - ) + cloud = get_cloud(distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle("modulename", cfg, cloud, None) m_user.assert_not_called() m_group.assert_not_called() @@ -59,11 +57,9 @@ def test_handle_users_in_cfg_calls_create_users(self, m_user, m_group): } } metadata = {} - cloud = self.tmp_cloud( - distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata - ) + cloud = get_cloud(distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle("modulename", cfg, cloud, None) - self.assertCountEqual( + assert_count_equal( m_user.call_args_list, [ mock.call( @@ -105,11 +101,11 @@ def test_handle_users_in_cfg_calls_create_users_on_bsd( with mock.patch( "cloudinit.distros.networking.subp.subp", return_value=("", None) ): - cloud = self.tmp_cloud( + cloud = get_cloud( distro="freebsd", sys_cfg=sys_cfg, metadata=metadata ) cc_users_groups.handle("modulename", cfg, cloud, None) - self.assertCountEqual( + assert_count_equal( m_fbsd_user.call_args_list, [ mock.call( @@ -141,11 +137,9 @@ def test_users_with_ssh_redirect_user_passes_keys(self, m_user, m_group): } } metadata = {"public-keys": ["key1"]} - cloud = self.tmp_cloud( - distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata - ) + cloud = get_cloud(distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle("modulename", cfg, cloud, None) - self.assertCountEqual( + assert_count_equal( m_user.call_args_list, [ mock.call( @@ -182,11 +176,9 @@ def test_users_with_ssh_redirect_user_default_str(self, m_user, m_group): } } metadata = {"public-keys": ["key1"]} - cloud = self.tmp_cloud( - distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata - ) + cloud = get_cloud(distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle("modulename", cfg, cloud, None) - self.assertCountEqual( + assert_count_equal( m_user.call_args_list, [ mock.call( @@ -216,15 +208,14 @@ def test_users_without_home_cannot_import_ssh_keys(self, m_user, m_group): }, ] } - cloud = self.tmp_cloud(distro="ubuntu", sys_cfg={}, metadata={}) - with self.assertRaises(ValueError) as context_manager: + cloud = get_cloud(distro="ubuntu", sys_cfg={}, metadata={}) + with pytest.raises( + ValueError, + match=r"Not creating user me2. Key\(s\) ssh_import_id cannot be" + " provided with no_create_home", + ): cc_users_groups.handle("modulename", cfg, cloud, None) m_group.assert_not_called() - self.assertEqual( - "Not creating user me2. Key(s) ssh_import_id cannot be provided" - " with no_create_home", - str(context_manager.exception), - ) def test_users_with_ssh_redirect_user_non_default(self, m_user, m_group): """Warn when ssh_redirect_user is not 'default'.""" @@ -244,17 +235,14 @@ def test_users_with_ssh_redirect_user_non_default(self, m_user, m_group): } } metadata = {"public-keys": ["key1"]} - cloud = self.tmp_cloud( - distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata - ) - with self.assertRaises(ValueError) as context_manager: + cloud = get_cloud(distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata) + with pytest.raises( + ValueError, + match="Not creating user me2. Invalid value of ssh_redirect_user:" + " snowflake. Expected values: true, default or false.", + ): cc_users_groups.handle("modulename", cfg, cloud, None) m_group.assert_not_called() - self.assertEqual( - "Not creating user me2. Invalid value of ssh_redirect_user:" - " snowflake. Expected values: true, default or false.", - str(context_manager.exception), - ) def test_users_with_ssh_redirect_user_default_false(self, m_user, m_group): """When unspecified ssh_redirect_user is false and not set up.""" @@ -269,11 +257,9 @@ def test_users_with_ssh_redirect_user_default_false(self, m_user, m_group): } } metadata = {"public-keys": ["key1"]} - cloud = self.tmp_cloud( - distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata - ) + cloud = get_cloud(distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle("modulename", cfg, cloud, None) - self.assertCountEqual( + assert_count_equal( m_user.call_args_list, [ mock.call( @@ -287,7 +273,9 @@ def test_users_with_ssh_redirect_user_default_false(self, m_user, m_group): ) m_group.assert_not_called() - def test_users_ssh_redirect_user_and_no_default(self, m_user, m_group): + def test_users_ssh_redirect_user_and_no_default( + self, m_user, m_group, caplog + ): """Warn when ssh_redirect_user is True and no default user present.""" cfg = { "users": ["default", {"name": "me2", "ssh_redirect_user": True}] @@ -295,18 +283,19 @@ def test_users_ssh_redirect_user_and_no_default(self, m_user, m_group): # System config defines *no* default user for the distro. sys_cfg = {} metadata = {} # no public-keys defined - cloud = self.tmp_cloud( - distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata - ) + cloud = get_cloud(distro="ubuntu", sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle("modulename", cfg, cloud, None) m_user.assert_called_once_with("me2", default=False) m_group.assert_not_called() - self.assertEqual( - "WARNING: Ignoring ssh_redirect_user: True for me2. No" - " default_user defined. Perhaps missing" - " cloud configuration users: [default, ..].\n", - self.logs.getvalue(), - ) + assert [ + ( + mock.ANY, + logging.WARNING, + "Ignoring ssh_redirect_user: True for me2. No" + " default_user defined. Perhaps missing" + " cloud configuration users: [default, ..].", + ) + ] == caplog.record_tuples class TestUsersGroupsSchema: diff --git a/tests/unittests/config/test_cc_wireguard.py b/tests/unittests/config/test_cc_wireguard.py index 6f46653a..6702eb93 100644 --- a/tests/unittests/config/test_cc_wireguard.py +++ b/tests/unittests/config/test_cc_wireguard.py @@ -1,4 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. +import logging + import pytest from cloudinit import subp, util @@ -8,7 +10,7 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from tests.unittests.helpers import mock, skipUnlessJsonSchema NL = "\n" # Module path used in mocks @@ -21,27 +23,18 @@ def __init__(self, distro): self.distro = distro -class TestWireGuard(CiTestCase): - with_logs = True - allowed_subp = [CiTestCase.SUBP_SHELL_TRUE] - - def setUp(self): - super(TestWireGuard, self).setUp() - self.tmp = self.tmp_dir() - +@pytest.mark.usefixtures("fake_filesystem") +class TestWireGuard: def test_readiness_probe_schema_non_string_values(self): """ValueError raised for any values expected as string type.""" wg_readinessprobes = [1, ["not-a-valid-command"]] - errors = [ - "Expected a string for readinessprobe at 0. Found 1", - "Expected a string for readinessprobe at 1." - " Found ['not-a-valid-command']", - ] - with self.assertRaises(ValueError) as context_mgr: + error = ( + r"Expected a string for readinessprobe at 0. Found 1" + r"\nExpected a string for readinessprobe at 1." + r" Found \['not-a-valid-command'\]" + ) + with pytest.raises(ValueError, match=error): cc_wireguard.readinessprobe_command_validation(wg_readinessprobes) - error_msg = str(context_mgr.exception) - for error in errors: - self.assertIn(error, error_msg) def test_suppl_schema_error_on_missing_keys(self): """ValueError raised reporting any missing required keys""" @@ -50,33 +43,30 @@ def test_suppl_schema_error_on_missing_keys(self): f"Invalid wireguard interface configuration:{NL}" "Missing required wg:interfaces keys: config_path, content, name" ) - with self.assertRaisesRegex(ValueError, match): + with pytest.raises(ValueError, match=match): cc_wireguard.supplemental_schema_validation(cfg) def test_suppl_schema_error_on_non_string_values(self): """ValueError raised for any values expected as string type.""" cfg = {"name": 1, "config_path": 2, "content": 3} - errors = [ - "Expected a string for wg:interfaces:config_path. Found 2", - "Expected a string for wg:interfaces:content. Found 3", - "Expected a string for wg:interfaces:name. Found 1", - ] - with self.assertRaises(ValueError) as context_mgr: + error = ( + "Expected a string for wg:interfaces:config_path. Found 2\n" + "Expected a string for wg:interfaces:content. Found 3\n" + "Expected a string for wg:interfaces:name. Found 1" + ) + with pytest.raises(ValueError, match=error): cc_wireguard.supplemental_schema_validation(cfg) - error_msg = str(context_mgr.exception) - for error in errors: - self.assertIn(error, error_msg) def test_write_config_failed(self): """Errors when writing config are raised.""" wg_int = {"name": "wg0", "config_path": "/no/valid/path"} - with self.assertRaises(RuntimeError) as context_mgr: + with pytest.raises( + RuntimeError, + match="Failure writing Wireguard configuration file" + " /no/valid/path:\n", + ): cc_wireguard.write_config(wg_int) - self.assertIn( - "Failure writing Wireguard configuration file /no/valid/path:\n", - str(context_mgr.exception), - ) @mock.patch("%s.subp.subp" % MPATH) def test_readiness_probe_invalid_command(self, m_subp): @@ -92,16 +82,15 @@ def fake_subp(cmd, capture=None, shell=None): m_subp.side_effect = fake_subp - with self.assertRaises(RuntimeError) as context_mgr: - cc_wireguard.readinessprobe(wg_readinessprobes) - self.assertIn( + error = ( "Failed running readinessprobe command:\n" "not-a-valid-command: Unexpected error while" " running command.\n" "Command: -\nExit code: -\nReason: -\n" - "Stdout: not-a-valid-command: command not found\nStderr: -", - str(context_mgr.exception), + "Stdout: not-a-valid-command: command not found\nStderr: -" ) + with pytest.raises(RuntimeError, match=error): + cc_wireguard.readinessprobe(wg_readinessprobes) @mock.patch("%s.subp.subp" % MPATH) def test_enable_wg_on_error(self, m_subp): @@ -112,16 +101,15 @@ def test_enable_wg_on_error(self, m_subp): "systemctl start wg-quik@wg0 failed: exit code 1" ) mycloud = FakeCloud(distro) - with self.assertRaises(RuntimeError) as context_mgr: - cc_wireguard.enable_wg(wg_int, mycloud) - self.assertEqual( - "Failed enabling/starting Wireguard interface(s):\n" + error = ( + r"Failed enabling/starting Wireguard interface\(s\):\n" "Unexpected error while running command.\n" "Command: -\nExit code: -\nReason: -\n" "Stdout: systemctl start wg-quik@wg0 failed: exit code 1\n" - "Stderr: -", - str(context_mgr.exception), + "Stderr: -" ) + with pytest.raises(RuntimeError, match=error): + cc_wireguard.enable_wg(wg_int, mycloud) @mock.patch("%s.subp.which" % MPATH) def test_maybe_install_wg_packages_noop_when_wg_tools_present( @@ -138,7 +126,7 @@ def test_maybe_install_wg_packages_noop_when_wg_tools_present( @mock.patch("%s.subp.which" % MPATH) @mock.patch("%s.util.kernel_version" % MPATH) def test_maybe_install_wf_tools_raises_update_errors( - self, m_kernel_version, m_which + self, m_kernel_version, m_which, caplog ): """maybe_install_wireguard_packages logs and raises apt update errors.""" @@ -148,17 +136,16 @@ def test_maybe_install_wf_tools_raises_update_errors( distro.update_package_sources.side_effect = RuntimeError( "Some apt error" ) - with self.assertRaises(RuntimeError) as context_manager: + with pytest.raises(RuntimeError, match="Some apt error"): cc_wireguard.maybe_install_wireguard_packages( cloud=FakeCloud(distro) ) - self.assertEqual("Some apt error", str(context_manager.exception)) - self.assertIn("Package update failed\nTraceback", self.logs.getvalue()) + assert "Package update failed\nTraceback" in caplog.text @mock.patch("%s.subp.which" % MPATH) @mock.patch("%s.util.kernel_version" % MPATH) def test_maybe_install_wg_raises_install_errors( - self, m_kernel_version, m_which + self, m_kernel_version, m_which, caplog ): """maybe_install_wireguard_packages logs and raises package install errors.""" @@ -169,34 +156,32 @@ def test_maybe_install_wg_raises_install_errors( distro.install_packages.side_effect = RuntimeError( "Some install error" ) - with self.assertRaises(RuntimeError) as context_manager: + with pytest.raises(RuntimeError, match="Some install error"): cc_wireguard.maybe_install_wireguard_packages( cloud=FakeCloud(distro) ) - self.assertEqual("Some install error", str(context_manager.exception)) - self.assertIn( - "Failed to install wireguard-tools\n", self.logs.getvalue() - ) + assert "Failed to install wireguard-tools\n" in caplog.text @mock.patch("%s.subp.subp" % MPATH) - def test_load_wg_module_failed(self, m_subp): + def test_load_wg_module_failed(self, m_subp, caplog): """load_wireguard_kernel_module logs and raises kernel modules loading error.""" m_subp.side_effect = subp.ProcessExecutionError( "Some kernel module load error" ) - with self.assertRaises(subp.ProcessExecutionError) as context_manager: - cc_wireguard.load_wireguard_kernel_module() - self.assertEqual( + error = ( "Unexpected error while running command.\n" "Command: -\nExit code: -\nReason: -\n" "Stdout: Some kernel module load error\n" - "Stderr: -", - str(context_manager.exception), - ) - self.assertIn( - "WARNING: Could not load wireguard module:\n", self.logs.getvalue() + "Stderr: -" ) + with pytest.raises(subp.ProcessExecutionError, match=error): + cc_wireguard.load_wireguard_kernel_module() + assert ( + mock.ANY, + logging.WARNING, + "Could not load wireguard module:\n" + error, + ) in caplog.record_tuples @mock.patch("%s.subp.which" % MPATH) @mock.patch("%s.util.kernel_version" % MPATH) @@ -217,29 +202,28 @@ def test_maybe_install_wg_packages_happy_path( distro.install_packages.assert_called_once_with(packages) @mock.patch("%s.maybe_install_wireguard_packages" % MPATH) - def test_handle_no_config(self, m_maybe_install_wireguard_packages): + def test_handle_no_config( + self, m_maybe_install_wireguard_packages, caplog + ): """When no wireguard configuration is provided, nothing happens.""" cfg = {} cc_wireguard.handle("wg", cfg=cfg, cloud=None, args=None) - self.assertIn( - "DEBUG: Skipping module named wg, no 'wireguard'" - " configuration found", - self.logs.getvalue(), - ) - self.assertEqual(m_maybe_install_wireguard_packages.call_count, 0) + assert ( + mock.ANY, + logging.DEBUG, + "Skipping module named wg, no 'wireguard' configuration found", + ) in caplog.record_tuples + assert m_maybe_install_wireguard_packages.call_count == 0 def test_readiness_probe_with_non_string_values(self): """ValueError raised for any values expected as string type.""" cfg = [1, 2] - errors = [ - "Expected a string for readinessprobe at 0. Found 1", - "Expected a string for readinessprobe at 1. Found 2", - ] - with self.assertRaises(ValueError) as context_manager: + error = ( + "Expected a string for readinessprobe at 0. Found 1\n" + "Expected a string for readinessprobe at 1. Found 2" + ) + with pytest.raises(ValueError, match=error): cc_wireguard.readinessprobe_command_validation(cfg) - error_msg = str(context_manager.exception) - for error in errors: - self.assertIn(error, error_msg) class TestWireguardSchema: diff --git a/tests/unittests/config/test_cc_write_files.py b/tests/unittests/config/test_cc_write_files.py index 7f7f1740..398fea66 100644 --- a/tests/unittests/config/test_cc_write_files.py +++ b/tests/unittests/config/test_cc_write_files.py @@ -5,8 +5,7 @@ import io import logging import re -import shutil -import tempfile +from unittest import mock import pytest import responses @@ -18,12 +17,8 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import ( - SCHEMA_EMPTY_ERROR, - CiTestCase, - FilesystemMockingTestCase, - skipUnlessJsonSchema, -) +from tests.unittests.helpers import SCHEMA_EMPTY_ERROR, skipUnlessJsonSchema +from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) @@ -64,39 +59,47 @@ } -class TestWriteFiles(FilesystemMockingTestCase): +OWNER = "root:root" +USER = "root" +GROUP = "root" + - with_logs = True - owner = "root:root" +@pytest.fixture +def cloud(): + cc = get_cloud("ubuntu") + with mock.patch.object(cc.distro, "default_owner", OWNER): + yield cc - def setUp(self): - super(TestWriteFiles, self).setUp() - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - def test_simple(self): - self.patchUtils(self.tmp) +@pytest.mark.usefixtures("fake_filesystem") +@mock.patch("cloudinit.config.cc_write_files.util.chownbyname") +class TestWriteFiles: + def test_simple(self, m_chownbyname): expected = "hello world\n" - filename = "/tmp/my.file" + filename = str("/tmp/my.file") write_files( "test_simple", [{"content": expected, "path": filename}], - self.owner, + OWNER, ) - self.assertEqual(util.load_text_file(filename), expected) + assert util.load_text_file(filename) == expected + assert [ + mock.call(mock.ANY, USER, GROUP) + ] == m_chownbyname.call_args_list - def test_empty(self): - self.patchUtils(self.tmp) - filename = "/tmp/my.file" + def test_empty(self, m_chownbyname): + filename = str("/tmp/my.file") write_files( "test_empty", [{"path": filename}], - self.owner, + OWNER, ) - self.assertEqual(util.load_text_file(filename), "") + assert util.load_text_file(filename) == "" + assert [ + mock.call(mock.ANY, USER, GROUP) + ] == m_chownbyname.call_args_list - def test_append(self): - self.patchUtils(self.tmp) + def test_append(self, m_chownbyname): existing = "hello " added = "world\n" expected = existing + added @@ -105,20 +108,26 @@ def test_append(self): write_files( "test_append", [{"content": added, "path": filename, "append": "true"}], - self.owner, + OWNER, ) - self.assertEqual(util.load_text_file(filename), expected) - - def test_yaml_binary(self): - self.patchUtils(self.tmp) - data = util.load_yaml(YAML_TEXT) - write_files("testname", data["write_files"], self.owner) + assert util.load_text_file(filename) == expected + assert [ + mock.call(mock.ANY, USER, GROUP) + ] == m_chownbyname.call_args_list + + def test_yaml_binary(self, m_chownbyname): + data_wrong_paths = util.load_yaml(YAML_TEXT) + data = [] + for content in data_wrong_paths["write_files"]: + content["path"] = content["path"] + data.append(content) + + write_files("testname", data, OWNER) for path, content in YAML_CONTENT_EXPECTED.items(): - self.assertEqual(util.load_text_file(path), content) - - def test_all_decodings(self): - self.patchUtils(self.tmp) + assert util.load_text_file(path) == content + assert 5 == m_chownbyname.call_count + def test_all_decodings(self, m_chownbyname): # build a 'files' array that has a dictionary of encodings # for 'gz', 'gzip', 'gz+base64' ... data = b"foobzr" @@ -147,19 +156,19 @@ def test_all_decodings(self): files.append(cur) expected.append((path, data)) - write_files("test_decoding", files, self.owner) + write_files("test_decoding", files, OWNER) for path, content in expected: - self.assertEqual(util.load_binary_file(path), content) + assert util.load_binary_file(path) == content # make sure we actually wrote *some* files. flen_expected = len(gz_aliases + gz_b64_aliases + b64_aliases) * len( datum ) - self.assertEqual(len(expected), flen_expected) + assert len(expected) == flen_expected + assert len(expected) == m_chownbyname.call_count - def test_handle_plain_text(self): - self.patchUtils(self.tmp) + def test_handle_plain_text(self, m_chownbyname, caplog, cloud): file_path = "/tmp/file-text-plain" content = "asdf" cfg = { @@ -172,15 +181,14 @@ def test_handle_plain_text(self): } ] } - cc = self.tmp_cloud("ubuntu") - handle("ignored", cfg, cc, []) + handle("ignored", cfg, cloud, []) assert content == util.load_text_file(file_path) - self.assertNotIn( - "Unknown encoding type text/plain", self.logs.getvalue() - ) + assert "Unknown encoding type text/plain" not in caplog.text + assert [ + mock.call(mock.ANY, USER, GROUP) + ] == m_chownbyname.call_args_list - def test_file_uri(self): - self.patchUtils(self.tmp) + def test_file_uri(self, m_chownbyname, cloud): src_path = "/tmp/file-uri" dst_path = "/tmp/file-uri-target" content = "asdf" @@ -193,15 +201,12 @@ def test_file_uri(self): } ] } - cc = self.tmp_cloud("ubuntu") - handle("ignored", cfg, cc, []) - self.assertEqual( - util.load_text_file(src_path), util.load_text_file(dst_path) - ) + handle("ignored", cfg, cloud, []) + assert util.load_text_file(src_path) == util.load_text_file(dst_path) + assert m_chownbyname.call_count @responses.activate - def test_http_uri(self): - self.patchUtils(self.tmp) + def test_http_uri(self, m_chownbyname, cloud): path = "/tmp/http-uri-target" url = "http://hostname/path" content = "more asdf" @@ -220,13 +225,14 @@ def test_http_uri(self): } ] } - cc = self.tmp_cloud("ubuntu") - handle("ignored", cfg, cc, []) - self.assertEqual(content, util.load_text_file(path)) - - def test_uri_fallback(self): - self.patchUtils(self.tmp) - src_path = "/tmp/INVALID" + handle("ignored", cfg, cloud, []) + assert content == util.load_text_file(path) + assert [ + mock.call(mock.ANY, USER, GROUP) + ] == m_chownbyname.call_args_list + + def test_uri_fallback(self, m_chownbyname, cloud): + src_path = "tmp/INVALID" dst_path = "/tmp/uri-fallback-target" content = "asdf" util.del_file(src_path) @@ -240,45 +246,47 @@ def test_uri_fallback(self): } ] } - cc = self.tmp_cloud("ubuntu") - handle("ignored", cfg, cc, []) - self.assertEqual(content, util.load_text_file(dst_path)) + handle("ignored", cfg, cloud, []) + assert content == util.load_text_file(dst_path) + assert [ + mock.call(mock.ANY, USER, GROUP) + ] == m_chownbyname.call_args_list - def test_deferred(self): - self.patchUtils(self.tmp) + def test_deferred(self, m_chownbyname, cloud): file_path = "/tmp/deferred.file" config = {"write_files": [{"path": file_path, "defer": True}]} - cc = self.tmp_cloud("ubuntu") - handle("cc_write_file", config, cc, []) - with self.assertRaises(FileNotFoundError): + handle("cc_write_file", config, cloud, []) + with pytest.raises(FileNotFoundError): util.load_text_file(file_path) + assert [] == m_chownbyname.call_args_list -class TestDecodePerms(CiTestCase): - - with_logs = True - +class TestDecodePerms: def test_none_returns_default(self): """If None is passed as perms, then default should be returned.""" default = object() found = decode_perms(None, default) - self.assertEqual(default, found) + assert default == found def test_integer(self): """A valid integer should return itself.""" found = decode_perms(0o755, None) - self.assertEqual(0o755, found) + assert 0o755 == found def test_valid_octal_string(self): """A string should be read as octal.""" found = decode_perms("644", None) - self.assertEqual(0o644, found) + assert 0o644 == found - def test_invalid_octal_string_returns_default_and_warns(self): + def test_invalid_octal_string_returns_default_and_warns(self, caplog): """A string with invalid octal should warn and return default.""" found = decode_perms("999", None) - self.assertIsNone(found) - self.assertIn("WARNING: Undecodable", self.logs.getvalue()) + assert found is None + assert ( + mock.ANY, + logging.WARNING, + "Undecodable permissions '999', returning default None", + ) in caplog.record_tuples def _gzip_bytes(data): diff --git a/tests/unittests/config/test_cc_write_files_deferred.py b/tests/unittests/config/test_cc_write_files_deferred.py index 3fbf800d..f12c9fd9 100644 --- a/tests/unittests/config/test_cc_write_files_deferred.py +++ b/tests/unittests/config/test_cc_write_files_deferred.py @@ -1,8 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging -import shutil -import tempfile +from unittest import mock import pytest @@ -13,25 +12,19 @@ get_schema, validate_cloudconfig_schema, ) -from tests.unittests.helpers import ( - FilesystemMockingTestCase, - skipUnlessJsonSchema, -) +from tests.unittests.helpers import skipUnlessJsonSchema +from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) -class TestWriteFilesDeferred(FilesystemMockingTestCase): - - with_logs = True +@pytest.mark.usefixtures("fake_filesystem") +class TestWriteFilesDeferred: - def setUp(self): - super(TestWriteFilesDeferred, self).setUp() - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) + USER = "root" - def test_filtering_deferred_files(self): - self.patchUtils(self.tmp) + @mock.patch("cloudinit.config.cc_write_files.util.chownbyname") + def test_filtering_deferred_files(self, m_chownbyname): expected = "hello world\n" config = { "write_files": [ @@ -43,11 +36,18 @@ def test_filtering_deferred_files(self): {"path": "/tmp/not_deferred.file"}, ] } - cc = self.tmp_cloud("ubuntu") - handle("cc_write_files_deferred", config, cc, []) - self.assertEqual(util.load_text_file("/tmp/deferred.file"), expected) - with self.assertRaises(FileNotFoundError): + cc = get_cloud("ubuntu") + # fake_filesytem's tree is owned by $USER:$USER + with mock.patch.object( + cc.distro, "default_owner", f"{self.USER}:{self.USER}" + ): + handle("cc_write_files_deferred", config, cc, []) + assert util.load_text_file("/tmp/deferred.file") == expected + with pytest.raises(FileNotFoundError): util.load_text_file("/tmp/not_deferred.file") + assert [ + mock.call(mock.ANY, self.USER, self.USER) + ] == m_chownbyname.call_args_list class TestWriteFilesDeferredSchema: diff --git a/tests/unittests/config/test_cc_yum_add_repo.py b/tests/unittests/config/test_cc_yum_add_repo.py index c77262f5..5d420631 100644 --- a/tests/unittests/config/test_cc_yum_add_repo.py +++ b/tests/unittests/config/test_cc_yum_add_repo.py @@ -3,8 +3,6 @@ import configparser import logging import re -import shutil -import tempfile import pytest @@ -20,12 +18,8 @@ LOG = logging.getLogger(__name__) -class TestConfig(helpers.FilesystemMockingTestCase): - def setUp(self): - super(TestConfig, self).setUp() - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - +@pytest.mark.usefixtures("fake_filesystem") +class TestConfig: def test_bad_config(self): cfg = { "yum_repos": { @@ -42,11 +36,9 @@ def test_bad_config(self): }, }, } - self.patchUtils(self.tmp) cc_yum_add_repo.handle("yum_add_repo", cfg, None, []) - self.assertRaises( - IOError, util.load_text_file, "/etc/yum.repos.d/epel_testing.repo" - ) + with pytest.raises(IOError): + util.load_text_file("/etc/yum.repos.d/epel_testing.repo") def test_metalink_config(self): cfg = { @@ -61,8 +53,6 @@ def test_metalink_config(self): }, }, } - self.patchUtils(self.tmp) - self.patchOS(self.tmp) cc_yum_add_repo.handle("yum_add_repo", cfg, None, []) contents = util.load_text_file("/etc/yum.repos.d/epel-testing.repo") parser = configparser.ConfigParser() @@ -78,12 +68,11 @@ def test_metalink_config(self): } } for section in expected: - self.assertTrue( - parser.has_section(section), - "Contains section {0}".format(section), + assert parser.has_section(section), "Contains section {0}".format( + section ) for k, v in expected[section].items(): - self.assertEqual(parser.get(section, k), v) + assert parser.get(section, k) == v def test_mirrorlist_config(self): cfg = { @@ -98,8 +87,6 @@ def test_mirrorlist_config(self): }, }, } - self.patchUtils(self.tmp) - self.patchOS(self.tmp) cc_yum_add_repo.handle("yum_add_repo", cfg, None, []) contents = util.load_text_file("/etc/yum.repos.d/epel-testing.repo") parser = configparser.ConfigParser() @@ -115,12 +102,11 @@ def test_mirrorlist_config(self): } } for section in expected: - self.assertTrue( - parser.has_section(section), - "Contains section {0}".format(section), + assert parser.has_section(section), "Contains section {0}".format( + section ) for k, v in expected[section].items(): - self.assertEqual(parser.get(section, k), v) + assert parser.get(section, k) == v def test_write_config(self): cfg = { @@ -135,8 +121,6 @@ def test_write_config(self): }, }, } - self.patchUtils(self.tmp) - self.patchOS(self.tmp) cc_yum_add_repo.handle("yum_add_repo", cfg, None, []) contents = util.load_text_file("/etc/yum.repos.d/epel-testing.repo") parser = configparser.ConfigParser() @@ -152,12 +136,11 @@ def test_write_config(self): } } for section in expected: - self.assertTrue( - parser.has_section(section), - "Contains section {0}".format(section), + assert parser.has_section(section), "Contains section {0}".format( + section ) for k, v in expected[section].items(): - self.assertEqual(parser.get(section, k), v) + assert parser.get(section, k) == v def test_write_config_array(self): cfg = { @@ -176,7 +159,6 @@ def test_write_config_array(self): } } } - self.patchUtils(self.tmp) cc_yum_add_repo.handle("yum_add_repo", cfg, None, []) contents = util.load_text_file( "/etc/yum.repos.d/puppetlabs-products.repo" @@ -196,12 +178,11 @@ def test_write_config_array(self): } } for section in expected: - self.assertTrue( - parser.has_section(section), - "Contains section {0}".format(section), + assert parser.has_section(section), "Contains section {0}".format( + section ) for k, v in expected[section].items(): - self.assertEqual(parser.get(section, k), v) + assert parser.get(section, k) == v class TestAddYumRepoSchema: diff --git a/tests/unittests/config/test_cc_zypper_add_repo.py b/tests/unittests/config/test_cc_zypper_add_repo.py index 21f033af..aab273f6 100644 --- a/tests/unittests/config/test_cc_zypper_add_repo.py +++ b/tests/unittests/config/test_cc_zypper_add_repo.py @@ -5,6 +5,8 @@ import logging import os +import pytest + from cloudinit import util from cloudinit.config import cc_zypper_add_repo from tests.unittests import helpers @@ -12,11 +14,11 @@ LOG = logging.getLogger(__name__) -class TestConfig(helpers.FilesystemMockingTestCase): - def setUp(self): - super(TestConfig, self).setUp() - self.tmp = self.tmp_dir() - self.zypp_conf = "etc/zypp/zypp.conf" +ZYPP_CONF = "etc/zypp/zypp.conf" + + +@pytest.mark.usefixtures("fake_filesystem") +class TestConfig: def test_bad_repo_config(self): """Config has no baseurl, no file should be written""" @@ -25,16 +27,14 @@ def test_bad_repo_config(self): {"id": "foo", "name": "suse-test", "enabled": "1"}, ] } - self.patchUtils(self.tmp) cc_zypper_add_repo._write_repos(cfg["repos"], "/etc/zypp/repos.d") - self.assertRaises( - IOError, util.load_text_file, "/etc/zypp/repos.d/foo.repo" - ) + with pytest.raises(IOError): + util.load_text_file("/etc/zypp/repos.d/foo.repo") - def test_write_repos(self): + def test_write_repos(self, tmp_path): """Verify valid repos get written""" cfg = self._get_base_config_repos() - root_d = self.tmp_dir() + root_d = str(tmp_path) cc_zypper_add_repo._write_repos(cfg["zypper"]["repos"], root_d) repos = glob.glob("%s/*.repo" % root_d) expected_repos = ["testing-foo.repo", "testing-bar.repo"] @@ -46,7 +46,7 @@ def test_write_repos(self): assert 'Found repo with name "%s"; unexpected' % repo_name # Validation that the content gets properly written is in another test - def test_write_repo(self): + def test_write_repo(self, tmp_path): """Verify the content of a repo file""" cfg = { "repos": [ @@ -57,7 +57,7 @@ def test_write_repo(self): }, ] } - root_d = self.tmp_dir() + root_d = str(tmp_path) cc_zypper_add_repo._write_repos(cfg["repos"], root_d) contents = util.load_text_file("%s/testing-foo.repo" % root_d) parser = configparser.ConfigParser() @@ -71,21 +71,19 @@ def test_write_repo(self): } } for section in expected: - self.assertTrue( - parser.has_section(section), - "Contains section {0}".format(section), + assert parser.has_section(section), "Contains section {0}".format( + section ) for k, v in expected[section].items(): - self.assertEqual(parser.get(section, k), v) + assert parser.get(section, k) == v - def test_config_write(self): + def test_config_write(self, tmp_path): """Write valid configuration data""" cfg = {"config": {"download.deltarpm": "False", "reposdir": "foo"}} - root_d = self.tmp_dir() - helpers.populate_dir(root_d, {self.zypp_conf: "# Zypp config\n"}) - self.reRoot(root_d) + root_d = str(tmp_path) + helpers.populate_dir(root_d, {ZYPP_CONF: "# Zypp config\n"}) cc_zypper_add_repo._write_zypp_config(cfg["config"]) - cfg_out = os.path.join(root_d, self.zypp_conf) + cfg_out = os.path.join(root_d, ZYPP_CONF) contents = util.load_text_file(cfg_out) expected = [ "# Zypp config", @@ -95,9 +93,9 @@ def test_config_write(self): ] for item in contents.split("\n"): if item not in expected: - self.assertIsNone(item) + assert item is None - def test_config_write_skip_configdir(self): + def test_config_write_skip_configdir(self, tmp_path): """Write configuration but skip writing 'configdir' setting""" cfg = { "config": { @@ -106,11 +104,10 @@ def test_config_write_skip_configdir(self): "configdir": "bar", } } - root_d = self.tmp_dir() - helpers.populate_dir(root_d, {self.zypp_conf: "# Zypp config\n"}) - self.reRoot(root_d) + root_d = str(tmp_path) + helpers.populate_dir(root_d, {ZYPP_CONF: "# Zypp config\n"}) cc_zypper_add_repo._write_zypp_config(cfg["config"]) - cfg_out = os.path.join(root_d, self.zypp_conf) + cfg_out = os.path.join(root_d, ZYPP_CONF) contents = util.load_text_file(cfg_out) expected = [ "# Zypp config", @@ -120,48 +117,45 @@ def test_config_write_skip_configdir(self): ] for item in contents.split("\n"): if item not in expected: - self.assertIsNone(item) + assert item is None # Not finding teh right path for mocking :( # assert mock_logging.warning.called - def test_empty_config_section_no_new_data(self): + def test_empty_config_section_no_new_data(self, tmp_path): """When the config section is empty no new data should be written to zypp.conf""" cfg = self._get_base_config_repos() cfg["zypper"]["config"] = None - root_d = self.tmp_dir() - helpers.populate_dir(root_d, {self.zypp_conf: "# No data"}) - self.reRoot(root_d) + root_d = str(tmp_path) + helpers.populate_dir(root_d, {ZYPP_CONF: "# No data"}) cc_zypper_add_repo._write_zypp_config(cfg.get("config", {})) - cfg_out = os.path.join(root_d, self.zypp_conf) + cfg_out = os.path.join(root_d, ZYPP_CONF) contents = util.load_text_file(cfg_out) - self.assertEqual(contents, "# No data") + assert contents == "# No data" - def test_empty_config_value_no_new_data(self): + def test_empty_config_value_no_new_data(self, tmp_path): """When the config section is not empty but there are no values no new data should be written to zypp.conf""" cfg = self._get_base_config_repos() cfg["zypper"]["config"] = {"download.deltarpm": None} - root_d = self.tmp_dir() - helpers.populate_dir(root_d, {self.zypp_conf: "# No data"}) - self.reRoot(root_d) + root_d = str(tmp_path) + helpers.populate_dir(root_d, {ZYPP_CONF: "# No data"}) cc_zypper_add_repo._write_zypp_config(cfg.get("config", {})) - cfg_out = os.path.join(root_d, self.zypp_conf) + cfg_out = os.path.join(root_d, ZYPP_CONF) contents = util.load_text_file(cfg_out) - self.assertEqual(contents, "# No data") + assert contents == "# No data" - def test_handler_full_setup(self): + def test_handler_full_setup(self, tmp_path): """Test that the handler ends up calling the renderers""" cfg = self._get_base_config_repos() cfg["zypper"]["config"] = { "download.deltarpm": "False", } - root_d = self.tmp_dir() + root_d = str(tmp_path) os.makedirs("%s/etc/zypp/repos.d" % root_d) - helpers.populate_dir(root_d, {self.zypp_conf: "# Zypp config\n"}) - self.reRoot(root_d) + helpers.populate_dir(root_d, {ZYPP_CONF: "# Zypp config\n"}) cc_zypper_add_repo.handle("zypper_add_repo", cfg, None, []) - cfg_out = os.path.join(root_d, self.zypp_conf) + cfg_out = os.path.join(root_d, ZYPP_CONF) contents = util.load_text_file(cfg_out) expected = [ "# Zypp config", @@ -170,7 +164,7 @@ def test_handler_full_setup(self): ] for item in contents.split("\n"): if item not in expected: - self.assertIsNone(item) + assert item is None repos = glob.glob("%s/etc/zypp/repos.d/*.repo" % root_d) expected_repos = ["testing-foo.repo", "testing-bar.repo"] if len(repos) != 2: @@ -180,25 +174,24 @@ def test_handler_full_setup(self): if repo_name not in expected_repos: assert 'Found repo with name "%s"; unexpected' % repo_name - def test_no_config_section_no_new_data(self): + def test_no_config_section_no_new_data(self, tmp_path): """When there is no config section no new data should be written to zypp.conf""" cfg = self._get_base_config_repos() - root_d = self.tmp_dir() - helpers.populate_dir(root_d, {self.zypp_conf: "# No data"}) - self.reRoot(root_d) + root_d = str(tmp_path) + helpers.populate_dir(root_d, {ZYPP_CONF: "# No data"}) cc_zypper_add_repo._write_zypp_config(cfg.get("config", {})) - cfg_out = os.path.join(root_d, self.zypp_conf) + cfg_out = os.path.join(root_d, ZYPP_CONF) contents = util.load_text_file(cfg_out) - self.assertEqual(contents, "# No data") + assert contents == "# No data" - def test_no_repo_data(self): + def test_no_repo_data(self, tmp_path): """When there is no repo data nothing should happen""" - root_d = self.tmp_dir() - self.reRoot(root_d) + # fake_filesystem creates a `tmp` dir a more under tmp_path + root_d = str(tmp_path / "isolated") cc_zypper_add_repo._write_repos(None, root_d) content = glob.glob("%s/*" % root_d) - self.assertEqual(len(content), 0) + assert len(content) == 0 def _get_base_config_repos(self): """Basic valid repo configuration""" diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 19195b2b..d158761e 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -47,7 +47,6 @@ from tests.hypothesis import given from tests.hypothesis_jsonschema import from_schema from tests.unittests.helpers import ( - CiTestCase, does_not_raise, mock, skipUnlessHypothesisJsonSchema, @@ -183,13 +182,13 @@ def check_deprecation_keys(schema, search_key): }, "new", ) - with self.assertRaises(AssertionError): + with pytest.raises(AssertionError): check_deprecation_keys({"changed": True}, "changed") - with self.assertRaises(AssertionError): + with pytest.raises(AssertionError): check_deprecation_keys( {"properties": {"deprecated": True}}, "deprecated" ) - with self.assertRaises(AssertionError): + with pytest.raises(AssertionError): check_deprecation_keys( {"properties": {"properties": {"new": True}}}, "new" ) @@ -254,6 +253,7 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_phone_home"}, {"$ref": "#/$defs/cc_power_state_change"}, {"$ref": "#/$defs/cc_puppet"}, + {"$ref": "#/$defs/cc_raspberry_pi"}, {"$ref": "#/$defs/cc_resizefs"}, {"$ref": "#/$defs/cc_resolv_conf"}, {"$ref": "#/$defs/cc_rh_subscription"}, @@ -323,7 +323,7 @@ def test_validate_data_file_schema(self, mocker, paths): ) -class SchemaValidationErrorTest(CiTestCase): +class SchemaValidationErrorTest: """Test validate_cloudconfig_schema""" def test_schema_validation_error_expects_schema_errors(self): @@ -335,14 +335,14 @@ def test_schema_validation_error_expects_schema_errors(self): ), ] exception = SchemaValidationError(schema_errors=errors) - self.assertIsInstance(exception, Exception) - self.assertEqual(exception.schema_errors, errors) - self.assertEqual( + assert isinstance(exception, Exception) + assert exception.schema_errors == errors + assert ( 'Cloud config schema errors: key.path: unexpected key "junk", ' - 'key2.path: "-123" is not a valid "hostname" format', - str(exception), + 'key2.path: "-123" is not a valid "hostname" format' + == str(exception) ) - self.assertTrue(isinstance(exception, ValueError)) + assert isinstance(exception, ValueError) class FakeNetplanParserException(Exception): @@ -456,6 +456,9 @@ def test_validateconfig_schema_non_strict_emits_warnings(self, caplog): """When strict is False validate_cloudconfig_schema emits warnings.""" schema = {"properties": {"p1": {"type": "string"}}} validate_cloudconfig_schema({"p1": -1}, schema=schema, strict=False) + assert ( + caplog.record_tuples and len(caplog.record_tuples) == 1 + ), caplog.record_tuples [(module, log_level, log_msg)] = caplog.record_tuples assert "cloudinit.config.schema" == module assert logging.WARNING == log_level @@ -1503,6 +1506,13 @@ class TestSchemaDocExamples: def test_cloud_config_schema_doc_examples(self, example_path): validate_cloudconfig_file(example_path, self.schema) + # Assert no use of deprecated keys + validate_cloudconfig_schema( + config=yaml.safe_load(open(example_path)), + schema=self.schema, + strict=True, + ) + @pytest.mark.parametrize( "example_path", _get_meta_doc_examples(file_glob="network-config-v1*yaml"), @@ -1798,6 +1808,27 @@ class TestNetworkSchema: "", id="GH-4710_mtu_none_and_str_address", ), + pytest.param( + { + "network": { + "version": 1, + "config": [ + { + "type": "physical", + "name": "eth0", + "subnets": [ + {"type": "dhcp4", "metric": 100}, + {"type": "dhcp6", "metric": 1000}, + ], + } + ], + } + }, + SchemaType.NETWORK_CONFIG_V1, + does_not_raise(), + "", + id="subnet_metric_validation", + ), ), ) @mock.patch("cloudinit.net.netplan.available", return_value=False) diff --git a/tests/unittests/conftest.py b/tests/unittests/conftest.py index 3042dee1..af0ed891 100644 --- a/tests/unittests/conftest.py +++ b/tests/unittests/conftest.py @@ -7,11 +7,26 @@ import pytest -from cloudinit import atomic_helper, lifecycle, util +from cloudinit import atomic_helper, distros, helpers, lifecycle, temp_utils +from cloudinit import user_data as ud +from cloudinit import util from cloudinit.gpg import GPG from cloudinit.log import loggers from tests.hypothesis import HAS_HYPOTHESIS -from tests.unittests.helpers import example_netdev, retarget_many_wrapper +from tests.unittests.helpers import ( + example_netdev, + rebase_path, + retarget_many_wrapper, +) + + +@pytest.fixture +def Distro(paths): + def _get_distro(name, cfg=None): + cls = distros.fetch(name) + return cls(name, cfg or {}, paths) + + return _get_distro @pytest.fixture @@ -39,13 +54,16 @@ def m_gpg(): ("relpath", 1), ], os: [ + ("chmod", 2), + ("chown", 2), ("listdir", 1), + ("lstat", 1), ("mkdir", 1), + ("rename", 2), ("rmdir", 1), - ("lstat", 1), - ("symlink", 2), - ("stat", 1), ("scandir", 1), + ("stat", 1), + ("symlink", 2), ], util: [ ("write_file", 1), @@ -74,21 +92,51 @@ def m_gpg(): ], } +FS_VARS = { + temp_utils: ["_ROOT_TMPDIR", "_EXE_ROOT_TMPDIR"], +} + @pytest.fixture -def fake_filesystem(mocker, tmpdir): - """Mocks fs functions to operate under `tmpdir`""" +def fake_filesystem_hook(): + """A hook to interact with the real filesystem before mocking it in + fake_filesystem. + + Fixtures needing to access the real filesystem in tests that use + fake_filesystem, can depend on this fixture to ensure they run before + fake_filesystem. + + See in action in tests/unittests/runs/test_simple_run.py. + """ + + +@pytest.fixture +def fake_filesystem(mocker, tmpdir, fake_filesystem_hook): + """Mocks fs functions to operate under `tmpdir` + + This fixture is sorted after fix_cloud_init_hook to allow fixtures sorted + before fake_cloud_init_hook to access the real filesystem. + """ # This allows fake_filesystem to be used with production code that # creates temporary directories. Functions like TemporaryDirectory() - # attempt to create a directory under "/tmp" assuming that it already - # exists, but then it fails because of the retargeting that happens here. - tmpdir.mkdir("tmp") + # attempt to create a directory under $TMPDIR (among other locations) + # assuming that it already exists, but then it fails because of the + # retargeting that happens here. + TMPDIR = os.getenv("TMPDIR", "/tmp") + Path(tmpdir, TMPDIR[1:]).mkdir(exist_ok=True) for mod, funcs in FS_FUNCS.items(): for f, nargs in funcs: func = getattr(mod, f) trap_func = retarget_many_wrapper(str(tmpdir), nargs, func) mocker.patch.object(mod, f, trap_func) + + for mod, vars in FS_VARS.items(): + for var_name in vars: + var_val = getattr(mod, var_name) + new_var_val = rebase_path(var_val, str(tmpdir)) + mocker.patch.object(mod, var_name, new_var_val) + yield str(tmpdir) @@ -170,3 +218,24 @@ def tmp_path(tmpdir): settings.register_profile("ci", max_examples=1000) settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) + + +@pytest.fixture +def paths(tmpdir) -> helpers.Paths: + """ + Return a helpers.Paths object configured to use a tmpdir. + + (This uses the builtin tmpdir fixture.) + """ + dirs = { + "cloud_dir": tmpdir.mkdir("cloud_dir").strpath, + "docs_dir": tmpdir.mkdir("docs_dir").strpath, + "run_dir": tmpdir.mkdir("run_dir").strpath, + "templates_dir": tmpdir.mkdir("templates_dir").strpath, + } + return helpers.Paths(dirs) + + +@pytest.fixture +def ud_proc(paths): + return ud.UserDataProcessor(paths) diff --git a/tests/unittests/distros/__init__.py b/tests/unittests/distros/__init__.py index e66b9446..da6365a5 100644 --- a/tests/unittests/distros/__init__.py +++ b/tests/unittests/distros/__init__.py @@ -1,19 +1 @@ # This file is part of cloud-init. See LICENSE file for license information. -import copy - -from cloudinit import distros, helpers, settings - - -def _get_distro(dtype, system_info=None): - """Return a Distro class of distro 'dtype'. - - cfg is format of CFG_BUILTIN['system_info']. - - example: _get_distro("debian") - """ - if system_info is None: - system_info = copy.deepcopy(settings.CFG_BUILTIN["system_info"]) - system_info["distro"] = dtype - paths = helpers.Paths(system_info["paths"]) - distro_cls = distros.fetch(dtype) - return distro_cls(dtype, system_info, paths) diff --git a/tests/unittests/distros/package_management/test_apt.py b/tests/unittests/distros/package_management/test_apt.py index 00523873..8eefa07a 100644 --- a/tests/unittests/distros/package_management/test_apt.py +++ b/tests/unittests/distros/package_management/test_apt.py @@ -8,7 +8,6 @@ from cloudinit import helpers, subp from cloudinit.distros.package_management import apt from cloudinit.distros.package_management.apt import APT_GET_COMMAND, Apt -from tests.unittests.helpers import get_mock_paths from tests.unittests.util import FakeDataSource M_PATH = "cloudinit.distros.package_management.apt.Apt." @@ -120,14 +119,14 @@ def test_search_stem(self, m_subp, m_which, mocker): @pytest.fixture(scope="function") -def apt_paths(tmpdir): - MockPaths = get_mock_paths(str(tmpdir)) +def apt_paths(paths, tmpdir): with mock.patch.object( apt, "APT_LOCK_FILES", [f"{tmpdir}/{FILE}" for FILE in apt.APT_LOCK_FILES], ): - yield MockPaths({}, FakeDataSource()) + paths.datasource = FakeDataSource() + yield paths class TestUpdatePackageSources: diff --git a/tests/unittests/distros/test__init__.py b/tests/unittests/distros/test__init__.py index 39583b13..2447d313 100644 --- a/tests/unittests/distros/test__init__.py +++ b/tests/unittests/distros/test__init__.py @@ -1,8 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -import shutil -import tempfile from unittest import mock import pytest @@ -10,7 +8,6 @@ from cloudinit import distros, util from cloudinit.distros.ubuntu import Distro from cloudinit.net.dhcp import Dhcpcd, IscDhclient, Udhcpc -from tests.unittests import helpers M_PATH = "cloudinit.distros." @@ -54,22 +51,13 @@ gapmi = distros._get_arch_package_mirror_info -class TestGenericDistro(helpers.FilesystemMockingTestCase): - with_logs = True - - def setUp(self): - super(TestGenericDistro, self).setUp() - # Make a temp directoy for tests to use. - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - +@pytest.mark.usefixtures("fake_filesystem") +class TestGenericDistro: def _write_load_doas(self, user, rules): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - if not os.path.exists(os.path.join(self.tmp, "etc")): - os.makedirs(os.path.join(self.tmp, "etc")) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) + if not os.path.exists("/etc"): + os.makedirs("/etc") d.write_doas_rules(user, rules) contents = util.load_text_file(d.doas_fn) return contents, cls, d @@ -77,10 +65,8 @@ def _write_load_doas(self, user, rules): def _write_load_sudoers(self, _user, rules): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - os.makedirs(os.path.join(self.tmp, "etc")) - os.makedirs(os.path.join(self.tmp, "etc", "sudoers.d")) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) + os.makedirs("/etc") + os.makedirs(os.path.join("/etc", "sudoers.d")) d.write_sudo_rules("harlowja", rules) contents = util.load_text_file(d.ci_sudoers_fn) return contents, cls, d @@ -98,7 +84,7 @@ def test_doas_ensure_rules(self): rules = ["permit nopass harlowja"] contents = self._write_load_doas("harlowja", rules)[0] expected = ["permit nopass harlowja"] - self.assertEqual(len(expected), self._count_in(expected, contents)) + assert len(expected) == self._count_in(expected, contents) def test_doas_ensure_rules_list(self): rules = [ @@ -112,7 +98,7 @@ def test_doas_ensure_rules_list(self): "permit nopass harlowja cmd pwd", "permit nopass harlowja cmd df", ] - self.assertEqual(len(expected), self._count_in(expected, contents)) + assert len(expected) == self._count_in(expected, contents) def test_doas_ensure_handle_duplicates(self): rules = [ @@ -129,7 +115,7 @@ def test_doas_ensure_handle_duplicates(self): "permit nopass harlowja cmd pwd", "permit nopass harlowja cmd df", ] - self.assertEqual(len(expected), self._count_in(expected, contents)) + assert len(expected) == self._count_in(expected, contents) def test_doas_ensure_new(self): rules = [ @@ -138,12 +124,11 @@ def test_doas_ensure_new(self): "permit nopass harlowja cmd df", ] contents = self._write_load_doas("harlowja", rules)[0] - self.assertIn("# Created by cloud-init v.", contents) - self.assertIn("harlowja", contents) - self.assertEqual(4, contents.count("harlowja")) + assert "# Created by cloud-init v." in contents + assert "harlowja" in contents + assert 4 == contents.count("harlowja") def test_doas_ensure_append(self): - self.patchUtils(self.tmp) util.write_file("/etc/doas.conf", "# root user\npermit nopass root\n") rules = [ "permit nopass harlowja cmd ls", @@ -151,22 +136,22 @@ def test_doas_ensure_append(self): "permit nopass harlowja cmd df", ] contents = self._write_load_doas("harlowja", rules)[0] - self.assertIn("root", contents) - self.assertEqual(2, contents.count("root")) - self.assertIn("harlowja", contents) - self.assertEqual(4, contents.count("harlowja")) + assert "root" in contents + assert 2 == contents.count("root") + assert "harlowja" in contents + assert 4 == contents.count("harlowja") def test_sudoers_ensure_rules(self): rules = "ALL=(ALL:ALL) ALL" contents = self._write_load_sudoers("harlowja", rules)[0] expected = ["harlowja ALL=(ALL:ALL) ALL"] - self.assertEqual(len(expected), self._count_in(expected, contents)) + assert len(expected) == self._count_in(expected, contents) not_expected = [ "harlowja A", "harlowja L", "harlowja L", ] - self.assertEqual(0, self._count_in(not_expected, contents)) + assert 0 == self._count_in(not_expected, contents) def test_sudoers_ensure_rules_list(self): rules = [ @@ -180,13 +165,13 @@ def test_sudoers_ensure_rules_list(self): "harlowja B-ALL=(ALL:ALL) ALL", "harlowja C-ALL=(ALL:ALL) ALL", ] - self.assertEqual(len(expected), self._count_in(expected, contents)) + assert len(expected) == self._count_in(expected, contents) not_expected = [ "harlowja A", "harlowja L", "harlowja L", ] - self.assertEqual(0, self._count_in(not_expected, contents)) + assert 0 == self._count_in(not_expected, contents) def test_sudoers_ensure_handle_duplicates(self): rules = [ @@ -203,123 +188,101 @@ def test_sudoers_ensure_handle_duplicates(self): "harlowja B-ALL=(ALL:ALL) ALL", "harlowja C-ALL=(ALL:ALL) ALL", ] - self.assertEqual(len(expected), self._count_in(expected, contents)) + assert len(expected) == self._count_in(expected, contents) not_expected = [ "harlowja A", "harlowja L", "harlowja L", ] - self.assertEqual(0, self._count_in(not_expected, contents)) + assert 0 == self._count_in(not_expected, contents) def test_sudoers_ensure_new(self): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) d.ensure_sudo_dir("/b") contents = util.load_text_file("/etc/sudoers") - self.assertIn("includedir /b", contents) - self.assertTrue(os.path.isdir("/b")) + assert "includedir /b" in contents + assert os.path.isdir("/b") - def test_sudoers_ensure_append(self): + def test_sudoers_ensure_append(self, caplog): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) util.write_file("/etc/sudoers", "josh, josh\n") d.ensure_sudo_dir("/b") contents = util.load_text_file("/etc/sudoers") - self.assertIn("includedir /b", contents) - self.assertTrue(os.path.isdir("/b")) - self.assertIn("josh", contents) - self.assertEqual(2, contents.count("josh")) - self.assertIn( - "Added '#includedir /b' to /etc/sudoers", self.logs.getvalue() - ) + assert "includedir /b" in contents + assert os.path.isdir("/b") + assert "josh" in contents + assert 2 == contents.count("josh") + assert "Added '#includedir /b' to /etc/sudoers" in caplog.text def test_sudoers_ensure_append_sudoer_file(self): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) util.write_file("/etc/sudoers", "josh, josh\n") d.ensure_sudo_dir("/b", "/etc/sudoers") contents = util.load_text_file("/etc/sudoers") - self.assertIn("includedir /b", contents) - self.assertTrue(os.path.isdir("/b")) - self.assertIn("josh", contents) - self.assertEqual(2, contents.count("josh")) + assert "includedir /b" in contents + assert os.path.isdir("/b") + assert "josh" in contents + assert 2 == contents.count("josh") - def test_usr_sudoers_ensure_new(self): + def test_usr_sudoers_ensure_new(self, caplog): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) util.write_file("/usr/etc/sudoers", "josh, josh\n") d.ensure_sudo_dir("/b") contents = util.load_text_file("/etc/sudoers") - self.assertIn("josh", contents) - self.assertEqual(2, contents.count("josh")) - self.assertIn("includedir /b", contents) - self.assertTrue(os.path.isdir("/b")) - self.assertIn( - "Using content from '/usr/etc/sudoers", self.logs.getvalue() - ) + assert "josh" in contents + assert 2 == contents.count("josh") + assert "includedir /b" in contents + assert os.path.isdir("/b") + assert "Using content from '/usr/etc/sudoers" in caplog.text def test_usr_sudoers_ensure_no_etc_create_when_include_in_usr_etc(self): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) util.write_file("/usr/etc/sudoers", "#includedir /b") d.ensure_sudo_dir("/b") - self.assertTrue(not os.path.exists("/etc/sudoers")) + assert not os.path.exists("/etc/sudoers") def test_sudoers_ensure_only_one_includedir(self): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) for char in ["#", "@"]: util.write_file("/etc/sudoers", "{}includedir /b".format(char)) d.ensure_sudo_dir("/b") contents = util.load_text_file("/etc/sudoers") - self.assertIn("includedir /b", contents) - self.assertTrue(os.path.isdir("/b")) - self.assertEqual(1, contents.count("includedir /b")) + assert "includedir /b" in contents + assert os.path.isdir("/b") + assert 1 == contents.count("includedir /b") def test_arch_package_mirror_info_unknown(self): """for an unknown arch, we should get back that with arch 'default'.""" arch_mirrors = gapmi(package_mirrors, arch="unknown") - self.assertEqual(unknown_arch_info, arch_mirrors) + assert unknown_arch_info == arch_mirrors def test_arch_package_mirror_info_known(self): arch_mirrors = gapmi(package_mirrors, arch="amd64") - self.assertEqual(package_mirrors[0], arch_mirrors) + assert package_mirrors[0] == arch_mirrors def test_systemd_in_use(self): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) os.makedirs("/run/systemd/system") - self.assertTrue(d.uses_systemd()) + assert d.uses_systemd() def test_systemd_not_in_use(self): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) - self.assertFalse(d.uses_systemd()) + assert not d.uses_systemd() def test_systemd_symlink(self): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) - self.patchOS(self.tmp) - self.patchUtils(self.tmp) os.makedirs("/run/systemd") os.symlink("/", "/run/systemd/system") - self.assertFalse(d.uses_systemd()) + assert not d.uses_systemd() @mock.patch("cloudinit.distros.debian.read_system_locale") def test_get_locale_ubuntu(self, m_locale): @@ -328,7 +291,7 @@ def test_get_locale_ubuntu(self, m_locale): cls = distros.fetch("ubuntu") d = cls("ubuntu", {}, None) locale = d.get_locale() - self.assertEqual("C.UTF-8", locale) + assert "C.UTF-8" == locale @mock.patch("cloudinit.distros.rhel.Distro._read_system_locale") def test_get_locale_rhel(self, m_locale): @@ -337,7 +300,7 @@ def test_get_locale_rhel(self, m_locale): cls = distros.fetch("rhel") d = cls("rhel", {}, None) locale = d.get_locale() - self.assertEqual("C.UTF-8", locale) + assert "C.UTF-8" == locale def test_expire_passwd_uses_chpasswd(self): """Test ubuntu.expire_passwd uses the passwd command.""" diff --git a/tests/unittests/distros/test_aosc.py b/tests/unittests/distros/test_aosc.py index e8a66b7a..9bac093a 100644 --- a/tests/unittests/distros/test_aosc.py +++ b/tests/unittests/distros/test_aosc.py @@ -1,10 +1,9 @@ # This file is part of cloud-init. See LICENSE file for license information. -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import CiTestCase +from tests.unittests.helpers import get_distro -class TestAOSC(CiTestCase): +class TestAOSC: def test_get_distro(self): - distro = _get_distro("aosc") - self.assertEqual(distro.osfamily, "aosc") + distro = get_distro("aosc") + assert distro.osfamily == "aosc" diff --git a/tests/unittests/distros/test_arch.py b/tests/unittests/distros/test_arch.py index 5a8c5fe9..962fa62b 100644 --- a/tests/unittests/distros/test_arch.py +++ b/tests/unittests/distros/test_arch.py @@ -1,14 +1,13 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import util -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import CiTestCase +from tests.unittests.helpers import get_distro -class TestArch(CiTestCase): - def test_get_distro(self): - distro = _get_distro("arch") +class TestArch: + def test_get_distro(self, tmp_path): + distro = get_distro("arch") hostname = "myhostname" - hostfile = self.tmp_path("hostfile") + hostfile = tmp_path / "hostfile" distro._write_hostname(hostname, hostfile) - self.assertEqual(hostname + "\n", util.load_text_file(hostfile)) + assert hostname + "\n" == util.load_text_file(hostfile) diff --git a/tests/unittests/distros/test_azurelinux.py b/tests/unittests/distros/test_azurelinux.py index 03c895bc..f8134f3a 100644 --- a/tests/unittests/distros/test_azurelinux.py +++ b/tests/unittests/distros/test_azurelinux.py @@ -1,8 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from tests.unittests.helpers import CiTestCase - -from . import _get_distro +from tests.unittests.helpers import get_distro SYSTEM_INFO = { "paths": { @@ -13,13 +11,12 @@ } -class TestAzurelinux(CiTestCase): - with_logs = True - distro = _get_distro("azurelinux", SYSTEM_INFO) +class TestAzurelinux: + distro = get_distro("azurelinux", SYSTEM_INFO) expected_log_line = "Rely on Azure Linux default network config" def test_network_renderer(self): - self.assertEqual(self.distro._cfg["network"]["renderers"], "networkd") + assert self.distro._cfg["network"]["renderers"] == "networkd" def test_get_distro(self): - self.assertEqual(self.distro.osfamily, "azurelinux") + assert self.distro.osfamily == "azurelinux" diff --git a/tests/unittests/distros/test_bsd_utils.py b/tests/unittests/distros/test_bsd_utils.py index ae2bdf5b..f5a7efd8 100644 --- a/tests/unittests/distros/test_bsd_utils.py +++ b/tests/unittests/distros/test_bsd_utils.py @@ -1,7 +1,8 @@ # This file is part of cloud-init. See LICENSE file for license information. +import pytest import cloudinit.distros.bsd_utils as bsd_utils -from tests.unittests.helpers import CiTestCase, ExitStack, mock +from tests.unittests.helpers import mock RC_FILE = """ if something; then @@ -11,56 +12,37 @@ """ -class TestBsdUtils(CiTestCase): - def setUp(self): - super().setUp() - patches = ExitStack() - self.addCleanup(patches.close) - - self.load_file = patches.enter_context( - mock.patch.object(bsd_utils.util, "load_text_file") - ) - - self.write_file = patches.enter_context( - mock.patch.object(bsd_utils.util, "write_file") - ) - - def test_get_rc_config_value(self): - self.load_file.return_value = "hostname=foo\n" - self.assertEqual(bsd_utils.get_rc_config_value("hostname"), "foo") - self.load_file.assert_called_with("/etc/rc.conf") - - self.load_file.return_value = "hostname=foo" - self.assertEqual(bsd_utils.get_rc_config_value("hostname"), "foo") - - self.load_file.return_value = 'hostname="foo"' - self.assertEqual(bsd_utils.get_rc_config_value("hostname"), "foo") - - self.load_file.return_value = "hostname='foo'" - self.assertEqual(bsd_utils.get_rc_config_value("hostname"), "foo") - - self.load_file.return_value = "hostname='foo\"" - self.assertEqual(bsd_utils.get_rc_config_value("hostname"), "'foo\"") - - self.load_file.return_value = "" - self.assertEqual(bsd_utils.get_rc_config_value("hostname"), None) - - self.load_file.return_value = RC_FILE.format(hostname="foo") - self.assertEqual(bsd_utils.get_rc_config_value("hostname"), "foo") - - def test_set_rc_config_value_unchanged(self): - # bsd_utils.set_rc_config_value('hostname', 'foo') - # self.write_file.assert_called_with('/etc/rc.conf', 'hostname=foo\n') - - self.load_file.return_value = RC_FILE.format(hostname="foo") - self.write_file.assert_not_called() - - def test_set_rc_config_value(self): +@mock.patch("cloudinit.distros.bsd_utils.util.load_text_file") +class TestBsdUtils: + @pytest.mark.parametrize( + "content,expected", + ( + ("hostname=foo\n", "foo"), + ("hostname=foo", "foo"), + ('hostname="foo"', "foo"), + ("hostname='foo'", "foo"), + ("hostname='foo\"", "'foo\""), + ("", None), + (RC_FILE.format(hostname="foo"), "foo"), + ), + ) + def test_get_rc_config_value(self, m_load_file, content, expected): + m_load_file.return_value = content + assert bsd_utils.get_rc_config_value("hostname") == expected + m_load_file.assert_called_with("/etc/rc.conf") + + @mock.patch("cloudinit.distros.bsd_utils.util.write_file") + def test_set_rc_config_value_unchanged(self, m_write_file, m_load_file): + m_load_file.return_value = RC_FILE.format(hostname="foo") + m_write_file.assert_not_called() + + @mock.patch("cloudinit.distros.bsd_utils.util.write_file") + def test_set_rc_config_value(self, m_write_file, m_load_file): bsd_utils.set_rc_config_value("hostname", "foo") - self.write_file.assert_called_with("/etc/rc.conf", "hostname=foo\n") + m_write_file.assert_called_with("/etc/rc.conf", "hostname=foo\n") - self.load_file.return_value = RC_FILE.format(hostname="foo") + m_load_file.return_value = RC_FILE.format(hostname="foo") bsd_utils.set_rc_config_value("hostname", "bar") - self.write_file.assert_called_with( + m_write_file.assert_called_with( "/etc/rc.conf", RC_FILE.format(hostname="bar") ) diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index 819e2b9b..69b844b6 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -6,8 +6,7 @@ import pytest from cloudinit import distros, features, lifecycle, ssh_util -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import mock +from tests.unittests.helpers import get_distro, mock from tests.unittests.util import abstract_to_concrete USER = "foo_user" @@ -368,7 +367,7 @@ def test_avoid_unlock_preexisting_user_empty_password( mocker, tmpdir, ): - dist = _get_distro(distro_name) + dist = get_distro(distro_name) dist.shadow_fn = tmpdir.join(dist.shadow_fn).strpath dist.shadow_extrausers_fn = tmpdir.join( dist.shadow_extrausers_fn diff --git a/tests/unittests/distros/test_dragonflybsd.py b/tests/unittests/distros/test_dragonflybsd.py index 5419eeea..7222aef7 100644 --- a/tests/unittests/distros/test_dragonflybsd.py +++ b/tests/unittests/distros/test_dragonflybsd.py @@ -1,8 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import cloudinit.util -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import mock +from tests.unittests.helpers import get_distro, mock M_PATH = "cloudinit.distros." @@ -10,7 +9,7 @@ class TestDragonFlyBSD: @mock.patch(M_PATH + "subp.subp") def test_add_user(self, m_subp): - distro = _get_distro("dragonflybsd") + distro = get_distro("dragonflybsd") assert True is distro.add_user("me2", uid=1234, default=False) assert [ mock.call( @@ -29,7 +28,7 @@ def test_add_user(self, m_subp): ] == m_subp.call_args_list def test_unlock_passwd(self, caplog): - distro = _get_distro("dragonflybsd") + distro = get_distro("dragonflybsd") distro.unlock_passwd("me2") assert ( "Dragonfly BSD/FreeBSD password lock is not reversible, " diff --git a/tests/unittests/distros/test_freebsd.py b/tests/unittests/distros/test_freebsd.py index 50fb8e9f..ac1402bd 100644 --- a/tests/unittests/distros/test_freebsd.py +++ b/tests/unittests/distros/test_freebsd.py @@ -3,8 +3,7 @@ import os from cloudinit.util import find_freebsd_part, get_path_dev_freebsd -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import get_distro, mock M_PATH = "cloudinit.distros.freebsd." @@ -12,7 +11,7 @@ class TestFreeBSD: @mock.patch(M_PATH + "subp.subp") def test_add_user(self, m_subp): - distro = _get_distro("freebsd") + distro = get_distro("freebsd") assert True is distro.add_user("me2", uid=1234, default=False) assert [ mock.call( @@ -38,7 +37,7 @@ def test_add_user(self, m_subp): ] == m_subp.call_args_list def test_unlock_passwd(self, caplog): - distro = _get_distro("freebsd") + distro = get_distro("freebsd") distro.unlock_passwd("me2") assert ( "Dragonfly BSD/FreeBSD password lock is not reversible, " @@ -46,7 +45,7 @@ def test_unlock_passwd(self, caplog): ) -class TestDeviceLookUp(CiTestCase): +class TestDeviceLookUp: @mock.patch("cloudinit.subp.subp") def test_find_freebsd_part_label(self, mock_subp): glabel_out = """ @@ -56,7 +55,7 @@ def test_find_freebsd_part_label(self, mock_subp): """ mock_subp.return_value = (glabel_out, "") res = find_freebsd_part("/dev/label/rootfs") - self.assertEqual("da0p2", res) + assert "da0p2" == res @mock.patch("cloudinit.subp.subp") def test_find_freebsd_part_gpt(self, mock_subp): @@ -69,7 +68,7 @@ def test_find_freebsd_part_gpt(self, mock_subp): """ mock_subp.return_value = (glabel_out, "") res = find_freebsd_part("/dev/gpt/rootfs") - self.assertEqual("vtbd0p3", res) + assert "vtbd0p3" == res @mock.patch("cloudinit.subp.subp") def test_find_freebsd_part_gptid(self, mock_subp): @@ -83,7 +82,7 @@ def test_find_freebsd_part_gptid(self, mock_subp): res = find_freebsd_part( "/dev/gptid/4cd084b4-7fb4-11ee-a7ba-002590ec5bf2" ) - self.assertEqual("vtbd0p4", res) + assert "vtbd0p4" == res @mock.patch("cloudinit.subp.subp") def test_find_freebsd_part_ufsid(self, mock_subp): @@ -95,7 +94,7 @@ def test_find_freebsd_part_ufsid(self, mock_subp): """ mock_subp.return_value = (glabel_out, "") res = find_freebsd_part("/dev/ufsid/654e0663786f5131") - self.assertEqual("vtbd0p4", res) + assert "vtbd0p4" == res def test_get_path_dev_freebsd_label(self): mnt_list = """ @@ -106,4 +105,4 @@ def test_get_path_dev_freebsd_label(self): """ with mock.patch.object(os.path, "exists", return_value=True): res = get_path_dev_freebsd("/etc", mnt_list) - self.assertIsNotNone(res) + assert res is not None diff --git a/tests/unittests/distros/test_gentoo.py b/tests/unittests/distros/test_gentoo.py index 979e6d82..71916629 100644 --- a/tests/unittests/distros/test_gentoo.py +++ b/tests/unittests/distros/test_gentoo.py @@ -1,42 +1,45 @@ # This file is part of cloud-init. See LICENSE file for license information. +import pytest from cloudinit import atomic_helper, util -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import get_distro -class TestGentoo(CiTestCase): - def test_write_hostname(self, whatever=False): - distro = _get_distro("gentoo") +class TestGentoo: + def test_write_hostname(self, tmp_path): + distro = get_distro("gentoo") hostname = "myhostname" - hostfile = self.tmp_path("hostfile") + hostfile = tmp_path / "hostfile" distro._write_hostname(hostname, hostfile) if distro.uses_systemd(): - self.assertEqual("myhostname\n", util.load_text_file(hostfile)) + assert "myhostname\n" == util.load_text_file(hostfile) else: - self.assertEqual( - 'hostname="myhostname"\n', util.load_text_file(hostfile) - ) + assert 'hostname="myhostname"\n' == util.load_text_file(hostfile) - def test_write_existing_hostname_with_comments(self, whatever=False): - distro = _get_distro("gentoo") + def test_write_existing_hostname_with_comments(self, tmp_path): + distro = get_distro("gentoo") hostname = "myhostname" contents = '#This is the hostname\nhostname="localhost"' - hostfile = self.tmp_path("hostfile") + hostfile = tmp_path / "hostfile" atomic_helper.write_file(hostfile, contents, omode="w") distro._write_hostname(hostname, hostfile) if distro.uses_systemd(): - self.assertEqual( - "#This is the hostname\nmyhostname\n", - util.load_text_file(hostfile), + assert ( + "#This is the hostname\nmyhostname\n" + == util.load_text_file(hostfile) ) else: - self.assertEqual( - '#This is the hostname\nhostname="myhostname"\n', - util.load_text_file(hostfile), + assert ( + '#This is the hostname\nhostname="myhostname"\n' + == util.load_text_file(hostfile) ) -@mock.patch("cloudinit.distros.uses_systemd", return_value=False) +@pytest.fixture +def no_systemd(mocker): + mocker.patch("cloudinit.distros.uses_systemd", return_value=False) + + +@pytest.mark.usefixtures("no_systemd") class TestGentooOpenRC(TestGentoo): pass diff --git a/tests/unittests/distros/test_init.py b/tests/unittests/distros/test_init.py index 7809899f..4319c35d 100644 --- a/tests/unittests/distros/test_init.py +++ b/tests/unittests/distros/test_init.py @@ -14,7 +14,7 @@ PackageInstallerError, _get_package_mirror_info, ) -from tests.unittests.distros import _get_distro +from tests.unittests.helpers import get_distro # In newer versions of Python, these characters will be omitted instead # of substituted because of security concerns. @@ -293,7 +293,7 @@ def test_log_errors_with_updating_package_source( M_PATH + "snap.Snap.update_package_sources", side_effect=snap_error, ) - _get_distro("ubuntu").update_package_sources() + get_distro("ubuntu").update_package_sources() for log in expected_logs: assert log in caplog.text @@ -337,7 +337,7 @@ def test_run_available_package_managers( m_snap_update = mocker.patch( M_PATH + "snap.Snap.update_package_sources" ) - _get_distro("ubuntu").update_package_sources() + get_distro("ubuntu").update_package_sources() if not snap_available: m_snap_update.assert_not_called() else: @@ -381,12 +381,12 @@ def m_subp(self, mocker): def test_invalid_yaml(self, m_apt_install): """Test that an invalid YAML raises an exception.""" with pytest.raises(ValueError): - _get_distro("debian").install_packages([["invalid"]]) + get_distro("debian").install_packages([["invalid"]]) m_apt_install.assert_not_called() def test_unknown_package_manager(self, m_apt_install, caplog): """Test that an unknown package manager raises an exception.""" - _get_distro("debian").install_packages( + get_distro("debian").install_packages( [{"apt": ["pkg1"]}, "pkg2", {"invalid": ["pkg3"]}] ) assert ( @@ -400,7 +400,7 @@ def test_unknown_package_manager(self, m_apt_install, caplog): def test_non_default_package_manager(self, m_apt_install, m_snap_install): """Test success from package manager not supported by distro.""" - _get_distro("debian").install_packages( + get_distro("debian").install_packages( [{"apt": ["pkg1"]}, "pkg2", {"snap": ["pkg3"]}] ) apt_install_args = m_apt_install.call_args_list[0][0][0] @@ -422,7 +422,7 @@ def test_non_default_package_manager_fail( PackageInstallerError, match="Failed to install the following packages: {'pkg3'}", ): - _get_distro("debian").install_packages( + get_distro("debian").install_packages( [{"apt": ["pkg1"]}, "pkg2", {"snap": ["pkg3"]}] ) @@ -432,7 +432,7 @@ def test_default_and_specific_package_manager( self, m_apt_install, m_snap_install ): """Test success from package manager not supported by distro.""" - _get_distro("ubuntu").install_packages( + get_distro("ubuntu").install_packages( ["pkg1", ["pkg3", "ver3"], {"apt": [["pkg2", "ver2"]]}] ) apt_install_args = m_apt_install.call_args_list[0][0][0] @@ -451,7 +451,7 @@ def test_specific_package_manager_fail_doesnt_retry( return_value=["pkg1"], ) with pytest.raises(PackageInstallerError): - _get_distro("ubuntu").install_packages([{"apt": ["pkg1"]}]) + get_distro("ubuntu").install_packages([{"apt": ["pkg1"]}]) apt_install_args = m_apt_install.call_args_list[0][0][0] assert "pkg1" in apt_install_args m_snap_install.assert_not_called() @@ -463,7 +463,7 @@ def test_no_attempt_if_no_package_manager( mocker.patch(M_PATH + "apt.Apt.available", return_value=False) mocker.patch(M_PATH + "snap.Snap.available", return_value=False) with pytest.raises(PackageInstallerError): - _get_distro("ubuntu").install_packages( + get_distro("ubuntu").install_packages( ["pkg1", "pkg2", {"other": "pkg3"}] ) m_apt_install.assert_not_called() @@ -544,7 +544,7 @@ def test_failed( return_value=snap_failed, ) with pytest.raises(PackageInstallerError) as exc: - _get_distro(distro).install_packages(pkg_list) + get_distro(distro).install_packages(pkg_list) message = exc.value.args[0] assert "Failed to install the following packages" in message for pkg in total_failed: diff --git a/tests/unittests/distros/test_manage_service.py b/tests/unittests/distros/test_manage_service.py index d7637d38..8b872897 100644 --- a/tests/unittests/distros/test_manage_service.py +++ b/tests/unittests/distros/test_manage_service.py @@ -1,17 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import get_distro, mock from tests.unittests.util import MockDistro -class TestManageService(CiTestCase): - - with_logs = True - - def setUp(self): - super(TestManageService, self).setUp() - self.dist = MockDistro() +class TestManageService: + dist = MockDistro() @mock.patch.object(MockDistro, "uses_systemd", return_value=True) @mock.patch("cloudinit.distros.subp.subp") @@ -34,7 +28,7 @@ def test_manage_service_service_initcmd(self, m_subp, m_sysd): @mock.patch.object(MockDistro, "uses_systemd", return_value=False) @mock.patch("cloudinit.distros.subp.subp") def test_manage_service_rcservice_initcmd(self, m_subp, m_sysd): - dist = _get_distro("alpine") + dist = get_distro("alpine") dist.init_cmd = ["rc-service", "--nocolor"] dist.manage_service("start", "myssh") m_subp.assert_called_with( @@ -45,7 +39,7 @@ def test_manage_service_rcservice_initcmd(self, m_subp, m_sysd): @mock.patch("cloudinit.distros.subp.subp") def test_manage_service_alpine_rcupdate_cmd(self, m_subp): - dist = _get_distro("alpine") + dist = get_distro("alpine") dist.update_cmd = ["rc-update", "--nocolor"] dist.manage_service("enable", "myssh") m_subp.assert_called_with( @@ -54,7 +48,7 @@ def test_manage_service_alpine_rcupdate_cmd(self, m_subp): @mock.patch("cloudinit.distros.subp.subp") def test_manage_service_rcctl_initcmd(self, m_subp): - dist = _get_distro("openbsd") + dist = get_distro("openbsd") dist.init_cmd = ["rcctl"] dist.manage_service("start", "myssh") m_subp.assert_called_with( @@ -63,7 +57,7 @@ def test_manage_service_rcctl_initcmd(self, m_subp): @mock.patch("cloudinit.distros.subp.subp") def test_manage_service_fbsd_service_initcmd(self, m_subp): - dist = _get_distro("freebsd") + dist = get_distro("freebsd") dist.init_cmd = ["service"] dist.manage_service("enable", "myssh") m_subp.assert_called_with( diff --git a/tests/unittests/distros/test_mariner.py b/tests/unittests/distros/test_mariner.py index 57f8d498..afda7d27 100644 --- a/tests/unittests/distros/test_mariner.py +++ b/tests/unittests/distros/test_mariner.py @@ -1,8 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from tests.unittests.helpers import CiTestCase - -from . import _get_distro +from tests.unittests.helpers import get_distro SYSTEM_INFO = { "paths": { @@ -13,13 +11,12 @@ } -class TestMariner(CiTestCase): - with_logs = True - distro = _get_distro("mariner", SYSTEM_INFO) +class TestMariner: + distro = get_distro("mariner", SYSTEM_INFO) expected_log_line = "Rely on MarinerOS default network config" def test_network_renderer(self): - self.assertEqual(self.distro._cfg["network"]["renderers"], "networkd") + assert self.distro._cfg["network"]["renderers"] == "networkd" def test_get_distro(self): - self.assertEqual(self.distro.osfamily, "mariner") + assert self.distro.osfamily == "mariner" diff --git a/tests/unittests/distros/test_netbsd.py b/tests/unittests/distros/test_netbsd.py index c4cb9a55..e5df07d5 100644 --- a/tests/unittests/distros/test_netbsd.py +++ b/tests/unittests/distros/test_netbsd.py @@ -2,7 +2,7 @@ import pytest -from tests.unittests.distros import _get_distro +from tests.unittests.helpers import get_distro try: # Blowfish not available in < 3.7, so this has never worked. Ignore failure @@ -18,7 +18,7 @@ class TestNetBSD: @mock.patch(M_PATH + "subp.subp") def test_add_user(self, m_subp): - distro = _get_distro("netbsd") + distro = get_distro("netbsd") assert True is distro.add_user("me2", uid=1234, default=False) assert [ mock.call( @@ -28,7 +28,7 @@ def test_add_user(self, m_subp): @mock.patch(M_PATH + "subp.subp") def test_unlock_passwd(self, m_subp, caplog): - distro = _get_distro("netbsd") + distro = get_distro("netbsd") distro.unlock_passwd("me2") assert [ mock.call(["usermod", "-C", "no", "me2"]) diff --git a/tests/unittests/distros/test_netconfig.py b/tests/unittests/distros/test_netconfig.py index 54b387ea..8fba089d 100644 --- a/tests/unittests/distros/test_netconfig.py +++ b/tests/unittests/distros/test_netconfig.py @@ -3,20 +3,19 @@ import copy import os import re +import shutil from io import StringIO from textwrap import dedent from unittest import mock +import pytest import yaml -from cloudinit import distros, features, helpers, settings, subp, util +from cloudinit import features, subp, util from cloudinit.distros.parsers.sys_conf import SysConf from cloudinit.net.activators import IfUpDownActivator -from tests.unittests.helpers import ( - FilesystemMockingTestCase, - dir2dict, - readResource, -) +from tests.helpers import cloud_init_project_dir +from tests.unittests.helpers import dir2dict, get_distro BASE_NET_CFG = """ auto lo @@ -290,76 +289,77 @@ def __str__(self): return self.buffer.getvalue() -class TestNetCfgDistroBase(FilesystemMockingTestCase): - def setUp(self): - super(TestNetCfgDistroBase, self).setUp() - self.add_patch("cloudinit.util.system_is_snappy", "m_snappy") - - def _get_distro(self, dname, renderers=None, activators=None): - cls = distros.fetch(dname) - cfg = copy.deepcopy(settings.CFG_BUILTIN) - cfg["system_info"]["distro"] = dname - system_info_network_cfg = {} - if renderers: - system_info_network_cfg["renderers"] = renderers - if activators: - system_info_network_cfg["activators"] = activators - if system_info_network_cfg: - cfg["system_info"]["network"] = system_info_network_cfg - paths = helpers.Paths({}) - return cls(dname, cfg.get("system_info"), paths) - - def assertCfgEquals(self, blob1, blob2): - b1 = dict(SysConf(blob1.strip().splitlines())) - b2 = dict(SysConf(blob2.strip().splitlines())) - self.assertEqual(b1, b2) - for k, v in b1.items(): - self.assertIn(k, b2) - for k, v in b2.items(): - self.assertIn(k, b1) - for k, v in b1.items(): - self.assertEqual(v, b2[k]) - - -class TestNetCfgDistroFreeBSD(TestNetCfgDistroBase): - def setUp(self): - super(TestNetCfgDistroFreeBSD, self).setUp() - ifs_txt = readResource("netinfo/freebsd-ifconfig-output") - with mock.patch( +@pytest.fixture(autouse=True) +def system_is_snappy(): + mock.patch("cloudinit.util.system_is_snappy") + + +def assertCfgEquals(blob1, blob2): + b1 = dict(SysConf(blob1.strip().splitlines())) + b2 = dict(SysConf(blob2.strip().splitlines())) + assert b1 == b2 + for k, v in b1.items(): + assert k in b2 + for k, v in b2.items(): + assert k in b1 + for k, v in b1.items(): + assert v == b2[k] + + +@pytest.fixture +def distro_freebsd(mocker): + with open("tests/data/netinfo/freebsd-ifconfig-output", "r") as fh: + ifs_txt = fh.read() + mocker.patch( "cloudinit.distros.networking.subp.subp", return_value=(ifs_txt, None), - ): - self.distro = self._get_distro("freebsd", renderers=["freebsd"]) + ) + return get_distro("freebsd", renderers=["freebsd"]) + + +@pytest.fixture(autouse=True) +def with_test_data(tmp_path, fake_filesystem_hook): + shutil.copytree( + str(cloud_init_project_dir("tests/data")), + str(tmp_path / "tests/data"), + dirs_exist_ok=True, + ) + +@pytest.mark.usefixtures("fake_filesystem") +class TestNetCfgDistroFreeBSD: def _apply_and_verify_freebsd( - self, apply_fn, config, expected_cfgs=None, bringup=False + self, apply_fn, config, tmp_path, expected_cfgs=None, bringup=False ): + rootd = tmp_path if not expected_cfgs: raise ValueError("expected_cfg must not be None") - tmpd = None with mock.patch("cloudinit.net.freebsd.available") as m_avail: m_avail.return_value = True - with self.reRooted(tmpd) as tmpd: - util.ensure_dir("/etc") - util.ensure_file("/etc/rc.conf") - util.ensure_file("/etc/resolv.conf") - apply_fn(config, bringup) + util.ensure_dir(rootd / "etc") + util.ensure_file(rootd / "etc/rc.conf") + util.ensure_file(rootd / "etc/resolv.conf") + apply_fn(config, bringup) - results = dir2dict(tmpd) + results = dir2dict( + str(rootd), filter=lambda fn: fn.startswith(str(rootd / "etc")) + ) for cfgpath, expected in expected_cfgs.items(): print("----------") print(expected) print("^^^^ expected | rendered VVVVVVV") print(results[cfgpath]) print("----------") - self.assertEqual( - set(expected.split("\n")), set(results[cfgpath].split("\n")) + assert set(expected.split("\n")) == set( + results[cfgpath].split("\n") ) - self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + assert 0o644 == get_mode(cfgpath, str(rootd)) @mock.patch("cloudinit.net.get_interfaces_by_mac") - def test_apply_network_config_freebsd_standard(self, ifaces_mac): + def test_apply_network_config_freebsd_standard( + self, ifaces_mac, distro_freebsd, tmp_path + ): ifaces_mac.return_value = { "00:15:5d:4c:73:00": "eth0", } @@ -374,13 +374,16 @@ def test_apply_network_config_freebsd_standard(self, ifaces_mac): "/etc/resolv.conf": "", } self._apply_and_verify_freebsd( - self.distro.apply_network_config, + distro_freebsd.apply_network_config, V1_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs.copy(), ) @mock.patch("cloudinit.net.get_interfaces_by_mac") - def test_apply_network_config_freebsd_ipv6_standard(self, ifaces_mac): + def test_apply_network_config_freebsd_ipv6_standard( + self, ifaces_mac, distro_freebsd, tmp_path + ): ifaces_mac.return_value = { "00:15:5d:4c:73:00": "eth0", } @@ -395,13 +398,16 @@ def test_apply_network_config_freebsd_ipv6_standard(self, ifaces_mac): "/etc/resolv.conf": "", } self._apply_and_verify_freebsd( - self.distro.apply_network_config, + distro_freebsd.apply_network_config, V1_NET_CFG_IPV6, + tmp_path, expected_cfgs=expected_cfgs.copy(), ) @mock.patch("cloudinit.net.get_interfaces_by_mac") - def test_apply_network_config_freebsd_ifrename(self, ifaces_mac): + def test_apply_network_config_freebsd_ifrename( + self, ifaces_mac, distro_freebsd, tmp_path + ): ifaces_mac.return_value = { "00:15:5d:4c:73:00": "vtnet0", } @@ -420,13 +426,16 @@ def test_apply_network_config_freebsd_ifrename(self, ifaces_mac): "/etc/resolv.conf": "", } self._apply_and_verify_freebsd( - self.distro.apply_network_config, + distro_freebsd.apply_network_config, V1_NET_CFG_RENAME, + tmp_path, expected_cfgs=expected_cfgs.copy(), ) @mock.patch("cloudinit.net.get_interfaces_by_mac") - def test_apply_network_config_freebsd_nameserver(self, ifaces_mac): + def test_apply_network_config_freebsd_nameserver( + self, ifaces_mac, distro_freebsd, tmp_path + ): ifaces_mac.return_value = { "00:15:5d:4c:73:00": "eth0", } @@ -436,19 +445,25 @@ def test_apply_network_config_freebsd_nameserver(self, ifaces_mac): V1_NET_CFG_DNS["config"][0]["subnets"][0]["dns_nameservers"] = ns expected_cfgs = {"/etc/resolv.conf": "nameserver 1.2.3.4\n"} self._apply_and_verify_freebsd( - self.distro.apply_network_config, + distro_freebsd.apply_network_config, V1_NET_CFG_DNS, + tmp_path, expected_cfgs=expected_cfgs.copy(), ) -class TestNetCfgDistroUbuntuEni(TestNetCfgDistroBase): - def setUp(self): - super(TestNetCfgDistroUbuntuEni, self).setUp() - self.distro = self._get_distro( - "ubuntu", renderers=["eni"], activators=["eni"] - ) +@pytest.fixture +def distro_eni(): + return get_distro("ubuntu", renderers=["eni"], activators=["eni"]) + + +@pytest.fixture +def m_activators_subp(mocker): + mocker.patch("cloudinit.net.activators.subp.subp", return_value=("", "")) + +@pytest.mark.usefixtures("fake_filesystem", "m_activators_subp") +class TestNetCfgDistroUbuntuEni: def eni_path(self): return "/etc/network/interfaces.d/50-cloud-init.cfg" @@ -459,6 +474,7 @@ def _apply_and_verify_eni( self, apply_fn, config, + tmp_path, expected_cfgs=None, bringup=False, previous_files=(), @@ -466,106 +482,122 @@ def _apply_and_verify_eni( if not expected_cfgs: raise ValueError("expected_cfg must not be None") - tmpd = None with mock.patch("cloudinit.net.eni.available") as m_avail: m_avail.return_value = True path_modes = {} - with self.reRooted(tmpd) as tmpd: - for previous_path, content, mode in previous_files: - util.write_file(previous_path, content, mode=mode) - path_modes[previous_path] = mode - apply_fn(config, bringup) - - results = dir2dict(tmpd) + for previous_path, content, mode in previous_files: + util.write_file(previous_path, content, mode=mode) + path_modes[previous_path] = mode + apply_fn(config, bringup) + + results = dir2dict( + str(tmp_path), + filter=lambda fn: fn.startswith(str(tmp_path / "etc")), + ) for cfgpath, expected in expected_cfgs.items(): print("----------") print(expected) print("^^^^ expected | rendered VVVVVVV") print(results[cfgpath]) print("----------") - self.assertEqual(expected, results[cfgpath]) - self.assertEqual( - path_modes.get(cfgpath, 0o644), get_mode(cfgpath, tmpd) + assert expected == results[cfgpath] + assert path_modes.get(cfgpath, 0o644) == get_mode( + cfgpath, str(tmp_path) ) - def test_apply_network_config_and_bringup_filters_priority_eni_ub(self): + def test_apply_network_config_and_bringup_filters_priority_eni_ub( + self, distro_eni, tmp_path + ): """Network activator search priority can be overridden from config.""" expected_cfgs = { self.eni_path(): V1_NET_CFG_OUTPUT, } + with mock.patch( "cloudinit.net.activators.select_activator" ) as select_activator: select_activator.return_value = IfUpDownActivator self._apply_and_verify_eni( - self.distro.apply_network_config, + distro_eni.apply_network_config, V1_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs.copy(), bringup=True, ) # 2nd call to select_activator via distro.network_activator prop - assert IfUpDownActivator == self.distro.network_activator - self.assertEqual( - [mock.call(priority=["eni"])] * 2, select_activator.call_args_list - ) + assert IfUpDownActivator == distro_eni.network_activator + assert [ + mock.call(priority=["eni"]) + ] * 2 == select_activator.call_args_list - def test_apply_network_config_and_bringup_activator_defaults_ub(self): + def test_apply_network_config_and_bringup_activator_defaults_ub( + self, tmp_path + ): """Network activator search priority defaults when unspecified.""" expected_cfgs = { self.eni_path(): V1_NET_CFG_OUTPUT, } # Don't set activators to see DEFAULT_PRIORITY - self.distro = self._get_distro("ubuntu", renderers=["eni"]) + distro = get_distro("ubuntu", renderers=["eni"]) with mock.patch( "cloudinit.net.activators.select_activator" ) as select_activator: select_activator.return_value = IfUpDownActivator self._apply_and_verify_eni( - self.distro.apply_network_config, + distro.apply_network_config, V1_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs.copy(), bringup=True, ) # 2nd call to select_activator via distro.network_activator prop - assert IfUpDownActivator == self.distro.network_activator - self.assertEqual( - [mock.call(priority=None)] * 2, select_activator.call_args_list - ) + assert IfUpDownActivator == distro.network_activator + assert [ + mock.call(priority=None) + ] * 2 == select_activator.call_args_list - def test_apply_network_config_eni_ub(self): + def test_apply_network_config_eni_ub(self, distro_eni, tmp_path): expected_cfgs = { self.eni_path(): V1_NET_CFG_OUTPUT, self.rules_path(): "", } self._apply_and_verify_eni( - self.distro.apply_network_config, + distro_eni.apply_network_config, V1_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs.copy(), previous_files=((self.rules_path(), "something", 0o660),), ) - def test_apply_network_config_ipv6_ub(self): + def test_apply_network_config_ipv6_ub(self, distro_eni, tmp_path): expected_cfgs = {self.eni_path(): V1_NET_CFG_IPV6_OUTPUT} self._apply_and_verify_eni( - self.distro.apply_network_config, + distro_eni.apply_network_config, V1_NET_CFG_IPV6, + tmp_path, expected_cfgs=expected_cfgs.copy(), ) -class TestNetCfgDistroUbuntuNetplan(TestNetCfgDistroBase): +@pytest.fixture +def distro_netplan(): + return get_distro("ubuntu", renderers=["netplan"]) + + +@pytest.fixture +def m_netplan_subp(mocker): + mocker.patch("cloudinit.net.netplan.subp.subp", return_value=("", "")) - with_logs = True - def setUp(self): - super(TestNetCfgDistroUbuntuNetplan, self).setUp() - self.distro = self._get_distro("ubuntu", renderers=["netplan"]) - self.devlist = ["eth0", "lo"] +@pytest.mark.usefixtures("fake_filesystem", "m_netplan_subp") +class TestNetCfgDistroUbuntuNetplan: + DEV_LIST = ["eth0", "lo"] def _apply_and_verify_netplan( self, apply_fn, config, + tmp_path, expected_cfgs=None, bringup=False, previous_files=(), @@ -573,63 +605,75 @@ def _apply_and_verify_netplan( if not expected_cfgs: raise ValueError("expected_cfg must not be None") - tmpd = None with mock.patch("cloudinit.net.netplan.available", return_value=True): with mock.patch( "cloudinit.net.netplan.get_devicelist", - return_value=self.devlist, + return_value=self.DEV_LIST, ): - with self.reRooted(tmpd) as tmpd: - for previous_path, content, mode in previous_files: - util.write_file(previous_path, content, mode=mode) - apply_fn(config, bringup) + for previous_path, content, mode in previous_files: + util.write_file(previous_path, content, mode=mode) + apply_fn(config, bringup) - results = dir2dict(tmpd) + results = dir2dict( + str(tmp_path), + filter=lambda fn: fn.startswith(str(tmp_path / "etc")), + ) for cfgpath, expected, mode in expected_cfgs: print("----------") print(expected) print("^^^^ expected | rendered VVVVVVV") print(results[cfgpath]) print("----------") - self.assertEqual(expected, results[cfgpath]) - self.assertEqual(mode, get_mode(cfgpath, tmpd)) + assert expected == results[cfgpath] + assert mode == get_mode(cfgpath, str(tmp_path)) def netplan_path(self): return "/etc/netplan/50-cloud-init.yaml" - def test_apply_network_config_v1_to_netplan_ub(self): + def test_apply_network_config_v1_to_netplan_ub( + self, distro_netplan, tmp_path + ): expected_cfgs = ( (self.netplan_path(), V1_TO_V2_NET_CFG_OUTPUT, 0o600), ) self._apply_and_verify_netplan( - self.distro.apply_network_config, + distro_netplan.apply_network_config, V1_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs, ) - def test_apply_network_config_v1_ipv6_to_netplan_ub(self): + def test_apply_network_config_v1_ipv6_to_netplan_ub( + self, distro_netplan, tmp_path + ): expected_cfgs = ( (self.netplan_path(), V1_TO_V2_NET_CFG_IPV6_OUTPUT, 0o600), ) self._apply_and_verify_netplan( - self.distro.apply_network_config, + distro_netplan.apply_network_config, V1_NET_CFG_IPV6, + tmp_path, expected_cfgs=expected_cfgs, ) - def test_apply_network_config_v2_passthrough_ub(self): + def test_apply_network_config_v2_passthrough_ub( + self, distro_netplan, tmp_path + ): expected_cfgs = ( (self.netplan_path(), V2_TO_V2_NET_CFG_OUTPUT, 0o600), ) self._apply_and_verify_netplan( - self.distro.apply_network_config, + distro_netplan.apply_network_config, V2_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs, ) - def test_apply_network_config_v2_passthrough_retain_orig_perms(self): + def test_apply_network_config_v2_passthrough_retain_orig_perms( + self, distro_netplan, tmp_path + ): """Custom permissions on existing netplan is kept when more strict.""" expected_cfgs = ( (self.netplan_path(), V2_TO_V2_NET_CFG_OUTPUT, 0o640), @@ -641,15 +685,18 @@ def test_apply_network_config_v2_passthrough_retain_orig_perms(self): # we keep 640 because it's more strict. # 1640 is used to assert sticky bit preserved across write self._apply_and_verify_netplan( - self.distro.apply_network_config, + distro_netplan.apply_network_config, V2_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs, previous_files=( ("/etc/netplan/50-cloud-init.yaml", "a", 0o640), ), ) - def test_apply_network_config_v2_passthrough_ub_old_behavior(self): + def test_apply_network_config_v2_passthrough_ub_old_behavior( + self, distro_netplan, tmp_path + ): """Kinetic and earlier have 50-cloud-init.yaml world-readable""" expected_cfgs = ( (self.netplan_path(), V2_TO_V2_NET_CFG_OUTPUT, 0o644), @@ -658,31 +705,38 @@ def test_apply_network_config_v2_passthrough_ub_old_behavior(self): features, "NETPLAN_CONFIG_ROOT_READ_ONLY", False ): self._apply_and_verify_netplan( - self.distro.apply_network_config, + distro_netplan.apply_network_config, V2_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs, ) - def test_apply_network_config_v2_full_passthrough_ub(self): + def test_apply_network_config_v2_full_passthrough_ub( + self, caplog, distro_netplan, tmp_path + ): expected_cfgs = ( (self.netplan_path(), V2_PASSTHROUGH_NET_CFG_OUTPUT, 0o600), ) self._apply_and_verify_netplan( - self.distro.apply_network_config, + distro_netplan.apply_network_config, V2_PASSTHROUGH_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs, ) - self.assertIn("Passthrough netplan v2 config", self.logs.getvalue()) - self.assertIn( - "Selected renderer 'netplan' from priority list: ['netplan']", - self.logs.getvalue(), + assert "Passthrough netplan v2 config" in caplog.text + assert ( + "Selected renderer 'netplan' from priority list: ['netplan']" + in caplog.text ) -class TestNetCfgDistroRedhat(TestNetCfgDistroBase): - def setUp(self): - super(TestNetCfgDistroRedhat, self).setUp() - self.distro = self._get_distro("rhel", renderers=["sysconfig"]) +@pytest.fixture +def distro_redhat(): + return get_distro("rhel", renderers=["sysconfig"]) + + +@pytest.mark.usefixtures("fake_filesystem") +class TestNetCfgDistroRedhat: def ifcfg_path(self, ifname): return "/etc/sysconfig/network-scripts/ifcfg-%s" % ifname @@ -694,24 +748,26 @@ def _apply_and_verify( self, apply_fn, config, + tmp_path, expected_cfgs=None, bringup=False, - tmpd=None, ): if not expected_cfgs: raise ValueError("expected_cfg must not be None") with mock.patch("cloudinit.net.sysconfig.available") as m_avail: m_avail.return_value = True - with self.reRooted(tmpd) as tmpd: - apply_fn(config, bringup) + apply_fn(config, bringup) - results = dir2dict(tmpd) + results = dir2dict( + str(tmp_path), + filter=lambda fn: fn.startswith(str(tmp_path / "etc")), + ) for cfgpath, expected in expected_cfgs.items(): - self.assertCfgEquals(expected, results[cfgpath]) - self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + assertCfgEquals(expected, results[cfgpath]) + assert 0o644 == get_mode(cfgpath, str(tmp_path)) - def test_apply_network_config_rh(self): + def test_apply_network_config_rh(self, distro_redhat, tmp_path): expected_cfgs = { self.ifcfg_path("eth0"): dedent( """\ @@ -743,12 +799,13 @@ def test_apply_network_config_rh(self): } # rh_distro.apply_network_config(V1_NET_CFG, False) self._apply_and_verify( - self.distro.apply_network_config, + distro_redhat.apply_network_config, V1_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs.copy(), ) - def test_apply_network_config_ipv6_rh(self): + def test_apply_network_config_ipv6_rh(self, distro_redhat, tmp_path): expected_cfgs = { self.ifcfg_path("eth0"): dedent( """\ @@ -784,12 +841,15 @@ def test_apply_network_config_ipv6_rh(self): } # rh_distro.apply_network_config(V1_NET_CFG_IPV6, False) self._apply_and_verify( - self.distro.apply_network_config, + distro_redhat.apply_network_config, V1_NET_CFG_IPV6, + tmp_path, expected_cfgs=expected_cfgs.copy(), ) - def test_sysconfig_network_no_overwite_ipv6_rh(self): + def test_sysconfig_network_no_overwite_ipv6_rh( + self, distro_redhat, tmp_path + ): expected_cfgs = { self.ifcfg_path("eth0"): dedent( """\ @@ -824,24 +884,22 @@ def test_sysconfig_network_no_overwite_ipv6_rh(self): """ ), } - tmpdir = self.tmp_dir() file_mode = 0o644 # pre-existing config in /etc/sysconfig/network should not be removed - with self.reRooted(tmpdir) as tmpdir: - util.write_file( - self.control_path(), - "".join("NOZEROCONF=yes") + "\n", - file_mode, - ) + util.write_file( + self.control_path(), + "".join("NOZEROCONF=yes") + "\n", + file_mode, + ) self._apply_and_verify( - self.distro.apply_network_config, + distro_redhat.apply_network_config, V1_NET_CFG_IPV6, + tmp_path, expected_cfgs=expected_cfgs.copy(), - tmpd=tmpdir, ) - def test_vlan_render_unsupported(self): + def test_vlan_render_unsupported(self, distro_redhat, tmp_path): """Render officially unsupported vlan names.""" cfg = { "version": 2, @@ -891,10 +949,13 @@ def test_vlan_render_unsupported(self): ), } self._apply_and_verify( - self.distro.apply_network_config, cfg, expected_cfgs=expected_cfgs + distro_redhat.apply_network_config, + cfg, + tmp_path, + expected_cfgs=expected_cfgs, ) - def test_vlan_render(self): + def test_vlan_render(self, distro_redhat, tmp_path): cfg = { "version": 2, "ethernets": {"eth0": {"addresses": ["192.10.1.2/24"]}}, @@ -937,36 +998,42 @@ def test_vlan_render(self): ), } self._apply_and_verify( - self.distro.apply_network_config, cfg, expected_cfgs=expected_cfgs + distro_redhat.apply_network_config, + cfg, + tmp_path, + expected_cfgs=expected_cfgs, ) -class TestNetCfgDistroOpensuse(TestNetCfgDistroBase): - def setUp(self): - super(TestNetCfgDistroOpensuse, self).setUp() - self.distro = self._get_distro("opensuse", renderers=["sysconfig"]) +@pytest.fixture +def distro_opensuse(): + return get_distro("opensuse", renderers=["sysconfig"]) + +@pytest.mark.usefixtures("fake_filesystem") +class TestNetCfgDistroOpensuse: def ifcfg_path(self, ifname): return "/etc/sysconfig/network/ifcfg-%s" % ifname def _apply_and_verify( - self, apply_fn, config, expected_cfgs=None, bringup=False + self, apply_fn, config, tmp_path, expected_cfgs=None, bringup=False ): if not expected_cfgs: raise ValueError("expected_cfg must not be None") - tmpd = None with mock.patch("cloudinit.net.sysconfig.available") as m_avail: m_avail.return_value = True - with self.reRooted(tmpd) as tmpd: - apply_fn(config, bringup) + apply_fn(config, bringup) - results = dir2dict(tmpd) + results = dir2dict( + str(tmp_path), + filter=lambda fn: fn.startswith(str(tmp_path / "etc")), + ) for cfgpath, expected in expected_cfgs.items(): - self.assertCfgEquals(expected, results[cfgpath]) - self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + assertCfgEquals(expected, results[cfgpath]) + assert 0o644 == get_mode(cfgpath, str(tmp_path)) - def test_apply_network_config_opensuse(self): + def test_apply_network_config_opensuse(self, distro_opensuse, tmp_path): """Opensuse uses apply_network_config and renders sysconfig""" expected_cfgs = { self.ifcfg_path("eth0"): dedent( @@ -985,12 +1052,15 @@ def test_apply_network_config_opensuse(self): ), } self._apply_and_verify( - self.distro.apply_network_config, + distro_opensuse.apply_network_config, V1_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs.copy(), ) - def test_apply_network_config_ipv6_opensuse(self): + def test_apply_network_config_ipv6_opensuse( + self, distro_opensuse, tmp_path + ): """Opensuse uses apply_network_config and renders sysconfig w/ipv6""" expected_cfgs = { self.ifcfg_path("eth0"): dedent( @@ -1008,21 +1078,25 @@ def test_apply_network_config_ipv6_opensuse(self): ), } self._apply_and_verify( - self.distro.apply_network_config, + distro_opensuse.apply_network_config, V1_NET_CFG_IPV6, + tmp_path, expected_cfgs=expected_cfgs.copy(), ) -class TestNetCfgDistroArch(TestNetCfgDistroBase): - def setUp(self): - super(TestNetCfgDistroArch, self).setUp() - self.distro = self._get_distro("arch", renderers=["netplan"]) +@pytest.fixture +def distro_arch(): + return get_distro("arch", renderers=["netplan"]) + +@pytest.mark.usefixtures("fake_filesystem", "m_netplan_subp") +class TestNetCfgDistroArch: def _apply_and_verify( self, apply_fn, config, + tmp_path, expected_cfgs=None, bringup=False, with_netplan=False, @@ -1030,14 +1104,15 @@ def _apply_and_verify( if not expected_cfgs: raise ValueError("expected_cfg must not be None") - tmpd = None with mock.patch( "cloudinit.net.netplan.available", return_value=with_netplan ): - with self.reRooted(tmpd) as tmpd: - apply_fn(config, bringup) + apply_fn(config, bringup) - results = dir2dict(tmpd) + results = dir2dict( + str(tmp_path), + filter=lambda fn: fn.startswith(str(tmp_path / "etc")), + ) mode = 0o600 if with_netplan else 0o644 for cfgpath, expected in expected_cfgs.items(): print("----------") @@ -1045,8 +1120,8 @@ def _apply_and_verify( print("^^^^ expected | rendered VVVVVVV") print(results[cfgpath]) print("----------") - self.assertEqual(expected, results[cfgpath]) - self.assertEqual(mode, get_mode(cfgpath, tmpd)) + assert expected == results[cfgpath] + assert mode == get_mode(cfgpath, str(tmp_path)) def netctl_path(self, iface): return "/etc/netctl/%s" % iface @@ -1054,7 +1129,7 @@ def netctl_path(self, iface): def netplan_path(self): return "/etc/netplan/50-cloud-init.yaml" - def test_apply_network_config_v1_with_netplan(self): + def test_apply_network_config_v1_with_netplan(self, distro_arch, tmp_path): expected_cfgs = { self.netplan_path(): dedent( """\ @@ -1078,18 +1153,22 @@ def test_apply_network_config_v1_with_netplan(self): "cloudinit.net.netplan.get_devicelist", return_value=[] ): self._apply_and_verify( - self.distro.apply_network_config, + distro_arch.apply_network_config, V1_NET_CFG, + tmp_path, expected_cfgs=expected_cfgs.copy(), with_netplan=True, ) -class TestNetCfgDistroPhoton(TestNetCfgDistroBase): - def setUp(self): - super(TestNetCfgDistroPhoton, self).setUp() - self.distro = self._get_distro("photon", renderers=["networkd"]) +@pytest.fixture +def distro_photon(mocker): + mocker.patch("cloudinit.net.networkd.util.chownbyname") + return get_distro("photon", renderers=["networkd"]) + +@pytest.mark.usefixtures("fake_filesystem") +class TestNetCfgDistroPhoton: def create_conf_dict(self, contents): content_dict = {} for line in contents: @@ -1106,25 +1185,26 @@ def create_conf_dict(self, contents): def compare_dicts(self, actual, expected): for k, v in actual.items(): - self.assertEqual(sorted(expected[k]), sorted(v)) + assert sorted(expected[k]) == sorted(v) def _apply_and_verify( - self, apply_fn, config, expected_cfgs=None, bringup=False + self, apply_fn, config, tmp_path, expected_cfgs=None, bringup=False ): if not expected_cfgs: raise ValueError("expected_cfg must not be None") - tmpd = None with mock.patch("cloudinit.net.networkd.available") as m_avail: m_avail.return_value = True - with self.reRooted(tmpd) as tmpd: - apply_fn(config, bringup) + apply_fn(config, bringup) - results = dir2dict(tmpd) + results = dir2dict( + str(tmp_path), + filter=lambda fn: fn.startswith(str(tmp_path / "etc")), + ) for cfgpath, expected in expected_cfgs.items(): actual = self.create_conf_dict(results[cfgpath].splitlines()) self.compare_dicts(actual, expected) - self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + assert 0o644 == get_mode(cfgpath, str(tmp_path)) def nwk_file_path(self, ifname): return "/etc/systemd/network/10-cloud-init-%s.network" % ifname @@ -1155,7 +1235,7 @@ def net_cfg_2(self, ifname): ) return ret - def test_photon_network_config_v1(self): + def test_photon_network_config_v1(self, distro_photon, tmp_path): tmp = self.net_cfg_1("eth0").splitlines() expected_eth0 = self.create_conf_dict(tmp) @@ -1168,10 +1248,13 @@ def test_photon_network_config_v1(self): } self._apply_and_verify( - self.distro.apply_network_config, V1_NET_CFG, expected_cfgs.copy() + distro_photon.apply_network_config, + V1_NET_CFG, + tmp_path, + expected_cfgs.copy(), ) - def test_photon_network_config_v2(self): + def test_photon_network_config_v2(self, distro_photon, tmp_path): tmp = self.net_cfg_1("eth7").splitlines() expected_eth7 = self.create_conf_dict(tmp) @@ -1184,10 +1267,15 @@ def test_photon_network_config_v2(self): } self._apply_and_verify( - self.distro.apply_network_config, V2_NET_CFG, expected_cfgs.copy() + distro_photon.apply_network_config, + V2_NET_CFG, + tmp_path, + expected_cfgs.copy(), ) - def test_photon_network_config_v1_with_duplicates(self): + def test_photon_network_config_v1_with_duplicates( + self, distro_photon, tmp_path + ): expected = """\ [Match] Name=eth0 @@ -1206,15 +1294,21 @@ def test_photon_network_config_v1_with_duplicates(self): } self._apply_and_verify( - self.distro.apply_network_config, net_cfg, expected_cfgs.copy() + distro_photon.apply_network_config, + net_cfg, + tmp_path, + expected_cfgs.copy(), ) -class TestNetCfgDistroMariner(TestNetCfgDistroBase): - def setUp(self): - super(TestNetCfgDistroMariner, self).setUp() - self.distro = self._get_distro("mariner", renderers=["networkd"]) +@pytest.fixture +def distro_mariner(mocker): + mocker.patch("cloudinit.net.networkd.util.chownbyname") + return get_distro("mariner", renderers=["networkd"]) + +@pytest.mark.usefixtures("fake_filesystem") +class TestNetCfgDistroMariner: def create_conf_dict(self, contents): content_dict = {} for line in contents: @@ -1231,25 +1325,26 @@ def create_conf_dict(self, contents): def compare_dicts(self, actual, expected): for k, v in actual.items(): - self.assertEqual(sorted(expected[k]), sorted(v)) + assert sorted(expected[k]) == sorted(v) def _apply_and_verify( - self, apply_fn, config, expected_cfgs=None, bringup=False + self, apply_fn, config, tmp_path, expected_cfgs=None, bringup=False ): if not expected_cfgs: raise ValueError("expected_cfg must not be None") - tmpd = None with mock.patch("cloudinit.net.networkd.available") as m_avail: m_avail.return_value = True - with self.reRooted(tmpd) as tmpd: - apply_fn(config, bringup) + apply_fn(config, bringup) - results = dir2dict(tmpd) + results = dir2dict( + str(tmp_path), + filter=lambda fn: fn.startswith(str(tmp_path / "etc")), + ) for cfgpath, expected in expected_cfgs.items(): actual = self.create_conf_dict(results[cfgpath].splitlines()) self.compare_dicts(actual, expected) - self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + assert 0o644 == get_mode(cfgpath, str(tmp_path)) def nwk_file_path(self, ifname): return "/etc/systemd/network/10-cloud-init-%s.network" % ifname @@ -1280,7 +1375,7 @@ def net_cfg_2(self, ifname): ) return ret - def test_mariner_network_config_v1(self): + def test_mariner_network_config_v1(self, distro_mariner, tmp_path): tmp = self.net_cfg_1("eth0").splitlines() expected_eth0 = self.create_conf_dict(tmp) @@ -1293,10 +1388,13 @@ def test_mariner_network_config_v1(self): } self._apply_and_verify( - self.distro.apply_network_config, V1_NET_CFG, expected_cfgs.copy() + distro_mariner.apply_network_config, + V1_NET_CFG, + tmp_path, + expected_cfgs.copy(), ) - def test_mariner_network_config_v2(self): + def test_mariner_network_config_v2(self, distro_mariner, tmp_path): tmp = self.net_cfg_1("eth7").splitlines() expected_eth7 = self.create_conf_dict(tmp) @@ -1309,10 +1407,15 @@ def test_mariner_network_config_v2(self): } self._apply_and_verify( - self.distro.apply_network_config, V2_NET_CFG, expected_cfgs.copy() + distro_mariner.apply_network_config, + V2_NET_CFG, + tmp_path, + expected_cfgs.copy(), ) - def test_mariner_network_config_v1_with_duplicates(self): + def test_mariner_network_config_v1_with_duplicates( + self, distro_mariner, tmp_path + ): expected = """\ [Match] Name=eth0 @@ -1331,15 +1434,21 @@ def test_mariner_network_config_v1_with_duplicates(self): } self._apply_and_verify( - self.distro.apply_network_config, net_cfg, expected_cfgs.copy() + distro_mariner.apply_network_config, + net_cfg, + tmp_path, + expected_cfgs.copy(), ) -class TestNetCfgDistroAzureLinux(TestNetCfgDistroBase): - def setUp(self): - super().setUp() - self.distro = self._get_distro("azurelinux", renderers=["networkd"]) +@pytest.fixture +def distro_azurelinux(mocker): + mocker.patch("cloudinit.net.networkd.util.chownbyname") + return get_distro("azurelinux", renderers=["networkd"]) + +@pytest.mark.usefixtures("fake_filesystem") +class TestNetCfgDistroAzureLinux: def create_conf_dict(self, contents): content_dict = {} for line in contents: @@ -1356,25 +1465,26 @@ def create_conf_dict(self, contents): def compare_dicts(self, actual, expected): for k, v in actual.items(): - self.assertEqual(sorted(expected[k]), sorted(v)) + assert sorted(expected[k]) == sorted(v) def _apply_and_verify( - self, apply_fn, config, expected_cfgs=None, bringup=False + self, apply_fn, config, tmp_path, expected_cfgs=None, bringup=False ): if not expected_cfgs: raise ValueError("expected_cfg must not be None") - tmpd = None with mock.patch("cloudinit.net.networkd.available") as m_avail: m_avail.return_value = True - with self.reRooted(tmpd) as tmpd: - apply_fn(config, bringup) + apply_fn(config, bringup) - results = dir2dict(tmpd) + results = dir2dict( + str(tmp_path), + filter=lambda fn: fn.startswith(str(tmp_path / "etc")), + ) for cfgpath, expected in expected_cfgs.items(): actual = self.create_conf_dict(results[cfgpath].splitlines()) self.compare_dicts(actual, expected) - self.assertEqual(0o644, get_mode(cfgpath, tmpd)) + assert 0o644 == get_mode(cfgpath, str(tmp_path)) def nwk_file_path(self, ifname): return "/etc/systemd/network/10-cloud-init-%s.network" % ifname @@ -1405,7 +1515,7 @@ def net_cfg_2(self, ifname): ) return ret - def test_azurelinux_network_config_v1(self): + def test_azurelinux_network_config_v1(self, distro_azurelinux, tmp_path): tmp = self.net_cfg_1("eth0").splitlines() expected_eth0 = self.create_conf_dict(tmp) @@ -1418,10 +1528,13 @@ def test_azurelinux_network_config_v1(self): } self._apply_and_verify( - self.distro.apply_network_config, V1_NET_CFG, expected_cfgs.copy() + distro_azurelinux.apply_network_config, + V1_NET_CFG, + tmp_path, + expected_cfgs.copy(), ) - def test_azurelinux_network_config_v2(self): + def test_azurelinux_network_config_v2(self, distro_azurelinux, tmp_path): tmp = self.net_cfg_1("eth7").splitlines() expected_eth7 = self.create_conf_dict(tmp) @@ -1434,10 +1547,15 @@ def test_azurelinux_network_config_v2(self): } self._apply_and_verify( - self.distro.apply_network_config, V2_NET_CFG, expected_cfgs.copy() + distro_azurelinux.apply_network_config, + V2_NET_CFG, + tmp_path, + expected_cfgs.copy(), ) - def test_azurelinux_network_config_v1_with_duplicates(self): + def test_azurelinux_network_config_v1_with_duplicates( + self, distro_azurelinux, tmp_path + ): expected = """\ [Match] Name=eth0 @@ -1456,7 +1574,10 @@ def test_azurelinux_network_config_v1_with_duplicates(self): } self._apply_and_verify( - self.distro.apply_network_config, net_cfg, expected_cfgs.copy() + distro_azurelinux.apply_network_config, + net_cfg, + tmp_path, + expected_cfgs.copy(), ) diff --git a/tests/unittests/distros/test_openbsd.py b/tests/unittests/distros/test_openbsd.py index 2bab0d3b..78342c17 100644 --- a/tests/unittests/distros/test_openbsd.py +++ b/tests/unittests/distros/test_openbsd.py @@ -1,7 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import mock +from tests.unittests.helpers import get_distro, mock M_PATH = "cloudinit.distros.openbsd." @@ -9,7 +8,7 @@ class TestOpenBSD: @mock.patch(M_PATH + "subp.subp") def test_add_user(self, m_subp): - distro = _get_distro("openbsd") + distro = get_distro("openbsd") assert True is distro.add_user("me2", uid=1234, default=False) assert [ mock.call( @@ -18,7 +17,7 @@ def test_add_user(self, m_subp): ] == m_subp.call_args_list def test_unlock_passwd(self, caplog): - distro = _get_distro("openbsd") + distro = get_distro("openbsd") distro.unlock_passwd("me2") assert ( "OpenBSD password lock is not reversible, " diff --git a/tests/unittests/distros/test_photon.py b/tests/unittests/distros/test_photon.py index c6c679ce..fdeade34 100644 --- a/tests/unittests/distros/test_photon.py +++ b/tests/unittests/distros/test_photon.py @@ -1,8 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import util -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import get_distro, mock SYSTEM_INFO = { "paths": { @@ -13,26 +12,25 @@ } -class TestPhoton(CiTestCase): - with_logs = True - distro = _get_distro("photon", SYSTEM_INFO) +class TestPhoton: + distro = get_distro("photon", SYSTEM_INFO) expected_log_line = "Rely on PhotonOS default network config" def test_network_renderer(self): - self.assertEqual(self.distro._cfg["network"]["renderers"], "networkd") + assert self.distro._cfg["network"]["renderers"] == "networkd" def test_get_distro(self): - self.assertEqual(self.distro.osfamily, "photon") + assert self.distro.osfamily == "photon" @mock.patch("cloudinit.distros.photon.subp.subp") - def test_write_hostname(self, m_subp): + def test_write_hostname(self, m_subp, caplog, tmp_path): hostname = "myhostname" - hostfile = self.tmp_path("previous-hostname") + hostfile = str(tmp_path / "previous-hostname") self.distro._write_hostname(hostname, hostfile) - self.assertEqual(hostname, util.load_text_file(hostfile)) + assert hostname == util.load_text_file(hostfile) ret = self.distro._read_hostname(hostfile) - self.assertEqual(ret, hostname) + assert ret == hostname m_subp.return_value = (None, None) hostfile += "hostfile" @@ -40,28 +38,28 @@ def test_write_hostname(self, m_subp): m_subp.return_value = (hostname, None) ret = self.distro._read_hostname(hostfile) - self.assertEqual(ret, hostname) + assert ret == hostname - self.logs.truncate(0) + caplog.clear() m_subp.return_value = (None, "bla") self.distro._write_hostname(hostname, None) - self.assertIn("Error while setting hostname", self.logs.getvalue()) + assert "Error while setting hostname" in caplog.text @mock.patch("cloudinit.net.generate_fallback_config") - def test_fallback_netcfg(self, m_fallback_cfg): + def test_fallback_netcfg(self, m_fallback_cfg, caplog): key = "disable_fallback_netcfg" # Don't use fallback if no setting given - self.logs.truncate(0) + caplog.clear() assert self.distro.generate_fallback_config() is None - self.assertIn(self.expected_log_line, self.logs.getvalue()) + assert self.expected_log_line in caplog.text - self.logs.truncate(0) + caplog.clear() self.distro._cfg[key] = True assert self.distro.generate_fallback_config() is None - self.assertIn(self.expected_log_line, self.logs.getvalue()) + assert self.expected_log_line in caplog.text - self.logs.truncate(0) + caplog.clear() self.distro._cfg[key] = False assert self.distro.generate_fallback_config() is not None - self.assertNotIn(self.expected_log_line, self.logs.getvalue()) + assert self.expected_log_line not in caplog.text diff --git a/tests/unittests/distros/test_raspberry_pi_os.py b/tests/unittests/distros/test_raspberry_pi_os.py new file mode 100644 index 00000000..508b84bb --- /dev/null +++ b/tests/unittests/distros/test_raspberry_pi_os.py @@ -0,0 +1,95 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import logging + +from cloudinit.distros import fetch +from cloudinit.subp import ProcessExecutionError +from tests.unittests.helpers import mock + +M_PATH = "cloudinit.distros.raspberry_pi_os." + + +class TestRaspberryPiOS: + @mock.patch(M_PATH + "subp.subp") + def test_set_keymap_calls_imager_custom(self, m_subp): + cls = fetch("raspberry_pi_os") + distro = cls("raspberry-pi-os", {}, None) + distro.set_keymap("us", "pc105", "basic", "") + m_subp.assert_called_once_with( + ["/usr/lib/raspberrypi-sys-mods/imager_custom", "set_keymap", "us"] + ) + + @mock.patch(M_PATH + "subp.subp") + def test_apply_locale_happy_path(self, m_subp): + cls = fetch("raspberry_pi_os") + distro = cls("raspberry-pi-os", {}, None) + distro.apply_locale("en_GB.UTF-8") + m_subp.assert_called_once_with( + [ + "/usr/bin/raspi-config", + "nonint", + "do_change_locale", + "en_GB.UTF-8", + ] + ) + + @mock.patch(M_PATH + "subp.subp") + def test_apply_locale_fallback_to_utf8(self, m_subp): + m_subp.side_effect = [ + ProcessExecutionError("Invalid locale"), # Simulate failure + None, # Fallback succeeds + ] + cls = fetch("raspberry_pi_os") + distro = cls("raspberry-pi-os", {}, None) + distro.apply_locale("en_GB") + assert m_subp.call_count == 2 + m_subp.assert_any_call( + ["/usr/bin/raspi-config", "nonint", "do_change_locale", "en_GB"] + ) + m_subp.assert_any_call( + [ + "/usr/bin/raspi-config", + "nonint", + "do_change_locale", + "en_GB.UTF-8", + ] + ) + + @mock.patch(M_PATH + "subp.subp") + def test_add_user_happy_path(self, m_subp): + cls = fetch("raspberry_pi_os") + distro = cls("raspberry-pi-os", {}, None) + # Mock the superclass add_user to return True + with mock.patch( + "cloudinit.distros.debian.Distro.add_user", return_value=True + ): + assert distro.add_user("pi") is True + m_subp.assert_called_once_with( + ["/usr/bin/rename-user", "-f", "-s"], + update_env={"SUDO_USER": "pi"}, + ) + + @mock.patch(M_PATH + "subp.subp") + def test_add_user_existing_user(self, m_subp): + cls = fetch("raspberry_pi_os") + distro = cls("raspberry-pi-os", {}, None) + with mock.patch( + "cloudinit.distros.debian.Distro.add_user", return_value=False + ): + assert distro.add_user("pi") is False + m_subp.assert_not_called() + + @mock.patch( + M_PATH + "subp.subp", + side_effect=ProcessExecutionError("rename-user failed"), + ) + @mock.patch("cloudinit.distros.debian.Distro.add_user", return_value=True) + def test_add_user_rename_fails_logs_error( + self, m_super_add_user, m_subp, caplog + ): + cls = fetch("raspberry_pi_os") + distro = cls("raspberry-pi-os", {}, None) + + with caplog.at_level(logging.ERROR): + assert distro.add_user("pi") is False + assert "Failed to setup user" in caplog.text diff --git a/tests/unittests/distros/test_sles.py b/tests/unittests/distros/test_sles.py index 7732c380..777e4267 100644 --- a/tests/unittests/distros/test_sles.py +++ b/tests/unittests/distros/test_sles.py @@ -1,10 +1,9 @@ # This file is part of cloud-init. See LICENSE file for license information. -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import CiTestCase +from tests.unittests.helpers import get_distro -class TestSLES(CiTestCase): +class TestSLES: def test_get_distro(self): - distro = _get_distro("sles") - self.assertEqual(distro.osfamily, "suse") + distro = get_distro("sles") + assert distro.osfamily == "suse" diff --git a/tests/unittests/filters/test_launch_index.py b/tests/unittests/filters/test_launch_index.py index 1b2ebfb2..7e8895b4 100644 --- a/tests/unittests/filters/test_launch_index.py +++ b/tests/unittests/filters/test_launch_index.py @@ -18,15 +18,15 @@ def count_messages(root): return am -class TestLaunchFilter(helpers.ResourceUsingTestCase): +class TestLaunchFilter: def assertCounts(self, message, expected_counts): orig_message = copy.deepcopy(message) for index, count in expected_counts.items(): index = util.safe_int(index) filtered_message = launch_index.Filter(index).apply(message) - self.assertEqual(count_messages(filtered_message), count) + assert count_messages(filtered_message) == count # Ensure original message still ok/not modified - self.assertTrue(self.equivalentMessage(message, orig_message)) + assert self.equivalentMessage(message, orig_message) is True def equivalentMessage(self, msg1, msg2): msg1_count = count_messages(msg1) @@ -51,11 +51,10 @@ def equivalentMessage(self, msg1, msg2): return False return True - def testMultiEmailIndex(self): + def testMultiEmailIndex(self, ud_proc): test_data = helpers.readResource("filter_cloud_multipart_2.email") - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) - self.assertTrue(count_messages(message) > 0) + assert count_messages(message) > 0 # This file should have the following # indexes -> amount mapping in it expected_counts = { @@ -66,11 +65,10 @@ def testMultiEmailIndex(self): } self.assertCounts(message, expected_counts) - def testHeaderEmailIndex(self): + def testHeaderEmailIndex(self, ud_proc): test_data = helpers.readResource("filter_cloud_multipart_header.email") - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) - self.assertTrue(count_messages(message) > 0) + assert count_messages(message) > 0 # This file should have the following # indexes -> amount mapping in it expected_counts = { @@ -81,11 +79,10 @@ def testHeaderEmailIndex(self): } self.assertCounts(message, expected_counts) - def testConfigEmailIndex(self): + def testConfigEmailIndex(self, ud_proc): test_data = helpers.readResource("filter_cloud_multipart_1.email") - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) - self.assertTrue(count_messages(message) > 0) + assert count_messages(message) > 0 # This file should have the following # indexes -> amount mapping in it expected_counts = { @@ -95,21 +92,19 @@ def testConfigEmailIndex(self): } self.assertCounts(message, expected_counts) - def testNoneIndex(self): + def testNoneIndex(self, ud_proc): test_data = helpers.readResource("filter_cloud_multipart.yaml") - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) start_count = count_messages(message) - self.assertTrue(start_count > 0) + assert start_count > 0 filtered_message = launch_index.Filter(None).apply(message) - self.assertTrue(self.equivalentMessage(message, filtered_message)) + assert self.equivalentMessage(message, filtered_message) - def testIndexes(self): + def testIndexes(self, ud_proc): test_data = helpers.readResource("filter_cloud_multipart.yaml") - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(test_data) start_count = count_messages(message) - self.assertTrue(start_count > 0) + assert start_count > 0 # This file should have the following # indexes -> amount mapping in it expected_counts = { diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index dfd9a508..7e9eda8c 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. # pylint: disable=attribute-defined-outside-init +import copy import functools import io import logging @@ -8,11 +9,10 @@ import random import shutil import string -import sys import tempfile import time import unittest -from contextlib import ExitStack, contextmanager +from contextlib import contextmanager from typing import ClassVar, List, Union from unittest import mock from unittest.util import strclass @@ -20,14 +20,12 @@ import responses -from cloudinit import atomic_helper, cloud, distros -from cloudinit import helpers as ch -from cloudinit import subp, util +from cloudinit import distros, helpers, settings, subp, util from cloudinit.config.schema import ( SchemaValidationError, validate_cloudconfig_schema, ) -from cloudinit.sources import DataSourceNone +from cloudinit.helpers import Paths from cloudinit.templater import JINJA_AVAILABLE from tests.helpers import cloud_init_project_dir from tests.hypothesis_jsonschema import HAS_HYPOTHESIS_JSONSCHEMA @@ -262,207 +260,58 @@ def tmp_path(self, path, dir=None): dir = self.tmp_dir() return os.path.normpath(os.path.abspath(os.path.join(dir, path))) - def tmp_cloud(self, distro, sys_cfg=None, metadata=None): - """Create a cloud with tmp working directory paths. - - @param distro: Name of the distro to attach to the cloud. - @param metadata: Optional metadata to set on the datasource. - - @return: The built cloud instance. - """ - self.new_root = self.tmp_dir() - if not sys_cfg: - sys_cfg = {} - MockPaths = get_mock_paths(self.new_root) - self.paths = MockPaths({}) - cls = distros.fetch(distro) - mydist = cls(distro, sys_cfg, self.paths) - myds = DataSourceNone.DataSourceNone(sys_cfg, mydist, self.paths) - if metadata: - myds.metadata.update(metadata) - return cloud.Cloud(myds, self.paths, sys_cfg, mydist, None) - @classmethod def random_string(cls, length=8): return random_string(length) -class ResourceUsingTestCase(CiTestCase): - def setUp(self): - super(ResourceUsingTestCase, self).setUp() - self.resource_path = None - - def getCloudPaths(self, ds=None): - tmpdir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmpdir) - cp = ch.Paths( - {"cloud_dir": tmpdir, "templates_dir": resourceLocation()}, ds=ds - ) - return cp - - -class FilesystemMockingTestCase(ResourceUsingTestCase): - def setUp(self): - super(FilesystemMockingTestCase, self).setUp() - self.patched_funcs = ExitStack() - - def tearDown(self): - self.patched_funcs.close() - ResourceUsingTestCase.tearDown(self) - - def replicateTestRoot(self, example_root, target_root): - real_root = resourceLocation() - real_root = os.path.join(real_root, "roots", example_root) - for dir_path, _dirnames, filenames in os.walk(real_root): - real_path = dir_path - make_path = rebase_path(real_path[len(real_root) :], target_root) - util.ensure_dir(make_path) - for f in filenames: - real_path = os.path.abspath(os.path.join(real_path, f)) - make_path = os.path.abspath(os.path.join(make_path, f)) - shutil.copy(real_path, make_path) - - def patchUtils(self, new_root): - patch_funcs = { - util: [ - ("write_file", 1), - ("append_file", 1), - ("load_binary_file", 1), - ("load_text_file", 1), - ("ensure_dir", 1), - ("chmod", 1), - ("delete_dir_contents", 1), - ("del_file", 1), - ("sym_link", -1), - ("copy", -1), - ], - atomic_helper: [ - ("write_json", 1), - ], - } - for mod, funcs in patch_funcs.items(): - for f, am in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, am, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func) - ) - - # Handle subprocess calls - func = getattr(subp, "subp") - - def nsubp(*_args, **_kwargs): - return ("", "") - - self.patched_funcs.enter_context( - mock.patch.object(subp, "subp", nsubp) - ) - - def null_func(*_args, **_kwargs): - return None - - for f in ["chownbyid", "chownbyname"]: - self.patched_funcs.enter_context( - mock.patch.object(util, f, null_func) - ) - - def patchOS(self, new_root): - patch_funcs = { - os.path: [ - ("isfile", 1), - ("exists", 1), - ("islink", 1), - ("isdir", 1), - ("lexists", 1), - ], - os: [ - ("listdir", 1), - ("mkdir", 1), - ("lstat", 1), - ("symlink", 2), - ("stat", 1), - ], - } - - if hasattr(os, "scandir"): - # py27 does not have scandir - patch_funcs[os].append(("scandir", 1)) - - for mod, funcs in patch_funcs.items(): - for f, nargs in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, nargs, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func) - ) - - def patchOpen(self, new_root): - trap_func = retarget_many_wrapper(new_root, 1, open) - self.patched_funcs.enter_context( - mock.patch("builtins.open", trap_func) - ) - - def patchStdoutAndStderr(self, stdout=None, stderr=None): - if stdout is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, "stdout", stdout) - ) - if stderr is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, "stderr", stderr) - ) - - def reRoot(self, root=None): - if root is None: - root = self.tmp_dir() - self.patchUtils(root) - self.patchOS(root) - self.patchOpen(root) - return root - - @contextmanager - def reRooted(self, root=None): - try: - yield self.reRoot(root) - finally: - self.patched_funcs.close() +def replicate_test_root(example_root, target_root): + real_root = resourceLocation() + real_root = os.path.join(real_root, "roots", example_root) + for dir_path, _dirnames, filenames in os.walk(real_root): + real_path = dir_path + make_path = rebase_path(real_path[len(real_root) :], target_root) + util.ensure_dir(make_path) + for f in filenames: + real_path = os.path.abspath(os.path.join(real_path, f)) + make_path = os.path.abspath(os.path.join(make_path, f)) + shutil.copy(real_path, make_path) -class CiRequestsMock(responses.RequestsMock): - def assert_call_count(self, url: str, count: int) -> bool: - """Focal and older have a version of responses which does - not carry this attribute. This can be removed when focal - is no longer supported. - """ - if hasattr(super(), "_ensure_url_default_path"): - return super().assert_call_count(url, count) - - def _ensure_url_default_path(url): - if isinstance(url, str): - url_parts = list(urlsplit(url)) - if url_parts[2] == "": - url_parts[2] = "/" - url = urlunsplit(url_parts) - return url - - call_count = len( - [ - 1 - for call in self.calls - if call.request.url == _ensure_url_default_path(url) - ] +def responses_assert_call_count(url: str, count: int) -> bool: + """Focal and older have a version of responses which does + not carry this attribute. This can be removed when focal + is no longer supported. + """ + if hasattr(responses, "assert_call_count"): + return responses.assert_call_count(url, count) + + def _ensure_url_default_path(url): + if isinstance(url, str): + url_parts = list(urlsplit(url)) + if url_parts[2] == "": + url_parts[2] = "/" + url = urlunsplit(url_parts) + return url + + call_count = len( + [ + 1 + for call in responses.calls + if call.request.url == _ensure_url_default_path(url) + ] + ) + if call_count == count: + return True + else: + raise AssertionError( + f"Expected URL '{url}' to be called {count} times. " + f"Called {call_count} times." ) - if call_count == count: - return True - else: - raise AssertionError( - f"Expected URL '{url}' to be called {count} times. " - f"Called {call_count} times." - ) def get_mock_paths(temp_dir): - class MockPaths(ch.Paths): + class MockPaths(Paths): def __init__(self, path_cfgs: dict, ds=None): super().__init__(path_cfgs=path_cfgs, ds=ds) @@ -479,18 +328,6 @@ def __init__(self, path_cfgs: dict, ds=None): return MockPaths -class ResponsesTestCase(CiTestCase): - def setUp(self): - super().setUp() - self.responses = CiRequestsMock(assert_all_requests_are_fired=False) - self.responses.start() - - def tearDown(self): - self.responses.stop() - self.responses.reset() - super().tearDown() - - class SchemaTestCaseMixin(unittest.TestCase): def assertSchemaValid(self, cfg, msg="Valid Schema failed validation."): """Assert the config is valid per self.schema. @@ -534,7 +371,7 @@ def populate_dir_with_ts(path, data): os.utime(os.path.sep.join((path, fpath)), (ts, ts)) -def dir2dict(startdir, prefix=None): +def dir2dict(startdir, prefix=None, filter=None): flist = {} if prefix is None: prefix = startdir @@ -542,6 +379,8 @@ def dir2dict(startdir, prefix=None): for fname in files: fpath = os.path.join(root, fname) key = fpath[len(prefix) :] + if filter is not None and not filter(fpath): + continue flist[key] = util.load_text_file(fpath) return flist @@ -636,6 +475,16 @@ def skipUnlessJinja(): return skipIf(not JINJA_AVAILABLE, "No jinja dependency present.") +@skipUnlessJinja() +def skipUnlessJinjaVersionGreaterThan(version=(0, 0, 0)): + import jinja2 + + return skipIf( + condition=tuple(map(int, jinja2.__version__.split("."))) < version, + reason=f"jinj2 version is less than {version}", + ) + + def skipIfJinja(): return skipIf(JINJA_AVAILABLE, "Jinja dependency present.") @@ -685,3 +534,32 @@ def does_not_raise(): """ yield + + +def get_distro(dname, system_info=None, /, renderers=None, activators=None): + """Return a Distro class of distro 'dname'. + + system_info has the format of CFG_BUILTIN['system_info']. + + Example: get_distro("debian") + """ + if system_info is None: + system_info = copy.deepcopy(settings.CFG_BUILTIN["system_info"]) + system_info["distro"] = dname + if renderers: + system_info["network"]["renderers"] = renderers + if activators: + system_info["network"]["activators"] = activators + paths = helpers.Paths(system_info["paths"]) + distro_cls = distros.fetch(dname) + return distro_cls(dname, system_info, paths) + + +def assert_count_equal(a, b): + """ + Equivalent to unittests.TestCase.assertCountEqual. + + https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertCountEqual + """ + case = unittest.TestCase() + case.assertCountEqual(a, b) diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond.200.netdev b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond.200.netdev new file mode 100644 index 00000000..c82fd6f5 --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond.200.netdev @@ -0,0 +1,8 @@ +[NetDev] +Kind=vlan +MACAddress=00:11:22:33:44:99 +MTUBytes=111 +Name=bond.200 + +[VLAN] +Id=200 diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond.200.network b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond.200.network new file mode 100644 index 00000000..64563364 --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond.200.network @@ -0,0 +1,14 @@ +[Address] +Address=192.168.200.10/24 + +[Link] +MACAddress=00:11:22:33:44:99 +MTUBytes=111 + +[Match] +Name=bond.200 + +[Network] +DHCP=ipv6 +DNS=1.1.1.1 8.8.4.4 +Domains=bond.vlan.test diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond0.netdev b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond0.netdev new file mode 100644 index 00000000..dd2da41d --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond0.netdev @@ -0,0 +1,11 @@ +[Bond] +LACPTransmitRate=fast +MIIMonitorSec=100ms +Mode=802.3ad +TransmitHashPolicy=layer2+3 + +[NetDev] +Kind=bond +MACAddress=00:11:22:33:44:77 +MTUBytes=333 +Name=bond0 diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond0.network b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond0.network new file mode 100644 index 00000000..bbebbf26 --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-bond0.network @@ -0,0 +1,10 @@ +[Link] +MACAddress=00:11:22:33:44:77 +MTUBytes=333 + +[Match] +Name=bond0 + +[Network] +DHCP=no +VLAN=bond.200 diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth0.network b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth0.network new file mode 100644 index 00000000..e3a8358f --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth0.network @@ -0,0 +1,11 @@ +[Link] +MTUBytes=1500 + +[Match] +MACAddress=00:11:22:33:44:55 +Name=eth0 + +[Network] +Bond=bond0 +DHCP=no +VLAN=vlan100 diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth1.network b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth1.network new file mode 100644 index 00000000..19e5db2f --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth1.network @@ -0,0 +1,5 @@ +[Match] +Name=eth1 + +[Network] +DHCP=yes diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth2.network b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth2.network new file mode 100644 index 00000000..712c52b7 --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-eth2.network @@ -0,0 +1,16 @@ +[Address] +Address=10.20.30.40/24 + +[Link] +MTUBytes=1400 + +[Match] +MACAddress=00:aa:bb:cc:dd:ee +Name=eth2 + +[Network] +Bond=bond0 +DHCP=no +DNS=9.9.9.9 8.8.8.8 +Domains=example.org +VLAN=vl101 diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vl101.netdev b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vl101.netdev new file mode 100644 index 00000000..3fa9ff33 --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vl101.netdev @@ -0,0 +1,6 @@ +[NetDev] +Kind=vlan +Name=vl101 + +[VLAN] +Id=101 diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vl101.network b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vl101.network new file mode 100644 index 00000000..fb82925b --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vl101.network @@ -0,0 +1,5 @@ +[Match] +Name=vl101 + +[Network] +DHCP=no diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vlan100.netdev b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vlan100.netdev new file mode 100644 index 00000000..5c6a1c09 --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vlan100.netdev @@ -0,0 +1,8 @@ +[NetDev] +Kind=vlan +MACAddress=00:11:22:33:44:66 +MTUBytes=901 +Name=vlan100 + +[VLAN] +Id=100 diff --git a/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vlan100.network b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vlan100.network new file mode 100644 index 00000000..f8afc472 --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config/etc/systemd/network/10-cloud-init-vlan100.network @@ -0,0 +1,27 @@ +[Address] +Address=192.168.100.10/24 + +[Address] +Address=192.168.100.11/24 + +[Link] +MACAddress=00:11:22:33:44:66 +MTUBytes=901 + +[Match] +Name=vlan100 + +[Network] +DHCP=ipv6 +DNS=8.8.8.8 1.1.1.1 +Domains=corp.example.com vlan.test + +[Route] +Destination=10.10.200.0/24 +Gateway=192.168.100.1 +Metric=50 + +[Route] +Destination=10.10.201.0/24 +Gateway=192.168.100.2 +Metric=150 diff --git a/tests/unittests/net/artifacts/photon_net_config_v2.yaml b/tests/unittests/net/artifacts/photon_net_config_v2.yaml new file mode 100644 index 00000000..7984f843 --- /dev/null +++ b/tests/unittests/net/artifacts/photon_net_config_v2.yaml @@ -0,0 +1,86 @@ +network: + version: 2 + + ethernets: + eth0: + match: + macaddress: "00:11:22:33:44:55" + set-name: eth0 + dhcp4: false + mtu: 1500 + + eth1: + dhcp4: true + dhcp6: true + + eth2: + match: + macaddress: "00:aa:bb:cc:dd:ee" + set-name: eth2 + mtu: 1400 + dhcp4: false + addresses: + - 10.20.30.40/24 + nameservers: + addresses: + - 9.9.9.9 + - 8.8.8.8 + search: + - example.org + + bonds: + bond0: + interfaces: [eth0, eth2] + parameters: + mode: 802.3ad + mii-monitor-interval: 100 + transmit-hash-policy: layer2+3 + lacp-rate: fast + dhcp4: false + dhcp6: false + mtu: 333 + macaddress: "00:11:22:33:44:77" + + vlans: + vlan100: + id: 100 + link: eth0 + addresses: + - 192.168.100.10/24 + - 192.168.100.11/24 + dhcp6: true + mtu: 901 + macaddress: "00:11:22:33:44:66" + nameservers: + addresses: + - 8.8.8.8 + - 1.1.1.1 + search: + - corp.example.com + - vlan.test + routes: + - to: 10.10.200.0/24 + via: 192.168.100.1 + metric: 50 + - to: 10.10.201.0/24 + via: 192.168.100.2 + metric: 150 + + vl101: + id: 101 + link: eth2 + + bond.200: + id: 200 + link: bond0 + macaddress: "00:11:22:33:44:99" + addresses: + - 192.168.200.10/24 + nameservers: + addresses: + - 1.1.1.1 + - 8.8.4.4 + search: + - bond.vlan.test + mtu: 111 + dhcp6: true diff --git a/tests/unittests/net/network_configs.py b/tests/unittests/net/network_configs.py index 6e870f6e..c2fd524a 100644 --- a/tests/unittests/net/network_configs.py +++ b/tests/unittests/net/network_configs.py @@ -937,7 +937,7 @@ auto iface0 iface iface0 inet6 dhcp - accept_ra 1 + accept-ra 1 """ ).rstrip(" "), "expected_netplan": textwrap.dedent( @@ -1000,7 +1000,7 @@ Name=iface0 [Network] DHCP=ipv6 - IPv6AcceptRA=True + IPv6AcceptRA=yes """ ).rstrip(" "), }, @@ -1012,7 +1012,7 @@ auto iface0 iface iface0 inet6 dhcp - accept_ra 0 + accept-ra 0 """ ).rstrip(" "), "expected_netplan": textwrap.dedent( @@ -1075,7 +1075,7 @@ Name=iface0 [Network] DHCP=ipv6 - IPv6AcceptRA=False + IPv6AcceptRA=no """ ).rstrip(" "), }, @@ -1509,16 +1509,16 @@ auto eth1 iface eth1 inet manual bond-master bond0 + bond-miimon 100 bond-mode active-backup bond-xmit-hash-policy layer3+4 - bond_miimon 100 auto eth2 iface eth2 inet manual bond-master bond0 + bond-miimon 100 bond-mode active-backup bond-xmit-hash-policy layer3+4 - bond_miimon 100 iface eth3 inet manual @@ -1535,29 +1535,29 @@ auto bond0 iface bond0 inet6 dhcp + bond-miimon 100 bond-mode active-backup bond-slaves none bond-xmit-hash-policy layer3+4 - bond_miimon 100 hwaddress aa:bb:cc:dd:ee:ff auto br0 iface br0 inet static address 192.168.14.2/24 - bridge_ageing 250 - bridge_bridgeprio 22 - bridge_fd 1 - bridge_gcint 2 - bridge_hello 1 - bridge_maxage 10 - bridge_pathcost eth3 50 - bridge_pathcost eth4 75 - bridge_portprio eth3 28 - bridge_portprio eth4 14 - bridge_ports eth3 eth4 - bridge_stp off - bridge_waitport 1 eth3 - bridge_waitport 2 eth4 + bridge-ageing 250 + bridge-bridgeprio 22 + bridge-fd 1 + bridge-gcint 2 + bridge-hello 1 + bridge-maxage 10 + bridge-pathcost eth3 50 + bridge-pathcost eth4 75 + bridge-portprio eth3 28 + bridge-portprio eth4 14 + bridge-ports eth3 eth4 + bridge-stp off + bridge-waitport 1 eth3 + bridge-waitport 2 eth4 hwaddress bb:bb:bb:bb:bb:aa # control-alias br0 @@ -1568,8 +1568,8 @@ auto bond0.200 iface bond0.200 inet dhcp + vlan-id 200 vlan-raw-device bond0 - vlan_id 200 auto eth0.101 iface eth0.101 inet static @@ -1579,12 +1579,14 @@ gateway 192.168.0.1 mtu 1500 hwaddress aa:bb:cc:dd:ee:11 + vlan-id 101 vlan-raw-device eth0 - vlan_id 101 # control-alias eth0.101 iface eth0.101 inet static address 192.168.2.10/24 + dns-nameservers 192.168.0.10 10.23.23.134 + dns-search barley.maas sacchromyces.maas brettanomyces.maas post-up route add -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true @@ -2350,16 +2352,16 @@ auto eth1 iface eth1 inet manual bond-master bond0 + bond-miimon 100 bond-mode active-backup bond-xmit-hash-policy layer3+4 - bond_miimon 100 auto eth2 iface eth2 inet manual bond-master bond0 + bond-miimon 100 bond-mode active-backup bond-xmit-hash-policy layer3+4 - bond_miimon 100 iface eth3 inet manual @@ -2376,29 +2378,29 @@ auto bond0 iface bond0 inet6 dhcp + bond-miimon 100 bond-mode active-backup bond-slaves none bond-xmit-hash-policy layer3+4 - bond_miimon 100 hwaddress aa:bb:cc:dd:ee:ff auto br0 iface br0 inet static address 192.168.14.2/24 - bridge_ageing 250 - bridge_bridgeprio 22 - bridge_fd 1 - bridge_gcint 2 - bridge_hello 1 - bridge_maxage 10 - bridge_pathcost eth3 50 - bridge_pathcost eth4 75 - bridge_portprio eth3 28 - bridge_portprio eth4 14 - bridge_ports eth3 eth4 - bridge_stp off - bridge_waitport 1 eth3 - bridge_waitport 2 eth4 + bridge-ageing 250 + bridge-bridgeprio 22 + bridge-fd 1 + bridge-gcint 2 + bridge-hello 1 + bridge-maxage 10 + bridge-pathcost eth3 50 + bridge-pathcost eth4 75 + bridge-portprio eth3 28 + bridge-portprio eth4 14 + bridge-ports eth3 eth4 + bridge-stp off + bridge-waitport 1 eth3 + bridge-waitport 2 eth4 hwaddress bb:bb:bb:bb:bb:aa # control-alias br0 @@ -2409,8 +2411,8 @@ auto bond0.200 iface bond0.200 inet dhcp + vlan-id 200 vlan-raw-device bond0 - vlan_id 200 auto eth0.101 iface eth0.101 inet static @@ -2420,8 +2422,8 @@ gateway 192.168.0.1 mtu 1500 hwaddress aa:bb:cc:dd:ee:11 + vlan-id 101 vlan-raw-device eth0 - vlan_id 101 # control-alias eth0.101 iface eth0.101 inet static @@ -3138,43 +3140,43 @@ auto bond0s0 iface bond0s0 inet manual bond-downdelay 10 - bond-fail_over_mac active + bond-fail-over-mac active bond-master bond0 + bond-miimon 100 bond-mode active-backup - bond-num_grat_arp 5 + bond-num-grat-arp 5 bond-primary bond0s0 - bond-primary_reselect always + bond-primary-reselect always bond-updelay 20 - bond-xmit_hash_policy layer3+4 - bond_miimon 100 + bond-xmit-hash-policy layer3+4 auto bond0s1 iface bond0s1 inet manual bond-downdelay 10 - bond-fail_over_mac active + bond-fail-over-mac active bond-master bond0 + bond-miimon 100 bond-mode active-backup - bond-num_grat_arp 5 + bond-num-grat-arp 5 bond-primary bond0s0 - bond-primary_reselect always + bond-primary-reselect always bond-updelay 20 - bond-xmit_hash_policy layer3+4 - bond_miimon 100 + bond-xmit-hash-policy layer3+4 auto bond0 iface bond0 inet static address 192.168.0.2/24 gateway 192.168.0.1 bond-downdelay 10 - bond-fail_over_mac active + bond-fail-over-mac active + bond-miimon 100 bond-mode active-backup - bond-num_grat_arp 5 + bond-num-grat-arp 5 bond-primary bond0s0 - bond-primary_reselect always + bond-primary-reselect always bond-slaves none bond-updelay 20 - bond-xmit_hash_policy layer3+4 - bond_miimon 100 + bond-xmit-hash-policy layer3+4 hwaddress aa:bb:cc:dd:e8:ff mtu 9000 post-up route add -net 10.1.3.0/24 gw 192.168.0.3 || true @@ -4602,6 +4604,39 @@ ), }, "v2-mixed-routes": { + "expected_eni": textwrap.dedent( + """\ + auto lo + iface lo inet loopback + + auto eth0 + iface eth0 inet dhcp + mtu 500 + post-up route add -host 169.254.42.42/32 gw 62.210.0.1 || true + pre-down route del -host 169.254.42.42/32 gw 62.210.0.1 || true + post-up route add -host 169.254.42.43/32 gw 62.210.0.2 || true + pre-down route del -host 169.254.42.43/32 gw 62.210.0.2 || true + + # control-alias eth0 + iface eth0 inet6 dhcp + post-up route add -A inet6 default gw fe80::dc00:ff:fe20:186 || true + pre-down route del -A inet6 default gw fe80::dc00:ff:fe20:186 || true + post-up route add -A inet6 fe80::dc00:ff:fe20:188/64 gw fe80::dc00:ff:fe20:187 || true + pre-down route del -A inet6 fe80::dc00:ff:fe20:188/64 gw fe80::dc00:ff:fe20:187 || true + + # control-alias eth0 + iface eth0 inet static + address 192.168.1.20/16 + dns-nameservers 8.8.8.8 + dns-search lab home + + # control-alias eth0 + iface eth0 inet6 static + address 2001:bc8:1210:232:dc00:ff:fe20:185/64 + dns-nameservers FEDC::1 + dns-search lab home + """ # noqa: E501 + ), "expected_network_manager": { "cloud-init-eth0.nmconnection": textwrap.dedent( """\ @@ -4675,6 +4710,190 @@ """ ), }, + "v2-mixed-routes-reversed": { + "expected_eni": textwrap.dedent( + """\ + auto lo + iface lo inet loopback + + auto eth0 + iface eth0 inet dhcp + mtu 500 + post-up route add -host 169.254.42.42/32 gw 62.210.0.1 || true + pre-down route del -host 169.254.42.42/32 gw 62.210.0.1 || true + post-up route add -host 169.254.42.43/32 gw 62.210.0.2 || true + pre-down route del -host 169.254.42.43/32 gw 62.210.0.2 || true + + # control-alias eth0 + iface eth0 inet6 dhcp + post-up route add -A inet6 default gw fe80::dc00:ff:fe20:186 || true + pre-down route del -A inet6 default gw fe80::dc00:ff:fe20:186 || true + post-up route add -A inet6 fe80::dc00:ff:fe20:188/64 gw fe80::dc00:ff:fe20:187 || true + pre-down route del -A inet6 fe80::dc00:ff:fe20:188/64 gw fe80::dc00:ff:fe20:187 || true + + # control-alias eth0 + iface eth0 inet6 static + address 2001:bc8:1210:232:dc00:ff:fe20:185/64 + dns-nameservers FEDC::1 + dns-search home lab + + # control-alias eth0 + iface eth0 inet static + address 192.168.1.20/16 + dns-nameservers 8.8.8.8 + dns-search home lab + """ # noqa: E501 + ), + "expected_network_manager": { + "cloud-init-eth0.nmconnection": textwrap.dedent( + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init eth0 + uuid=1dd9a779-d327-56e1-8454-c65e2556c12c + autoconnect-priority=120 + type=ethernet + interface-name=eth0 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + mtu=500 + + [ipv4] + method=auto + may-fail=true + route1=169.254.42.42/32,62.210.0.1 + route1_options=mtu=400 + route2=169.254.42.43/32,62.210.0.2 + route2_options=mtu=200 + address1=192.168.1.20/16 + dns=8.8.8.8; + dns-search=home;lab; + + [ipv6] + route1=::/0,fe80::dc00:ff:fe20:186 + route1_options=mtu=300 + route2=fe80::dc00:ff:fe20:188/64,fe80::dc00:ff:fe20:187 + route2_options=mtu=100 + method=auto + may-fail=true + address1=2001:bc8:1210:232:dc00:ff:fe20:185/64 + dns=FEDC::1; + dns-search=home;lab; + + """ + ) + }, + "yaml": textwrap.dedent( + """\ + version: 2 + ethernets: + eth0: + dhcp6: true + dhcp4: true + mtu: 500 + nameservers: + search: [home, lab] + addresses: ["FEDC::1", 8.8.8.8] + routes: + - via: fe80::dc00:ff:fe20:186 + to: ::/0 + mtu: 300 + - to: 169.254.42.42/32 + via: 62.210.0.1 + mtu: 400 + - via: fe80::dc00:ff:fe20:187 + to: fe80::dc00:ff:fe20:188 + mtu: 100 + - to: 169.254.42.43/32 + via: 62.210.0.2 + mtu: 200 + addresses: + - 2001:bc8:1210:232:dc00:ff:fe20:185/64 + - 192.168.1.20/16 + """ + ), + }, + "v2-mixed-routes-no-ipv6-addr": { + "expected_eni": textwrap.dedent( + """\ + auto lo + iface lo inet loopback + + auto eth0 + iface eth0 inet dhcp + post-up route add -host 169.254.42.42/32 gw 62.210.0.1 || true + pre-down route del -host 169.254.42.42/32 gw 62.210.0.1 || true + + # control-alias eth0 + iface eth0 inet static + address 192.168.1.20/16 + dns-nameservers 8.8.8.8 + dns-search lab home + + # control-alias eth0 + iface eth0 inet6 static + dns-nameservers FEDC::1 + dns-search lab home + post-up route add -A inet6 default gw fe80::dc00:ff:fe20:186 || true + pre-down route del -A inet6 default gw fe80::dc00:ff:fe20:186 || true + """ # noqa: E501 + ), + "expected_network_manager": { + "cloud-init-eth0.nmconnection": textwrap.dedent( + """\ + # Generated by cloud-init. Changes will be lost. + + [connection] + id=cloud-init eth0 + uuid=1dd9a779-d327-56e1-8454-c65e2556c12c + autoconnect-priority=120 + type=ethernet + interface-name=eth0 + + [user] + org.freedesktop.NetworkManager.origin=cloud-init + + [ethernet] + + [ipv4] + method=auto + may-fail=false + route1=169.254.42.42/32,62.210.0.1 + address1=192.168.1.20/16 + dns=8.8.8.8; + dns-search=lab;home; + + [ipv6] + route1=::/0,fe80::dc00:ff:fe20:186 + dns=FEDC::1; + dns-search=lab;home; + + """ + ) + }, + "yaml": textwrap.dedent( + """\ + version: 2 + ethernets: + eth0: + dhcp4: true + nameservers: + search: [lab, home] + addresses: [8.8.8.8, "FEDC::1"] + routes: + - to: 169.254.42.42/32 + via: 62.210.0.1 + - via: fe80::dc00:ff:fe20:186 + to: ::/0 + addresses: + - 192.168.1.20/16 + """ + ), + }, "v2-dns": { "expected_networkd": textwrap.dedent( """\ @@ -4695,11 +4914,6 @@ ), "expected_eni": textwrap.dedent( """\ - # This file is generated from information provided by the datasource. Changes - # to it will not persist across an instance reboot. To disable cloud-init's - # network configuration capabilities, write a file - # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: - # network: {config: disabled} auto lo iface lo inet loopback @@ -4800,6 +5014,18 @@ ), }, "v2-dns-no-if-ips": { + "expected_eni": textwrap.dedent( + """\ + auto lo + iface lo inet loopback + + auto eth0 + iface eth0 inet dhcp + + # control-alias eth0 + iface eth0 inet6 dhcp + """ # noqa: E501 + ), "expected_network_manager": { "cloud-init-eth0.nmconnection": textwrap.dedent( """\ @@ -4846,6 +5072,14 @@ ), }, "v2-dns-no-dhcp": { + "expected_eni": textwrap.dedent( + """\ + auto lo + iface lo inet loopback + + iface eth0 inet manual + """ # noqa: E501 + ), "expected_network_manager": { "cloud-init-eth0.nmconnection": textwrap.dedent( """\ diff --git a/tests/unittests/net/test_dhcp.py b/tests/unittests/net/test_dhcp.py index 704b8e85..70856642 100644 --- a/tests/unittests/net/test_dhcp.py +++ b/tests/unittests/net/test_dhcp.py @@ -7,6 +7,7 @@ from textwrap import dedent import pytest +import responses from cloudinit.distros import alpine, amazon, centos, debian, freebsd, rhel from cloudinit.distros.ubuntu import Distro @@ -27,7 +28,6 @@ from cloudinit.util import ensure_file, load_binary_file, subp, write_file from tests.unittests.helpers import ( CiTestCase, - ResponsesTestCase, example_netdev, mock, populate_dir, @@ -686,7 +686,8 @@ def test_dhcp_output_error_stream( my_pid = 1 write_file(pid_file, "%d\n" % my_pid) - def dhcp_log_func(out, err): + def dhcp_log_func(interface, out, err): + assert interface == "eth9" assert out == dhclient_out assert err == dhclient_err @@ -813,9 +814,10 @@ def test_multiple_files(self): ) +@responses.activate @pytest.mark.usefixtures("disable_netdev_info") @mock.patch("cloudinit.net.ephemeral._check_connectivity_to_imds") -class TestEphemeralDhcpNoNetworkSetup(ResponsesTestCase): +class TestEphemeralDhcpNoNetworkSetup(CiTestCase): @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") def test_ephemeral_dhcp_no_network_if_url_connectivity( self, m_dhcp, m_imds diff --git a/tests/unittests/net/test_init.py b/tests/unittests/net/test_init.py index 6226a024..af43a87c 100644 --- a/tests/unittests/net/test_init.py +++ b/tests/unittests/net/test_init.py @@ -4,6 +4,7 @@ import copy import errno import ipaddress +import logging import os from pathlib import Path from typing import Optional @@ -404,6 +405,7 @@ def create_fake_interface( bridge: bool = False, failover_standby: bool = False, operstate: Optional[str] = None, + renamed: bool = True, ): interface_path = self.sys_path / name interface_path.mkdir(parents=True) @@ -441,8 +443,14 @@ def create_fake_interface( (device_path / driver).write_text(driver) (device_path / "driver").symlink_to(driver) + if renamed: + (interface_path / "name_assign_type").write_text("4") + else: + (interface_path / "name_assign_type").write_text("0") + @pytest.fixture(autouse=True) def setup(self, monkeypatch, tmpdir): + """Fixture to set up the test environment.""" self.sys_path = Path(tmpdir) / "sys" monkeypatch.setattr( net, "get_sys_class_path", lambda: str(self.sys_path) + "/" @@ -452,7 +460,11 @@ def setup(self, monkeypatch, tmpdir): "is_container", lambda: False, ) - monkeypatch.setattr(net.util, "udevadm_settle", lambda: None) + + self.m_settle = mock.Mock(return_value=None) + self.m_cmdline = mock.Mock(return_value="") + monkeypatch.setattr("cloudinit.net.util.udevadm_settle", self.m_settle) + monkeypatch.setattr("cloudinit.net.util.get_cmdline", self.m_cmdline) def test_ignored_interfaces(self): self.create_fake_interface( @@ -562,6 +574,48 @@ def test_eth0_preferred_after_carrier(self, dormant, operstate): def test_no_nics(self): assert net.find_candidate_nics_on_linux() == [] + def test_udevadm_settle_called_for_predictable_names(self): + self.create_fake_interface(name="eth0", renamed=False) + + result = net.find_candidate_nics_on_linux() + + assert result == ["eth0"] + self.m_settle.assert_called_once() + + def test_udevadm_settle_not_called_for_unpredictable_names(self): + self.create_fake_interface(name="eth0", renamed=False) + self.m_cmdline.return_value = "net.ifnames=0 biosdevname=0" + + result = net.find_candidate_nics_on_linux() + + assert result == ["eth0"] + self.m_settle.assert_not_called() + + def test_udevadm_settle_not_called_for_renamed_interface(self): + self.create_fake_interface(name="eth0", renamed=True) + + result = net.find_candidate_nics_on_linux() + + assert result == ["eth0"] + self.m_settle.assert_not_called() + + def test_udevadm_settle_failure_handled_gracefully(self, caplog): + self.create_fake_interface(name="eth0", renamed=False) + + self.m_settle.side_effect = subp.ProcessExecutionError( + cmd="xcmd", stderr="xstderr", stdout="xstdout", exit_code=1 + ) + + with caplog.at_level(logging.WARNING): + result = net.find_candidate_nics_on_linux() + assert ( + "udevadm failed to settle: cmd='xcmd' " + "stderr='xstderr' stdout='xstdout' exit_code=1" in caplog.text + ) + + assert result == ["eth0"] + self.m_settle.assert_called_once() + class TestGetDeviceList(CiTestCase): def setUp(self): diff --git a/tests/unittests/net/test_net_rendering.py b/tests/unittests/net/test_net_rendering.py index 0f3c766f..72fe3b1b 100644 --- a/tests/unittests/net/test_net_rendering.py +++ b/tests/unittests/net/test_net_rendering.py @@ -35,6 +35,8 @@ from cloudinit.net.netplan import Renderer as NetplanRenderer from cloudinit.net.network_manager import Renderer as NetworkManagerRenderer from cloudinit.net.network_state import NetworkState, parse_net_config_data +from cloudinit.net.networkd import Renderer as NetworkdRenderer +from tests.unittests.helpers import mock ARTIFACT_DIR = Path(__file__).parent.absolute() / "artifacts" @@ -50,6 +52,19 @@ def setup(mocker): mocker.patch("cloudinit.net.network_state.get_interfaces_by_mac") +def _check_file_diff(expected_paths: list, tmp_path: Path): + for expected_path in expected_paths: + expected_contents = Path(expected_path).read_text() + actual_path = tmp_path / expected_path.split( + str(ARTIFACT_DIR), maxsplit=1 + )[1].lstrip("/") + assert ( + actual_path.exists() + ), f"Expected {actual_path} to exist, but it does not" + actual_contents = actual_path.read_text() + assert expected_contents.strip() == actual_contents.strip() + + def _check_netplan( network_state: NetworkState, netplan_path: Path, expected_config ): @@ -73,21 +88,30 @@ def _check_network_manager(network_state: NetworkState, tmp_path: Path): str(ARTIFACT_DIR / "no_matching_mac" / "**/*.nmconnection"), recursive=True, ) - for expected_path in expected_paths: - expected_contents = Path(expected_path).read_text() - actual_path = tmp_path / expected_path.split( - str(ARTIFACT_DIR), maxsplit=1 - )[1].lstrip("/") - assert ( - actual_path.exists() - ), f"Expected {actual_path} to exist, but it does not" - actual_contents = actual_path.read_text() - assert expected_contents.strip() == actual_contents.strip() + _check_file_diff(expected_paths, tmp_path) + + +@mock.patch("cloudinit.net.util.chownbyname", return_value=True) +def _check_networkd_renderer( + network_state: NetworkState, tmp_path: Path, m_chown +): + renderer = NetworkdRenderer() + renderer.render_network_state( + network_state, target=str(tmp_path / "photon_net_config") + ) + expected_paths = glob.glob( + str(ARTIFACT_DIR / "photon_net_config" / "**/*.net*"), + recursive=True, + ) + _check_file_diff(expected_paths, tmp_path) @pytest.mark.parametrize( "test_name, renderers", - [("no_matching_mac_v2", Renderer.Netplan | Renderer.NetworkManager)], + [ + ("no_matching_mac_v2", Renderer.Netplan | Renderer.NetworkManager), + ("photon_net_config_v2", Renderer.Networkd), + ], ) def test_convert(test_name, renderers, tmp_path): network_config = yaml.safe_load( @@ -100,3 +124,5 @@ def test_convert(test_name, renderers, tmp_path): ) if Renderer.NetworkManager in renderers: _check_network_manager(network_state, tmp_path) + if Renderer.Networkd in renderers: + _check_networkd_renderer(network_state, tmp_path) diff --git a/tests/unittests/net/test_netplan.py b/tests/unittests/net/test_netplan.py index 86bb32b1..26a4e48b 100644 --- a/tests/unittests/net/test_netplan.py +++ b/tests/unittests/net/test_netplan.py @@ -51,3 +51,55 @@ def test_no_netplan_python_api(self, caplog): "No netplan python module. Fallback to write" f" {netplan.CLOUDINIT_NETPLAN_FILE}" in caplog.text ) + + +SIMPLE_V2_CFG_MTU = """\ +network: + version: 2 + ethernets: + eno1: + match: + macaddress: 08:94:ef:51:ae:e0 + mtu: 0 + eno2: + match: + macaddress: 08:94:ef:51:ae:e1 + mtu: 100 +""" + + +REDACTED_V2_CFG_MTU = """\ +network: + version: 2 + ethernets: + eno1: + match: + macaddress: 08:94:ef:51:ae:e0 + eno2: + match: + macaddress: 08:94:ef:51:ae:e1 + mtu: 100 +""" + + +class TestMaybeStripInvalidMTU: + @pytest.mark.parametrize( + "netcfg,expected,strip_enabled", + ( + pytest.param( + SIMPLE_V2_CFG_MTU, + SIMPLE_V2_CFG_MTU, + False, + id="valid_mtu_unchanged", + ), + pytest.param( + SIMPLE_V2_CFG_MTU, + REDACTED_V2_CFG_MTU, + True, + id="invalid_mtu_redatcted", + ), + ), + ) + def test__strip_invalid_mtu(self, netcfg, expected, strip_enabled, mocker): + mocker.patch("cloudinit.features.STRIP_INVALID_MTU", strip_enabled) + assert expected == netplan._maybe_strip_invalid_mtu(netcfg) diff --git a/tests/unittests/runs/test_merge_run.py b/tests/unittests/runs/test_merge_run.py index 7cd43c63..876f8c69 100644 --- a/tests/unittests/runs/test_merge_run.py +++ b/tests/unittests/runs/test_merge_run.py @@ -1,57 +1,59 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -import shutil -import tempfile + +import pytest from cloudinit import safeyaml, stages, util from cloudinit.config.modules import Modules from cloudinit.settings import PER_INSTANCE from tests.unittests import helpers +from tests.unittests.helpers import replicate_test_root + +@pytest.mark.usefixtures("fake_filesystem_hook") +@pytest.fixture(autouse=True) +def user_data(tmp_path): + replicate_test_root("simple_ubuntu", str(tmp_path)) + return helpers.readResource("user_data.1.txt") -class TestMergeRun(helpers.FilesystemMockingTestCase): - def _patchIn(self, root): - self.patchOS(root) - self.patchUtils(root) - def test_none_ds(self): - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self.replicateTestRoot("simple_ubuntu", new_root) - cfg = { - "datasource_list": ["None"], - "cloud_init_modules": ["write_files"], - "system_info": { - "paths": {"run_dir": new_root}, - "package_mirrors": [ - { - "arches": ["i386", "amd64", "blah"], - "failsafe": { - "primary": "http://my.archive.mydomain.com/ubuntu", - "security": ( - "http://my.security.mydomain.com/ubuntu" - ), - }, - "search": {"primary": [], "security": []}, +@pytest.fixture(autouse=True) +def cfg(tmp_path, mocker): + mocker.patch("cloudinit.util.os.chown") + new_root = str(tmp_path) + cfg = { + "datasource_list": ["None"], + "cloud_init_modules": ["write_files"], + "system_info": { + "paths": {"run_dir": new_root}, + "package_mirrors": [ + { + "arches": ["i386", "amd64", "blah"], + "failsafe": { + "primary": "http://my.archive.mydomain.com/ubuntu", + "security": ("http://my.security.mydomain.com/ubuntu"), }, - ], - }, - } - ud = helpers.readResource("user_data.1.txt") - cloud_cfg = safeyaml.dumps(cfg) - util.ensure_dir(os.path.join(new_root, "etc", "cloud")) - util.write_file( - os.path.join(new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg - ) - self._patchIn(new_root) + "search": {"primary": [], "security": []}, + }, + ], + }, + } + cloud_cfg = safeyaml.dumps(cfg) + util.ensure_dir(os.path.join(new_root, "etc", "cloud")) + util.write_file( + os.path.join(new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg + ) + - # Now start verifying whats created +@pytest.mark.usefixtures("fake_filesystem") +class TestMergeRun: + def test_none_ds(self, user_data): initer = stages.Init() initer.read_cfg() initer.initialize() initer.fetch() - initer.datasource.userdata_raw = ud + initer.datasource.userdata_raw = user_data initer.instancify() initer.update() initer.cloudify().run( @@ -61,13 +63,13 @@ def test_none_ds(self): freq=PER_INSTANCE, ) mirrors = initer.distro.get_option("package_mirrors") - self.assertEqual(1, len(mirrors)) + assert 1 == len(mirrors) mirror = mirrors[0] - self.assertEqual(mirror["arches"], ["i386", "amd64", "blah"]) + assert mirror["arches"] == ["i386", "amd64", "blah"] mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertFalse(failures) - self.assertTrue(os.path.exists("/etc/blah.ini")) - self.assertIn("write_files", which_ran) + assert not failures + assert os.path.exists("/etc/blah.ini") + assert "write_files" in which_ran contents = util.load_text_file("/etc/blah.ini") - self.assertEqual(contents, "blah") + assert contents == "blah" diff --git a/tests/unittests/runs/test_simple_run.py b/tests/unittests/runs/test_simple_run.py index e8737ea2..93493e72 100644 --- a/tests/unittests/runs/test_simple_run.py +++ b/tests/unittests/runs/test_simple_run.py @@ -3,48 +3,52 @@ import copy import os +import pytest + from cloudinit import atomic_helper, safeyaml, stages, util from cloudinit.config.modules import Modules from cloudinit.settings import PER_INSTANCE from cloudinit.sources import NetworkConfigSource -from tests.unittests import helpers - - -class TestSimpleRun(helpers.FilesystemMockingTestCase): - - with_logs = True - - def setUp(self): - super(TestSimpleRun, self).setUp() - self.new_root = self.tmp_dir() - self.replicateTestRoot("simple_ubuntu", self.new_root) - - # Seed cloud.cfg file for our tests - self.cfg = { - "datasource_list": ["None"], - "runcmd": ["ls /etc"], # test ALL_DISTROS - "spacewalk": {}, # test non-ubuntu distros module definition - "system_info": { - "paths": {"run_dir": self.new_root}, - "distro": "ubuntu", +from tests.unittests.helpers import replicate_test_root + + +@pytest.mark.usefixtures("fake_filesystem_hook") +@pytest.fixture(autouse=True) +def replicate_root(tmp_path): + replicate_test_root("simple_ubuntu", str(tmp_path)) + + +@pytest.fixture(autouse=True) +def cfg(mocker, tmp_path): + new_root = str(tmp_path) + _cfg = { + "datasource_list": ["None"], + "runcmd": ["ls /etc"], # test ALL_DISTROS + "spacewalk": {}, # test non-ubuntu distros module definition + "system_info": { + "paths": {"run_dir": new_root}, + "distro": "ubuntu", + }, + "write_files": [ + { + "path": "/etc/blah.ini", + "content": "blah", + "permissions": 0o755, }, - "write_files": [ - { - "path": "/etc/blah.ini", - "content": "blah", - "permissions": 0o755, - }, - ], - "cloud_init_modules": ["write_files", "spacewalk", "runcmd"], - } - cloud_cfg = safeyaml.dumps(self.cfg) - util.ensure_dir(os.path.join(self.new_root, "etc", "cloud")) - util.write_file( - os.path.join(self.new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg - ) - self.patchOS(self.new_root) - self.patchUtils(self.new_root) - + ], + "cloud_init_modules": ["write_files", "spacewalk", "runcmd"], + } + cloud_cfg = safeyaml.dumps(_cfg) + util.ensure_dir(os.path.join(new_root, "etc", "cloud")) + util.write_file( + os.path.join(new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg + ) + mocker.patch("cloudinit.util.os.chown") + return _cfg + + +@pytest.mark.usefixtures("fake_filesystem") +class TestSimpleRun: def test_none_ds_populates_var_lib_cloud(self): """Init and run_section default behavior creates appropriate dirs.""" # Now start verifying whats created @@ -56,31 +60,30 @@ def test_none_ds_populates_var_lib_cloud(self): def fake_network_config(): return netcfg, NetworkConfigSource.FALLBACK - self.assertFalse(os.path.exists("/var/lib/cloud")) + assert not os.path.exists("/var/lib/cloud") initer = stages.Init() initer.read_cfg() initer.initialize() - self.assertTrue(os.path.exists("/var/lib/cloud")) + assert os.path.exists("/var/lib/cloud") for d in ["scripts", "seed", "instances", "handlers", "sem", "data"]: - self.assertTrue(os.path.isdir(os.path.join("/var/lib/cloud", d))) + assert os.path.isdir(os.path.join("/var/lib/cloud", d)) initer.fetch() - self.assertFalse(os.path.islink("var/lib/cloud/instance")) + assert not os.path.islink("var/lib/cloud/instance") iid = initer.instancify() - self.assertEqual(iid, "iid-datasource-none") + assert iid == "iid-datasource-none" initer.update() - self.assertTrue(os.path.islink("var/lib/cloud/instance")) + assert os.path.islink("var/lib/cloud/instance") initer._find_networking_config = fake_network_config - self.assertFalse( - os.path.exists("/var/lib/cloud/instance/network-config.json") + assert not os.path.exists( + "/var/lib/cloud/instance/network-config.json" ) initer.apply_network_config(False) - self.assertEqual( - f"{atomic_helper.json_dumps(netcfg)}\n", - util.load_text_file("/var/lib/cloud/instance/network-config.json"), + assert f"{atomic_helper.json_dumps(netcfg)}\n" == util.load_text_file( + "/var/lib/cloud/instance/network-config.json" ) - def test_none_ds_runs_modules_which_do_not_define_distros(self): + def test_none_ds_runs_modules_which_do_not_define_distros(self, caplog): """Any modules which do not define a distros attribute are run.""" initer = stages.Init() initer.read_cfg() @@ -97,18 +100,19 @@ def test_none_ds_runs_modules_which_do_not_define_distros(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertFalse(failures) - self.assertTrue(os.path.exists("/etc/blah.ini")) - self.assertIn("write_files", which_ran) + assert not failures + assert os.path.exists("/etc/blah.ini") + assert "write_files" in which_ran contents = util.load_text_file("/etc/blah.ini") - self.assertEqual(contents, "blah") - self.assertNotIn( + assert contents == "blah" + assert ( "Skipping modules ['write_files'] because they are not verified on" - " distro 'ubuntu'", - self.logs.getvalue(), + " distro 'ubuntu'" not in caplog.text ) - def test_none_ds_skips_modules_which_define_unmatched_distros(self): + def test_none_ds_skips_modules_which_define_unmatched_distros( + self, caplog + ): """Skip modules which define distros which don't match the current.""" initer = stages.Init() initer.read_cfg() @@ -125,15 +129,14 @@ def test_none_ds_skips_modules_which_define_unmatched_distros(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertFalse(failures) - self.assertIn( + assert not failures + assert ( "Skipping modules 'spacewalk' because they are not verified on" - " distro 'ubuntu'", - self.logs.getvalue(), + " distro 'ubuntu'" in caplog.text ) - self.assertNotIn("spacewalk", which_ran) + assert "spacewalk" not in which_ran - def test_none_ds_runs_modules_which_distros_all(self): + def test_none_ds_runs_modules_which_distros_all(self, caplog): """Skip modules which define distros attribute as supporting 'all'. This is done in the module with the declaration: @@ -154,25 +157,22 @@ def test_none_ds_runs_modules_which_distros_all(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertFalse(failures) - self.assertIn("runcmd", which_ran) - self.assertNotIn( + assert not failures + assert "runcmd" in which_ran + assert ( "Skipping modules 'runcmd' because they are not verified on" - " distro 'ubuntu'", - self.logs.getvalue(), + " distro 'ubuntu'" not in caplog.text ) - def test_none_ds_forces_run_via_unverified_modules(self): + def test_none_ds_forces_run_via_unverified_modules(self, caplog, cfg): """run_section forced skipped modules by using unverified_modules.""" # re-write cloud.cfg with unverified_modules override - cfg = copy.deepcopy(self.cfg) + cfg = copy.deepcopy(cfg) cfg["unverified_modules"] = ["spacewalk"] # Would have skipped cloud_cfg = safeyaml.dumps(cfg) - util.ensure_dir(os.path.join(self.new_root, "etc", "cloud")) - util.write_file( - os.path.join(self.new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg - ) + util.ensure_dir(os.path.join("/etc", "cloud")) + util.write_file(os.path.join("/etc", "cloud", "cloud.cfg"), cloud_cfg) initer = stages.Init() initer.read_cfg() @@ -189,24 +189,20 @@ def test_none_ds_forces_run_via_unverified_modules(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertFalse(failures) - self.assertIn("spacewalk", which_ran) - self.assertIn( - "running unverified_modules: 'spacewalk'", self.logs.getvalue() - ) + assert not failures + assert "spacewalk" in which_ran + assert "running unverified_modules: 'spacewalk'" in caplog.text - def test_none_ds_run_with_no_config_modules(self): + def test_none_ds_run_with_no_config_modules(self, cfg): """run_section will report no modules run when none are configured.""" # re-write cloud.cfg with unverified_modules override - cfg = copy.deepcopy(self.cfg) + cfg = copy.deepcopy(cfg) # Represent empty configuration in /etc/cloud/cloud.cfg cfg["cloud_init_modules"] = None cloud_cfg = safeyaml.dumps(cfg) - util.ensure_dir(os.path.join(self.new_root, "etc", "cloud")) - util.write_file( - os.path.join(self.new_root, "etc", "cloud", "cloud.cfg"), cloud_cfg - ) + util.ensure_dir(os.path.join("/etc", "cloud")) + util.write_file(os.path.join("/etc", "cloud", "cloud.cfg"), cloud_cfg) initer = stages.Init() initer.read_cfg() @@ -223,5 +219,5 @@ def test_none_ds_run_with_no_config_modules(self): mods = Modules(initer) (which_ran, failures) = mods.run_section("cloud_init_modules") - self.assertFalse(failures) - self.assertEqual([], which_ran) + assert not failures + assert [] == which_ran diff --git a/tests/unittests/sources/azure/test_errors.py b/tests/unittests/sources/azure/test_errors.py index 43c0da61..5bbfe250 100644 --- a/tests/unittests/sources/azure/test_errors.py +++ b/tests/unittests/sources/azure/test_errors.py @@ -120,7 +120,7 @@ def test_reportable_errors( "documentation_url=https://aka.ms/linuxprovisioningerror", ] - assert error.as_encoded_report() == "|".join(data) + assert error.as_encoded_report(vm_id=fake_vm_id) == "|".join(data) def test_dhcp_lease(mocker): @@ -239,19 +239,40 @@ def test_unhandled_exception(): assert isinstance(traceback_base64, str) trace = base64.b64decode(traceback_base64).decode("utf-8") - assert trace.startswith("Traceback") + assert trace.startswith("\nValueError: my value error\n") assert "raise ValueError" in trace - assert trace.endswith("ValueError: my value error\n") + assert trace.endswith("Traceback (most recent call last):") quoted_value = quote_csv_value(f"exception={source_error!r}") - assert f"|{quoted_value}|" in error.as_encoded_report() + assert f"|{quoted_value}|" in error.as_encoded_report(vm_id="test-vm-id") -def test_imds_invalid_metadata(): +@pytest.mark.parametrize( + "value", + [ + "Running", + "None", + None, + ], +) +def test_imds_invalid_metadata(value): key = "compute" - value = "Running" error = errors.ReportableErrorImdsInvalidMetadata(key=key, value=value) assert error.reason == "invalid IMDS metadata for key=compute" assert error.supporting_data["key"] == key - assert error.supporting_data["value"] == repr(value) + assert error.supporting_data["value"] == value + assert error.supporting_data["type"] == type(value).__name__ + + +def test_vm_identification_exception(): + exception = ValueError("foobar") + system_uuid = "1234-5678-90ab-cdef" + + error = errors.ReportableErrorVmIdentification( + exception=exception, system_uuid=system_uuid + ) + + assert error.reason == "failure to identify Azure VM ID" + assert error.supporting_data["exception"] == repr(exception) + assert error.supporting_data["system_uuid"] == system_uuid diff --git a/tests/unittests/sources/azure/test_kvp.py b/tests/unittests/sources/azure/test_kvp.py index 0404dcfc..666ad88c 100644 --- a/tests/unittests/sources/azure/test_kvp.py +++ b/tests/unittests/sources/azure/test_kvp.py @@ -40,29 +40,23 @@ def telemetry_reporter(tmp_path): class TestReportFailureToHost: - def test_report_failure_to_host(self, caplog, telemetry_reporter, mocker): - mocker.patch( - "cloudinit.sources.azure.identity.query_vm_id", return_value="foo" - ) + def test_report_via_kvp(self, caplog, telemetry_reporter): error = errors.ReportableError(reason="test") - assert kvp.report_failure_to_host(error) is True + encoded_report = error.as_encoded_report(vm_id="fake-vm-id") + + assert kvp.report_via_kvp(encoded_report) is True assert ( "KVP handler not enabled, skipping host report." not in caplog.text ) report = { "key": "PROVISIONING_REPORT", - "value": error.as_encoded_report(), + "value": encoded_report, } assert report in list(telemetry_reporter._iterate_kvps(0)) - def test_report_skipped_without_telemetry(self, caplog, mocker): - mocker.patch( - "cloudinit.sources.azure.identity.query_vm_id", return_value="foo" - ) - error = errors.ReportableError(reason="test") - - assert kvp.report_failure_to_host(error) is False + def test_report_skipped_without_telemetry(self, caplog): + assert kvp.report_via_kvp("test report") is False assert "KVP handler not enabled, skipping host report." in caplog.text @@ -70,7 +64,7 @@ class TestReportSuccessToHost: def test_report_success_to_host( self, caplog, fake_utcnow, fake_vm_id, telemetry_reporter ): - assert kvp.report_success_to_host() is True + assert kvp.report_success_to_host(vm_id=fake_vm_id) is True assert ( "KVP handler not enabled, skipping host report." not in caplog.text ) @@ -94,5 +88,5 @@ def test_report_skipped_without_telemetry(self, caplog, mocker): mocker.patch( "cloudinit.sources.azure.identity.query_vm_id", return_value="foo" ) - assert kvp.report_success_to_host() is False + assert kvp.report_success_to_host(vm_id="fake") is False assert "KVP handler not enabled, skipping host report." in caplog.text diff --git a/tests/unittests/sources/conftest.py b/tests/unittests/sources/conftest.py index ef4d7e61..5a58f4a7 100644 --- a/tests/unittests/sources/conftest.py +++ b/tests/unittests/sources/conftest.py @@ -1,3 +1,4 @@ +import os from unittest import mock import pytest @@ -7,3 +8,16 @@ def mock_util_get_cmdline(): with mock.patch("cloudinit.util.get_cmdline", return_value="") as m: yield m + + +@pytest.fixture(autouse=True) +def hide_resource_disk(): + """GitHub runner may have a resource disk which may affect tests.""" + real_exists = os.path.exists + with mock.patch("os.path.exists") as mock_exists: + mock_exists.side_effect = lambda path: ( + False + if path == "/dev/disk/cloud/azure_resource" + else real_exists(path) + ) + yield diff --git a/tests/unittests/sources/helpers/test_ec2.py b/tests/unittests/sources/helpers/test_ec2.py index aa250893..2f4cdf06 100644 --- a/tests/unittests/sources/helpers/test_ec2.py +++ b/tests/unittests/sources/helpers/test_ec2.py @@ -4,163 +4,170 @@ from cloudinit import url_helper as uh from cloudinit.sources.helpers import ec2 -from tests.unittests import helpers -class TestEc2Util(helpers.ResponsesTestCase): +class TestEc2Util: VERSION = "latest" + @responses.activate def test_userdata_fetch(self): - self.responses.add( + responses.add( responses.GET, "http://169.254.169.254/%s/user-data" % (self.VERSION), body="stuff", status=200, ) userdata = ec2.get_instance_userdata(self.VERSION) - self.assertEqual("stuff", userdata.decode("utf-8")) + assert "stuff" == userdata.decode("utf-8") + @responses.activate def test_userdata_fetch_fail_not_found(self): - self.responses.add( + responses.add( responses.GET, "http://169.254.169.254/%s/user-data" % (self.VERSION), status=404, ) userdata = ec2.get_instance_userdata(self.VERSION, retries=0) - self.assertEqual(b"", userdata) + assert b"" == userdata + @responses.activate def test_userdata_fetch_fail_server_dead(self): - self.responses.add( + responses.add( responses.GET, "http://169.254.169.254/%s/user-data" % (self.VERSION), status=500, ) userdata = ec2.get_instance_userdata(self.VERSION, retries=0) - self.assertEqual(b"", userdata) + assert b"" == userdata + @responses.activate def test_userdata_fetch_fail_server_not_found(self): - self.responses.add( + responses.add( responses.GET, "http://169.254.169.254/%s/user-data" % (self.VERSION), status=404, ) userdata = ec2.get_instance_userdata(self.VERSION) - self.assertEqual(b"", userdata) + assert b"" == userdata + @responses.activate def test_metadata_fetch_no_keys(self): base_url = "http://169.254.169.254/%s/meta-data/" % (self.VERSION) - self.responses.add( + responses.add( responses.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "ami-launch-index"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "ami-launch-index"), status=200, body="1", ) md = ec2.get_instance_metadata(self.VERSION, retries=0) - self.assertEqual(md["hostname"], "ec2.fake.host.name.com") - self.assertEqual(md["instance-id"], "123") - self.assertEqual(md["ami-launch-index"], "1") + assert md["hostname"] == "ec2.fake.host.name.com" + assert md["instance-id"] == "123" + assert md["ami-launch-index"] == "1" + @responses.activate def test_metadata_fetch_key(self): base_url = "http://169.254.169.254/%s/meta-data/" % (self.VERSION) - self.responses.add( + responses.add( responses.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "public-keys/"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "public-keys/"), status=200, body="0=my-public-key", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "public-keys/0/openssh-key"), status=200, body="ssh-rsa AAAA.....wZEf my-public-key", ) md = ec2.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEqual(md["hostname"], "ec2.fake.host.name.com") - self.assertEqual(md["instance-id"], "123") - self.assertEqual(1, len(md["public-keys"])) + assert md["hostname"] == "ec2.fake.host.name.com" + assert md["instance-id"] == "123" + assert 1 == len(md["public-keys"]) + @responses.activate def test_metadata_fetch_with_2_keys(self): base_url = "http://169.254.169.254/%s/meta-data/" % (self.VERSION) - self.responses.add( + responses.add( responses.GET, base_url, status=200, body="\n".join(["hostname", "instance-id", "public-keys/"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "public-keys/"), status=200, body="\n".join(["0=my-public-key", "1=my-other-key"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "public-keys/0/openssh-key"), status=200, body="ssh-rsa AAAA.....wZEf my-public-key", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "public-keys/1/openssh-key"), status=200, body="ssh-rsa AAAA.....wZEf my-other-key", ) md = ec2.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEqual(md["hostname"], "ec2.fake.host.name.com") - self.assertEqual(md["instance-id"], "123") - self.assertEqual(2, len(md["public-keys"])) + assert md["hostname"] == "ec2.fake.host.name.com" + assert md["instance-id"] == "123" + assert 2 == len(md["public-keys"]) + @responses.activate def test_metadata_fetch_bdm(self): base_url = "http://169.254.169.254/%s/meta-data/" % (self.VERSION) - self.responses.add( + responses.add( responses.GET, base_url, status=200, @@ -168,89 +175,90 @@ def test_metadata_fetch_bdm(self): ["hostname", "instance-id", "block-device-mapping/"] ), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "hostname"), status=200, body="ec2.fake.host.name.com", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "instance-id"), status=200, body="123", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "block-device-mapping/"), status=200, body="\n".join(["ami", "ephemeral0"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "block-device-mapping/ami"), status=200, body="sdb", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "block-device-mapping/ephemeral0"), status=200, body="sdc", ) md = ec2.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEqual(md["hostname"], "ec2.fake.host.name.com") - self.assertEqual(md["instance-id"], "123") + assert md["hostname"] == "ec2.fake.host.name.com" + assert md["instance-id"] == "123" bdm = md["block-device-mapping"] - self.assertEqual(2, len(bdm)) - self.assertEqual(bdm["ami"], "sdb") - self.assertEqual(bdm["ephemeral0"], "sdc") + assert 2 == len(bdm) + assert bdm["ami"] == "sdb" + assert bdm["ephemeral0"] == "sdc" + @responses.activate def test_metadata_no_security_credentials(self): base_url = "http://169.254.169.254/%s/meta-data/" % (self.VERSION) - self.responses.add( + responses.add( responses.GET, base_url, status=200, body="\n".join(["instance-id", "iam/"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "instance-id"), status=200, body="i-0123451689abcdef0", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "iam/"), status=200, body="\n".join(["info/", "security-credentials/"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "iam/info/"), status=200, body="LastUpdated", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "iam/info/LastUpdated"), status=200, body="2016-10-27T17:29:39Z", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "iam/security-credentials/"), status=200, body="ReadOnly/", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "iam/security-credentials/ReadOnly/"), status=200, body="\n".join(["LastUpdated", "Expiration"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url( base_url, "iam/security-credentials/ReadOnly/LastUpdated" @@ -258,7 +266,7 @@ def test_metadata_no_security_credentials(self): status=200, body="2016-10-27T17:28:17Z", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url( base_url, "iam/security-credentials/ReadOnly/Expiration" @@ -267,12 +275,13 @@ def test_metadata_no_security_credentials(self): body="2016-10-28T00:00:34Z", ) md = ec2.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEqual(md["instance-id"], "i-0123451689abcdef0") + assert md["instance-id"] == "i-0123451689abcdef0" iam = md["iam"] - self.assertEqual(1, len(iam)) - self.assertEqual(iam["info"]["LastUpdated"], "2016-10-27T17:29:39Z") - self.assertNotIn("security-credentials", iam) + assert 1 == len(iam) + assert iam["info"]["LastUpdated"] == "2016-10-27T17:29:39Z" + assert "security-credentials" not in iam + @responses.activate def test_metadata_children_with_invalid_character(self): def _skip_tags(exception): if isinstance(exception, uh.UrlError) and exception.code == 404: @@ -282,30 +291,30 @@ def _skip_tags(exception): return False base_url = "http://169.254.169.254/%s/meta-data/" % (self.VERSION) - self.responses.add( + responses.add( responses.GET, base_url, status=200, body="\n".join(["tags/", "ami-launch-index"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "tags/"), status=200, body="\n".join(["test/invalid", "valid"]), ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "tags/valid"), status=200, body="OK", ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "tags/test/invalid"), status=404, ) - self.responses.add( + responses.add( responses.GET, uh.combine_url(base_url, "ami-launch-index"), status=200, @@ -317,8 +326,8 @@ def _skip_tags(exception): timeout=0.1, retrieval_exception_ignore_cb=_skip_tags, ) - self.assertEqual(md["tags"]["valid"], "OK") - self.assertEqual(md["tags"]["test/invalid"], "(skipped)") - self.assertEqual(md["ami-launch-index"], "1") + assert md["tags"]["valid"] == "OK" + assert md["tags"]["test/invalid"] == "(skipped)" + assert md["ami-launch-index"] == "1" md = ec2.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEqual(len(md), 0) + assert len(md) == 0 diff --git a/tests/unittests/sources/helpers/test_netlink.py b/tests/unittests/sources/helpers/test_netlink.py index b68c8006..010b8d31 100644 --- a/tests/unittests/sources/helpers/test_netlink.py +++ b/tests/unittests/sources/helpers/test_netlink.py @@ -6,6 +6,8 @@ import socket import struct +import pytest + from cloudinit.sources.helpers.netlink import ( MAX_SIZE, OPER_DORMANT, @@ -29,7 +31,7 @@ wait_for_nic_attach_event, wait_for_nic_detach_event, ) -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import mock def int_to_bytes(i): @@ -39,22 +41,22 @@ def int_to_bytes(i): return codecs.decode(hex_value, "hex_codec") -class TestCreateBoundNetlinkSocket(CiTestCase): +class TestCreateBoundNetlinkSocket: @mock.patch("cloudinit.sources.helpers.netlink.socket.socket") def test_socket_error_on_create(self, m_socket): """create_bound_netlink_socket catches socket creation exception""" # NetlinkCreateSocketError is raised when socket creation errors. m_socket.side_effect = socket.error("Fake socket failure") - with self.assertRaises(NetlinkCreateSocketError) as ctx_mgr: + with pytest.raises( + NetlinkCreateSocketError, + match="Exception during netlink socket create: Fake socket" + " failure", + ): create_bound_netlink_socket() - self.assertEqual( - "Exception during netlink socket create: Fake socket failure", - str(ctx_mgr.exception), - ) -class TestReadNetlinkSocket(CiTestCase): +class TestReadNetlinkSocket: @mock.patch("cloudinit.sources.helpers.netlink.socket.socket") @mock.patch("cloudinit.sources.helpers.netlink.select.select") def test_read_netlink_socket(self, m_select, m_socket): @@ -65,8 +67,8 @@ def test_read_netlink_socket(self, m_select, m_socket): recv_data = read_netlink_socket(m_socket, 2) m_select.assert_called_with([m_socket], [], [], 2) m_socket.recv.assert_called_with(MAX_SIZE) - self.assertIsNotNone(recv_data) - self.assertEqual(recv_data, data) + assert recv_data is not None + assert recv_data == data @mock.patch("cloudinit.sources.helpers.netlink.socket.socket") @mock.patch("cloudinit.sources.helpers.netlink.select.select") @@ -75,18 +77,17 @@ def test_netlink_read_timeout(self, m_select, m_socket): m_select.return_value = [], None, None data = read_netlink_socket(m_socket, 1) m_select.assert_called_with([m_socket], [], [], 1) - self.assertEqual(m_socket.recv.call_count, 0) - self.assertIsNone(data) + assert m_socket.recv.call_count == 0 + assert data is None def test_read_invalid_socket(self): """read_netlink_socket raises assert error if socket is invalid""" socket = None - with self.assertRaises(AssertionError) as context: + with pytest.raises(AssertionError, match="netlink socket is none"): read_netlink_socket(socket, 1) - self.assertTrue("netlink socket is none" in str(context.exception)) -class TestParseNetlinkMessage(CiTestCase): +class TestParseNetlinkMessage: def test_read_rta_oper_state(self): """read_rta_oper_state could parse netlink message and extract data""" ifname = "eth0" @@ -104,15 +105,14 @@ def test_read_rta_oper_state(self): int_to_bytes(OPER_DOWN), ) interface_state = read_rta_oper_state(buf) - self.assertEqual(interface_state.ifname, ifname) - self.assertEqual(interface_state.operstate, OPER_DOWN) + assert interface_state.ifname == ifname + assert interface_state.operstate == OPER_DOWN def test_read_none_data(self): """read_rta_oper_state raises assert error if data is none""" data = None - with self.assertRaises(AssertionError) as context: + with pytest.raises(AssertionError, match="data is none"): read_rta_oper_state(data) - self.assertEqual("data is none", str(context.exception)) def test_read_invalid_rta_operstate_none(self): """read_rta_oper_state returns none if operstate is none""" @@ -121,7 +121,7 @@ def test_read_invalid_rta_operstate_none(self): bytes = ifname.encode("utf-8") struct.pack_into("HH4s", buf, RTATTR_START_OFFSET, 8, 3, bytes) interface_state = read_rta_oper_state(buf) - self.assertIsNone(interface_state) + assert interface_state is None def test_read_invalid_rta_ifname_none(self): """read_rta_oper_state returns none if ifname is none""" @@ -130,41 +130,37 @@ def test_read_invalid_rta_ifname_none(self): "HHc", buf, RTATTR_START_OFFSET, 5, 16, int_to_bytes(OPER_DOWN) ) interface_state = read_rta_oper_state(buf) - self.assertIsNone(interface_state) + assert interface_state is None def test_read_invalid_data_len(self): """raise assert error if data size is smaller than required size""" buf = bytearray(32) - with self.assertRaises(AssertionError) as context: + with pytest.raises( + AssertionError, + match="length of data is smaller than RTATTR_START_OFFSET", + ): read_rta_oper_state(buf) - self.assertTrue( - "length of data is smaller than RTATTR_START_OFFSET" - in str(context.exception) - ) def test_unpack_rta_attr_none_data(self): """unpack_rta_attr raises assert error if data is none""" data = None - with self.assertRaises(AssertionError) as context: + with pytest.raises(AssertionError, match="data is none"): unpack_rta_attr(data, RTATTR_START_OFFSET) - self.assertTrue("data is none" in str(context.exception)) def test_unpack_rta_attr_invalid_offset(self): """unpack_rta_attr raises assert error if offset is invalid""" data = bytearray(48) - with self.assertRaises(AssertionError) as context: + with pytest.raises(AssertionError, match="offset is not integer"): unpack_rta_attr(data, "offset") - self.assertTrue("offset is not integer" in str(context.exception)) - with self.assertRaises(AssertionError) as context: + with pytest.raises( + AssertionError, match="rta offset is less than expected length" + ): unpack_rta_attr(data, 31) - self.assertTrue( - "rta offset is less than expected length" in str(context.exception) - ) @mock.patch("cloudinit.sources.helpers.netlink.socket.socket") @mock.patch("cloudinit.sources.helpers.netlink.read_netlink_socket") -class TestNicAttachDetach(CiTestCase): +class TestNicAttachDetach: with_logs = True def _media_switch_data(self, ifname, msg_type, operstate): @@ -206,8 +202,8 @@ def test_nic_attached_oper_down(self, m_read_netlink_socket, m_socket): data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) m_read_netlink_socket.side_effect = [data_op_down] ifread = wait_for_nic_attach_event(m_socket, []) - self.assertEqual(m_read_netlink_socket.call_count, 1) - self.assertEqual(ifname, ifread) + assert m_read_netlink_socket.call_count == 1 + assert ifname == ifread def test_nic_attached_oper_up(self, m_read_netlink_socket, m_socket): """Test for a new nic attached""" @@ -215,8 +211,8 @@ def test_nic_attached_oper_up(self, m_read_netlink_socket, m_socket): data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) m_read_netlink_socket.side_effect = [data_op_up] ifread = wait_for_nic_attach_event(m_socket, []) - self.assertEqual(m_read_netlink_socket.call_count, 1) - self.assertEqual(ifname, ifread) + assert m_read_netlink_socket.call_count == 1 + assert ifname == ifread def test_nic_attach_ignore_existing(self, m_read_netlink_socket, m_socket): """Test that we read only the interfaces we are interested in.""" @@ -224,8 +220,8 @@ def test_nic_attach_ignore_existing(self, m_read_netlink_socket, m_socket): data_eth1 = self._media_switch_data("eth1", RTM_NEWLINK, OPER_DOWN) m_read_netlink_socket.side_effect = [data_eth0, data_eth1] ifread = wait_for_nic_attach_event(m_socket, ["eth0"]) - self.assertEqual(m_read_netlink_socket.call_count, 2) - self.assertEqual("eth1", ifread) + assert m_read_netlink_socket.call_count == 2 + assert "eth1" == ifread def test_nic_attach_read_first(self, m_read_netlink_socket, m_socket): """Test that we read only the interfaces we are interested in.""" @@ -233,8 +229,8 @@ def test_nic_attach_read_first(self, m_read_netlink_socket, m_socket): data_eth1 = self._media_switch_data("eth1", RTM_NEWLINK, OPER_DOWN) m_read_netlink_socket.side_effect = [data_eth0, data_eth1] ifread = wait_for_nic_attach_event(m_socket, ["eth1"]) - self.assertEqual(m_read_netlink_socket.call_count, 1) - self.assertEqual("eth0", ifread) + assert m_read_netlink_socket.call_count == 1 + assert "eth0" == ifread def test_nic_detached(self, m_read_netlink_socket, m_socket): """Test for an existing nic detached""" @@ -242,13 +238,13 @@ def test_nic_detached(self, m_read_netlink_socket, m_socket): data_op_down = self._media_switch_data(ifname, RTM_DELLINK, OPER_DOWN) m_read_netlink_socket.side_effect = [data_op_down] ifread = wait_for_nic_detach_event(m_socket) - self.assertEqual(m_read_netlink_socket.call_count, 1) - self.assertEqual(ifname, ifread) + assert m_read_netlink_socket.call_count == 1 + assert ifname == ifread @mock.patch("cloudinit.sources.helpers.netlink.socket.socket") @mock.patch("cloudinit.sources.helpers.netlink.read_netlink_socket") -class TestWaitForMediaDisconnectConnect(CiTestCase): +class TestWaitForMediaDisconnectConnect: with_logs = True def _media_switch_data(self, ifname, msg_type, operstate): @@ -293,10 +289,10 @@ def test_media_down_up_scenario(self, m_read_netlink_socket, m_socket): data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) m_read_netlink_socket.side_effect = [data_op_down, data_op_up] wait_for_media_disconnect_connect(m_socket, ifname) - self.assertEqual(m_read_netlink_socket.call_count, 2) + assert m_read_netlink_socket.call_count == 2 def test_wait_for_media_switch_diff_interface( - self, m_read_netlink_socket, m_socket + self, m_read_netlink_socket, m_socket, caplog ): """wait_for_media_disconnect_connect ignores unexpected interfaces. @@ -326,11 +322,11 @@ def test_wait_for_media_switch_diff_interface( data_op_up_eth0, ] wait_for_media_disconnect_connect(m_socket, expected_ifname) - self.assertIn( - "Ignored netlink event on interface %s" % other_ifname, - self.logs.getvalue(), + assert ( + "Ignored netlink event on interface %s" % other_ifname + in caplog.text ) - self.assertEqual(m_read_netlink_socket.call_count, 4) + assert m_read_netlink_socket.call_count == 4 def test_invalid_msgtype_getlink(self, m_read_netlink_socket, m_socket): """wait_for_media_disconnect_connect ignores GETLINK events. @@ -357,7 +353,7 @@ def test_invalid_msgtype_getlink(self, m_read_netlink_socket, m_socket): data_newlink_up, ] wait_for_media_disconnect_connect(m_socket, ifname) - self.assertEqual(m_read_netlink_socket.call_count, 4) + assert m_read_netlink_socket.call_count == 4 def test_invalid_msgtype_setlink(self, m_read_netlink_socket, m_socket): """wait_for_media_disconnect_connect ignores SETLINK events. @@ -387,7 +383,7 @@ def test_invalid_msgtype_setlink(self, m_read_netlink_socket, m_socket): data_newlink_up, ] wait_for_media_disconnect_connect(m_socket, ifname) - self.assertEqual(m_read_netlink_socket.call_count, 4) + assert m_read_netlink_socket.call_count == 4 def test_netlink_invalid_switch_scenario( self, m_read_netlink_socket, m_socket @@ -428,7 +424,7 @@ def test_netlink_invalid_switch_scenario( data_op_up, ] wait_for_media_disconnect_connect(m_socket, ifname) - self.assertEqual(m_read_netlink_socket.call_count, 14) + assert m_read_netlink_socket.call_count == 14 def test_netlink_valid_inbetween_transitions( self, m_read_netlink_socket, m_socket @@ -450,7 +446,7 @@ def test_netlink_valid_inbetween_transitions( data_op_up, ] wait_for_media_disconnect_connect(m_socket, ifname) - self.assertEqual(m_read_netlink_socket.call_count, 4) + assert m_read_netlink_socket.call_count == 4 def test_netlink_invalid_operstate(self, m_read_netlink_socket, m_socket): """wait_for_media_disconnect_connect should handle invalid operstates. @@ -470,28 +466,25 @@ def test_netlink_invalid_operstate(self, m_read_netlink_socket, m_socket): data_op_up, ] wait_for_media_disconnect_connect(m_socket, ifname) - self.assertEqual(m_read_netlink_socket.call_count, 5) + assert m_read_netlink_socket.call_count == 5 def test_wait_invalid_socket(self, m_read_netlink_socket, m_socket): """wait_for_media_disconnect_connect handle none netlink socket.""" socket = None ifname = "eth0" - with self.assertRaises(AssertionError) as context: + with pytest.raises(AssertionError, match="netlink socket is none"): wait_for_media_disconnect_connect(socket, ifname) - self.assertTrue("netlink socket is none" in str(context.exception)) def test_wait_invalid_ifname(self, m_read_netlink_socket, m_socket): """wait_for_media_disconnect_connect handle none interface name""" ifname = None - with self.assertRaises(AssertionError) as context: + with pytest.raises(AssertionError, match="interface name is none"): wait_for_media_disconnect_connect(m_socket, ifname) - self.assertTrue("interface name is none" in str(context.exception)) ifname = "" - with self.assertRaises(AssertionError) as context: + with pytest.raises( + AssertionError, match="interface name cannot be empty" + ): wait_for_media_disconnect_connect(m_socket, ifname) - self.assertTrue( - "interface name cannot be empty" in str(context.exception) - ) def test_wait_invalid_rta_attr(self, m_read_netlink_socket, m_socket): """wait_for_media_disconnect_connect handles invalid rta data""" @@ -507,7 +500,7 @@ def test_wait_invalid_rta_attr(self, m_read_netlink_socket, m_socket): data_op_up, ] wait_for_media_disconnect_connect(m_socket, ifname) - self.assertEqual(m_read_netlink_socket.call_count, 4) + assert m_read_netlink_socket.call_count == 4 def test_read_multiple_netlink_msgs(self, m_read_netlink_socket, m_socket): """Read multiple messages in single receive call""" @@ -540,7 +533,7 @@ def test_read_multiple_netlink_msgs(self, m_read_netlink_socket, m_socket): ) m_read_netlink_socket.return_value = data wait_for_media_disconnect_connect(m_socket, ifname) - self.assertEqual(m_read_netlink_socket.call_count, 1) + assert m_read_netlink_socket.call_count == 1 def test_read_partial_netlink_msgs(self, m_read_netlink_socket, m_socket): """Read partial messages in receive call""" @@ -570,4 +563,4 @@ def test_read_partial_netlink_msgs(self, m_read_netlink_socket, m_socket): ) m_read_netlink_socket.side_effect = [data1, data2] wait_for_media_disconnect_connect(m_socket, ifname) - self.assertEqual(m_read_netlink_socket.call_count, 2) + assert m_read_netlink_socket.call_count == 2 diff --git a/tests/unittests/sources/test_aliyun.py b/tests/unittests/sources/test_aliyun.py index 2d61ff8a..5a201dbd 100644 --- a/tests/unittests/sources/test_aliyun.py +++ b/tests/unittests/sources/test_aliyun.py @@ -7,13 +7,11 @@ import pytest import responses -from cloudinit import helpers from cloudinit.sources import DataSourceAliYun as ay from cloudinit.sources.helpers.aliyun import ( convert_ecs_metadata_network_config, ) from cloudinit.util import load_json -from tests.unittests import helpers as test_helpers DEFAULT_METADATA_RAW = r"""{ "disks": { @@ -97,183 +95,149 @@ - echo hello world > /tmp/vendor""" -class TestAliYunDatasource(test_helpers.ResponsesTestCase): - def setUp(self): - super(TestAliYunDatasource, self).setUp() - cfg = {"datasource": {"AliYun": {"timeout": "1", "max_wait": "1"}}} - distro = {} - paths = helpers.Paths({"run_dir": self.tmp_dir()}) - self.ds = ay.DataSourceAliYun(cfg, distro, paths) - self.metadata_address = self.ds.metadata_urls[0] - - @property - def default_metadata(self): - return DEFAULT_METADATA - - @property - def default_userdata(self): - return DEFAULT_USERDATA - - @property - def default_vendordata(self): - return DEFAULT_VENDORDATA - - @property - def metadata_url(self): - return ( - os.path.join( - self.metadata_address, - self.ds.min_metadata_version, - "meta-data", - ) - + "/" +@pytest.fixture +def ds(paths): + cfg = {"datasource": {"AliYun": {"timeout": "1", "max_wait": "1"}}} + distro = {} + return ay.DataSourceAliYun(cfg, distro, paths) + + +@pytest.fixture +def metadata_address(ds): + return ds.metadata_urls[0] + + +def register_mock_metaserver(base_url, data): + def register_helper(register, base_url, body): + if isinstance(body, str): + register(base_url, body) + elif isinstance(body, list): + register(base_url.rstrip("/"), "\n".join(body) + "\n") + elif isinstance(body, dict): + if not body: + register(base_url.rstrip("/") + "/", "not found", status=404) + vals = [] + for k, v in body.items(): + if isinstance(v, (str, list)): + suffix = k.rstrip("/") + else: + suffix = k.rstrip("/") + "/" + vals.append(suffix) + url = base_url.rstrip("/") + "/" + suffix + register_helper(register, url, v) + register(base_url, "\n".join(vals) + "\n") + + register = functools.partial(responses.add, responses.GET) + register_helper(register, base_url, data) + + +@pytest.fixture +def regist_default_server(ds, metadata_address): + metadata_url = ( + os.path.join( + metadata_address, + ds.min_metadata_version, + "meta-data", ) + + "/" + ) + register_mock_metaserver(metadata_url, DEFAULT_METADATA) - @property - def metadata_all_url(self): - return ( - os.path.join( - self.metadata_address, - self.ds.min_metadata_version, - "meta-data", - ) - + "/all" - ) + userdata_url = os.path.join( + metadata_address, ds.min_metadata_version, "user-data" + ) + register_mock_metaserver(userdata_url, DEFAULT_USERDATA) - @property - def userdata_url(self): - return os.path.join( - self.metadata_address, self.ds.min_metadata_version, "user-data" - ) - - @property - def vendordata_url(self): - return os.path.join( - self.metadata_address, self.ds.min_metadata_version, "vendor-data" - ) + vendordata_url = os.path.join( + metadata_address, ds.min_metadata_version, "vendor-data" + ) + register_mock_metaserver(vendordata_url, DEFAULT_USERDATA) # EC2 provides an instance-identity document which must return 404 here # for this test to pass. - @property - def default_identity(self): - return {} - - @property - def identity_url(self): - return os.path.join( - self.metadata_address, - self.ds.min_metadata_version, - "dynamic", - "instance-identity", + default_identity = {} + identity_url = os.path.join( + metadata_address, + ds.min_metadata_version, + "dynamic", + "instance-identity", + ) + register_mock_metaserver(identity_url, default_identity) + + token_url = os.path.join(metadata_address, "latest", "api", "token") + responses.add(responses.PUT, token_url, "API-TOKEN") + + +@pytest.fixture +def regist_json_meta_path(ds, metadata_address): + metadata_all_url = ( + os.path.join( + metadata_address, + ds.min_metadata_version, + "meta-data", ) + + "/all" + ) + register_mock_metaserver(metadata_all_url, DEFAULT_METADATA_RAW) - @property - def token_url(self): - return os.path.join( - self.metadata_address, - "latest", - "api", - "token", - ) - def register_mock_metaserver(self, base_url, data): - def register_helper(register, base_url, body): - if isinstance(body, str): - register(base_url, body) - elif isinstance(body, list): - register(base_url.rstrip("/"), "\n".join(body) + "\n") - elif isinstance(body, dict): - if not body: - register( - base_url.rstrip("/") + "/", "not found", status=404 - ) - vals = [] - for k, v in body.items(): - if isinstance(v, (str, list)): - suffix = k.rstrip("/") - else: - suffix = k.rstrip("/") + "/" - vals.append(suffix) - url = base_url.rstrip("/") + "/" + suffix - register_helper(register, url, v) - register(base_url, "\n".join(vals) + "\n") - - register = functools.partial(self.responses.add, responses.GET) - register_helper(register, base_url, data) - - def regist_default_server(self, register_json_meta_path=True): - self.register_mock_metaserver(self.metadata_url, self.default_metadata) - if register_json_meta_path: - self.register_mock_metaserver( - self.metadata_all_url, DEFAULT_METADATA_RAW - ) - self.register_mock_metaserver(self.userdata_url, self.default_userdata) - self.register_mock_metaserver( - self.vendordata_url, self.default_userdata - ) +class TestAliYunDatasource: - self.register_mock_metaserver(self.identity_url, self.default_identity) - self.responses.add(responses.PUT, self.token_url, "API-TOKEN") + def _test_get_data(self, ds): + assert ds.metadata == DEFAULT_METADATA + assert ds.userdata_raw == DEFAULT_USERDATA.encode("utf8") - def _test_get_data(self): - self.assertEqual(self.ds.metadata, self.default_metadata) - self.assertEqual( - self.ds.userdata_raw, self.default_userdata.encode("utf8") - ) - - def _test_get_sshkey(self): + def _test_get_sshkey(self, ds): pub_keys = [ v["openssh-key"] - for (_, v) in self.default_metadata["public-keys"].items() + for (_, v) in DEFAULT_METADATA["public-keys"].items() ] - self.assertEqual(self.ds.get_public_ssh_keys(), pub_keys) + assert ds.get_public_ssh_keys() == pub_keys - def _test_get_iid(self): - self.assertEqual( - self.default_metadata["instance-id"], self.ds.get_instance_id() - ) + def _test_get_iid(self, ds): + assert DEFAULT_METADATA["instance-id"] == ds.get_instance_id() - def _test_host_name(self): - self.assertEqual( - self.default_metadata["hostname"], self.ds.get_hostname().hostname - ) + def _test_host_name(self, ds): + assert DEFAULT_METADATA["hostname"] == ds.get_hostname().hostname + @responses.activate + @pytest.mark.usefixtures("regist_default_server", "regist_json_meta_path") @mock.patch("cloudinit.sources.DataSourceEc2.util.is_resolvable") @mock.patch("cloudinit.sources.DataSourceAliYun._is_aliyun") - def test_with_mock_server(self, m_is_aliyun, m_resolv): + def test_with_mock_server(self, m_is_aliyun, m_resolv, ds): m_is_aliyun.return_value = True - self.regist_default_server() - ret = self.ds.get_data() - self.assertEqual(True, ret) - self.assertEqual(1, m_is_aliyun.call_count) - self._test_get_data() - self._test_get_sshkey() - self._test_get_iid() - self._test_host_name() - self.assertEqual("aliyun", self.ds.cloud_name) - self.assertEqual("aliyun", self.ds.platform) - self.assertEqual( - "metadata (http://100.100.100.200)", self.ds.subplatform - ) - + ret = ds.get_data() + assert True is ret + assert 1 == m_is_aliyun.call_count + self._test_get_data(ds) + self._test_get_sshkey(ds) + self._test_get_iid(ds) + self._test_host_name(ds) + assert "aliyun" == ds.cloud_name + assert "aliyun" == ds.platform + assert "metadata (http://100.100.100.200)" == ds.subplatform + + @responses.activate + @pytest.mark.usefixtures("regist_default_server") @mock.patch("cloudinit.sources.DataSourceEc2.util.is_resolvable") @mock.patch("cloudinit.sources.DataSourceAliYun._is_aliyun") - def test_with_mock_server_without_json_path(self, m_is_aliyun, m_resolv): + def test_with_mock_server_without_json_path( + self, m_is_aliyun, m_resolv, ds + ): m_is_aliyun.return_value = True - self.regist_default_server(register_json_meta_path=False) - ret = self.ds.get_data() - self.assertEqual(True, ret) - self.assertEqual(1, m_is_aliyun.call_count) - self._test_get_data() - self._test_get_sshkey() - self._test_get_iid() - self._test_host_name() - self.assertEqual("aliyun", self.ds.cloud_name) - self.assertEqual("aliyun", self.ds.platform) - self.assertEqual( - "metadata (http://100.100.100.200)", self.ds.subplatform - ) - + ret = ds.get_data() + assert True is ret + assert 1 == m_is_aliyun.call_count + self._test_get_data(ds) + self._test_get_sshkey(ds) + self._test_get_iid(ds) + self._test_host_name(ds) + assert "aliyun" == ds.cloud_name + assert "aliyun" == ds.platform + assert "metadata (http://100.100.100.200)" == ds.subplatform + + @responses.activate + @pytest.mark.usefixtures("regist_default_server", "regist_json_meta_path") @mock.patch("cloudinit.net.ephemeral.EphemeralIPv6Network") @mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") @mock.patch("cloudinit.sources.DataSourceEc2.util.is_resolvable") @@ -291,6 +255,8 @@ def test_aliyun_local_with_mock_server( m_resolva, m_net4, m_net6, + ds, + paths, ): m_is_aliyun.return_value = True m_fallback_nic.return_value = "eth9" @@ -304,66 +270,57 @@ def test_aliyun_local_with_mock_server( m_is_bsd.return_value = False cfg = {"datasource": {"AliYun": {"timeout": "1", "max_wait": "1"}}} distro = mock.MagicMock() - paths = helpers.Paths({"run_dir": self.tmp_dir()}) - self.ds = ay.DataSourceAliYunLocal(cfg, distro, paths) - self.regist_default_server() - ret = self.ds.get_data() - self.assertEqual(True, ret) - self.assertEqual(1, m_is_aliyun.call_count) - self._test_get_data() - self._test_get_sshkey() - self._test_get_iid() - self._test_host_name() - self.assertEqual("aliyun", self.ds.cloud_name) - self.assertEqual("aliyun", self.ds.platform) - self.assertEqual( - "metadata (http://100.100.100.200)", self.ds.subplatform - ) - + ds = ay.DataSourceAliYunLocal(cfg, distro, paths) + ret = ds.get_data() + assert True is ret + assert 1 == m_is_aliyun.call_count + self._test_get_data(ds) + self._test_get_sshkey(ds) + self._test_get_iid(ds) + self._test_host_name(ds) + assert "aliyun" == ds.cloud_name + assert "aliyun" == ds.platform + assert "metadata (http://100.100.100.200)" == ds.subplatform + + @responses.activate + @pytest.mark.usefixtures("regist_default_server", "regist_json_meta_path") @mock.patch("cloudinit.sources.DataSourceAliYun._is_aliyun") - def test_returns_false_when_not_on_aliyun(self, m_is_aliyun): + def test_returns_false_when_not_on_aliyun(self, m_is_aliyun, ds): """If is_aliyun returns false, then get_data should return False.""" m_is_aliyun.return_value = False - self.regist_default_server() - ret = self.ds.get_data() - self.assertEqual(1, m_is_aliyun.call_count) - self.assertEqual(False, ret) + ret = ds.get_data() + assert 1 == m_is_aliyun.call_count + assert False is ret def test_parse_public_keys(self): public_keys = {} - self.assertEqual(ay.parse_public_keys(public_keys), []) + assert ay.parse_public_keys(public_keys) == [] public_keys = {"key-pair-0": "ssh-key-0"} - self.assertEqual( - ay.parse_public_keys(public_keys), [public_keys["key-pair-0"]] - ) + assert ay.parse_public_keys(public_keys) == [public_keys["key-pair-0"]] public_keys = {"key-pair-0": "ssh-key-0", "key-pair-1": "ssh-key-1"} - self.assertEqual( - set(ay.parse_public_keys(public_keys)), - set([public_keys["key-pair-0"], public_keys["key-pair-1"]]), + assert set(ay.parse_public_keys(public_keys)) == set( + [public_keys["key-pair-0"], public_keys["key-pair-1"]] ) public_keys = {"key-pair-0": ["ssh-key-0", "ssh-key-1"]} - self.assertEqual( - ay.parse_public_keys(public_keys), public_keys["key-pair-0"] - ) + assert ay.parse_public_keys(public_keys) == public_keys["key-pair-0"] public_keys = {"key-pair-0": {"openssh-key": []}} - self.assertEqual(ay.parse_public_keys(public_keys), []) + assert ay.parse_public_keys(public_keys) == [] public_keys = {"key-pair-0": {"openssh-key": "ssh-key-0"}} - self.assertEqual( - ay.parse_public_keys(public_keys), - [public_keys["key-pair-0"]["openssh-key"]], - ) + assert ay.parse_public_keys(public_keys) == [ + public_keys["key-pair-0"]["openssh-key"] + ] public_keys = { "key-pair-0": {"openssh-key": ["ssh-key-0", "ssh-key-1"]} } - self.assertEqual( - ay.parse_public_keys(public_keys), - public_keys["key-pair-0"]["openssh-key"], + assert ( + ay.parse_public_keys(public_keys) + == public_keys["key-pair-0"]["openssh-key"] ) def test_route_metric_calculated_with_multiple_network_cards(self): @@ -428,7 +385,7 @@ def test_route_metric_calculated_with_multiple_network_cards(self): assert "dhcp4-overrides" not in met0 -class TestIsAliYun(test_helpers.CiTestCase): +class TestIsAliYun: ALIYUN_PRODUCT = "Alibaba Cloud ECS" read_dmi_data_expected = [mock.call("system-product-name")] @@ -437,27 +394,21 @@ def test_true_on_aliyun_product(self, m_read_dmi_data): """Should return true if the dmi product data has expected value.""" m_read_dmi_data.return_value = self.ALIYUN_PRODUCT ret = ay._is_aliyun() - self.assertEqual( - self.read_dmi_data_expected, m_read_dmi_data.call_args_list - ) - self.assertEqual(True, ret) + assert self.read_dmi_data_expected == m_read_dmi_data.call_args_list + assert True is ret @mock.patch("cloudinit.sources.DataSourceAliYun.dmi.read_dmi_data") def test_false_on_empty_string(self, m_read_dmi_data): """Should return false on empty value returned.""" m_read_dmi_data.return_value = "" ret = ay._is_aliyun() - self.assertEqual( - self.read_dmi_data_expected, m_read_dmi_data.call_args_list - ) - self.assertEqual(False, ret) + assert self.read_dmi_data_expected == m_read_dmi_data.call_args_list + assert False is ret @mock.patch("cloudinit.sources.DataSourceAliYun.dmi.read_dmi_data") def test_false_on_unknown_string(self, m_read_dmi_data): """Should return false on an unrelated string.""" m_read_dmi_data.return_value = "cubs win" ret = ay._is_aliyun() - self.assertEqual( - self.read_dmi_data_expected, m_read_dmi_data.call_args_list - ) - self.assertEqual(False, ret) + assert self.read_dmi_data_expected == m_read_dmi_data.call_args_list + assert False is ret diff --git a/tests/unittests/sources/test_altcloud.py b/tests/unittests/sources/test_altcloud.py index dba0f6a4..74a8d336 100644 --- a/tests/unittests/sources/test_altcloud.py +++ b/tests/unittests/sources/test_altcloud.py @@ -12,11 +12,12 @@ import os import shutil -import tempfile + +import pytest import cloudinit.sources.DataSourceAltCloud as dsac -from cloudinit import dmi, helpers, subp, util -from tests.unittests.helpers import CiTestCase, mock +from cloudinit import subp, util +from tests.unittests.helpers import mock OS_UNAME_ORIG = getattr(os, "uname") @@ -73,381 +74,324 @@ def _data(key): return _data -class TestGetCloudType(CiTestCase): +@pytest.fixture +def force_x86_64(): + # We have a different code path for arm to deal with LP1243287 + # We have to switch arch to x86_64 to avoid test failure + force_arch("x86_64") + yield + # Return back to original arch + force_arch() + + +@pytest.mark.usefixtures("fake_filesystem", "force_x86_64") +class TestGetCloudType: """Test to exercise method: DataSourceAltCloud.get_cloud_type()""" - with_logs = True - - def setUp(self): - """Set up.""" - super(TestGetCloudType, self).setUp() - self.tmp = self.tmp_dir() - self.paths = helpers.Paths({"cloud_dir": self.tmp}) - self.dmi_data = dmi.read_dmi_data - # We have a different code path for arm to deal with LP1243287 - # We have to switch arch to x86_64 to avoid test failure - force_arch("x86_64") - - def tearDown(self): - # Reset - dmi.read_dmi_data = self.dmi_data - force_arch() - super().tearDown() - - def test_cloud_info_file_ioerror(self): + def test_cloud_info_file_ioerror(self, caplog, paths, tmp_path): """Return UNKNOWN when /etc/sysconfig/cloud-info exists but errors.""" - self.assertEqual("/etc/sysconfig/cloud-info", dsac.CLOUD_INFO_FILE) - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) + assert "/etc/sysconfig/cloud-info" == dsac.CLOUD_INFO_FILE + dsrc = dsac.DataSourceAltCloud({}, None, paths) # Attempting to read the directory generates IOError - with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.tmp): - self.assertEqual("UNKNOWN", dsrc.get_cloud_type()) - self.assertIn( - "[Errno 21] Is a directory: '%s'" % self.tmp, self.logs.getvalue() - ) + with mock.patch.object(dsac, "CLOUD_INFO_FILE", str(tmp_path)): + assert "UNKNOWN" == dsrc.get_cloud_type() + assert "[Errno 21] Is a directory: '%s'" % tmp_path in caplog.text - def test_cloud_info_file(self): + def test_cloud_info_file(self, paths): """Return uppercase stripped content from /etc/sysconfig/cloud-info.""" - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - cloud_info = self.tmp_path("cloud-info", dir=self.tmp) + dsrc = dsac.DataSourceAltCloud({}, None, paths) + cloud_info = "cloud-info" util.write_file(cloud_info, " OverRiDdeN CloudType ") # Attempting to read the directory generates IOError with mock.patch.object(dsac, "CLOUD_INFO_FILE", cloud_info): - self.assertEqual("OVERRIDDEN CLOUDTYPE", dsrc.get_cloud_type()) + assert "OVERRIDDEN CLOUDTYPE" == dsrc.get_cloud_type() - def test_rhev(self): + @mock.patch("cloudinit.dmi.read_dmi_data", side_effect=_dmi_data("RHEV")) + def test_rhev(self, m_read_dmi_data, paths): """ Test method get_cloud_type() for RHEVm systems. Forcing read_dmi_data return to match a RHEVm system: RHEV Hypervisor """ - dmi.read_dmi_data = _dmi_data("RHEV") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual("RHEV", dsrc.get_cloud_type()) - - def test_vsphere(self): + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert "RHEV" == dsrc.get_cloud_type() + + @mock.patch( + "cloudinit.dmi.read_dmi_data", + side_effect=_dmi_data("VMware Virtual Platform"), + ) + def test_vsphere(self, m_read_dmi_data, paths): """ Test method get_cloud_type() for vSphere systems. Forcing read_dmi_data return to match a vSphere system: RHEV Hypervisor """ - dmi.read_dmi_data = _dmi_data("VMware Virtual Platform") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual("VSPHERE", dsrc.get_cloud_type()) - - def test_unknown(self): + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert "VSPHERE" == dsrc.get_cloud_type() + + @mock.patch( + "cloudinit.dmi.read_dmi_data", + side_effect=_dmi_data("Unrecognized Platform"), + ) + def test_unknown(self, m_read_dmi_data, paths): """ Test method get_cloud_type() for unknown systems. Forcing read_dmi_data return to match an unrecognized return. """ - dmi.read_dmi_data = _dmi_data("Unrecognized Platform") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual("UNKNOWN", dsrc.get_cloud_type()) + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert "UNKNOWN" == dsrc.get_cloud_type() -class TestGetDataCloudInfoFile(CiTestCase): +@pytest.mark.usefixtures("fake_filesystem") +class TestGetDataCloudInfoFile: """ Test to exercise method: DataSourceAltCloud.get_data() With a contrived CLOUD_INFO_FILE """ - def setUp(self): - """Set up.""" - self.tmp = self.tmp_dir() - self.paths = helpers.Paths( - {"cloud_dir": self.tmp, "run_dir": self.tmp} - ) - self.cloud_info_file = self.tmp_path("cloud-info", dir=self.tmp) + CLOUD_INFO_FILE = "cloud-info" - def test_rhev(self): + def test_rhev(self, paths): """Success Test module get_data() forcing RHEV.""" - util.write_file(self.cloud_info_file, "RHEV") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) + util.write_file(self.CLOUD_INFO_FILE, "RHEV") + dsrc = dsac.DataSourceAltCloud({}, None, paths) dsrc.user_data_rhevm = lambda: True - with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.cloud_info_file): - self.assertEqual(True, dsrc.get_data()) - self.assertEqual("altcloud", dsrc.cloud_name) - self.assertEqual("altcloud", dsrc.platform_type) - self.assertEqual("rhev (/dev/fd0)", dsrc.subplatform) + with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.CLOUD_INFO_FILE): + assert True is dsrc.get_data() + assert "altcloud" == dsrc.cloud_name + assert "altcloud" == dsrc.platform_type + assert "rhev (/dev/fd0)" == dsrc.subplatform - def test_vsphere(self): + def test_vsphere(self, paths): """Success Test module get_data() forcing VSPHERE.""" - util.write_file(self.cloud_info_file, "VSPHERE") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) + util.write_file(self.CLOUD_INFO_FILE, "VSPHERE") + dsrc = dsac.DataSourceAltCloud({}, None, paths) dsrc.user_data_vsphere = lambda: True - with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.cloud_info_file): - self.assertEqual(True, dsrc.get_data()) - self.assertEqual("altcloud", dsrc.cloud_name) - self.assertEqual("altcloud", dsrc.platform_type) - self.assertEqual("vsphere (unknown)", dsrc.subplatform) + with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.CLOUD_INFO_FILE): + assert True is dsrc.get_data() + assert "altcloud" == dsrc.cloud_name + assert "altcloud" == dsrc.platform_type + assert "vsphere (unknown)" == dsrc.subplatform - def test_fail_rhev(self): + def test_fail_rhev(self, paths): """Failure Test module get_data() forcing RHEV.""" - util.write_file(self.cloud_info_file, "RHEV") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) + util.write_file(self.CLOUD_INFO_FILE, "RHEV") + dsrc = dsac.DataSourceAltCloud({}, None, paths) dsrc.user_data_rhevm = lambda: False - with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.cloud_info_file): - self.assertEqual(False, dsrc.get_data()) + with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.CLOUD_INFO_FILE): + assert False is dsrc.get_data() - def test_fail_vsphere(self): + def test_fail_vsphere(self, paths): """Failure Test module get_data() forcing VSPHERE.""" - util.write_file(self.cloud_info_file, "VSPHERE") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) + util.write_file(self.CLOUD_INFO_FILE, "VSPHERE") + dsrc = dsac.DataSourceAltCloud({}, None, paths) dsrc.user_data_vsphere = lambda: False - with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.cloud_info_file): - self.assertEqual(False, dsrc.get_data()) + with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.CLOUD_INFO_FILE): + assert False is dsrc.get_data() - def test_unrecognized(self): + def test_unrecognized(self, paths): """Failure Test module get_data() forcing unrecognized.""" - util.write_file(self.cloud_info_file, "unrecognized") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.cloud_info_file): - self.assertEqual(False, dsrc.get_data()) + util.write_file(self.CLOUD_INFO_FILE, "unrecognized") + dsrc = dsac.DataSourceAltCloud({}, None, paths) + with mock.patch.object(dsac, "CLOUD_INFO_FILE", self.CLOUD_INFO_FILE): + assert False is dsrc.get_data() -class TestGetDataNoCloudInfoFile(CiTestCase): +@pytest.fixture +def fake_dsca_cloud_info(): + dsac.CLOUD_INFO_FILE = "no such file" + yield + dsac.CLOUD_INFO_FILE = "/etc/sysconfig/cloud-info" + + +@pytest.mark.usefixtures( + "fake_filesystem", "force_x86_64", "fake_dsca_cloud_info" +) +class TestGetDataNoCloudInfoFile: """ Test to exercise method: DataSourceAltCloud.get_data() Without a CLOUD_INFO_FILE """ - def setUp(self): - """Set up.""" - self.tmp = self.tmp_dir() - self.paths = helpers.Paths( - {"cloud_dir": self.tmp, "run_dir": self.tmp} - ) - self.dmi_data = dmi.read_dmi_data - dsac.CLOUD_INFO_FILE = "no such file" - # We have a different code path for arm to deal with LP1243287 - # We have to switch arch to x86_64 to avoid test failure - force_arch("x86_64") - - def tearDown(self): - # Reset - dsac.CLOUD_INFO_FILE = "/etc/sysconfig/cloud-info" - dmi.read_dmi_data = self.dmi_data - # Return back to original arch - force_arch() - super().tearDown() - - def test_rhev_no_cloud_file(self): + @mock.patch( + "cloudinit.dmi.read_dmi_data", side_effect=_dmi_data("RHEV Hypervisor") + ) + def test_rhev_no_cloud_file(self, m_read_dmi_data, paths): """Test No cloud info file module get_data() forcing RHEV.""" - - dmi.read_dmi_data = _dmi_data("RHEV Hypervisor") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) + dsrc = dsac.DataSourceAltCloud({}, None, paths) dsrc.user_data_rhevm = lambda: True - self.assertEqual(True, dsrc.get_data()) + assert True is dsrc.get_data() - def test_vsphere_no_cloud_file(self): + @mock.patch( + "cloudinit.dmi.read_dmi_data", + side_effect=_dmi_data("VMware Virtual Platform"), + ) + def test_vsphere_no_cloud_file(self, m_read_dmi_data, paths): """Test No cloud info file module get_data() forcing VSPHERE.""" - - dmi.read_dmi_data = _dmi_data("VMware Virtual Platform") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) + dsrc = dsac.DataSourceAltCloud({}, None, paths) dsrc.user_data_vsphere = lambda: True - self.assertEqual(True, dsrc.get_data()) + assert True is dsrc.get_data() - def test_failure_no_cloud_file(self): + @mock.patch( + "cloudinit.dmi.read_dmi_data", + side_effect=_dmi_data("Unrecognized Platform"), + ) + def test_failure_no_cloud_file(self, m_read_dmi_data, paths): """Test No cloud info file module get_data() forcing unrecognized.""" - - dmi.read_dmi_data = _dmi_data("Unrecognized Platform") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual(False, dsrc.get_data()) - - -class TestUserDataRhevm(CiTestCase): + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert False is dsrc.get_data() + + +@pytest.fixture +def user_data(tmp_path): + mount_dir = str(tmp_path) + _write_user_data_files(mount_dir, "test user data") + yield + _remove_user_data_files(mount_dir) + shutil.rmtree(mount_dir) + + +@pytest.mark.usefixtures("user_data") +@mock.patch( + "cloudinit.sources.DataSourceAltCloud.modprobe_floppy", + return_value=None, +) +@mock.patch( + "cloudinit.sources.DataSourceAltCloud.util.udevadm_settle", + return_value=("", ""), +) +@mock.patch("cloudinit.sources.DataSourceAltCloud.util.mount_cb") +class TestUserDataRhevm: """ Test to exercise method: DataSourceAltCloud.user_data_rhevm() """ - def setUp(self): - """Set up.""" - self.paths = helpers.Paths({"cloud_dir": "/tmp"}) - self.mount_dir = self.tmp_dir() - _write_user_data_files(self.mount_dir, "test user data") - self.add_patch( - "cloudinit.sources.DataSourceAltCloud.modprobe_floppy", - "m_modprobe_floppy", - return_value=None, - ) - self.add_patch( - "cloudinit.sources.DataSourceAltCloud.util.udevadm_settle", - "m_udevadm_settle", - return_value=("", ""), - ) - self.add_patch( - "cloudinit.sources.DataSourceAltCloud.util.mount_cb", "m_mount_cb" - ) - - def test_mount_cb_fails(self): + def test_mount_cb_fails( + self, m_mount_cb, m_udevadm_settle, m_modprobe_floppy, paths + ): """Test user_data_rhevm() where mount_cb fails.""" + m_mount_cb.side_effect = util.MountFailedError("Failed Mount") + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert False is dsrc.user_data_rhevm() - self.m_mount_cb.side_effect = util.MountFailedError("Failed Mount") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual(False, dsrc.user_data_rhevm()) - - def test_modprobe_fails(self): + def test_modprobe_fails( + self, m_mount_cb, m_udevadm_settle, m_modprobe_floppy, paths + ): """Test user_data_rhevm() where modprobe fails.""" - - self.m_modprobe_floppy.side_effect = subp.ProcessExecutionError( + m_modprobe_floppy.side_effect = subp.ProcessExecutionError( "Failed modprobe" ) - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual(False, dsrc.user_data_rhevm()) + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert False is dsrc.user_data_rhevm() - def test_no_modprobe_cmd(self): + def test_no_modprobe_cmd( + self, m_mount_cb, m_udevadm_settle, m_modprobe_floppy, paths + ): """Test user_data_rhevm() with no modprobe command.""" - - self.m_modprobe_floppy.side_effect = subp.ProcessExecutionError( + m_modprobe_floppy.side_effect = subp.ProcessExecutionError( "No such file or dir" ) - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual(False, dsrc.user_data_rhevm()) + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert False is dsrc.user_data_rhevm() - def test_udevadm_fails(self): + def test_udevadm_fails( + self, m_mount_cb, m_udevadm_settle, m_modprobe_floppy, paths + ): """Test user_data_rhevm() where udevadm fails.""" - - self.m_udevadm_settle.side_effect = subp.ProcessExecutionError( + m_udevadm_settle.side_effect = subp.ProcessExecutionError( "Failed settle." ) - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual(False, dsrc.user_data_rhevm()) + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert False is dsrc.user_data_rhevm() - def test_no_udevadm_cmd(self): + def test_no_udevadm_cmd( + self, m_mount_cb, m_udevadm_settle, m_modprobe_floppy, paths + ): """Test user_data_rhevm() with no udevadm command.""" + m_udevadm_settle.side_effect = OSError("No such file or dir") + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert False is dsrc.user_data_rhevm() - self.m_udevadm_settle.side_effect = OSError("No such file or dir") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual(False, dsrc.user_data_rhevm()) - -class TestUserDataVsphere(CiTestCase): +@pytest.mark.usefixtures("user_data") +class TestUserDataVsphere: """ Test to exercise method: DataSourceAltCloud.user_data_vsphere() """ - def setUp(self): - """Set up.""" - self.tmp = self.tmp_dir() - self.paths = helpers.Paths({"cloud_dir": self.tmp}) - self.mount_dir = tempfile.mkdtemp() - - _write_user_data_files(self.mount_dir, "test user data") - - def tearDown(self): - # Reset - - _remove_user_data_files(self.mount_dir) - - # Attempt to remove the temp dir ignoring errors - try: - shutil.rmtree(self.mount_dir) - except OSError: - pass - - dsac.CLOUD_INFO_FILE = "/etc/sysconfig/cloud-info" - super().tearDown() - @mock.patch("cloudinit.sources.DataSourceAltCloud.util.find_devs_with") @mock.patch("cloudinit.sources.DataSourceAltCloud.util.mount_cb") - def test_user_data_vsphere_no_cdrom(self, m_mount_cb, m_find_devs_with): + def test_user_data_vsphere_no_cdrom( + self, m_mount_cb, m_find_devs_with, paths + ): """Test user_data_vsphere() where mount_cb fails.""" m_mount_cb.return_value = [] - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual(False, dsrc.user_data_vsphere()) - self.assertEqual(0, m_mount_cb.call_count) + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert False is dsrc.user_data_vsphere() + assert 0 == m_mount_cb.call_count @mock.patch("cloudinit.sources.DataSourceAltCloud.util.find_devs_with") @mock.patch("cloudinit.sources.DataSourceAltCloud.util.mount_cb") - def test_user_data_vsphere_mcb_fail(self, m_mount_cb, m_find_devs_with): + def test_user_data_vsphere_mcb_fail( + self, m_mount_cb, m_find_devs_with, paths + ): """Test user_data_vsphere() where mount_cb fails.""" m_find_devs_with.return_value = ["/dev/mock/cdrom"] m_mount_cb.side_effect = util.MountFailedError("Unable To mount") - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - self.assertEqual(False, dsrc.user_data_vsphere()) - self.assertEqual(1, m_find_devs_with.call_count) - self.assertEqual(1, m_mount_cb.call_count) + dsrc = dsac.DataSourceAltCloud({}, None, paths) + assert False is dsrc.user_data_vsphere() + assert 1 == m_find_devs_with.call_count + assert 1 == m_mount_cb.call_count @mock.patch("cloudinit.sources.DataSourceAltCloud.util.find_devs_with") @mock.patch("cloudinit.sources.DataSourceAltCloud.util.mount_cb") - def test_user_data_vsphere_success(self, m_mount_cb, m_find_devs_with): + def test_user_data_vsphere_success( + self, m_mount_cb, m_find_devs_with, tmp_path, paths + ): """Test user_data_vsphere() where successful.""" m_find_devs_with.return_value = ["/dev/mock/cdrom"] m_mount_cb.return_value = "raw userdata from cdrom" - dsrc = dsac.DataSourceAltCloud({}, None, self.paths) - cloud_info = self.tmp_path("cloud-info", dir=self.tmp) + dsrc = dsac.DataSourceAltCloud({}, None, paths) + cloud_info = tmp_path / "cloud-info" util.write_file(cloud_info, "VSPHERE") - self.assertEqual(True, dsrc.user_data_vsphere()) + assert True is dsrc.user_data_vsphere() m_find_devs_with.assert_called_once_with("LABEL=CDROM") m_mount_cb.assert_called_once_with( "/dev/mock/cdrom", dsac.read_user_data_callback ) with mock.patch.object(dsrc, "get_cloud_type", return_value="VSPHERE"): - self.assertEqual("vsphere (/dev/mock/cdrom)", dsrc.subplatform) + assert "vsphere (/dev/mock/cdrom)" == dsrc.subplatform -class TestReadUserDataCallback(CiTestCase): +@pytest.mark.usefixtures("user_data") +class TestReadUserDataCallback: """ Test to exercise method: DataSourceAltCloud.read_user_data_callback() """ - def setUp(self): - """Set up.""" - self.paths = helpers.Paths({"cloud_dir": "/tmp"}) - self.mount_dir = tempfile.mkdtemp() - - _write_user_data_files(self.mount_dir, "test user data") - - def tearDown(self): - # Reset - - _remove_user_data_files(self.mount_dir) - - # Attempt to remove the temp dir ignoring errors - try: - shutil.rmtree(self.mount_dir) - except OSError: - pass - super().tearDown() - - def test_callback_both(self): + def test_callback_both(self, tmp_path): """Test read_user_data_callback() with both files.""" + assert "test user data" == dsac.read_user_data_callback(str(tmp_path)) - self.assertEqual( - "test user data", dsac.read_user_data_callback(self.mount_dir) - ) - - def test_callback_dc(self): + def test_callback_dc(self, tmp_path): """Test read_user_data_callback() with only DC file.""" + _remove_user_data_files(str(tmp_path), dc_file=False, non_dc_file=True) + assert "test user data" == dsac.read_user_data_callback(str(tmp_path)) - _remove_user_data_files( - self.mount_dir, dc_file=False, non_dc_file=True - ) - - self.assertEqual( - "test user data", dsac.read_user_data_callback(self.mount_dir) - ) - - def test_callback_non_dc(self): + def test_callback_non_dc(self, tmp_path): """Test read_user_data_callback() with only non-DC file.""" + _remove_user_data_files(str(tmp_path), dc_file=True, non_dc_file=False) + assert "test user data" == dsac.read_user_data_callback(str(tmp_path)) - _remove_user_data_files( - self.mount_dir, dc_file=True, non_dc_file=False - ) - - self.assertEqual( - "test user data", dsac.read_user_data_callback(self.mount_dir) - ) - - def test_callback_none(self): + def test_callback_none(self, tmp_path): """Test read_user_data_callback() no files are found.""" - - _remove_user_data_files(self.mount_dir) - self.assertIsNone(dsac.read_user_data_callback(self.mount_dir)) + _remove_user_data_files(str(tmp_path)) + assert dsac.read_user_data_callback(str(tmp_path)) is None def force_arch(arch=None): diff --git a/tests/unittests/sources/test_azure.py b/tests/unittests/sources/test_azure.py index 32edb832..d7f0adb4 100644 --- a/tests/unittests/sources/test_azure.py +++ b/tests/unittests/sources/test_azure.py @@ -4,6 +4,7 @@ import copy import datetime import json +import logging import os import stat import xml.etree.ElementTree as ET @@ -15,6 +16,7 @@ from cloudinit import distros, dmi, helpers, subp, url_helper from cloudinit.atomic_helper import b64e, json_dumps +from cloudinit.config import cc_mounts from cloudinit.net import dhcp, ephemeral from cloudinit.sources import UNSET from cloudinit.sources import DataSourceAzure as dsaz @@ -27,8 +29,7 @@ write_file, ) from tests.unittests.helpers import ( - CiTestCase, - ExitStack, + assert_count_equal, example_netdev, mock, populate_dir, @@ -141,7 +142,7 @@ def mock_monotonic(): yield m -@pytest.fixture +@pytest.fixture(autouse=True) def mock_dmi_read_dmi_data(): def fake_read(key: str) -> str: if key == "system-uuid": @@ -209,9 +210,9 @@ def mock_report_dmesg_to_kvp(): @pytest.fixture -def mock_kvp_report_failure_to_host(): +def mock_kvp_report_via_kvp(): with mock.patch( - MOCKPATH + "kvp.report_failure_to_host", + MOCKPATH + "kvp.report_via_kvp", return_value=True, autospec=True, ) as m: @@ -1050,60 +1051,49 @@ def test_uses_fallback_cfg_when_no_interface_metadata( assert azure_ds.network_config == self.fallback_config -class TestAzureDataSource(CiTestCase): - with_logs = True +@pytest.fixture +def waagent_d(tmp_path): + return str(tmp_path / "var/lib/waagent") - def setUp(self): - super(TestAzureDataSource, self).setUp() - self.tmp = self.tmp_dir() +# @pytest.mark.usefixtures("fake_filesystem") +class TestAzureDataSource: + @pytest.fixture(autouse=True) + def fixture(self, mocker): # patch cloud_dir, so our 'seed_dir' is guaranteed empty - self.paths = helpers.Paths( - {"cloud_dir": self.tmp, "run_dir": self.tmp} - ) - self.waagent_d = os.path.join(self.tmp, "var", "lib", "waagent") + mocker.patch.object(dsaz, "_get_random_seed", return_value="wild") - self.patches = ExitStack() - self.addCleanup(self.patches.close) - - self.patches.enter_context( - mock.patch.object(dsaz, "_get_random_seed", return_value="wild") - ) - - self.m_dhcp = self.patches.enter_context( - mock.patch.object( - dsaz, - "EphemeralDHCPv4", - ) + self.m_dhcp = mocker.patch.object( + dsaz, + "EphemeralDHCPv4", ) self.m_dhcp.return_value.lease = {} self.m_dhcp.return_value.iface = "eth4" - self.m_fetch = self.patches.enter_context( - mock.patch.object( - dsaz.imds, - "fetch_metadata_with_api_fallback", - mock.MagicMock(return_value=NETWORK_METADATA), - ) + self.m_fetch = mocker.patch.object( + dsaz.imds, + "fetch_metadata_with_api_fallback", + mock.MagicMock(return_value=NETWORK_METADATA), ) - self.m_fallback_nic = self.patches.enter_context( - mock.patch( - "cloudinit.sources.net.find_fallback_nic", return_value="eth9" - ) + self.m_fallback_nic = mocker.patch( + "cloudinit.sources.net.find_fallback_nic", return_value="eth9" ) - self.m_remove_ubuntu_network_scripts = self.patches.enter_context( - mock.patch.object( - dsaz, - "maybe_remove_ubuntu_network_config_scripts", - mock.MagicMock(), - ) + self.m_remove_ubuntu_network_scripts = mocker.patch.object( + dsaz, + "maybe_remove_ubuntu_network_config_scripts", + mock.MagicMock(), ) - def apply_patches(self, patches): - for module, name, new in patches: - self.patches.enter_context(mock.patch.object(module, name, new)) + @pytest.fixture + def apply_patches(self, mocker): + def _apply_patches(patches): + for module, name, new in patches: + mocker.patch.object(module, name, new) + + return _apply_patches - def _get_mockds(self): + @pytest.fixture + def _get_mockds(self, apply_patches): sysctl_out = ( "dev.storvsc.3.%pnpinfo: " "classid=ba6163d9-04a1-4d29-b605-72e2ffb1dc7f " @@ -1133,7 +1123,7 @@ def _get_mockds(self): at scbus2 target 0 lun 0 (da0,pass1) at scbus3 target 1 lun 0 (da1,pass2) """ - self.apply_patches( + apply_patches( [ ( dsaz, @@ -1154,105 +1144,108 @@ def _get_mockds(self): ) return dsaz - def _get_ds( - self, - data, - distro="ubuntu", - apply_network=None, - instance_id=None, - write_ovf_to_data_dir: bool = False, - write_ovf_to_seed_dir: bool = True, - ): - def _wait_for_files(flist, _maxwait=None, _naplen=None): - data["waited"] = flist - return [] - - def _load_possible_azure_ds(seed_dir, cache_dir): - yield seed_dir - yield dsaz.DEFAULT_PROVISIONING_ISO_DEV - yield from data.get("dsdevs", []) - if cache_dir: - yield cache_dir - - seed_dir = os.path.join(self.paths.seed_dir, "azure") - if write_ovf_to_seed_dir and data.get("ovfcontent") is not None: - populate_dir(seed_dir, {"ovf-env.xml": data["ovfcontent"]}) - - if write_ovf_to_data_dir and data.get("ovfcontent") is not None: - populate_dir(self.waagent_d, {"ovf-env.xml": data["ovfcontent"]}) - - dsaz.BUILTIN_DS_CONFIG["data_dir"] = self.waagent_d - - self.m_get_metadata_from_fabric = mock.MagicMock(return_value=[]) - self.m_report_failure_to_fabric = mock.MagicMock(autospec=True) - self.m_list_possible_azure_ds = mock.MagicMock( - side_effect=_load_possible_azure_ds - ) + @pytest.fixture + def get_ds(self, apply_patches, paths, tmp_path, waagent_d): + def _get_ds( + data, + distro="ubuntu", + apply_network=None, + instance_id=None, + write_ovf_to_data_dir: bool = False, + write_ovf_to_seed_dir: bool = True, + ): + def _wait_for_files(flist, _maxwait=None, _naplen=None): + data["waited"] = flist + return [] + + def _load_possible_azure_ds(seed_dir, cache_dir): + yield seed_dir + yield dsaz.DEFAULT_PROVISIONING_ISO_DEV + yield from data.get("dsdevs", []) + if cache_dir: + yield cache_dir + + seed_dir = os.path.join(paths.seed_dir, "azure") + if write_ovf_to_seed_dir and data.get("ovfcontent") is not None: + populate_dir(seed_dir, {"ovf-env.xml": data["ovfcontent"]}) + + if write_ovf_to_data_dir and data.get("ovfcontent") is not None: + populate_dir(waagent_d, {"ovf-env.xml": data["ovfcontent"]}) + + dsaz.BUILTIN_DS_CONFIG["data_dir"] = waagent_d + + self.m_get_metadata_from_fabric = mock.MagicMock(return_value=[]) + self.m_report_failure_to_fabric = mock.MagicMock(autospec=True) + self.m_list_possible_azure_ds = mock.MagicMock( + side_effect=_load_possible_azure_ds + ) - if instance_id: - self.instance_id = instance_id - else: - self.instance_id = EXAMPLE_UUID + if instance_id: + self.instance_id = instance_id + else: + self.instance_id = EXAMPLE_UUID - def _dmi_mocks(key): - if key == "system-uuid": - return self.instance_id - elif key == "chassis-asset-tag": - return "7783-7084-3265-9085-8269-3286-77" - raise RuntimeError() + def _dmi_mocks(key): + if key == "system-uuid": + return self.instance_id + elif key == "chassis-asset-tag": + return "7783-7084-3265-9085-8269-3286-77" + raise RuntimeError() - self.m_read_dmi_data = mock.MagicMock(autospec=True) - self.m_read_dmi_data.side_effect = _dmi_mocks + self.m_read_dmi_data = mock.MagicMock(autospec=True) + self.m_read_dmi_data.side_effect = _dmi_mocks - self.apply_patches( - [ - ( - dsaz, - "list_possible_azure_ds", - self.m_list_possible_azure_ds, - ), - ( - dsaz, - "get_metadata_from_fabric", - self.m_get_metadata_from_fabric, - ), - ( - dsaz, - "report_failure_to_fabric", - self.m_report_failure_to_fabric, - ), - (dsaz, "get_boot_telemetry", mock.MagicMock()), - (dsaz, "get_system_info", mock.MagicMock()), - ( - dsaz.net, - "get_interface_mac", - mock.MagicMock(return_value="00:15:5d:69:63:ba"), - ), - (dsaz.subp, "which", lambda x: True), - ( - dmi, - "read_dmi_data", - self.m_read_dmi_data, - ), - ( - dsaz.util, - "wait_for_files", - mock.MagicMock(side_effect=_wait_for_files), - ), - ] - ) + apply_patches( + [ + ( + dsaz, + "list_possible_azure_ds", + self.m_list_possible_azure_ds, + ), + ( + dsaz, + "get_metadata_from_fabric", + self.m_get_metadata_from_fabric, + ), + ( + dsaz, + "report_failure_to_fabric", + self.m_report_failure_to_fabric, + ), + (dsaz, "get_boot_telemetry", mock.MagicMock()), + (dsaz, "get_system_info", mock.MagicMock()), + ( + dsaz.net, + "get_interface_mac", + mock.MagicMock(return_value="00:15:5d:69:63:ba"), + ), + (dsaz.subp, "which", lambda x: True), + ( + dmi, + "read_dmi_data", + self.m_read_dmi_data, + ), + ( + dsaz.util, + "wait_for_files", + mock.MagicMock(side_effect=_wait_for_files), + ), + ] + ) - if isinstance(distro, str): - distro_cls = distros.fetch(distro) - distro = distro_cls(distro, data.get("sys_cfg", {}), self.paths) - distro.get_tmp_exec_path = mock.Mock(side_effect=self.tmp_dir) - dsrc = dsaz.DataSourceAzure( - data.get("sys_cfg", {}), distro=distro, paths=self.paths - ) - if apply_network is not None: - dsrc.ds_cfg["apply_network_config"] = apply_network + if isinstance(distro, str): + distro_cls = distros.fetch(distro) + distro = distro_cls(distro, data.get("sys_cfg", {}), paths) + distro.get_tmp_exec_path = mock.Mock(side_effect=str(tmp_path)) + dsrc = dsaz.DataSourceAzure( + data.get("sys_cfg", {}), distro=distro, paths=paths + ) + if apply_network is not None: + dsrc.ds_cfg["apply_network_config"] = apply_network - return dsrc + return dsrc + + return _get_ds def _get_and_setup(self, dsrc): ret = dsrc.get_data() @@ -1293,16 +1286,16 @@ def xml_notequals(self, oxml, nxml): return raise AssertionError("XML is the same") - def test_get_resource_disk(self): - ds = self._get_mockds() + def test_get_resource_disk(self, _get_mockds): + ds = _get_mockds dev = ds.get_resource_disk_on_freebsd(1) - self.assertEqual("da1", dev) + assert "da1" == dev - def test_not_ds_detect_seed_should_return_no_datasource(self): + def test_not_ds_detect_seed_should_return_no_datasource(self, get_ds): """Check seed_dir using ds_detect and return False.""" # Return a non-matching asset tag value data = {} - dsrc = self._get_ds(data) + dsrc = get_ds(data) self.m_read_dmi_data.side_effect = lambda x: "notazure" with mock.patch.object( dsrc, "crawl_metadata" @@ -1313,38 +1306,40 @@ def test_not_ds_detect_seed_should_return_no_datasource(self): assert self.m_read_dmi_data.mock_calls == [ mock.call("chassis-asset-tag") ] - self.assertFalse(ret) + assert not ret # Assert that for non viable platforms, # there is no communication with the Azure datasource. - self.assertEqual(0, m_crawl_metadata.call_count) - self.assertEqual(0, m_report_failure.call_count) + assert 0 == m_crawl_metadata.call_count + assert 0 == m_report_failure.call_count - def test_platform_viable_but_no_devs_should_return_no_datasource(self): + def test_platform_viable_but_no_devs_should_return_no_datasource( + self, get_ds, mocker + ): """For platforms where the Azure platform is viable (which is indicated by the matching asset tag), the absence of any devs at all (devs == candidate sources for crawling Azure datasource) is NOT expected. - Report failure to Azure as this is an unexpected fatal error. + Report failure to Azure as this is an unexpected error. """ data = {} - dsrc = self._get_ds(data) + dsrc = get_ds(data) + mocker.patch(MOCKPATH + "list_possible_azure_ds", return_value=[]) with mock.patch.object(dsrc, "_report_failure") as m_report_failure: - ret = dsrc.get_data() - self.assertFalse(ret) - self.assertEqual(1, m_report_failure.call_count) + assert dsrc.get_data() is True + assert 1 == m_report_failure.call_count - def test_crawl_metadata_exception_returns_no_datasource(self): + def test_crawl_metadata_exception_returns_no_datasource(self, get_ds): data = {} - dsrc = self._get_ds(data) + dsrc = get_ds(data) with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: m_crawl_metadata.side_effect = Exception ret = dsrc.get_data() - self.assertEqual(1, m_crawl_metadata.call_count) - self.assertFalse(ret) + assert 1 == m_crawl_metadata.call_count + assert not ret - def test_crawl_metadata_exception_should_report_failure(self): + def test_crawl_metadata_exception_should_report_failure(self, get_ds): data = {} - dsrc = self._get_ds(data) + dsrc = get_ds(data) with mock.patch.object( dsrc, "crawl_metadata" ) as m_crawl_metadata, mock.patch.object( @@ -1352,45 +1347,42 @@ def test_crawl_metadata_exception_should_report_failure(self): ) as m_report_failure: m_crawl_metadata.side_effect = Exception dsrc.get_data() - self.assertEqual(1, m_crawl_metadata.call_count) + assert 1 == m_crawl_metadata.call_count m_report_failure.assert_called_once_with(mock.ANY) - def test_crawl_metadata_exc_should_log_could_not_crawl_msg(self): + def test_crawl_metadata_exc_should_log_could_not_crawl_msg( + self, caplog, get_ds + ): data = {} - dsrc = self._get_ds(data) + dsrc = get_ds(data) with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: m_crawl_metadata.side_effect = Exception dsrc.get_data() - self.assertEqual(1, m_crawl_metadata.call_count) - self.assertIn( - "Azure datasource failure occurred:", self.logs.getvalue() - ) + assert 1 == m_crawl_metadata.call_count + assert "Azure datasource failure occurred:" in caplog.text - def test_basic_seed_dir(self): + def test_basic_seed_dir(self, get_ds, paths, waagent_d): data = { "ovfcontent": construct_ovf_env(hostname="myhost"), "sys_cfg": {}, } - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, "") - self.assertEqual(dsrc.metadata["local-hostname"], "myhost") - self.assertTrue( - os.path.isfile(os.path.join(self.waagent_d, "ovf-env.xml")) - ) - self.assertEqual("azure", dsrc.cloud_name) - self.assertEqual("azure", dsrc.platform_type) - self.assertEqual( - "seed-dir (%s/seed/azure)" % self.tmp, dsrc.subplatform - ) - - def test_data_dir_without_imds_data(self): + assert ret + assert dsrc.userdata_raw == "" + assert dsrc.metadata["local-hostname"] == "myhost" + assert os.path.isfile(os.path.join(waagent_d, "ovf-env.xml")) + assert "azure" == dsrc.cloud_name + assert "azure" == dsrc.platform_type + seed_dir = os.path.join(paths.seed_dir, "azure") + assert "seed-dir (%s)" % seed_dir == dsrc.subplatform + + def test_data_dir_without_imds_data(self, get_ds, waagent_d): data = { "ovfcontent": construct_ovf_env(hostname="myhost"), "sys_cfg": {}, } - dsrc = self._get_ds( + dsrc = get_ds( data, write_ovf_to_data_dir=True, write_ovf_to_seed_dir=False ) @@ -1402,20 +1394,18 @@ def test_data_dir_without_imds_data(self): ] ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, "") - self.assertEqual(dsrc.metadata["local-hostname"], "myhost") - self.assertTrue( - os.path.isfile(os.path.join(self.waagent_d, "ovf-env.xml")) - ) - self.assertEqual("azure", dsrc.cloud_name) - self.assertEqual("azure", dsrc.platform_type) - self.assertEqual("seed-dir (%s)" % self.waagent_d, dsrc.subplatform) + assert ret + assert dsrc.userdata_raw == "" + assert dsrc.metadata["local-hostname"] == "myhost" + assert os.path.isfile(os.path.join(waagent_d, "ovf-env.xml")) + assert "azure" == dsrc.cloud_name + assert "azure" == dsrc.platform_type + assert "seed-dir (%s)" % waagent_d == dsrc.subplatform - def test_basic_dev_file(self): + def test_basic_dev_file(self, get_ds): """When a device path is used, present that in subplatform.""" data = {"sys_cfg": {}, "dsdevs": ["/dev/cd0"]} - dsrc = self._get_ds(data) + dsrc = get_ds(data) # DSAzure will attempt to mount /dev/sr0 first, which should # fail with mount error since the list of devices doesn't have # /dev/sr0 @@ -1424,14 +1414,14 @@ def test_basic_dev_file(self): MountFailedError("fail"), ({"local-hostname": "me"}, "ud", {"cfg": ""}, {}), ] - self.assertTrue(dsrc.get_data()) - self.assertEqual(dsrc.userdata_raw, "ud") - self.assertEqual(dsrc.metadata["local-hostname"], "me") - self.assertEqual("azure", dsrc.cloud_name) - self.assertEqual("azure", dsrc.platform_type) - self.assertEqual("config-disk (/dev/cd0)", dsrc.subplatform) - - def test_get_data_non_ubuntu_will_not_remove_network_scripts(self): + assert dsrc.get_data() + assert dsrc.userdata_raw == "ud" + assert dsrc.metadata["local-hostname"] == "me" + assert "azure" == dsrc.cloud_name + assert "azure" == dsrc.platform_type + assert "config-disk (/dev/cd0)" == dsrc.subplatform + + def test_get_data_non_ubuntu_will_not_remove_network_scripts(self, get_ds): """get_data on non-Ubuntu will not remove ubuntu net scripts.""" data = { "ovfcontent": construct_ovf_env( @@ -1440,11 +1430,11 @@ def test_get_data_non_ubuntu_will_not_remove_network_scripts(self): "sys_cfg": {}, } - dsrc = self._get_ds(data, distro="debian") + dsrc = get_ds(data, distro="debian") dsrc.get_data() self.m_remove_ubuntu_network_scripts.assert_not_called() - def test_get_data_on_ubuntu_will_remove_network_scripts(self): + def test_get_data_on_ubuntu_will_remove_network_scripts(self, get_ds): """get_data will remove ubuntu net scripts on Ubuntu distro.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { @@ -1452,11 +1442,13 @@ def test_get_data_on_ubuntu_will_remove_network_scripts(self): "sys_cfg": sys_cfg, } - dsrc = self._get_ds(data, distro="ubuntu") + dsrc = get_ds(data, distro="ubuntu") dsrc.get_data() self.m_remove_ubuntu_network_scripts.assert_called_once_with() - def test_get_data_on_ubuntu_will_not_remove_network_scripts_disabled(self): + def test_get_data_on_ubuntu_will_not_remove_network_scripts_disabled( + self, get_ds + ): """When apply_network_config false, do not remove scripts on Ubuntu.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": False}}} data = { @@ -1464,11 +1456,13 @@ def test_get_data_on_ubuntu_will_not_remove_network_scripts_disabled(self): "sys_cfg": sys_cfg, } - dsrc = self._get_ds(data, distro="ubuntu") + dsrc = get_ds(data, distro="ubuntu") dsrc.get_data() self.m_remove_ubuntu_network_scripts.assert_not_called() - def test_crawl_metadata_returns_structured_data_and_caches_nothing(self): + def test_crawl_metadata_returns_structured_data_and_caches_nothing( + self, get_ds, waagent_d + ): """Return all structured metadata and cache no class attributes.""" data = { "ovfcontent": construct_ovf_env( @@ -1476,7 +1470,7 @@ def test_crawl_metadata_returns_structured_data_and_caches_nothing(self): ), "sys_cfg": {}, } - dsrc = self._get_ds(data) + dsrc = get_ds(data) expected_cfg = { "PreprovisionedVMType": None, "PreprovisionedVm": False, @@ -1492,46 +1486,40 @@ def test_crawl_metadata_returns_structured_data_and_caches_nothing(self): crawled_metadata = dsrc.crawl_metadata() - self.assertCountEqual( + assert_count_equal( crawled_metadata.keys(), ["cfg", "files", "metadata", "userdata_raw"], ) - self.assertEqual(crawled_metadata["cfg"], expected_cfg) - self.assertEqual( - list(crawled_metadata["files"].keys()), ["ovf-env.xml"] - ) - self.assertIn( - b"myhost", - crawled_metadata["files"]["ovf-env.xml"], - ) - self.assertEqual(crawled_metadata["metadata"], expected_metadata) - self.assertEqual(crawled_metadata["userdata_raw"], b"FOOBAR") - self.assertEqual(dsrc.userdata_raw, None) - self.assertEqual(dsrc.metadata, {}) - self.assertEqual(dsrc._metadata_imds, UNSET) - self.assertFalse( - os.path.isfile(os.path.join(self.waagent_d, "ovf-env.xml")) + assert crawled_metadata["cfg"] == expected_cfg + assert list(crawled_metadata["files"].keys()) == ["ovf-env.xml"] + assert ( + b"myhost" + in crawled_metadata["files"]["ovf-env.xml"] ) - - def test_crawl_metadata_raises_invalid_metadata_on_error(self): + assert crawled_metadata["metadata"] == expected_metadata + assert crawled_metadata["userdata_raw"] == b"FOOBAR" + assert dsrc.userdata_raw is None + assert dsrc.metadata == {} + assert dsrc._metadata_imds == UNSET + assert not os.path.isfile(os.path.join(waagent_d, "ovf-env.xml")) + + def test_crawl_metadata_raises_invalid_metadata_on_error(self, get_ds): """crawl_metadata raises an exception on invalid ovf-env.xml.""" data = {"ovfcontent": "BOGUS", "sys_cfg": {}} - dsrc = self._get_ds(data) + dsrc = get_ds(data) error_msg = "error parsing ovf-env.xml: syntax error: line 1, column 0" - with self.assertRaises( - errors.ReportableErrorOvfParsingException - ) as cm: + with pytest.raises(errors.ReportableErrorOvfParsingException) as cm: dsrc.crawl_metadata() - self.assertEqual(cm.exception.reason, error_msg) + assert cm.value.reason == error_msg - def test_crawl_metadata_call_imds_once_no_reprovision(self): + def test_crawl_metadata_call_imds_once_no_reprovision(self, get_ds): """If reprovisioning, report ready at the end""" ovfenv = construct_ovf_env(preprovisioned_vm=False) data = {"ovfcontent": ovfenv, "sys_cfg": {}} - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.crawl_metadata() - self.assertEqual(1, self.m_fetch.call_count) + assert 1 == self.m_fetch.call_count @mock.patch("cloudinit.sources.DataSourceAzure.util.write_file") @mock.patch( @@ -1539,26 +1527,26 @@ def test_crawl_metadata_call_imds_once_no_reprovision(self): ) @mock.patch("cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds") def test_crawl_metadata_call_imds_twice_with_reprovision( - self, poll_imds_func, m_report_ready, m_write + self, poll_imds_func, m_report_ready, m_write, get_ds ): """If reprovisioning, imds metadata will be fetched twice""" ovfenv = construct_ovf_env(preprovisioned_vm=True) data = {"ovfcontent": ovfenv, "sys_cfg": {}} - dsrc = self._get_ds(data) + dsrc = get_ds(data) poll_imds_func.return_value = ovfenv dsrc.crawl_metadata() - self.assertEqual(2, self.m_fetch.call_count) + assert 2 == self.m_fetch.call_count - def test_waagent_d_has_0700_perms(self): + def test_waagent_d_has_0700_perms(self, get_ds, waagent_d): # we expect /var/lib/waagent to be created 0700 - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(os.path.isdir(self.waagent_d)) - self.assertEqual(stat.S_IMODE(os.stat(self.waagent_d).st_mode), 0o700) + assert ret + assert os.path.isdir(waagent_d) + assert stat.S_IMODE(os.stat(waagent_d).st_mode) == 0o700 - def test_network_config_set_from_imds(self): + def test_network_config_set_from_imds(self, get_ds): """Datasource.network_config returns IMDS network data.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { @@ -1577,11 +1565,13 @@ def test_network_config_set_from_imds(self): }, "version": 2, } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertEqual(expected_network_config, dsrc.network_config) + assert expected_network_config == dsrc.network_config - def test_network_config_set_from_imds_route_metric_for_secondary_nic(self): + def test_network_config_set_from_imds_route_metric_for_secondary_nic( + self, get_ds + ): """Datasource.network_config adds route-metric to secondary nics.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { @@ -1623,11 +1613,13 @@ def test_network_config_set_from_imds_route_metric_for_secondary_nic(self): imds_data["network"]["interface"].append(third_intf) self.m_fetch.return_value = imds_data - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertEqual(expected_network_config, dsrc.network_config) + assert expected_network_config == dsrc.network_config - def test_network_config_set_from_imds_for_secondary_nic_no_ip(self): + def test_network_config_set_from_imds_for_secondary_nic_no_ip( + self, get_ds + ): """If an IP address is empty then there should no config for it.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { @@ -1649,33 +1641,33 @@ def test_network_config_set_from_imds_for_secondary_nic_no_ip(self): imds_data = copy.deepcopy(NETWORK_METADATA) imds_data["network"]["interface"].append(SECONDARY_INTERFACE_NO_IP) self.m_fetch.return_value = imds_data - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertEqual(expected_network_config, dsrc.network_config) + assert expected_network_config == dsrc.network_config - def test_availability_zone_set_from_imds(self): + def test_availability_zone_set_from_imds(self, get_ds): """Datasource.availability returns IMDS platformFaultDomain.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertEqual("0", dsrc.availability_zone) + assert "0" == dsrc.availability_zone - def test_region_set_from_imds(self): + def test_region_set_from_imds(self, get_ds): """Datasource.region returns IMDS region location.""" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertEqual("eastus2", dsrc.region) + assert "eastus2" == dsrc.region - def test_sys_cfg_set_never_destroy_ntfs(self): + def test_sys_cfg_set_never_destroy_ntfs(self, get_ds): sys_cfg = { "datasource": { "Azure": {"never_destroy_ntfs": "user-supplied-value"} @@ -1686,54 +1678,63 @@ def test_sys_cfg_set_never_destroy_ntfs(self): "sys_cfg": sys_cfg, } - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = self._get_and_setup(dsrc) - self.assertTrue(ret) - self.assertEqual( - dsrc.ds_cfg.get(dsaz.DS_CFG_KEY_PRESERVE_NTFS), - "user-supplied-value", + assert ret + assert ( + dsrc.ds_cfg.get(dsaz.DS_CFG_KEY_PRESERVE_NTFS) + == "user-supplied-value" ) - def test_username_used(self): + def test_no_admin_username(self, get_ds): + data = {"ovfcontent": construct_ovf_env(username=None)} + + dsrc = get_ds(data) + ret = dsrc.get_data() + assert ret + + assert dsrc.cfg == { + "PreprovisionedVMType": None, + "PreprovisionedVm": False, + "ProvisionGuestProxyAgent": False, + } + + def test_username_used(self, get_ds): data = {"ovfcontent": construct_ovf_env(username="myuser")} - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - dsrc.cfg["system_info"]["default_user"]["name"], "myuser" - ) + assert ret + assert dsrc.cfg["system_info"]["default_user"]["name"] == "myuser" assert "ssh_pwauth" not in dsrc.cfg - def test_password_given(self): + def test_password_given(self, get_ds): data = { "ovfcontent": construct_ovf_env( username="myuser", password="mypass" ) } - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) - self.assertIn("default_user", dsrc.cfg["system_info"]) + assert ret + assert "default_user" in dsrc.cfg["system_info"] defuser = dsrc.cfg["system_info"]["default_user"] # default user should be updated username and should not be locked. - self.assertEqual(defuser["name"], "myuser") - self.assertFalse(defuser["lock_passwd"]) + assert defuser["name"] == "myuser" + assert not defuser["lock_passwd"] # passwd is crypt formated string $id$salt$encrypted # encrypting plaintext with salt value of everything up to final '$' # should equal that after the '$' - self.assertTrue( - passlib.hash.sha512_crypt.verify( - "mypass", defuser["hashed_passwd"] - ) + assert passlib.hash.sha512_crypt.verify( + "mypass", defuser["hashed_passwd"] ) assert dsrc.cfg["ssh_pwauth"] is True - def test_password_with_disable_ssh_pw_auth_true(self): + def test_password_with_disable_ssh_pw_auth_true(self, get_ds): data = { "ovfcontent": construct_ovf_env( username="myuser", @@ -1742,12 +1743,12 @@ def test_password_with_disable_ssh_pw_auth_true(self): ) } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() assert dsrc.cfg["ssh_pwauth"] is False - def test_password_with_disable_ssh_pw_auth_false(self): + def test_password_with_disable_ssh_pw_auth_false(self, get_ds): data = { "ovfcontent": construct_ovf_env( username="myuser", @@ -1756,12 +1757,12 @@ def test_password_with_disable_ssh_pw_auth_false(self): ) } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() assert dsrc.cfg["ssh_pwauth"] is True - def test_password_with_disable_ssh_pw_auth_unspecified(self): + def test_password_with_disable_ssh_pw_auth_unspecified(self, get_ds): data = { "ovfcontent": construct_ovf_env( username="myuser", @@ -1770,12 +1771,12 @@ def test_password_with_disable_ssh_pw_auth_unspecified(self): ) } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() assert dsrc.cfg["ssh_pwauth"] is True - def test_no_password_with_disable_ssh_pw_auth_true(self): + def test_no_password_with_disable_ssh_pw_auth_true(self, get_ds): data = { "ovfcontent": construct_ovf_env( username="myuser", @@ -1783,12 +1784,12 @@ def test_no_password_with_disable_ssh_pw_auth_true(self): ) } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() assert dsrc.cfg["ssh_pwauth"] is False - def test_no_password_with_disable_ssh_pw_auth_false(self): + def test_no_password_with_disable_ssh_pw_auth_false(self, get_ds): data = { "ovfcontent": construct_ovf_env( username="myuser", @@ -1796,12 +1797,12 @@ def test_no_password_with_disable_ssh_pw_auth_false(self): ) } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() assert dsrc.cfg["ssh_pwauth"] is True - def test_no_password_with_disable_ssh_pw_auth_unspecified(self): + def test_no_password_with_disable_ssh_pw_auth_unspecified(self, get_ds): data = { "ovfcontent": construct_ovf_env( username="myuser", @@ -1809,12 +1810,12 @@ def test_no_password_with_disable_ssh_pw_auth_unspecified(self): ) } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() assert "ssh_pwauth" not in dsrc.cfg - def test_user_not_locked_if_password_redacted(self): + def test_user_not_locked_if_password_redacted(self, get_ds): data = { "ovfcontent": construct_ovf_env( username="myuser", @@ -1822,27 +1823,27 @@ def test_user_not_locked_if_password_redacted(self): ) } - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) - self.assertIn("default_user", dsrc.cfg["system_info"]) + assert ret + assert "default_user" in dsrc.cfg["system_info"] defuser = dsrc.cfg["system_info"]["default_user"] # default user should be updated username and should not be locked. - self.assertEqual(defuser["name"], "myuser") - self.assertIn("lock_passwd", defuser) - self.assertFalse(defuser["lock_passwd"]) + assert defuser["name"] == "myuser" + assert "lock_passwd" in defuser + assert not defuser["lock_passwd"] - def test_userdata_found(self): + def test_userdata_found(self, get_ds): mydata = "FOOBAR" data = {"ovfcontent": construct_ovf_env(custom_data=mydata)} - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, mydata.encode("utf-8")) + assert ret + assert dsrc.userdata_raw == mydata.encode("utf-8") - def test_default_ephemeral_configs_ephemeral_exists(self): + def test_default_ephemeral_configs_ephemeral_exists(self, get_ds): # make sure the ephemeral configs are correct if disk present data = { "ovfcontent": construct_ovf_env(), @@ -1857,21 +1858,21 @@ def changed_exists(path): ) with mock.patch(MOCKPATH + "os.path.exists", new=changed_exists): - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) + assert ret cfg = dsrc.get_config_obj() - self.assertEqual( - dsrc.device_name_to_device("ephemeral0"), - dsaz.RESOURCE_DISK_PATH, + assert ( + dsrc.device_name_to_device("ephemeral0") + == dsaz.RESOURCE_DISK_PATH ) assert "disk_setup" in cfg assert "fs_setup" in cfg - self.assertIsInstance(cfg["disk_setup"], dict) - self.assertIsInstance(cfg["fs_setup"], list) + assert isinstance(cfg["disk_setup"], dict) + assert isinstance(cfg["fs_setup"], list) - def test_default_ephemeral_configs_ephemeral_does_not_exist(self): + def test_default_ephemeral_configs_ephemeral_does_not_exist(self, get_ds): # make sure the ephemeral configs are correct if disk not present data = { "ovfcontent": construct_ovf_env(), @@ -1886,74 +1887,74 @@ def changed_exists(path): ) with mock.patch(MOCKPATH + "os.path.exists", new=changed_exists): - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) + assert ret cfg = dsrc.get_config_obj() assert "disk_setup" not in cfg assert "fs_setup" not in cfg - def test_userdata_arrives(self): + def test_userdata_arrives(self, get_ds): userdata = "This is my user-data" xml = construct_ovf_env(custom_data=userdata) data = {"ovfcontent": xml} - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertEqual(userdata.encode("us-ascii"), dsrc.userdata_raw) + assert userdata.encode("us-ascii") == dsrc.userdata_raw - def test_password_redacted_in_ovf(self): + def test_password_redacted_in_ovf(self, get_ds, waagent_d): data = { "ovfcontent": construct_ovf_env( username="myuser", password="mypass" ) } - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) - ovf_env_path = os.path.join(self.waagent_d, "ovf-env.xml") + assert ret + ovf_env_path = os.path.join(waagent_d, "ovf-env.xml") # The XML should not be same since the user password is redacted on_disk_ovf = load_text_file(ovf_env_path) self.xml_notequals(data["ovfcontent"], on_disk_ovf) # Make sure that the redacted password on disk is not used by CI - self.assertNotEqual( - dsrc.cfg.get("password"), dsaz.DEF_PASSWD_REDACTION - ) + assert dsrc.cfg.get("password") != dsaz.DEF_PASSWD_REDACTION # Make sure that the password was really encrypted et = ET.fromstring(on_disk_ovf) for elem in et.iter(): if "UserPassword" in elem.tag: - self.assertEqual(dsaz.DEF_PASSWD_REDACTION, elem.text) + assert dsaz.DEF_PASSWD_REDACTION == elem.text - def test_ovf_env_arrives_in_waagent_dir(self): + def test_ovf_env_arrives_in_waagent_dir(self, get_ds, waagent_d): xml = construct_ovf_env(custom_data="FOODATA") - dsrc = self._get_ds({"ovfcontent": xml}) + dsrc = get_ds({"ovfcontent": xml}) dsrc.get_data() # 'data_dir' is '/var/lib/waagent' (walinux-agent's state dir) # we expect that the ovf-env.xml file is copied there. - ovf_env_path = os.path.join(self.waagent_d, "ovf-env.xml") - self.assertTrue(os.path.exists(ovf_env_path)) + ovf_env_path = os.path.join(waagent_d, "ovf-env.xml") + assert os.path.exists(ovf_env_path) self.xml_equals(xml, load_text_file(ovf_env_path)) - def test_ovf_can_include_unicode(self): + def test_ovf_can_include_unicode(self, get_ds): xml = construct_ovf_env() xml = "\ufeff{0}".format(xml) - dsrc = self._get_ds({"ovfcontent": xml}) + dsrc = get_ds({"ovfcontent": xml}) dsrc.get_data() - def test_dsaz_report_ready_returns_true_when_report_succeeds(self): - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + def test_dsaz_report_ready_returns_true_when_report_succeeds(self, get_ds): + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) assert dsrc._report_ready() == [] @mock.patch(MOCKPATH + "report_diagnostic_event") - def test_dsaz_report_ready_failure_reports_telemetry(self, m_report_diag): - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + def test_dsaz_report_ready_failure_reports_telemetry( + self, m_report_diag, get_ds + ): + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) self.m_get_metadata_from_fabric.side_effect = Exception("foo") with pytest.raises(Exception): @@ -1967,21 +1968,23 @@ def test_dsaz_report_ready_failure_reports_telemetry(self, m_report_diag): ) ] - def test_dsaz_report_failure_returns_true_when_report_succeeds(self): - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + def test_dsaz_report_failure_returns_true_when_report_succeeds( + self, get_ds + ): + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: # mock crawl metadata failure to cause report failure m_crawl_metadata.side_effect = Exception error = errors.ReportableError(reason="foo") - self.assertTrue(dsrc._report_failure(error)) - self.assertEqual(1, self.m_report_failure_to_fabric.call_count) + assert dsrc._report_failure(error) + assert 1 == self.m_report_failure_to_fabric.call_count def test_dsaz_report_failure_returns_false_and_does_not_propagate_exc( - self, + self, get_ds ): - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object( dsrc, "crawl_metadata" @@ -2007,23 +2010,26 @@ def test_dsaz_report_failure_returns_false_and_does_not_propagate_exc( self.m_report_failure_to_fabric.side_effect = Exception error = errors.ReportableError(reason="foo") - self.assertFalse(dsrc._report_failure(error)) - self.assertEqual(2, self.m_report_failure_to_fabric.call_count) + assert not dsrc._report_failure(error) + assert 2 == self.m_report_failure_to_fabric.call_count - def test_dsaz_report_failure(self): - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + def test_dsaz_report_failure(self, get_ds): + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: m_crawl_metadata.side_effect = Exception error = errors.ReportableError(reason="foo") - self.assertTrue(dsrc._report_failure(error)) + assert dsrc._report_failure(error) self.m_report_failure_to_fabric.assert_called_once_with( - endpoint="168.63.129.16", error=error + endpoint="168.63.129.16", + encoded_report=error.as_encoded_report(vm_id=dsrc._vm_id), ) - def test_dsaz_report_failure_uses_cached_ephemeral_dhcp_ctx_lease(self): - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + def test_dsaz_report_failure_uses_cached_ephemeral_dhcp_ctx_lease( + self, get_ds + ): + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object( dsrc, "crawl_metadata" @@ -2034,15 +2040,18 @@ def test_dsaz_report_failure_uses_cached_ephemeral_dhcp_ctx_lease(self): m_crawl_metadata.side_effect = Exception error = errors.ReportableError(reason="foo") - self.assertTrue(dsrc._report_failure(error)) + assert dsrc._report_failure(error) # ensure called with cached ephemeral dhcp lease option 245 self.m_report_failure_to_fabric.assert_called_once_with( - endpoint="test-ep", error=error + endpoint="test-ep", + encoded_report=error.as_encoded_report(vm_id=dsrc._vm_id), ) - def test_dsaz_report_failure_no_net_uses_new_ephemeral_dhcp_lease(self): - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + def test_dsaz_report_failure_no_net_uses_new_ephemeral_dhcp_lease( + self, get_ds + ): + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) with mock.patch.object(dsrc, "crawl_metadata") as m_crawl_metadata: # mock crawl metadata failure to cause report failure @@ -2056,85 +2065,83 @@ def test_dsaz_report_failure_no_net_uses_new_ephemeral_dhcp_lease(self): self.m_dhcp.return_value.obtain_lease.return_value = test_lease error = errors.ReportableError(reason="foo") - self.assertTrue(dsrc._report_failure(error)) + assert dsrc._report_failure(error) # ensure called with the newly discovered # ephemeral dhcp lease option 245 self.m_report_failure_to_fabric.assert_called_once_with( - endpoint="1.2.3.4", error=error + endpoint="1.2.3.4", + encoded_report=error.as_encoded_report(vm_id=dsrc._vm_id), ) - def test_exception_fetching_fabric_data_doesnt_propagate(self): + def test_exception_fetching_fabric_data_doesnt_propagate(self, get_ds): """Errors communicating with fabric should warn, but return True.""" - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) self.m_get_metadata_from_fabric.side_effect = Exception ret = self._get_and_setup(dsrc) - self.assertTrue(ret) + assert ret - def test_fabric_data_included_in_metadata(self): - dsrc = self._get_ds({"ovfcontent": construct_ovf_env()}) + def test_fabric_data_included_in_metadata(self, get_ds): + dsrc = get_ds({"ovfcontent": construct_ovf_env()}) self.m_get_metadata_from_fabric.return_value = ["ssh-key-value"] ret = self._get_and_setup(dsrc) - self.assertTrue(ret) - self.assertEqual(["ssh-key-value"], dsrc.metadata["public-keys"]) + assert ret + assert ["ssh-key-value"] == dsrc.metadata["public-keys"] - def test_instance_id_case_insensitive(self): + def test_instance_id_case_insensitive(self, get_ds, paths): """Return the previous iid when current is a case-insensitive match.""" lower_iid = EXAMPLE_UUID.lower() upper_iid = EXAMPLE_UUID.upper() # lowercase current UUID - ds = self._get_ds( - {"ovfcontent": construct_ovf_env()}, instance_id=lower_iid - ) + ds = get_ds({"ovfcontent": construct_ovf_env()}, instance_id=lower_iid) # UPPERCASE previous write_file( - os.path.join(self.paths.cloud_dir, "data", "instance-id"), + os.path.join(paths.cloud_dir, "data", "instance-id"), upper_iid, ) ds.get_data() - self.assertEqual(upper_iid, ds.metadata["instance-id"]) + assert upper_iid == ds.metadata["instance-id"] # UPPERCASE current UUID - ds = self._get_ds( - {"ovfcontent": construct_ovf_env()}, instance_id=upper_iid - ) + ds = get_ds({"ovfcontent": construct_ovf_env()}, instance_id=upper_iid) # lowercase previous write_file( - os.path.join(self.paths.cloud_dir, "data", "instance-id"), + os.path.join(paths.cloud_dir, "data", "instance-id"), lower_iid, ) ds.get_data() - self.assertEqual(lower_iid, ds.metadata["instance-id"]) + assert lower_iid == ds.metadata["instance-id"] - def test_instance_id_endianness(self): + def test_instance_id_endianness(self, get_ds, paths): """Return the previous iid when dmi uuid is the byteswapped iid.""" - ds = self._get_ds({"ovfcontent": construct_ovf_env()}) + ds = get_ds({"ovfcontent": construct_ovf_env()}) # byte-swapped previous write_file( - os.path.join(self.paths.cloud_dir, "data", "instance-id"), + os.path.join(paths.cloud_dir, "data", "instance-id"), "544CDFD0-CB4E-4B4A-9954-5BDF3ED5C3B8", ) ds.get_data() - self.assertEqual( - "544CDFD0-CB4E-4B4A-9954-5BDF3ED5C3B8", ds.metadata["instance-id"] + assert ( + "544CDFD0-CB4E-4B4A-9954-5BDF3ED5C3B8" + == ds.metadata["instance-id"] ) # not byte-swapped previous write_file( - os.path.join(self.paths.cloud_dir, "data", "instance-id"), + os.path.join(paths.cloud_dir, "data", "instance-id"), "644CDFD0-CB4E-4B4A-9954-5BDF3ED5C3B8", ) ds.get_data() - self.assertEqual(self.instance_id, ds.metadata["instance-id"]) + assert self.instance_id == ds.metadata["instance-id"] - def test_instance_id_from_dmidecode_used(self): - ds = self._get_ds({"ovfcontent": construct_ovf_env()}) + def test_instance_id_from_dmidecode_used(self, get_ds): + ds = get_ds({"ovfcontent": construct_ovf_env()}) ds.get_data() - self.assertEqual(self.instance_id, ds.metadata["instance-id"]) + assert self.instance_id == ds.metadata["instance-id"] - def test_instance_id_from_dmidecode_used_for_builtin(self): - ds = self._get_ds({"ovfcontent": construct_ovf_env()}) + def test_instance_id_from_dmidecode_used_for_builtin(self, get_ds): + ds = get_ds({"ovfcontent": construct_ovf_env()}) ds.get_data() - self.assertEqual(self.instance_id, ds.metadata["instance-id"]) + assert self.instance_id == ds.metadata["instance-id"] @mock.patch(MOCKPATH + "util.is_FreeBSD") @mock.patch(MOCKPATH + "_check_freebsd_cdrom") @@ -2145,21 +2152,18 @@ def test_list_possible_azure_ds(self, m_check_fbsd_cdrom, m_is_FreeBSD): possible_ds = [] for src in dsaz.list_possible_azure_ds("seed_dir", "cache_dir"): possible_ds.append(src) - self.assertEqual( - possible_ds, - [ - "seed_dir", - dsaz.DEFAULT_PROVISIONING_ISO_DEV, - "/dev/cd0", - "cache_dir", - ], - ) - self.assertEqual( - [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list - ) + assert possible_ds == [ + "seed_dir", + dsaz.DEFAULT_PROVISIONING_ISO_DEV, + "/dev/cd0", + "cache_dir", + ] + assert [mock.call("/dev/cd0")] == m_check_fbsd_cdrom.call_args_list @mock.patch(MOCKPATH + "net.get_interfaces") - def test_blacklist_through_distro(self, m_net_get_interfaces): + def test_blacklist_through_distro( + self, m_net_get_interfaces, get_ds, paths + ): """Verify Azure DS updates blacklist drivers in the distro's networking object.""" data = { @@ -2168,8 +2172,8 @@ def test_blacklist_through_distro(self, m_net_get_interfaces): } distro_cls = distros.fetch("ubuntu") - distro = distro_cls("ubuntu", {}, self.paths) - dsrc = self._get_ds(data, distro=distro) + distro = distro_cls("ubuntu", {}, paths) + dsrc = get_ds(data, distro=distro) dsrc.get_data() distro.networking.get_interfaces_by_mac() @@ -2178,18 +2182,18 @@ def test_blacklist_through_distro(self, m_net_get_interfaces): @mock.patch( "cloudinit.sources.helpers.azure.OpenSSLManager.parse_certificates" ) - def test_get_public_ssh_keys_with_imds(self, m_parse_certificates): + def test_get_public_ssh_keys_with_imds(self, m_parse_certificates, get_ds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() dsrc.setup(True) ssh_keys = dsrc.get_public_ssh_keys() - self.assertEqual(ssh_keys, ["ssh-rsa key1"]) - self.assertEqual(m_parse_certificates.call_count, 0) + assert ssh_keys == ["ssh-rsa key1"] + assert m_parse_certificates.call_count == 0 def test_key_without_crlf_valid(self): test_key = "ssh-rsa somerandomkeystuff some comment" @@ -2207,7 +2211,7 @@ def test_key_endswith_crlf_valid(self): "cloudinit.sources.helpers.azure.OpenSSLManager.parse_certificates" ) def test_get_public_ssh_keys_with_no_openssh_format( - self, m_parse_certificates + self, m_parse_certificates, get_ds ): imds_data = copy.deepcopy(NETWORK_METADATA) imds_data["compute"]["publicKeys"][0]["keyData"] = "no-openssh-format" @@ -2217,28 +2221,28 @@ def test_get_public_ssh_keys_with_no_openssh_format( "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() dsrc.setup(True) ssh_keys = dsrc.get_public_ssh_keys() - self.assertEqual(ssh_keys, []) - self.assertEqual(m_parse_certificates.call_count, 0) + assert ssh_keys == [] + assert m_parse_certificates.call_count == 0 - def test_get_public_ssh_keys_without_imds(self): + def test_get_public_ssh_keys_without_imds(self, get_ds): self.m_fetch.return_value = dict() sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), "sys_cfg": sys_cfg, } - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsaz.get_metadata_from_fabric.return_value = ["key2"] dsrc.get_data() dsrc.setup(True) ssh_keys = dsrc.get_public_ssh_keys() - self.assertEqual(ssh_keys, ["key2"]) + assert ssh_keys == ["key2"] - def test_hostname_from_imds(self): + def test_hostname_from_imds(self, get_ds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), @@ -2251,11 +2255,11 @@ def test_hostname_from_imds(self): disablePasswordAuthentication="true", ) self.m_fetch.return_value = imds_data_with_os_profile - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertEqual(dsrc.metadata["local-hostname"], "hostname1") + assert dsrc.metadata["local-hostname"] == "hostname1" - def test_username_from_imds(self): + def test_username_from_imds(self, get_ds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), @@ -2268,13 +2272,11 @@ def test_username_from_imds(self): disablePasswordAuthentication="true", ) self.m_fetch.return_value = imds_data_with_os_profile - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertEqual( - dsrc.cfg["system_info"]["default_user"]["name"], "username1" - ) + assert dsrc.cfg["system_info"]["default_user"]["name"] == "username1" - def test_disable_password_from_imds_true(self): + def test_disable_password_from_imds_true(self, get_ds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), @@ -2287,11 +2289,11 @@ def test_disable_password_from_imds_true(self): disablePasswordAuthentication="true", ) self.m_fetch.return_value = imds_data_with_os_profile - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertFalse(dsrc.cfg["ssh_pwauth"]) + assert not dsrc.cfg["ssh_pwauth"] - def test_disable_password_from_imds_false(self): + def test_disable_password_from_imds_false(self, get_ds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), @@ -2305,11 +2307,11 @@ def test_disable_password_from_imds_false(self): disablePasswordAuthentication="false", ) self.m_fetch.return_value = imds_data_with_os_profile - dsrc = self._get_ds(data) + dsrc = get_ds(data) dsrc.get_data() - self.assertTrue(dsrc.cfg["ssh_pwauth"]) + assert dsrc.cfg["ssh_pwauth"] - def test_userdata_from_imds(self): + def test_userdata_from_imds(self, get_ds): sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { "ovfcontent": construct_ovf_env(), @@ -2324,12 +2326,12 @@ def test_userdata_from_imds(self): ) imds_data["compute"]["userData"] = b64e(userdata) self.m_fetch.return_value = imds_data - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, userdata.encode("utf-8")) + assert ret + assert dsrc.userdata_raw == userdata.encode("utf-8") - def test_userdata_from_imds_with_customdata_from_OVF(self): + def test_userdata_from_imds_with_customdata_from_OVF(self, get_ds): userdataOVF = "userdataOVF" sys_cfg = {"datasource": {"Azure": {"apply_network_config": True}}} data = { @@ -2346,116 +2348,157 @@ def test_userdata_from_imds_with_customdata_from_OVF(self): ) imds_data["compute"]["userData"] = b64e(userdataImds) self.m_fetch.return_value = imds_data - dsrc = self._get_ds(data) + dsrc = get_ds(data) ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, userdataOVF.encode("utf-8")) + assert ret + assert dsrc.userdata_raw == userdataOVF.encode("utf-8") + + @pytest.mark.usefixtures("fake_filesystem") + def test_cleanup_resourcedisk_fstab(self, get_ds): + """Ensure that cloud-init clean will remove resource disk entries + from /etc/fstab""" + fstab_original_content = ( + "UUID=abc123 / ext4 defaults 0 0\n" + "/dev/disk/cloud/azure_resource-part1 /mnt " + "auto defaults,nofail,x-systemd.after=" + "cloud-init.service,_netdev,comment=cloudconfig 0 2\n" + ) + fstab_expected_content = "UUID=abc123 / ext4 defaults 0 0\n" + etc_path = "/etc" + if not os.path.exists(etc_path): + os.makedirs(etc_path) + fstab_path = cc_mounts.FSTAB_PATH + with open(fstab_path, "w") as fd: + fd.write(fstab_original_content) -class TestLoadAzureDsDir(CiTestCase): - """Tests for load_azure_ds_dir.""" + data = {} + dsrc = get_ds(data) + dsrc.clean() - def setUp(self): - self.source_dir = self.tmp_dir() - super(TestLoadAzureDsDir, self).setUp() + with open(fstab_path, "r") as fd: + fstab_new_content = fd.read() + assert fstab_expected_content == fstab_new_content - def test_missing_ovf_env_xml_raises_non_azure_datasource_error(self): + +class TestLoadAzureDsDir: + """Tests for load_azure_ds_dir.""" + + def test_missing_ovf_env_xml_raises_non_azure_datasource_error( + self, tmp_path + ): """load_azure_ds_dir raises an error When ovf-env.xml doesn't exit.""" - with self.assertRaises(dsaz.NonAzureDataSource) as context_manager: - dsaz.load_azure_ds_dir(self.source_dir) - self.assertEqual( - "No ovf-env file found", str(context_manager.exception) - ) + with pytest.raises( + dsaz.NonAzureDataSource, match="No ovf-env file found" + ): + dsaz.load_azure_ds_dir(str(tmp_path)) - def test_wb_invalid_ovf_env_xml_calls_read_azure_ovf(self): + def test_wb_invalid_ovf_env_xml_calls_read_azure_ovf(self, tmp_path): """load_azure_ds_dir calls read_azure_ovf to parse the xml.""" - ovf_path = os.path.join(self.source_dir, "ovf-env.xml") + ovf_path = os.path.join(str(tmp_path), "ovf-env.xml") with open(ovf_path, "wb") as stream: stream.write(b"invalid xml") - with self.assertRaises( - errors.ReportableErrorOvfParsingException - ) as context_manager: - dsaz.load_azure_ds_dir(self.source_dir) - self.assertEqual( - "error parsing ovf-env.xml: syntax error: line 1, column 0", - context_manager.exception.reason, + with pytest.raises(errors.ReportableErrorOvfParsingException) as cm: + dsaz.load_azure_ds_dir(str(tmp_path)) + assert ( + "error parsing ovf-env.xml: syntax error: line 1, column 0" + == cm.value.reason ) -class TestReadAzureOvf(CiTestCase): +class TestReadAzureOvf: def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_ovf_env() - self.assertRaises( - errors.ReportableErrorOvfParsingException, - dsaz.read_azure_ovf, - invalid_xml, - ) + with pytest.raises(errors.ReportableErrorOvfParsingException): + dsaz.read_azure_ovf( + invalid_xml, + ) def test_load_with_pubkeys(self): public_keys = [{"fingerprint": "fp1", "path": "path1", "value": ""}] content = construct_ovf_env(public_keys=public_keys) (_md, _ud, cfg) = dsaz.read_azure_ovf(content) for pk in public_keys: - self.assertIn(pk, cfg["_pubkeys"]) + assert pk in cfg["_pubkeys"] -class TestCanDevBeReformatted(CiTestCase): - with_logs = True +class TestCanDevBeReformatted: warning_file = "dataloss_warning_readme.txt" - def _domock(self, mockpath, sattr=None): - patcher = mock.patch(mockpath) - setattr(self, sattr, patcher.start()) - self.addCleanup(patcher.stop) - - def patchup(self, devs): - bypath = {} - for path, data in devs.items(): - bypath[path] = data - if "realpath" in data: - bypath[data["realpath"]] = data - for ppath, pdata in data.get("partitions", {}).items(): - bypath[ppath] = pdata + @pytest.fixture + def patchup(self, mocker): + def _patchup(devs): + bypath = {} + for path, data in devs.items(): + bypath[path] = data if "realpath" in data: - bypath[pdata["realpath"]] = pdata - - def realpath(d): - return bypath[d].get("realpath", d) - - def partitions_on_device(devpath): - parts = bypath.get(devpath, {}).get("partitions", {}) - ret = [] - for path, data in parts.items(): - ret.append((data.get("num"), realpath(path))) - # return sorted by partition number - return sorted(ret, key=lambda d: d[0]) - - def has_ntfs_fs(device): - return bypath.get(device, {}).get("fs") == "ntfs" - - p = MOCKPATH - self._domock(p + "_partitions_on_device", "m_partitions_on_device") - self._domock(p + "_has_ntfs_filesystem", "m_has_ntfs_filesystem") - self._domock(p + "os.path.realpath", "m_realpath") - self._domock(p + "os.path.exists", "m_exists") - self._domock(p + "util.SeLinuxGuard", "m_selguard") - - self.m_exists.side_effect = lambda p: p in bypath - self.m_realpath.side_effect = realpath - self.m_has_ntfs_filesystem.side_effect = has_ntfs_fs - self.m_partitions_on_device.side_effect = partitions_on_device - self.m_selguard.__enter__ = mock.Mock(return_value=False) - self.m_selguard.__exit__ = mock.Mock() - - return bypath + bypath[data["realpath"]] = data + for ppath, pdata in data.get("partitions", {}).items(): + bypath[ppath] = pdata + if "realpath" in data: + bypath[pdata["realpath"]] = pdata + + def realpath(d): + return bypath[d].get("realpath", d) + + def partitions_on_device(devpath): + parts = bypath.get(devpath, {}).get("partitions", {}) + ret = [] + for path, data in parts.items(): + ret.append((data.get("num"), realpath(path))) + # return sorted by partition number + return sorted(ret, key=lambda d: d[0]) + + def has_ntfs_fs(device): + return bypath.get(device, {}).get("fs") == "ntfs" + + p = MOCKPATH + self.m_partitions_on_device = mocker.patch( + p + "_partitions_on_device" + ) + self.m_has_ntfs_filesystem = mocker.patch( + p + "_has_ntfs_filesystem" + ) + self.m_realpath = mocker.patch(p + "os.path.realpath") + self.m_exists = mocker.patch(p + "os.path.exists") + self.m_selguard = mocker.patch(p + "util.SeLinuxGuard") + + self.m_exists.side_effect = lambda p: p in bypath + self.m_realpath.side_effect = realpath + self.m_has_ntfs_filesystem.side_effect = has_ntfs_fs + self.m_partitions_on_device.side_effect = partitions_on_device + self.m_selguard.__enter__ = mock.Mock(return_value=False) + self.m_selguard.__exit__ = mock.Mock() + + return bypath + + return _patchup + + @pytest.fixture + def domock_mount_cb(self, mocker, tmp_path): + def _do_mock_mount_cb(bypath): + def mount_cb( + device, callback, mtype, update_env_for_mount, log_error=False + ): + assert "ntfs" == mtype + assert "C" == update_env_for_mount.get("LANG") + for f in bypath.get(device).get("files", []): + write_file(os.path.join(tmp_path, f), content=f) + return callback(str(tmp_path)) + + p = MOCKPATH + self.m_mount_cb = mocker.patch(p + "util.mount_cb") + self.m_mount_cb.side_effect = mount_cb + + return _do_mock_mount_cb M_PATH = "cloudinit.util." @mock.patch(M_PATH + "subp.subp") - def test_ntfs_mount_logs(self, m_subp): + def test_ntfs_mount_logs(self, m_subp, caplog, patchup): """can_dev_be_reformatted does not log errors in case of unknown filesystem 'ntfs'.""" - self.patchup( + patchup( { "/dev/sda": { "partitions": { @@ -2472,26 +2515,11 @@ def test_ntfs_mount_logs(self, m_subp): ) dsaz.can_dev_be_reformatted("/dev/sda", preserve_ntfs=False) - self.assertNotIn(log_msg, self.logs.getvalue()) + assert log_msg not in caplog.text - def _domock_mount_cb(self, bypath): - def mount_cb( - device, callback, mtype, update_env_for_mount, log_error=False - ): - self.assertEqual("ntfs", mtype) - self.assertEqual("C", update_env_for_mount.get("LANG")) - p = self.tmp_dir() - for f in bypath.get(device).get("files", []): - write_file(os.path.join(p, f), content=f) - return callback(p) - - p = MOCKPATH - self._domock(p + "util.mount_cb", "m_mount_cb") - self.m_mount_cb.side_effect = mount_cb - - def test_three_partitions_is_false(self): + def test_three_partitions_is_false(self, domock_mount_cb, patchup): """A disk with 3 partitions can not be formatted.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2502,26 +2530,26 @@ def test_three_partitions_is_false(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertFalse(value) - self.assertIn("3 or more", msg.lower()) + assert not value + assert "3 or more" in msg.lower() - def test_no_partitions_is_false(self): + def test_no_partitions_is_false(self, patchup, domock_mount_cb): """A disk with no partitions can not be formatted.""" - bypath = self.patchup({"/dev/sda": {}}) - self._domock_mount_cb(bypath) + bypath = patchup({"/dev/sda": {}}) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertFalse(value) - self.assertIn("not partitioned", msg.lower()) + assert not value + assert "not partitioned" in msg.lower() - def test_two_partitions_not_ntfs_false(self): + def test_two_partitions_not_ntfs_false(self, patchup, domock_mount_cb): """2 partitions and 2nd not ntfs can not be formatted.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2531,16 +2559,18 @@ def test_two_partitions_not_ntfs_false(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertFalse(value) - self.assertIn("not ntfs", msg.lower()) + assert not value + assert "not ntfs" in msg.lower() - def test_two_partitions_ntfs_populated_false(self): + def test_two_partitions_ntfs_populated_false( + self, patchup, domock_mount_cb + ): """2 partitions and populated ntfs fs on 2nd can not be formatted.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2554,16 +2584,16 @@ def test_two_partitions_ntfs_populated_false(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertFalse(value) - self.assertIn("files on it", msg.lower()) + assert not value + assert "files on it" in msg.lower() - def test_two_partitions_ntfs_empty_is_true(self): + def test_two_partitions_ntfs_empty_is_true(self, patchup, domock_mount_cb): """2 partitions and empty ntfs fs on 2nd can be formatted.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2573,16 +2603,16 @@ def test_two_partitions_ntfs_empty_is_true(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertTrue(value) - self.assertIn("safe for", msg.lower()) + assert value + assert "safe for" in msg.lower() - def test_one_partition_not_ntfs_false(self): + def test_one_partition_not_ntfs_false(self, patchup, domock_mount_cb): """1 partition witih fs other than ntfs can not be formatted.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2591,16 +2621,18 @@ def test_one_partition_not_ntfs_false(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertFalse(value) - self.assertIn("not ntfs", msg.lower()) + assert not value + assert "not ntfs" in msg.lower() - def test_one_partition_ntfs_populated_false(self): + def test_one_partition_ntfs_populated_false( + self, patchup, domock_mount_cb + ): """1 mountable ntfs partition with many files can not be formatted.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2613,21 +2645,19 @@ def test_one_partition_ntfs_populated_false(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) with mock.patch.object(dsaz.LOG, "warning") as warning: value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) wmsg = warning.call_args[0][0] - self.assertIn( - "looks like you're using NTFS on the ephemeral disk", wmsg - ) - self.assertFalse(value) - self.assertIn("files on it", msg.lower()) + assert "looks like you're using NTFS on the ephemeral disk" in wmsg + assert not value + assert "files on it" in msg.lower() - def test_one_partition_ntfs_empty_is_true(self): + def test_one_partition_ntfs_empty_is_true(self, patchup, domock_mount_cb): """1 mountable ntfs partition and no files can be formatted.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2636,16 +2666,18 @@ def test_one_partition_ntfs_empty_is_true(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertTrue(value) - self.assertIn("safe for", msg.lower()) + assert value + assert "safe for" in msg.lower() - def test_one_partition_ntfs_empty_with_dataloss_file_is_true(self): + def test_one_partition_ntfs_empty_with_dataloss_file_is_true( + self, patchup, domock_mount_cb + ): """1 mountable ntfs partition and only warn file can be formatted.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2658,16 +2690,18 @@ def test_one_partition_ntfs_empty_with_dataloss_file_is_true(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertTrue(value) - self.assertIn("safe for", msg.lower()) + assert value + assert "safe for" in msg.lower() - def test_one_partition_ntfs_empty_with_svi_file_is_true(self): + def test_one_partition_ntfs_empty_with_svi_file_is_true( + self, patchup, domock_mount_cb + ): """1 mountable ntfs partition and only warn file can be formatted.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2680,17 +2714,19 @@ def test_one_partition_ntfs_empty_with_svi_file_is_true(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertTrue(value) - self.assertIn("safe for", msg.lower()) + assert value + assert "safe for" in msg.lower() - def test_one_partition_through_realpath_is_true(self): + def test_one_partition_through_realpath_is_true( + self, patchup, domock_mount_cb + ): """A symlink to a device with 1 ntfs partition can be formatted.""" epath = "/dev/disk/cloud/azure_resource" - bypath = self.patchup( + bypath = patchup( { epath: { "realpath": "/dev/sdb", @@ -2706,15 +2742,17 @@ def test_one_partition_through_realpath_is_true(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted(epath, preserve_ntfs=False) - self.assertTrue(value) - self.assertIn("safe for", msg.lower()) + assert value + assert "safe for" in msg.lower() - def test_three_partition_through_realpath_is_false(self): + def test_three_partition_through_realpath_is_false( + self, patchup, domock_mount_cb + ): """A symlink to a device with 3 partitions can not be formatted.""" epath = "/dev/disk/cloud/azure_resource" - bypath = self.patchup( + bypath = patchup( { epath: { "realpath": "/dev/sdb", @@ -2742,14 +2780,14 @@ def test_three_partition_through_realpath_is_false(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted(epath, preserve_ntfs=False) - self.assertFalse(value) - self.assertIn("3 or more", msg.lower()) + assert not value + assert "3 or more" in msg.lower() - def test_ntfs_mount_errors_true(self): + def test_ntfs_mount_errors_true(self, patchup, domock_mount_cb): """can_dev_be_reformatted does not fail if NTFS is unknown fstype.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2758,7 +2796,7 @@ def test_ntfs_mount_errors_true(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) error_msgs = [ "Stderr: mount: unknown filesystem type 'ntfs'", # RHEL @@ -2774,12 +2812,12 @@ def test_ntfs_mount_errors_true(self): value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=False ) - self.assertTrue(value) - self.assertIn("cannot mount NTFS, assuming", msg) + assert value + assert "cannot mount NTFS, assuming" in msg - def test_never_destroy_ntfs_config_false(self): + def test_never_destroy_ntfs_config_false(self, patchup, domock_mount_cb): """Normally formattable situation with never_destroy_ntfs set.""" - bypath = self.patchup( + bypath = patchup( { "/dev/sda": { "partitions": { @@ -2792,23 +2830,20 @@ def test_never_destroy_ntfs_config_false(self): } } ) - self._domock_mount_cb(bypath) + domock_mount_cb(bypath) value, msg = dsaz.can_dev_be_reformatted( "/dev/sda", preserve_ntfs=True ) - self.assertFalse(value) - self.assertIn( + assert not value + assert ( "config says to never destroy NTFS " - "(datasource.Azure.never_destroy_ntfs)", - msg, + "(datasource.Azure.never_destroy_ntfs)" in msg ) -class TestClearCachedData(CiTestCase): - def test_clear_cached_attrs_clears_imds(self): +class TestClearCachedData: + def test_clear_cached_attrs_clears_imds(self, paths): """All class attributes are reset to defaults, including imds data.""" - tmp = self.tmp_dir() - paths = helpers.Paths({"cloud_dir": tmp, "run_dir": tmp}) dsrc = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=paths) clean_values = [dsrc.metadata, dsrc.userdata, dsrc._metadata_imds] dsrc.metadata = "md" @@ -2816,26 +2851,28 @@ def test_clear_cached_attrs_clears_imds(self): dsrc._metadata_imds = "imds" dsrc._dirty_cache = True dsrc.clear_cached_attrs() - self.assertEqual( - [dsrc.metadata, dsrc.userdata, dsrc._metadata_imds], clean_values - ) + assert [ + dsrc.metadata, + dsrc.userdata, + dsrc._metadata_imds, + ] == clean_values -class TestAzureNetExists(CiTestCase): +class TestAzureNetExists: def test_azure_net_must_exist_for_legacy_objpkl(self): """DataSourceAzureNet must exist for old obj.pkl files that reference it.""" - self.assertTrue(hasattr(dsaz, "DataSourceAzureNet")) + assert hasattr(dsaz, "DataSourceAzureNet") -class TestPreprovisioningReadAzureOvfFlag(CiTestCase): +class TestPreprovisioningReadAzureOvfFlag: def test_read_azure_ovf_with_true_flag(self): """The read_azure_ovf method should set the PreprovisionedVM cfg flag if the proper setting is present.""" content = construct_ovf_env(preprovisioned_vm=True) ret = dsaz.read_azure_ovf(content) cfg = ret[2] - self.assertTrue(cfg["PreprovisionedVm"]) + assert cfg["PreprovisionedVm"] def test_read_azure_ovf_with_false_flag(self): """The read_azure_ovf method should set the PreprovisionedVM @@ -2843,7 +2880,7 @@ def test_read_azure_ovf_with_false_flag(self): content = construct_ovf_env(preprovisioned_vm=False) ret = dsaz.read_azure_ovf(content) cfg = ret[2] - self.assertFalse(cfg["PreprovisionedVm"]) + assert not cfg["PreprovisionedVm"] def test_read_azure_ovf_without_flag(self): """The read_azure_ovf method should not set the @@ -2851,8 +2888,8 @@ def test_read_azure_ovf_without_flag(self): content = construct_ovf_env() ret = dsaz.read_azure_ovf(content) cfg = ret[2] - self.assertFalse(cfg["PreprovisionedVm"]) - self.assertEqual(None, cfg["PreprovisionedVMType"]) + assert not cfg["PreprovisionedVm"] + assert None is cfg["PreprovisionedVMType"] def test_read_azure_ovf_with_running_type(self): """The read_azure_ovf method should set PreprovisionedVMType @@ -2862,8 +2899,8 @@ def test_read_azure_ovf_with_running_type(self): ) ret = dsaz.read_azure_ovf(content) cfg = ret[2] - self.assertTrue(cfg["PreprovisionedVm"]) - self.assertEqual("Running", cfg["PreprovisionedVMType"]) + assert cfg["PreprovisionedVm"] + assert "Running" == cfg["PreprovisionedVMType"] def test_read_azure_ovf_with_savable_type(self): """The read_azure_ovf method should set PreprovisionedVMType @@ -2873,8 +2910,8 @@ def test_read_azure_ovf_with_savable_type(self): ) ret = dsaz.read_azure_ovf(content) cfg = ret[2] - self.assertTrue(cfg["PreprovisionedVm"]) - self.assertEqual("Savable", cfg["PreprovisionedVMType"]) + assert cfg["PreprovisionedVm"] + assert "Savable" == cfg["PreprovisionedVMType"] def test_read_azure_ovf_with_proxy_guest_agent_true(self): """The read_azure_ovf method should set ProvisionGuestProxyAgent @@ -2973,14 +3010,10 @@ def test_determine_pps_with_reprovision_marker( ] -class TestPreprovisioningHotAttachNics(CiTestCase): - def setUp(self): - super(TestPreprovisioningHotAttachNics, self).setUp() - self.tmp = self.tmp_dir() - self.waagent_d = self.tmp_path("/var/lib/waagent", self.tmp) - self.paths = helpers.Paths({"cloud_dir": self.tmp}) - dsaz.BUILTIN_DS_CONFIG["data_dir"] = self.waagent_d - self.paths = helpers.Paths({"cloud_dir": self.tmp}) +class TestPreprovisioningHotAttachNics: + @pytest.fixture(autouse=True) + def fixtures(self, waagent_d): + dsaz.BUILTIN_DS_CONFIG["data_dir"] = waagent_d @mock.patch(MOCKPATH + "util.write_file", autospec=True) @mock.patch(MOCKPATH + "DataSourceAzure._report_ready") @@ -2994,14 +3027,15 @@ def test_detect_nic_attach_reports_ready_and_waits_for_detach( m_wait_for_hot_attached_primary_nic, m_report_ready, m_writefile, + paths, ): """Report ready first and then wait for nic detach""" - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=None, paths=paths) dsa._wait_for_pps_savable_reuse() - self.assertEqual(1, m_report_ready.call_count) - self.assertEqual(1, m_wait_for_hot_attached_primary_nic.call_count) - self.assertEqual(1, m_detach.call_count) - self.assertEqual(1, m_writefile.call_count) + assert 1 == m_report_ready.call_count + assert 1 == m_wait_for_hot_attached_primary_nic.call_count + assert 1 == m_detach.call_count + assert 1 == m_writefile.call_count m_writefile.assert_called_with( dsa._reported_ready_marker_file, mock.ANY ) @@ -3022,12 +3056,14 @@ def test_wait_for_nic_attach_multinic_attach( m_link_up, m_report_ready, m_writefile, + paths, + tmp_path, ): """Wait for nic attach if we do not have a fallback interface. Skip waiting for additional nics after we have found primary""" distro = mock.MagicMock() - distro.get_tmp_exec_path = self.tmp_dir - dsa = dsaz.DataSourceAzure({}, distro=distro, paths=self.paths) + distro.get_tmp_exec_path = str(tmp_path) + dsa = dsaz.DataSourceAzure({}, distro=distro, paths=paths) lease = { "interface": "eth9", "fixed-address": "192.168.2.9", @@ -3062,13 +3098,13 @@ def test_wait_for_nic_attach_multinic_attach( dsa._wait_for_pps_savable_reuse() - self.assertEqual(1, m_detach.call_count) + assert 1 == m_detach.call_count # only wait for primary nic - self.assertEqual(1, m_attach.call_count) + assert 1 == m_attach.call_count # DHCP and network metadata calls will only happen on the primary NIC. - self.assertEqual(1, m_dhcpv4.call_count) + assert 1 == m_dhcpv4.call_count # no call to bring link up on secondary nic - self.assertEqual(1, m_link_up.call_count) + assert 1 == m_link_up.call_count # reset mock to test again with primary nic being eth1 dhcp_ctx_primary.interface = "eth1" @@ -3090,59 +3126,60 @@ def test_wait_for_nic_attach_multinic_attach( m_dhcpv4.side_effect = [dhcp_ctx_secondary, dhcp_ctx_primary] m_link_up.reset_mock() m_attach.side_effect = ["eth0", "eth1"] - dsa = dsaz.DataSourceAzure({}, distro=distro, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=distro, paths=paths) dsa._wait_for_pps_savable_reuse() - self.assertEqual(1, m_detach.call_count) - self.assertEqual(2, m_attach.call_count) - self.assertEqual(2, m_dhcpv4.call_count) - self.assertEqual(2, m_link_up.call_count) + assert 1 == m_detach.call_count + assert 2 == m_attach.call_count + assert 2 == m_dhcpv4.call_count + assert 2 == m_link_up.call_count @mock.patch("cloudinit.distros.networking.LinuxNetworking.try_set_link_up") - def test_wait_for_link_up_returns_if_already_up(self, m_is_link_up): + def test_wait_for_link_up_returns_if_already_up(self, m_is_link_up, paths): """Waiting for link to be up should return immediately if the link is already up.""" distro_cls = distros.fetch("ubuntu") - distro = distro_cls("ubuntu", {}, self.paths) - dsa = dsaz.DataSourceAzure({}, distro=distro, paths=self.paths) + distro = distro_cls("ubuntu", {}, paths) + dsa = dsaz.DataSourceAzure({}, distro=distro, paths=paths) m_is_link_up.return_value = True dsa.wait_for_link_up("eth0") - self.assertEqual(1, m_is_link_up.call_count) + assert 1 == m_is_link_up.call_count @mock.patch("cloudinit.distros.networking.LinuxNetworking.try_set_link_up") @mock.patch(MOCKPATH + "sleep") def test_wait_for_link_up_checks_link_after_sleep( - self, m_sleep, m_try_set_link_up + self, m_sleep, m_try_set_link_up, paths ): """Waiting for link to be up should return immediately if the link is already up.""" distro_cls = distros.fetch("ubuntu") - distro = distro_cls("ubuntu", {}, self.paths) - dsa = dsaz.DataSourceAzure({}, distro=distro, paths=self.paths) + distro = distro_cls("ubuntu", {}, paths) + dsa = dsaz.DataSourceAzure({}, distro=distro, paths=paths) m_try_set_link_up.return_value = False dsa.wait_for_link_up("eth0") - self.assertEqual(100, m_try_set_link_up.call_count) - self.assertEqual(99 * [mock.call(0.1)], m_sleep.mock_calls) + assert 100 == m_try_set_link_up.call_count + assert 99 * [mock.call(0.1)] == m_sleep.mock_calls @mock.patch( "cloudinit.sources.helpers.netlink.create_bound_netlink_socket" ) - def test_wait_for_all_nics_ready_raises_if_socket_fails(self, m_socket): + def test_wait_for_all_nics_ready_raises_if_socket_fails( + self, m_socket, paths + ): """Waiting for all nics should raise exception if netlink socket creation fails.""" m_socket.side_effect = netlink.NetlinkCreateSocketError distro_cls = distros.fetch("ubuntu") - distro = distro_cls("ubuntu", {}, self.paths) - dsa = dsaz.DataSourceAzure({}, distro=distro, paths=self.paths) + distro = distro_cls("ubuntu", {}, paths) + dsa = dsaz.DataSourceAzure({}, distro=distro, paths=paths) - self.assertRaises( - netlink.NetlinkCreateSocketError, dsa._wait_for_pps_savable_reuse - ) + with pytest.raises(netlink.NetlinkCreateSocketError): + dsa._wait_for_pps_savable_reuse() @mock.patch("cloudinit.net.find_fallback_nic", return_value="eth9") @@ -3151,13 +3188,10 @@ def test_wait_for_all_nics_ready_raises_if_socket_fails(self, m_socket): "cloudinit.sources.helpers.netlink.wait_for_media_disconnect_connect" ) @mock.patch(MOCKPATH + "imds.fetch_reprovision_data") -class TestPreprovisioningPollIMDS(CiTestCase): - def setUp(self): - super(TestPreprovisioningPollIMDS, self).setUp() - self.tmp = self.tmp_dir() - self.waagent_d = self.tmp_path("/var/lib/waagent", self.tmp) - self.paths = helpers.Paths({"cloud_dir": self.tmp}) - dsaz.BUILTIN_DS_CONFIG["data_dir"] = self.waagent_d +class TestPreprovisioningPollIMDS: + @pytest.fixture(autouse=True) + def fixtures(self, waagent_d): + dsaz.BUILTIN_DS_CONFIG["data_dir"] = waagent_d @mock.patch("time.sleep", mock.MagicMock()) def test_poll_imds_re_dhcp_on_timeout( @@ -3166,6 +3200,7 @@ def test_poll_imds_re_dhcp_on_timeout( m_media_switch, m_dhcp, m_fallback, + paths, ): """The poll_imds will retry DHCP on IMDS timeout.""" m_fetch_reprovisiondata.side_effect = [ @@ -3185,11 +3220,11 @@ def test_poll_imds_re_dhcp_on_timeout( dhcp_ctx.obtain_lease.return_value = lease dhcp_ctx.iface = lease["interface"] - dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=mock.Mock(), paths=paths) dsa._ephemeral_dhcp_ctx = dhcp_ctx dsa._poll_imds() - self.assertEqual(1, m_dhcp.call_count, "Expected 1 DHCP calls") + assert 1 == m_dhcp.call_count, "Expected 1 DHCP calls" assert m_fetch_reprovisiondata.call_count == 2 @mock.patch("os.path.isfile") @@ -3200,6 +3235,7 @@ def test_poll_imds_skips_dhcp_if_ctx_present( m_media_switch, m_dhcp, m_fallback, + paths, ): """The poll_imds function should reuse the dhcp ctx if it is already present. This happens when we wait for nic to be hot-attached before @@ -3207,11 +3243,11 @@ def test_poll_imds_skips_dhcp_if_ctx_present( _poll_imds is called, then it is not expected to be waiting for media_disconnect_connect either.""" m_isfile.return_value = True - dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) + dsa = dsaz.DataSourceAzure({}, distro=None, paths=paths) dsa._ephemeral_dhcp_ctx = mock.Mock(lease={}) dsa._poll_imds() - self.assertEqual(0, m_dhcp.call_count) - self.assertEqual(0, m_media_switch.call_count) + assert 0 == m_dhcp.call_count + assert 0 == m_media_switch.call_count @mock.patch("os.path.isfile") def test_poll_imds_does_dhcp_on_retries_if_ctx_present( @@ -3221,6 +3257,8 @@ def test_poll_imds_does_dhcp_on_retries_if_ctx_present( m_media_switch, m_dhcp, m_fallback, + paths, + tmp_path, ): """The poll_imds function should reuse the dhcp ctx if it is already present. This happens when we wait for nic to be hot-attached before @@ -3238,70 +3276,69 @@ def test_poll_imds_does_dhcp_on_retries_if_ctx_present( ] m_isfile.return_value = True distro = mock.MagicMock() - distro.get_tmp_exec_path = self.tmp_dir - dsa = dsaz.DataSourceAzure({}, distro=distro, paths=self.paths) + distro.get_tmp_exec_path = str(tmp_path) + dsa = dsaz.DataSourceAzure({}, distro=distro, paths=paths) with mock.patch.object(dsa, "_ephemeral_dhcp_ctx") as m_dhcp_ctx: m_dhcp_ctx.obtain_lease.return_value = "Dummy lease" dsa._ephemeral_dhcp_ctx = m_dhcp_ctx dsa._poll_imds() - self.assertEqual(1, m_dhcp_ctx.clean_network.call_count) - self.assertEqual(1, m_dhcp.call_count) - self.assertEqual(0, m_media_switch.call_count) - self.assertEqual(2, m_fetch_reprovisiondata.call_count) - - -class TestRemoveUbuntuNetworkConfigScripts(CiTestCase): - with_logs = True + assert 1 == m_dhcp_ctx.clean_network.call_count + assert 1 == m_dhcp.call_count + assert 0 == m_media_switch.call_count + assert 2 == m_fetch_reprovisiondata.call_count - def setUp(self): - super(TestRemoveUbuntuNetworkConfigScripts, self).setUp() - self.tmp = self.tmp_dir() - def test_remove_network_scripts_removes_both_files_and_directories(self): +class TestRemoveUbuntuNetworkConfigScripts: + def test_remove_network_scripts_removes_both_files_and_directories( + self, caplog, tmp_path + ): """Any files or directories in paths are removed when present.""" - file1 = self.tmp_path("file1", dir=self.tmp) - subdir = self.tmp_path("sub1", dir=self.tmp) - subfile = self.tmp_path("leaf1", dir=subdir) + file1 = tmp_path / "file1" + subdir = tmp_path / "sub1" + subfile = subdir / "leaf1" write_file(file1, "file1content") write_file(subfile, "leafcontent") dsaz.maybe_remove_ubuntu_network_config_scripts(paths=[subdir, file1]) for path in (file1, subdir, subfile): - self.assertFalse( - os.path.exists(path), "Found unremoved: %s" % path - ) + assert not os.path.exists(path), "Found unremoved: %s" % path expected_logs = [ - "INFO: Removing Ubuntu extended network scripts because cloud-init" - " updates Azure network configuration on the following events:" - " ['boot', 'boot-legacy']", - "Recursively deleting %s" % subdir, - "Attempting to remove %s" % file1, + ( + mock.ANY, + logging.INFO, + "Removing Ubuntu extended network scripts because cloud-init" + " updates Azure network configuration on the following events:" + " ['boot', 'boot-legacy'].", + ), + (mock.ANY, logging.DEBUG, "Recursively deleting %s" % subdir), + (mock.ANY, logging.DEBUG, "Attempting to remove %s" % file1), ] for log in expected_logs: - self.assertIn(log, self.logs.getvalue()) + assert log in caplog.record_tuples - def test_remove_network_scripts_only_attempts_removal_if_path_exists(self): + def test_remove_network_scripts_only_attempts_removal_if_path_exists( + self, caplog, tmp_path + ): """Any files or directories absent are skipped without error.""" dsaz.maybe_remove_ubuntu_network_config_scripts( paths=[ - self.tmp_path("nodirhere/", dir=self.tmp), - self.tmp_path("notfilehere", dir=self.tmp), + tmp_path / "nodirhere/", + tmp_path / "notfilehere", ] ) - self.assertNotIn("/not/a", self.logs.getvalue()) # No delete logs + assert "/not/a" not in caplog.text # No delete logs - @mock.patch(MOCKPATH + "os.path.exists") + # Report path absent on all to avoid delete operation + @mock.patch(MOCKPATH + "os.path.exists", return_value=False) def test_remove_network_scripts_default_removes_stock_scripts( self, m_exists ): """Azure's stock ubuntu image scripts and artifacts are removed.""" - # Report path absent on all to avoid delete operation - m_exists.return_value = False dsaz.maybe_remove_ubuntu_network_config_scripts() calls = m_exists.call_args_list for path in dsaz.UBUNTU_EXTENDED_NETWORK_SCRIPTS: - self.assertIn(mock.call(path), calls) + assert mock.call(path) in calls class TestIsPlatformViable: @@ -3345,7 +3382,7 @@ def test_false_on_no_matching_azure_criteria( ) -class TestRandomSeed(CiTestCase): +class TestRandomSeed: """Test proper handling of random_seed""" def test_non_ascii_seed_is_serializable(self): @@ -3361,9 +3398,9 @@ def test_non_ascii_seed_is_serializable(self): serialized = json_dumps(obj) deserialized = load_json(serialized) except UnicodeDecodeError: - self.fail("Non-serializable random seed returned") + pytest.fail("Non-serializable random seed returned") - self.assertEqual(deserialized["seed"], result) + assert deserialized["seed"] == result class TestEphemeralNetworking: @@ -3464,7 +3501,7 @@ def test_retry_interface_error( self, azure_ds, mock_ephemeral_dhcp_v4, - mock_kvp_report_failure_to_host, + mock_kvp_report_via_kvp, mock_sleep, ): lease = { @@ -3490,11 +3527,11 @@ def test_retry_interface_error( assert azure_ds._wireserver_endpoint == "168.63.129.16" assert azure_ds._ephemeral_dhcp_ctx.iface == "fakeEth0" - error_reasons = [ - c[0][0].reason - for c in mock_kvp_report_failure_to_host.call_args_list - ] - assert error_reasons == ["failure to find DHCP interface"] + assert len(mock_kvp_report_via_kvp.call_args_list) == 1 + for call in mock_kvp_report_via_kvp.call_args_list: + assert call[0][0].startswith( + "result=error|reason=failure to find DHCP interface" + ) def test_retry_process_error( self, @@ -3568,7 +3605,7 @@ def test_retry_sleeps( self, azure_ds, mock_ephemeral_dhcp_v4, - mock_kvp_report_failure_to_host, + mock_kvp_report_via_kvp, mock_sleep, error_class, error_reason, @@ -3597,11 +3634,9 @@ def test_retry_sleeps( assert azure_ds._wireserver_endpoint == "168.63.129.16" assert azure_ds._ephemeral_dhcp_ctx.iface == "fakeEth0" - error_reasons = [ - c[0][0].reason - for c in mock_kvp_report_failure_to_host.call_args_list - ] - assert error_reasons == [error_reason] * 10 + assert len(mock_kvp_report_via_kvp.call_args_list) == 10 + for call in mock_kvp_report_via_kvp.call_args_list: + assert call[0][0].startswith(f"result=error|reason={error_reason}") @pytest.mark.parametrize( "error_class,error_reason", @@ -3614,7 +3649,7 @@ def test_retry_times_out( self, azure_ds, mock_ephemeral_dhcp_v4, - mock_kvp_report_failure_to_host, + mock_kvp_report_via_kvp, mock_sleep, mock_time, mock_monotonic, @@ -3645,11 +3680,9 @@ def test_retry_times_out( assert azure_ds._wireserver_endpoint == "168.63.129.16" assert azure_ds._ephemeral_dhcp_ctx is None - error_reasons = [ - c[0][0].reason - for c in mock_kvp_report_failure_to_host.call_args_list - ] - assert error_reasons == [error_reason] * 3 + assert len(mock_kvp_report_via_kvp.call_args_list) == 3 + for call in mock_kvp_report_via_kvp.call_args_list: + assert call[0][0].startswith(f"result=error|reason={error_reason}") class TestCheckIfPrimary: @@ -3739,7 +3772,7 @@ def provisioning_setup( mock_dmi_read_dmi_data, mock_get_interfaces, mock_get_interface_mac, - mock_kvp_report_failure_to_host, + mock_kvp_report_via_kvp, mock_kvp_report_success_to_host, mock_netlink, mock_readurl, @@ -3769,7 +3802,7 @@ def provisioning_setup( self.mock_dmi_read_dmi_data = mock_dmi_read_dmi_data self.mock_get_interfaces = mock_get_interfaces self.mock_get_interface_mac = mock_get_interface_mac - self.mock_kvp_report_failure_to_host = mock_kvp_report_failure_to_host + self.mock_kvp_report_via_kvp = mock_kvp_report_via_kvp self.mock_kvp_report_success_to_host = mock_kvp_report_success_to_host self.mock_netlink = mock_netlink self.mock_readurl = mock_readurl @@ -3882,7 +3915,7 @@ def test_no_pps(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert not self.mock_kvp_report_failure_to_host.mock_calls + assert not self.mock_kvp_report_via_kvp.mock_calls assert not self.mock_azure_report_failure_to_fabric.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 @@ -3967,7 +4000,7 @@ def test_no_pps_gpa(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert not self.mock_kvp_report_failure_to_host.mock_calls + assert not self.mock_kvp_report_via_kvp.mock_calls assert not self.mock_azure_report_failure_to_fabric.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 @@ -4030,7 +4063,6 @@ def test_no_pps_gpa_fail(self): assert self.mock_dmi_read_dmi_data.mock_calls == [ mock.call("chassis-asset-tag"), mock.call("system-uuid"), - mock.call("system-uuid"), ] assert ( self.azure_ds.metadata["instance-id"] @@ -4051,7 +4083,7 @@ def test_no_pps_gpa_fail(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 1 + assert len(self.mock_kvp_report_via_kvp.mock_calls) == 1 assert len(self.mock_azure_report_failure_to_fabric.mock_calls) == 1 assert not self.mock_kvp_report_success_to_host.mock_calls @@ -4108,17 +4140,16 @@ def test_stale_pps(self, pps_type): assert self.mock_dmi_read_dmi_data.mock_calls == [ mock.call("chassis-asset-tag"), mock.call("system-uuid"), - mock.call("system-uuid"), ] # Verify reports via KVP. assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 - assert self.mock_kvp_report_failure_to_host.mock_calls == [ + assert self.mock_kvp_report_via_kvp.mock_calls == [ mock.call( errors.ReportableErrorImdsInvalidMetadata( key="extended.compute.ppsType", value=pps_type - ), + ).as_encoded_report(vm_id=self.azure_ds._vm_id), ), ] @@ -4236,7 +4267,7 @@ def test_running_pps(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert not self.mock_kvp_report_failure_to_host.mock_calls + assert not self.mock_kvp_report_via_kvp.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 2 # Verify dmesg reported via KVP. @@ -4363,7 +4394,7 @@ def test_running_pps_gpa(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert not self.mock_kvp_report_failure_to_host.mock_calls + assert not self.mock_kvp_report_via_kvp.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 2 # Verify dmesg reported via KVP. @@ -4492,7 +4523,7 @@ def test_savable_pps(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert not self.mock_kvp_report_failure_to_host.mock_calls + assert not self.mock_kvp_report_via_kvp.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 2 # Verify dmesg reported via KVP. @@ -4628,7 +4659,7 @@ def test_savable_pps_gpa(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert not self.mock_kvp_report_failure_to_host.mock_calls + assert not self.mock_kvp_report_via_kvp.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 2 # Verify dmesg reported via KVP. @@ -4869,7 +4900,7 @@ def test_recovery_pps(self, pps_type): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert not self.mock_kvp_report_failure_to_host.mock_calls + assert not self.mock_kvp_report_via_kvp.mock_calls assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 @pytest.mark.parametrize("pps_type", ["Savable", "Running", "Unknown"]) @@ -4901,7 +4932,7 @@ def test_source_pps_fails_initial_dhcp(self, pps_type): assert self.mock_netlink.mock_calls == [] # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 2 + assert len(self.mock_kvp_report_via_kvp.mock_calls) == 2 assert not self.mock_kvp_report_success_to_host.mock_calls @pytest.mark.parametrize( @@ -4974,7 +5005,7 @@ def test_os_disk_pps(self, mock_sleep, subp_side_effect): assert self.wrapped_util_write_file.mock_calls == [] # Verify reports via KVP. Ignore failure reported after sleep(). - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 1 + assert len(self.mock_kvp_report_via_kvp.mock_calls) == 1 assert len(self.mock_kvp_report_success_to_host.mock_calls) == 1 def test_imds_failure_results_in_provisioning_failure(self): @@ -5017,7 +5048,7 @@ def test_imds_failure_results_in_provisioning_failure(self): assert self.patched_reported_ready_marker_path.exists() is False # Verify reports via KVP. - assert len(self.mock_kvp_report_failure_to_host.mock_calls) == 1 + assert len(self.mock_kvp_report_via_kvp.mock_calls) == 1 assert not self.mock_kvp_report_success_to_host.mock_calls @@ -5104,7 +5135,7 @@ def test_errors( exception, mock_azure_report_failure_to_fabric, mock_imds_fetch_metadata_with_api_fallback, - mock_kvp_report_failure_to_host, + mock_kvp_report_via_kvp, mock_time, mock_monotonic, monkeypatch, @@ -5139,12 +5170,8 @@ def test_errors( == expected_duration ) - reported_error = mock_kvp_report_failure_to_host.call_args[0][0] - assert isinstance(reported_error, reported_error_type) - assert reported_error.supporting_data["exception"] == repr(exception) - assert mock_kvp_report_failure_to_host.mock_calls == [ - mock.call(reported_error) - ] + reported_error = mock_kvp_report_via_kvp.call_args[0][0] + assert type(exception).__name__ in reported_error connection_error = isinstance( exception, url_helper.UrlError @@ -5152,7 +5179,7 @@ def test_errors( report_skipped = not route_configured_for_imds and connection_error if report_failure and not report_skipped: assert mock_azure_report_failure_to_fabric.mock_calls == [ - mock.call(endpoint=mock.ANY, error=reported_error) + mock.call(endpoint=mock.ANY, encoded_report=reported_error) ] else: assert mock_azure_report_failure_to_fabric.mock_calls == [] @@ -5165,16 +5192,18 @@ def test_report_host_only_kvp_enabled( azure_ds, kvp_enabled, mock_azure_report_failure_to_fabric, - mock_kvp_report_failure_to_host, + mock_kvp_report_via_kvp, mock_kvp_report_success_to_host, mock_report_dmesg_to_kvp, ): - mock_kvp_report_failure_to_host.return_value = kvp_enabled + mock_kvp_report_via_kvp.return_value = kvp_enabled error = errors.ReportableError(reason="foo") assert azure_ds._report_failure(error, host_only=True) == kvp_enabled - assert mock_kvp_report_failure_to_host.mock_calls == [mock.call(error)] + assert mock_kvp_report_via_kvp.mock_calls == [ + mock.call(error.as_encoded_report(vm_id=azure_ds._vm_id)) + ] assert mock_kvp_report_success_to_host.mock_calls == [] assert mock_azure_report_failure_to_fabric.mock_calls == [] assert mock_report_dmesg_to_kvp.mock_calls == [mock.call()] @@ -5403,3 +5432,67 @@ def test_dependency_fallback(self): Python versions """ assert dsaz.encrypt_pass("`") + + +class TestQueryVmId: + @mock.patch.object( + identity, "query_system_uuid", side_effect=["test-system-uuid"] + ) + @mock.patch.object( + identity, "convert_system_uuid_to_vm_id", side_effect=["test-vm-id"] + ) + def test_query_vm_id_success( + self, mock_convert_uuid, mock_query_system_uuid, azure_ds + ): + azure_ds._query_vm_id() + + assert azure_ds._system_uuid == "test-system-uuid" + assert azure_ds._vm_id == "test-vm-id" + + mock_query_system_uuid.assert_called_once() + mock_convert_uuid.assert_called_once_with("test-system-uuid") + + @mock.patch.object( + identity, + "query_system_uuid", + side_effect=[RuntimeError("test failure")], + ) + def test_query_vm_id_system_uuid_failure( + self, mock_query_system_uuid, azure_ds + ): + with pytest.raises(errors.ReportableErrorVmIdentification) as exc_info: + azure_ds._query_vm_id() + + assert azure_ds._system_uuid is None + assert azure_ds._vm_id is None + assert ( + exc_info.value.reason + == "Failed to query system UUID: test failure" + ) + + mock_query_system_uuid.assert_called_once() + + @mock.patch.object( + identity, "query_system_uuid", side_effect=["test-system-uuid"] + ) + @mock.patch.object( + identity, + "convert_system_uuid_to_vm_id", + side_effect=[ValueError("test failure")], + ) + def test_query_vm_id_vm_id_conversion_failure( + self, mock_convert_uuid, mock_query_system_uuid, azure_ds + ): + with pytest.raises(errors.ReportableErrorVmIdentification) as excinfo: + azure_ds._query_vm_id() + + assert azure_ds._system_uuid == "test-system-uuid" + assert azure_ds._vm_id is None + assert ( + excinfo.value.reason + == "Failed to convert system UUID 'test-system-uuid' " + "to Azure VM ID: test failure" + ) + + mock_query_system_uuid.assert_called_once() + mock_convert_uuid.assert_called_once_with("test-system-uuid") diff --git a/tests/unittests/sources/test_azure_helper.py b/tests/unittests/sources/test_azure_helper.py index 3a58486e..4fc252b9 100644 --- a/tests/unittests/sources/test_azure_helper.py +++ b/tests/unittests/sources/test_azure_helper.py @@ -5,6 +5,7 @@ import re import unittest from textwrap import dedent +from unittest import mock from xml.etree import ElementTree as ET from xml.sax.saxutils import escape, unescape @@ -17,7 +18,6 @@ from cloudinit.sources.helpers import azure as azure_helper from cloudinit.sources.helpers.azure import WALinuxAgentShim as wa_shim from cloudinit.util import load_text_file -from tests.unittests.helpers import CiTestCase, ExitStack, mock from tests.unittests.sources.test_azure import construct_ovf_env from tests.unittests.util import MockDistro @@ -158,7 +158,7 @@ def test_get_ip_from_lease_value(self, encoded_address, ip_address): ) -class TestGoalStateParsing(CiTestCase): +class TestGoalStateParsing: default_parameters = { "incarnation": 1, "container_id": "MyContainerId", @@ -188,17 +188,17 @@ def _get_goal_state(self, m_azure_endpoint_client=None, **kwargs): def test_incarnation_parsed_correctly(self): incarnation = "123" goal_state = self._get_goal_state(incarnation=incarnation) - self.assertEqual(incarnation, goal_state.incarnation) + assert incarnation == goal_state.incarnation def test_container_id_parsed_correctly(self): container_id = "TestContainerId" goal_state = self._get_goal_state(container_id=container_id) - self.assertEqual(container_id, goal_state.container_id) + assert container_id == goal_state.container_id def test_instance_id_parsed_correctly(self): instance_id = "TestInstanceId" goal_state = self._get_goal_state(instance_id=instance_id) - self.assertEqual(instance_id, goal_state.instance_id) + assert instance_id == goal_state.instance_id def test_certificates_xml_parsed_and_fetched_correctly(self): m_azure_endpoint_client = mock.MagicMock() @@ -208,15 +208,12 @@ def test_certificates_xml_parsed_and_fetched_correctly(self): certificates_url=certificates_url, ) certificates_xml = goal_state.certificates_xml - self.assertEqual(1, m_azure_endpoint_client.get.call_count) - self.assertEqual( - certificates_url, m_azure_endpoint_client.get.call_args[0][0] - ) - self.assertTrue( - m_azure_endpoint_client.get.call_args[1].get("secure", False) - ) - self.assertEqual( - m_azure_endpoint_client.get.return_value.contents, certificates_xml + assert 1 == m_azure_endpoint_client.get.call_count + assert certificates_url == m_azure_endpoint_client.get.call_args[0][0] + assert m_azure_endpoint_client.get.call_args[1].get("secure", False) + assert ( + m_azure_endpoint_client.get.return_value.contents + == certificates_xml ) def test_missing_certificates_skips_http_get(self): @@ -226,66 +223,60 @@ def test_missing_certificates_skips_http_get(self): certificates_url=None, ) certificates_xml = goal_state.certificates_xml - self.assertEqual(0, m_azure_endpoint_client.get.call_count) - self.assertIsNone(certificates_xml) + assert 0 == m_azure_endpoint_client.get.call_count + assert certificates_xml is None def test_invalid_goal_state_xml_raises_parse_error(self): xml = "random non-xml data" - with self.assertRaises(ET.ParseError): + with pytest.raises(ET.ParseError): azure_helper.GoalState(xml, mock.MagicMock()) def test_missing_container_id_in_goal_state_xml_raises_exc(self): xml = self._get_formatted_goal_state_xml_string() xml = re.sub(".*", "", xml) - with self.assertRaises(azure_helper.InvalidGoalStateXMLException): + with pytest.raises(azure_helper.InvalidGoalStateXMLException): azure_helper.GoalState(xml, mock.MagicMock()) def test_missing_instance_id_in_goal_state_xml_raises_exc(self): xml = self._get_formatted_goal_state_xml_string() xml = re.sub(".*", "", xml) - with self.assertRaises(azure_helper.InvalidGoalStateXMLException): + with pytest.raises(azure_helper.InvalidGoalStateXMLException): azure_helper.GoalState(xml, mock.MagicMock()) def test_missing_incarnation_in_goal_state_xml_raises_exc(self): xml = self._get_formatted_goal_state_xml_string() xml = re.sub(".*", "", xml) - with self.assertRaises(azure_helper.InvalidGoalStateXMLException): + with pytest.raises(azure_helper.InvalidGoalStateXMLException): azure_helper.GoalState(xml, mock.MagicMock()) -class TestAzureEndpointHttpClient(CiTestCase): +@mock.patch("cloudinit.sources.helpers.azure.http_with_retries") +class TestAzureEndpointHttpClient: regular_headers = { "x-ms-agent-name": "WALinuxAgent", "x-ms-version": "2012-11-30", } - def setUp(self): - super(TestAzureEndpointHttpClient, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - self.m_http_with_retries = patches.enter_context( - mock.patch.object(azure_helper, "http_with_retries") - ) - - def test_non_secure_get(self): + def test_non_secure_get(self, m_http_with_retries): client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) url = "MyTestUrl" response = client.get(url, secure=False) - self.assertEqual(1, self.m_http_with_retries.call_count) - self.assertEqual(self.m_http_with_retries.return_value, response) - self.assertEqual( - mock.call(url, headers=self.regular_headers), - self.m_http_with_retries.call_args, + assert 1 == m_http_with_retries.call_count + assert m_http_with_retries.return_value == response + assert ( + mock.call(url, headers=self.regular_headers) + == m_http_with_retries.call_args ) - def test_non_secure_get_raises_exception(self): + def test_non_secure_get_raises_exception(self, m_http_with_retries): client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) url = "MyTestUrl" - self.m_http_with_retries.side_effect = SentinelException - self.assertRaises(SentinelException, client.get, url, secure=False) - self.assertEqual(1, self.m_http_with_retries.call_count) + m_http_with_retries.side_effect = SentinelException + with pytest.raises(SentinelException): + client.get(url, secure=False) + assert 1 == m_http_with_retries.call_count - def test_secure_get(self): + def test_secure_get(self, m_http_with_retries): url = "MyTestUrl" m_certificate = mock.MagicMock() expected_headers = self.regular_headers.copy() @@ -297,67 +288,70 @@ def test_secure_get(self): ) client = azure_helper.AzureEndpointHttpClient(m_certificate) response = client.get(url, secure=True) - self.assertEqual(1, self.m_http_with_retries.call_count) - self.assertEqual(self.m_http_with_retries.return_value, response) - self.assertEqual( - mock.call(url, headers=expected_headers), - self.m_http_with_retries.call_args, + assert 1 == m_http_with_retries.call_count + assert m_http_with_retries.return_value == response + assert ( + mock.call(url, headers=expected_headers) + == m_http_with_retries.call_args ) - def test_secure_get_raises_exception(self): + def test_secure_get_raises_exception(self, m_http_with_retries): url = "MyTestUrl" client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) - self.m_http_with_retries.side_effect = SentinelException - self.assertRaises(SentinelException, client.get, url, secure=True) - self.assertEqual(1, self.m_http_with_retries.call_count) + m_http_with_retries.side_effect = SentinelException + with pytest.raises(SentinelException): + client.get(url, secure=True) + assert 1 == m_http_with_retries.call_count - def test_post(self): + def test_post(self, m_http_with_retries): m_data = mock.MagicMock() url = "MyTestUrl" client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) response = client.post(url, data=m_data) - self.assertEqual(1, self.m_http_with_retries.call_count) - self.assertEqual(self.m_http_with_retries.return_value, response) - self.assertEqual( - mock.call(url, data=m_data, headers=self.regular_headers), - self.m_http_with_retries.call_args, + assert 1 == m_http_with_retries.call_count + assert m_http_with_retries.return_value == response + assert ( + mock.call(url, data=m_data, headers=self.regular_headers) + == m_http_with_retries.call_args ) - def test_post_raises_exception(self): + def test_post_raises_exception(self, m_http_with_retries): m_data = mock.MagicMock() url = "MyTestUrl" client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) - self.m_http_with_retries.side_effect = SentinelException - self.assertRaises(SentinelException, client.post, url, data=m_data) - self.assertEqual(1, self.m_http_with_retries.call_count) + m_http_with_retries.side_effect = SentinelException + with pytest.raises(SentinelException): + client.post(url, data=m_data) + assert 1 == m_http_with_retries.call_count - def test_post_with_extra_headers(self): + def test_post_with_extra_headers(self, m_http_with_retries): url = "MyTestUrl" client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) extra_headers = {"test": "header"} client.post(url, extra_headers=extra_headers) expected_headers = self.regular_headers.copy() expected_headers.update(extra_headers) - self.assertEqual(1, self.m_http_with_retries.call_count) - self.assertEqual( - mock.call(url, data=mock.ANY, headers=expected_headers), - self.m_http_with_retries.call_args, + assert 1 == m_http_with_retries.call_count + assert ( + mock.call(url, data=mock.ANY, headers=expected_headers) + == m_http_with_retries.call_args ) - def test_post_with_sleep_with_extra_headers_raises_exception(self): + def test_post_with_sleep_with_extra_headers_raises_exception( + self, m_http_with_retries + ): m_data = mock.MagicMock() url = "MyTestUrl" extra_headers = {"test": "header"} client = azure_helper.AzureEndpointHttpClient(mock.MagicMock()) - self.m_http_with_retries.side_effect = SentinelException - self.assertRaises( - SentinelException, - client.post, - url, - data=m_data, - extra_headers=extra_headers, - ) - self.assertEqual(1, self.m_http_with_retries.call_count) + m_http_with_retries.side_effect = SentinelException + with pytest.raises(SentinelException): + client.post( + url, + data=m_data, + extra_headers=extra_headers, + ) + assert 1 == m_http_with_retries.call_count class TestHttpWithRetries: @@ -483,52 +477,42 @@ def test_success( ) in caplog.record_tuples -class TestOpenSSLManager(CiTestCase): - def setUp(self): - super(TestOpenSSLManager, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - - self.subp = patches.enter_context( - mock.patch.object(azure_helper.subp, "subp") - ) +@mock.patch("cloudinit.sources.helpers.azure.subp.subp") +class TestOpenSSLManager: + @pytest.fixture(autouse=True) + def fixtures(self, mocker): try: - self.open = patches.enter_context(mock.patch("__builtin__.open")) + mocker.patch("__builtin__.open") except ImportError: - self.open = patches.enter_context(mock.patch("builtins.open")) + mocker.patch("builtins.open") @mock.patch.object(azure_helper, "cd", mock.MagicMock()) @mock.patch.object(azure_helper.temp_utils, "mkdtemp") - def test_openssl_manager_creates_a_tmpdir(self, mkdtemp): + def test_openssl_manager_creates_a_tmpdir(self, mkdtemp, m_subp): manager = azure_helper.OpenSSLManager() - self.assertEqual(mkdtemp.return_value, manager.tmpdir) + assert mkdtemp.return_value == manager.tmpdir - def test_generate_certificate_uses_tmpdir(self): + def test_generate_certificate_uses_tmpdir(self, m_subp): subp_directory = {} def capture_directory(*args, **kwargs): subp_directory["path"] = os.getcwd() - self.subp.side_effect = capture_directory + m_subp.side_effect = capture_directory manager = azure_helper.OpenSSLManager() - self.assertEqual(manager.tmpdir, subp_directory["path"]) + assert manager.tmpdir == subp_directory["path"] manager.clean_up() @mock.patch.object(azure_helper, "cd", mock.MagicMock()) @mock.patch.object(azure_helper.temp_utils, "mkdtemp", mock.MagicMock()) @mock.patch.object(azure_helper.util, "del_dir") - def test_clean_up(self, del_dir): + def test_clean_up(self, del_dir, m_subp): manager = azure_helper.OpenSSLManager() manager.clean_up() - self.assertEqual([mock.call(manager.tmpdir)], del_dir.call_args_list) - - -class TestOpenSSLManagerActions(CiTestCase): - def setUp(self): - super(TestOpenSSLManagerActions, self).setUp() + assert [mock.call(manager.tmpdir)] == del_dir.call_args_list - self.allowed_subp = True +class TestOpenSSLManagerActions: def _data_file(self, name): path = "tests/data/azure" return os.path.join(path, name) @@ -539,11 +523,11 @@ def test_pubkey_extract(self): good_key = load_text_file(self._data_file("pubkey_extract_ssh_key")) sslmgr = azure_helper.OpenSSLManager() key = sslmgr._get_ssh_key_from_cert(cert) - self.assertEqual(good_key, key) + assert good_key == key good_fingerprint = "073E19D14D1C799224C6A0FD8DDAB6A8BF27D473" fingerprint = sslmgr._get_fingerprint_from_cert(cert) - self.assertEqual(good_fingerprint, fingerprint) + assert good_fingerprint == fingerprint @unittest.skip("todo move to cloud_test") @mock.patch.object(azure_helper.OpenSSLManager, "_decrypt_certs_from_xml") @@ -563,12 +547,12 @@ def test_parse_certificates(self, mock_decrypt_certs): sslmgr = azure_helper.OpenSSLManager() keys_by_fp = sslmgr.parse_certificates("") for fp in keys_by_fp.keys(): - self.assertIn(fp, fingerprints) + assert fp in fingerprints for fp in fingerprints: - self.assertIn(fp, keys_by_fp) + assert fp in keys_by_fp -class TestGoalStateHealthReporter(CiTestCase): +class TestGoalStateHealthReporter: maxDiff = None default_parameters = { @@ -590,25 +574,14 @@ class TestGoalStateHealthReporter(CiTestCase): "Test error message containing provisioning failure details" ) - def setUp(self): - super(TestGoalStateHealthReporter, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - - patches.enter_context( - mock.patch.object(azure_helper, "sleep", mock.MagicMock()) - ) - self.read_file_or_url = patches.enter_context( - mock.patch.object(azure_helper.url_helper, "read_file_or_url") - ) - - self.post = patches.enter_context( - mock.patch.object(azure_helper.AzureEndpointHttpClient, "post") - ) - - self.GoalState = patches.enter_context( - mock.patch.object(azure_helper, "GoalState") + @pytest.fixture(autouse=True) + def fixtures(self, mocker): + mocker.patch.object(azure_helper, "sleep", mock.MagicMock()) + mocker.patch.object(azure_helper.url_helper, "read_file_or_url") + self.post = mocker.patch.object( + azure_helper.AzureEndpointHttpClient, "post" ) + self.GoalState = mocker.patch.object(azure_helper, "GoalState") self.GoalState.return_value.container_id = self.default_parameters[ "container_id" ] @@ -667,14 +640,14 @@ def test_send_ready_signal_sends_post_request(self): ) reporter.send_ready_signal() - self.assertEqual(1, self.post.call_count) - self.assertEqual( + assert 1 == self.post.call_count + assert ( mock.call( self.test_health_report_url, data=m_build_report.return_value, extra_headers=self.test_default_headers, - ), - self.post.call_args, + ) + == self.post.call_args ) def test_send_failure_signal_sends_post_request(self): @@ -691,14 +664,14 @@ def test_send_failure_signal_sends_post_request(self): description=self.provisioning_failure_err_description ) - self.assertEqual(1, self.post.call_count) - self.assertEqual( + assert 1 == self.post.call_count + assert ( mock.call( self.test_health_report_url, data=m_build_report.return_value, extra_headers=self.test_default_headers, - ), - self.post.call_args, + ) + == self.post.call_args ) def test_build_report_for_ready_signal_health_document(self): @@ -715,51 +688,42 @@ def test_build_report_for_ready_signal_health_document(self): status=self.provisioning_success_status, ) - self.assertEqual(health_document, generated_health_document) + assert health_document == generated_health_document generated_xroot = ET.fromstring(generated_health_document) - self.assertEqual( - self._text_from_xpath_in_xroot( - generated_xroot, "./GoalStateIncarnation" - ), - str(self.default_parameters["incarnation"]), - ) - self.assertEqual( - self._text_from_xpath_in_xroot( - generated_xroot, "./Container/ContainerId" - ), - str(self.default_parameters["container_id"]), - ) - self.assertEqual( - self._text_from_xpath_in_xroot( - generated_xroot, "./Container/RoleInstanceList/Role/InstanceId" - ), - str(self.default_parameters["instance_id"]), - ) - self.assertEqual( - self._text_from_xpath_in_xroot( - generated_xroot, - "./Container/RoleInstanceList/Role/Health/State", - ), - escape(self.provisioning_success_status), - ) - self.assertIsNone( + assert self._text_from_xpath_in_xroot( + generated_xroot, "./GoalStateIncarnation" + ) == str(self.default_parameters["incarnation"]) + assert self._text_from_xpath_in_xroot( + generated_xroot, "./Container/ContainerId" + ) == str(self.default_parameters["container_id"]) + assert self._text_from_xpath_in_xroot( + generated_xroot, "./Container/RoleInstanceList/Role/InstanceId" + ) == str(self.default_parameters["instance_id"]) + assert self._text_from_xpath_in_xroot( + generated_xroot, + "./Container/RoleInstanceList/Role/Health/State", + ) == escape(self.provisioning_success_status) + assert ( self._text_from_xpath_in_xroot( generated_xroot, "./Container/RoleInstanceList/Role/Health/Details", ) + is None ) - self.assertIsNone( + assert ( self._text_from_xpath_in_xroot( generated_xroot, "./Container/RoleInstanceList/Role/Health/Details/SubStatus", ) + is None ) - self.assertIsNone( + assert ( self._text_from_xpath_in_xroot( generated_xroot, "./Container/RoleInstanceList/Role/Health/Details/Description", ) + is None ) def test_build_report_for_failure_signal_health_document(self): @@ -778,48 +742,36 @@ def test_build_report_for_failure_signal_health_document(self): description=self.provisioning_failure_err_description, ) - self.assertEqual(health_document, generated_health_document) + assert health_document == generated_health_document generated_xroot = ET.fromstring(generated_health_document) - self.assertEqual( - self._text_from_xpath_in_xroot( - generated_xroot, "./GoalStateIncarnation" - ), - str(self.default_parameters["incarnation"]), - ) - self.assertEqual( + assert self._text_from_xpath_in_xroot( + generated_xroot, "./GoalStateIncarnation" + ) == str(self.default_parameters["incarnation"]) + assert ( self._text_from_xpath_in_xroot( generated_xroot, "./Container/ContainerId" - ), - self.default_parameters["container_id"], + ) + == self.default_parameters["container_id"] ) - self.assertEqual( + assert ( self._text_from_xpath_in_xroot( generated_xroot, "./Container/RoleInstanceList/Role/InstanceId" - ), - self.default_parameters["instance_id"], - ) - self.assertEqual( - self._text_from_xpath_in_xroot( - generated_xroot, - "./Container/RoleInstanceList/Role/Health/State", - ), - escape(self.provisioning_not_ready_status), - ) - self.assertEqual( - self._text_from_xpath_in_xroot( - generated_xroot, - "./Container/RoleInstanceList/Role/Health/Details/SubStatus", - ), - escape(self.provisioning_failure_substatus), - ) - self.assertEqual( - self._text_from_xpath_in_xroot( - generated_xroot, - "./Container/RoleInstanceList/Role/Health/Details/Description", - ), - escape(self.provisioning_failure_err_description), + ) + == self.default_parameters["instance_id"] ) + assert self._text_from_xpath_in_xroot( + generated_xroot, + "./Container/RoleInstanceList/Role/Health/State", + ) == escape(self.provisioning_not_ready_status) + assert self._text_from_xpath_in_xroot( + generated_xroot, + "./Container/RoleInstanceList/Role/Health/Details/SubStatus", + ) == escape(self.provisioning_failure_substatus) + assert self._text_from_xpath_in_xroot( + generated_xroot, + "./Container/RoleInstanceList/Role/Health/Details/Description", + ) == escape(self.provisioning_failure_err_description) def test_send_ready_signal_calls_build_report(self): with mock.patch.object( @@ -832,15 +784,15 @@ def test_send_ready_signal_calls_build_report(self): ) reporter.send_ready_signal() - self.assertEqual(1, m_build_report.call_count) - self.assertEqual( + assert 1 == m_build_report.call_count + assert ( mock.call( incarnation=self.default_parameters["incarnation"], container_id=self.default_parameters["container_id"], instance_id=self.default_parameters["instance_id"], status=self.provisioning_success_status, - ), - m_build_report.call_args, + ) + == m_build_report.call_args ) def test_send_failure_signal_calls_build_report(self): @@ -856,8 +808,8 @@ def test_send_failure_signal_calls_build_report(self): description=self.provisioning_failure_err_description ) - self.assertEqual(1, m_build_report.call_count) - self.assertEqual( + assert 1 == m_build_report.call_count + assert ( mock.call( incarnation=self.default_parameters["incarnation"], container_id=self.default_parameters["container_id"], @@ -865,8 +817,8 @@ def test_send_failure_signal_calls_build_report(self): status=self.provisioning_not_ready_status, substatus=self.provisioning_failure_substatus, description=self.provisioning_failure_err_description, - ), - m_build_report.call_args, + ) + == m_build_report.call_args ) def test_build_report_escapes_chars(self): @@ -905,7 +857,7 @@ def test_build_report_escapes_chars(self): description=health_description, ) - self.assertEqual(health_document, generated_health_document) + assert health_document == generated_health_document def test_build_report_conforms_to_length_limits(self): reporter = azure_helper.GoalStateHealthReporter( @@ -928,9 +880,9 @@ def test_build_report_conforms_to_length_limits(self): generated_xroot, "./Container/RoleInstanceList/Role/Health/Details/Description", ) - self.assertEqual( - len(unescape(generated_health_report_description)), - HEALTH_REPORT_DESCRIPTION_TRIM_LEN, + assert ( + len(unescape(generated_health_report_description)) + == HEALTH_REPORT_DESCRIPTION_TRIM_LEN ) def test_trim_description_then_escape_conforms_to_len_limits_worst_case( @@ -983,27 +935,20 @@ def test_trim_description_then_escape_conforms_to_len_limits_worst_case( ) # The escaped description string should be less than # the Azure platform limit for the escaped description string. - self.assertLessEqual(len(generated_health_report_description), 4096) - + assert len(generated_health_report_description) <= 4096 -class TestWALinuxAgentShim(CiTestCase): - def setUp(self): - super(TestWALinuxAgentShim, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - self.AzureEndpointHttpClient = patches.enter_context( - mock.patch.object(azure_helper, "AzureEndpointHttpClient") - ) - self.GoalState = patches.enter_context( - mock.patch.object(azure_helper, "GoalState") - ) - self.OpenSSLManager = patches.enter_context( - mock.patch.object(azure_helper, "OpenSSLManager", autospec=True) +class TestWALinuxAgentShim: + @pytest.fixture(autouse=True) + def fixtures(self, mocker): + self.AzureEndpointHttpClient = mocker.patch.object( + azure_helper, "AzureEndpointHttpClient" ) - patches.enter_context( - mock.patch.object(azure_helper, "sleep", mock.MagicMock()) + self.GoalState = mocker.patch.object(azure_helper, "GoalState") + self.OpenSSLManager = mocker.patch.object( + azure_helper, "OpenSSLManager", autospec=True ) + mocker.patch.object(azure_helper, "sleep", mock.MagicMock()) self.test_incarnation = "TestIncarnation" self.test_container_id = "TestContainerId" @@ -1026,54 +971,42 @@ def test_eject_iso_is_called(self): def test_http_client_does_not_use_certificate_for_report_ready(self): shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data(distro=None) - self.assertEqual( - [mock.call(None)], self.AzureEndpointHttpClient.call_args_list - ) + assert [mock.call(None)] == self.AzureEndpointHttpClient.call_args_list def test_http_client_does_not_use_certificate_for_report_failure(self): shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") - self.assertEqual( - [mock.call(None)], self.AzureEndpointHttpClient.call_args_list - ) + assert [mock.call(None)] == self.AzureEndpointHttpClient.call_args_list def test_correct_url_used_for_goalstate_during_report_ready(self): shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data(distro=None) m_get = self.AzureEndpointHttpClient.return_value.get - self.assertEqual( - [mock.call("http://test_endpoint/machine/?comp=goalstate")], - m_get.call_args_list, - ) - self.assertEqual( - [ - mock.call( - m_get.return_value.contents, - self.AzureEndpointHttpClient.return_value, - False, - ) - ], - self.GoalState.call_args_list, - ) + assert [ + mock.call("http://test_endpoint/machine/?comp=goalstate") + ] == m_get.call_args_list + assert [ + mock.call( + m_get.return_value.contents, + self.AzureEndpointHttpClient.return_value, + False, + ) + ] == self.GoalState.call_args_list def test_correct_url_used_for_goalstate_during_report_failure(self): shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") m_get = self.AzureEndpointHttpClient.return_value.get - self.assertEqual( - [mock.call("http://test_endpoint/machine/?comp=goalstate")], - m_get.call_args_list, - ) - self.assertEqual( - [ - mock.call( - m_get.return_value.contents, - self.AzureEndpointHttpClient.return_value, - False, - ) - ], - self.GoalState.call_args_list, - ) + assert [ + mock.call("http://test_endpoint/machine/?comp=goalstate") + ] == m_get.call_args_list + assert [ + mock.call( + m_get.return_value.contents, + self.AzureEndpointHttpClient.return_value, + False, + ) + ] == self.GoalState.call_args_list def test_certificates_used_to_determine_public_keys(self): # if register_with_azure_and_fetch_data() isn't passed some info about @@ -1094,13 +1027,12 @@ def test_certificates_used_to_determine_public_keys(self): data = shim.register_with_azure_and_fetch_data( distro=None, pubkey_info=mypk ) - self.assertEqual( - [mock.call(self.GoalState.return_value.certificates_xml)], - sslmgr.parse_certificates.call_args_list, - ) - self.assertIn("expected-key", data) - self.assertIn("expected-no-value-key", data) - self.assertNotIn("should-not-be-found", data) + assert [ + mock.call(self.GoalState.return_value.certificates_xml) + ] == sslmgr.parse_certificates.call_args_list + assert "expected-key" in data + assert "expected-no-value-key" in data + assert "should-not-be-found" not in data def test_absent_certificates_produces_empty_public_keys(self): mypk = [{"fingerprint": "fp1", "path": "path1"}] @@ -1109,25 +1041,23 @@ def test_absent_certificates_produces_empty_public_keys(self): data = shim.register_with_azure_and_fetch_data( distro=None, pubkey_info=mypk ) - self.assertEqual([], data) + assert [] == data def test_correct_url_used_for_report_ready(self): shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data(distro=None) expected_url = "http://test_endpoint/machine?comp=health" - self.assertEqual( - [mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY)], - self.AzureEndpointHttpClient.return_value.post.call_args_list, - ) + assert [ + mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY) + ] == self.AzureEndpointHttpClient.return_value.post.call_args_list def test_correct_url_used_for_report_failure(self): shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_report_failure(description="TestDesc") expected_url = "http://test_endpoint/machine?comp=health" - self.assertEqual( - [mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY)], - self.AzureEndpointHttpClient.return_value.post.call_args_list, - ) + assert [ + mock.call(expected_url, data=mock.ANY, extra_headers=mock.ANY) + ] == self.AzureEndpointHttpClient.return_value.post.call_args_list def test_goal_state_values_used_for_report_ready(self): shim = wa_shim(endpoint="test_endpoint") @@ -1135,9 +1065,9 @@ def test_goal_state_values_used_for_report_ready(self): posted_document = ( self.AzureEndpointHttpClient.return_value.post.call_args[1]["data"] ) - self.assertIn(self.test_incarnation.encode("utf-8"), posted_document) - self.assertIn(self.test_container_id.encode("utf-8"), posted_document) - self.assertIn(self.test_instance_id.encode("utf-8"), posted_document) + assert self.test_incarnation.encode("utf-8") in posted_document + assert self.test_container_id.encode("utf-8") in posted_document + assert self.test_instance_id.encode("utf-8") in posted_document def test_goal_state_values_used_for_report_failure(self): shim = wa_shim(endpoint="test_endpoint") @@ -1145,9 +1075,9 @@ def test_goal_state_values_used_for_report_failure(self): posted_document = ( self.AzureEndpointHttpClient.return_value.post.call_args[1]["data"] ) - self.assertIn(self.test_incarnation.encode("utf-8"), posted_document) - self.assertIn(self.test_container_id.encode("utf-8"), posted_document) - self.assertIn(self.test_instance_id.encode("utf-8"), posted_document) + assert self.test_incarnation.encode("utf-8") in posted_document + assert self.test_container_id.encode("utf-8") in posted_document + assert self.test_instance_id.encode("utf-8") in posted_document def test_xml_elems_in_report_ready_post(self): shim = wa_shim(endpoint="test_endpoint") @@ -1162,7 +1092,7 @@ def test_xml_elems_in_report_ready_post(self): posted_document = ( self.AzureEndpointHttpClient.return_value.post.call_args[1]["data"] ) - self.assertEqual(health_document, posted_document) + assert health_document == posted_document def test_xml_elems_in_report_failure_post(self): shim = wa_shim(endpoint="test_endpoint") @@ -1182,7 +1112,7 @@ def test_xml_elems_in_report_failure_post(self): posted_document = ( self.AzureEndpointHttpClient.return_value.post.call_args[1]["data"] ) - self.assertEqual(health_document, posted_document) + assert health_document == posted_document @mock.patch.object(azure_helper, "GoalStateHealthReporter", autospec=True) def test_register_with_azure_and_fetch_data_calls_send_ready_signal( @@ -1190,9 +1120,9 @@ def test_register_with_azure_and_fetch_data_calls_send_ready_signal( ): shim = wa_shim(endpoint="test_endpoint") shim.register_with_azure_and_fetch_data(distro=None) - self.assertEqual( - 1, - m_goal_state_health_reporter.return_value.send_ready_signal.call_count, # noqa: E501 + assert ( + 1 + == m_goal_state_health_reporter.return_value.send_ready_signal.call_count # noqa: E501 ) @mock.patch.object(azure_helper, "GoalStateHealthReporter", autospec=True) @@ -1245,96 +1175,84 @@ def test_fetch_goalstate_during_report_ready_raises_exc_on_get_exc(self): url_helper.UrlError("retry", code=404) ) shim = wa_shim(endpoint="test_endpoint") - self.assertRaises( - url_helper.UrlError, shim.register_with_azure_and_fetch_data, None - ) + with pytest.raises(url_helper.UrlError): + shim.register_with_azure_and_fetch_data(None) def test_fetch_goalstate_during_report_failure_raises_exc_on_get_exc(self): self.AzureEndpointHttpClient.return_value.get.side_effect = ( url_helper.UrlError("retry", code=404) ) shim = wa_shim(endpoint="test_endpoint") - self.assertRaises( - url_helper.UrlError, - shim.register_with_azure_and_report_failure, - description="TestDesc", - ) + with pytest.raises(url_helper.UrlError): + shim.register_with_azure_and_report_failure( + description="TestDesc", + ) def test_fetch_goalstate_during_report_ready_raises_exc_on_parse_exc(self): self.GoalState.side_effect = url_helper.UrlError("retry", code=404) shim = wa_shim(endpoint="test_endpoint") - self.assertRaises( - url_helper.UrlError, shim.register_with_azure_and_fetch_data, None - ) + with pytest.raises(url_helper.UrlError): + shim.register_with_azure_and_fetch_data(None) def test_fetch_goalstate_during_report_failure_raises_exc_on_parse_exc( self, ): self.GoalState.side_effect = url_helper.UrlError("retry", code=404) shim = wa_shim(endpoint="test_endpoint") - self.assertRaises( - url_helper.UrlError, - shim.register_with_azure_and_report_failure, - description="TestDesc", - ) + with pytest.raises(url_helper.UrlError): + shim.register_with_azure_and_report_failure( + description="TestDesc", + ) def test_failure_to_send_report_ready_health_doc_bubbles_up(self): self.AzureEndpointHttpClient.return_value.post.side_effect = ( url_helper.UrlError("retry", code=404) ) shim = wa_shim(endpoint="test_endpoint") - self.assertRaises( - url_helper.UrlError, shim.register_with_azure_and_fetch_data, None - ) + with pytest.raises(url_helper.UrlError): + shim.register_with_azure_and_fetch_data(None) def test_failure_to_send_report_failure_health_doc_bubbles_up(self): self.AzureEndpointHttpClient.return_value.post.side_effect = ( url_helper.UrlError("retry", code=404) ) shim = wa_shim(endpoint="test_endpoint") - self.assertRaises( - url_helper.UrlError, - shim.register_with_azure_and_report_failure, - description="TestDesc", - ) - + with pytest.raises(url_helper.UrlError): + shim.register_with_azure_and_report_failure( + description="TestDesc", + ) -class TestGetMetadataGoalStateXMLAndReportReadyToFabric(CiTestCase): - def setUp(self): - super(TestGetMetadataGoalStateXMLAndReportReadyToFabric, self).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - self.m_shim = patches.enter_context( - mock.patch.object(azure_helper, "WALinuxAgentShim") - ) +class TestGetMetadataGoalStateXMLAndReportReadyToFabric: + @pytest.fixture(autouse=True) + def fixtures(self, mocker): + self.m_shim = mocker.patch.object(azure_helper, "WALinuxAgentShim") def test_data_from_shim_returned(self): ret = azure_helper.get_metadata_from_fabric( distro=None, endpoint="test_endpoint" ) - self.assertEqual( - self.m_shim.return_value.register_with_azure_and_fetch_data.return_value, # noqa: E501 - ret, + assert ( + self.m_shim.return_value.register_with_azure_and_fetch_data.return_value + == ret # noqa: E501 ) def test_success_calls_clean_up(self): azure_helper.get_metadata_from_fabric( distro=None, endpoint="test_endpoint" ) - self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) + assert 1 == self.m_shim.return_value.clean_up.call_count def test_failure_in_registration_propagates_exc_and_calls_clean_up(self): self.m_shim.return_value.register_with_azure_and_fetch_data.side_effect = url_helper.UrlError( # noqa: E501 "retry", code=404 ) - self.assertRaises( - url_helper.UrlError, - azure_helper.get_metadata_from_fabric, - "test_endpoint", - None, - ) - self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) + with pytest.raises(url_helper.UrlError): + azure_helper.get_metadata_from_fabric( + "test_endpoint", + None, + ) + assert 1 == self.m_shim.return_value.clean_up.call_count def test_calls_shim_register_with_azure_and_fetch_data(self): m_pubkey_info = mock.MagicMock() @@ -1344,46 +1262,35 @@ def test_calls_shim_register_with_azure_and_fetch_data(self): pubkey_info=m_pubkey_info, iso_dev="/dev/sr0", ) - self.assertEqual( - 1, - self.m_shim.return_value.register_with_azure_and_fetch_data.call_count, # noqa: E501 + assert ( + 1 + == self.m_shim.return_value.register_with_azure_and_fetch_data.call_count # noqa: E501 ) - self.assertEqual( + assert ( mock.call( distro=None, iso_dev="/dev/sr0", pubkey_info=m_pubkey_info - ), - self.m_shim.return_value.register_with_azure_and_fetch_data.call_args, # noqa: E501 + ) + == self.m_shim.return_value.register_with_azure_and_fetch_data.call_args # noqa: E501 ) def test_instantiates_shim_with_kwargs(self): azure_helper.get_metadata_from_fabric( endpoint="test_endpoint", distro=None ) - self.assertEqual(1, self.m_shim.call_count) - self.assertEqual( - mock.call(endpoint="test_endpoint"), - self.m_shim.call_args, - ) - + assert 1 == self.m_shim.call_count + assert mock.call(endpoint="test_endpoint") == self.m_shim.call_args -class TestGetMetadataGoalStateXMLAndReportFailureToFabric(CiTestCase): - def setUp(self): - super( - TestGetMetadataGoalStateXMLAndReportFailureToFabric, self - ).setUp() - patches = ExitStack() - self.addCleanup(patches.close) - self.m_shim = patches.enter_context( - mock.patch.object(azure_helper, "WALinuxAgentShim") - ) +class TestGetMetadataGoalStateXMLAndReportFailureToFabric: + @pytest.fixture(autouse=True) + def fixtures(self, mocker): + self.m_shim = mocker.patch.object(azure_helper, "WALinuxAgentShim") def test_success_calls_clean_up(self): - error = errors.ReportableError(reason="test") azure_helper.report_failure_to_fabric( - endpoint="test_endpoint", error=error + endpoint="test_endpoint", encoded_report="test" ) - self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) + assert 1 == self.m_shim.return_value.clean_up.call_count def test_failure_in_shim_report_failure_propagates_exc_and_calls_clean_up( self, @@ -1391,33 +1298,30 @@ def test_failure_in_shim_report_failure_propagates_exc_and_calls_clean_up( self.m_shim.return_value.register_with_azure_and_report_failure.side_effect = ( # noqa: E501 SentinelException ) - self.assertRaises( - SentinelException, - azure_helper.report_failure_to_fabric, - "test_endpoint", - errors.ReportableError(reason="test"), - ) - self.assertEqual(1, self.m_shim.return_value.clean_up.call_count) + with pytest.raises(SentinelException): + azure_helper.report_failure_to_fabric( + "test_endpoint", + encoded_report="test-report", + ) + assert 1 == self.m_shim.return_value.clean_up.call_count def test_report_failure_to_fabric_calls_shim_report_failure( self, ): - error = errors.ReportableError(reason="test") - azure_helper.report_failure_to_fabric( endpoint="test_endpoint", - error=error, + encoded_report="test", ) # default err message description should be shown to the user # if an empty description is passed in self.m_shim.return_value.register_with_azure_and_report_failure.assert_called_once_with( # noqa: E501 - description=error.as_encoded_report(), + description="test", ) def test_instantiates_shim_with_kwargs(self): azure_helper.report_failure_to_fabric( endpoint="test_endpoint", - error=errors.ReportableError(reason="test"), + encoded_report="test", ) self.m_shim.assert_called_once_with( endpoint="test_endpoint", @@ -1570,11 +1474,6 @@ def test_valid_ovf_scenarios(self, ovf, expected): @pytest.mark.parametrize( "ovf,error", [ - ( - construct_ovf_env(username=None), - "unexpected metadata parsing ovf-env.xml: " - "missing configuration for 'UserName'", - ), ( construct_ovf_env(hostname=None), "unexpected metadata parsing ovf-env.xml: " diff --git a/tests/unittests/sources/test_cloudsigma.py b/tests/unittests/sources/test_cloudsigma.py index 0410c3c7..c63bfbd0 100644 --- a/tests/unittests/sources/test_cloudsigma.py +++ b/tests/unittests/sources/test_cloudsigma.py @@ -4,10 +4,11 @@ import copy from unittest import mock -from cloudinit import distros, helpers, importer, sources +import pytest + +from cloudinit import distros, importer, sources from cloudinit.sources import DataSourceCloudSigma from cloudinit.sources.helpers.cloudsigma import Cepko -from tests.unittests import helpers as test_helpers SERVER_CONTEXT = { "cpu": 1000, @@ -40,104 +41,89 @@ def all(self): return self -class DataSourceCloudSigmaTest(test_helpers.CiTestCase): - def setUp(self): - super(DataSourceCloudSigmaTest, self).setUp() - self.paths = helpers.Paths({"run_dir": self.tmp_dir()}) - self.add_patch( - DS_PATH + ".override_ds_detect", - "m_is_container", - return_value=True, - ) +@pytest.fixture +def ds(mocker, paths): + mocker.patch(DS_PATH + ".override_ds_detect", return_value=True) + distro_cls = distros.fetch("ubuntu") + distro = distro_cls("ubuntu", cfg={}, paths=paths) + datasource = DataSourceCloudSigma.DataSourceCloudSigma( + sys_cfg={}, distro=distro, paths=paths + ) + datasource.cepko = CepkoMock(SERVER_CONTEXT) + return datasource - distro_cls = distros.fetch("ubuntu") - distro = distro_cls("ubuntu", cfg={}, paths=self.paths) - self.datasource = DataSourceCloudSigma.DataSourceCloudSigma( - sys_cfg={}, distro=distro, paths=self.paths - ) - self.datasource.cepko = CepkoMock(SERVER_CONTEXT) - def test_get_hostname(self): - self.datasource.get_data() - self.assertEqual( - "test_server", self.datasource.get_hostname().hostname - ) - self.datasource.metadata["name"] = "" - self.assertEqual("65b2fb23", self.datasource.get_hostname().hostname) +class TestDataSourceCloudSigma: + + def test_get_hostname(self, ds): + ds.get_data() + assert "test_server" == ds.get_hostname().hostname + ds.metadata["name"] = "" + assert "65b2fb23" == ds.get_hostname().hostname utf8_hostname = b"\xd1\x82\xd0\xb5\xd1\x81\xd1\x82".decode("utf-8") - self.datasource.metadata["name"] = utf8_hostname - self.assertEqual("65b2fb23", self.datasource.get_hostname().hostname) - - def test_get_public_ssh_keys(self): - self.datasource.get_data() - self.assertEqual( - [SERVER_CONTEXT["meta"]["ssh_public_key"]], - self.datasource.get_public_ssh_keys(), - ) + ds.metadata["name"] = utf8_hostname + assert "65b2fb23" == ds.get_hostname().hostname - def test_get_instance_id(self): - self.datasource.get_data() - self.assertEqual( - SERVER_CONTEXT["uuid"], self.datasource.get_instance_id() - ) + def test_get_public_ssh_keys(self, ds): + ds.get_data() + assert [ + SERVER_CONTEXT["meta"]["ssh_public_key"] + ] == ds.get_public_ssh_keys() + + def test_get_instance_id(self, ds): + ds.get_data() + assert SERVER_CONTEXT["uuid"] == ds.get_instance_id() - def test_platform(self): + def test_platform(self, ds): """All platform-related attributes are set.""" - self.datasource.get_data() - self.assertEqual(self.datasource.cloud_name, "cloudsigma") - self.assertEqual(self.datasource.platform_type, "cloudsigma") - self.assertEqual(self.datasource.subplatform, "cepko (/dev/ttyS1)") - - def test_metadata(self): - self.datasource.get_data() - self.assertEqual(self.datasource.metadata, SERVER_CONTEXT) - - def test_user_data(self): - self.datasource.get_data() - self.assertEqual( - self.datasource.userdata_raw, - SERVER_CONTEXT["meta"]["cloudinit-user-data"], - ) + ds.get_data() + assert ds.cloud_name == "cloudsigma" + assert ds.platform_type == "cloudsigma" + assert ds.subplatform == "cepko (/dev/ttyS1)" + + def test_metadata(self, ds): + ds.get_data() + assert ds.metadata == SERVER_CONTEXT - def test_encoded_user_data(self): + def test_user_data(self, ds): + ds.get_data() + assert ds.userdata_raw == SERVER_CONTEXT["meta"]["cloudinit-user-data"] + def test_encoded_user_data(self, ds): encoded_context = copy.deepcopy(SERVER_CONTEXT) encoded_context["meta"]["base64_fields"] = "cloudinit-user-data" encoded_context["meta"]["cloudinit-user-data"] = "aGkgd29ybGQK" - self.datasource.cepko = CepkoMock(encoded_context) - self.datasource.get_data() + ds.cepko = CepkoMock(encoded_context) + ds.get_data() - self.assertEqual(self.datasource.userdata_raw, b"hi world\n") + assert ds.userdata_raw == b"hi world\n" - def test_vendor_data(self): - self.datasource.get_data() - self.assertEqual( - self.datasource.vendordata_raw, - SERVER_CONTEXT["vendor_data"]["cloudinit"], - ) + def test_vendor_data(self, ds): + ds.get_data() + assert ds.vendordata_raw == SERVER_CONTEXT["vendor_data"]["cloudinit"] - def test_lack_of_vendor_data(self): + def test_lack_of_vendor_data(self, ds): stripped_context = copy.deepcopy(SERVER_CONTEXT) del stripped_context["vendor_data"] - self.datasource.cepko = CepkoMock(stripped_context) - self.datasource.get_data() + ds.cepko = CepkoMock(stripped_context) + ds.get_data() - self.assertIsNone(self.datasource.vendordata_raw) + assert ds.vendordata_raw is None - def test_lack_of_cloudinit_key_in_vendor_data(self): + def test_lack_of_cloudinit_key_in_vendor_data(self, ds): stripped_context = copy.deepcopy(SERVER_CONTEXT) del stripped_context["vendor_data"]["cloudinit"] - self.datasource.cepko = CepkoMock(stripped_context) - self.datasource.get_data() + ds.cepko = CepkoMock(stripped_context) + ds.get_data() - self.assertIsNone(self.datasource.vendordata_raw) + assert ds.vendordata_raw is None -class DsLoads(test_helpers.TestCase): +class TestDsLoads: def test_get_datasource_list_returns_in_local(self): deps = (sources.DEP_FILESYSTEM,) ds_list = DataSourceCloudSigma.get_datasource_list(deps) - self.assertEqual(ds_list, [DataSourceCloudSigma.DataSourceCloudSigma]) + assert ds_list == [DataSourceCloudSigma.DataSourceCloudSigma] @mock.patch.object( importer, @@ -150,4 +136,4 @@ def test_list_sources_finds_ds(self): (sources.DEP_FILESYSTEM,), ["cloudinit.sources"], ) - self.assertEqual([DataSourceCloudSigma.DataSourceCloudSigma], found) + assert [DataSourceCloudSigma.DataSourceCloudSigma] == found diff --git a/tests/unittests/sources/test_cloudstack.py b/tests/unittests/sources/test_cloudstack.py index a64d80f3..990c8a7f 100644 --- a/tests/unittests/sources/test_cloudstack.py +++ b/tests/unittests/sources/test_cloudstack.py @@ -1,147 +1,99 @@ # This file is part of cloud-init. See LICENSE file for license information. +from socket import gaierror from textwrap import dedent import pytest from cloudinit import helpers -from cloudinit.distros import rhel, ubuntu +from cloudinit.net.dhcp import NoDHCPLeaseError from cloudinit.sources import DataSourceHostname -from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack -from tests.unittests.helpers import CiTestCase, ExitStack, mock +from cloudinit.sources.DataSourceCloudStack import ( + CLOUD_STACK_DMI_NAME, + DataSourceCloudStack, + DataSourceCloudStackLocal, + get_data_server, + get_vr_address, +) +from tests.unittests.helpers import mock from tests.unittests.util import MockDistro SOURCES_PATH = "cloudinit.sources" MOD_PATH = SOURCES_PATH + ".DataSourceCloudStack" DS_PATH = MOD_PATH + ".DataSourceCloudStack" DHCP_MOD_PATH = "cloudinit.net.dhcp" +FAKE_LEASE = { + "interface": "eth0", + "fixed-address": "192.168.0.1", + "subnet-mask": "255.255.255.0", + "routers": "192.168.0.1", + "domain-name": "dhclient.local", + "renew": "4 2017/07/27 18:02:30", + "expire": "5 2017/07/28 07:08:15", +} + +FAKE_LEASE_WITH_SERVER_IDENT = """\ +lease { + interface "eth0"; + fixed-address 10.0.0.5; + server-name "DSM111070915004"; + option subnet-mask 255.255.255.0; + option dhcp-lease-time 4294967295; + option routers 10.0.0.1; + option dhcp-message-type 5; + option dhcp-server-identifier 168.63.129.16; + option domain-name-servers 168.63.129.16; + option dhcp-renewal-time 4294967295; + option rfc3442-classless-static-routes 0,10,0,0,1,32,168,63,129,16,10,0,0,1,32,169,254,169,254,10,0,0,1; + option unknown-245 a8:3f:81:10; + option dhcp-rebinding-time 4294967295; + renew 0 2160/02/17 02:22:33; + rebind 0 2160/02/17 02:22:33; + expire 0 2160/02/17 02:22:33; +} +""" # noqa: E501 + + +@pytest.fixture +def cloudstack_ds(request, paths): + yield DataSourceCloudStack(sys_cfg={}, distro=MockDistro(), paths=paths) @pytest.mark.usefixtures("dhclient_exists") -class TestCloudStackHostname(CiTestCase): - def setUp(self): - super(TestCloudStackHostname, self).setUp() - self.patches = ExitStack() - self.addCleanup(self.patches.close) +class TestCloudStackHostname: + @pytest.fixture(autouse=True) + def setup(self, mocker, tmp_path): self.hostname = "vm-hostname" self.networkd_domainname = "networkd.local" self.isc_dhclient_domainname = "dhclient.local" - # Mock the parent class get_hostname() method to return - # a non-fqdn hostname get_hostname_parent = mock.MagicMock( return_value=DataSourceHostname(self.hostname, True) ) - self.patches.enter_context( - mock.patch( - SOURCES_PATH + ".DataSource.get_hostname", get_hostname_parent - ) + mocker.patch( + SOURCES_PATH + ".DataSource.get_hostname", get_hostname_parent ) - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".util.load_text_file", - return_value=dedent( - """ - lease { - interface "eth0"; - fixed-address 10.0.0.5; - server-name "DSM111070915004"; - option subnet-mask 255.255.255.0; - option dhcp-lease-time 4294967295; - option routers 10.0.0.1; - option dhcp-message-type 5; - option dhcp-server-identifier 168.63.129.16; - option domain-name-servers 168.63.129.16; - option dhcp-renewal-time 4294967295; - option rfc3442-classless-static-routes """ - """0,10,0,0,1,32,168,63,129,16,10,0,0,1,32,169,254,""" - """169,254,10,0,0,1; - option unknown-245 a8:3f:81:10; - option dhcp-rebinding-time 4294967295; - """ - """renew 0 2160/02/17 02:22:33; - rebind 0 2160/02/17 02:22:33; - expire 0 2160/02/17 02:22:33; - } - """ - ), - ) + mocker.patch( + DHCP_MOD_PATH + ".util.load_text_file", + return_value=FAKE_LEASE_WITH_SERVER_IDENT, ) - # Mock cloudinit.net.dhcp.networkd_get_option_from_leases() method \ # result since we don't have a DHCP client running networkd_get_option_from_leases = mock.MagicMock( return_value=self.networkd_domainname ) - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".networkd_get_option_from_leases", - networkd_get_option_from_leases, - ) - ) - - # Mock cloudinit.net.dhcp.get_newest_lease_file_from_distro() method \ - # result since we don't have a DHCP client running - isc_dhclient_get_newest_lease_file_from_distro = mock.MagicMock( - return_value="/var/lib/NetworkManager/dhclient-u-u-i-d-eth0.lease" - ) - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH - + ".IscDhclient.get_newest_lease_file_from_distro", - isc_dhclient_get_newest_lease_file_from_distro, - ) - ) - - # Mock cloudinit.net.dhcp.networkd_get_option_from_leases() method \ - # result since we don't have a DHCP client running - lease = { - "interface": "eth0", - "fixed-address": "192.168.0.1", - "subnet-mask": "255.255.255.0", - "routers": "192.168.0.1", - "domain-name": self.isc_dhclient_domainname, - "renew": "4 2017/07/27 18:02:30", - "expire": "5 2017/07/28 07:08:15", - } - get_newest_lease = mock.MagicMock(return_value=lease) - - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", - get_newest_lease, - ) - ) - - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".IscDhclient.parse_leases", - mock.MagicMock(return_value=[lease]), - ) - ) - - # Mock get_vr_address() method as it relies to - # parsing DHCP/networkd files - self.patches.enter_context( - mock.patch( - MOD_PATH + ".get_vr_address", - mock.MagicMock(return_value="192.168.0.1"), - ) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + networkd_get_option_from_leases, ) - self.tmp = self.tmp_dir() - - def test_get_domainname_networkd(self): + def test_get_domainname_networkd(self, cloudstack_ds): """ Test if DataSourceCloudStack._get_domainname() gets domain name from systemd-networkd leases. """ - ds = DataSourceCloudStack( - {}, ubuntu.Distro, helpers.Paths({"run_dir": self.tmp}) - ) - result = ds._get_domainname() - self.assertEqual(self.networkd_domainname, result) + assert self.networkd_domainname == cloudstack_ds._get_domainname() - def test_get_domainname_isc_dhclient(self): + def test_get_domainname_isc_dhclient(self, cloudstack_ds, mocker): """ Test if DataSourceCloudStack._get_domainname() gets domain name from isc-dhcp-client leases @@ -150,17 +102,12 @@ def test_get_domainname_isc_dhclient(self): # Override systemd-networkd reply mock to None # to force the code to fallback to IscDhclient get_networkd_domain = mock.MagicMock(return_value=None) - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".networkd_get_option_from_leases", - get_networkd_domain, - ) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, ) - ds = DataSourceCloudStack( - {}, rhel.Distro, helpers.Paths({"run_dir": self.tmp}) - ) - with mock.patch( + with mocker.patch( MOD_PATH + ".util.load_text_file", return_value=dedent( """ @@ -189,10 +136,10 @@ def test_get_domainname_isc_dhclient(self): """ ), ): - result = ds._get_domainname() - self.assertEqual(self.isc_dhclient_domainname, result) + result = cloudstack_ds._get_domainname() + assert self.isc_dhclient_domainname == result - def test_get_hostname_non_fqdn(self): + def test_get_hostname_non_fqdn(self, cloudstack_ds): """ Test get_hostname() method implementation with fqdn parameter=False. @@ -200,14 +147,10 @@ def test_get_hostname_non_fqdn(self): return its response intact. """ expected = DataSourceHostname(self.hostname, True) + result = cloudstack_ds.get_hostname(fqdn=False) + assert expected == result - ds = DataSourceCloudStack( - {}, ubuntu.Distro, helpers.Paths({"run_dir": self.tmp}) - ) - result = ds.get_hostname(fqdn=False) - self.assertTupleEqual(expected, result) - - def test_get_hostname_fqdn(self): + def test_get_hostname_fqdn(self, cloudstack_ds): """ Test get_hostname() method implementation with fqdn parameter=True. @@ -216,14 +159,10 @@ def test_get_hostname_fqdn(self): expected = DataSourceHostname( self.hostname + "." + self.networkd_domainname, True ) + result = cloudstack_ds.get_hostname(fqdn=True) + assert expected == result - ds = DataSourceCloudStack( - {}, ubuntu.Distro, helpers.Paths({"run_dir": self.tmp}) - ) - result = ds.get_hostname(fqdn=True) - self.assertTupleEqual(expected, result) - - def test_get_hostname_fqdn_fallback(self): + def test_get_hostname_fqdn_fallback(self, cloudstack_ds, mocker): """ Test get_hostname() when some error happens during domainname discovery. @@ -240,32 +179,23 @@ def test_get_hostname_fqdn_fallback(self): # Override systemd-networkd reply mock to None # to force the code to fallback to IscDhclient get_networkd_domain = mock.MagicMock(return_value=None) - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".networkd_get_option_from_leases", - get_networkd_domain, - ) + mocker.patch( + DHCP_MOD_PATH + ".networkd_get_option_from_leases", + get_networkd_domain, ) - self.patches.enter_context( - mock.patch( - "cloudinit.distros.net.find_fallback_nic", - return_value="eth0", - ) + mocker.patch( + "cloudinit.distros.net.find_fallback_nic", + return_value="eth0", ) - self.patches.enter_context( - mock.patch( - MOD_PATH - + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", - return_value=True, - ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.get_newest_lease_file_from_distro", + return_value=True, ) - self.patches.enter_context( - mock.patch( - MOD_PATH + ".dhcp.IscDhclient.parse_leases", return_value=[] - ) + mocker.patch( + MOD_PATH + ".dhcp.IscDhclient.parse_leases", return_value=[] ) lease = { @@ -276,155 +206,165 @@ def test_get_hostname_fqdn_fallback(self): "renew": "4 2017/07/27 18:02:30", "expire": "5 2017/07/28 07:08:15", } - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", - return_value=lease, - ) - ) - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".Dhcpcd.get_newest_lease", return_value=lease - ) - ) - - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".util.load_text_file", - return_value=dedent( - """ - lease { - interface "eth0"; - fixed-address 10.0.0.5; - server-name "DSM111070915004"; - option subnet-mask 255.255.255.0; - option dhcp-lease-time 4294967295; - option routers 10.0.0.1; - option dhcp-message-type 5; - option dhcp-server-identifier 168.63.129.16; - option domain-name-servers 168.63.129.16; - option dhcp-renewal-time 4294967295; - option rfc3442-classless-static-routes """ - """0,10,0,0,1,32,168,63,129,16,10,0,0,1,32,169,254,""" - """169,254,10,0,0,1; - option unknown-245 a8:3f:81:10; - option dhcp-rebinding-time 4294967295; - """ - """renew 0 2160/02/17 02:22:33; - rebind 0 2160/02/17 02:22:33; - expire 0 2160/02/17 02:22:33; - } - """ - ), - ) - ) - - ds = DataSourceCloudStack( - {}, ubuntu.Distro("", {}, {}), helpers.Paths({"run_dir": self.tmp}) - ) - ds.distro.fallback_interface = "eth0" - with mock.patch(MOD_PATH + ".util.load_text_file"): - result = ds.get_hostname(fqdn=True) - self.assertTupleEqual(expected, result) + mocker.patch( + DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", + return_value=lease, + ) + mocker.patch( + DHCP_MOD_PATH + ".Dhcpcd.get_newest_lease", return_value=lease + ) + + cloudstack_ds.distro.fallback_interface = "eth0" + with mocker.patch(MOD_PATH + ".util.load_text_file"): + result = cloudstack_ds.get_hostname(fqdn=True) + assert expected == result + + +class TestGetDataServer: + @pytest.mark.parametrize( + "addrinfo,expected,expected_log", + ( + pytest.param( + # Fake addrinfo + [("_", "_", "_", "_", ("10.1.35.171", 80)), "_"], + "10.1.35.171", + None, + id="success_on_dns_resolution", + ), + pytest.param( + gaierror("Name or service not known"), + None, + "DNS Entry data-server not found", + id="none_on_no_dns_resolution", + ), + ), + ) + def test_data_server_from_dns( + self, addrinfo, expected, expected_log, mocker, caplog + ): + """Lookup data-server from DNS.""" + if isinstance(addrinfo, Exception): + mocker.patch(MOD_PATH + ".getaddrinfo", side_effect=addrinfo) + assert expected == get_data_server() + else: + mocker.patch(MOD_PATH + ".getaddrinfo", return_value=addrinfo) + assert expected is get_data_server() + if expected_log: + assert expected_log in caplog.text + + +@mock.patch(MOD_PATH + ".get_data_server", return_value="10.1.37.131") +@mock.patch( + MOD_PATH + ".dhcp.networkd_get_option_from_leases", + return_value="10.1.37.132", +) +class TestGetVrAddress: + def test_get_vr_addr_from_dns( + self, m_networkd_option_from_leases, m_get_data_server, caplog + ): + """cloud-init first obtains data-server if resolved by DNS""" + assert "10.1.37.131" == get_vr_address(MockDistro()) + assert ( + "Found metadata server '10.1.37.131' via data-server DNS entry" + in caplog.text + ) + assert 0 == m_networkd_option_from_leases.call_count + + def test_get_vr_addr_from_networkd_leases( + self, m_networkd_option_from_leases, m_get_data_server, mocker, caplog + ): + """When no DNS for data-server use networkd dhcp-server-identifier""" + mocker.patch(MOD_PATH + ".get_data_server", return_value=None) + assert "10.1.37.132" == get_vr_address(MockDistro()) + assert ( + "Found SERVER_ADDRESS '10.1.37.132' via networkd_leases" + in caplog.text + ) + m_networkd_option_from_leases.assert_called_once_with("SERVER_ADDRESS") @pytest.mark.usefixtures("dhclient_exists") -class TestCloudStackPasswordFetching(CiTestCase): - def setUp(self): - super(TestCloudStackPasswordFetching, self).setUp() - self.patches = ExitStack() - self.addCleanup(self.patches.close) - mod_name = MOD_PATH - self.patches.enter_context(mock.patch("{0}.ec2".format(mod_name))) - self.patches.enter_context(mock.patch("{0}.uhelp".format(mod_name))) +@mock.patch(MOD_PATH + ".dmi.read_dmi_data", return_value=CLOUD_STACK_DMI_NAME) +class TestCloudStackPasswordFetching: + @pytest.fixture(autouse=True) + def setup(self, mocker, tmp_path): + mocker.patch(f"{MOD_PATH}.ec2") + mocker.patch(f"{MOD_PATH}.uhelp") default_gw = "192.201.20.0" - - get_newest_lease_file_from_distro = mock.MagicMock(return_value=None) - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", - return_value={ - "interface": "eth0", - "fixed-address": "192.168.0.1", - "subnet-mask": "255.255.255.0", - "routers": "192.168.0.1", - "renew": "4 2017/07/27 18:02:30", - "expire": "5 2017/07/28 07:08:15", - }, - ) + mocker.patch( + DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", + return_value={ + "interface": "eth0", + "fixed-address": "192.168.0.1", + "subnet-mask": "255.255.255.0", + "routers": "192.168.0.1", + "renew": "4 2017/07/27 18:02:30", + "expire": "5 2017/07/28 07:08:15", + "dhcp-server-identifier": "168.63.129.16", + }, ) - self.patches.enter_context( - mock.patch( - DHCP_MOD_PATH - + ".IscDhclient.get_newest_lease_file_from_distro", - get_newest_lease_file_from_distro, - ) + get_newest_lease_file_from_distro = mock.MagicMock(return_value=None) + mocker.patch( + DHCP_MOD_PATH + ".IscDhclient.get_newest_lease", + return_value={ + "interface": "eth0", + "fixed-address": "192.168.0.1", + "subnet-mask": "255.255.255.0", + "routers": "192.168.0.1", + "renew": "4 2017/07/27 18:02:30", + "expire": "5 2017/07/28 07:08:15", + "dhcp-server-identifier": "168.63.129.16", + }, + ) + mocker.patch( + DHCP_MOD_PATH + ".IscDhclient.get_newest_lease_file_from_distro", + get_newest_lease_file_from_distro, ) - get_default_gw = mock.MagicMock(return_value=default_gw) - self.patches.enter_context( - mock.patch(mod_name + ".get_default_gateway", get_default_gw) - ) + mocker.patch(MOD_PATH + ".get_default_gateway", get_default_gw) get_networkd_server_address = mock.MagicMock(return_value=None) - self.patches.enter_context( - mock.patch( - mod_name + ".dhcp.networkd_get_option_from_leases", - get_networkd_server_address, - ) + mocker.patch( + MOD_PATH + ".dhcp.networkd_get_option_from_leases", + get_networkd_server_address, ) get_data_server = mock.MagicMock(return_value=None) - self.patches.enter_context( - mock.patch(mod_name + ".get_data_server", get_data_server) - ) - - self.tmp = self.tmp_dir() + mocker.patch(MOD_PATH + ".get_data_server", get_data_server) - def _set_password_server_response(self, response_string): + def _set_password_server_response(self, response_string, mocker): subp = mock.MagicMock(return_value=(response_string, "")) - self.patches.enter_context( - mock.patch( - "cloudinit.sources.DataSourceCloudStack.subp.subp", subp - ) - ) + mocker.patch("cloudinit.sources.DataSourceCloudStack.subp.subp", subp) return subp - def test_empty_password_doesnt_create_config(self): - self._set_password_server_response("") - ds = DataSourceCloudStack( - {}, MockDistro(), helpers.Paths({"run_dir": self.tmp}) - ) - ds.get_data() - self.assertEqual({}, ds.get_config_obj()) + def test_empty_password_doesnt_create_config( + self, _dmi, cloudstack_ds, mocker + ): + self._set_password_server_response("", mocker) + cloudstack_ds.get_data() + assert {} == cloudstack_ds.get_config_obj() - def test_saved_password_doesnt_create_config(self): - self._set_password_server_response("saved_password") - ds = DataSourceCloudStack( - {}, MockDistro(), helpers.Paths({"run_dir": self.tmp}) - ) - ds.get_data() - self.assertEqual({}, ds.get_config_obj()) + def test_saved_password_doesnt_create_config( + self, _dmi, cloudstack_ds, mocker + ): + self._set_password_server_response("saved_password", mocker) + cloudstack_ds.get_data() + assert {} == cloudstack_ds.get_config_obj() @mock.patch(DS_PATH + ".wait_for_metadata_service") - def test_password_sets_password(self, m_wait): + def test_password_sets_password(self, m_wait, _dmi, cloudstack_ds, mocker): m_wait.return_value = True password = "SekritSquirrel" - self._set_password_server_response(password) - ds = DataSourceCloudStack( - {}, MockDistro(), helpers.Paths({"run_dir": self.tmp}) - ) - ds.get_data() - self.assertEqual(password, ds.get_config_obj()["password"]) + self._set_password_server_response(password, mocker) + cloudstack_ds.get_data() + assert password == cloudstack_ds.get_config_obj()["password"] @mock.patch(DS_PATH + ".wait_for_metadata_service") - def test_bad_request_doesnt_stop_ds_from_working(self, m_wait): + def test_bad_request_doesnt_stop_ds_from_working( + self, m_wait, _dmi, cloudstack_ds, mocker + ): m_wait.return_value = True - self._set_password_server_response("bad_request") - # with mock.patch(DHCP_MOD_PATH + ".util.load_text_file"): - ds = DataSourceCloudStack( - {}, MockDistro(), helpers.Paths({"run_dir": self.tmp}) - ) - self.assertTrue(ds.get_data()) + self._set_password_server_response("bad_request", mocker) + assert cloudstack_ds.get_data() is True def assertRequestTypesSent(self, subp, expected_request_types): request_types = [] @@ -433,36 +373,122 @@ def assertRequestTypesSent(self, subp, expected_request_types): for arg in args: if arg.startswith("DomU_Request"): request_types.append(arg.split()[1]) - self.assertEqual(expected_request_types, request_types) + assert expected_request_types == request_types @mock.patch(DS_PATH + ".wait_for_metadata_service") - def test_valid_response_means_password_marked_as_saved(self, m_wait): + def test_valid_response_means_password_marked_as_saved( + self, m_wait, _dmi, cloudstack_ds, mocker + ): m_wait.return_value = True password = "SekritSquirrel" - subp = self._set_password_server_response(password) - ds = DataSourceCloudStack( - {}, MockDistro(), helpers.Paths({"run_dir": self.tmp}) - ) - ds.get_data() + subp = self._set_password_server_response(password, mocker) + cloudstack_ds.get_data() self.assertRequestTypesSent( subp, ["send_my_password", "saved_password"] ) - def _check_password_not_saved_for(self, response_string): - subp = self._set_password_server_response(response_string) - ds = DataSourceCloudStack( - {}, MockDistro(), helpers.Paths({"run_dir": self.tmp}) + def _check_password_not_saved_for( + self, response_string, cloudstack_ds, mocker + ): + subp = self._set_password_server_response( + response_string, mocker=mocker ) with mock.patch(DS_PATH + ".wait_for_metadata_service") as m_wait: m_wait.return_value = True - ds.get_data() + cloudstack_ds.get_data() self.assertRequestTypesSent(subp, ["send_my_password"]) - def test_password_not_saved_if_empty(self): - self._check_password_not_saved_for("") + def test_password_not_saved_if_empty(self, _dmi, cloudstack_ds, mocker): + self._check_password_not_saved_for("", cloudstack_ds, mocker) + + def test_password_not_saved_if_already_saved( + self, _dmi, cloudstack_ds, mocker + ): + self._check_password_not_saved_for( + "saved_password", cloudstack_ds, mocker + ) - def test_password_not_saved_if_already_saved(self): - self._check_password_not_saved_for("saved_password") + def test_password_not_saved_if_bad_request( + self, _dmi, cloudstack_ds, mocker + ): + self._check_password_not_saved_for( + "bad_request", cloudstack_ds, mocker + ) - def test_password_not_saved_if_bad_request(self): - self._check_password_not_saved_for("bad_request") + +class TestDataSourceCloudStackLocal: + + @mock.patch(MOD_PATH + ".EphemeralIPNetwork", autospec=True) + @mock.patch(MOD_PATH + ".net.find_fallback_nic") + @mock.patch(MOD_PATH + ".get_vr_address", return_value="10.1.37.131") + def test_local_datasource_fails_ephemeral_dhcp( + self, m_get_vr_address, m_find_fallback_nic, m_dhcp, caplog, tmpdir + ): + distro = MockDistro() + ds = DataSourceCloudStackLocal( + {}, distro, helpers.Paths({"run_dir": tmpdir}) + ) + fallback_nic_cases = ["enp0s1", "enp0s1"] + dhcp_results = [NoDHCPLeaseError, Exception("Something unexpected")] + expected_logs = [ + ( + "Attempting DHCP on: enp0s1", + "Unable to obtain a DHCP lease on enp0s1", + ), + ( + "Attempting DHCP on: enp0s1", + "Failed fetching metadata service: Something unexpected", + ), + ] + + # Each of the above cases, except the first, increments the m_dhcp call + # count by one. Therefore start at and expect 0, incrementing the + # expectation with each iteration + dhcp_module_call_count = 0 + for fallback_nic, dhcp_result, logs in zip( + fallback_nic_cases, dhcp_results, expected_logs + ): + m_find_fallback_nic.return_value = fallback_nic + m_dhcp.return_value.__enter__.side_effect = dhcp_result + dhcp_module_call_count += 1 + assert m_dhcp.call_count == dhcp_module_call_count - 1 + assert ds._get_data() is False + for msg in logs: + assert msg in caplog.text + + @mock.patch(MOD_PATH + ".CloudStackPasswordServerClient.get_password") + @mock.patch(SOURCES_PATH + ".helpers.ec2.get_instance_metadata") + @mock.patch(SOURCES_PATH + ".helpers.ec2.get_instance_userdata") + @mock.patch(DS_PATH + ".wait_for_metadata_service") + @mock.patch( + MOD_PATH + ".EphemeralIPNetwork", + autospec=True, + ) + @mock.patch(MOD_PATH + ".net.find_fallback_nic") + # @mock.patch(MOD_PATH + ".get_vr_address", return_value="10.1.37.131") + def test_local_datasource_success( + self, + # m_get_vr_address, + m_find_fallback_nic, + m_dhcp, + m_wait_for_mds, + m_get_userdata, + m_get_metadata, + m_get_password, + tmpdir, + ): + distro = MockDistro() + ds = DataSourceCloudStackLocal( + {}, distro, helpers.Paths({"run_dir": tmpdir}) + ) + + m_find_fallback_nic.return_value = "enp0s1" + m_dhcp.return_value.__enter__.side_effect = (None,) + m_wait_for_mds.return_value = (True,) + m_get_userdata.return_value = "ud" + m_get_metadata.return_value = "md" + m_get_password.return_value = True + + assert ds._get_data() is True + assert ds.userdata_raw == "ud" + assert ds.metadata == "md" diff --git a/tests/unittests/sources/test_common.py b/tests/unittests/sources/test_common.py index 4f6e847b..51bd404e 100644 --- a/tests/unittests/sources/test_common.py +++ b/tests/unittests/sources/test_common.py @@ -63,6 +63,7 @@ NWCS.DataSourceNWCS, Akamai.DataSourceAkamaiLocal, WSL.DataSourceWSL, + CloudStack.DataSourceCloudStackLocal, ] DEFAULT_NETWORK = [ diff --git a/tests/unittests/sources/test_configdrive.py b/tests/unittests/sources/test_configdrive.py index 6e97b992..089bdef4 100644 --- a/tests/unittests/sources/test_configdrive.py +++ b/tests/unittests/sources/test_configdrive.py @@ -2,13 +2,16 @@ import json import os +from contextlib import ExitStack from copy import copy, deepcopy +import pytest + from cloudinit import helpers, settings, util from cloudinit.net import eni, network_state from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack -from tests.unittests.helpers import CiTestCase, ExitStack, mock, populate_dir +from tests.unittests.helpers import mock, populate_dir PUBKEY = "ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n" EC2_META = { @@ -382,27 +385,23 @@ M_PATH = "cloudinit.sources.DataSourceConfigDrive." -class TestConfigDriveDataSource(CiTestCase): - def setUp(self): - super(TestConfigDriveDataSource, self).setUp() - self.add_patch( - M_PATH + "util.find_devs_with", "m_find_devs_with", return_value=[] - ) - self.tmp = self.tmp_dir() +class TestConfigDriveDataSource: + @pytest.fixture(autouse=True) + def fixtures(self, mocker, tmp_path): + mocker.patch(M_PATH + "util.find_devs_with", return_value=[]) + populate_dir(str(tmp_path), CFG_DRIVE_FILES_V2) - def test_ec2_metadata(self): - populate_dir(self.tmp, CFG_DRIVE_FILES_V2) - found = ds.read_config_drive(self.tmp) - self.assertTrue("ec2-metadata" in found) + def test_ec2_metadata(self, tmp_path): + found = ds.read_config_drive(tmp_path) + assert "ec2-metadata" in found ec2_md = found["ec2-metadata"] - self.assertEqual(EC2_META, ec2_md) + assert EC2_META == ec2_md - def test_dev_os_remap(self): - populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + def test_dev_os_remap(self, tmp_path): cfg_ds = ds.DataSourceConfigDrive( settings.CFG_BUILTIN, None, helpers.Paths({}) ) - found = ds.read_config_drive(self.tmp) + found = ds.read_config_drive(tmp_path) cfg_ds.metadata = found["metadata"] name_tests = { "ami": "/dev/vda1", @@ -433,17 +432,16 @@ def exists_side_effect(): os.path, "exists", side_effect=exists_side_effect() ) ) - self.assertEqual(dev_name, cfg_ds.device_name_to_device(name)) + assert dev_name == cfg_ds.device_name_to_device(name) find_mock.assert_called_once_with(mock.ANY) - self.assertEqual(exists_mock.call_count, 2) + assert exists_mock.call_count == 2 - def test_dev_os_map(self): - populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + def test_dev_os_map(self, tmp_path): cfg_ds = ds.DataSourceConfigDrive( settings.CFG_BUILTIN, None, helpers.Paths({}) ) - found = ds.read_config_drive(self.tmp) + found = ds.read_config_drive(tmp_path) os_md = found["metadata"] cfg_ds.metadata = os_md name_tests = { @@ -462,17 +460,16 @@ def test_dev_os_map(self): exists_mock = mocks.enter_context( mock.patch.object(os.path, "exists", return_value=True) ) - self.assertEqual(dev_name, cfg_ds.device_name_to_device(name)) + assert dev_name == cfg_ds.device_name_to_device(name) find_mock.assert_called_once_with(mock.ANY) exists_mock.assert_called_once_with(mock.ANY) - def test_dev_ec2_remap(self): - populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + def test_dev_ec2_remap(self, tmp_path): cfg_ds = ds.DataSourceConfigDrive( settings.CFG_BUILTIN, None, helpers.Paths({}) ) - found = ds.read_config_drive(self.tmp) + found = ds.read_config_drive(tmp_path) ec2_md = found["ec2-metadata"] os_md = found["metadata"] cfg_ds.ec2_metadata = ec2_md @@ -498,18 +495,17 @@ def exists_side_effect(): with mock.patch.object( os.path, "exists", side_effect=exists_side_effect() ): - self.assertEqual(dev_name, cfg_ds.device_name_to_device(name)) + assert dev_name == cfg_ds.device_name_to_device(name) # We don't assert the call count for os.path.exists() because # not all of the entries in name_tests results in two calls to # that function. Specifically, 'root2k' doesn't seem to call # it at all. - def test_dev_ec2_map(self): - populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + def test_dev_ec2_map(self, tmp_path): cfg_ds = ds.DataSourceConfigDrive( settings.CFG_BUILTIN, None, helpers.Paths({}) ) - found = ds.read_config_drive(self.tmp) + found = ds.read_config_drive(tmp_path) ec2_md = found["ec2-metadata"] os_md = found["metadata"] cfg_ds.ec2_metadata = ec2_md @@ -525,45 +521,43 @@ def test_dev_ec2_map(self): } for name, dev_name in name_tests.items(): with mock.patch.object(os.path, "exists", return_value=True): - self.assertEqual(dev_name, cfg_ds.device_name_to_device(name)) + assert dev_name == cfg_ds.device_name_to_device(name) - def test_dir_valid(self): + def test_dir_valid(self, tmp_path): """Verify a dir is read as such.""" - populate_dir(self.tmp, CFG_DRIVE_FILES_V2) - - found = ds.read_config_drive(self.tmp) + found = ds.read_config_drive(tmp_path) expected_md = copy(OSTACK_META) expected_md["instance-id"] = expected_md["uuid"] expected_md["local-hostname"] = expected_md["hostname"] - self.assertEqual(USER_DATA, found["userdata"]) - self.assertEqual(expected_md, found["metadata"]) - self.assertEqual(NETWORK_DATA, found["networkdata"]) - self.assertEqual(VENDOR_DATA, found["vendordata"]) - self.assertEqual(VENDOR_DATA2, found["vendordata2"]) - self.assertEqual(found["files"]["/etc/foo.cfg"], CONTENT_0) - self.assertEqual(found["files"]["/etc/bar/bar.cfg"], CONTENT_1) + assert USER_DATA == found["userdata"] + assert expected_md == found["metadata"] + assert NETWORK_DATA == found["networkdata"] + assert VENDOR_DATA == found["vendordata"] + assert VENDOR_DATA2 == found["vendordata2"] + assert found["files"]["/etc/foo.cfg"] == CONTENT_0 + assert found["files"]["/etc/bar/bar.cfg"] == CONTENT_1 - def test_seed_dir_valid_extra(self): + def test_seed_dir_valid_extra(self, tmp_path): """Verify extra files do not affect datasource validity.""" data = copy(CFG_DRIVE_FILES_V2) data["myfoofile.txt"] = "myfoocontent" data["openstack/latest/random-file.txt"] = "random-content" - populate_dir(self.tmp, data) + populate_dir(str(tmp_path), data) - found = ds.read_config_drive(self.tmp) + found = ds.read_config_drive(tmp_path) expected_md = copy(OSTACK_META) expected_md["instance-id"] = expected_md["uuid"] expected_md["local-hostname"] = expected_md["hostname"] - self.assertEqual(expected_md, found["metadata"]) + assert expected_md == found["metadata"] - def test_seed_dir_bad_json_metadata(self): + def test_seed_dir_bad_json_metadata(self, tmp_path): """Verify that bad json in metadata raises BrokenConfigDriveDir.""" data = copy(CFG_DRIVE_FILES_V2) @@ -571,27 +565,28 @@ def test_seed_dir_bad_json_metadata(self): data["openstack/2015-10-15/meta_data.json"] = "non-json garbage {}" data["openstack/latest/meta_data.json"] = "non-json garbage {}" - populate_dir(self.tmp, data) + populate_dir(str(tmp_path), data) - self.assertRaises( - openstack.BrokenMetadata, ds.read_config_drive, self.tmp - ) + with pytest.raises(openstack.BrokenMetadata): + ds.read_config_drive(tmp_path) - def test_seed_dir_no_configdrive(self): + def test_seed_dir_no_configdrive(self, tmp_path): """Verify that no metadata raises NonConfigDriveDir.""" - my_d = os.path.join(self.tmp, "non-configdrive") + my_d = os.path.join(tmp_path, "non-configdrive") data = copy(CFG_DRIVE_FILES_V2) data["myfoofile.txt"] = "myfoocontent" data["openstack/latest/random-file.txt"] = "random-content" data["content/foo"] = "foocontent" - self.assertRaises(openstack.NonReadable, ds.read_config_drive, my_d) + with pytest.raises(openstack.NonReadable): + ds.read_config_drive(my_d) - def test_seed_dir_missing(self): + def test_seed_dir_missing(self, tmp_path): """Verify that missing seed_dir raises NonConfigDriveDir.""" - my_d = os.path.join(self.tmp, "nonexistantdirectory") - self.assertRaises(openstack.NonReadable, ds.read_config_drive, my_d) + my_d = os.path.join(tmp_path, "nonexistantdirectory") + with pytest.raises(openstack.NonReadable): + ds.read_config_drive(my_d) def test_find_candidates(self): devs_with_answers = {} @@ -614,14 +609,12 @@ def my_is_partition(dev): "TYPE=iso9660": ["/dev/vdb"], "LABEL=config-2": ["/dev/vdb"], } - self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) + assert ["/dev/vdb"] == ds.find_candidate_devs() # add a vfat item # zdd reverse sorts after vdb, but config-2 label is preferred devs_with_answers["TYPE=vfat"] = ["/dev/zdd"] - self.assertEqual( - ["/dev/vdb", "/dev/zdd"], ds.find_candidate_devs() - ) + assert ["/dev/vdb", "/dev/zdd"] == ds.find_candidate_devs() # verify that partitions are considered, that have correct label. devs_with_answers = { @@ -629,7 +622,7 @@ def my_is_partition(dev): "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"], } - self.assertEqual(["/dev/vdb3"], ds.find_candidate_devs()) + assert ["/dev/vdb3"] == ds.find_candidate_devs() # Verify that uppercase labels are also found. devs_with_answers = { @@ -637,22 +630,22 @@ def my_is_partition(dev): "TYPE=iso9660": ["/dev/vdb"], "LABEL=CONFIG-2": ["/dev/vdb"], } - self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) + assert ["/dev/vdb"] == ds.find_candidate_devs() finally: util.find_devs_with = orig_find_devs_with util.is_partition = orig_is_partition @mock.patch(M_PATH + "on_first_boot") - def test_pubkeys_v2(self, on_first_boot): + def test_pubkeys_v2(self, on_first_boot, tmp_path): """Verify that public-keys work in config-drive-v2.""" - myds = cfg_ds_from_dir(self.tmp, files=CFG_DRIVE_FILES_V2) - self.assertEqual( - myds.get_public_ssh_keys(), [OSTACK_META["public_keys"]["mykey"]] - ) - self.assertEqual("configdrive", myds.cloud_name) - self.assertEqual("openstack", myds.platform) - self.assertEqual("seed-dir (%s/seed)" % self.tmp, myds.subplatform) + myds = cfg_ds_from_dir(tmp_path, files=CFG_DRIVE_FILES_V2) + assert myds.get_public_ssh_keys() == [ + OSTACK_META["public_keys"]["mykey"] + ] + assert "configdrive" == myds.cloud_name + assert "openstack" == myds.platform + assert "seed-dir (%s/seed)" % tmp_path == myds.subplatform def test_subplatform_config_drive_when_starts_with_dev(self): """subplatform reports config-drive when source starts with /dev/.""" @@ -663,34 +656,29 @@ def test_subplatform_config_drive_when_starts_with_dev(self): with mock.patch(M_PATH + "util.mount_cb"): with mock.patch(M_PATH + "on_first_boot"): m_find_devs.return_value = ["/dev/anything"] - self.assertEqual(True, cfg_ds.get_data()) - self.assertEqual("config-disk (/dev/anything)", cfg_ds.subplatform) + assert True is cfg_ds.get_data() + assert "config-disk (/dev/anything)" == cfg_ds.subplatform @mock.patch( "cloudinit.net.is_openvswitch_internal_interface", mock.Mock(return_value=False), ) -class TestNetJson(CiTestCase): - def setUp(self): - super(TestNetJson, self).setUp() - self.tmp = self.tmp_dir() - self.maxDiff = None - +class TestNetJson: @mock.patch(M_PATH + "on_first_boot") - def test_network_data_is_found(self, on_first_boot): + def test_network_data_is_found(self, on_first_boot, tmp_path): """Verify that network_data is present in ds in config-drive-v2.""" - myds = cfg_ds_from_dir(self.tmp, files=CFG_DRIVE_FILES_V2) - self.assertIsNotNone(myds.network_json) + myds = cfg_ds_from_dir(tmp_path, files=CFG_DRIVE_FILES_V2) + assert myds.network_json is not None @mock.patch(M_PATH + "on_first_boot") - def test_network_config_is_converted(self, on_first_boot): + def test_network_config_is_converted(self, on_first_boot, tmp_path): """Verify that network_data is converted and present on ds object.""" - myds = cfg_ds_from_dir(self.tmp, files=CFG_DRIVE_FILES_V2) + myds = cfg_ds_from_dir(tmp_path, files=CFG_DRIVE_FILES_V2) network_config = openstack.convert_net_json( NETWORK_DATA, known_macs=KNOWN_MACS ) - self.assertEqual(myds.network_config, network_config) + assert myds.network_config == network_config def test_network_config_conversion_dhcp6(self): """Test some ipv6 input network json and check the expected @@ -748,7 +736,7 @@ def test_network_config_conversion_dhcp6(self): ], } conv_data = openstack.convert_net_json(in_data, known_macs=KNOWN_MACS) - self.assertEqual(out_data, conv_data) + assert out_data == conv_data def test_network_config_conversions(self): """Tests a bunch of input network json and checks the @@ -855,21 +843,14 @@ def test_network_config_conversions(self): conv_data = openstack.convert_net_json( in_data, known_macs=KNOWN_MACS ) - self.assertEqual(out_data, conv_data) + assert out_data == conv_data @mock.patch( "cloudinit.net.is_openvswitch_internal_interface", mock.Mock(return_value=False), ) -class TestConvertNetworkData(CiTestCase): - - with_logs = True - - def setUp(self): - super(TestConvertNetworkData, self).setUp() - self.tmp = self.tmp_dir() - +class TestConvertNetworkData: def _getnames_in_config(self, ncfg): return set( [n["name"] for n in ncfg["config"] if n["type"] == "physical"] @@ -879,7 +860,7 @@ def test_conversion_fills_names(self): ncfg = openstack.convert_net_json(NETWORK_DATA, known_macs=KNOWN_MACS) expected = set(["nic0", "enp0s1", "enp0s2"]) found = self._getnames_in_config(ncfg) - self.assertEqual(found, expected) + assert found == expected @mock.patch("cloudinit.net.get_interfaces_by_mac") def test_convert_reads_system_prefers_name(self, get_interfaces_by_mac): @@ -892,21 +873,20 @@ def test_convert_reads_system_prefers_name(self, get_interfaces_by_mac): ncfg = openstack.convert_net_json(NETWORK_DATA) expected = set(["nic0", "ens1", "enp0s2"]) found = self._getnames_in_config(ncfg) - self.assertEqual(found, expected) + assert found == expected def test_convert_raises_value_error_on_missing_name(self): macs = {"aa:aa:aa:aa:aa:00": "ens1"} with mock.patch( "cloudinit.sources.helpers.openstack.util.udevadm_settle" ): - self.assertRaises( - ValueError, - openstack.convert_net_json, - NETWORK_DATA, - known_macs=macs, - ) + with pytest.raises(ValueError): + openstack.convert_net_json( + NETWORK_DATA, + known_macs=macs, + ) - def test_conversion_with_route(self): + def test_conversion_with_route(self, tmp_path): ncfg = openstack.convert_net_json( NETWORK_DATA_2, known_macs=KNOWN_MACS ) @@ -916,19 +896,20 @@ def test_conversion_with_route(self): for n in ncfg["config"]: for s in n.get("subnets", []): routes.extend(s.get("routes", [])) - self.assertIn( - {"network": "0.0.0.0", "netmask": "0.0.0.0", "gateway": "2.2.2.9"}, - routes, - ) + assert { + "network": "0.0.0.0", + "netmask": "0.0.0.0", + "gateway": "2.2.2.9", + } in routes eni_renderer = eni.Renderer() eni_renderer.render_network_state( - network_state.parse_net_config_data(ncfg), target=self.tmp + network_state.parse_net_config_data(ncfg), target=str(tmp_path) ) with open( - os.path.join(self.tmp, "etc", "network", "interfaces"), "r" + os.path.join(tmp_path, "etc", "network", "interfaces"), "r" ) as f: eni_rendering = f.read() - self.assertIn("route add default gw 2.2.2.9", eni_rendering) + assert "route add default gw 2.2.2.9" in eni_rendering def test_conversion_with_tap(self): ncfg = openstack.convert_net_json( @@ -938,9 +919,9 @@ def test_conversion_with_tap(self): for i in ncfg["config"]: if i.get("type") == "physical": physicals.add(i["name"]) - self.assertEqual(physicals, set(("foo1", "foo2"))) + assert physicals == set(("foo1", "foo2")) - def test_bond_conversion(self): + def test_bond_conversion(self, tmp_path): # light testing of bond conversion and eni rendering of bond ncfg = openstack.convert_net_json( NETWORK_DATA_BOND, known_macs=KNOWN_MACS @@ -948,10 +929,10 @@ def test_bond_conversion(self): eni_renderer = eni.Renderer() eni_renderer.render_network_state( - network_state.parse_net_config_data(ncfg), target=self.tmp + network_state.parse_net_config_data(ncfg), target=str(tmp_path) ) with open( - os.path.join(self.tmp, "etc", "network", "interfaces"), "r" + os.path.join(tmp_path, "etc", "network", "interfaces"), "r" ) as f: eni_rendering = f.read() @@ -963,43 +944,43 @@ def test_bond_conversion(self): if i["type"] in ("vlan", "bond", "physical") ] ) - self.assertEqual( - sorted(["oeth0", "oeth1", "bond0", "bond0.602", "bond0.612"]), - interfaces, + assert ( + sorted(["oeth0", "oeth1", "bond0", "bond0.602", "bond0.612"]) + == interfaces ) words = eni_rendering.split() # 'eth0' and 'eth1' are the ids. because their mac adresses # map to other names, we should not see them in the ENI - self.assertNotIn("eth0", words) - self.assertNotIn("eth1", words) + assert "eth0" not in words + assert "eth1" not in words # oeth0 and oeth1 are the interface names for eni. # bond0 will be generated for the bond. Each should be auto. - self.assertIn("auto oeth0", eni_rendering) - self.assertIn("auto oeth1", eni_rendering) - self.assertIn("auto bond0", eni_rendering) + assert "auto oeth0" in eni_rendering + assert "auto oeth1" in eni_rendering + assert "auto bond0" in eni_rendering # The bond should have the given mac address pos = eni_rendering.find("auto bond0") - self.assertIn(BOND_MAC, eni_rendering[pos:]) + assert BOND_MAC in eni_rendering[pos:] - def test_vlan(self): + def test_vlan(self, tmp_path): # light testing of vlan config conversion and eni rendering ncfg = openstack.convert_net_json( NETWORK_DATA_VLAN, known_macs=KNOWN_MACS ) eni_renderer = eni.Renderer() eni_renderer.render_network_state( - network_state.parse_net_config_data(ncfg), target=self.tmp + network_state.parse_net_config_data(ncfg), target=str(tmp_path) ) with open( - os.path.join(self.tmp, "etc", "network", "interfaces"), "r" + os.path.join(tmp_path, "etc", "network", "interfaces"), "r" ) as f: eni_rendering = f.read() - self.assertIn("iface enp0s1", eni_rendering) - self.assertIn("address 10.0.1.5", eni_rendering) - self.assertIn("auto enp0s1.602", eni_rendering) + assert "iface enp0s1" in eni_rendering + assert "address 10.0.1.5" in eni_rendering + assert "auto enp0s1.602" in eni_rendering def test_mac_addrs_can_be_upper_case(self): # input mac addresses on rackspace may be upper case @@ -1018,9 +999,9 @@ def test_mac_addrs_can_be_upper_case(self): "enp0s1": "fa:16:3e:69:b0:58", "enp0s2": "fa:16:3e:d4:57:ad", } - self.assertEqual(expected, config_name2mac) + assert expected == config_name2mac - def test_unknown_device_types_accepted(self): + def test_unknown_device_types_accepted(self, caplog): # If we don't recognise a link, we should treat it as physical for a # best-effort boot my_netdata = deepcopy(NETWORK_DATA) @@ -1037,12 +1018,12 @@ def test_unknown_device_types_accepted(self): "enp0s1": "fa:16:3e:69:b0:58", "enp0s2": "fa:16:3e:d4:57:ad", } - self.assertEqual(expected, config_name2mac) + assert expected == config_name2mac # We should, however, warn the user that we don't recognise the type - self.assertIn( - "Unknown network_data link type (my-special-link-type)", - self.logs.getvalue(), + assert ( + "Unknown network_data link type (my-special-link-type)" + in caplog.text ) diff --git a/tests/unittests/sources/test_digitalocean.py b/tests/unittests/sources/test_digitalocean.py index c111e710..0ef2e51a 100644 --- a/tests/unittests/sources/test_digitalocean.py +++ b/tests/unittests/sources/test_digitalocean.py @@ -8,10 +8,12 @@ import json -from cloudinit import helpers, settings +import pytest + +from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from cloudinit.sources.helpers import digitalocean -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import mock DO_MULTIPLE_KEYS = [ "ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co", @@ -139,36 +141,39 @@ def _mock_dmi(): return (True, DO_META.get("id")) -class TestDataSourceDigitalOcean(CiTestCase): - """ - Test reading the meta-data - """ - - def setUp(self): - super(TestDataSourceDigitalOcean, self).setUp() - self.tmp = self.tmp_dir() - - def get_ds(self, get_sysinfo=_mock_dmi): +@pytest.fixture +def get_ds(paths): + def _get_ds(get_sysinfo=_mock_dmi): ds = DataSourceDigitalOcean.DataSourceDigitalOcean( - settings.CFG_BUILTIN, None, helpers.Paths({"run_dir": self.tmp}) + settings.CFG_BUILTIN, None, paths ) ds.use_ip4LL = False if get_sysinfo is not None: ds._get_sysinfo = get_sysinfo return ds + return _get_ds + + +class TestDataSourceDigitalOcean: + """ + Test reading the meta-data + """ + @mock.patch("cloudinit.sources.helpers.digitalocean.read_sysinfo") - def test_returns_false_not_on_docean(self, m_read_sysinfo): + def test_returns_false_not_on_docean(self, m_read_sysinfo, get_ds): m_read_sysinfo.return_value = (False, None) - ds = self.get_ds(get_sysinfo=None) - self.assertEqual(False, ds.get_data()) - self.assertTrue(m_read_sysinfo.called) + ds = get_ds(get_sysinfo=None) + assert False is ds.get_data() + m_read_sysinfo.assert_called_once() @mock.patch("cloudinit.sources.helpers.digitalocean.read_metadata") @mock.patch("cloudinit.sources.lifecycle.deprecate") - def test_deprecation_log_on_init(self, mock_deprecate, _mock_readmd): - ds = self.get_ds() - self.assertTrue(ds.get_data()) + def test_deprecation_log_on_init( + self, mock_deprecate, _mock_readmd, get_ds + ): + ds = get_ds() + assert ds.get_data() mock_deprecate.assert_called_with( deprecated="DataSourceDigitalOcean", deprecated_version="23.2", @@ -177,11 +182,13 @@ def test_deprecation_log_on_init(self, mock_deprecate, _mock_readmd): @mock.patch("cloudinit.sources.helpers.digitalocean.read_metadata") @mock.patch("cloudinit.sources.lifecycle.deprecate") - def test_deprecation_log_on_unpick(self, mock_deprecate, _mock_readmd): - ds = self.get_ds() - self.assertTrue(ds.get_data()) + def test_deprecation_log_on_unpick( + self, mock_deprecate, _mock_readmd, get_ds + ): + ds = get_ds() + assert ds.get_data() ds._unpickle(0) - self.assertEqual(mock_deprecate.call_count, 2) + assert mock_deprecate.call_count == 2 mock_deprecate.assert_has_calls( [ mock.call( @@ -202,46 +209,44 @@ def test_deprecation_log_on_unpick(self, mock_deprecate, _mock_readmd): ) @mock.patch("cloudinit.sources.helpers.digitalocean.read_metadata") - def test_metadata(self, mock_readmd): + def test_metadata(self, mock_readmd, get_ds): mock_readmd.return_value = DO_META.copy() - ds = self.get_ds() + ds = get_ds() ret = ds.get_data() - self.assertTrue(ret) + assert ret - self.assertTrue(mock_readmd.called) + mock_readmd.assert_called_once() - self.assertEqual(DO_META.get("user_data"), ds.get_userdata_raw()) - self.assertEqual(DO_META.get("vendor_data"), ds.get_vendordata_raw()) - self.assertEqual(DO_META.get("region"), ds.availability_zone) - self.assertEqual(DO_META.get("droplet_id"), ds.get_instance_id()) - self.assertEqual(DO_META.get("hostname"), ds.get_hostname().hostname) + assert DO_META.get("user_data") == ds.get_userdata_raw() + assert DO_META.get("vendor_data") == ds.get_vendordata_raw() + assert DO_META.get("region") == ds.availability_zone + assert DO_META.get("droplet_id") == ds.get_instance_id() + assert DO_META.get("hostname") == ds.get_hostname().hostname # Single key - self.assertEqual( - [DO_META.get("public_keys")], ds.get_public_ssh_keys() - ) + assert [DO_META.get("public_keys")] == ds.get_public_ssh_keys() - self.assertIsInstance(ds.get_public_ssh_keys(), list) + assert isinstance(ds.get_public_ssh_keys(), list) @mock.patch("cloudinit.sources.helpers.digitalocean.read_metadata") - def test_multiple_ssh_keys(self, mock_readmd): + def test_multiple_ssh_keys(self, mock_readmd, get_ds): metadata = DO_META.copy() metadata["public_keys"] = DO_MULTIPLE_KEYS mock_readmd.return_value = metadata.copy() - ds = self.get_ds() + ds = get_ds() ret = ds.get_data() - self.assertTrue(ret) + assert ret is True - self.assertTrue(mock_readmd.called) + mock_readmd.assert_called_once() # Multiple keys - self.assertEqual(metadata["public_keys"], ds.get_public_ssh_keys()) - self.assertIsInstance(ds.get_public_ssh_keys(), list) + assert metadata["public_keys"] == ds.get_public_ssh_keys() + assert isinstance(ds.get_public_ssh_keys(), list) -class TestNetworkConvert(CiTestCase): +class TestNetworkConvert: def _get_networking(self): self.m_get_by_mac.return_value = { "04:01:57:d1:9e:01": "ens1", @@ -252,16 +257,16 @@ def _get_networking(self): netcfg = digitalocean.convert_network_configuration( DO_META["interfaces"], DO_META["dns"]["nameservers"] ) - self.assertIn("config", netcfg) + assert "config" in netcfg return netcfg - def setUp(self): - super(TestNetworkConvert, self).setUp() - self.add_patch("cloudinit.net.get_interfaces_by_mac", "m_get_by_mac") + @pytest.fixture(autouse=True) + def fix(self, mocker): + self.m_get_by_mac = mocker.patch("cloudinit.net.get_interfaces_by_mac") def test_networking_defined(self): netcfg = self._get_networking() - self.assertIsNotNone(netcfg) + assert netcfg is not None dns_defined = False for part in netcfg.get("config"): @@ -270,12 +275,12 @@ def test_networking_defined(self): if n_type == "nameserver": n_address = part.get("address") - self.assertIsNotNone(n_address) - self.assertEqual(len(n_address), 3) + assert n_address is not None + assert len(n_address) == 3 dns_resolvers = DO_META["dns"]["nameservers"] for x in n_address: - self.assertIn(x, dns_resolvers) + assert x in dns_resolvers dns_defined = True else: @@ -283,12 +288,12 @@ def test_networking_defined(self): n_name = part.get("name") n_mac = part.get("mac_address") - self.assertIsNotNone(n_type) - self.assertIsNotNone(n_subnets) - self.assertIsNotNone(n_name) - self.assertIsNotNone(n_mac) + assert n_type is not None + assert n_subnets is not None + assert n_name is not None + assert n_mac is not None - self.assertTrue(dns_defined) + assert dns_defined def _get_nic_definition(self, int_type, expected_name): """helper function to return if_type (i.e. public) and the expected @@ -296,7 +301,7 @@ def _get_nic_definition(self, int_type, expected_name): netcfg = self._get_networking() meta_def = (DO_META.get("interfaces")).get(int_type)[0] - self.assertEqual(int_type, meta_def.get("type")) + assert int_type == meta_def.get("type") for nic_def in netcfg.get("config"): print(nic_def) @@ -307,7 +312,7 @@ def _get_match_subn(self, subnets, ip_addr): """get the matching subnet definition based on ip address""" for subn in subnets: address = subn.get("address") - self.assertIsNotNone(address) + assert address is not None # equals won't work because of ipv6 addressing being in # cidr notation, i.e fe00::1/64 @@ -327,36 +332,36 @@ def test_correct_gateways_defined(self): gateways.append(subn.get("gateway")) # we should have two gateways, one ipv4 and ipv6 - self.assertEqual(len(gateways), 2) + assert len(gateways) == 2 # make that the ipv6 gateway is there (nic_def, meta_def) = self._get_nic_definition("public", "eth0") ipv4_def = meta_def.get("ipv4") - self.assertIn(ipv4_def.get("gateway"), gateways) + assert ipv4_def.get("gateway") in gateways # make sure the the ipv6 gateway is there ipv6_def = meta_def.get("ipv6") - self.assertIn(ipv6_def.get("gateway"), gateways) + assert ipv6_def.get("gateway") in gateways def test_public_interface_defined(self): """test that the public interface is defined as eth0""" (nic_def, meta_def) = self._get_nic_definition("public", "eth0") - self.assertEqual("eth0", nic_def.get("name")) - self.assertEqual(meta_def.get("mac"), nic_def.get("mac_address")) - self.assertEqual("physical", nic_def.get("type")) + assert "eth0" == nic_def.get("name") + assert meta_def.get("mac") == nic_def.get("mac_address") + assert "physical" == nic_def.get("type") def test_private_interface_defined(self): """test that the private interface is defined as eth1""" (nic_def, meta_def) = self._get_nic_definition("private", "eth1") - self.assertEqual("eth1", nic_def.get("name")) - self.assertEqual(meta_def.get("mac"), nic_def.get("mac_address")) - self.assertEqual("physical", nic_def.get("type")) + assert "eth1" == nic_def.get("name") + assert meta_def.get("mac") == nic_def.get("mac_address") + assert "physical" == nic_def.get("type") def test_public_interface_ipv6(self): """test public ipv6 addressing""" (nic_def, meta_def) = self._get_nic_definition("public", "eth0") ipv6_def = meta_def.get("ipv6") - self.assertIsNotNone(ipv6_def) + assert ipv6_def is not None subn_def = self._get_match_subn( nic_def.get("subnets"), ipv6_def.get("ip_address") @@ -366,34 +371,34 @@ def test_public_interface_ipv6(self): ipv6_def.get("ip_address"), ipv6_def.get("cidr") ) - self.assertEqual(cidr_notated_address, subn_def.get("address")) - self.assertEqual(ipv6_def.get("gateway"), subn_def.get("gateway")) + assert cidr_notated_address == subn_def.get("address") + assert ipv6_def.get("gateway") == subn_def.get("gateway") def test_public_interface_ipv4(self): """test public ipv4 addressing""" (nic_def, meta_def) = self._get_nic_definition("public", "eth0") ipv4_def = meta_def.get("ipv4") - self.assertIsNotNone(ipv4_def) + assert ipv4_def is not None subn_def = self._get_match_subn( nic_def.get("subnets"), ipv4_def.get("ip_address") ) - self.assertEqual(ipv4_def.get("netmask"), subn_def.get("netmask")) - self.assertEqual(ipv4_def.get("gateway"), subn_def.get("gateway")) + assert ipv4_def.get("netmask") == subn_def.get("netmask") + assert ipv4_def.get("gateway") == subn_def.get("gateway") def test_public_interface_anchor_ipv4(self): """test public ipv4 addressing""" (nic_def, meta_def) = self._get_nic_definition("public", "eth0") ipv4_def = meta_def.get("anchor_ipv4") - self.assertIsNotNone(ipv4_def) + assert ipv4_def is not None subn_def = self._get_match_subn( nic_def.get("subnets"), ipv4_def.get("ip_address") ) - self.assertEqual(ipv4_def.get("netmask"), subn_def.get("netmask")) - self.assertNotIn("gateway", subn_def) + assert ipv4_def.get("netmask") == subn_def.get("netmask") + assert "gateway" not in subn_def @mock.patch("cloudinit.net.get_interfaces_by_mac") def test_convert_without_private(self, m_get_by_mac): @@ -414,10 +419,9 @@ def test_convert_without_private(self, m_get_by_mac): "name '%s' in config twice: %s" % (i["name"], netcfg) ) byname[i["name"]] = i - self.assertTrue("eth0" in byname) - self.assertTrue("subnets" in byname["eth0"]) + assert "eth0" in byname + assert "subnets" in byname["eth0"] eth0 = byname["eth0"] - self.assertEqual( - sorted(["45.55.249.133", "10.17.0.5"]), - sorted([i["address"] for i in eth0["subnets"]]), + assert sorted(["45.55.249.133", "10.17.0.5"]) == sorted( + [i["address"] for i in eth0["subnets"]] ) diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py index 3b0b7046..62bc9e95 100644 --- a/tests/unittests/sources/test_ec2.py +++ b/tests/unittests/sources/test_ec2.py @@ -918,14 +918,14 @@ def test_ec2_local_returns_false_on_bsd( @pytest.mark.usefixtures("disable_netdev_info") @mock.patch("cloudinit.net.ephemeral.EphemeralIPv6Network") @mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") - @mock.patch("cloudinit.distros.net.find_fallback_nic") + @mock.patch("cloudinit.distros.net.find_candidate_nics") @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD") def test_ec2_local_performs_dhcp_on_non_bsd( self, m_is_bsd, m_dhcp, - m_fallback_nic, + m_candidate_nics, m_net4, m_net6, caplog, @@ -940,7 +940,7 @@ def test_ec2_local_performs_dhcp_on_non_bsd( When the platform data is valid, return True. """ - m_fallback_nic.return_value = "eth9" + m_candidate_nics.return_value = ["eth9"] m_is_bsd.return_value = False m_dhcp.return_value = { "interface": "eth9", @@ -973,6 +973,73 @@ def test_ec2_local_performs_dhcp_on_non_bsd( static_routes=None, ) + @responses.activate + @pytest.mark.usefixtures("disable_netdev_info") + @mock.patch("cloudinit.net.ephemeral.EphemeralIPv6Network") + @mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") + @mock.patch("cloudinit.distros.net.find_candidate_nics") + @mock.patch("cloudinit.net.ephemeral.maybe_perform_dhcp_discovery") + @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD") + def test_ec2_local_get_metadata_via_iterating_nics( + self, + m_is_bsd, + m_dhcp, + m_candidate_nics, + m_net4, + m_net6, + caplog, + mocker, + tmpdir, + disable_netdev_info, + ): + """DataSourceEc2Local iterates over candidate NICs and fetches metadata + until successful""" + + m_candidate_nics.return_value = ["eth0", "eth1", "eth2"] + m_is_bsd.return_value = False + m_dhcp.side_effect = ( + { + "interface": "eth0", + "fixed-address": "10.0.2.6", + "routers": "10.0.2.1", + "subnet-mask": "255.255.255.0", + "broadcast-address": "10.0.2.255", + }, + { + "interface": "eth1", + "fixed-address": "192.168.2.7", + "routers": "192.168.2.1", + "subnet-mask": "255.255.255.0", + "broadcast-address": "192.168.2.255", + }, + { + "interface": "eth2", + "fixed-address": "10.0.2.8", + "routers": "10.0.2.1", + "subnet-mask": "255.255.255.0", + "broadcast-address": "10.0.2.255", + }, + ) + self.datasource = ec2.DataSourceEc2Local + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={"datasource": {"Ec2": {"strict_id": False}}}, + md={"md": DEFAULT_METADATA}, + distro=MockDistro("", {}, {}), + mocker=mocker, + tmpdir=tmpdir, + ) + crawled_metadata = ds.crawl_metadata() + mocker.patch.object( + ds, "crawl_metadata", side_effect=[{}, crawled_metadata] + ) + + ret = ds.get_data() + assert True is ret + assert 2 == m_dhcp.call_count + assert 2 == m_net4.call_count + assert "eth1" == ds.distro.fallback_interface + @responses.activate def test_get_instance_tags(self, mocker, tmpdir): ds = self._setup_ds( diff --git a/tests/unittests/sources/test_exoscale.py b/tests/unittests/sources/test_exoscale.py index 82b567d7..08cbeeb9 100644 --- a/tests/unittests/sources/test_exoscale.py +++ b/tests/unittests/sources/test_exoscale.py @@ -16,7 +16,7 @@ get_password, read_metadata, ) -from tests.unittests.helpers import ResponsesTestCase, mock +from tests.unittests.helpers import mock TEST_PASSWORD_URL = "{}:{}/{}/".format( METADATA_URL, PASSWORD_SERVER_PORT, API_VERSION @@ -27,176 +27,157 @@ TEST_USERDATA_URL = "{}/{}/user-data".format(METADATA_URL, API_VERSION) -class TestDatasourceExoscale(ResponsesTestCase): - def setUp(self): - super(TestDatasourceExoscale, self).setUp() - self.tmp = self.tmp_dir() - self.password_url = TEST_PASSWORD_URL - self.metadata_url = TEST_METADATA_URL - self.userdata_url = TEST_USERDATA_URL - +class TestDatasourceExoscale: + @responses.activate def test_password_saved(self): """The password is not set when it is not found in the metadata service.""" - self.responses.add( - responses.GET, self.password_url, body="saved_password" - ) - self.assertFalse(get_password()) + responses.add(responses.GET, TEST_PASSWORD_URL, body="saved_password") + assert not get_password() + @responses.activate def test_password_empty(self): """No password is set if the metadata service returns an empty string.""" - self.responses.add(responses.GET, self.password_url, body="") - self.assertFalse(get_password()) + responses.add(responses.GET, TEST_PASSWORD_URL, body="") + assert not get_password() + @responses.activate def test_password(self): """The password is set to what is found in the metadata service.""" expected_password = "p@ssw0rd" - self.responses.add( - responses.GET, self.password_url, body=expected_password - ) + responses.add(responses.GET, TEST_PASSWORD_URL, body=expected_password) password = get_password() - self.assertEqual(expected_password, password) + assert expected_password == password - def test_activate_removes_set_passwords_semaphore(self): + def test_activate_removes_set_passwords_semaphore(self, tmp_path): """Allow set_passwords to run every boot by removing the semaphore.""" - path = helpers.Paths({"cloud_dir": self.tmp}) - sem_dir = self.tmp_path("instance/sem", dir=self.tmp) + path = helpers.Paths({"cloud_dir": str(tmp_path)}) + sem_dir = str(tmp_path / "instance/sem") util.ensure_dir(sem_dir) sem_file = os.path.join(sem_dir, "config_set_passwords") with open(sem_file, "w") as stream: stream.write("") ds = DataSourceExoscale({}, None, path) ds.activate(None, None) - self.assertFalse(os.path.exists(sem_file)) + assert not os.path.exists(sem_file) - def test_get_data(self): + @responses.activate + def test_get_data(self, tmp_path): """The datasource conforms to expected behavior when supplied full test data.""" - path = helpers.Paths({"run_dir": self.tmp}) + path = helpers.Paths({"run_dir": str(tmp_path)}) ds = DataSourceExoscale({}, None, path) ds.ds_detect = lambda: True expected_password = "p@ssw0rd" expected_id = "12345" expected_hostname = "myname" expected_userdata = "#cloud-config" - self.responses.add( - responses.GET, self.userdata_url, body=expected_userdata - ) - self.responses.add( - responses.GET, self.password_url, body=expected_password - ) - self.responses.add( + responses.add(responses.GET, TEST_USERDATA_URL, body=expected_userdata) + responses.add(responses.GET, TEST_PASSWORD_URL, body=expected_password) + responses.add( responses.GET, - self.metadata_url, + TEST_METADATA_URL, body="instance-id\nlocal-hostname", ) - self.responses.add( + responses.add( responses.GET, - "{}local-hostname".format(self.metadata_url), + "{}local-hostname".format(TEST_METADATA_URL), body=expected_hostname, ) - self.responses.add( + responses.add( responses.GET, - "{}instance-id".format(self.metadata_url), + "{}instance-id".format(TEST_METADATA_URL), body=expected_id, ) - self.assertTrue(ds._check_and_get_data()) - self.assertEqual(ds.userdata_raw.decode("utf-8"), "#cloud-config") - self.assertEqual( - ds.metadata, - {"instance-id": expected_id, "local-hostname": expected_hostname}, - ) - self.assertEqual( - ds.get_config_obj(), - { - "ssh_pwauth": True, - "password": expected_password, - "chpasswd": { - "expire": False, - }, + assert ds._check_and_get_data() + assert ds.userdata_raw.decode("utf-8") == "#cloud-config" + assert ds.metadata == { + "instance-id": expected_id, + "local-hostname": expected_hostname, + } + assert ds.get_config_obj() == { + "ssh_pwauth": True, + "password": expected_password, + "chpasswd": { + "expire": False, }, - ) + } - def test_get_data_saved_password(self): + @responses.activate + def test_get_data_saved_password(self, tmp_path): """The datasource conforms to expected behavior when saved_password is returned by the password server.""" - path = helpers.Paths({"run_dir": self.tmp}) + path = helpers.Paths({"run_dir": str(tmp_path)}) ds = DataSourceExoscale({}, None, path) ds.ds_detect = lambda: True expected_answer = "saved_password" expected_id = "12345" expected_hostname = "myname" expected_userdata = "#cloud-config" - self.responses.add( - responses.GET, self.userdata_url, body=expected_userdata - ) - self.responses.add( - responses.GET, self.password_url, body=expected_answer - ) - self.responses.add( + responses.add(responses.GET, TEST_USERDATA_URL, body=expected_userdata) + responses.add(responses.GET, TEST_PASSWORD_URL, body=expected_answer) + responses.add( responses.GET, - self.metadata_url, + TEST_METADATA_URL, body="instance-id\nlocal-hostname", ) - self.responses.add( + responses.add( responses.GET, - "{}local-hostname".format(self.metadata_url), + "{}local-hostname".format(TEST_METADATA_URL), body=expected_hostname, ) - self.responses.add( + responses.add( responses.GET, - "{}instance-id".format(self.metadata_url), + "{}instance-id".format(TEST_METADATA_URL), body=expected_id, ) - self.assertTrue(ds._check_and_get_data()) - self.assertEqual(ds.userdata_raw.decode("utf-8"), "#cloud-config") - self.assertEqual( - ds.metadata, - {"instance-id": expected_id, "local-hostname": expected_hostname}, - ) - self.assertEqual(ds.get_config_obj(), {}) + assert ds._check_and_get_data() + assert ds.userdata_raw.decode("utf-8") == "#cloud-config" + assert ds.metadata == { + "instance-id": expected_id, + "local-hostname": expected_hostname, + } + assert ds.get_config_obj() == {} - def test_get_data_no_password(self): + @responses.activate + def test_get_data_no_password(self, tmp_path): """The datasource conforms to expected behavior when no password is returned by the password server.""" - path = helpers.Paths({"run_dir": self.tmp}) + path = helpers.Paths({"run_dir": str(tmp_path)}) ds = DataSourceExoscale({}, None, path) ds.ds_detect = lambda: True expected_answer = "" expected_id = "12345" expected_hostname = "myname" expected_userdata = "#cloud-config" - self.responses.add( - responses.GET, self.userdata_url, body=expected_userdata - ) - self.responses.add( - responses.GET, self.password_url, body=expected_answer - ) - self.responses.add( + responses.add(responses.GET, TEST_USERDATA_URL, body=expected_userdata) + responses.add(responses.GET, TEST_PASSWORD_URL, body=expected_answer) + responses.add( responses.GET, - self.metadata_url, + TEST_METADATA_URL, body="instance-id\nlocal-hostname", ) - self.responses.add( + responses.add( responses.GET, - "{}local-hostname".format(self.metadata_url), + "{}local-hostname".format(TEST_METADATA_URL), body=expected_hostname, ) - self.responses.add( + responses.add( responses.GET, - "{}instance-id".format(self.metadata_url), + "{}instance-id".format(TEST_METADATA_URL), body=expected_id, ) - self.assertTrue(ds._check_and_get_data()) - self.assertEqual(ds.userdata_raw.decode("utf-8"), "#cloud-config") - self.assertEqual( - ds.metadata, - {"instance-id": expected_id, "local-hostname": expected_hostname}, - ) - self.assertEqual(ds.get_config_obj(), {}) + assert ds._check_and_get_data() + assert ds.userdata_raw.decode("utf-8") == "#cloud-config" + assert ds.metadata == { + "instance-id": expected_id, + "local-hostname": expected_hostname, + } + assert ds.get_config_obj() == {} + @responses.activate @mock.patch("cloudinit.sources.DataSourceExoscale.get_password") def test_read_metadata_when_password_server_unreachable(self, m_password): """The read_metadata function returns partial results in case the @@ -206,35 +187,31 @@ def test_read_metadata_when_password_server_unreachable(self, m_password): expected_userdata = "#cloud-config" m_password.side_effect = requests.Timeout("Fake Connection Timeout") - self.responses.add( - responses.GET, self.userdata_url, body=expected_userdata - ) - self.responses.add( + responses.add(responses.GET, TEST_USERDATA_URL, body=expected_userdata) + responses.add( responses.GET, - self.metadata_url, + TEST_METADATA_URL, body="instance-id\nlocal-hostname", ) - self.responses.add( + responses.add( responses.GET, - "{}local-hostname".format(self.metadata_url), + "{}local-hostname".format(TEST_METADATA_URL), body=expected_hostname, ) - self.responses.add( + responses.add( responses.GET, - "{}instance-id".format(self.metadata_url), + "{}instance-id".format(TEST_METADATA_URL), body=expected_id, ) result = read_metadata() - self.assertIsNone(result.get("password")) - self.assertEqual( - result.get("user-data").decode("utf-8"), expected_userdata - ) + assert result.get("password") is None + assert result.get("user-data").decode("utf-8") == expected_userdata - def test_non_viable_platform(self): + def test_non_viable_platform(self, tmp_path): """The datasource fails fast when the platform is not viable.""" - path = helpers.Paths({"run_dir": self.tmp}) + path = helpers.Paths({"run_dir": str(tmp_path)}) ds = DataSourceExoscale({}, None, path) ds.ds_detect = lambda: False - self.assertFalse(ds._check_and_get_data()) + assert not ds._check_and_get_data() diff --git a/tests/unittests/sources/test_gce.py b/tests/unittests/sources/test_gce.py index 350ebd12..d1e60472 100644 --- a/tests/unittests/sources/test_gce.py +++ b/tests/unittests/sources/test_gce.py @@ -11,12 +11,12 @@ from unittest import mock from urllib.parse import urlparse +import pytest import responses from cloudinit import distros, helpers, settings from cloudinit.net.dhcp import NoDHCPLeaseError from cloudinit.sources import DataSourceGCE -from tests.unittests import helpers as test_helpers M_PATH = "cloudinit.sources.DataSourceGCE." @@ -61,7 +61,7 @@ ) -class TestDataSourceGCE(test_helpers.ResponsesTestCase): +class TestDataSourceGCE: with_logs = True def _make_distro(self, dtype, def_user=None): @@ -74,27 +74,18 @@ def _make_distro(self, dtype, def_user=None): distro = distro_cls(dtype, cfg["system_info"], paths) return distro - def setUp(self): - tmp = self.tmp_dir() + @pytest.fixture(autouse=True) + def fixtures(self, mocker, paths): self.ds = DataSourceGCE.DataSourceGCE( - settings.CFG_BUILTIN, None, helpers.Paths({"run_dir": tmp}) + settings.CFG_BUILTIN, None, paths ) - - ppatch = self.m_platform_reports_gce = mock.patch( - M_PATH + "platform_reports_gce" + self.m_platform_reports_gce = mocker.patch( + M_PATH + "platform_reports_gce", return_value=True ) - self.m_platform_reports_gce = ppatch.start() - self.m_platform_reports_gce.return_value = True - self.addCleanup(ppatch.stop) - - pppatch = self.m_is_resolvable_url = mock.patch( + self.m_is_resolvable_url = mocker.patch( M_PATH + "util.is_resolvable_url", return_value=True ) - self.m_is_resolvable_url = pppatch.start() - self.addCleanup(pppatch.stop) - - self.add_patch("time.sleep", "m_sleep") # just to speed up tests - super(TestDataSourceGCE, self).setUp() + mocker.patch("time.sleep") def _set_mock_metadata(self, gce_meta=None, *, check_headers=None): if gce_meta is None: @@ -115,22 +106,24 @@ def _request_callback(request): response = json.dumps(response) if check_headers is not None: for k in check_headers.keys(): - self.assertEqual(check_headers[k], request.headers[k]) + assert check_headers[k] == request.headers[k] return (200, request.headers, response) else: return (404, request.headers, "") - self.responses.add_callback( + responses.add_callback( responses.GET, MD_URL_RE, callback=_request_callback, ) + @responses.activate def test_connection(self): self._set_mock_metadata(check_headers=HEADERS) success = self.ds.get_data() - self.assertTrue(success) + assert success + @responses.activate def test_metadata(self): # UnicodeDecodeError if set to ds.userdata instead of userdata_raw meta = GCE_META.copy() @@ -140,38 +133,37 @@ def test_metadata(self): self.ds.get_data() shostname = GCE_META.get("instance/hostname").split(".")[0] - self.assertEqual(shostname, self.ds.get_hostname().hostname) + assert shostname == self.ds.get_hostname().hostname - self.assertEqual( - GCE_META.get("instance/id"), self.ds.get_instance_id() - ) + assert GCE_META.get("instance/id") == self.ds.get_instance_id() - self.assertEqual( - GCE_META.get("instance/attributes/user-data"), - self.ds.get_userdata_raw(), + assert ( + GCE_META.get("instance/attributes/user-data") + == self.ds.get_userdata_raw() ) + @responses.activate def test_metadata_partial(self): """test partial metadata (missing user-data in particular)""" self._set_mock_metadata(GCE_META_PARTIAL) self.ds.get_data() - self.assertEqual( - GCE_META_PARTIAL.get("instance/id"), self.ds.get_instance_id() - ) + assert GCE_META_PARTIAL.get("instance/id") == self.ds.get_instance_id() shostname = GCE_META_PARTIAL.get("instance/hostname").split(".")[0] - self.assertEqual(shostname, self.ds.get_hostname().hostname) + assert shostname == self.ds.get_hostname().hostname + @responses.activate def test_userdata_no_encoding(self): """check that user-data is read.""" self._set_mock_metadata(GCE_USER_DATA_TEXT) self.ds.get_data() - self.assertEqual( - GCE_USER_DATA_TEXT["instance/attributes"]["user-data"].encode(), - self.ds.get_userdata_raw(), + assert ( + GCE_USER_DATA_TEXT["instance/attributes"]["user-data"].encode() + == self.ds.get_userdata_raw() ) + @responses.activate def test_metadata_encoding(self): """user-data is base64 encoded if user-data-encoding is 'base64'.""" self._set_mock_metadata(GCE_META_ENCODING) @@ -179,8 +171,9 @@ def test_metadata_encoding(self): instance_data = GCE_META_ENCODING.get("instance/attributes") decoded = b64decode(instance_data.get("user-data")) - self.assertEqual(decoded, self.ds.get_userdata_raw()) + assert decoded == self.ds.get_userdata_raw() + @responses.activate def test_missing_required_keys_return_false(self): for required_key in [ "instance/id", @@ -190,14 +183,16 @@ def test_missing_required_keys_return_false(self): meta = GCE_META_PARTIAL.copy() del meta[required_key] self._set_mock_metadata(meta) - self.assertEqual(False, self.ds.get_data()) - self.responses.reset() + assert False is self.ds.get_data() + responses.reset() + @responses.activate def test_no_ssh_keys_metadata(self): self._set_mock_metadata() self.ds.get_data() - self.assertEqual([], self.ds.get_public_ssh_keys()) + assert [] == self.ds.get_public_ssh_keys() + @responses.activate def test_cloudinit_ssh_keys(self): valid_key = "ssh-rsa VALID {0}" invalid_key = "ssh-rsa INVALID {0}" @@ -233,16 +228,17 @@ def test_cloudinit_ssh_keys(self): self.ds.get_data() expected = [valid_key.format(key) for key in range(3)] - self.assertEqual(set(expected), set(self.ds.get_public_ssh_keys())) + assert set(expected) == set(self.ds.get_public_ssh_keys()) + @responses.activate @mock.patch(M_PATH + "ug_util") - def test_default_user_ssh_keys(self, mock_ug_util): + def test_default_user_ssh_keys(self, mock_ug_util, paths): mock_ug_util.normalize_users_groups.return_value = None, None mock_ug_util.extract_default.return_value = "ubuntu", None ubuntu_ds = DataSourceGCE.DataSourceGCE( settings.CFG_BUILTIN, self._make_distro("ubuntu"), - helpers.Paths({"run_dir": self.tmp_dir()}), + paths, ) valid_key = "ssh-rsa VALID {0}" @@ -279,8 +275,9 @@ def test_default_user_ssh_keys(self, mock_ug_util): ubuntu_ds.get_data() expected = [valid_key.format(key) for key in range(3)] - self.assertEqual(set(expected), set(ubuntu_ds.get_public_ssh_keys())) + assert set(expected) == set(ubuntu_ds.get_public_ssh_keys()) + @responses.activate def test_instance_ssh_keys_override(self): valid_key = "ssh-rsa VALID {0}" invalid_key = "ssh-rsa INVALID {0}" @@ -302,8 +299,9 @@ def test_instance_ssh_keys_override(self): self.ds.get_data() expected = [valid_key.format(key) for key in range(2)] - self.assertEqual(set(expected), set(self.ds.get_public_ssh_keys())) + assert set(expected) == set(self.ds.get_public_ssh_keys()) + @responses.activate def test_block_project_ssh_keys_override(self): valid_key = "ssh-rsa VALID {0}" invalid_key = "ssh-rsa INVALID {0}" @@ -324,19 +322,19 @@ def test_block_project_ssh_keys_override(self): self.ds.get_data() expected = [valid_key.format(0)] - self.assertEqual(set(expected), set(self.ds.get_public_ssh_keys())) + assert set(expected) == set(self.ds.get_public_ssh_keys()) + @responses.activate def test_only_last_part_of_zone_used_for_availability_zone(self): self._set_mock_metadata() - r = self.ds.get_data() - self.assertEqual(True, r) - self.assertEqual("bar", self.ds.availability_zone) + assert True is self.ds.get_data() + assert "bar" == self.ds.availability_zone @mock.patch("cloudinit.sources.DataSourceGCE.GoogleMetadataFetcher") def test_get_data_returns_false_if_not_on_gce(self, m_fetcher): self.m_platform_reports_gce.return_value = False ret = self.ds.get_data() - self.assertEqual(False, ret) + assert False is ret m_fetcher.assert_not_called() def test_has_expired(self): @@ -363,7 +361,7 @@ def _get_timestamp(days): } for key, expired in ssh_keys.items(): - self.assertEqual(DataSourceGCE._has_expired(key), expired) + assert DataSourceGCE._has_expired(key) == expired def test_parse_public_keys_non_ascii(self): public_key_data = [ @@ -377,7 +375,7 @@ def test_parse_public_keys_non_ascii(self): found = DataSourceGCE._parse_public_keys( public_key_data, default_user="default" ) - self.assertEqual(sorted(found), sorted(expected)) + assert sorted(found) == sorted(expected) @mock.patch("cloudinit.url_helper.readurl") def test_publish_host_keys(self, m_readurl): @@ -401,23 +399,25 @@ def test_publish_host_keys(self, m_readurl): self.ds.publish_host_keys(hostkeys) m_readurl.assert_has_calls(readurl_expected_calls, any_order=True) + @responses.activate @mock.patch( M_PATH + "EphemeralDHCPv4", autospec=True, ) @mock.patch(M_PATH + "net.find_candidate_nics", return_value=["ens4"]) def test_local_datasource_uses_ephemeral_dhcp( - self, _m_find_candidate_nics, m_dhcp + self, _m_find_candidate_nics, m_dhcp, tmp_path ): self._set_mock_metadata() distro = mock.MagicMock() - distro.get_tmp_exec_path = self.tmp_dir + distro.get_tmp_exec_path = str(tmp_path) ds = DataSourceGCE.DataSourceGCELocal( sys_cfg={}, distro=distro, paths=None ) ds._get_data() assert m_dhcp.call_count == 1 + @responses.activate @mock.patch(M_PATH + "read_md") @mock.patch( M_PATH + "EphemeralDHCPv4", @@ -425,11 +425,11 @@ def test_local_datasource_uses_ephemeral_dhcp( ) @mock.patch(M_PATH + "net.find_candidate_nics") def test_local_datasource_tries_on_multi_nic( - self, m_find_candidate_nics, m_dhcp, m_read_md + self, m_find_candidate_nics, m_dhcp, m_read_md, caplog, tmp_path ): self._set_mock_metadata() distro = mock.MagicMock() - distro.get_tmp_exec_path = self.tmp_dir + distro.get_tmp_exec_path = str(tmp_path) ds = DataSourceGCE.DataSourceGCELocal( sys_cfg={}, distro=distro, paths=None ) @@ -474,11 +474,9 @@ def test_local_datasource_tries_on_multi_nic( " whoopsie, not this one", ) for msg in expected_logs: - self.assertIn( - msg, - self.logs.getvalue(), - ) + assert msg in caplog.text + @responses.activate @mock.patch( M_PATH + "EphemeralDHCPv4", autospec=True, @@ -489,17 +487,18 @@ def test_datasource_doesnt_use_ephemeral_dhcp(self, m_dhcp): ds._get_data() assert m_dhcp.call_count == 0 + @responses.activate @mock.patch( M_PATH + "EphemeralDHCPv4", autospec=True, ) @mock.patch(M_PATH + "net.find_candidate_nics") def test_datasource_on_dhcp_lease_failure( - self, m_find_candidate_nics, m_dhcp + self, m_find_candidate_nics, m_dhcp, caplog, tmp_path ): self._set_mock_metadata() distro = mock.MagicMock() - distro.get_tmp_exec_path = self.tmp_dir + distro.get_tmp_exec_path = str(tmp_path) ds = DataSourceGCE.DataSourceGCELocal( sys_cfg={}, distro=distro, paths=None ) @@ -523,7 +522,28 @@ def test_datasource_on_dhcp_lease_failure( "Unable to obtain a DHCP lease for ens0p5", ) for msg in expected_logs: - self.assertIn( - msg, - self.logs.getvalue(), - ) + assert msg in caplog.text + + @responses.activate + def test_instance_and_project_data_decoded(self): + """instance_data and project_data is decoded and can be queried.""" + md = { + "instance/id": "123", + "instance/zone": "foo/bar", + "instance/hostname": "server.project-foo.local", + "instance/attributes": {"ikey": "ivalue"}, + "project/attributes": {"pkey": "pvalue"}, + } + + self._set_mock_metadata(md) + assert self.ds.get_data() is True + + assert "instance-data" in self.ds.metadata + assert isinstance(self.ds.metadata["instance-data"], dict) + assert "ikey" in self.ds.metadata["instance-data"] + assert "ivalue" == self.ds.metadata["instance-data"]["ikey"] + + assert "project-data" in self.ds.metadata + assert isinstance(self.ds.metadata["project-data"], dict) + assert "pkey" in self.ds.metadata["project-data"] + assert "pvalue" == self.ds.metadata["project-data"]["pkey"] diff --git a/tests/unittests/sources/test_hetzner.py b/tests/unittests/sources/test_hetzner.py index 5867a4fa..26883756 100644 --- a/tests/unittests/sources/test_hetzner.py +++ b/tests/unittests/sources/test_hetzner.py @@ -4,9 +4,11 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import helpers, settings, util +import pytest + +from cloudinit import settings, util from cloudinit.sources import DataSourceHetzner -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import mock METADATA = util.load_yaml( """ @@ -51,20 +53,17 @@ """ -class TestDataSourceHetzner(CiTestCase): +class TestDataSourceHetzner: """ Test reading the meta-data """ - def setUp(self): - super(TestDataSourceHetzner, self).setUp() - self.tmp = self.tmp_dir() - - def get_ds(self): + @pytest.fixture + def ds(self, paths, tmp_path): distro = mock.MagicMock() - distro.get_tmp_exec_path = self.tmp_dir + distro.get_tmp_exec_path = str(tmp_path) ds = DataSourceHetzner.DataSourceHetzner( - settings.CFG_BUILTIN, distro, helpers.Paths({"run_dir": self.tmp}) + settings.CFG_BUILTIN, distro, paths ) return ds @@ -82,6 +81,7 @@ def test_read_data( m_fallback_nic, m_net, m_dhcp, + ds, ): m_get_hcloud_data.return_value = ( True, @@ -100,9 +100,7 @@ def test_read_data( } ] - ds = self.get_ds() - ret = ds.get_data() - self.assertTrue(ret) + assert True is ds.get_data() m_net.assert_called_once_with( ds.distro, @@ -114,29 +112,28 @@ def test_read_data( ], ) - self.assertTrue(m_readmd.called) + assert 0 != m_readmd.call_count - self.assertEqual(METADATA.get("hostname"), ds.get_hostname().hostname) + assert METADATA.get("hostname") == ds.get_hostname().hostname - self.assertEqual(METADATA.get("public-keys"), ds.get_public_ssh_keys()) + assert METADATA.get("public-keys") == ds.get_public_ssh_keys() - self.assertIsInstance(ds.get_public_ssh_keys(), list) - self.assertEqual(ds.get_userdata_raw(), USERDATA) - self.assertEqual(ds.get_vendordata_raw(), METADATA.get("vendor_data")) + assert isinstance(ds.get_public_ssh_keys(), list) + assert ds.get_userdata_raw() == USERDATA + assert ds.get_vendordata_raw() == METADATA.get("vendor_data") @mock.patch("cloudinit.sources.helpers.hetzner.read_metadata") @mock.patch("cloudinit.net.find_fallback_nic") @mock.patch("cloudinit.sources.DataSourceHetzner.get_hcloud_data") def test_not_on_hetzner_returns_false( - self, m_get_hcloud_data, m_find_fallback, m_read_md + self, m_get_hcloud_data, m_find_fallback, m_read_md, ds ): """If helper 'get_hcloud_data' returns False, return False from get_data.""" m_get_hcloud_data.return_value = (False, None) - ds = self.get_ds() ret = ds.get_data() - self.assertFalse(ret) + assert not ret # These are a white box attempt to ensure it did not search. - m_find_fallback.assert_not_called() - m_read_md.assert_not_called() + assert 0 == m_find_fallback.call_count + assert 0 == m_read_md.call_count diff --git a/tests/unittests/sources/test_ibmcloud.py b/tests/unittests/sources/test_ibmcloud.py index 37c2594d..2589871a 100644 --- a/tests/unittests/sources/test_ibmcloud.py +++ b/tests/unittests/sources/test_ibmcloud.py @@ -5,8 +5,9 @@ import json from textwrap import dedent +import pytest + from cloudinit import util -from cloudinit.helpers import Paths from cloudinit.sources import DataSourceIBMCloud as ibm from tests.unittests import helpers as test_helpers @@ -18,7 +19,7 @@ @mock.patch(D_PATH + "_is_xen", return_value=True) @mock.patch(D_PATH + "_is_ibm_provisioning") @mock.patch(D_PATH + "util.blkid") -class TestGetIBMPlatform(test_helpers.CiTestCase): +class TestGetIBMPlatform: """Test the get_ibm_platform helper.""" blkid_base = { @@ -55,7 +56,8 @@ class TestGetIBMPlatform(test_helpers.CiTestCase): } } - def setUp(self): + @pytest.fixture(autouse=True) + def fixtures(self): self.blkid_metadata = copy.deepcopy(self.blkid_base) self.blkid_metadata.update(copy.deepcopy(self.blkid_metadata_disk)) @@ -66,36 +68,34 @@ def test_id_template_live_metadata(self, m_blkid, m_is_prov, _m_xen): """identify TEMPLATE_LIVE_METADATA.""" m_blkid.return_value = self.blkid_metadata m_is_prov.return_value = False - self.assertEqual( - (ibm.Platforms.TEMPLATE_LIVE_METADATA, "/dev/xvdh1"), - ibm.get_ibm_platform(), - ) + assert ( + ibm.Platforms.TEMPLATE_LIVE_METADATA, + "/dev/xvdh1", + ) == ibm.get_ibm_platform() def test_id_template_prov_metadata(self, m_blkid, m_is_prov, _m_xen): """identify TEMPLATE_PROVISIONING_METADATA.""" m_blkid.return_value = self.blkid_metadata m_is_prov.return_value = True - self.assertEqual( - (ibm.Platforms.TEMPLATE_PROVISIONING_METADATA, "/dev/xvdh1"), - ibm.get_ibm_platform(), - ) + assert ( + ibm.Platforms.TEMPLATE_PROVISIONING_METADATA, + "/dev/xvdh1", + ) == ibm.get_ibm_platform() def test_id_template_prov_nodata(self, m_blkid, m_is_prov, _m_xen): """identify TEMPLATE_PROVISIONING_NODATA.""" m_blkid.return_value = self.blkid_base m_is_prov.return_value = True - self.assertEqual( - (ibm.Platforms.TEMPLATE_PROVISIONING_NODATA, None), - ibm.get_ibm_platform(), - ) + assert ( + ibm.Platforms.TEMPLATE_PROVISIONING_NODATA, + None, + ) == ibm.get_ibm_platform() def test_id_os_code(self, m_blkid, m_is_prov, _m_xen): """Identify OS_CODE.""" m_blkid.return_value = self.blkid_oscode m_is_prov.return_value = False - self.assertEqual( - (ibm.Platforms.OS_CODE, "/dev/xvdh"), ibm.get_ibm_platform() - ) + assert (ibm.Platforms.OS_CODE, "/dev/xvdh") == ibm.get_ibm_platform() def test_id_os_code_must_match_uuid(self, m_blkid, m_is_prov, _m_xen): """Test against false positive on openstack with non-ibm UUID.""" @@ -103,12 +103,12 @@ def test_id_os_code_must_match_uuid(self, m_blkid, m_is_prov, _m_xen): blkid["/dev/xvdh"]["UUID"] = "9999-9999" m_blkid.return_value = blkid m_is_prov.return_value = False - self.assertEqual((None, None), ibm.get_ibm_platform()) + assert (None, None) == ibm.get_ibm_platform() @mock.patch(D_PATH + "_read_system_uuid", return_value=None) @mock.patch(D_PATH + "get_ibm_platform") -class TestReadMD(test_helpers.CiTestCase): +class TestReadMD: """Test the read_datasource helper.""" template_md = { @@ -235,7 +235,7 @@ def test_provisioning_md(self, m_platform, m_sysuuid): ibm.Platforms.TEMPLATE_PROVISIONING_METADATA, "/dev/xvdh", ) - self.assertIsNone(ibm.read_md()) + assert ibm.read_md() is None def test_provisioning_no_metadata(self, m_platform, m_sysuuid): """Provisioning env with no metadata disk should return None.""" @@ -243,16 +243,16 @@ def test_provisioning_no_metadata(self, m_platform, m_sysuuid): ibm.Platforms.TEMPLATE_PROVISIONING_NODATA, None, ) - self.assertIsNone(ibm.read_md()) + assert ibm.read_md() is None def test_provisioning_not_ibm(self, m_platform, m_sysuuid): """Provisioning env but not identified as IBM should return None.""" m_platform.return_value = (None, None) - self.assertIsNone(ibm.read_md()) + assert ibm.read_md() is None - def test_template_live(self, m_platform, m_sysuuid): + def test_template_live(self, m_platform, m_sysuuid, tmp_path): """Template live environment should be identified.""" - tmpdir = self.tmp_dir() + tmpdir = str(tmp_path) m_platform.return_value = ( ibm.Platforms.TEMPLATE_LIVE_METADATA, tmpdir, @@ -273,18 +273,16 @@ def test_template_live(self, m_platform, m_sysuuid): ret = ibm.read_md() if ret is None: # this is needed for mypy - ensures ret is not None - self.fail("read_md returned None unexpectedly") - self.assertEqual(ibm.Platforms.TEMPLATE_LIVE_METADATA, ret["platform"]) - self.assertEqual(tmpdir, ret["source"]) - self.assertEqual(self.userdata, ret["userdata"]) - self.assertEqual( - self._get_expected_metadata(self.template_md), ret["metadata"] - ) - self.assertEqual(self.sysuuid, ret["system-uuid"]) - - def test_os_code_live(self, m_platform, m_sysuuid): + pytest.fail("read_md returned None unexpectedly") + assert ibm.Platforms.TEMPLATE_LIVE_METADATA == ret["platform"] + assert tmpdir == ret["source"] + assert self.userdata == ret["userdata"] + assert self._get_expected_metadata(self.template_md) == ret["metadata"] + assert self.sysuuid == ret["system-uuid"] + + def test_os_code_live(self, m_platform, m_sysuuid, tmp_path): """Verify an os_code metadata path.""" - tmpdir = self.tmp_dir() + tmpdir = str(tmp_path) m_platform.return_value = (ibm.Platforms.OS_CODE, tmpdir) netdata = json.dumps(self.network_data) test_helpers.populate_dir( @@ -301,17 +299,15 @@ def test_os_code_live(self, m_platform, m_sysuuid): ret = ibm.read_md() if ret is None: # this is needed for mypy - ensures ret is not None - self.fail("read_md returned None unexpectedly") - self.assertEqual(ibm.Platforms.OS_CODE, ret["platform"]) - self.assertEqual(tmpdir, ret["source"]) - self.assertEqual(self.userdata, ret["userdata"]) - self.assertEqual( - self._get_expected_metadata(self.oscode_md), ret["metadata"] - ) + pytest.fail("read_md returned None unexpectedly") + assert ibm.Platforms.OS_CODE == ret["platform"] + assert tmpdir == ret["source"] + assert self.userdata == ret["userdata"] + assert self._get_expected_metadata(self.oscode_md) == ret["metadata"] - def test_os_code_live_no_userdata(self, m_platform, m_sysuuid): + def test_os_code_live_no_userdata(self, m_platform, m_sysuuid, tmp_path): """Verify os_code without user-data.""" - tmpdir = self.tmp_dir() + tmpdir = str(tmp_path) m_platform.return_value = (ibm.Platforms.OS_CODE, tmpdir) test_helpers.populate_dir( tmpdir, @@ -325,84 +321,72 @@ def test_os_code_live_no_userdata(self, m_platform, m_sysuuid): ret = ibm.read_md() if ret is None: # this is needed for mypy - ensures ret is not None - self.fail("read_md returned None unexpectedly") - self.assertEqual(ibm.Platforms.OS_CODE, ret["platform"]) - self.assertEqual(tmpdir, ret["source"]) - self.assertIsNone(ret["userdata"]) - self.assertEqual( - self._get_expected_metadata(self.oscode_md), ret["metadata"] - ) + pytest.fail("read_md returned None unexpectedly") + assert ibm.Platforms.OS_CODE == ret["platform"] + assert tmpdir == ret["source"] + assert ret["userdata"] is None + assert self._get_expected_metadata(self.oscode_md) == ret["metadata"] -class TestIsIBMProvisioning(test_helpers.FilesystemMockingTestCase): +@pytest.mark.usefixtures("fake_filesystem") +class TestIsIBMProvisioning: """Test the _is_ibm_provisioning method.""" inst_log = "/root/swinstall.log" prov_cfg = "/root/provisioningConfiguration.cfg" boot_ref = "/proc/1/environ" - with_logs = True - - def _call_with_root(self, rootd): - self.reRoot(rootd) - return ibm._is_ibm_provisioning() def test_no_config(self): """No provisioning config means not provisioning.""" - self.assertFalse(self._call_with_root(self.tmp_dir())) + assert not ibm._is_ibm_provisioning() - def test_config_only(self): + def test_config_only(self, tmp_path): """A provisioning config without a log means provisioning.""" - rootd = self.tmp_dir() - test_helpers.populate_dir(rootd, {self.prov_cfg: "key=value"}) - self.assertTrue(self._call_with_root(rootd)) + test_helpers.populate_dir(str(tmp_path), {self.prov_cfg: "key=value"}) + assert ibm._is_ibm_provisioning() - def test_config_with_old_log(self): + def test_config_with_old_log(self, caplog, tmp_path): """A config with a log from previous boot is not provisioning.""" - rootd = self.tmp_dir() data = { self.prov_cfg: ("key=value\nkey2=val2\n", -10), self.inst_log: ("log data\n", -30), self.boot_ref: ("PWD=/", 0), } - test_helpers.populate_dir_with_ts(rootd, data) - self.assertFalse(self._call_with_root(rootd=rootd)) - self.assertIn("from previous boot", self.logs.getvalue()) + test_helpers.populate_dir_with_ts(str(tmp_path), data) + assert not ibm._is_ibm_provisioning() + assert "from previous boot" in caplog.text - def test_config_with_new_log(self): + def test_config_with_new_log(self, caplog, tmp_path): """A config with a log from this boot is provisioning.""" - rootd = self.tmp_dir() data = { self.prov_cfg: ("key=value\nkey2=val2\n", -10), self.inst_log: ("log data\n", 30), self.boot_ref: ("PWD=/", 0), } - test_helpers.populate_dir_with_ts(rootd, data) - self.assertTrue(self._call_with_root(rootd=rootd)) - self.assertIn("from current boot", self.logs.getvalue()) + test_helpers.populate_dir_with_ts(str(tmp_path), data) + assert ibm._is_ibm_provisioning() + assert "from current boot" in caplog.text - def test_config_and_log_no_reference(self): + def test_config_and_log_no_reference(self, caplog, tmp_path): """If the config and log existed, but no reference, assume not.""" - rootd = self.tmp_dir() test_helpers.populate_dir( - rootd, {self.prov_cfg: "key=value", self.inst_log: "log data\n"} + str(tmp_path), + {self.prov_cfg: "key=value", self.inst_log: "log data\n"}, ) - self.assertFalse(self._call_with_root(rootd=rootd)) - self.assertIn("no reference file", self.logs.getvalue()) + assert not ibm._is_ibm_provisioning() + assert "no reference file" in caplog.text -class TestDataSourceIBMCloud(test_helpers.CiTestCase): - def setUp(self): - super(TestDataSourceIBMCloud, self).setUp() - self.tmp = self.tmp_dir() - self.cloud_dir = self.tmp_path("cloud", dir=self.tmp) - util.ensure_dir(self.cloud_dir) - paths = Paths({"run_dir": self.tmp, "cloud_dir": self.cloud_dir}) +class TestDataSourceIBMCloud: + @pytest.fixture(autouse=True) + def fixture(self, paths): + util.ensure_dir(paths.cloud_dir) self.ds = ibm.DataSourceIBMCloud(sys_cfg={}, distro=None, paths=paths) def test_get_data_false(self): """When read_md returns None, get_data returns False.""" with mock.patch(D_PATH + "read_md", return_value=None): - self.assertFalse(self.ds.get_data()) + assert not self.ds.get_data() def test_get_data_processes_read_md(self): """get_data processes and caches content returned by read_md.""" @@ -416,13 +400,13 @@ def test_get_data_processes_read_md(self): "vendordata": "vd", } with mock.patch(D_PATH + "read_md", return_value=md): - self.assertTrue(self.ds.get_data()) - self.assertEqual("src", self.ds.source) - self.assertEqual("plat", self.ds.platform) - self.assertEqual({}, self.ds.metadata) - self.assertEqual("ud", self.ds.userdata_raw) - self.assertEqual("net", self.ds.network_json) - self.assertEqual("uuid", self.ds.system_uuid) - self.assertEqual("ibmcloud", self.ds.cloud_name) - self.assertEqual("ibmcloud", self.ds.platform_type) - self.assertEqual("plat (src)", self.ds.subplatform) + assert self.ds.get_data() + assert "src" == self.ds.source + assert "plat" == self.ds.platform + assert {} == self.ds.metadata + assert "ud" == self.ds.userdata_raw + assert "net" == self.ds.network_json + assert "uuid" == self.ds.system_uuid + assert "ibmcloud" == self.ds.cloud_name + assert "ibmcloud" == self.ds.platform_type + assert "plat (src)" == self.ds.subplatform diff --git a/tests/unittests/sources/test_init.py b/tests/unittests/sources/test_init.py index de8ded4a..05bebd6e 100644 --- a/tests/unittests/sources/test_init.py +++ b/tests/unittests/sources/test_init.py @@ -3,9 +3,12 @@ import copy import inspect +import logging import os import stat +import pytest + from cloudinit import importer, util from cloudinit.distros import ubuntu from cloudinit.event import EventScope, EventType @@ -21,7 +24,7 @@ redact_sensitive_keys, ) from cloudinit.user_data import UserDataProcessor -from tests.unittests.helpers import CiTestCase, mock +from tests.unittests.helpers import CiTestCase, assert_count_equal, mock class DataSourceTestSubclassNet(DataSource): @@ -67,61 +70,60 @@ class InvalidDataSourceTestSubclassNet(DataSource): pass -class TestDataSource(CiTestCase): - - with_logs = True - maxDiff = None +class TestDataSource: - def setUp(self): - super(TestDataSource, self).setUp() + @pytest.fixture(autouse=True) + def fixtures(self, paths): self.sys_cfg = {"datasource": {"_undef": {"key1": False}}} self.distro = ubuntu.Distro("somedistro", {}, {}) - self.paths = Paths({}) - self.datasource = DataSource(self.sys_cfg, self.distro, self.paths) + self.datasource = DataSource(self.sys_cfg, self.distro, paths) + + @pytest.fixture + def datasource(self, paths): + return DataSourceTestSubclassNet(self.sys_cfg, self.distro, paths) def test_datasource_init(self): """DataSource initializes metadata attributes, ds_cfg and ud_proc.""" - self.assertEqual(self.paths, self.datasource.paths) - self.assertEqual(self.sys_cfg, self.datasource.sys_cfg) - self.assertEqual(self.distro, self.datasource.distro) - self.assertIsNone(self.datasource.userdata) - self.assertEqual({}, self.datasource.metadata) - self.assertIsNone(self.datasource.userdata_raw) - self.assertIsNone(self.datasource.vendordata) - self.assertIsNone(self.datasource.vendordata_raw) - self.assertEqual({"key1": False}, self.datasource.ds_cfg) - self.assertIsInstance(self.datasource.ud_proc, UserDataProcessor) - - def test_datasource_init_gets_ds_cfg_using_dsname(self): + assert self.sys_cfg == self.datasource.sys_cfg + assert self.distro == self.datasource.distro + assert self.datasource.userdata is None + assert {} == self.datasource.metadata + assert self.datasource.userdata_raw is None + assert self.datasource.vendordata is None + assert self.datasource.vendordata_raw is None + assert {"key1": False} == self.datasource.ds_cfg + assert isinstance(self.datasource.ud_proc, UserDataProcessor) + + def test_datasource_init_gets_ds_cfg_using_dsname(self, paths): """Init uses DataSource.dsname for sourcing ds_cfg.""" sys_cfg = {"datasource": {"MyTestSubclass": {"key2": False}}} distro = "distrotest" # generally should be a Distro object - datasource = DataSourceTestSubclassNet(sys_cfg, distro, self.paths) - self.assertEqual({"key2": False}, datasource.ds_cfg) + datasource = DataSourceTestSubclassNet(sys_cfg, distro, paths) + assert {"key2": False} == datasource.ds_cfg - def test_str_is_classname(self): + def test_str_is_classname(self, paths): """The string representation of the datasource is the classname.""" - self.assertEqual("DataSource", str(self.datasource)) - self.assertEqual( - "DataSourceTestSubclassNet", - str(DataSourceTestSubclassNet("", "", self.paths)), + assert "DataSource" == str(self.datasource) + assert "DataSourceTestSubclassNet" == str( + DataSourceTestSubclassNet("", "", paths) ) def test_datasource_get_url_params_defaults(self): """get_url_params default url config settings for the datasource.""" params = self.datasource.get_url_params() - self.assertEqual(params.max_wait_seconds, self.datasource.url_max_wait) - self.assertEqual(params.timeout_seconds, self.datasource.url_timeout) - self.assertEqual(params.num_retries, self.datasource.url_retries) - self.assertEqual( - params.sec_between_retries, self.datasource.url_sec_between_retries + assert params.max_wait_seconds == self.datasource.url_max_wait + assert params.timeout_seconds == self.datasource.url_timeout + assert params.num_retries == self.datasource.url_retries + assert ( + params.sec_between_retries + == self.datasource.url_sec_between_retries ) - def test_datasource_get_url_params_subclassed(self): + def test_datasource_get_url_params_subclassed(self, paths): """Subclasses can override get_url_params defaults.""" sys_cfg = {"datasource": {"MyTestSubclass": {"key2": False}}} distro = "distrotest" # generally should be a Distro object - datasource = DataSourceTestSubclassNet(sys_cfg, distro, self.paths) + datasource = DataSourceTestSubclassNet(sys_cfg, distro, paths) expected = ( datasource.url_max_wait, datasource.url_timeout, @@ -129,10 +131,10 @@ def test_datasource_get_url_params_subclassed(self): datasource.url_sec_between_retries, ) url_params = datasource.get_url_params() - self.assertNotEqual(self.datasource.get_url_params(), url_params) - self.assertEqual(expected, url_params) + assert self.datasource.get_url_params() != url_params + assert expected == url_params - def test_datasource_get_url_params_ds_config_override(self): + def test_datasource_get_url_params_ds_config_override(self, paths): """Datasource configuration options can override url param defaults.""" sys_cfg = { "datasource": { @@ -144,36 +146,31 @@ def test_datasource_get_url_params_ds_config_override(self): } } } - datasource = DataSourceTestSubclassNet( - sys_cfg, self.distro, self.paths - ) + datasource = DataSourceTestSubclassNet(sys_cfg, self.distro, paths) expected = (1, 2, 3, 4) url_params = datasource.get_url_params() - self.assertNotEqual( - ( - datasource.url_max_wait, - datasource.url_timeout, - datasource.url_retries, - datasource.url_sec_between_retries, - ), - url_params, - ) - self.assertEqual(expected, url_params) + assert ( + datasource.url_max_wait, + datasource.url_timeout, + datasource.url_retries, + datasource.url_sec_between_retries, + ) != url_params + assert expected == url_params - def test_datasource_get_url_params_is_zero_or_greater(self): + def test_datasource_get_url_params_is_zero_or_greater(self, paths): """get_url_params ignores timeouts with a value below 0.""" # Set an override that is below 0 which gets ignored. sys_cfg = {"datasource": {"_undef": {"timeout": "-1"}}} - datasource = DataSource(sys_cfg, self.distro, self.paths) + datasource = DataSource(sys_cfg, self.distro, paths) ( _max_wait, timeout, _retries, _sec_between_retries, ) = datasource.get_url_params() - self.assertEqual(0, timeout) + assert 0 == timeout - def test_datasource_get_url_uses_defaults_on_errors(self): + def test_datasource_get_url_uses_defaults_on_errors(self, caplog, paths): """On invalid system config values for url_params defaults are used.""" # All invalid values should be logged sys_cfg = { @@ -185,7 +182,7 @@ def test_datasource_get_url_uses_defaults_on_errors(self): } } } - datasource = DataSource(sys_cfg, self.distro, self.paths) + datasource = DataSource(sys_cfg, self.distro, paths) url_params = datasource.get_url_params() expected = ( datasource.url_max_wait, @@ -193,188 +190,158 @@ def test_datasource_get_url_uses_defaults_on_errors(self): datasource.url_retries, datasource.url_sec_between_retries, ) - self.assertEqual(expected, url_params) - logs = self.logs.getvalue() + assert expected == url_params expected_logs = [ "Config max_wait 'nope' is not an int, using default '-1'", "Config timeout 'bug' is not an int, using default '10'", "Config retries 'nonint' is not an int, using default '5'", ] for log in expected_logs: - self.assertIn(log, logs) + assert log in caplog.text @mock.patch("cloudinit.distros.net.find_fallback_nic") def test_fallback_interface_is_discovered(self, m_get_fallback_nic): """The fallback_interface is discovered via find_fallback_nic.""" m_get_fallback_nic.return_value = "nic9" - self.assertEqual("nic9", self.datasource.distro.fallback_interface) + assert "nic9" == self.datasource.distro.fallback_interface @mock.patch("cloudinit.sources.net.find_fallback_nic") - def test_fallback_interface_logs_undiscovered(self, m_get_fallback_nic): + def test_fallback_interface_logs_undiscovered( + self, m_get_fallback_nic, caplog + ): """Log a warning when fallback_interface can not discover the nic.""" m_get_fallback_nic.return_value = None # Couldn't discover nic - self.assertIsNone(self.datasource.distro.fallback_interface) - self.assertEqual( - "WARNING: Did not find a fallback interface on distro: " - "somedistro.\n", - self.logs.getvalue(), - ) + assert self.datasource.distro.fallback_interface is None + assert ( + mock.ANY, + logging.WARNING, + "Did not find a fallback interface on distro: somedistro.", + ) in caplog.record_tuples @mock.patch("cloudinit.sources.net.find_fallback_nic") def test_wb_fallback_interface_is_cached(self, m_get_fallback_nic): """The fallback_interface is cached and won't be rediscovered.""" self.datasource.distro.fallback_interface = "nic10" - self.assertEqual("nic10", self.datasource.distro.fallback_interface) + assert "nic10" == self.datasource.distro.fallback_interface m_get_fallback_nic.assert_not_called() - def test__get_data_unimplemented(self): + def test__get_data_unimplemented(self, paths): """Raise an error when _get_data is not implemented.""" - with self.assertRaises(NotImplementedError) as context_manager: + with pytest.raises( + NotImplementedError, + match="Subclasses of DataSource must implement _get_data", + ): self.datasource.get_data() - self.assertIn( - "Subclasses of DataSource must implement _get_data", - str(context_manager.exception), - ) datasource2 = InvalidDataSourceTestSubclassNet( - self.sys_cfg, self.distro, self.paths + self.sys_cfg, self.distro, paths ) - with self.assertRaises(NotImplementedError) as context_manager: + with pytest.raises( + NotImplementedError, + match="Subclasses of DataSource must implement _get_data", + ): datasource2.get_data() - self.assertIn( - "Subclasses of DataSource must implement _get_data", - str(context_manager.exception), - ) - def test_get_data_calls_subclass__get_data(self): + def test_get_data_calls_subclass__get_data(self, datasource): """Datasource.get_data uses the subclass' version of _get_data.""" - tmp = self.tmp_dir() - datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) - ) - self.assertTrue(datasource.get_data()) - self.assertEqual( - { - "availability_zone": "myaz", - "local-hostname": "test-subclass-hostname", - "region": "myregion", - }, - datasource.metadata, - ) - self.assertEqual("userdata_raw", datasource.userdata_raw) - self.assertEqual("vendordata_raw", datasource.vendordata_raw) - - def test_get_hostname_strips_local_hostname_without_domain(self): + assert datasource.get_data() is True + assert { + "availability_zone": "myaz", + "local-hostname": "test-subclass-hostname", + "region": "myregion", + } == datasource.metadata + assert "userdata_raw" == datasource.userdata_raw + assert "vendordata_raw" == datasource.vendordata_raw + + def test_get_hostname_strips_local_hostname_without_domain( + self, datasource + ): """Datasource.get_hostname strips metadata local-hostname of domain.""" - tmp = self.tmp_dir() - datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) - ) - self.assertTrue(datasource.get_data()) - self.assertEqual( - "test-subclass-hostname", datasource.metadata["local-hostname"] - ) - self.assertEqual( - "test-subclass-hostname", datasource.get_hostname().hostname + assert datasource.get_data() + assert ( + "test-subclass-hostname" == datasource.metadata["local-hostname"] ) + assert "test-subclass-hostname" == datasource.get_hostname().hostname datasource.metadata["local-hostname"] = "hostname.my.domain.com" - self.assertEqual("hostname", datasource.get_hostname().hostname) + assert "hostname" == datasource.get_hostname().hostname - def test_get_hostname_with_fqdn_returns_local_hostname_with_domain(self): + def test_get_hostname_with_fqdn_returns_local_hostname_with_domain( + self, datasource + ): """Datasource.get_hostname with fqdn set gets qualified hostname.""" - tmp = self.tmp_dir() - datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) - ) - self.assertTrue(datasource.get_data()) + assert datasource.get_data() datasource.metadata["local-hostname"] = "hostname.my.domain.com" - self.assertEqual( - "hostname.my.domain.com", - datasource.get_hostname(fqdn=True).hostname, + assert ( + "hostname.my.domain.com" + == datasource.get_hostname(fqdn=True).hostname ) - def test_get_hostname_without_metadata_uses_system_hostname(self): + def test_get_hostname_without_metadata_uses_system_hostname( + self, datasource + ): """Datasource.gethostname runs util.get_hostname when no metadata.""" - tmp = self.tmp_dir() - datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) - ) - self.assertEqual({}, datasource.metadata) + assert {} == datasource.metadata mock_fqdn = "cloudinit.sources.util.get_fqdn_from_hosts" with mock.patch("cloudinit.sources.util.get_hostname") as m_gethost: with mock.patch(mock_fqdn) as m_fqdn: m_gethost.return_value = "systemhostname.domain.com" m_fqdn.return_value = None # No maching fqdn in /etc/hosts - self.assertEqual( - "systemhostname", datasource.get_hostname().hostname - ) - self.assertEqual( - "systemhostname.domain.com", - datasource.get_hostname(fqdn=True).hostname, + assert "systemhostname" == datasource.get_hostname().hostname + assert ( + "systemhostname.domain.com" + == datasource.get_hostname(fqdn=True).hostname ) - def test_get_hostname_without_metadata_returns_none(self): + def test_get_hostname_without_metadata_returns_none(self, datasource): """Datasource.gethostname returns None when metadata_only and no MD.""" - tmp = self.tmp_dir() - datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) - ) - self.assertEqual({}, datasource.metadata) + assert {} == datasource.metadata mock_fqdn = "cloudinit.sources.util.get_fqdn_from_hosts" with mock.patch("cloudinit.sources.util.get_hostname") as m_gethost: with mock.patch(mock_fqdn) as m_fqdn: - self.assertIsNone( + assert ( datasource.get_hostname(metadata_only=True).hostname + is None ) - self.assertIsNone( + assert ( datasource.get_hostname( fqdn=True, metadata_only=True ).hostname + is None ) - self.assertEqual([], m_gethost.call_args_list) - self.assertEqual([], m_fqdn.call_args_list) + assert [] == m_gethost.call_args_list + assert [] == m_fqdn.call_args_list - def test_get_hostname_without_metadata_prefers_etc_hosts(self): + def test_get_hostname_without_metadata_prefers_etc_hosts(self, datasource): """Datasource.gethostname prefers /etc/hosts to util.get_hostname.""" - tmp = self.tmp_dir() - datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) - ) - self.assertEqual({}, datasource.metadata) + assert {} == datasource.metadata mock_fqdn = "cloudinit.sources.util.get_fqdn_from_hosts" with mock.patch("cloudinit.sources.util.get_hostname") as m_gethost: with mock.patch(mock_fqdn) as m_fqdn: m_gethost.return_value = "systemhostname.domain.com" m_fqdn.return_value = "fqdnhostname.domain.com" - self.assertEqual( - "fqdnhostname", datasource.get_hostname().hostname - ) - self.assertEqual( - "fqdnhostname.domain.com", - datasource.get_hostname(fqdn=True).hostname, + assert "fqdnhostname" == datasource.get_hostname().hostname + assert ( + "fqdnhostname.domain.com" + == datasource.get_hostname(fqdn=True).hostname ) - def test_get_data_does_not_write_instance_data_on_failure(self): + def test_get_data_does_not_write_instance_data_on_failure(self, paths): """get_data does not write INSTANCE_JSON_FILE on get_data False.""" - tmp = self.tmp_dir() - paths = Paths({"run_dir": tmp}) datasource = DataSourceTestSubclassNet( self.sys_cfg, self.distro, paths, get_data_retval=False, ) - self.assertFalse(datasource.get_data()) + assert datasource.get_data() is False json_file = paths.get_runpath("instance_data") - self.assertFalse( - os.path.exists(json_file), f"Found unexpected file {json_file}" - ) + assert not os.path.exists( + json_file + ), f"Found unexpected file {json_file}" - def test_get_data_writes_json_instance_data_on_success(self): + def test_get_data_writes_json_instance_data_on_success( + self, datasource, paths + ): """get_data writes INSTANCE_JSON_FILE to run_dir as world readable.""" - tmp = self.tmp_dir() - datasource = DataSourceTestSubclassNet( - self.sys_cfg, self.distro, Paths({"run_dir": tmp}) - ) sys_info = { "python": "3.7", "platform": ( @@ -396,7 +363,7 @@ def test_get_data_writes_json_instance_data_on_success(self): return_value="canonical_cloud_id", ): datasource.get_data() - json_file = Paths({"run_dir": tmp}).get_runpath("instance_data") + json_file = paths.get_runpath("instance_data") content = util.load_text_file(json_file) expected = { "base64_encoded_keys": [], @@ -439,18 +406,17 @@ def test_get_data_writes_json_instance_data_on_success(self): }, }, } - self.assertEqual(expected, util.load_json(content)) + assert expected == util.load_json(content) file_stat = os.stat(json_file) - self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode)) - self.assertEqual(expected, util.load_json(content)) + assert 0o644 == stat.S_IMODE(file_stat.st_mode) + assert expected == util.load_json(content) - def test_get_data_writes_redacted_public_json_instance_data(self): + def test_get_data_writes_redacted_public_json_instance_data(self, paths): """get_data writes redacted content to public INSTANCE_JSON_FILE.""" - tmp = self.tmp_dir() datasource = DataSourceTestSubclassNet( self.sys_cfg, self.distro, - Paths({"run_dir": tmp}), + paths, custom_metadata={ "availability_zone": "myaz", "local-hostname": "test-subclass-hostname", @@ -469,7 +435,7 @@ def test_get_data_writes_redacted_public_json_instance_data(self): "VENDOR-DAta": "HIDE ME TOO", }, ) - self.assertCountEqual( + assert_count_equal( ( "combined_cloud_config", "merged_cfg", @@ -501,7 +467,7 @@ def test_get_data_writes_redacted_public_json_instance_data(self): } with mock.patch("cloudinit.util.system_info", return_value=sys_info): datasource.get_data() - json_file = Paths({"run_dir": tmp}).get_runpath("instance_data") + json_file = paths.get_runpath("instance_data") redacted = util.load_json(util.load_text_file(json_file)) expected = { "base64_encoded_keys": [], @@ -555,19 +521,18 @@ def test_get_data_writes_redacted_public_json_instance_data(self): }, }, } - self.assertEqual(expected, redacted) + assert expected == redacted file_stat = os.stat(json_file) - self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode)) + assert 0o644 == stat.S_IMODE(file_stat.st_mode) - def test_get_data_writes_json_instance_data_sensitive(self): + def test_get_data_writes_json_instance_data_sensitive(self, paths): """ get_data writes unmodified data to sensitive file as root-readonly. """ - tmp = self.tmp_dir() datasource = DataSourceTestSubclassNet( self.sys_cfg, self.distro, - Paths({"run_dir": tmp}), + paths, custom_metadata={ "availability_zone": "myaz", "local-hostname": "test-subclass-hostname", @@ -596,7 +561,7 @@ def test_get_data_writes_json_instance_data_sensitive(self): "dist": ["ubuntu", "20.04", "focal"], } - self.assertCountEqual( + assert_count_equal( ( "combined_cloud_config", "merged_cfg", @@ -617,9 +582,7 @@ def test_get_data_writes_json_instance_data_sensitive(self): return_value="canonical-cloud-id", ): datasource.get_data() - sensitive_json_file = Paths({"run_dir": tmp}).get_runpath( - "instance_data_sensitive" - ) + sensitive_json_file = paths.get_runpath("instance_data_sensitive") content = util.load_text_file(sensitive_json_file) expected = { "base64_encoded_keys": [], @@ -684,20 +647,18 @@ def test_get_data_writes_json_instance_data_sensitive(self): }, }, } - self.assertCountEqual(expected, util.load_json(content)) + assert_count_equal(expected, util.load_json(content)) file_stat = os.stat(sensitive_json_file) - self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode)) - self.assertEqual(expected, util.load_json(content)) + assert 0o600 == stat.S_IMODE(file_stat.st_mode) + assert expected == util.load_json(content) - def test_get_data_handles_redacted_unserializable_content(self): + def test_get_data_handles_redacted_unserializable_content(self, paths): """get_data warns unserializable content in INSTANCE_JSON_FILE.""" - tmp = self.tmp_dir() - paths = Paths({"run_dir": tmp}) datasource = DataSourceTestSubclassNet( self.sys_cfg, self.distro, paths, - custom_metadata={"key1": "val1", "key2": {"key2.1": self.paths}}, + custom_metadata={"key1": "val1", "key2": {"key2.1": paths}}, ) datasource.get_data() json_file = paths.get_runpath("instance_data") @@ -712,14 +673,13 @@ def test_get_data_handles_redacted_unserializable_content(self): }, } instance_json = util.load_json(content) - self.assertEqual(expected_metadata, instance_json["ds"]["meta_data"]) + assert expected_metadata == instance_json["ds"]["meta_data"] - def test_persist_instance_data_writes_ec2_metadata_when_set(self): + def test_persist_instance_data_writes_ec2_metadata_when_set( + self, datasource, paths + ): """When ec2_metadata class attribute is set, persist to json.""" - tmp = self.tmp_dir() - cloud_dir = os.path.join(tmp, "cloud") - util.ensure_dir(cloud_dir) - paths = Paths({"run_dir": tmp, "cloud_dir": cloud_dir}) + util.ensure_dir(paths.cloud_dir) datasource = DataSourceTestSubclassNet( self.sys_cfg, self.distro, @@ -729,17 +689,17 @@ def test_persist_instance_data_writes_ec2_metadata_when_set(self): datasource.get_data() json_file = paths.get_runpath("instance_data") instance_data = util.load_json(util.load_text_file(json_file)) - self.assertNotIn("ec2_metadata", instance_data["ds"]) + assert "ec2_metadata" not in instance_data["ds"] datasource.ec2_metadata = {"ec2stuff": "is good"} datasource.persist_instance_data() instance_data = util.load_json(util.load_text_file(json_file)) - self.assertEqual( - {"ec2stuff": "is good"}, instance_data["ds"]["ec2_metadata"] - ) + assert {"ec2stuff": "is good"} == instance_data["ds"]["ec2_metadata"] - def test_persist_instance_data_writes_canonical_cloud_id_and_symlink(self): + def test_persist_instance_data_writes_canonical_cloud_id_and_symlink( + self, tmp_path + ): """canonical-cloud-id class attribute is set, persist to json.""" - tmp = self.tmp_dir() + tmp = str(tmp_path) cloud_dir = os.path.join(tmp, "cloud") util.ensure_dir(cloud_dir) datasource = DataSourceTestSubclassNet( @@ -751,35 +711,39 @@ def test_persist_instance_data_writes_canonical_cloud_id_and_symlink(self): cloud_id_file = os.path.join(tmp, "cloud-id-my-cloud") cloud_id2_file = os.path.join(tmp, "cloud-id-my-cloud2") for filename in (cloud_id_file, cloud_id_link, cloud_id2_file): - self.assertFalse( - os.path.exists(filename), "Unexpected link found {filename}" - ) + assert not os.path.exists( + filename + ), f"Unexpected link found {filename}" with mock.patch( "cloudinit.sources.canonical_cloud_id", return_value="my-cloud" ): datasource.get_data() - self.assertEqual("my-cloud\n", util.load_text_file(cloud_id_link)) + assert "my-cloud\n" == util.load_text_file(cloud_id_link) # A symlink with the generic /run/cloud-init/cloud-id # link is present - self.assertTrue(util.is_link(cloud_id_link)) + assert util.is_link(cloud_id_link) datasource.persist_instance_data() # cloud-id not deleted: no cloud-id change - self.assertTrue(os.path.exists(cloud_id_file)) + assert os.path.exists(cloud_id_file) # When cloud-id changes, symlink and content change with mock.patch( "cloudinit.sources.canonical_cloud_id", return_value="my-cloud2" ): datasource.persist_instance_data() - self.assertEqual("my-cloud2\n", util.load_text_file(cloud_id2_file)) + assert "my-cloud2\n" == util.load_text_file(cloud_id2_file) # Previous cloud-id- file removed - self.assertFalse(os.path.exists(cloud_id_file)) + assert not os.path.exists( + cloud_id_file + ), f"Unexpected {filename} not removed" # Generic link persisted which contains canonical-cloud-id as content - self.assertTrue(util.is_link(cloud_id_link)) - self.assertEqual("my-cloud2\n", util.load_text_file(cloud_id_link)) + assert util.is_link(cloud_id_link) + assert "my-cloud2\n" == util.load_text_file(cloud_id_link) - def test_persist_instance_data_writes_network_json_when_set(self): + def test_persist_instance_data_writes_network_json_when_set( + self, tmp_path + ): """When network_data.json class attribute is set, persist to json.""" - tmp = self.tmp_dir() + tmp = str(tmp_path) cloud_dir = os.path.join(tmp, "cloud") util.ensure_dir(cloud_dir) paths = Paths({"run_dir": tmp, "cloud_dir": cloud_dir}) @@ -791,17 +755,17 @@ def test_persist_instance_data_writes_network_json_when_set(self): datasource.get_data() json_file = paths.get_runpath("instance_data") instance_data = util.load_json(util.load_text_file(json_file)) - self.assertNotIn("network_json", instance_data["ds"]) + assert "network_json" not in instance_data["ds"] datasource.network_json = {"network_json": "is good"} datasource.persist_instance_data() instance_data = util.load_json(util.load_text_file(json_file)) - self.assertEqual( - {"network_json": "is good"}, instance_data["ds"]["network_json"] - ) + assert {"network_json": "is good"} == instance_data["ds"][ + "network_json" + ] - def test_persist_instance_serializes_datasource_pickle(self): + def test_persist_instance_serializes_datasource_pickle(self, tmp_path): """obj.pkl is written when instance link present and write_cache.""" - tmp = self.tmp_dir() + tmp = str(tmp_path) cloud_dir = os.path.join(tmp, "cloud") util.ensure_dir(cloud_dir) datasource = DataSourceTestSubclassNet( @@ -810,26 +774,26 @@ def test_persist_instance_serializes_datasource_pickle(self): Paths({"run_dir": tmp, "cloud_dir": cloud_dir}), ) pkl_cache_file = os.path.join(cloud_dir, "instance/obj.pkl") - self.assertFalse(os.path.exists(pkl_cache_file)) + assert not os.path.exists(pkl_cache_file) datasource.network_json = {"network_json": "is good"} # No /var/lib/cloud/instance symlink datasource.persist_instance_data(write_cache=True) - self.assertFalse(os.path.exists(pkl_cache_file)) + assert not os.path.exists(pkl_cache_file) # Symlink /var/lib/cloud/instance but write_cache=False util.sym_link(cloud_dir, os.path.join(cloud_dir, "instance")) datasource.persist_instance_data(write_cache=False) - self.assertFalse(os.path.exists(pkl_cache_file)) + assert not os.path.exists(pkl_cache_file) # Symlink /var/lib/cloud/instance and write_cache=True datasource.persist_instance_data(write_cache=True) - self.assertTrue(os.path.exists(pkl_cache_file)) + assert os.path.exists(pkl_cache_file) ds = pkl_load(pkl_cache_file) - self.assertEqual(datasource.network_json, ds.network_json) + assert datasource.network_json == ds.network_json - def test_get_data_base64encodes_unserializable_bytes(self): + def test_get_data_base64encodes_unserializable_bytes(self, tmp_path): """On py3, get_data base64encodes any unserializable content.""" - tmp = self.tmp_dir() + tmp = str(tmp_path) paths = Paths({"run_dir": tmp}) datasource = DataSourceTestSubclassNet( self.sys_cfg, @@ -837,17 +801,16 @@ def test_get_data_base64encodes_unserializable_bytes(self): paths, custom_metadata={"key1": "val1", "key2": {"key2.1": b"\x123"}}, ) - self.assertTrue(datasource.get_data()) + assert datasource.get_data() is True json_file = paths.get_runpath("instance_data") content = util.load_text_file(json_file) instance_json = util.load_json(content) - self.assertCountEqual( + assert_count_equal( ["ds/meta_data/key2/key2.1"], instance_json["base64_encoded_keys"] ) - self.assertEqual( - {"key1": "val1", "key2": {"key2.1": "EjM="}}, - instance_json["ds"]["meta_data"], - ) + assert {"key1": "val1", "key2": {"key2.1": "EjM="}} == instance_json[ + "ds" + ]["meta_data"] def test_get_hostname_subclass_support(self): """Validate get_hostname signature on all subclasses of DataSource.""" @@ -863,17 +826,15 @@ def test_get_hostname_subclass_support(self): for child in DataSource.__subclasses__(): if "Test" in child.dsname: continue - self.assertEqual( - base_args, - inspect.getfullargspec(child.get_hostname), - "%s does not implement DataSource.get_hostname params" % child, + assert base_args == inspect.getfullargspec(child.get_hostname), ( + "%s does not implement DataSource.get_hostname params" % child ) for grandchild in child.__subclasses__(): - self.assertEqual( - base_args, - inspect.getfullargspec(grandchild.get_hostname), + assert base_args == inspect.getfullargspec( + grandchild.get_hostname + ), ( "%s does not implement DataSource.get_hostname params" - % grandchild, + % grandchild ) def test_clear_cached_attrs_resets_cached_attr_class_attributes(self): @@ -886,7 +847,7 @@ def test_clear_cached_attrs_resets_cached_attr_class_attributes(self): self.datasource._dirty_cache = True self.datasource.clear_cached_attrs() for attr, value in self.datasource.cached_attr_defaults: - self.assertEqual(value, getattr(self.datasource, attr)) + assert value == getattr(self.datasource, attr) def test_clear_cached_attrs_noops_on_clean_cache(self): """Class attributes listed in cached_attr_defaults are reset.""" @@ -899,14 +860,14 @@ def test_clear_cached_attrs_noops_on_clean_cache(self): self.datasource.clear_cached_attrs() count = 0 for attr, _ in self.datasource.cached_attr_defaults: - self.assertEqual(count, getattr(self.datasource, attr)) + assert count == getattr(self.datasource, attr) count += 1 def test_clear_cached_attrs_skips_non_attr_class_attributes(self): """Skip any cached_attr_defaults which aren't class attributes.""" self.datasource._dirty_cache = True self.datasource.clear_cached_attrs(attr_defaults=(("some", "value"),)) - self.assertFalse(hasattr(self.datasource, "some")) + assert not hasattr(self.datasource, "some") def test_clear_cached_attrs_of_custom_attrs(self): """Custom attr_values can be passed to clear_cached_attrs.""" @@ -917,8 +878,8 @@ def test_clear_cached_attrs_of_custom_attrs(self): self.datasource.clear_cached_attrs( attr_defaults=(("myattr", "updated"),) ) - self.assertEqual("himom", getattr(self.datasource, cached_attr_name)) - self.assertEqual("updated", self.datasource.myattr) + assert "himom" == getattr(self.datasource, cached_attr_name) + assert "updated" == self.datasource.myattr @mock.patch.dict( DataSource.default_update_events, @@ -930,25 +891,24 @@ def test_clear_cached_attrs_of_custom_attrs(self): ) def test_update_metadata_only_acts_on_supported_update_events(self): """update_metadata_if_supported wont get_data on unsupported events.""" - self.assertEqual( - {EventScope.NETWORK: set([EventType.BOOT_NEW_INSTANCE])}, - self.datasource.default_update_events, - ) + assert { + EventScope.NETWORK: set([EventType.BOOT_NEW_INSTANCE]) + } == self.datasource.default_update_events fake_get_data = mock.Mock() self.datasource.get_data = fake_get_data - self.assertFalse( - self.datasource.update_metadata_if_supported( - source_event_types=[EventType.BOOT] - ) + assert not self.datasource.update_metadata_if_supported( + source_event_types=[EventType.BOOT] ) - self.assertEqual([], fake_get_data.call_args_list) + assert [] == fake_get_data.call_args_list @mock.patch.dict( DataSource.supported_update_events, {EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}}, ) - def test_update_metadata_returns_true_on_supported_update_event(self): + def test_update_metadata_returns_true_on_supported_update_event( + self, caplog + ): """update_metadata_if_supported returns get_data on supported events""" def fake_get_data(): @@ -957,34 +917,29 @@ def fake_get_data(): self.datasource.get_data = fake_get_data self.datasource._network_config = "something" self.datasource._dirty_cache = True - self.assertTrue( - self.datasource.update_metadata_if_supported( - source_event_types=[ - EventType.BOOT, - EventType.BOOT_NEW_INSTANCE, - ] - ) - ) - self.assertEqual(UNSET, self.datasource._network_config) - - self.assertIn( - "DEBUG: Update datasource metadata and network config due to" + assert self.datasource.update_metadata_if_supported( + source_event_types=[ + EventType.BOOT, + EventType.BOOT_NEW_INSTANCE, + ] + ) + assert UNSET == self.datasource._network_config + + assert ( + mock.ANY, + logging.DEBUG, + "Update datasource metadata and network config due to" " events: boot-new-instance", - self.logs.getvalue(), - ) + ) in caplog.record_tuples class TestRedactSensitiveData(CiTestCase): def test_redact_sensitive_data_noop_when_no_sensitive_keys_present(self): """When sensitive_keys is absent or empty from metadata do nothing.""" md = {"my": "data"} - self.assertEqual( - md, redact_sensitive_keys(md, redact_value="redacted") - ) + assert md == redact_sensitive_keys(md, redact_value="redacted") md["sensitive_keys"] = [] - self.assertEqual( - md, redact_sensitive_keys(md, redact_value="redacted") - ) + assert md == redact_sensitive_keys(md, redact_value="redacted") def test_redact_sensitive_data_redacts_exact_match_name(self): """Only exact matched sensitive_keys are redacted from metadata.""" @@ -994,9 +949,7 @@ def test_redact_sensitive_data_redacts_exact_match_name(self): } secure_md = copy.deepcopy(md) secure_md["md"]["secure"] = "redacted" - self.assertEqual( - secure_md, redact_sensitive_keys(md, redact_value="redacted") - ) + assert secure_md == redact_sensitive_keys(md, redact_value="redacted") def test_redact_sensitive_data_does_redacts_with_default_string(self): """When redact_value is absent, REDACT_SENSITIVE_VALUE is used.""" @@ -1006,89 +959,59 @@ def test_redact_sensitive_data_does_redacts_with_default_string(self): } secure_md = copy.deepcopy(md) secure_md["md"]["secure"] = "redacted for non-root user" - self.assertEqual(secure_md, redact_sensitive_keys(md)) + assert secure_md == redact_sensitive_keys(md) class TestCanonicalCloudID(CiTestCase): def test_cloud_id_returns_platform_on_unknowns(self): """When region and cloud_name are unknown, return platform.""" - self.assertEqual( - "platform", - canonical_cloud_id( - cloud_name=METADATA_UNKNOWN, - region=METADATA_UNKNOWN, - platform="platform", - ), + assert "platform" == canonical_cloud_id( + cloud_name=METADATA_UNKNOWN, + region=METADATA_UNKNOWN, + platform="platform", ) def test_cloud_id_returns_platform_on_none(self): """When region and cloud_name are unknown, return platform.""" - self.assertEqual( - "platform", - canonical_cloud_id( - cloud_name=None, region=None, platform="platform" - ), + assert "platform" == canonical_cloud_id( + cloud_name=None, region=None, platform="platform" ) def test_cloud_id_returns_cloud_name_on_unknown_region(self): """When region is unknown, return cloud_name.""" for region in (None, METADATA_UNKNOWN): - self.assertEqual( - "cloudname", - canonical_cloud_id( - cloud_name="cloudname", region=region, platform="platform" - ), + assert "cloudname" == canonical_cloud_id( + cloud_name="cloudname", region=region, platform="platform" ) def test_cloud_id_returns_platform_on_unknown_cloud_name(self): """When region is set but cloud_name is unknown return cloud_name.""" - self.assertEqual( - "platform", - canonical_cloud_id( - cloud_name=METADATA_UNKNOWN, - region="region", - platform="platform", - ), + assert "platform" == canonical_cloud_id( + cloud_name=METADATA_UNKNOWN, + region="region", + platform="platform", ) def test_cloud_id_aws_based_on_region_and_cloud_name(self): """When cloud_name is aws, return proper cloud-id based on region.""" - self.assertEqual( - "aws-china", - canonical_cloud_id( - cloud_name="aws", region="cn-north-1", platform="platform" - ), + assert "aws-china" == canonical_cloud_id( + cloud_name="aws", region="cn-north-1", platform="platform" ) - self.assertEqual( - "aws", - canonical_cloud_id( - cloud_name="aws", region="us-east-1", platform="platform" - ), + assert "aws" == canonical_cloud_id( + cloud_name="aws", region="us-east-1", platform="platform" ) - self.assertEqual( - "aws-gov", - canonical_cloud_id( - cloud_name="aws", region="us-gov-1", platform="platform" - ), + assert "aws-gov" == canonical_cloud_id( + cloud_name="aws", region="us-gov-1", platform="platform" ) - self.assertEqual( # Overrideen non-aws cloud_name is returned - "!aws", - canonical_cloud_id( - cloud_name="!aws", region="us-gov-1", platform="platform" - ), + assert "!aws" == canonical_cloud_id( + cloud_name="!aws", region="us-gov-1", platform="platform" ) def test_cloud_id_azure_based_on_region_and_cloud_name(self): """Report cloud-id when cloud_name is azure and region is in china.""" - self.assertEqual( - "azure-china", - canonical_cloud_id( - cloud_name="azure", region="chinaeast", platform="platform" - ), + assert "azure-china" == canonical_cloud_id( + cloud_name="azure", region="chinaeast", platform="platform" ) - self.assertEqual( - "azure", - canonical_cloud_id( - cloud_name="azure", region="!chinaeast", platform="platform" - ), + assert "azure" == canonical_cloud_id( + cloud_name="azure", region="!chinaeast", platform="platform" ) diff --git a/tests/unittests/sources/test_lxd.py b/tests/unittests/sources/test_lxd.py index 9d3ae417..f1996e2b 100644 --- a/tests/unittests/sources/test_lxd.py +++ b/tests/unittests/sources/test_lxd.py @@ -192,7 +192,9 @@ def mocks(self, mocker): def test_provided_network_config(self, lxd_ds, mocker): def _get_data(self): self._crawled_metadata = copy.deepcopy(DEVICES) - self._crawled_metadata["network-config"] = "hi" + self._crawled_metadata["network-config"] = { + "test-key": {"test-inner-key": "hi"} + } mocker.patch.object( lxd.DataSourceLXD, @@ -200,7 +202,7 @@ def _get_data(self): autospec=True, side_effect=_get_data, ) - assert lxd_ds.network_config == "hi" + assert lxd_ds.network_config == {"test-key": {"test-inner-key": "hi"}} @pytest.mark.parametrize( "devices_to_remove,expected_config", diff --git a/tests/unittests/sources/test_maas.py b/tests/unittests/sources/test_maas.py index 5d31c916..980307f5 100644 --- a/tests/unittests/sources/test_maas.py +++ b/tests/unittests/sources/test_maas.py @@ -9,7 +9,7 @@ from cloudinit import helpers, settings, url_helper from cloudinit.sources import DataSourceMAAS -from tests.unittests.helpers import get_mock_paths, populate_dir +from tests.unittests.helpers import populate_dir from tests.unittests.util import MockDistro @@ -224,7 +224,7 @@ def tests_wb_local_stage_detects_datasource_on_initramfs_network( assert expected == ds.get_data() @responses.activate - def test_get_data_with_retry(self, mocker, tmp_path, caplog): + def test_get_data_with_retry(self, mocker, paths, caplog): """Ensure we can get data from IMDS even if some attempts fail.""" mocker.patch("time.sleep") metadata_url = "http://169.254.169.254/MAAS/metadata" @@ -260,9 +260,7 @@ def test_get_data_with_retry(self, mocker, tmp_path, caplog): ) cfg = {"datasource": {"MAAS": {"metadata_url": metadata_url}}} - ds = DataSourceMAAS.DataSourceMAAS( - cfg, MockDistro(), get_mock_paths(tmp_path)({}) - ) + ds = DataSourceMAAS.DataSourceMAAS(cfg, MockDistro(), paths) assert ds.get_data() assert ds.metadata["instance-id"] == "i-123" assert ds.metadata["local-hostname"] == "myhostname" diff --git a/tests/unittests/sources/test_openstack.py b/tests/unittests/sources/test_openstack.py index 669148d8..2d2e8a71 100644 --- a/tests/unittests/sources/test_openstack.py +++ b/tests/unittests/sources/test_openstack.py @@ -14,7 +14,6 @@ import responses from cloudinit import helpers, settings, util -from cloudinit.distros import Distro from cloudinit.sources import UNSET, BrokenMetadata from cloudinit.sources import DataSourceOpenStack as ds from cloudinit.sources import convert_vendordata @@ -146,7 +145,7 @@ def _read_metadata_service(): return ds.read_metadata_service(BASE_URL, retries=0, timeout=0.1) -class TestOpenStackDataSource(test_helpers.ResponsesTestCase): +class TestOpenStackDataSource(test_helpers.CiTestCase): with_logs = True VERSION = "latest" @@ -155,13 +154,14 @@ def setUp(self): super(TestOpenStackDataSource, self).setUp() self.tmp = self.tmp_dir() + @responses.activate def test_successful(self): _register_uris( self.VERSION, EC2_FILES, EC2_META, OS_FILES, - responses_mock=self.responses, + responses_mock=responses, ) f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get("vendordata")) @@ -186,9 +186,10 @@ def test_successful(self): "b0fa911b-69d4-4476-bbe2-1c92bff6535c", metadata.get("instance-id") ) + @responses.activate def test_no_ec2(self): _register_uris( - self.VERSION, {}, {}, OS_FILES, responses_mock=self.responses + self.VERSION, {}, {}, OS_FILES, responses_mock=responses ) f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get("vendordata")) @@ -199,16 +200,18 @@ def test_no_ec2(self): self.assertEqual({}, f.get("ec2-metadata")) self.assertEqual(2, f.get("version")) + @responses.activate def test_bad_metadata(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith("meta_data.json"): os_files.pop(k, None) _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) self.assertRaises(openstack.NonReadable, _read_metadata_service) + @responses.activate def test_bad_uuid(self): os_files = copy.deepcopy(OS_FILES) os_meta = copy.deepcopy(OSTACK_META) @@ -217,17 +220,18 @@ def test_bad_uuid(self): if k.endswith("meta_data.json"): os_files[k] = json.dumps(os_meta) _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) self.assertRaises(BrokenMetadata, _read_metadata_service) + @responses.activate def test_userdata_empty(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith("user_data"): os_files.pop(k, None) _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get("vendordata")) @@ -236,62 +240,68 @@ def test_userdata_empty(self): self.assertEqual(CONTENT_1, f["files"]["/etc/bar/bar.cfg"]) self.assertFalse(f.get("userdata")) + @responses.activate def test_vendordata_empty(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith("vendor_data.json"): os_files.pop(k, None) _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) f = _read_metadata_service() self.assertEqual(CONTENT_0, f["files"]["/etc/foo.cfg"]) self.assertEqual(CONTENT_1, f["files"]["/etc/bar/bar.cfg"]) self.assertFalse(f.get("vendordata")) + @responses.activate def test_vendordata2_empty(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith("vendor_data2.json"): os_files.pop(k, None) _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) f = _read_metadata_service() self.assertEqual(CONTENT_0, f["files"]["/etc/foo.cfg"]) self.assertEqual(CONTENT_1, f["files"]["/etc/bar/bar.cfg"]) self.assertFalse(f.get("vendordata2")) + @responses.activate def test_vendordata_invalid(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith("vendor_data.json"): os_files[k] = "{" # some invalid json _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) self.assertRaises(BrokenMetadata, _read_metadata_service) + @responses.activate def test_vendordata2_invalid(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith("vendor_data2.json"): os_files[k] = "{" # some invalid json _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) self.assertRaises(BrokenMetadata, _read_metadata_service) + @responses.activate def test_metadata_invalid(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith("meta_data.json"): os_files[k] = "{" # some invalid json _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) self.assertRaises(BrokenMetadata, _read_metadata_service) + @responses.activate @test_helpers.mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery") def test_datasource(self, m_dhcp): _register_uris( @@ -299,11 +309,12 @@ def test_datasource(self, m_dhcp): EC2_FILES, EC2_META, OS_FILES, - responses_mock=self.responses, + responses_mock=responses, ) - distro = mock.MagicMock(spec=Distro) ds_os = ds.DataSourceOpenStack( - settings.CFG_BUILTIN, distro, helpers.Paths({"run_dir": self.tmp}) + settings.CFG_BUILTIN, + test_util.MockDistro(), + helpers.Paths({"run_dir": self.tmp}), ) self.assertIsNone(ds_os.version) with mock.patch.object(ds_os, "override_ds_detect", return_value=True): @@ -319,6 +330,7 @@ def test_datasource(self, m_dhcp): self.assertIsNone(ds_os.vendordata_raw) m_dhcp.assert_not_called() + @responses.activate @test_helpers.mock.patch("cloudinit.net.ephemeral.EphemeralIPv4Network") @test_helpers.mock.patch( "cloudinit.net.ephemeral.maybe_perform_dhcp_discovery" @@ -331,7 +343,7 @@ def test_local_datasource(self, m_dhcp, m_net): EC2_FILES, EC2_META, OS_FILES, - responses_mock=self.responses, + responses_mock=responses, ) distro = mock.MagicMock() distro.get_tmp_exec_path = self.tmp_dir @@ -365,18 +377,19 @@ def test_local_datasource(self, m_dhcp, m_net): self.assertIsNone(ds_os_local.vendordata_raw) m_dhcp.assert_called_with(distro, "eth9", None) + @responses.activate def test_bad_datasource_meta(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith("meta_data.json"): os_files[k] = "{" # some invalid json _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) - distro = mock.MagicMock(spec=Distro) - distro.is_virtual = True ds_os = ds.DataSourceOpenStack( - settings.CFG_BUILTIN, distro, helpers.Paths({"run_dir": self.tmp}) + settings.CFG_BUILTIN, + test_util.MockDistro(), + helpers.Paths({"run_dir": self.tmp}), ) self.assertIsNone(ds_os.version) with test_helpers.mock.patch.object( @@ -392,18 +405,19 @@ def test_bad_datasource_meta(self): r" http://(169.254.169.254|\[fe80::a9fe:a9fe\])", ) + @responses.activate def test_no_datasource(self): os_files = copy.deepcopy(OS_FILES) for k in list(os_files.keys()): if k.endswith("meta_data.json"): os_files.pop(k) _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) - distro = mock.MagicMock(spec=Distro) - distro.is_virtual = True ds_os = ds.DataSourceOpenStack( - settings.CFG_BUILTIN, distro, helpers.Paths({"run_dir": self.tmp}) + settings.CFG_BUILTIN, + test_util.MockDistro(), + helpers.Paths({"run_dir": self.tmp}), ) ds_os.ds_cfg = { "max_wait": 0, @@ -464,6 +478,7 @@ def test_network_config_cached(self): self.assertEqual(example_cfg, ds_os.network_config) m_convert_json.assert_not_called() + @responses.activate def test_disabled_datasource(self): os_files = copy.deepcopy(OS_FILES) os_meta = copy.deepcopy(OSTACK_META) @@ -474,12 +489,12 @@ def test_disabled_datasource(self): if k.endswith("meta_data.json"): os_files[k] = json.dumps(os_meta) _register_uris( - self.VERSION, {}, {}, os_files, responses_mock=self.responses + self.VERSION, {}, {}, os_files, responses_mock=responses ) - distro = mock.MagicMock(spec=Distro) - distro.is_virtual = True ds_os = ds.DataSourceOpenStack( - settings.CFG_BUILTIN, distro, helpers.Paths({"run_dir": self.tmp}) + settings.CFG_BUILTIN, + test_util.MockDistro(), + helpers.Paths({"run_dir": self.tmp}), ) ds_os.ds_cfg = { "max_wait": 0, @@ -494,6 +509,7 @@ def test_disabled_datasource(self): self.assertFalse(found) self.assertIsNone(ds_os.version) + @responses.activate def test_wb__crawl_metadata_does_not_persist(self): """_crawl_metadata returns current metadata and does not cache.""" _register_uris( @@ -501,7 +517,7 @@ def test_wb__crawl_metadata_does_not_persist(self): EC2_FILES, EC2_META, OS_FILES, - responses_mock=self.responses, + responses_mock=responses, ) ds_os = ds.DataSourceOpenStack( settings.CFG_BUILTIN, @@ -582,10 +598,10 @@ def setUp(self): self.tmp = self.tmp_dir() def _fake_ds(self) -> ds.DataSourceOpenStack: - distro = mock.MagicMock(spec=Distro) - distro.is_virtual = True return ds.DataSourceOpenStack( - settings.CFG_BUILTIN, distro, helpers.Paths({"run_dir": self.tmp}) + settings.CFG_BUILTIN, + test_util.MockDistro(), + helpers.Paths({"run_dir": self.tmp}), ) def test_ds_detect_non_intel_x86(self, m_is_x86): @@ -790,7 +806,7 @@ def fake_dmi_read(dmi_key): m_proc_env.assert_called_with(1) -class TestMetadataReader(test_helpers.ResponsesTestCase): +class TestMetadataReader(test_helpers.CiTestCase): """Test the MetadataReader.""" burl = "http://169.254.169.254/" @@ -807,7 +823,7 @@ class TestMetadataReader(test_helpers.ResponsesTestCase): def register(self, path, body=None, status=200): content = body if not isinstance(body, str) else body.encode("utf-8") - self.responses.add( + responses.add( responses.GET, self.burl + "openstack" + path, status=status, @@ -828,6 +844,7 @@ def register_version(self, version, data): if "user_data" not in data: self.register("/%s/user_data" % version, "nodata", status=404) + @responses.activate def test__find_working_version(self): """Test a working version ignores unsupported.""" unsup = "2016-11-09" @@ -844,6 +861,7 @@ def test__find_working_version(self): openstack.MetadataReader(self.burl)._find_working_version(), ) + @responses.activate def test__find_working_version_uses_latest(self): """'latest' should be used if no supported versions.""" unsup1, unsup2 = ("2016-11-09", "2017-06-06") @@ -853,6 +871,7 @@ def test__find_working_version_uses_latest(self): openstack.MetadataReader(self.burl)._find_working_version(), ) + @responses.activate def test_read_v2_os_ocata(self): """Validate return value of read_v2 for os_ocata data.""" md = copy.deepcopy(self.md_base) diff --git a/tests/unittests/sources/test_oracle.py b/tests/unittests/sources/test_oracle.py index d773acf7..1a61510a 100644 --- a/tests/unittests/sources/test_oracle.py +++ b/tests/unittests/sources/test_oracle.py @@ -485,48 +485,6 @@ def test_imds_nic_setup_v1(self, set_primary, oracle_ds): assert "10.0.0.231/24" == secondary_cfg["subnets"][0]["address"] assert "static" == secondary_cfg["subnets"][0]["type"] - @pytest.mark.parametrize( - "set_primary", - [True, False], - ) - def test_secondary_nic_v2(self, set_primary, oracle_ds): - oracle_ds._vnics_data = json.loads(OPC_VM_SECONDARY_VNIC_RESPONSE) - oracle_ds._network_config = { - "version": 2, - "ethernets": {"primary": {"nic": {}}}, - } - with mock.patch( - f"{DS_PATH}.get_interfaces_by_mac", - return_value={ - "02:00:17:05:d1:db": "ens3", - "00:00:17:02:2b:b1": "ens4", - }, - ): - oracle_ds._add_network_config_from_opc_imds( - set_primary=set_primary - ) - - nic_cfg = oracle_ds.network_config["ethernets"] - if set_primary: - assert "ens3" in nic_cfg - primary_cfg = nic_cfg["ens3"] - - assert primary_cfg["dhcp4"] is True - assert primary_cfg["dhcp6"] is False - assert "02:00:17:05:d1:db" == primary_cfg["match"]["macaddress"] - assert 9000 == primary_cfg["mtu"] - assert "addresses" not in primary_cfg - - assert "ens4" in nic_cfg - secondary_cfg = nic_cfg["ens4"] - assert secondary_cfg["dhcp4"] is False - assert secondary_cfg["dhcp6"] is False - assert "00:00:17:02:2b:b1" == secondary_cfg["match"]["macaddress"] - assert 9000 == secondary_cfg["mtu"] - - assert 1 == len(secondary_cfg["addresses"]) - assert "10.0.0.231/24" == secondary_cfg["addresses"][0] - @pytest.mark.parametrize( "set_primary", [ @@ -578,53 +536,6 @@ def test_imds_nic_setup_v1_ipv6_only(self, set_primary, oracle_ds): ) assert "static" == secondary_cfg["subnets"][0]["type"] - @pytest.mark.parametrize( - "set_primary", - [True, False], - ) - def test_secondary_nic_v2_ipv6_only(self, set_primary, oracle_ds): - oracle_ds._vnics_data = json.loads( - OPC_VM_IPV6_ONLY_SECONDARY_VNIC_RESPONSE - ) - oracle_ds._network_config = { - "version": 2, - "ethernets": {"primary": {"nic": {}}}, - } - with mock.patch( - f"{DS_PATH}.get_interfaces_by_mac", - return_value={ - "02:00:17:0d:6b:be": "ens3", - "02:00:17:18:f6:ff": "ens4", - }, - ): - oracle_ds._add_network_config_from_opc_imds( - set_primary=set_primary - ) - - nic_cfg = oracle_ds.network_config["ethernets"] - if set_primary: - assert "ens3" in nic_cfg - primary_cfg = nic_cfg["ens3"] - - assert primary_cfg["dhcp4"] is False - assert primary_cfg["dhcp6"] is True - assert "02:00:17:0d:6b:be" == primary_cfg["match"]["macaddress"] - assert 9000 == primary_cfg["mtu"] - assert "addresses" not in primary_cfg - - assert "ens4" in nic_cfg - secondary_cfg = nic_cfg["ens4"] - assert secondary_cfg["dhcp4"] is False - assert secondary_cfg["dhcp6"] is False - assert "02:00:17:18:f6:ff" == secondary_cfg["match"]["macaddress"] - assert 9000 == secondary_cfg["mtu"] - - assert 1 == len(secondary_cfg["addresses"]) - assert ( - "2603:c020:400d:5d7e:aacc:8e5f:3b1b:3a4a/128" - == secondary_cfg["addresses"][0] - ) - @pytest.mark.parametrize("error_add_network", [None, Exception]) @pytest.mark.parametrize( "configure_secondary_nics", @@ -675,7 +586,7 @@ def test_network_config_log_errors( ) == caplog.record_tuples[-1][1:] assert ( - logging.WARNING, + logging.DEBUG, "Could not obtain network configuration from initramfs." " Falling back to IMDS.", ) == caplog.record_tuples[log_initramfs_index][1:] diff --git a/tests/unittests/sources/test_scaleway.py b/tests/unittests/sources/test_scaleway.py index eefa6275..bb1f2136 100644 --- a/tests/unittests/sources/test_scaleway.py +++ b/tests/unittests/sources/test_scaleway.py @@ -2,7 +2,6 @@ import json import socket -import sys from urllib.parse import SplitResult, urlsplit import requests @@ -12,7 +11,11 @@ from cloudinit import helpers, settings, sources from cloudinit.distros import ubuntu from cloudinit.sources import DataSourceScaleway -from tests.unittests.helpers import CiTestCase, ResponsesTestCase, mock +from tests.unittests.helpers import ( + CiTestCase, + mock, + responses_assert_call_count, +) class DataResponses: @@ -188,7 +191,7 @@ def _fix_mocking_url(url: str) -> str: ).geturl() -class TestDataSourceScaleway(ResponsesTestCase): +class TestDataSourceScaleway(CiTestCase): def setUp(self): tmp = self.tmp_dir() distro = ubuntu.Distro("", {}, {}) @@ -204,7 +207,7 @@ def setUp(self): # The trailing / at the end of the URL is needed to # workaround a bug in responses 3.6 which does not match # the URL otherwise. - self.responses.add_callback( + responses.add_callback( responses.GET, f"{url}/", callback=MetadataResponses.get_ok, @@ -223,18 +226,21 @@ def setUp(self): return_value="scalewaynic0", ) + @responses.activate def test_set_metadata_url_ipv4_ok(self): self.datasource._set_metadata_url([self.base_urls[0]]) self.assertTrue(self.base_urls[0] in self.datasource.metadata_url) + @responses.activate def test_set_metadata_url_ipv6_ok(self): self.datasource._set_metadata_url([self.base_urls[1]]) self.assertTrue(self.base_urls[1] in self.datasource.metadata_url) + @responses.activate @mock.patch( "cloudinit.sources.DataSourceScaleway.DataSourceScaleway" ".override_ds_detect" @@ -248,34 +254,34 @@ def test_ipv4_metadata_ok(self, dhcpv4, ds_detect): self.datasource._set_metadata_url([self.base_urls[0]]) - self.responses.reset() - self.responses.add_callback( + responses.reset() + responses.add_callback( responses.GET, f"{self.base_urls[0]}/", callback=MetadataResponses.get_ok, ) - self.responses.add_callback( + responses.add_callback( responses.GET, f"{self.base_urls[1]}/", callback=MetadataResponses.get_ok, ) # Use _fix_mocking_url to workaround py3.6 bug in responses - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(f"{self.base_urls[0]}/conf?format=json"), callback=MetadataResponses.get_ok, ) - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(f"{self.base_urls[1]}/conf?format=json"), callback=MetadataResponses.get_ok, ) - self.responses.add_callback( + responses.add_callback( responses.GET, f"{self.base_urls[0]}/user_data/cloud-init", callback=DataResponses.get_ok, ) - self.responses.add_callback( + responses.add_callback( responses.GET, f"{self.base_urls[0]}/vendor_data/cloud-init", callback=DataResponses.get_ok, @@ -309,6 +315,7 @@ def test_ipv4_metadata_ok(self, dhcpv4, ds_detect): self.assertIsNone(self.datasource.availability_zone) self.assertIsNone(self.datasource.region) + @responses.activate @mock.patch( "cloudinit.sources.DataSourceScaleway.DataSourceScaleway" ".override_ds_detect" @@ -324,46 +331,45 @@ def test_ipv4_metadata_timeout_ipv6_ok(self, dhcpv4, inet6, ds_detect): self.datasource._set_metadata_url([self.base_urls[0]]) - self.responses.reset() - self.responses.add_callback( + responses.reset() + responses.add_callback( responses.GET, f"{self.base_urls[0]}/", callback=ConnectTimeout, ) - self.responses.add_callback( + responses.add_callback( responses.GET, f"{self.base_urls[1]}/", callback=MetadataResponses.get_ok, ) - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(f"{self.base_urls[1]}/conf?format=json"), callback=MetadataResponses.get_ok, ) - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(f"{self.base_urls[1]}/user_data/cloud-init"), callback=DataResponses.get_ok, ) - self.responses.add_callback( + responses.add_callback( responses.GET, f"{self.base_urls[1]}/vendor_data/cloud-init", callback=DataResponses.get_ok, ) self.datasource.get_data() - if sys.version_info.minor >= 7: - self.responses.assert_call_count( - f"{self.datasource.metadata_urls[0]}", - 1, - ) - self.responses.assert_call_count( - f"{self.datasource.metadata_urls[1]}", - 1, - ) - self.responses.assert_call_count( - f"{self.datasource.metadata_urls[1]}/conf?format=json", 1 - ) + responses_assert_call_count( + f"{self.datasource.metadata_urls[0]}", + 1, + ) + responses_assert_call_count( + f"{self.datasource.metadata_urls[1]}", + 1, + ) + responses_assert_call_count( + f"{self.datasource.metadata_urls[1]}/conf?format=json", 1 + ) self.assertEqual( self.datasource.get_instance_id(), @@ -392,6 +398,7 @@ def test_ipv4_metadata_timeout_ipv6_ok(self, dhcpv4, inet6, ds_detect): self.assertIsNone(self.datasource.availability_zone) self.assertIsNone(self.datasource.region) + @responses.activate @mock.patch( "cloudinit.sources.DataSourceScaleway.DataSourceScaleway" ".override_ds_detect" @@ -409,35 +416,28 @@ def test_ipv4_ipv6_metadata_timeout(self, dhcpv4, inet6, sleep, ds_detect): self.datasource._set_metadata_url([self.base_urls[0]]) # Remove callbacks defined at class initialization - self.responses.reset() - self.responses.add_callback( + responses.reset() + responses.add_callback( responses.GET, f"{self.base_urls[0]}/", callback=ConnectTimeout, ) - self.responses.add_callback( + responses.add_callback( responses.GET, f"{self.base_urls[1]}/", callback=ConnectTimeout, ) self.datasource.max_wait = 0 ret = self.datasource.get_data() - # assert_call_count is not available in responses for py3.6 - if sys.version_info.minor >= 7: - self.responses.assert_call_count( - f"{self.datasource.metadata_urls[0]}", - 2, - ) - self.responses.assert_call_count( - f"{self.datasource.metadata_urls[1]}", - 2, - ) + responses_assert_call_count(f"{self.datasource.metadata_urls[0]}", 2) + responses_assert_call_count(f"{self.datasource.metadata_urls[1]}", 2) self.assertFalse(ret) self.assertEqual(self.datasource.metadata, {}) self.assertIsNone(self.datasource.get_userdata_raw()) self.assertIsNone(self.datasource.get_vendordata_raw()) + @responses.activate @mock.patch( "cloudinit.sources.DataSourceScaleway.DataSourceScaleway" ".override_ds_detect" @@ -454,17 +454,17 @@ def test_metadata_ipv4_404(self, dhcpv4, ds_detect): # Make user and vendor data APIs return HTTP/404, which means there is # no user / vendor data for the server. - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(self.datasource.metadata_url), callback=MetadataResponses.get_ok, ) - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(self.datasource.userdata_url), callback=DataResponses.empty, ) - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(self.datasource.vendordata_url), callback=DataResponses.empty, @@ -476,6 +476,7 @@ def test_metadata_ipv4_404(self, dhcpv4, ds_detect): self.assertIsNone(self.datasource.get_userdata_raw()) self.assertIsNone(self.datasource.get_vendordata_raw()) + @responses.activate @mock.patch("cloudinit.url_helper.time.sleep", lambda x: None) @mock.patch("cloudinit.sources.DataSourceScaleway.EphemeralDHCPv4") def test_metadata_connection_errors_legacy_ipv4_url(self, dhcpv4): @@ -490,9 +491,9 @@ def test_metadata_connection_errors_legacy_ipv4_url(self, dhcpv4): "http://169.254.42.42", ] - self.responses.reset() + responses.reset() with self.assertRaises(ConnectionError): - self.responses.add_callback( + responses.add_callback( responses.GET, f"{self.datasource.metadata_urls[0]}/", callback=ConnectionError, @@ -502,6 +503,7 @@ def test_metadata_connection_errors_legacy_ipv4_url(self, dhcpv4): self.assertIsNone(self.datasource.get_userdata_raw()) self.assertIsNone(self.datasource.get_vendordata_raw()) + @responses.activate @mock.patch( "cloudinit.sources.DataSourceScaleway.DataSourceScaleway" ".override_ds_detect" @@ -536,13 +538,13 @@ def test_metadata_connection_errors_two_urls( ] self.datasource.has_ipv4 = True - self.responses.reset() - self.responses.add_callback( + responses.reset() + responses.add_callback( responses.GET, self.datasource.metadata_urls[0], callback=ConnectionError, ) - self.responses.add_callback( + responses.add_callback( responses.GET, self.datasource.metadata_urls[1], callback=ConnectionError, @@ -551,18 +553,18 @@ def test_metadata_connection_errors_two_urls( self.datasource.get_data() # url_helper.wait_on_url tests both URL in list each time so # called twice for each URL - if sys.version_info.minor >= 7: - self.responses.assert_call_count( - f"{self.datasource.metadata_urls[0]}", - 2, - ) - self.responses.assert_call_count( - f"{self.datasource.metadata_urls[1]}", - 2, - ) + responses_assert_call_count( + f"{self.datasource.metadata_urls[0]}", + 2, + ) + responses_assert_call_count( + f"{self.datasource.metadata_urls[1]}", + 2, + ) self.assertIsNone(self.datasource.get_userdata_raw()) self.assertIsNone(self.datasource.get_vendordata_raw()) + @responses.activate @mock.patch( "cloudinit.sources.DataSourceScaleway.DataSourceScaleway" ".override_ds_detect" @@ -578,12 +580,12 @@ def test_metadata_ipv4_rate_limit(self, sleep, dhcpv4, ds_detect): self.datasource._set_metadata_url([self.base_urls[0]]) - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(self.datasource.metadata_url), callback=MetadataResponses.get_ok, ) - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(self.datasource.vendordata_url), callback=DataResponses.empty, @@ -603,7 +605,7 @@ def _callback(request): return DataResponses.rate_limited(request) return DataResponses.get_ok(request) - self.responses.add_callback( + responses.add_callback( responses.GET, _fix_mocking_url(self.datasource.userdata_url), callback=_callback, diff --git a/tests/unittests/sources/test_smartos.py b/tests/unittests/sources/test_smartos.py index fbb7ee62..e73b7c4d 100644 --- a/tests/unittests/sources/test_smartos.py +++ b/tests/unittests/sources/test_smartos.py @@ -21,13 +21,13 @@ import re import signal import stat -import unittest import uuid from binascii import crc32 +from collections import namedtuple +import pytest import serial -from cloudinit import helpers as c_helpers from cloudinit.atomic_helper import b64e from cloudinit.event import EventScope, EventType from cloudinit.sources import DataSourceSmartOS @@ -41,12 +41,7 @@ ) from cloudinit.subp import ProcessExecutionError, subp, which from cloudinit.util import write_file -from tests.unittests.helpers import ( - CiTestCase, - FilesystemMockingTestCase, - mock, - skipIf, -) +from tests.unittests.helpers import mock, skipIf DSMOS = "cloudinit.sources.DataSourceSmartOS" SDC_NICS = json.loads( @@ -404,7 +399,7 @@ def exists(self): return True def open_transport(self): - assert not self._is_open + assert self._is_open is False self._is_open = True def close_transport(self): @@ -412,188 +407,178 @@ def close_transport(self): self._is_open = False -class TestSmartOSDataSource(FilesystemMockingTestCase): - jmc_cfact = None - get_smartos_environ = None +@pytest.fixture +def legacy_user_d(tmp_path): + legacy_user_dir = str(tmp_path / "legacy_user_tmp") + os.mkdir(legacy_user_dir) + return legacy_user_dir - def setUp(self): - super(TestSmartOSDataSource, self).setUp() - - self.add_patch(DSMOS + ".get_smartos_environ", "get_smartos_environ") - self.add_patch(DSMOS + ".jmc_client_factory", "jmc_cfact") - self.legacy_user_d = self.tmp_path("legacy_user_tmp") - os.mkdir(self.legacy_user_d) - self.add_patch( - DSMOS + ".LEGACY_USER_D", - "m_legacy_user_d", - autospec=False, - new=self.legacy_user_d, - ) - self.add_patch( - DSMOS + ".identify_file", - "m_identify_file", - return_value="text/plain", - ) - def _get_ds( - self, - mockdata=None, - mode=DataSourceSmartOS.SMARTOS_ENV_KVM, - sys_cfg=None, - ds_cfg=None, - ): - self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata) - self.get_smartos_environ.return_value = mode +@pytest.fixture +def m_jmc_client_factory(mocker): + return mocker.patch( + DSMOS + ".jmc_client_factory", + return_value=PsuedoJoyentClient(MOCK_RETURNS), + ) - tmpd = self.tmp_dir() - dirs = { - "cloud_dir": self.tmp_path("cloud_dir", tmpd), - "run_dir": self.tmp_path("run_dir"), - } - for d in dirs.values(): - os.mkdir(d) - paths = c_helpers.Paths(dirs) - if sys_cfg is None: - sys_cfg = {} +@pytest.fixture +def mocks(legacy_user_d, mocker, m_jmc_client_factory): + mocker.patch( + DSMOS + ".get_smartos_environ", + return_value=DataSourceSmartOS.SMARTOS_ENV_KVM, + ) + mocker.patch( + DSMOS + ".LEGACY_USER_D", + autospec=False, + new=legacy_user_d, + ) + mocker.patch( + DSMOS + ".identify_file", + return_value="text/plain", + ) + mocker.patch("cloudinit.net.activators.subp.subp", return_value=("", "")) - if ds_cfg is not None: - sys_cfg["datasource"] = sys_cfg.get("datasource", {}) - sys_cfg["datasource"]["SmartOS"] = ds_cfg - return DataSourceSmartOS.DataSourceSmartOS( - sys_cfg, distro=None, paths=paths - ) +def _get_ds(paths, ds_cfg=None): + sys_cfg = {} + if ds_cfg is not None: + sys_cfg["datasource"] = {} + sys_cfg["datasource"]["SmartOS"] = ds_cfg + + return DataSourceSmartOS.DataSourceSmartOS( + sys_cfg, distro=None, paths=paths + ) + + +@pytest.fixture +def ds(paths): + return lambda: _get_ds(paths) + + +@pytest.mark.usefixtures("fake_filesystem", "mocks") +class TestSmartOSDataSource: + jmc_cfact = None + get_smartos_environ = None - def test_no_base64(self): + def test_no_base64(self, paths): ds_cfg = {"no_base64_decode": ["test_var1"], "all_base": True} - dsrc = self._get_ds(ds_cfg=ds_cfg) + dsrc = _get_ds(paths, ds_cfg=ds_cfg) ret = dsrc.get_data() - self.assertTrue(ret) + assert ret is True - def test_uuid(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_uuid(self, ds, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["sdc:uuid"], dsrc.metadata["instance-id"] - ) + assert ret is True + assert MOCK_RETURNS["sdc:uuid"] == dsrc.metadata["instance-id"] - def test_platform_info(self): + def test_platform_info(self, ds, m_jmc_client_factory): """All platform-related attributes are properly set.""" - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - self.assertEqual("joyent", dsrc.cloud_name) - self.assertEqual("joyent", dsrc.platform_type) - self.assertEqual("serial (/dev/ttyS1)", dsrc.subplatform) + dsrc = ds() + assert "joyent" == dsrc.cloud_name + assert "joyent" == dsrc.platform_type + assert "serial (/dev/ttyS1)" == dsrc.subplatform - def test_root_keys(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_root_keys(self, ds, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["root_authorized_keys"], dsrc.metadata["public-keys"] + assert ret is True + assert ( + MOCK_RETURNS["root_authorized_keys"] + == dsrc.metadata["public-keys"] ) - def test_hostname_b64(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_hostname_b64(self, ds, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["hostname"], dsrc.metadata["local-hostname"] - ) + assert ret is True + assert MOCK_RETURNS["hostname"] == dsrc.metadata["local-hostname"] - def test_hostname(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_hostname(self, ds, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["hostname"], dsrc.metadata["local-hostname"] - ) + assert ret is True + assert MOCK_RETURNS["hostname"] == dsrc.metadata["local-hostname"] - def test_hostname_if_no_sdc_hostname(self): + def test_hostname_if_no_sdc_hostname(self, ds, m_jmc_client_factory): my_returns = MOCK_RETURNS.copy() my_returns["sdc:hostname"] = "sdc-" + my_returns["hostname"] - dsrc = self._get_ds(mockdata=my_returns) + m_jmc_client_factory.return_value = PsuedoJoyentClient(my_returns) + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - my_returns["hostname"], dsrc.metadata["local-hostname"] - ) + assert ret is True + assert my_returns["hostname"] == dsrc.metadata["local-hostname"] - def test_sdc_hostname_if_no_hostname(self): + def test_sdc_hostname_if_no_hostname(self, ds, m_jmc_client_factory): my_returns = MOCK_RETURNS.copy() my_returns["sdc:hostname"] = "sdc-" + my_returns["hostname"] del my_returns["hostname"] - dsrc = self._get_ds(mockdata=my_returns) + m_jmc_client_factory.return_value = PsuedoJoyentClient(my_returns) + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - my_returns["sdc:hostname"], dsrc.metadata["local-hostname"] - ) + assert ret is True + assert my_returns["sdc:hostname"] == dsrc.metadata["local-hostname"] - def test_sdc_uuid_if_no_hostname_or_sdc_hostname(self): + def test_sdc_uuid_if_no_hostname_or_sdc_hostname( + self, ds, m_jmc_client_factory + ): my_returns = MOCK_RETURNS.copy() del my_returns["hostname"] - dsrc = self._get_ds(mockdata=my_returns) + m_jmc_client_factory.return_value = PsuedoJoyentClient(my_returns) + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - my_returns["sdc:uuid"], dsrc.metadata["local-hostname"] - ) + assert ret is True + assert my_returns["sdc:uuid"] == dsrc.metadata["local-hostname"] - def test_userdata(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_userdata(self, ds, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["user-data"], dsrc.metadata["legacy-user-data"] - ) - self.assertEqual( - MOCK_RETURNS["cloud-init:user-data"], dsrc.userdata_raw - ) + assert ret is True + assert MOCK_RETURNS["user-data"] == dsrc.metadata["legacy-user-data"] + assert MOCK_RETURNS["cloud-init:user-data"] == dsrc.userdata_raw - def test_sdc_nics(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_sdc_nics(self, ds, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - json.loads(MOCK_RETURNS["sdc:nics"]), dsrc.metadata["network-data"] + assert ret is True + assert ( + json.loads(MOCK_RETURNS["sdc:nics"]) + == dsrc.metadata["network-data"] ) - def test_sdc_scripts(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_sdc_scripts(self, ds, legacy_user_d, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["user-script"], dsrc.metadata["user-script"] - ) + assert ret is True + assert MOCK_RETURNS["user-script"] == dsrc.metadata["user-script"] - legacy_script_f = "%s/user-script" % self.legacy_user_d + legacy_script_f = "%s/user-script" % legacy_user_d print("legacy_script_f=%s" % legacy_script_f) - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) + assert os.path.exists(legacy_script_f) + assert os.path.islink(legacy_script_f) user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEqual(user_script_perm, "700") + assert user_script_perm == "700" - def test_scripts_shebanged(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_scripts_shebanged(self, ds, legacy_user_d, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["user-script"], dsrc.metadata["user-script"] - ) + assert ret is True + assert MOCK_RETURNS["user-script"] == dsrc.metadata["user-script"] - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) + legacy_script_f = "%s/user-script" % legacy_user_d + assert os.path.exists(legacy_script_f) + assert os.path.islink(legacy_script_f) shebang = None with open(legacy_script_f, "r") as f: shebang = f.readlines()[0].strip() - self.assertEqual(shebang, "#!/bin/bash") + assert shebang == "#!/bin/bash" user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEqual(user_script_perm, "700") + assert user_script_perm == "700" - def test_scripts_shebang_not_added(self): + def test_scripts_shebang_not_added( + self, ds, legacy_user_d, m_jmc_client_factory + ): """ Test that the SmartOS requirement that plain text scripts are executable. This test makes sure that plain texts scripts @@ -605,22 +590,21 @@ def test_scripts_shebang_not_added(self): ["#!/usr/bin/perl", 'print("hi")', ""] ) - dsrc = self._get_ds(mockdata=my_returns) + m_jmc_client_factory.return_value = PsuedoJoyentClient(my_returns) + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - my_returns["user-script"], dsrc.metadata["user-script"] - ) + assert ret is True + assert my_returns["user-script"] == dsrc.metadata["user-script"] - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) + legacy_script_f = "%s/user-script" % legacy_user_d + assert os.path.exists(legacy_script_f) + assert os.path.islink(legacy_script_f) shebang = None with open(legacy_script_f, "r") as f: shebang = f.readlines()[0].strip() - self.assertEqual(shebang, "#!/usr/bin/perl") + assert shebang == "#!/usr/bin/perl" - def test_userdata_removed(self): + def test_userdata_removed(self, ds, legacy_user_d, m_jmc_client_factory): """ User-data in the SmartOS world is supposed to be written to a file each and every boot. This tests to make sure that in the event the @@ -628,83 +612,82 @@ def test_userdata_removed(self): and there is no /var/db/user-data left. """ - user_data_f = "%s/mdata-user-data" % self.legacy_user_d + user_data_f = "%s/mdata-user-data" % legacy_user_d with open(user_data_f, "w") as f: f.write("PREVIOUS") my_returns = MOCK_RETURNS.copy() del my_returns["user-data"] - dsrc = self._get_ds(mockdata=my_returns) + m_jmc_client_factory.return_value = PsuedoJoyentClient(my_returns) + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertFalse(dsrc.metadata.get("legacy-user-data")) + assert ret is True + assert dsrc.metadata.get("legacy-user-data") is None found_new = False - for root, _dirs, files in os.walk(self.legacy_user_d): + for root, _dirs, files in os.walk(legacy_user_d): for name in files: name_f = os.path.join(root, name) permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] if re.match(r".*\/mdata-user-data$", name_f): found_new = True print(name_f) - self.assertEqual(permissions, "400") + assert permissions == "400" - self.assertFalse(found_new) + assert found_new is False - def test_vendor_data_not_default(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_vendor_data_not_default(self, ds, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["sdc:vendor-data"], dsrc.metadata["vendor-data"] - ) + assert ret is True + assert MOCK_RETURNS["sdc:vendor-data"] == dsrc.metadata["vendor-data"] - def test_default_vendor_data(self): + def test_default_vendor_data(self, ds, m_jmc_client_factory): my_returns = MOCK_RETURNS.copy() def_op_script = my_returns["sdc:vendor-data"] del my_returns["sdc:vendor-data"] - dsrc = self._get_ds(mockdata=my_returns) + m_jmc_client_factory.return_value = PsuedoJoyentClient(my_returns) + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertNotEqual(def_op_script, dsrc.metadata["vendor-data"]) + assert ret is True + assert def_op_script != dsrc.metadata["vendor-data"] # we expect default vendor-data is a boothook - self.assertTrue(dsrc.vendordata_raw.startswith("#cloud-boothook")) + assert dsrc.vendordata_raw.startswith("#cloud-boothook") - def test_disable_iptables_flag(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_disable_iptables_flag(self, ds, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["disable_iptables_flag"], - dsrc.metadata["iptables_disable"], + assert ret is True + assert ( + MOCK_RETURNS["disable_iptables_flag"] + == dsrc.metadata["iptables_disable"] ) - def test_motd_sys_info(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) + def test_motd_sys_info(self, ds, m_jmc_client_factory): + dsrc = ds() ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual( - MOCK_RETURNS["enable_motd_sys_info"], - dsrc.metadata["motd_sys_info"], + assert ret is True + assert ( + MOCK_RETURNS["enable_motd_sys_info"] + == dsrc.metadata["motd_sys_info"] ) - def test_default_ephemeral(self): + def test_default_ephemeral(self, ds): # Test to make sure that the builtin config has the ephemeral # configuration. - dsrc = self._get_ds() + dsrc = ds() cfg = dsrc.get_config_obj() - ret = dsrc.get_data() - self.assertTrue(ret) + assert ret is True assert "disk_setup" in cfg assert "fs_setup" in cfg - self.assertIsInstance(cfg["disk_setup"], dict) - self.assertIsInstance(cfg["fs_setup"], list) + assert isinstance(cfg["disk_setup"], dict) + assert isinstance(cfg["fs_setup"], list) - def test_override_disk_aliases(self): + def test_override_disk_aliases(self, paths): # Test to make sure that the built-in DS is overriden builtin = DataSourceSmartOS.BUILTIN_DS_CONFIG @@ -712,55 +695,51 @@ def test_override_disk_aliases(self): # expect that these values are in builtin, or this is pointless for k in mydscfg: - self.assertIn(k, builtin) + assert k in builtin - dsrc = self._get_ds(ds_cfg=mydscfg) + dsrc = _get_ds(paths, ds_cfg=mydscfg) ret = dsrc.get_data() - self.assertTrue(ret) + assert ret is True - self.assertEqual( - mydscfg["disk_aliases"]["FOO"], dsrc.ds_cfg["disk_aliases"]["FOO"] + assert ( + mydscfg["disk_aliases"]["FOO"] + == dsrc.ds_cfg["disk_aliases"]["FOO"] ) - self.assertEqual( - dsrc.device_name_to_device("FOO"), mydscfg["disk_aliases"]["FOO"] + assert ( + dsrc.device_name_to_device("FOO") == mydscfg["disk_aliases"]["FOO"] ) - def test_reconfig_network_on_boot(self): + def test_reconfig_network_on_boot(self, ds, m_jmc_client_factory): # Test to ensure that network is configured from metadata on each boot - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - self.assertSetEqual( - { - EventType.BOOT_NEW_INSTANCE, - EventType.BOOT, - EventType.BOOT_LEGACY, - }, - dsrc.default_update_events[EventScope.NETWORK], - ) + assert { + EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, + EventType.BOOT_LEGACY, + } == ds().default_update_events[EventScope.NETWORK] -class TestIdentifyFile(CiTestCase): +class TestIdentifyFile: """Test the 'identify_file' utility.""" + @pytest.mark.allow_subp_for("file") @skipIf(not which("file"), "command 'file' not available.") - def test_file_happy_path(self): + def test_file_happy_path(self, tmp_path): """Test file is available and functional on plain text.""" - fname = self.tmp_path("myfile") + fname = str(tmp_path / "myfile") write_file(fname, "plain text content here\n") - with self.allow_subp(["file"]): - self.assertEqual("text/plain", identify_file(fname)) + assert "text/plain" == identify_file(fname) @mock.patch(DSMOS + ".subp.subp") - def test_returns_none_on_error(self, m_subp): + def test_returns_none_on_error(self, m_subp, tmp_path): """On 'file' execution error, None should be returned.""" m_subp.side_effect = ProcessExecutionError("FILE_FAILED", exit_code=99) - fname = self.tmp_path("myfile") + fname = str(tmp_path / "myfile") write_file(fname, "plain text content here\n") - self.assertEqual(None, identify_file(fname)) - self.assertEqual( - [mock.call(["file", "--brief", "--mime-type", fname])], - m_subp.call_args_list, - ) + assert None is identify_file(fname) + assert [ + mock.call(["file", "--brief", "--mime-type", fname]) + ] == m_subp.call_args_list class ShortReader: @@ -798,249 +777,300 @@ def read(self, size=-1): return ret -class TestJoyentMetadataClient(FilesystemMockingTestCase): +@pytest.fixture +def m_serial(mocker): + return mocker.MagicMock(spec=serial.Serial) - invalid = b"invalid command\n" - failure = b"FAILURE\n" - v2_ok = b"V2_OK\n" - - def setUp(self): - super(TestJoyentMetadataClient, self).setUp() - - self.serial = mock.MagicMock(spec=serial.Serial) - self.request_id = 0xABCDEF12 - self.metadata_value = "value" - self.response_parts = { - "command": "SUCCESS", - "crc": "b5a9ff00", - "length": SUCCESS_LEN + len(b64e(self.metadata_value)), - "payload": b64e(self.metadata_value), - "request_id": "{0:08x}".format(self.request_id), - } - def make_response(): - payloadstr = "" - if "payload" in self.response_parts: - payloadstr = " {0}".format(self.response_parts["payload"]) - return ( - "V2 {length} {crc} {request_id} " - "{command}{payloadstr}\n".format( - payloadstr=payloadstr, **self.response_parts - ).encode("ascii") - ) +@pytest.fixture +def joyent_metadata(mocker, m_serial): + res = namedtuple( + "joyent", + [ + "serial", + "request_id", + "metadata_value", + "response_parts", + "meta_source_data", + "metasource_data_len", + ], + defaults=(None, None, None, None, None, None), + ) + + res.serial = m_serial + res.request_id = 0xABCDEF12 + res.metadata_value = "value" + res.response_parts = { + "command": "SUCCESS", + "crc": "b5a9ff00", + "length": SUCCESS_LEN + len(b64e(res.metadata_value)), + "payload": b64e(res.metadata_value), + "request_id": "{0:08x}".format(res.request_id), + } - self.metasource_data = None - - def read_response(length): - if not self.metasource_data: - self.metasource_data = make_response() - self.metasource_data_len = len(self.metasource_data) - resp = self.metasource_data[:length] - self.metasource_data = self.metasource_data[length:] - return resp - - self.serial.read.side_effect = read_response - self.patched_funcs.enter_context( - mock.patch( - "cloudinit.sources.DataSourceSmartOS.random.randint", - mock.Mock(return_value=self.request_id), - ) + def make_response(): + payloadstr = "" + if "payload" in res.response_parts: + payloadstr = " {0}".format(res.response_parts["payload"]) + return ( + "V2 {length} {crc} {request_id} " + "{command}{payloadstr}\n".format( + payloadstr=payloadstr, **res.response_parts + ).encode("ascii") ) - def _get_client(self): - return DataSourceSmartOS.JoyentMetadataClient( - fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM - ) + res.metasource_data = None + + def read_response(length): + if not res.metasource_data: + res.metasource_data = make_response() + res.metasource_data_len = len(res.metasource_data) + resp = res.metasource_data[:length] + res.metasource_data = res.metasource_data[length:] + return resp + + res.serial.read.side_effect = read_response + mocker.patch( + "cloudinit.sources.DataSourceSmartOS.random.randint", + mock.Mock(return_value=res.request_id), + ) + return res + + +@pytest.fixture +def joyent_client(joyent_metadata): + return DataSourceSmartOS.JoyentMetadataClient( + fp=joyent_metadata.serial, + smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM, + ) + + +def _get_written_line(joyent_client, m_serial, key="some_key"): + joyent_client.get(key) + return m_serial.write.call_args[0][0] + + +@pytest.fixture +def joyent_serial_client(joyent_metadata): + joyent_metadata.serial.timeout = 1 + return DataSourceSmartOS.JoyentMetadataSerialClient( + None, fp=joyent_metadata.serial + ) - def _get_serial_client(self): - self.serial.timeout = 1 - return DataSourceSmartOS.JoyentMetadataSerialClient( - None, fp=self.serial - ) + +@pytest.mark.usefixtures("fake_filesystem") +class TestJoyentMetadataClient: + + invalid = b"invalid command\n" + failure = b"FAILURE\n" + v2_ok = b"V2_OK\n" def assertEndsWith(self, haystack, prefix): - self.assertTrue( - haystack.endswith(prefix), - "{0} does not end with '{1}'".format(repr(haystack), prefix), + assert haystack.endswith(prefix), "{0} does not end with '{1}'".format( + repr(haystack), prefix ) def assertStartsWith(self, haystack, prefix): - self.assertTrue( - haystack.startswith(prefix), - "{0} does not start with '{1}'".format(repr(haystack), prefix), - ) + assert haystack.startswith( + prefix + ), "{0} does not start with '{1}'".format(repr(haystack), prefix) def assertNoMoreSideEffects(self, obj): - self.assertRaises(StopIteration, obj) + with pytest.raises(StopIteration): + obj() - def test_get_metadata_writes_a_single_line(self): - client = self._get_client() - client.get("some_key") - self.assertEqual(1, self.serial.write.call_count) - written_line = self.serial.write.call_args[0][0] + def test_get_metadata_writes_a_single_line(self, m_serial, joyent_client): + joyent_client.get("some_key") + assert 1 == m_serial.write.call_count + written_line = m_serial.write.call_args[0][0] self.assertEndsWith( written_line.decode("ascii"), b"\n".decode("ascii") ) - self.assertEqual(1, written_line.count(b"\n")) - - def _get_written_line(self, key="some_key"): - client = self._get_client() - client.get(key) - return self.serial.write.call_args[0][0] + assert 1 == written_line.count(b"\n") - def test_get_metadata_writes_bytes(self): - self.assertIsInstance(self._get_written_line(), bytes) + def test_get_metadata_writes_bytes(self, joyent_client, m_serial): + assert isinstance(_get_written_line(joyent_client, m_serial), bytes) - def test_get_metadata_line_starts_with_v2(self): - foo = self._get_written_line() + def test_get_metadata_line_starts_with_v2(self, joyent_client, m_serial): + foo = _get_written_line(joyent_client, m_serial) self.assertStartsWith(foo.decode("ascii"), b"V2".decode("ascii")) - def test_get_metadata_uses_get_command(self): - parts = self._get_written_line().decode("ascii").strip().split(" ") - self.assertEqual("GET", parts[4]) + def test_get_metadata_uses_get_command(self, joyent_client, m_serial): + parts = ( + _get_written_line(joyent_client, m_serial) + .decode("ascii") + .strip() + .split(" ") + ) + assert "GET" == parts[4] - def test_get_metadata_base64_encodes_argument(self): + def test_get_metadata_base64_encodes_argument( + self, joyent_client, m_serial + ): key = "my_key" - parts = self._get_written_line(key).decode("ascii").strip().split(" ") - self.assertEqual(b64e(key), parts[5]) + parts = ( + _get_written_line(joyent_client, m_serial, key) + .decode("ascii") + .strip() + .split(" ") + ) + assert b64e(key) == parts[5] - def test_get_metadata_calculates_length_correctly(self): - parts = self._get_written_line().decode("ascii").strip().split(" ") + def test_get_metadata_calculates_length_correctly( + self, joyent_client, m_serial + ): + parts = ( + _get_written_line(joyent_client, m_serial) + .decode("ascii") + .strip() + .split(" ") + ) expected_length = len(" ".join(parts[3:])) - self.assertEqual(expected_length, int(parts[1])) + assert expected_length == int(parts[1]) - def test_get_metadata_uses_appropriate_request_id(self): - parts = self._get_written_line().decode("ascii").strip().split(" ") + def test_get_metadata_uses_appropriate_request_id( + self, joyent_client, m_serial + ): + parts = ( + _get_written_line(joyent_client, m_serial) + .decode("ascii") + .strip() + .split(" ") + ) request_id = parts[3] - self.assertEqual(8, len(request_id)) - self.assertEqual(request_id, request_id.lower()) + assert 8 == len(request_id) + assert request_id == request_id.lower() - def test_get_metadata_uses_random_number_for_request_id(self): - line = self._get_written_line() + def test_get_metadata_uses_random_number_for_request_id( + self, joyent_client, joyent_metadata, m_serial + ): + line = _get_written_line(joyent_client, m_serial) request_id = line.decode("ascii").strip().split(" ")[3] - self.assertEqual("{0:08x}".format(self.request_id), request_id) - - def test_get_metadata_checksums_correctly(self): - parts = self._get_written_line().decode("ascii").strip().split(" ") + assert "{0:08x}".format(joyent_metadata.request_id) == request_id + + def test_get_metadata_checksums_correctly(self, joyent_client, m_serial): + parts = ( + _get_written_line(joyent_client, m_serial) + .decode("ascii") + .strip() + .split(" ") + ) expected_checksum = "{0:08x}".format( crc32(" ".join(parts[3:]).encode("utf-8")) & 0xFFFFFFFF ) checksum = parts[2] - self.assertEqual(expected_checksum, checksum) - - def test_get_metadata_reads_a_line(self): - client = self._get_client() - client.get("some_key") - self.assertEqual(self.metasource_data_len, self.serial.read.call_count) - - def test_get_metadata_returns_valid_value(self): - client = self._get_client() - value = client.get("some_key") - self.assertEqual(self.metadata_value, value) - - def test_get_metadata_throws_exception_for_incorrect_length(self): - self.response_parts["length"] = 0 - client = self._get_client() - self.assertRaises( - DataSourceSmartOS.JoyentMetadataFetchException, - client.get, - "some_key", - ) + assert expected_checksum == checksum - def test_get_metadata_throws_exception_for_incorrect_crc(self): - self.response_parts["crc"] = "deadbeef" - client = self._get_client() - self.assertRaises( - DataSourceSmartOS.JoyentMetadataFetchException, - client.get, - "some_key", - ) + def test_get_metadata_reads_a_line( + self, joyent_client, joyent_metadata, m_serial + ): + joyent_client.get("some_key") + assert joyent_metadata.metasource_data_len == m_serial.read.call_count - def test_get_metadata_throws_exception_for_request_id_mismatch(self): - self.response_parts["request_id"] = "deadbeef" - client = self._get_client() - client._checksum = lambda _: self.response_parts["crc"] - self.assertRaises( - DataSourceSmartOS.JoyentMetadataFetchException, - client.get, - "some_key", - ) + def test_get_metadata_returns_valid_value( + self, joyent_client, joyent_metadata + ): + value = joyent_client.get("some_key") + assert joyent_metadata.metadata_value == value + + def test_get_metadata_throws_exception_for_incorrect_length( + self, joyent_client, joyent_metadata + ): + joyent_metadata.response_parts["length"] = 0 + with pytest.raises(DataSourceSmartOS.JoyentMetadataFetchException): + joyent_client.get( + "some_key", + ) - def test_get_metadata_returns_None_if_value_not_found(self): - self.response_parts["payload"] = "" - self.response_parts["command"] = "NOTFOUND" - self.response_parts["length"] = NOTFOUND_LEN - client = self._get_client() - client._checksum = lambda _: self.response_parts["crc"] - self.assertIsNone(client.get("some_key")) + def test_get_metadata_throws_exception_for_incorrect_crc( + self, joyent_client, joyent_metadata + ): + joyent_metadata.response_parts["crc"] = "deadbeef" + with pytest.raises(DataSourceSmartOS.JoyentMetadataFetchException): + joyent_client.get( + "some_key", + ) - def test_negotiate(self): - client = self._get_client() + def test_get_metadata_throws_exception_for_request_id_mismatch( + self, joyent_client, joyent_metadata + ): + joyent_metadata.response_parts["request_id"] = "deadbeef" + joyent_client._checksum = lambda _: joyent_metadata.response_parts[ + "crc" + ] + with pytest.raises(DataSourceSmartOS.JoyentMetadataFetchException): + joyent_client.get("some_key") + + def test_get_metadata_returns_None_if_value_not_found( + self, joyent_client, joyent_metadata + ): + joyent_metadata.response_parts["payload"] = "" + joyent_metadata.response_parts["command"] = "NOTFOUND" + joyent_metadata.response_parts["length"] = NOTFOUND_LEN + joyent_client._checksum = lambda _: joyent_metadata.response_parts[ + "crc" + ] + assert joyent_client.get("some_key") is None + + def test_negotiate(self, joyent_client): reader = ShortReader(self.v2_ok) - client.fp.read.side_effect = reader.read - client._negotiate() - self.assertTrue(reader.emptied) + joyent_client.fp.read.side_effect = reader.read + joyent_client._negotiate() + assert reader.emptied - def test_negotiate_short_response(self): - client = self._get_client() + def test_negotiate_short_response(self, joyent_client): # chopped '\n' from v2_ok. reader = ShortReader(self.v2_ok[:-1] + b"\0") - client.fp.read.side_effect = reader.read - self.assertRaises( - DataSourceSmartOS.JoyentMetadataTimeoutException, client._negotiate - ) - self.assertTrue(reader.emptied) + joyent_client.fp.read.side_effect = reader.read + with pytest.raises(DataSourceSmartOS.JoyentMetadataTimeoutException): + joyent_client._negotiate() + assert reader.emptied - def test_negotiate_bad_response(self): - client = self._get_client() + def test_negotiate_bad_response(self, joyent_client): reader = ShortReader(b"garbage\n" + self.v2_ok) - client.fp.read.side_effect = reader.read - self.assertRaises( - DataSourceSmartOS.JoyentMetadataFetchException, client._negotiate - ) - self.assertEqual(self.v2_ok, client.fp.read()) + joyent_client.fp.read.side_effect = reader.read + with pytest.raises(DataSourceSmartOS.JoyentMetadataFetchException): + joyent_client._negotiate() + assert self.v2_ok == joyent_client.fp.read() - def test_serial_open_transport(self): - client = self._get_serial_client() + def test_serial_open_transport(self, joyent_serial_client): reader = ShortReader(b"garbage\0" + self.invalid + self.v2_ok) - client.fp.read.side_effect = reader.read - client.open_transport() - self.assertTrue(reader.emptied) + joyent_serial_client.fp.read.side_effect = reader.read + joyent_serial_client.open_transport() + assert reader.emptied - def test_flush_failure(self): - client = self._get_serial_client() + def test_flush_failure(self, joyent_serial_client): reader = ShortReader( b"garbage" + b"\0" + self.failure + self.invalid + self.v2_ok ) - client.fp.read.side_effect = reader.read - client.open_transport() - self.assertTrue(reader.emptied) + joyent_serial_client.fp.read.side_effect = reader.read + joyent_serial_client.open_transport() + assert reader.emptied - def test_flush_many_timeouts(self): - client = self._get_serial_client() + def test_flush_many_timeouts(self, joyent_serial_client): reader = ShortReader(b"\0" * 100 + self.invalid + self.v2_ok) - client.fp.read.side_effect = reader.read - client.open_transport() - self.assertTrue(reader.emptied) + joyent_serial_client.fp.read.side_effect = reader.read + joyent_serial_client.open_transport() + assert reader.emptied - def test_list_metadata_returns_list(self): + def test_list_metadata_returns_list(self, joyent_client, joyent_metadata): parts = ["foo", "bar"] value = b64e("\n".join(parts)) - self.response_parts["payload"] = value - self.response_parts["crc"] = "40873553" - self.response_parts["length"] = SUCCESS_LEN + len(value) - client = self._get_client() - self.assertEqual(client.list(), parts) + joyent_metadata.response_parts["payload"] = value + joyent_metadata.response_parts["crc"] = "40873553" + joyent_metadata.response_parts["length"] = SUCCESS_LEN + len(value) + assert joyent_client.list() == parts - def test_list_metadata_returns_empty_list_if_no_customer_metadata(self): - del self.response_parts["payload"] - self.response_parts["length"] = SUCCESS_LEN - 1 - self.response_parts["crc"] = "14e563ba" - client = self._get_client() - self.assertEqual(client.list(), []) + def test_list_metadata_returns_empty_list_if_no_customer_metadata( + self, joyent_client, joyent_metadata + ): + del joyent_metadata.response_parts["payload"] + joyent_metadata.response_parts["length"] = SUCCESS_LEN - 1 + joyent_metadata.response_parts["crc"] = "14e563ba" + assert joyent_client.list() == [] -class TestNetworkConversion(CiTestCase): +class TestNetworkConversion: def test_convert_simple(self): expected = { "version": 1, @@ -1070,7 +1100,7 @@ def test_convert_simple(self): ], } found = convert_net(SDC_NICS) - self.assertEqual(expected, found) + assert expected == found def test_convert_simple_alt(self): expected = { @@ -1101,7 +1131,7 @@ def test_convert_simple_alt(self): ], } found = convert_net(SDC_NICS_ALT) - self.assertEqual(expected, found) + assert expected == found def test_convert_simple_dhcp(self): expected = { @@ -1130,7 +1160,7 @@ def test_convert_simple_dhcp(self): ], } found = convert_net(SDC_NICS_DHCP) - self.assertEqual(expected, found) + assert expected == found def test_convert_simple_multi_ip(self): expected = { @@ -1163,7 +1193,7 @@ def test_convert_simple_multi_ip(self): ], } found = convert_net(SDC_NICS_MIP) - self.assertEqual(expected, found) + assert expected == found def test_convert_with_dns(self): expected = { @@ -1201,7 +1231,7 @@ def test_convert_with_dns(self): dns_servers=["8.8.8.8", "8.8.8.1"], dns_domain="local", ) - self.assertEqual(expected, found) + assert expected == found def test_convert_simple_multi_ipv6(self): expected = { @@ -1238,7 +1268,7 @@ def test_convert_simple_multi_ipv6(self): ], } found = convert_net(SDC_NICS_MIP_IPV6) - self.assertEqual(expected, found) + assert expected == found def test_convert_simple_both_ipv4_ipv6(self): expected = { @@ -1276,7 +1306,7 @@ def test_convert_simple_both_ipv4_ipv6(self): ], } found = convert_net(SDC_NICS_IPV4_IPV6) - self.assertEqual(expected, found) + assert expected == found def test_gateways_not_on_all_nics(self): expected = { @@ -1307,7 +1337,7 @@ def test_gateways_not_on_all_nics(self): ], } found = convert_net(SDC_NICS_SINGLE_GATEWAY) - self.assertEqual(expected, found) + assert expected == found def test_routes_on_all_nics(self): routes = [ @@ -1366,7 +1396,7 @@ def test_routes_on_all_nics(self): } found = convert_net(SDC_NICS_SINGLE_GATEWAY, routes=routes) self.maxDiff = None - self.assertEqual(expected, found) + assert expected == found def test_ipv6_addrconf(self): expected = { @@ -1389,18 +1419,45 @@ def test_ipv6_addrconf(self): } found = convert_net(SDC_NICS_ADDRCONF) self.maxDiff = None - self.assertEqual(expected, found) + assert expected == found + + +@pytest.mark.allow_subp_for("mdata-get") +@pytest.fixture +def mdata_proc(): + mdata_proc = multiprocessing.Process(target=start_mdata_loop) + mdata_proc.start() + yield mdata_proc -@unittest.skipUnless( - get_smartos_environ() == SMARTOS_ENV_KVM, - "Only supported on KVM and bhyve guests under SmartOS", + # os.kill() rather than mdata_proc.terminate() to avoid console spam. + os.kill(mdata_proc.pid, signal.SIGKILL) + mdata_proc.join() + + +def start_mdata_loop(): + """ + The mdata-get command is repeatedly run in a separate process so + that it may try to race with metadata operations performed in the + main test process. Use of mdata-get is better than two processes + using the protocol implementation in DataSourceSmartOS because we + are testing to be sure that cloud-init and mdata-get respect each + others locks. + """ + rcs = list(range(256)) + while True: + subp(["mdata-get", "sdc:routes"], rcs=rcs) + + +@pytest.mark.skipif( + get_smartos_environ() != SMARTOS_ENV_KVM, + reason="Only supported on KVM and bhyve guests under SmartOS", ) -@unittest.skipUnless( - os.access(SERIAL_DEVICE, os.W_OK), - "Requires write access to " + SERIAL_DEVICE, +@pytest.mark.skipif( + not os.access(SERIAL_DEVICE, os.W_OK), + reason="Requires write access to " + SERIAL_DEVICE, ) -class TestSerialConcurrency(CiTestCase): +class TestSerialConcurrency: """ This class tests locking on an actual serial port, and as such can only be run in a kvm or bhyve guest running on a SmartOS host. A test run on @@ -1412,40 +1469,15 @@ class TestSerialConcurrency(CiTestCase): This takes on the order of 2 to 3 minutes to run. """ - allowed_subp = ["mdata-get"] - - def setUp(self): - self.mdata_proc = multiprocessing.Process(target=self.start_mdata_loop) - self.mdata_proc.start() - super(TestSerialConcurrency, self).setUp() - - def tearDown(self): - # os.kill() rather than mdata_proc.terminate() to avoid console spam. - os.kill(self.mdata_proc.pid, signal.SIGKILL) - self.mdata_proc.join() - super().tearDown() - - def start_mdata_loop(self): - """ - The mdata-get command is repeatedly run in a separate process so - that it may try to race with metadata operations performed in the - main test process. Use of mdata-get is better than two processes - using the protocol implementation in DataSourceSmartOS because we - are testing to be sure that cloud-init and mdata-get respect each - others locks. - """ - rcs = list(range(256)) - while True: - subp(["mdata-get", "sdc:routes"], rcs=rcs) - - def test_all_keys(self): - self.assertIsNotNone(self.mdata_proc.pid) + @pytest.mark.allow_subp_for("mdata-get") + def test_all_keys(self, mdata_proc): + assert mdata_proc.pid is not None ds = DataSourceSmartOS keys = [tup[0] for tup in ds.SMARTOS_ATTRIB_MAP.values()] keys.extend(ds.SMARTOS_ATTRIB_JSON.values()) client = ds.jmc_client_factory(smartos_type=SMARTOS_ENV_KVM) - self.assertIsNotNone(client) + assert client is not None # The behavior that we are testing for was observed mdata-get running # 10 times at roughly the same time as cloud-init fetched each key @@ -1457,4 +1489,4 @@ def test_all_keys(self): # thrown any exceptions. client.get(key) - self.assertIsNone(self.mdata_proc.exitcode) + assert mdata_proc.exitcode is None diff --git a/tests/unittests/sources/test_vmware.py b/tests/unittests/sources/test_vmware.py index cfeff6d5..dacc0a5a 100644 --- a/tests/unittests/sources/test_vmware.py +++ b/tests/unittests/sources/test_vmware.py @@ -1,7 +1,7 @@ -# Copyright (c) 2021-2022 VMware, Inc. All Rights Reserved. +# Copyright (c) 2021-2025 Broadcom. All Rights Reserved. # -# Authors: Andrew Kutz -# Pengpeng Sun +# Authors: Andrew Kutz +# Pengpeng Sun # # This file is part of cloud-init. See LICENSE file for license information. @@ -9,21 +9,17 @@ import gzip import os from contextlib import ExitStack +from logging import DEBUG from textwrap import dedent import pytest -from cloudinit import dmi, helpers, safeyaml, settings, util +from cloudinit import dmi, safeyaml, settings, util +from cloudinit.event import EventScope from cloudinit.sources import DataSourceVMware from cloudinit.sources.helpers.vmware.imc import guestcust_util from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import ( - CiTestCase, - FilesystemMockingTestCase, - mock, - populate_dir, - wrap_and_call, -) +from tests.unittests.helpers import mock, populate_dir, wrap_and_call MPATH = "cloudinit.sources.DataSourceVMware." PRODUCT_NAME_FILE_PATH = "/sys/class/dmi/id/product_name" @@ -40,7 +36,8 @@ ] VMW_SINGLE_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAA... test@vmw.com" -VMW_METADATA_YAML = """instance-id: cloud-vm +VMW_METADATA_YAML = """\ +instance-id: cloud-vm local-hostname: cloud-vm network: version: 2 @@ -51,13 +48,15 @@ dhcp4: yes """ -VMW_USERDATA_YAML = """## template: jinja +VMW_USERDATA_YAML = """\ +## template: jinja #cloud-config users: - default """ -VMW_VENDORDATA_YAML = """## template: jinja +VMW_VENDORDATA_YAML = """\ +## template: jinja #cloud-config runcmd: - echo "Hello, world." @@ -106,6 +105,27 @@ "addr": "fd42:baa2:3dd:17a:216:3eff:fe16:db54", } +# Please note this should be a constant, but uses formatting to avoid +# the line-length warning from the linter. +VMW_EXPECTED_EXTRA_HOTPLUG_UDEV_RULES = """ +ENV{ID_NET_DRIVER}=="e1000|e1000e|vlance|vmxnet2|vmxnet3|vrdma", GOTO="cloudinit_hook" +GOTO="cloudinit_end" +""" # noqa: E501 + + +VMW_METADATA_YAML_WITH_NET_DRIVERS = """\ +instance-id: cloud-vm +local-hostname: cloud-vm +network-drivers: +- vmxnet2 +- vmxnet3 +""" + +VMW_EXPECTED_EXTRA_HOTPLUG_UDEV_RULES_VMXNET = """ +ENV{ID_NET_DRIVER}=="vmxnet2|vmxnet3", GOTO="cloudinit_hook" +GOTO="cloudinit_end" +""" + def generate_test_netdev_data(ipv4=None, ipv6=None): ipv4 = ipv4 or [] @@ -144,85 +164,75 @@ def common_patches(): yield -class TestDataSourceVMware(CiTestCase): +class TestDataSourceVMware: """ Test common functionality that is not transport specific. """ - with_logs = True - - def setUp(self): - super(TestDataSourceVMware, self).setUp() - self.tmp = self.tmp_dir() - - def test_no_data_access_method(self): - ds = get_ds(self.tmp) + def test_no_data_access_method(self, DS): + ds = DS(settings.CFG_BUILTIN) with mock.patch( "cloudinit.sources.DataSourceVMware.is_vmware_platform", return_value=False, ): ret = ds.get_data() - self.assertFalse(ret) + assert not ret def test_convert_to_netifaces_ipv4_format(self): netifaces_format = DataSourceVMware.convert_to_netifaces_ipv4_format( VMW_IPV4_NETDEV_ADDR ) - self.assertEqual(netifaces_format, VMW_IPV4_NETIFACES_ADDR) + assert netifaces_format == VMW_IPV4_NETIFACES_ADDR def test_convert_to_netifaces_ipv6_format(self): netifaces_format = DataSourceVMware.convert_to_netifaces_ipv6_format( VMW_IPV6_NETDEV_ADDR ) - self.assertEqual(netifaces_format, VMW_IPV6_NETIFACES_ADDR) + assert netifaces_format == VMW_IPV6_NETIFACES_ADDR netifaces_format = DataSourceVMware.convert_to_netifaces_ipv6_format( VMW_IPV6_NETDEV_PEER_ADDR ) - self.assertEqual(netifaces_format, VMW_IPV6_NETIFACES_PEER_ADDR) + assert netifaces_format == VMW_IPV6_NETIFACES_PEER_ADDR @mock.patch("cloudinit.sources.DataSourceVMware.get_default_ip_addrs") def test_get_host_info_ipv4(self, m_fn_ipaddr): m_fn_ipaddr.return_value = ("10.10.10.1", None) host_info = DataSourceVMware.get_host_info() - self.assertTrue(host_info) - self.assertTrue(host_info["hostname"]) - self.assertTrue(host_info["hostname"] == "host.cloudinit.test") - self.assertTrue(host_info["local-hostname"]) - self.assertTrue(host_info["local_hostname"]) - self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4]) - self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4] == "10.10.10.1") - self.assertFalse(host_info.get(DataSourceVMware.LOCAL_IPV6)) + assert host_info + assert host_info["hostname"] + assert host_info["hostname"] == "host.cloudinit.test" + assert host_info["local-hostname"] + assert host_info["local_hostname"] + assert host_info[DataSourceVMware.LOCAL_IPV4] + assert host_info[DataSourceVMware.LOCAL_IPV4] == "10.10.10.1" + assert not host_info.get(DataSourceVMware.LOCAL_IPV6) @mock.patch("cloudinit.sources.DataSourceVMware.get_default_ip_addrs") def test_get_host_info_ipv6(self, m_fn_ipaddr): m_fn_ipaddr.return_value = (None, "2001:db8::::::8888") host_info = DataSourceVMware.get_host_info() - self.assertTrue(host_info) - self.assertTrue(host_info["hostname"]) - self.assertTrue(host_info["hostname"] == "host.cloudinit.test") - self.assertTrue(host_info["local-hostname"]) - self.assertTrue(host_info["local_hostname"]) - self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV6]) - self.assertTrue( - host_info[DataSourceVMware.LOCAL_IPV6] == "2001:db8::::::8888" - ) - self.assertFalse(host_info.get(DataSourceVMware.LOCAL_IPV4)) + assert host_info + assert host_info["hostname"] + assert host_info["hostname"] == "host.cloudinit.test" + assert host_info["local-hostname"] + assert host_info["local_hostname"] + assert host_info[DataSourceVMware.LOCAL_IPV6] + assert host_info[DataSourceVMware.LOCAL_IPV6] == "2001:db8::::::8888" + assert not host_info.get(DataSourceVMware.LOCAL_IPV4) @mock.patch("cloudinit.sources.DataSourceVMware.get_default_ip_addrs") def test_get_host_info_dual(self, m_fn_ipaddr): m_fn_ipaddr.return_value = ("10.10.10.1", "2001:db8::::::8888") host_info = DataSourceVMware.get_host_info() - self.assertTrue(host_info) - self.assertTrue(host_info["hostname"]) - self.assertTrue(host_info["hostname"] == "host.cloudinit.test") - self.assertTrue(host_info["local-hostname"]) - self.assertTrue(host_info["local_hostname"]) - self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4]) - self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4] == "10.10.10.1") - self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV6]) - self.assertTrue( - host_info[DataSourceVMware.LOCAL_IPV6] == "2001:db8::::::8888" - ) + assert host_info + assert host_info["hostname"] + assert host_info["hostname"] == "host.cloudinit.test" + assert host_info["local-hostname"] + assert host_info["local_hostname"] + assert host_info[DataSourceVMware.LOCAL_IPV4] + assert host_info[DataSourceVMware.LOCAL_IPV4] == "10.10.10.1" + assert host_info[DataSourceVMware.LOCAL_IPV6] + assert host_info[DataSourceVMware.LOCAL_IPV6] == "2001:db8::::::8888" # TODO migrate this entire test suite to pytest then parameterize @mock.patch("cloudinit.netinfo.route_info") @@ -241,8 +251,8 @@ def test_get_default_ip_addrs_ipv4only( ipv4=[VMW_IPV4_NETDEV_ADDR] ) ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() - self.assertEqual(ipv4, "10.85.130.116") - self.assertEqual(ipv6, None) + assert ipv4 == "10.85.130.116" + assert ipv6 is None @mock.patch("cloudinit.netinfo.route_info") @mock.patch("cloudinit.netinfo.netdev_info") @@ -259,8 +269,8 @@ def test_get_default_ip_addrs_ipv6only( ipv6=[VMW_IPV6_NETDEV_ADDR] ) ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() - self.assertEqual(ipv4, None) - self.assertEqual(ipv6, "fd42:baa2:3dd:17a:216:3eff:fe16:db54/64") + assert ipv4 is None + assert ipv6 == "fd42:baa2:3dd:17a:216:3eff:fe16:db54/64" @mock.patch("cloudinit.netinfo.route_info") @mock.patch("cloudinit.netinfo.netdev_info") @@ -278,8 +288,8 @@ def test_get_default_ip_addrs_dualstack( ipv6=[VMW_IPV6_NETDEV_ADDR], ) ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() - self.assertEqual(ipv4, "10.85.130.116") - self.assertEqual(ipv6, "fd42:baa2:3dd:17a:216:3eff:fe16:db54/64") + assert ipv4 == "10.85.130.116" + assert ipv6 == "fd42:baa2:3dd:17a:216:3eff:fe16:db54/64" @mock.patch("cloudinit.netinfo.route_info") @mock.patch("cloudinit.netinfo.netdev_info") @@ -311,8 +321,8 @@ def test_get_default_ip_addrs_multiaddr( ], ) ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() - self.assertEqual(ipv4, None) - self.assertEqual(ipv6, None) + assert ipv4 is None + assert ipv6 is None @mock.patch("cloudinit.netinfo.route_info") @mock.patch("cloudinit.netinfo.netdev_info") @@ -339,11 +349,11 @@ def test_get_default_ip_addrs_nodefault( ipv6=[VMW_IPV6_NETDEV_ADDR], ) ipv4, ipv6 = DataSourceVMware.get_default_ip_addrs() - self.assertEqual(ipv4, None) - self.assertEqual(ipv6, None) + assert ipv4 is None + assert ipv6 is None @mock.patch("cloudinit.sources.DataSourceVMware.get_host_info") - def test_wait_on_network(self, m_fn): + def test_wait_on_network(self, m_fn, caplog): metadata = { DataSourceVMware.WAIT_ON_NETWORK: { DataSourceVMware.WAIT_ON_NETWORK_IPV4: True, @@ -397,359 +407,494 @@ def test_wait_on_network(self, m_fn): host_info = DataSourceVMware.wait_on_network(metadata) - logs = self.logs.getvalue() expected_logs = [ - "DEBUG: waiting on network: wait4=True, " - "ready4=False, wait6=False, ready6=False\n", - "DEBUG: waiting on network complete\n", + ( + "cloudinit.sources.DataSourceVMware", + DEBUG, + ( + "waiting on network: wait4=True, " + "ready4=False, wait6=False, ready6=False" + ), + ), + ( + "cloudinit.sources.DataSourceVMware", + DEBUG, + "waiting on network complete", + ), ] for log in expected_logs: - self.assertIn(log, logs) - - self.assertTrue(host_info) - self.assertTrue(host_info["hostname"]) - self.assertTrue(host_info["hostname"] == "host.cloudinit.test") - self.assertTrue(host_info["local-hostname"]) - self.assertTrue(host_info["local_hostname"]) - self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4]) - self.assertTrue(host_info[DataSourceVMware.LOCAL_IPV4] == "10.10.10.1") + assert log in caplog.record_tuples + + assert host_info + assert host_info["hostname"] + assert host_info["hostname"] == "host.cloudinit.test" + assert host_info["local-hostname"] + assert host_info["local_hostname"] + assert host_info[DataSourceVMware.LOCAL_IPV4] + assert host_info[DataSourceVMware.LOCAL_IPV4] == "10.10.10.1" + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_set_value") + def test_advertise_update_events(self, m_set_fn): + ( + supported_events, + enabled_events, + ) = DataSourceVMware.advertise_update_events( + DataSourceVMware.SUPPORTED_UPDATE_EVENTS, + DataSourceVMware.DEFAULT_UPDATE_EVENTS, + "rpctool", + len, + ) + assert 2 == m_set_fn.call_count + assert "network=boot;boot-new-instance;hotplug" == supported_events + assert "network=boot-new-instance;hotplug" == enabled_events + + def test_extra_hotplug_udev_rules(self, DS): + ds = DS(settings.CFG_BUILTIN) + assert ( + VMW_EXPECTED_EXTRA_HOTPLUG_UDEV_RULES + == ds.extra_hotplug_udev_rules + ) -class TestDataSourceVMwareEnvVars(FilesystemMockingTestCase): +class TestDataSourceVMwareEnvVars: """ Test the envvar transport. """ - def setUp(self): - super(TestDataSourceVMwareEnvVars, self).setUp() - self.tmp = self.tmp_dir() - os.environ[DataSourceVMware.VMX_GUESTINFO] = "1" - self.create_system_files() - - def tearDown(self): - del os.environ[DataSourceVMware.VMX_GUESTINFO] - return super().tearDown() - - def create_system_files(self): - rootd = self.tmp_dir() + @pytest.fixture(autouse=True) + def env_and_files(self, fake_filesystem, monkeypatch, tmpdir): + monkeypatch.setenv(DataSourceVMware.VMX_GUESTINFO, "1") populate_dir( - rootd, - { - DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID, - }, + str(tmpdir), + {DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID}, ) - self.assertTrue(self.reRoot(rootd)) - def assert_get_data_ok(self, m_fn, m_fn_call_count=6): - ds = get_ds(self.tmp) + def assert_get_data_ok(self, DS, m_fn, m_fn_call_count=6): + ds = DS(settings.CFG_BUILTIN) ret = ds.get_data() - self.assertTrue(ret) - self.assertEqual(m_fn_call_count, m_fn.call_count) - self.assertEqual( - ds.data_access_method, DataSourceVMware.DATA_ACCESS_METHOD_ENVVAR + assert ret + assert m_fn_call_count == m_fn.call_count + assert ( + ds.data_access_method == DataSourceVMware.DATA_ACCESS_METHOD_ENVVAR ) return ds - def assert_metadata(self, metadata, m_fn, m_fn_call_count=6): - ds = self.assert_get_data_ok(m_fn, m_fn_call_count) - assert_metadata(self, ds, metadata) + def assert_metadata(self, DS, metadata, m_fn, m_fn_call_count=6): + ds = self.assert_get_data_ok(DS, m_fn, m_fn_call_count) + assert_metadata(ds, metadata) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_get_subplatform(self, m_fn): + def test_get_subplatform(self, m_fn, DS): m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] - ds = self.assert_get_data_ok(m_fn, m_fn_call_count=4) - self.assertEqual( - ds.subplatform, - "%s (%s)" - % ( - DataSourceVMware.DATA_ACCESS_METHOD_ENVVAR, - DataSourceVMware.get_guestinfo_envvar_key_name("metadata"), - ), + ds = self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) + assert ds.subplatform == "%s (%s)" % ( + DataSourceVMware.DATA_ACCESS_METHOD_ENVVAR, + DataSourceVMware.get_guestinfo_envvar_key_name("metadata"), + ) + + # Test to ensure that network is configured from metadata on each boot. + assert ( + DataSourceVMware.DEFAULT_UPDATE_EVENTS[EventScope.NETWORK] + == ds.default_update_events[EventScope.NETWORK] + ) + assert ( + DataSourceVMware.SUPPORTED_UPDATE_EVENTS[EventScope.NETWORK] + == ds.supported_update_events[EventScope.NETWORK] ) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_get_data_metadata_only(self, m_fn): + def test_get_data_metadata_only(self, m_fn, DS): m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_get_data_userdata_only(self, m_fn): + def test_get_data_userdata_only(self, m_fn, DS): m_fn.side_effect = ["", VMW_USERDATA_YAML, "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_get_data_vendordata_only(self, m_fn): + def test_get_data_vendordata_only(self, m_fn, DS): m_fn.side_effect = ["", "", VMW_VENDORDATA_YAML, ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_get_data_metadata_base64(self, m_fn): + def test_get_data_metadata_base64(self, m_fn, DS): data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8")) m_fn.side_effect = [data, "base64", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_get_data_metadata_b64(self, m_fn): + def test_get_data_metadata_b64(self, m_fn, DS): data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8")) m_fn.side_effect = [data, "b64", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_get_data_metadata_gzip_base64(self, m_fn): + def test_get_data_metadata_gzip_base64(self, m_fn, DS): data = VMW_METADATA_YAML.encode("utf-8") data = gzip.compress(data) data = base64.b64encode(data) m_fn.side_effect = [data, "gzip+base64", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_get_data_metadata_gz_b64(self, m_fn): + def test_get_data_metadata_gz_b64(self, m_fn, DS): data = VMW_METADATA_YAML.encode("utf-8") data = gzip.compress(data) data = base64.b64encode(data) m_fn.side_effect = [data, "gz+b64", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_metadata_single_ssh_key(self, m_fn): + def test_metadata_single_ssh_key(self, m_fn, DS): metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML) metadata["public_keys"] = VMW_SINGLE_KEY metadata_yaml = safeyaml.dumps(metadata) m_fn.side_effect = [metadata_yaml, "", "", ""] - self.assert_metadata(metadata, m_fn, m_fn_call_count=4) + self.assert_metadata(DS, metadata, m_fn, m_fn_call_count=4) @mock.patch( "cloudinit.sources.DataSourceVMware.guestinfo_envvar_get_value" ) - def test_metadata_multiple_ssh_keys(self, m_fn): + def test_metadata_multiple_ssh_keys(self, m_fn, DS): metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML) metadata["public_keys"] = VMW_MULTIPLE_KEYS metadata_yaml = safeyaml.dumps(metadata) m_fn.side_effect = [metadata_yaml, "", "", ""] - self.assert_metadata(metadata, m_fn, m_fn_call_count=4) + self.assert_metadata(DS, metadata, m_fn, m_fn_call_count=4) -class TestDataSourceVMwareGuestInfo(FilesystemMockingTestCase): +class TestDataSourceVMwareGuestInfo: """ Test the guestinfo transport on a VMware platform. """ - def setUp(self): - super(TestDataSourceVMwareGuestInfo, self).setUp() - self.tmp = self.tmp_dir() - self.create_system_files() - - def create_system_files(self): - rootd = self.tmp_dir() + @pytest.fixture(autouse=True) + def create_files(self, fake_filesystem, tmpdir): populate_dir( - rootd, + str(tmpdir), { DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID, PRODUCT_NAME_FILE_PATH: PRODUCT_NAME, }, ) - self.assertTrue(self.reRoot(rootd)) - def assert_get_data_ok(self, m_fn, m_fn_call_count=6): - ds = get_ds(self.tmp) + def assert_get_data_ok(self, DS, m_fn, m_fn_call_count=6): + ds = DS(settings.CFG_BUILTIN) ret = ds.get_data() - self.assertTrue(ret) - self.assertEqual(m_fn_call_count, m_fn.call_count) - self.assertEqual( - ds.data_access_method, - DataSourceVMware.DATA_ACCESS_METHOD_GUESTINFO, + assert ret + assert m_fn_call_count == m_fn.call_count + assert ( + ds.data_access_method + == DataSourceVMware.DATA_ACCESS_METHOD_GUESTINFO ) return ds - def assert_metadata(self, metadata, m_fn, m_fn_call_count=6): - ds = self.assert_get_data_ok(m_fn, m_fn_call_count) - assert_metadata(self, ds, metadata) + def assert_metadata(self, DS, metadata, m_fn, m_fn_call_count=6): + ds = self.assert_get_data_ok(DS, m_fn, m_fn_call_count) + assert_metadata(ds, metadata) def test_ds_valid_on_vmware_platform(self): system_type = dmi.read_dmi_data("system-product-name") - self.assertEqual(system_type, PRODUCT_NAME) + assert system_type == PRODUCT_NAME @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_get_subplatform(self, m_which_fn, m_fn): + def test_get_subplatform(self, m_which_fn, m_fn, DS): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] - ds = self.assert_get_data_ok(m_fn, m_fn_call_count=4) - self.assertEqual( - ds.subplatform, - "%s (%s)" - % ( - DataSourceVMware.DATA_ACCESS_METHOD_GUESTINFO, - DataSourceVMware.get_guestinfo_key_name("metadata"), - ), + ds = self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) + assert ds.subplatform == "%s (%s)" % ( + DataSourceVMware.DATA_ACCESS_METHOD_GUESTINFO, + DataSourceVMware.get_guestinfo_key_name("metadata"), + ) + + # Test to ensure that network is configured from metadata on each boot. + assert ( + DataSourceVMware.DEFAULT_UPDATE_EVENTS[EventScope.NETWORK] + == ds.default_update_events[EventScope.NETWORK] + ) + assert ( + DataSourceVMware.SUPPORTED_UPDATE_EVENTS[EventScope.NETWORK] + == ds.supported_update_events[EventScope.NETWORK] ) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_get_data_metadata_with_vmware_rpctool(self, m_which_fn, m_fn): + def test_get_data_metadata_with_vmware_rpctool( + self, m_which_fn, m_fn, DS, tmpdir + ): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] m_fn.side_effect = [VMW_METADATA_YAML, "", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.exec_vmware_rpctool") @mock.patch("cloudinit.sources.DataSourceVMware.which") def test_get_data_metadata_non_zero_exit_code_fallback_to_vmtoolsd( - self, m_which_fn, m_exec_vmware_rpctool_fn, m_fn + self, m_which_fn, m_exec_vmware_rpctool_fn, m_fn, DS, tmpdir ): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] m_exec_vmware_rpctool_fn.side_effect = ProcessExecutionError( exit_code=1 ) m_fn.side_effect = [VMW_METADATA_YAML, "", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.exec_vmware_rpctool") @mock.patch("cloudinit.sources.DataSourceVMware.which") def test_get_data_metadata_vmware_rpctool_not_found_fallback_to_vmtoolsd( - self, m_which_fn, m_exec_vmware_rpctool_fn, m_fn + self, m_which_fn, m_exec_vmware_rpctool_fn, m_fn, DS, tmpdir ): m_which_fn.side_effect = ["vmtoolsd", None] m_fn.side_effect = [VMW_METADATA_YAML, "", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_get_data_userdata_only(self, m_which_fn, m_fn): + def test_get_data_userdata_only(self, m_which_fn, m_fn, DS): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] m_fn.side_effect = ["", VMW_USERDATA_YAML, "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_get_data_vendordata_only(self, m_which_fn, m_fn): + def test_get_data_vendordata_only(self, m_which_fn, m_fn, DS): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] m_fn.side_effect = ["", "", VMW_VENDORDATA_YAML, ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_metadata_single_ssh_key(self, m_which_fn, m_fn): + def test_metadata_single_ssh_key(self, m_which_fn, m_fn, DS): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML) metadata["public_keys"] = VMW_SINGLE_KEY metadata_yaml = safeyaml.dumps(metadata) m_fn.side_effect = [metadata_yaml, "", "", ""] - self.assert_metadata(metadata, m_fn, m_fn_call_count=4) + self.assert_metadata(DS, metadata, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_metadata_multiple_ssh_keys(self, m_which_fn, m_fn): + def test_metadata_multiple_ssh_keys(self, m_which_fn, m_fn, DS): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] metadata = DataSourceVMware.load_json_or_yaml(VMW_METADATA_YAML) metadata["public_keys"] = VMW_MULTIPLE_KEYS metadata_yaml = safeyaml.dumps(metadata) m_fn.side_effect = [metadata_yaml, "", "", ""] - self.assert_metadata(metadata, m_fn, m_fn_call_count=4) + self.assert_metadata(DS, metadata, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_get_data_metadata_base64(self, m_which_fn, m_fn): + def test_get_data_metadata_base64(self, m_which_fn, m_fn, DS): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8")) m_fn.side_effect = [data, "base64", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_get_data_metadata_b64(self, m_which_fn, m_fn): + def test_get_data_metadata_b64(self, m_which_fn, m_fn, DS): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] data = base64.b64encode(VMW_METADATA_YAML.encode("utf-8")) m_fn.side_effect = [data, "b64", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_get_data_metadata_gzip_base64(self, m_which_fn, m_fn): + def test_get_data_metadata_gzip_base64(self, m_which_fn, m_fn, DS): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] data = VMW_METADATA_YAML.encode("utf-8") data = gzip.compress(data) data = base64.b64encode(data) m_fn.side_effect = [data, "gzip+base64", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") @mock.patch("cloudinit.sources.DataSourceVMware.which") - def test_get_data_metadata_gz_b64(self, m_which_fn, m_fn): + def test_get_data_metadata_gz_b64(self, m_which_fn, m_fn, DS): m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] data = VMW_METADATA_YAML.encode("utf-8") data = gzip.compress(data) data = base64.b64encode(data) m_fn.side_effect = [data, "gz+b64", "", ""] - self.assert_get_data_ok(m_fn, m_fn_call_count=4) + self.assert_get_data_ok(DS, m_fn, m_fn_call_count=4) + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_set_value") + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + @mock.patch("cloudinit.sources.DataSourceVMware.which") + def test_advertise_update_events( + self, m_which_fn, m_get_fn, m_set_fn, DS, tmpdir + ): + m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] + m_get_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] + ds = self.assert_get_data_ok(DS, m_get_fn, m_fn_call_count=4) + supported_events, enabled_events = ds.advertise_update_events({}) + assert 2 == m_set_fn.call_count + assert "network=boot;boot-new-instance;hotplug" == supported_events + assert "network=boot-new-instance;hotplug" == enabled_events + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_set_value") + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + @mock.patch("cloudinit.sources.DataSourceVMware.which") + def test_advertise_update_events_with_events_from_user_data( + self, m_which_fn, m_get_fn, m_set_fn, DS, tmpdir + ): + m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] + m_get_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] + ds = self.assert_get_data_ok(DS, m_get_fn, m_fn_call_count=4) + supported_events, enabled_events = ds.advertise_update_events( + { + "updates": { + "network": { + "when": ["boot"], + }, + }, + } + ) + assert 2 == m_set_fn.call_count + assert "network=boot;boot-new-instance;hotplug" == supported_events + assert "network=boot" == enabled_events + + @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") + @mock.patch("cloudinit.sources.DataSourceVMware.which") + def test_extra_hotplug_udev_rules_with_net_drivers( + self, m_which_fn, m_get_fn, DS, tmpdir + ): + m_which_fn.side_effect = ["vmtoolsd", "vmware-rpctool"] + m_get_fn.side_effect = [ + VMW_METADATA_YAML_WITH_NET_DRIVERS, + "", + "", + "", + "", + "", + ] + ds = self.assert_get_data_ok(DS, m_get_fn, m_fn_call_count=4) + ds.init_extra_hotplug_udev_rules() + assert ( + VMW_EXPECTED_EXTRA_HOTPLUG_UDEV_RULES_VMXNET + == ds.extra_hotplug_udev_rules + ) -class TestDataSourceVMwareGuestInfo_InvalidPlatform(FilesystemMockingTestCase): + +class TestDataSourceVMwareGuestInfo_InvalidPlatform: """ Test the guestinfo transport on a non-VMware platform. """ - def setUp(self): - super(TestDataSourceVMwareGuestInfo_InvalidPlatform, self).setUp() - self.tmp = self.tmp_dir() - self.create_system_files() - - def create_system_files(self): - rootd = self.tmp_dir() + @pytest.fixture(autouse=True) + def create_files(self, fake_filesystem, tmpdir): populate_dir( - rootd, - { - DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID, - }, + str(tmpdir), + {DataSourceVMware.PRODUCT_UUID_FILE_PATH: PRODUCT_UUID}, ) - self.assertTrue(self.reRoot(rootd)) @mock.patch("cloudinit.sources.DataSourceVMware.guestinfo_get_value") - def test_ds_invalid_on_non_vmware_platform(self, m_fn): + def test_ds_invalid_on_non_vmware_platform(self, m_fn, DS): system_type = dmi.read_dmi_data("system-product-name") - self.assertEqual(system_type, None) + assert system_type is None m_fn.side_effect = [VMW_METADATA_YAML, "", "", "", "", ""] - ds = get_ds(self.tmp) + ds = DS(settings.CFG_BUILTIN) ret = ds.get_data() - self.assertFalse(ret) + assert not ret -class TestDataSourceVMwareIMC(CiTestCase): +class TestDataSourceVMwareIMC: """ Test the VMware Guest OS Customization transport """ - with_logs = True + def test_get_subplatform(self, DS, tmpdir): + ds = DS({"disable_vmware_customization": True}) + # Prepare the conf file + conf_file = os.path.join(tmpdir, "test-cust") + conf_content = dedent( + """\ + [CLOUDINIT] + METADATA = test-meta + """ + ) + util.write_file(conf_file, conf_content) + # Prepare the meta data file + metadata_file = os.path.join(tmpdir, "test-meta") + metadata_content = dedent( + """\ + { + "instance-id": "cloud-vm", + "local-hostname": "my-host.domain.com", + "network": { + "version": 2, + "ethernets": { + "eths": { + "match": { + "name": "ens*" + }, + "dhcp4": true + } + } + } + } + """ + ) + util.write_file(metadata_file, metadata_content) + + with mock.patch( + MPATH + "guestcust_util.set_customization_status", + return_value=("msg", b""), + ): + result = wrap_and_call( + "cloudinit.sources.DataSourceVMware", + { + "dmi.read_dmi_data": "vmware", + "util.del_dir": True, + "guestcust_util.search_file": tmpdir, + "guestcust_util.wait_for_cust_cfg_file": conf_file, + "guestcust_util.get_imc_dir_path": tmpdir, + }, + ds._get_data, + ) + assert result + + assert ds.subplatform == "%s (%s)" % ( + DataSourceVMware.DATA_ACCESS_METHOD_IMC, + DataSourceVMware.get_imc_key_name("metadata"), + ) - def setUp(self): - super(TestDataSourceVMwareIMC, self).setUp() - self.datasource = DataSourceVMware.DataSourceVMware - self.tdir = self.tmp_dir() + # Test to ensure that network is configured from metadata on each boot. + assert ( + DataSourceVMware.DEFAULT_UPDATE_EVENTS[EventScope.NETWORK] + == ds.default_update_events[EventScope.NETWORK] + ) + assert ( + DataSourceVMware.SUPPORTED_UPDATE_EVENTS[EventScope.NETWORK] + == ds.supported_update_events[EventScope.NETWORK] + ) - def test_get_data_false_on_none_dmi_data(self): + def test_get_data_false_on_none_dmi_data(self, caplog, DS): """When dmi for system-product-name is None, get_data returns False.""" - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource(sys_cfg={}, distro={}, paths=paths) + ds = DS({}) result = wrap_and_call( "cloudinit.sources.DataSourceVMware", { @@ -757,24 +902,23 @@ def test_get_data_false_on_none_dmi_data(self): }, ds.get_data, ) - self.assertFalse(result, "Expected False return from ds.get_data") - self.assertIn("No system-product-name found", self.logs.getvalue()) + assert not result, "Expected False return from ds.get_data" + assert "No system-product-name found" in caplog.text - def test_get_imc_data_vmware_customization_disabled(self): + def test_get_imc_data_vmware_customization_disabled( + self, caplog, DS, tmpdir + ): """ When vmware customization is disabled via sys_cfg and allow_raw_data is disabled via ds_cfg, log a message. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={ + ds = DS( + { "disable_vmware_customization": True, "datasource": {"VMware": {"allow_raw_data": False}}, }, - distro={}, - paths=paths, ) - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [MISC] @@ -789,27 +933,23 @@ def test_get_imc_data_vmware_customization_disabled(self): }, ds.get_imc_data_fn, ) - self.assertEqual(result, (None, None, None)) - self.assertIn( - "Customization for VMware platform is disabled", - self.logs.getvalue(), - ) + assert result == (None, None, None) + assert "Customization for VMware platform is disabled" in caplog.text - def test_get_imc_data_vmware_customization_sys_cfg_disabled(self): + def test_get_imc_data_vmware_customization_sys_cfg_disabled( + self, caplog, DS, tmpdir + ): """ When vmware customization is disabled via sys_cfg and no meta data is found, log a message. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={ + ds = DS( + { "disable_vmware_customization": True, "datasource": {"VMware": {"allow_raw_data": True}}, }, - distro={}, - paths=paths, ) - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [MISC] @@ -822,34 +962,30 @@ def test_get_imc_data_vmware_customization_sys_cfg_disabled(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, }, ds.get_imc_data_fn, ) - self.assertEqual(result, (None, None, None)) - self.assertIn( - "No allowed customization configuration data found", - self.logs.getvalue(), + assert result == (None, None, None) + assert ( + "No allowed customization configuration data found" in caplog.text ) - def test_get_imc_data_allow_raw_data_disabled(self): + def test_get_imc_data_allow_raw_data_disabled(self, caplog, DS, tmpdir): """ When allow_raw_data is disabled via ds_cfg and meta data is found, log a message. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={ + ds = DS( + { "disable_vmware_customization": False, "datasource": {"VMware": {"allow_raw_data": False}}, }, - distro={}, - paths=paths, ) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CLOUDINIT] @@ -862,29 +998,26 @@ def test_get_imc_data_allow_raw_data_disabled(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, }, ds.get_imc_data_fn, ) - self.assertEqual(result, (None, None, None)) - self.assertIn( - "No allowed customization configuration data found", - self.logs.getvalue(), + assert result == (None, None, None) + assert ( + "No allowed customization configuration data found" in caplog.text ) - def test_get_imc_data_vmware_customization_enabled(self): + @pytest.mark.allow_subp_for("vmware-rpctool") + def test_get_imc_data_vmware_customization_enabled( + self, caplog, DS, tmpdir + ): """ When cloud-init workflow for vmware is enabled via sys_cfg log a message. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": False}, - distro={}, - paths=paths, - ) - conf_file = self.tmp_path("test-cust", self.tdir) + ds = DS({"disable_vmware_customization": False}) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CUSTOM-SCRIPT] @@ -903,31 +1036,23 @@ def test_get_imc_data_vmware_customization_enabled(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, }, ds.get_imc_data_fn, ) - self.assertEqual(result, (None, None, None)) - custom_script = self.tmp_path("test-script", self.tdir) - self.assertIn( - "Script %s not found!!" % custom_script, - self.logs.getvalue(), - ) + assert result == (None, None, None) + custom_script = os.path.join(tmpdir, "test-script") + assert "Script %s not found!!" % custom_script in caplog.text - def test_get_imc_data_cust_script_disabled(self): + def test_get_imc_data_cust_script_disabled(self, caplog, DS, tmpdir): """ If custom script is disabled by VMware tools configuration, log a message. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": False}, - distro={}, - paths=paths, - ) + ds = DS({"disable_vmware_customization": False}) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CUSTOM-SCRIPT] @@ -938,7 +1063,7 @@ def test_get_imc_data_cust_script_disabled(self): ) util.write_file(conf_file, conf_content) # Prepare the custom sript - customscript = self.tmp_path("test-script", self.tdir) + customscript = os.path.join(tmpdir, "test-script") util.write_file(customscript, "This is the post cust script") with mock.patch( @@ -954,30 +1079,22 @@ def test_get_imc_data_cust_script_disabled(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, }, ds.get_imc_data_fn, ) - self.assertEqual(result, (None, None, None)) - self.assertIn( - "Custom script is disabled by VM Administrator", - self.logs.getvalue(), - ) + assert result == (None, None, None) + assert "Custom script is disabled by VM Administrator" in caplog.text - def test_get_imc_data_cust_script_enabled(self): + def test_get_imc_data_cust_script_enabled(self, caplog, DS, tmpdir): """ If custom script is enabled by VMware tools configuration, execute the script. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": False}, - distro={}, - paths=paths, - ) + ds = DS({"disable_vmware_customization": False}) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CUSTOM-SCRIPT] @@ -1003,32 +1120,26 @@ def test_get_imc_data_cust_script_enabled(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, }, ds.get_imc_data_fn, ) - self.assertEqual(result, (None, None, None)) + assert result == (None, None, None) # Verify custom script is trying to be executed - custom_script = self.tmp_path("test-script", self.tdir) - self.assertIn( - "Script %s not found!!" % custom_script, - self.logs.getvalue(), - ) + custom_script = os.path.join(tmpdir, "test-script") + assert "Script %s not found!!" % custom_script in caplog.text - def test_get_imc_data_force_run_post_script_is_yes(self): + def test_get_imc_data_force_run_post_script_is_yes( + self, caplog, DS, tmpdir + ): """ If DEFAULT-RUN-POST-CUST-SCRIPT is yes, custom script could run if enable-custom-scripts is not defined in VM Tools configuration """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": False}, - distro={}, - paths=paths, - ) + ds = DS({"disable_vmware_customization": False}) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") # set DEFAULT-RUN-POST-CUST-SCRIPT = yes so that enable-custom-scripts # default value is TRUE conf_content = dedent( @@ -1060,33 +1171,25 @@ def my_get_tools_config(*args, **kwargs): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, }, ds.get_imc_data_fn, ) - self.assertEqual(result, (None, None, None)) + assert result == (None, None, None) # Verify custom script still runs although it is # disabled by VMware Tools - custom_script = self.tmp_path("test-script", self.tdir) - self.assertIn( - "Script %s not found!!" % custom_script, - self.logs.getvalue(), - ) + custom_script = os.path.join(tmpdir, "test-script") + assert "Script %s not found!!" % custom_script in caplog.text - def test_get_data_cloudinit_metadata_json(self): + def test_get_data_cloudinit_metadata_json(self, DS, tmpdir): """ Test metadata can be loaded to cloud-init metadata and network. The metadata format is json. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": True}, - distro={}, - paths=paths, - ) + ds = DS({"disable_vmware_customization": True}) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CLOUDINIT] @@ -1095,7 +1198,7 @@ def test_get_data_cloudinit_metadata_json(self): ) util.write_file(conf_file, conf_content) # Prepare the meta data file - metadata_file = self.tmp_path("test-meta", self.tdir) + metadata_file = os.path.join(tmpdir, "test-meta") metadata_content = dedent( """\ { @@ -1126,31 +1229,26 @@ def test_get_data_cloudinit_metadata_json(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, - "guestcust_util.get_imc_dir_path": self.tdir, + "guestcust_util.get_imc_dir_path": tmpdir, }, ds._get_data, ) - self.assertTrue(result) - self.assertEqual("cloud-vm", ds.metadata["instance-id"]) - self.assertEqual("my-host.domain.com", ds.metadata["local-hostname"]) - self.assertEqual(2, ds.network_config["version"]) - self.assertTrue(ds.network_config["ethernets"]["eths"]["dhcp4"]) + assert result + assert "cloud-vm" == ds.metadata["instance-id"] + assert "my-host.domain.com" == ds.metadata["local-hostname"] + assert 2 == ds.network_config["version"] + assert ds.network_config["ethernets"]["eths"]["dhcp4"] - def test_get_data_cloudinit_metadata_yaml(self): + def test_get_data_cloudinit_metadata_yaml(self, DS, tmpdir): """ Test metadata can be loaded to cloud-init metadata and network. The metadata format is yaml. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": True}, - distro={}, - paths=paths, - ) + ds = DS({"disable_vmware_customization": True}) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CLOUDINIT] @@ -1159,7 +1257,7 @@ def test_get_data_cloudinit_metadata_yaml(self): ) util.write_file(conf_file, conf_content) # Prepare the meta data file - metadata_file = self.tmp_path("test-meta", self.tdir) + metadata_file = os.path.join(tmpdir, "test-meta") metadata_content = dedent( """\ instance-id: cloud-vm @@ -1184,31 +1282,28 @@ def test_get_data_cloudinit_metadata_yaml(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, - "guestcust_util.get_imc_dir_path": self.tdir, + "guestcust_util.get_imc_dir_path": tmpdir, }, ds._get_data, ) - self.assertTrue(result) - self.assertEqual("cloud-vm", ds.metadata["instance-id"]) - self.assertEqual("my-host.domain.com", ds.metadata["local-hostname"]) - self.assertEqual(2, ds.network_config["version"]) - self.assertTrue(ds.network_config["ethernets"]["nics"]["dhcp4"]) - - def test_get_imc_data_cloudinit_metadata_not_valid(self): + assert result + assert "cloud-vm" == ds.metadata["instance-id"] + assert "my-host.domain.com" == ds.metadata["local-hostname"] + assert 2 == ds.network_config["version"] + assert ds.network_config["ethernets"]["nics"]["dhcp4"] + + def test_get_imc_data_cloudinit_metadata_not_valid( + self, caplog, DS, tmpdir + ): """ Test metadata is not JSON or YAML format, log a message """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": True}, - distro={}, - paths=paths, - ) + ds = DS({"disable_vmware_customization": True}) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CLOUDINIT] @@ -1218,7 +1313,7 @@ def test_get_imc_data_cloudinit_metadata_not_valid(self): util.write_file(conf_file, conf_content) # Prepare the meta data file - metadata_file = self.tmp_path("test-meta", self.tdir) + metadata_file = os.path.join(tmpdir, "test-meta") metadata_content = "[This is not json or yaml format]a=b" util.write_file(metadata_file, metadata_content) @@ -1231,30 +1326,26 @@ def test_get_imc_data_cloudinit_metadata_not_valid(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, - "guestcust_util.get_imc_dir_path": self.tdir, + "guestcust_util.get_imc_dir_path": tmpdir, }, ds.get_data, ) - self.assertFalse(result) - self.assertIn( - "expected '', but found ''", - self.logs.getvalue(), + assert not result + assert ( + "expected '', but found ''" in caplog.text ) - def test_get_imc_data_cloudinit_metadata_not_found(self): + def test_get_imc_data_cloudinit_metadata_not_found( + self, caplog, DS, tmpdir + ): """ Test metadata file can't be found, log a message """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": True}, - distro={}, - paths=paths, - ) + ds = DS({"disable_vmware_customization": True}) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CLOUDINIT] @@ -1273,28 +1364,23 @@ def test_get_imc_data_cloudinit_metadata_not_found(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, - "guestcust_util.get_imc_dir_path": self.tdir, + "guestcust_util.get_imc_dir_path": tmpdir, }, ds.get_imc_data_fn, ) - self.assertEqual(result, (None, None, None)) - self.assertIn("Meta data file is not found", self.logs.getvalue()) + assert result == (None, None, None) + assert "Meta data file is not found" in caplog.text - def test_get_data_cloudinit_userdata(self): + def test_get_data_cloudinit_userdata(self, DS, tmpdir): """ Test user data can be loaded to cloud-init user data. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": False}, - distro={}, - paths=paths, - ) + ds = DS({"disable_vmware_customization": False}) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CLOUDINIT] @@ -1305,7 +1391,7 @@ def test_get_data_cloudinit_userdata(self): util.write_file(conf_file, conf_content) # Prepare the meta data file - metadata_file = self.tmp_path("test-meta", self.tdir) + metadata_file = os.path.join(tmpdir, "test-meta") metadata_content = dedent( """\ instance-id: cloud-vm @@ -1322,7 +1408,7 @@ def test_get_data_cloudinit_userdata(self): util.write_file(metadata_file, metadata_content) # Prepare the user data file - userdata_file = self.tmp_path("test-user", self.tdir) + userdata_file = os.path.join(tmpdir, "test-user") userdata_content = "This is the user data" util.write_file(userdata_file, userdata_content) @@ -1335,29 +1421,26 @@ def test_get_data_cloudinit_userdata(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, - "guestcust_util.get_imc_dir_path": self.tdir, + "guestcust_util.get_imc_dir_path": tmpdir, }, ds._get_data, ) - self.assertTrue(result) - self.assertEqual("cloud-vm", ds.metadata["instance-id"]) - self.assertEqual(userdata_content, ds.userdata_raw) + assert result + assert "cloud-vm" == ds.metadata["instance-id"] + assert userdata_content == ds.userdata_raw - def test_get_imc_data_cloudinit_userdata_not_found(self): + def test_get_imc_data_cloudinit_userdata_not_found( + self, caplog, DS, tmpdir + ): """ Test userdata file can't be found. """ - paths = helpers.Paths({"cloud_dir": self.tdir}) - ds = self.datasource( - sys_cfg={"disable_vmware_customization": True}, - distro={}, - paths=paths, - ) + ds = DS({"disable_vmware_customization": True}) # Prepare the conf file - conf_file = self.tmp_path("test-cust", self.tdir) + conf_file = os.path.join(tmpdir, "test-cust") conf_content = dedent( """\ [CLOUDINIT] @@ -1368,7 +1451,7 @@ def test_get_imc_data_cloudinit_userdata_not_found(self): util.write_file(conf_file, conf_content) # Prepare the meta data file - metadata_file = self.tmp_path("test-meta", self.tdir) + metadata_file = os.path.join(tmpdir, "test-meta") metadata_content = dedent( """\ instance-id: cloud-vm @@ -1395,61 +1478,54 @@ def test_get_imc_data_cloudinit_userdata_not_found(self): { "dmi.read_dmi_data": "vmware", "util.del_dir": True, - "guestcust_util.search_file": self.tdir, + "guestcust_util.search_file": tmpdir, "guestcust_util.wait_for_cust_cfg_file": conf_file, - "guestcust_util.get_imc_dir_path": self.tdir, + "guestcust_util.get_imc_dir_path": tmpdir, }, ds.get_imc_data_fn, ) - self.assertEqual(result, (None, None, None)) - self.assertIn("Userdata file is not found", self.logs.getvalue()) + assert result == (None, None, None) + assert "Userdata file is not found" in caplog.text -class TestDataSourceVMwareIMC_MarkerFiles(CiTestCase): - def setUp(self): - super(TestDataSourceVMwareIMC_MarkerFiles, self).setUp() - self.tdir = self.tmp_dir() +class TestDataSourceVMwareIMC_MarkerFiles: - def test_false_when_markerid_none(self): + def test_false_when_markerid_none(self, tmpdir): """Return False when markerid provided is None.""" - self.assertFalse( - guestcust_util.check_marker_exists( - markerid=None, marker_dir=self.tdir - ) + assert not guestcust_util.check_marker_exists( + markerid=None, marker_dir=tmpdir ) - def test_markerid_file_exist(self): + def test_markerid_file_exist(self, tmpdir): """Return False when markerid file path does not exist, True otherwise.""" - self.assertFalse(guestcust_util.check_marker_exists("123", self.tdir)) - marker_file = self.tmp_path(".markerfile-123.txt", self.tdir) + assert not guestcust_util.check_marker_exists("123", tmpdir) + marker_file = os.path.join(tmpdir, ".markerfile-123.txt") util.write_file(marker_file, "") - self.assertTrue(guestcust_util.check_marker_exists("123", self.tdir)) + assert guestcust_util.check_marker_exists("123", tmpdir) - def test_marker_file_setup(self): + def test_marker_file_setup(self, tmpdir): """Test creation of marker files.""" - markerfilepath = self.tmp_path(".markerfile-hi.txt", self.tdir) - self.assertFalse(os.path.exists(markerfilepath)) - guestcust_util.setup_marker_files(marker_id="hi", marker_dir=self.tdir) - self.assertTrue(os.path.exists(markerfilepath)) + markerfilepath = os.path.join(tmpdir, ".markerfile-hi.txt") + assert not os.path.exists(markerfilepath) + guestcust_util.setup_marker_files(marker_id="hi", marker_dir=tmpdir) + assert os.path.exists(markerfilepath) -def assert_metadata(test_obj, ds, metadata): - test_obj.assertEqual(metadata.get("instance-id"), ds.get_instance_id()) - test_obj.assertEqual( - metadata.get("local-hostname"), ds.get_hostname().hostname - ) +def assert_metadata(ds, metadata): + assert metadata.get("instance-id") == ds.get_instance_id() + assert metadata.get("local-hostname") == ds.get_hostname().hostname expected_public_keys = metadata.get("public_keys") if not isinstance(expected_public_keys, list): expected_public_keys = [expected_public_keys] - test_obj.assertEqual(expected_public_keys, ds.get_public_ssh_keys()) - test_obj.assertIsInstance(ds.get_public_ssh_keys(), list) + assert expected_public_keys == ds.get_public_ssh_keys() + assert isinstance(ds.get_public_ssh_keys(), list) -def get_ds(temp_dir): - ds = DataSourceVMware.DataSourceVMware( - settings.CFG_BUILTIN, None, helpers.Paths({"run_dir": temp_dir}) +@pytest.fixture +def DS(paths): + return lambda sys_cfg: DataSourceVMware.DataSourceVMware( + sys_cfg, {}, paths ) - return ds diff --git a/tests/unittests/sources/test_wsl.py b/tests/unittests/sources/test_wsl.py index ebe01827..9c062706 100644 --- a/tests/unittests/sources/test_wsl.py +++ b/tests/unittests/sources/test_wsl.py @@ -15,8 +15,7 @@ from cloudinit import util from cloudinit.sources import DataSourceWSL as wsl -from tests.unittests.distros import _get_distro -from tests.unittests.helpers import does_not_raise, mock +from tests.unittests.helpers import does_not_raise, get_distro, mock INSTANCE_NAME = "Noble-MLKit" GOOD_MOUNTS = { @@ -182,18 +181,29 @@ def test_candidate_files(self, m_gld, linux_distro_value, files): assert files == wsl.candidate_user_data_file_names(INSTANCE_NAME) @pytest.mark.parametrize( - "md_content,raises,errors,warnings,md_expected", + "md_content,is_from_pro,raises,errors,warnings,md_expected", ( pytest.param( None, + False, does_not_raise(), [], [], {"instance-id": "iid-datasource-wsl"}, id="default_md_on_no_md_file", ), + pytest.param( + '{"instance-id":"iid-load-from-pro"}', + True, + does_not_raise(), + [], + [], + {"instance-id": "iid-load-from-pro"}, + id="metadata_from_pro", + ), pytest.param( "{}", + False, pytest.raises( ValueError, match=( @@ -207,6 +217,7 @@ def test_candidate_files(self, m_gld, linux_distro_value, files): ), pytest.param( "{", + True, pytest.raises( ValueError, match=( @@ -221,15 +232,29 @@ def test_candidate_files(self, m_gld, linux_distro_value, files): ), ) def test_load_instance_metadata( - self, md_content, raises, errors, warnings, md_expected, tmpdir, caplog + self, + md_content, + is_from_pro, + raises, + errors, + warnings, + md_expected, + tmpdir, + caplog, ): """meta-data file is optional. Errors are raised on invalid content.""" + path = ".cloud-init" + if is_from_pro: + path = ".ubuntupro/.cloud-init" + if md_content is not None: - tmpdir.join("myinstance.meta-data").write(md_content) + dir = tmpdir.join(path) + os.makedirs(dir) + dir.join("myinstance.meta-data").write(md_content) with caplog.at_level(logging.WARNING): with raises: assert md_expected == wsl.load_instance_metadata( - PurePath(tmpdir), "myinstance" + tmpdir, "myinstance" ) warning_logs = "\n".join( [ @@ -256,6 +281,28 @@ def test_load_instance_metadata( else: assert "" == error_logs + @mock.patch("cloudinit.util.subp.subp") + def test_landscape_supports_field(self, m_subp): + LANDSCAPE_HELP_OUTPUT = """\ + Usage: landscape-config [options] + + Options: + --version show program's version number and exit + -h, --help show this help message and exit + --installation-request-id Only set this value if this computer is a + instance managed by Landscape, in which + it to be the request id that Landscape + to the installation activity for the host + --access-group=ACCESS_GROUP + Suggested access group for this computer. + --tags=TAGS Comma separated list of tag names to be sent + to the server. + """ + m_subp.return_value = util.subp.SubpResult(LANDSCAPE_HELP_OUTPUT, "") + assert wsl.landscape_supports_field("non_sense") is False + assert wsl.landscape_supports_field("tags") is True + assert wsl.landscape_supports_field("installation-request-id") is True + SAMPLE_CFG = {"datasource_list": ["NoCloud", "WSL"]} @@ -317,11 +364,11 @@ def test_merged_data_excludes_empty_or_none( if agent_yaml is not None: agent_path = tmpdir.join("agent.yaml") agent_path.write(agent_yaml) - agent_data = wsl.ConfigData(agent_path) + agent_data = wsl.ConfigData(agent_path, "") if landscape_user_data is not None: landscape_ud_path = tmpdir.join("instance_name.user_data") landscape_ud_path.write(landscape_user_data) - user_data = wsl.ConfigData(landscape_ud_path) + user_data = wsl.ConfigData(landscape_ud_path, "") assert expected == wsl.merge_agent_landscape_data( agent_data, user_data ) @@ -342,6 +389,10 @@ def setup(self, mocker, tmpdir): "cloudinit.sources.DataSourceWSL.subp.which", return_value="/usr/bin/wslpath", ) + mocker.patch( + "cloudinit.sources.DataSourceWSL.landscape_supports_field", + return_value=True, + ) def test_metadata_id_default(self, tmpdir, paths): """ @@ -351,7 +402,7 @@ def test_metadata_id_default(self, tmpdir, paths): ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) ds.get_data() @@ -374,7 +425,7 @@ def test_metadata_id(self, tmpdir, paths): ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) ds.get_data() @@ -390,7 +441,7 @@ def test_get_data_cc(self, m_lsb_release, paths, tmpdir): ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -413,7 +464,7 @@ def test_get_data_sh(self, m_lsb_release, tmpdir, paths): data_path.write(f"#!/bin/sh\n{COMMAND}\n") ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -442,7 +493,7 @@ def test_get_data_jinja(self, m_lsb_release, paths, tmpdir): ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -478,7 +529,7 @@ def test_get_data_x( ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -527,7 +578,7 @@ def test_data_precedence(self, m_get_linux_dist, tmpdir, paths): # Run the datasource ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -551,9 +602,9 @@ def test_data_precedence(self, m_get_linux_dist, tmpdir, paths): def test_interaction_with_pro(self, m_get_linux_dist, tmpdir, paths): """Validates the interaction of user-data and Pro For WSL agent data""" - m_get_linux_dist.return_value = SAMPLE_LINUX_DISTRO + m_get_linux_dist.return_value = ("ubuntu", "25.10", "plucky") - user_file = tmpdir.join(".cloud-init", "ubuntu-24.04.user-data") + user_file = tmpdir.join(".cloud-init", "ubuntu-25.10.user-data") user_file.dirpath().mkdir() user_file.write("#cloud-config\nwrite_files:\n- path: /etc/wsl.conf") @@ -575,11 +626,16 @@ def test_interaction_with_pro(self, m_get_linux_dist, tmpdir, paths): ubuntu_pro: token: testtoken""" ) + SAMPLE_ID = "Nice-ID" + agent_metadata_path = ubuntu_pro_tmp.join(f"{INSTANCE_NAME}.meta-data") + agent_metadata_path.write( + f'{{"instance-id":"{SAMPLE_ID}"}}', + ) # Run the datasource ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -596,6 +652,8 @@ def test_interaction_with_pro(self, m_get_linux_dist, tmpdir, paths): assert "ubuntu_pro" in userdata assert "landscape" in userdata assert "agenttest" in userdata + assert "installation_request_id" in userdata + assert SAMPLE_ID in userdata @mock.patch("cloudinit.util.get_linux_distro") def test_landscape_vs_local_user(self, m_get_linux_dist, tmpdir, paths): @@ -620,7 +678,7 @@ def test_landscape_vs_local_user(self, m_get_linux_dist, tmpdir, paths): # Run the datasource ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -676,7 +734,7 @@ def test_landscape_provided_data(self, m_get_linux_dist, tmpdir, paths): # Run the datasource ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -734,7 +792,7 @@ def test_landscape_empty_data(self, m_get_linux_dist, tmpdir, paths): # Run the datasource ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -783,7 +841,7 @@ def test_landscape_shell_script(self, m_get_linux_dist, tmpdir, paths): # Run the datasource ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -842,7 +900,7 @@ def test_with_landscape_no_tags(self, m_get_linux_dist, tmpdir, paths): # Run the datasource ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -891,7 +949,7 @@ def test_with_no_tags_at_all(self, m_get_linux_dist, tmpdir, paths): # Run the datasource ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) @@ -939,7 +997,7 @@ def test_with_no_client_subkey(self, m_get_linux_dist, tmpdir, paths): # Run the datasource ds = wsl.DataSourceWSL( sys_cfg=SAMPLE_CFG, - distro=_get_distro("ubuntu"), + distro=get_distro("ubuntu"), paths=paths, ) diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 10cacf4b..501009e3 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -4,12 +4,13 @@ import os import shutil import tempfile +from contextlib import ExitStack import pytest from cloudinit import handlers, helpers, settings, url_helper, util from cloudinit.cmd import main -from tests.unittests.helpers import ExitStack, TestCase, mock +from tests.unittests.helpers import TestCase, mock class FakeModule(handlers.Handler): diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 0be75e65..d9f392f8 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -277,6 +277,22 @@ def test_subcommand_parser(self, subcommand, mock_status_wrapper): self._call_main(["cloud-init", subcommand, "-h"]) assert f"usage: cloud-init {subcommand}" in out.getvalue() + @pytest.mark.parametrize( + "subcommand", + [ + "clean", + "collect-logs", + "status", + ], + ) + def test_subcommand_parser_shows_usage(self, subcommand, capsys): + """cloud-init `subcommand` shows usage on error.""" + # Provide --invalid-arg to `subcommand` to trigger error. + exit_code = self._call_main(["cloud-init", subcommand, "--invalid"]) + _out, err = capsys.readouterr() + assert f"usage: cloud-init {subcommand}" in err + assert 2 == exit_code + @pytest.mark.parametrize( "args,expected_subcommands", [ @@ -330,6 +346,32 @@ def test_features_hook_subcommand(self, m_features): assert False is parseargs.debug assert False is parseargs.force + def test_all_stages_with_tty(self, mocker): + """Ensure all stages get called when using a tty.""" + mocker.patch("cloudinit.cmd.main.os.isatty", return_value=True) + mocker.patch("cloudinit.cmd.main.sys.stdin.fileno") + mocker.patch("cloudinit.cmd.main.socket.sd_notify") + mocker.patch("cloudinit.cmd.main.socket.os.makedirs") + mocker.patch("cloudinit.cmd.main.socket.os.remove") + mocker.patch("cloudinit.cmd.main.socket.socket") + m_sub_main = mocker.patch( + "cloudinit.cmd.main.sub_main", return_value=0 + ) + + self._call_main(["cloud-init", "--all-stages"]) + assert m_sub_main.call_count == 4 + assert m_sub_main.call_args_list[0][0][0].subcommand == "init" + assert m_sub_main.call_args_list[0][0][0].local is True + + assert m_sub_main.call_args_list[1][0][0].subcommand == "init" + assert m_sub_main.call_args_list[1][0][0].local is False + + assert m_sub_main.call_args_list[2][0][0].subcommand == "modules" + assert m_sub_main.call_args_list[2][0][0].mode == "config" + + assert m_sub_main.call_args_list[3][0][0].subcommand == "modules" + assert m_sub_main.call_args_list[3][0][0].mode == "final" + class TestSignalHandling: @mock.patch("cloudinit.cmd.main.atomic_helper.write_json") diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 78e6e4bd..204765e4 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -472,20 +472,7 @@ def test_mime_text_plain(self, init_tmp, caplog): init_tmp.paths.get_ipath("cloud_config"), "", 0o600 ) - # Since features are intended to be overridden downstream, mock them - # all here so new feature flags don't require a new change to this - # unit test. - @mock.patch.multiple( - "cloudinit.features", - ERROR_ON_USER_DATA_FAILURE=True, - ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES=True, - EXPIRE_APPLIES_TO_HASHED_USERS=False, - NETPLAN_CONFIG_ROOT_READ_ONLY=True, - DEPRECATION_INFO_BOUNDARY="devel", - NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, - APT_DEB822_SOURCE_LIST_FILE=True, - ) - def test_shellscript(self, init_tmp, tmpdir, caplog): + def test_shellscript(self, init_tmp, caplog): """Raw text starting #!/bin/sh is treated as script.""" script = "#!/bin/sh\necho hello\n" init_tmp.datasource = FakeDataSource(script) @@ -507,16 +494,32 @@ def test_shellscript(self, init_tmp, tmpdir, caplog): mock.call(init_tmp.paths.get_ipath("cloud_config"), "", 0o600), ] ) + + def test_expected_artifacts(self, init_tmp, tmpdir, caplog, mocker): + """Test combined_cloud_config and instance_data_sensitive contents.""" + init_tmp.datasource = FakeDataSource() + + mocker.patch("cloudinit.util.write_file") + mocker.patch.object(init_tmp, "_reset") + mocker.patch( + "cloudinit.features.get_features", + return_value={ + "ERROR_ON_USER_DATA_FAILURE": True, + "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES": False, + "SOME_FAKE_FEATURE": True, + }, + ) + + with caplog.at_level(logging.WARNING): + init_tmp.fetch() + init_tmp.consume_data() + assert caplog.records == [] # No warnings + expected = { "features": { "ERROR_ON_USER_DATA_FAILURE": True, - "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES": True, - "EXPIRE_APPLIES_TO_HASHED_USERS": False, - "NETPLAN_CONFIG_ROOT_READ_ONLY": True, - "DEPRECATION_INFO_BOUNDARY": "devel", - "NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH": False, - "APT_DEB822_SOURCE_LIST_FILE": True, - "MANUAL_NETWORK_WAIT": False, + "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES": False, + "SOME_FAKE_FEATURE": True, }, "system_info": { "default_user": {"name": "ubuntu"}, @@ -770,26 +773,21 @@ def test_include_bad_url_no_fail( assert cc.get("included") is True -class TestUDProcess(helpers.ResourceUsingTestCase): - def test_bytes_in_userdata(self): +class TestUDProcess: + def test_bytes_in_userdata(self, ud_proc): msg = b"#cloud-config\napt_update: True\n" - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(msg) - self.assertTrue(count_messages(message) == 1) + assert count_messages(message) == 1 - def test_string_in_userdata(self): + def test_string_in_userdata(self, ud_proc): msg = "#cloud-config\napt_update: True\n" - - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(msg) - self.assertTrue(count_messages(message) == 1) + assert count_messages(message) == 1 - def test_compressed_in_userdata(self): + def test_compressed_in_userdata(self, ud_proc): msg = gzip_text("#cloud-config\napt_update: True\n") - - ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(msg) - self.assertTrue(count_messages(message) == 1) + assert count_messages(message) == 1 class TestConvertString(helpers.TestCase): diff --git a/tests/unittests/test_dmi.py b/tests/unittests/test_dmi.py index 61fd210a..e8eeb372 100644 --- a/tests/unittests/test_dmi.py +++ b/tests/unittests/test_dmi.py @@ -1,31 +1,25 @@ import os -import shutil -import tempfile from unittest import mock import pytest from cloudinit import dmi, subp, util from cloudinit.subp import SubpResult -from tests.unittests import helpers - - -class TestReadDMIData(helpers.FilesystemMockingTestCase): - def setUp(self): - super(TestReadDMIData, self).setUp() - self.new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.new_root) - self.reRoot(self.new_root) - p = mock.patch("cloudinit.dmi.is_container", return_value=False) - self.addCleanup(p.stop) - self._m_is_container = p.start() - p = mock.patch("cloudinit.dmi.is_FreeBSD", return_value=False) - self.addCleanup(p.stop) - self._m_is_FreeBSD = p.start() - - p = mock.patch("cloudinit.dmi.is_OpenBSD", return_value=False) - self.addCleanup(p.stop) - self._m_is_OpenBSD = p.start() + + +@pytest.mark.usefixtures("fake_filesystem") +class TestReadDMIData: + @pytest.fixture(autouse=True) + def common_mocks(self, mocker): + self.m_is_container = mocker.patch( + "cloudinit.dmi.is_container", return_value=False + ) + self.m_is_freebsd = mocker.patch( + "cloudinit.dmi.is_FreeBSD", return_value=False + ) + self.m_is_openbsd = mocker.patch( + "cloudinit.dmi.is_OpenBSD", return_value=False + ) def _create_sysfs_parent_directory(self): util.ensure_dir(os.path.join("sys", "class", "dmi", "id")) @@ -36,7 +30,7 @@ def _create_sysfs_file(self, key, content): dmi_key = "/sys/class/dmi/id/{0}".format(key) util.write_file(dmi_key, content) - def _configure_dmidecode_return(self, key, content, error=None): + def _configure_dmidecode_return(self, mocker, key, content, error=None): """ In order to test a missing sys path and call outs to dmidecode, this function fakes the results of dmidecode to test the results. @@ -47,14 +41,10 @@ def _dmidecode_subp(cmd) -> SubpResult: raise subp.ProcessExecutionError() return SubpResult(content, error) - self.patched_funcs.enter_context( - mock.patch("cloudinit.dmi.subp.which", side_effect=lambda _: True) - ) - self.patched_funcs.enter_context( - mock.patch("cloudinit.dmi.subp.subp", side_effect=_dmidecode_subp) - ) + mocker.patch("cloudinit.dmi.subp.which", side_effect=lambda _: True) + mocker.patch("cloudinit.dmi.subp.subp", side_effect=_dmidecode_subp) - def _configure_kenv_return(self, key, content, error=None): + def _configure_kenv_return(self, mocker, key, content, error=None): """ In order to test a FreeBSD system call outs to kenv, this function fakes the results of kenv to test the results. @@ -65,11 +55,9 @@ def _kenv_subp(cmd) -> SubpResult: raise subp.ProcessExecutionError() return SubpResult(content, error) - self.patched_funcs.enter_context( - mock.patch("cloudinit.dmi.subp.subp", side_effect=_kenv_subp) - ) + mocker.patch("cloudinit.dmi.subp.subp", side_effect=_kenv_subp) - def _configure_sysctl_return(self, key, content, error=None): + def _configure_sysctl_return(self, mocker, key, content, error=None): """ In order to test an OpenBSD system call outs to sysctl, this function fakes the results of kenv to test the results. @@ -80,29 +68,27 @@ def _sysctl_subp(cmd) -> SubpResult: raise subp.ProcessExecutionError() return SubpResult(content, error) - self.patched_funcs.enter_context( - mock.patch("cloudinit.dmi.subp.subp", side_effect=_sysctl_subp) - ) - - def patch_mapping(self, new_mapping): - self.patched_funcs.enter_context( - mock.patch("cloudinit.dmi.DMIDECODE_TO_KERNEL", new_mapping) - ) + mocker.patch("cloudinit.dmi.subp.subp", side_effect=_sysctl_subp) - def test_sysfs_used_with_key_in_mapping_and_file_on_disk(self): - self.patch_mapping( - {"mapped-key": dmi.KernelNames("mapped-value", None, None)} + def test_sysfs_used_with_key_in_mapping_and_file_on_disk(self, mocker): + mocker.patch( + "cloudinit.dmi.DMIDECODE_TO_KERNEL", + {"mapped-key": dmi.KernelNames("mapped-value", None, None)}, ) expected_dmi_value = "sys-used-correctly" self._create_sysfs_file("mapped-value", expected_dmi_value) - self._configure_dmidecode_return("mapped-key", "wrong-wrong-wrong") - self.assertEqual(expected_dmi_value, dmi.read_dmi_data("mapped-key")) + self._configure_dmidecode_return( + mocker, "mapped-key", "wrong-wrong-wrong" + ) + assert expected_dmi_value == dmi.read_dmi_data("mapped-key") - def test_dmidecode_used_if_no_sysfs_file_on_disk(self): - self.patch_mapping({}) + def test_dmidecode_used_if_no_sysfs_file_on_disk(self, mocker): + mocker.patch("cloudinit.dmi.DMIDECODE_TO_KERNEL", {}) self._create_sysfs_parent_directory() expected_dmi_value = "dmidecode-used" - self._configure_dmidecode_return("use-dmidecode", expected_dmi_value) + self._configure_dmidecode_return( + mocker, "use-dmidecode", expected_dmi_value + ) with mock.patch("cloudinit.util.os.uname") as m_uname: m_uname.return_value = ( "x-sysname", @@ -111,17 +97,15 @@ def test_dmidecode_used_if_no_sysfs_file_on_disk(self): "x-version", "x86_64", ) - self.assertEqual( - expected_dmi_value, dmi.read_dmi_data("use-dmidecode") - ) + expected_dmi_value == dmi.read_dmi_data("use-dmidecode") - def test_dmidecode_not_used_on_arm(self): - self.patch_mapping({}) + def test_dmidecode_not_used_on_arm(self, mocker): + mocker.patch("cloudinit.dmi.DMIDECODE_TO_KERNEL", {}) print("current =%s", subp) self._create_sysfs_parent_directory() dmi_val = "from-dmidecode" dmi_name = "use-dmidecode" - self._configure_dmidecode_return(dmi_name, dmi_val) + self._configure_dmidecode_return(mocker, dmi_name, dmi_val) print("now =%s", subp) expected = {"armel": None, "aarch64": dmi_val, "x86_64": dmi_val} @@ -140,19 +124,17 @@ def test_dmidecode_not_used_on_arm(self): ) print("now2 =%s", subp) found[arch] = dmi.read_dmi_data(dmi_name) - self.assertEqual(expected, found) + assert expected == found - def test_none_returned_if_neither_source_has_data(self): - self.patch_mapping({}) - self._configure_dmidecode_return("key", "value") - self.assertIsNone(dmi.read_dmi_data("expect-fail")) + def test_none_returned_if_neither_source_has_data(self, mocker): + mocker.patch("cloudinit.dmi.DMIDECODE_TO_KERNEL", {}) + self._configure_dmidecode_return(mocker, "key", "value") + assert dmi.read_dmi_data("expect-fail") is None - def test_none_returned_if_dmidecode_not_in_path(self): - self.patched_funcs.enter_context( - mock.patch.object(subp, "which", lambda _: False) - ) - self.patch_mapping({}) - self.assertIsNone(dmi.read_dmi_data("expect-fail")) + def test_none_returned_if_dmidecode_not_in_path(self, mocker): + mocker.patch.object(subp, "which", lambda _: False) + mocker.patch("cloudinit.dmi.DMIDECODE_TO_KERNEL", {}) + assert dmi.read_dmi_data("expect-fail") is None def test_empty_string_returned_instead_of_foxfox(self): # uninitialized dmi values show as \xff, return empty string @@ -162,41 +144,41 @@ def test_empty_string_returned_instead_of_foxfox(self): dmi_key = "system-product-name" sysfs_key = "product_name" self._create_sysfs_file(sysfs_key, dmi_value) - self.assertEqual(expected, dmi.read_dmi_data(dmi_key)) + assert expected == dmi.read_dmi_data(dmi_key) def test_container_returns_none(self): """In a container read_dmi_data should always return None.""" # first verify we get the value if not in container - self._m_is_container.return_value = False + self.m_is_container.return_value = False key, val = "system-product-name", "my_product" self._create_sysfs_file("product_name", val) - self.assertEqual(val, dmi.read_dmi_data(key)) + assert val == dmi.read_dmi_data(key) # then verify in container returns None - self._m_is_container.return_value = True - self.assertIsNone(dmi.read_dmi_data(key)) + self.m_is_container.return_value = True + assert dmi.read_dmi_data(key) is None def test_container_returns_none_on_unknown(self): """In a container even bogus keys return None.""" - self._m_is_container.return_value = True + self.m_is_container.return_value = True self._create_sysfs_file("product_name", "should-be-ignored") - self.assertIsNone(dmi.read_dmi_data("bogus")) - self.assertIsNone(dmi.read_dmi_data("system-product-name")) + assert dmi.read_dmi_data("bogus") is None + assert dmi.read_dmi_data("system-product-name") is None - def test_freebsd_uses_kenv(self): + def test_freebsd_uses_kenv(self, mocker): """On a FreeBSD system, kenv is called.""" - self._m_is_FreeBSD.return_value = True + self.m_is_freebsd.return_value = True key, val = "system-product-name", "my_product" - self._configure_kenv_return(key, val) - self.assertEqual(dmi.read_dmi_data(key), val) + self._configure_kenv_return(mocker, key, val) + assert dmi.read_dmi_data(key) == val - def test_openbsd_uses_kenv(self): + def test_openbsd_uses_kenv(self, mocker): """On a OpenBSD system, sysctl is called.""" - self._m_is_OpenBSD.return_value = True + self.m_is_openbsd.return_value = True key, val = "system-product-name", "my_product" - self._configure_sysctl_return(key, val) - self.assertEqual(dmi.read_dmi_data(key), val) + self._configure_sysctl_return(mocker, key, val) + assert dmi.read_dmi_data(key) == val class TestSubDMIVars: diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 45894d35..63d415b6 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -18,7 +18,6 @@ from cloudinit.sources import DataSourceSmartOS as ds_smartos from tests.helpers import cloud_init_project_dir from tests.unittests.helpers import ( - CiTestCase, dir2dict, populate_dir, populate_dir_with_ts, @@ -143,7 +142,6 @@ - mcollective - salt-minion - reset_rmc - - refresh_rmc_and_interface - rightscale_userdata - scripts-vendor - scripts-per-once @@ -305,16 +303,15 @@ ) -class DsIdentifyBase(CiTestCase): +class DsIdentifyBase: dsid_path = cloud_init_project_dir("tools/ds-identify") - allowed_subp = ["sh"] # set to true to write out the mocked ds-identify for inspection debug_mode = True def call( self, - rootd=None, + rootd, mocks=None, no_mocks=None, func="main", @@ -337,11 +334,8 @@ def call( if cloudcfg not in files: files[cloudcfg] = DEFAULT_CLOUD_CONFIG - if rootd is None: - rootd = self.tmp_dir() - unset = "_unset" - wrap = self.tmp_path(path="_shwrap", dir=rootd) + wrap = os.path.join(rootd, "_shwrap") populate_dir(rootd, files) # DI_DEFAULT_POLICY* are declared always as to not rely @@ -451,7 +445,7 @@ def write_mock(data): return CallReturn(rc, out, err, cfg, dir2dict(rootd)) - def _call_via_dict(self, data, rootd=None, **kwargs): + def _call_via_dict(self, data, rootd, **kwargs): # return output of self.call with a dict input like VALID_CFG[item] xwargs = {"rootd": rootd} passthrough = ( @@ -471,25 +465,25 @@ def _call_via_dict(self, data, rootd=None, **kwargs): xwargs[k] = kwargs[k] return self.call(**xwargs) - def _test_ds_found(self, name): + def _test_ds_found(self, name, rootd): data = copy.deepcopy(VALID_CFG[name]) dslist = [] for ds in data.pop("ds").split(","): dslist.append(ds.strip()) dslist.append(DS_NONE) - return self._check_via_dict(data, RC_FOUND, dslist=dslist) + return self._check_via_dict(data, rootd, RC_FOUND, dslist=dslist) - def _test_ds_not_found(self, name): + def _test_ds_not_found(self, name, rootd): data = copy.deepcopy(VALID_CFG[name]) - return self._check_via_dict(data, RC_NOT_FOUND) + return self._check_via_dict(data, rootd, RC_NOT_FOUND) - def _check_via_dict(self, data, rc, dslist=None, **kwargs): - ret = self._call_via_dict(data, **kwargs) + def _check_via_dict(self, data, rootd, rc, dslist=None, **kwargs): + ret = self._call_via_dict(data, rootd, **kwargs) good = False try: - self.assertEqual(rc, ret.rc) + assert rc == ret.rc if dslist is not None: - self.assertEqual(dslist, ret.cfg.get("datasource_list")) + assert dslist == ret.cfg.get("datasource_list") good = True finally: if not good: @@ -499,11 +493,12 @@ def _check_via_dict(self, data, rc, dslist=None, **kwargs): return ret +@pytest.mark.allow_subp_for("sh") class TestDsIdentify(DsIdentifyBase): - def test_wb_print_variables(self): + def test_wb_print_variables(self, tmp_path): """_print_info reports an array of discovered variables to stderr.""" data = VALID_CFG["Azure-dmi-detection"] - _, _, err, _, _ = self._call_via_dict(data) + _, _, err, _, _ = self._call_via_dict(data, str(tmp_path)) expected_vars = [ "DMI_PRODUCT_NAME", "DMI_SYS_VENDOR", @@ -525,199 +520,405 @@ def test_wb_print_variables(self): "ON_NOTFOUND", ] for var in expected_vars: - self.assertIn("{0}=".format(var), err) - - @pytest.mark.xfail(reason="GH-4796") - def test_maas_not_detected_1(self): - """Don't incorrectly identify maas - - In ds-identify the function check_config() attempts to parse yaml keys - in bash, but it sometimes introduces false positives. The maas - datasource uses check_config() and the existence of a "MAAS" key to - identify itself (which is a very poor identifier - clouds should have - stricter identifiers). Since the MAAS datasource is at the begining of - the list, this is particularly troublesome and more concerning than - NoCloud false positives, for example. - """ - config = "LXD-kvm-not-MAAS-1" - self._test_ds_found(config) - - def test_maas_not_detected_2(self): - """Don't incorrectly identify maas - - The bug reported in 4794 combined with the previously existing bug - reported in 4796 made for very loose MAAS false-positives. - - In ds-identify the function check_config() attempts to parse yaml keys - in bash, but it sometimes introduces false positives. The maas - datasource uses check_config() and the existence of a "MAAS" key to - identify itself (which is a very poor identifier - clouds should have - stricter identifiers). Since the MAAS datasource is at the begining of - the list, this is particularly troublesome and more concerning than - NoCloud false positives, for example. - """ - config = "LXD-kvm-not-MAAS-2" - self._test_ds_found(config) - - @pytest.mark.xfail(reason="GH-4796") - def test_maas_not_detected_3(self): - """Don't incorrectly identify maas - - The bug reported in 4794 combined with the previously existing bug - reported in 4796 made for very loose MAAS false-positives. - - In ds-identify the function check_config() attempts to parse yaml keys - in bash, but it sometimes introduces false positives. The maas - datasource uses check_config() and the existence of a "MAAS" key to - identify itself (which is a very poor identifier - clouds should have - stricter identifiers). Since the MAAS datasource is at the begining of - the list, this is particularly troublesome and more concerning than - NoCloud false positives, for example. - """ - config = "LXD-kvm-not-MAAS-3" - self._test_ds_found(config) + assert "{0}=".format(var) in err + + @pytest.mark.parametrize( + "config,found", + [ + # Don't incorrectly identify maas + # + # The bug reported in 4794 combined with the previously existing + # bug reported in 4796 made for very loose MAAS false-positives. + # + # In ds-identify the function check_config() attempts to parse yaml + # keys in bash, but it sometimes introduces false positives. The + # maas datasource uses check_config() and the existence of a "MAAS" + # key to identify itself (which is a very poor identifier - clouds + # should have stricter identifiers). Since the MAAS datasource is + # at the begining of the list, this is particularly troublesome and + # more concerning than NoCloud false positives, for example. + pytest.param("LXD-kvm-not-MAAS-2", True, id="mass_not_detected_2"), + # Don't detect incorrect config when invalid datasource_list + # provided + # + # If unparsable list is provided we just ignore it. Some users + # might assume that since the rest of the configuration is yaml + # that multi-line yaml lists are valid (they aren't). When this + # happens, just run ds-identify and figure it out for ourselves + # which platform to run. + pytest.param( + "Azure-parse-invalid", True, id="azure_invalid_configuration" + ), + # Azure datasource is detected from DMI chassis-asset-tag + pytest.param( + "Azure-dmi-detection", + True, + id="azure_dmi_detection_from_chassis_asset_tag", + ), + # Azure datasource is detected due to presence of a seed file. + # + # The seed file tested is /var/lib/cloud/seed/azure/ovf-env.xml. + pytest.param( + "Azure-seed-detection", True, id="azure_seed_file_detection" + ), + # EC2: hvm instances use dmi serial and uuid starting with 'ec2'. + pytest.param("Ec2-hvm", True, id="aws_ec2_hvm"), + # EC2: hvm instances use dmi serial and uuid starting with 'ec2' + # + # test using SYSTEMD_VIRTUALIZATION, not systemd-detect-virt + pytest.param("Ec2-hvm-env", True, id="aws_ec2_hvm_env"), + # EC2: hvm instances use system-uuid and may have swapped + # endianness + # + # test using SYSTEMD_VIRTUALIZATION, not systemd-detect-virt + pytest.param( + "Ec2-hvm-swap-endianness", True, id="aws_ec2_hvm_endian" + ), + # EC2: sys/hypervisor/uuid starts with ec2. + pytest.param("Ec2-xen", True, id="aws_ec2_xen"), + # EC2: product_serial ends with '.brightbox.com' + pytest.param("Ec2-brightbox", True, id="brightbox_is_ec2"), + # EC2: bobrightbox.com in product_serial is not brightbox + pytest.param( + "Ec2-brightbox-negative", + False, + id="brightbox_is_not_brightbox", + ), + # NoCloud identified on FreeBSD via label by geom. + pytest.param("NoCloud-fbsd", True, id="freebsd_nocloud"), + # GCE identifies itself with product_name. + pytest.param("GCE", True, id="gce_by_product_name"), + # GCE identifies itself with product_name. + # + # Uses SYSTEMD_VIRTUALIZATION + pytest.param("GCE_ENV", True, id="gce_by_product_name_env"), + # Older gce compute instances must be identified by serial. + pytest.param("GCE-serial", True, id="gce_by_serial"), + # LXD KVM has race on absent /dev/lxd/socket. Use DMI board_name. + pytest.param("LXD-kvm", True, id="lxd_kvm"), + # LXD KVM on host systems with a kernel > 5.10 need to match "qemu" + # + # LXD provides `hv_passthrough` when launching kvm instances when + # host kernel is > 5.10. This results in systemd being unable to + # detect the virtualized CPUID="Linux KVM Hv" as type "kvm" and + # results in systemd-detect-virt returning "qemu" in this case. + # + # Assert ds-identify can match systemd-detect-virt="qemu" and + # /sys/class/dmi/id/board_name = LXD. + # Once systemd 251 is available on a target distro, the virtualized + # CPUID will be represented properly as "kvm" + pytest.param( + "LXD-kvm-qemu-kernel-gt-5.10", True, id="lxd_kvm_jammy" + ), + # LXD KVM on host systems with a kernel > 5.10 need to match "qemu" + # + # LXD provides `hv_passthrough` when launching kvm instances when + # host kernel is > 5.10. This results in systemd being unable to + # detect the virtualized CPUID="Linux KVM Hv" as type "kvm" and + # results in systemd-detect-virt returning "qemu" in this case. + # + # Assert ds-identify can match systemd-detect-virt="qemu" and + # /sys/class/dmi/id/board_name = LXD. + # Once systemd 251 is available on a target distro, the virtualized + # CPUID will be represented properly as "kvm" + pytest.param( + "LXD-kvm-qemu-kernel-gt-5.10-env", True, id="lxd_kvm_jammy_env" + ), + # LXD containers will have /dev/lxd/socket at generator time. + pytest.param("LXD", True, id="lxd_containers"), + # ConfigDrive datasource has a disk with LABEL=config-2. + pytest.param("ConfigDrive", True, id="config_drive"), + # Rbx datasource has a disk with LABEL=CLOUDMD. + pytest.param("RbxCloud", True, id="rbx_cloud"), + # Rbx datasource has a disk with LABEL=cloudmd. + pytest.param("RbxCloudLower", True, id="rbx_cloud_lower"), + # ConfigDrive datasource has a disk with LABEL=CONFIG-2. + pytest.param("ConfigDriveUpper", True, id="config_drive_upper"), + # Config Drive seed directory. + pytest.param("ConfigDrive-seed", True, id="config_drive_seed"), + # Multi-line yaml is unsupported + pytest.param( + "LXD-kvm-not-azure", + True, + marks=[ + pytest.mark.xfail( + reason=( + "not supported: yaml parser implemented in POSIX" + " shell" + ) + ) + ], + id="multiline_yaml", + ), + # Template provisioned with user-data first boot. + # + # Template provisioning with user-data has METADATA disk. + # datasource should return found. + pytest.param( + "IBMCloud-metadata", True, id="ibmcloud_template_userdata" + ), + # Launched by os code always has config-2 disk. + pytest.param("IBMCloud-config-2", True, id="ibmcloud_os_code"), + # Test that Aliyun cloud is identified by product id. + pytest.param("AliYun", True, id="ibmcloud_os_code"), + # On Intel, openstack must be identified. + pytest.param( + "OpenStack", True, id="default_openstack_intel_is_found" + ), + # Open Telecom identification. + pytest.param( + "OpenStack-OpenTelekom", + True, + id="openstack_open_telekom_cloud", + ), + # SAP Converged Cloud identification + pytest.param( + "OpenStack-SAPCCloud", True, id="openstack_sap_ccloud" + ), + pytest.param( + "OpenStack-SAPCCloud-env", True, id="openstack_sap_ccloud-env" + ), + # Open Huawei Cloud identification. + pytest.param( + "OpenStack-HuaweiCloud", True, id="openstack_huawei_cloud" + ), + # Open Samsung Cloud Platform identification. + pytest.param( + "OpenStack-SamsungCloudPlatform", + True, + id="openstack_samsung_cloud_platform", + ), + # OpenStack identification via asset tag OpenStack Nova. + pytest.param( + "OpenStack-AssetTag-Nova", True, id="openstack_asset_tag_nova" + ), + # OpenStack identification via asset tag OpenStack Compute. + pytest.param( + "OpenStack-AssetTag-Compute", + True, + id="openstack_asset_tag_compute", + ), + # OVF is identified found when ovf/ovf-env.xml seed file exists. + pytest.param("OVF-seed", True, id="default_ovf_is_found"), + # OVF is identified when iso9660 cdrom path contains ovf schema. + pytest.param( + "OVF", + True, + id="ovf_on_vmware_iso_found_by_cdrom_with_ovf_schema_match", + ), + # OVF guest info is found on vmware. + pytest.param( + "OVF-guestinfo", True, id="ovf_on_vmware_guestinfo_found" + ), + # NoCloud is found with iso9660 filesystem on non-cdrom disk. + pytest.param("NoCloud", True, id="default_nocloud_as_vdb_iso9660"), + # NoCloud is found with uppercase filesystem label. + pytest.param("NoCloudUpper", True, id="nocloud_upper"), + # NoCloud seed definition can go in /etc/cloud/cloud.cfg[.d] + pytest.param("NoCloud-cfg", True, id="nocloud_seed_in_cfg"), + # NoCloud fatboot label - LP: #184166. + pytest.param("NoCloud-fatboot", True, id="nocloud_fatboot"), + # Nocloud seed directory. + pytest.param("NoCloud-seed", True, id="nocloud_seed"), + # Nocloud seed directory ubuntu core writable + pytest.param( + "NoCloud-seed-ubuntu-core", + True, + id="nocloud_seed_ubuntu_core_writable", + ), + # Hetzner cloud is identified in sys_vendor. + pytest.param("Hetzner", True, id="hetzner_found"), + # CloudCIX cloud is identified in dmi product-name + pytest.param("CloudCIX", True, id="cloudcix_found"), + # NWCS is identified in sys_vendor. + pytest.param("NWCS", True, id="nwcs_found"), + # SmartOS cloud identified by SmartDC in dmi. + pytest.param("SmartOS-bhyve", True, id="smartos_bhyve"), + # SmartOS cloud identified on lxbrand container. + pytest.param("SmartOS-lxbrand", True, id="smartos_lxbrand"), + pytest.param( + "SmartOS-lxbrand-env", True, id="smartos_lxbrand-env" + ), + # EC2: chassis asset tag ends with 'zstack.io' + pytest.param("Ec2-ZStack", True, id="zstack_is_ec2"), + # EC2: e24cloud identified by sys_vendor + pytest.param("Ec2-E24Cloud", True, id="e24cloud_is_ec2"), + # EC2: bobrightbox.com in product_serial is not brightbox' + pytest.param( + "Ec2-E24Cloud-negative", False, id="e24cloud_not_active" + ), + # EC2: outscale identified by sys_vendor and product_name + pytest.param("Ec2-Outscale", True, id="outscale_is_ec2"), + # EC2: outscale in sys_vendor is not outscale' + pytest.param( + "Ec2-Outscale-negative-sysvendor", + False, + id="outscale_not_active_sysvendor", + ), + # EC2: outscale in product_name is not outscale' + pytest.param( + "Ec2-Outscale-negative-productname", + False, + id="outscale_not_active_productname", + ), + # VMware: no valid transports + pytest.param( + "VMware-NoValidTransports", + False, + id="vmware_no_valid_transports", + ), + # VMware is identified when vmware customization is enabled. + pytest.param( + "VMware-vmware-customization", + True, + id="vmware_on_vmware_when_vmware_customization_is_enabled", + ), + # VMware and OVF are identified when: + # 1. On VMware platform. + # 2. VMware customization is enabled. + # 3. iso9660 cdrom path contains ovf schema. + pytest.param( + "VMware-OVF-on-vmware-with-vmware-customization-and-ovf-schema", + True, + id="vmware_ovf_on_vmware_with_vmware_customization_and_ovf_" + "schema", + ), + # OVF is identified when: + # 1. Not on VMware platform. + # 2. VMware customization is enabled. + # 3. iso9660 cdrom path contains ovf schema. + pytest.param( + "OVF-not-on-vmware-with-vmware-customization-and-ovf-schema", + True, + id="ovf_not_on_vmware_with_vmware_customization_and_ovf_" + "schema", + ), + # VMware: envvar transport no data + pytest.param( + "VMware-EnvVar-NoData", False, id="vmware_envvar_no_data" + ), + # VMware: envvar transport success if no virt id + pytest.param( + "VMware-EnvVar-NoVirtID", True, id="vmware_envvar_no_virt_id" + ), + # VMware: envvar transport activated by metadata + pytest.param( + "VMware-EnvVar-Metadata", + True, + id="vmware_envvar_activated_by_metadata", + ), + # VMware: envvar transport activated by userdata + pytest.param( + "VMware-EnvVar-Userdata", + True, + id="vmware_envvar_activated_by_userdata", + ), + # VMware: envvar transport activated by vendordata + pytest.param( + "VMware-EnvVar-Vendordata", + True, + id="vmware_envvar_activated_by_vendordata", + ), + # VMware: guestinfo transport no data + pytest.param( + "VMware-GuestInfo-NoData-Rpctool", + False, + id="vmware_guestinfo_no_data_rcptool", + ), + pytest.param( + "VMware-GuestInfo-NoData-Vmtoolsd", + False, + id="vmware_guestinfo_no_data_vmtoolsd", + ), + # VMware: guestinfo transport fails if no virt id + pytest.param( + "VMware-GuestInfo-NoVirtID", + False, + id="vmware_guestinfo_no_virt_id", + ), + # VMware: guestinfo transport activated by metadata + pytest.param( + "VMware-GuestInfo-Metadata", + True, + id="vmware_guestinfo_activated_by_metadata", + ), + # VMware: guestinfo transport activated by userdata + pytest.param( + "VMware-GuestInfo-Userdata", + True, + id="vmware_guestinfo_activated_by_userdata", + ), + # VMware: guestinfo transport activated by vendordata + pytest.param( + "VMware-GuestInfo-Vendordata", + True, + id="vmware_guestinfo_activated_by_vendordata", + ), + # VMware and OVF are identified when: + # 1. On VMware platform. + # 2. guestinfo transport activated by metadata + # 3. iso9660 cdrom path contains ovf schema. + pytest.param( + "VMware-OVF-on-vmware-with-guestinfo-metadata-and-ovf-schema", + True, + id="vmware_ovf_on_vmware_with_guestinfo_metadata_and_ovf_" + "schema", + ), + # OVF is identified when: + # 1. Not on VMware platform. + # 2. guestinfo transport activated by metadata + # 3. iso9660 cdrom path contains ovf schema. + pytest.param( + "OVF-not-on-vmware-with-guestinfo-metadata-and-ovf-schema", + True, + id="ovf_not_on_vmware_with_guestinfo_metadata_and_ovf_schema", + ), + # ds-identify finds Akamai by system-manufacturer dmi field + pytest.param("Akamai", True, id="akamai_found_by_sys_vendor"), + # Test *BSD code paths + # + # FreeBSD doesn't have /sys so we use kenv(1) here. + # OpenBSD uses sysctl(8). + # Other BSD systems fallback to dmidecode(8). + # BSDs also doesn't have systemd-detect-virt(8), so we use + # sysctl(8) to query kern.vm_guest, and optionally map it: + # + # Test that kenv(1) works on systems which don't have /sys + pytest.param("Hetzner-kenv", True, id="bsd_dmi_kenv"), + # Test that sysctl(8) works on systems which don't have /sys + pytest.param("Hetzner-sysctl", True, id="bsd_dmi_sysctl"), + # Test that dmidecode(8) works on systems which don't have /sys + pytest.param("Hetzner-dmidecode", True, id="bsd_dmi_dmidecode"), + # Simple positive test of Oracle by chassis id. + pytest.param("Oracle", True, id="oracle_found_by_chassis"), + # Simple negative test for WSL due other virt. + pytest.param("Not-WSL", False, id="wsl_not_found_virt"), + # Negative test by lack of host filesystem mount points. + pytest.param("WSL-no-host-mounts", False, id="wsl_no_fs_mounts"), + ], + ) + def test_ds_found_not_found(self, config, found, tmp_path): + test_func = self._test_ds_found if found else self._test_ds_not_found + test_func(config, str(tmp_path)) - def test_flow_sequence_control(self): + def test_flow_sequence_control(self, tmp_path): """ensure that an invalid key in the flow_sequence tests produces no datasource list match control test: this test serves as a control test for test_flow_sequence """ data = copy.deepcopy(VALID_CFG["flow_sequence-control"]) - self._check_via_dict(data, RC_NOT_FOUND) + self._check_via_dict(data, str(tmp_path), RC_NOT_FOUND) - def test_flow_sequence(self): + def test_flow_sequence(self, tmp_path): """correctly identify flow sequences""" for i in range(1, 10): data = copy.deepcopy(VALID_CFG[f"flow_sequence-{i}"]) - self._check_via_dict(data, RC_FOUND, dslist=[data.get("ds")]) - - def test_azure_invalid_configuration(self): - """Don't detect incorrect config when invalid datasource_list provided - - If unparsable list is provided we just ignore it. Some users - might assume that since the rest of the configuration is yaml that - multi-line yaml lists are valid (they aren't). When this happens, just - run ds-identify and figure it out for ourselves which platform to run. - """ - self._test_ds_found("Azure-parse-invalid") - - def test_azure_dmi_detection_from_chassis_asset_tag(self): - """Azure datasource is detected from DMI chassis-asset-tag""" - self._test_ds_found("Azure-dmi-detection") - - def test_azure_seed_file_detection(self): - """Azure datasource is detected due to presence of a seed file. - - The seed file tested is /var/lib/cloud/seed/azure/ovf-env.xml.""" - self._test_ds_found("Azure-seed-detection") - - def test_aws_ec2_hvm(self): - """EC2: hvm instances use dmi serial and uuid starting with 'ec2'.""" - self._test_ds_found("Ec2-hvm") - - def test_aws_ec2_hvm_env(self): - """EC2: hvm instances use dmi serial and uuid starting with 'ec2' - - test using SYSTEMD_VIRTUALIZATION, not systemd-detect-virt - """ - self._test_ds_found("Ec2-hvm-env") - - def test_aws_ec2_hvm_endian(self): - """EC2: hvm instances use system-uuid and may have swapped endianness - - test using SYSTEMD_VIRTUALIZATION, not systemd-detect-virt - """ - self._test_ds_found("Ec2-hvm-swap-endianness") - - def test_aws_ec2_xen(self): - """EC2: sys/hypervisor/uuid starts with ec2.""" - self._test_ds_found("Ec2-xen") - - def test_brightbox_is_ec2(self): - """EC2: product_serial ends with '.brightbox.com'""" - self._test_ds_found("Ec2-brightbox") - - def test_bobrightbox_is_not_brightbox(self): - """EC2: bobrightbox.com in product_serial is not brightbox'""" - self._test_ds_not_found("Ec2-brightbox-negative") - - def test_freebsd_nocloud(self): - """NoCloud identified on FreeBSD via label by geom.""" - self._test_ds_found("NoCloud-fbsd") - - def test_gce_by_product_name(self): - """GCE identifies itself with product_name.""" - self._test_ds_found("GCE") - - def test_gce_by_product_name_env(self): - """GCE identifies itself with product_name. - - Uses SYSTEMD_VIRTUALIZATION - """ - self._test_ds_found("GCE_ENV") - - def test_gce_by_serial(self): - """Older gce compute instances must be identified by serial.""" - self._test_ds_found("GCE-serial") - - def test_lxd_kvm(self): - """LXD KVM has race on absent /dev/lxd/socket. Use DMI board_name.""" - self._test_ds_found("LXD-kvm") - - def test_lxd_kvm_jammy(self): - """LXD KVM on host systems with a kernel > 5.10 need to match "qemu". - LXD provides `hv_passthrough` when launching kvm instances when host - kernel is > 5.10. This results in systemd being unable to detect the - virtualized CPUID="Linux KVM Hv" as type "kvm" and results in - systemd-detect-virt returning "qemu" in this case. - - Assert ds-identify can match systemd-detect-virt="qemu" and - /sys/class/dmi/id/board_name = LXD. - Once systemd 251 is available on a target distro, the virtualized - CPUID will be represented properly as "kvm" - """ - self._test_ds_found("LXD-kvm-qemu-kernel-gt-5.10") - - def test_lxd_kvm_jammy_env(self): - """LXD KVM on host systems with a kernel > 5.10 need to match "qemu". - LXD provides `hv_passthrough` when launching kvm instances when host - kernel is > 5.10. This results in systemd being unable to detect the - virtualized CPUID="Linux KVM Hv" as type "kvm" and results in - systemd-detect-virt returning "qemu" in this case. - - Assert ds-identify can match systemd-detect-virt="qemu" and - /sys/class/dmi/id/board_name = LXD. - Once systemd 251 is available on a target distro, the virtualized - CPUID will be represented properly as "kvm" - """ - self._test_ds_found("LXD-kvm-qemu-kernel-gt-5.10-env") - - def test_lxd_containers(self): - """LXD containers will have /dev/lxd/socket at generator time.""" - self._test_ds_found("LXD") - - def test_config_drive(self): - """ConfigDrive datasource has a disk with LABEL=config-2.""" - self._test_ds_found("ConfigDrive") - - def test_rbx_cloud(self): - """Rbx datasource has a disk with LABEL=CLOUDMD.""" - self._test_ds_found("RbxCloud") - - def test_rbx_cloud_lower(self): - """Rbx datasource has a disk with LABEL=cloudmd.""" - self._test_ds_found("RbxCloudLower") - - def test_config_drive_upper(self): - """ConfigDrive datasource has a disk with LABEL=CONFIG-2.""" - self._test_ds_found("ConfigDriveUpper") - - def test_config_drive_seed(self): - """Config Drive seed directory.""" - self._test_ds_found("ConfigDrive-seed") + self._check_via_dict( + data, str(tmp_path), RC_FOUND, dslist=[data.get("ds")] + ) - def test_config_drive_interacts_with_ibmcloud_config_disk(self): + def test_config_drive_interacts_with_ibmcloud_config_disk(self, tmp_path): """Verify ConfigDrive interaction with IBMCloud. If ConfigDrive is enabled and not IBMCloud, then ConfigDrive @@ -731,24 +932,15 @@ def test_config_drive_interacts_with_ibmcloud_config_disk(self): # with list including IBMCloud, config drive should be not found. files[cfgpath] = "datasource_list: [ ConfigDrive, IBMCloud ]\n" - ret = self._check_via_dict(data, shell_true) - self.assertEqual(ret.cfg.get("datasource_list"), ["IBMCloud", "None"]) + ret = self._check_via_dict(data, str(tmp_path / "ibm"), shell_true) + assert ret.cfg.get("datasource_list") == ["IBMCloud", "None"] # But if IBMCloud is not enabled, config drive should claim this. files[cfgpath] = "datasource_list: [ ConfigDrive, NoCloud ]\n" - ret = self._check_via_dict(data, shell_true) - self.assertEqual( - ret.cfg.get("datasource_list"), ["ConfigDrive", "None"] - ) - - @pytest.mark.xfail( - reason=("not supported: yaml parser implemented in POSIX shell") - ) - def test_multiline_yaml(self): - """Multi-line yaml is unsupported""" - self._test_ds_found("LXD-kvm-not-azure") + ret = self._check_via_dict(data, str(tmp_path), shell_true) + assert ret.cfg.get("datasource_list") == ["ConfigDrive", "None"] - def test_ibmcloud_template_userdata_in_provisioning(self): + def test_ibmcloud_template_userdata_in_provisioning(self, tmp_path): """Template provisioned with user-data during provisioning stage. Template provisioning with user-data has METADATA disk, @@ -759,16 +951,9 @@ def test_ibmcloud_template_userdata_in_provisioning(self): m for m in data["mocks"] if m["name"] == "is_ibm_provisioning" ][0] isprov_m["ret"] = shell_true - self._check_via_dict(data, RC_NOT_FOUND) - - def test_ibmcloud_template_userdata(self): - """Template provisioned with user-data first boot. - - Template provisioning with user-data has METADATA disk. - datasource should return found.""" - self._test_ds_found("IBMCloud-metadata") + self._check_via_dict(data, str(tmp_path), RC_NOT_FOUND) - def test_ibmcloud_template_no_userdata_in_provisioning(self): + def test_ibmcloud_template_no_userdata_in_provisioning(self, tmp_path): """Template provisioned with no user-data during provisioning. no disks attached. Datasource should return not found.""" @@ -776,19 +961,17 @@ def test_ibmcloud_template_no_userdata_in_provisioning(self): data["mocks"].append( {"name": "is_ibm_provisioning", "ret": shell_true} ) - self._check_via_dict(data, RC_NOT_FOUND) + self._check_via_dict(data, str(tmp_path), RC_NOT_FOUND) - def test_ibmcloud_template_no_userdata(self): + def test_ibmcloud_template_no_userdata(self, tmp_path): """Template provisioned with no user-data first boot. no disks attached. Datasource should return found.""" - self._check_via_dict(VALID_CFG["IBMCloud-nodisks"], RC_NOT_FOUND) - - def test_ibmcloud_os_code(self): - """Launched by os code always has config-2 disk.""" - self._test_ds_found("IBMCloud-config-2") + self._check_via_dict( + VALID_CFG["IBMCloud-nodisks"], str(tmp_path), RC_NOT_FOUND + ) - def test_ibmcloud_os_code_different_uuid(self): + def test_ibmcloud_os_code_different_uuid(self, tmp_path): """IBM cloud config-2 disks must be explicit match on UUID. If the UUID is not 9796-932E then we actually expect ConfigDrive.""" @@ -804,10 +987,10 @@ def test_ibmcloud_os_code_different_uuid(self): ds_ibm.IBM_CONFIG_UUID, "DEAD-BEEF" ) self._check_via_dict( - data, rc=RC_FOUND, dslist=["ConfigDrive", DS_NONE] + data, str(tmp_path), rc=RC_FOUND, dslist=["ConfigDrive", DS_NONE] ) - def test_ibmcloud_with_nocloud_seed(self): + def test_ibmcloud_with_nocloud_seed(self, tmp_path): """NoCloud seed should be preferred over IBMCloud. A nocloud seed should be preferred over IBMCloud even if enabled. @@ -817,12 +1000,12 @@ def test_ibmcloud_with_nocloud_seed(self): if not files: data["files"] = files files.update(VALID_CFG["NoCloud-seed"]["files"]) - ret = self._check_via_dict(data, shell_true) - self.assertEqual( - ["NoCloud", "IBMCloud", "None"], ret.cfg.get("datasource_list") + ret = self._check_via_dict(data, str(tmp_path), shell_true) + assert ["NoCloud", "IBMCloud", "None"] == ret.cfg.get( + "datasource_list" ) - def test_ibmcloud_with_configdrive_seed(self): + def test_ibmcloud_with_configdrive_seed(self, tmp_path): """ConfigDrive seed should be preferred over IBMCloud. A ConfigDrive seed should be preferred over IBMCloud even if enabled. @@ -833,26 +1016,28 @@ def test_ibmcloud_with_configdrive_seed(self): if not files: data["files"] = files files.update(VALID_CFG["ConfigDrive-seed"]["files"]) - ret = self._check_via_dict(data, shell_true) - self.assertEqual( - ["ConfigDrive", "IBMCloud", "None"], ret.cfg.get("datasource_list") + ret = self._check_via_dict(data, str(tmp_path), shell_true) + assert ["ConfigDrive", "IBMCloud", "None"] == ret.cfg.get( + "datasource_list" ) - def test_policy_disabled(self): + def test_policy_disabled(self, tmp_path): """A Builtin policy of 'disabled' should return not found. Even though a search would find something, the builtin policy of disabled should cause the return of not found.""" mydata = copy.deepcopy(VALID_CFG["Ec2-hvm"]) - self._check_via_dict(mydata, rc=RC_NOT_FOUND, policy_dmi="disabled") + self._check_via_dict( + mydata, str(tmp_path), rc=RC_NOT_FOUND, policy_dmi="disabled" + ) - def test_policy_config_disable_overrides_builtin(self): + def test_policy_config_disable_overrides_builtin(self, tmp_path): """explicit policy: disabled in config file should cause not found.""" mydata = copy.deepcopy(VALID_CFG["Ec2-hvm"]) mydata["files"][P_DSID_CFG] = "\n".join(["policy: disabled", ""]) - self._check_via_dict(mydata, rc=RC_NOT_FOUND) + self._check_via_dict(mydata, str(tmp_path), rc=RC_NOT_FOUND) - def test_single_entry_defines_datasource(self): + def test_single_entry_defines_datasource(self, tmp_path): """If config has a single entry in datasource_list, that is used. Test the valid Ec2-hvm, but provide a config file that specifies @@ -861,9 +1046,11 @@ def test_single_entry_defines_datasource(self): mydata = copy.deepcopy(VALID_CFG["Ec2-hvm"]) cfgpath = "etc/cloud/cloud.cfg.d/myds.cfg" mydata["files"][cfgpath] = 'datasource_list: ["NoCloud"]\n' - self._check_via_dict(mydata, rc=RC_FOUND, dslist=["NoCloud"]) + self._check_via_dict( + mydata, str(tmp_path), rc=RC_FOUND, dslist=["NoCloud"] + ) - def test_configured_list_with_none(self): + def test_configured_list_with_none(self, tmp_path): """When datasource_list already contains None, None is not added. The explicitly configured datasource_list has 'None' in it. That @@ -871,9 +1058,11 @@ def test_configured_list_with_none(self): mydata = copy.deepcopy(VALID_CFG["GCE"]) cfgpath = "etc/cloud/cloud.cfg.d/myds.cfg" mydata["files"][cfgpath] = 'datasource_list: ["Ec2", "None"]\n' - self._check_via_dict(mydata, rc=RC_FOUND, dslist=["Ec2", DS_NONE]) + self._check_via_dict( + mydata, str(tmp_path), rc=RC_FOUND, dslist=["Ec2", DS_NONE] + ) - def test_nocloud_seedfrom(self): + def test_nocloud_seedfrom(self, tmp_path): """Check seedfrom system config detects nocloud. Verify that a cloud.cfg.d/ that contains more than two datasources in @@ -882,11 +1071,12 @@ def test_nocloud_seedfrom(self): """ self._check_via_dict( copy.deepcopy(VALID_CFG["NoCloud-seedfrom"]), + str(tmp_path), rc=RC_FOUND, dslist=["NoCloud", DS_NONE], ) - def test_nocloud_userdata_and_metadata(self): + def test_nocloud_userdata_and_metadata(self, tmp_path): """Check seedfrom system config detects nocloud. Verify that a cloud.cfg.d/ that contains more than two datasources in @@ -895,58 +1085,27 @@ def test_nocloud_userdata_and_metadata(self): """ self._check_via_dict( copy.deepcopy(VALID_CFG["NoCloud-user-data-meta-data"]), + str(tmp_path), rc=RC_FOUND, dslist=["NoCloud", DS_NONE], ) - def test_aliyun_identified(self): - """Test that Aliyun cloud is identified by product id.""" - self._test_ds_found("AliYun") - - def test_aliyun_over_ec2(self): + def test_aliyun_over_ec2(self, tmp_path): """Even if all other factors identified Ec2, AliYun should be used.""" mydata = copy.deepcopy(VALID_CFG["Ec2-xen"]) - self._test_ds_found("AliYun") + self._test_ds_found("AliYun", str(tmp_path)) prod_name = VALID_CFG["AliYun"]["files"][P_PRODUCT_NAME] mydata["files"][P_PRODUCT_NAME] = prod_name policy = "search,found=first,maybe=none,notfound=disabled" self._check_via_dict( - mydata, rc=RC_FOUND, dslist=["AliYun", DS_NONE], policy_dmi=policy + mydata, + str(tmp_path), + rc=RC_FOUND, + dslist=["AliYun", DS_NONE], + policy_dmi=policy, ) - def test_default_openstack_intel_is_found(self): - """On Intel, openstack must be identified.""" - self._test_ds_found("OpenStack") - - def test_openstack_open_telekom_cloud(self): - """Open Telecom identification.""" - self._test_ds_found("OpenStack-OpenTelekom") - - def test_openstack_sap_ccloud(self): - """SAP Converged Cloud identification""" - self._test_ds_found("OpenStack-SAPCCloud") - - def test_openstack_sap_ccloud_env(self): - """SAP Converged Cloud identification""" - self._test_ds_found("OpenStack-SAPCCloud-env") - - def test_openstack_huawei_cloud(self): - """Open Huawei Cloud identification.""" - self._test_ds_found("OpenStack-HuaweiCloud") - - def test_openstack_samsung_cloud_platform(self): - """Open Samsung Cloud Platform identification.""" - self._test_ds_found("OpenStack-SamsungCloudPlatform") - - def test_openstack_asset_tag_nova(self): - """OpenStack identification via asset tag OpenStack Nova.""" - self._test_ds_found("OpenStack-AssetTag-Nova") - - def test_openstack_asset_tag_copute(self): - """OpenStack identification via asset tag OpenStack Compute.""" - self._test_ds_found("OpenStack-AssetTag-Compute") - - def test_openstack_on_non_intel_is_maybe(self): + def test_openstack_on_non_intel_is_maybe(self, tmp_path): """On non-Intel, openstack without dmi info is none. nova does not identify itself on platforms other than intel. @@ -963,50 +1122,52 @@ def test_openstack_on_non_intel_is_maybe(self): # this should show not found as default uname in tests is intel. # and intel openstack requires positive identification. - self._check_via_dict(data, RC_NOT_FOUND, dslist=None) + self._check_via_dict( + data, str(tmp_path / "0"), RC_NOT_FOUND, dslist=None + ) # updating the uname to ppc64 though should get a maybe. data.update({"mocks": [MOCK_VIRT_IS_KVM, MOCK_UNAME_IS_PPC64]}) - (_, _, err, _, _) = self._check_via_dict(data, RC_NOT_FOUND) - self.assertIn("check for 'OpenStack' returned maybe", err) - self.assertIn("No ds found", err) - self.assertIn("Disabled cloud-init", err) - self.assertIn("returning 1", err) - - def test_default_ovf_is_found(self): - """OVF is identified found when ovf/ovf-env.xml seed file exists.""" - self._test_ds_found("OVF-seed") + (_, _, err, _, _) = self._check_via_dict( + data, str(tmp_path / "1"), RC_NOT_FOUND + ) + assert "check for 'OpenStack' returned maybe" in err + assert "No ds found" in err + assert "Disabled cloud-init" in err + assert "returning 1" in err - def test_default_ovf_with_detect_virt_none_not_found(self): + def test_default_ovf_with_detect_virt_none_not_found(self, tmp_path): """OVF identifies not found when detect_virt returns "none".""" self._check_via_dict( - {"ds": "OVF"}, rc=RC_NOT_FOUND, policy_dmi="disabled" + {"ds": "OVF"}, + str(tmp_path), + rc=RC_NOT_FOUND, + policy_dmi="disabled", ) - def test_default_ovf_returns_not_found_on_azure(self): + def test_default_ovf_returns_not_found_on_azure(self, tmp_path): """OVF datasource won't be found as false positive on Azure.""" ovfonazure = copy.deepcopy(VALID_CFG["OVF"]) # Set azure asset tag to assert OVF content not found ovfonazure["files"][ P_CHASSIS_ASSET_TAG ] = "7783-7084-3265-9085-8269-3286-77\n" - self._check_via_dict(ovfonazure, RC_FOUND, dslist=["Azure", DS_NONE]) - - def test_ovf_on_vmware_iso_found_by_cdrom_with_ovf_schema_match(self): - """OVF is identified when iso9660 cdrom path contains ovf schema.""" - self._test_ds_found("OVF") - - def test_ovf_on_vmware_guestinfo_found(self): - """OVF guest info is found on vmware.""" - self._test_ds_found("OVF-guestinfo") + self._check_via_dict( + ovfonazure, str(tmp_path), RC_FOUND, dslist=["Azure", DS_NONE] + ) - def test_ovf_on_vmware_iso_found_by_cdrom_with_matching_fs_label(self): + def test_ovf_on_vmware_iso_found_by_cdrom_with_matching_fs_label( + self, tmp_path + ): """OVF is identified by well-known iso9660 labels.""" ovf_cdrom_by_label = copy.deepcopy(VALID_CFG["OVF"]) # Unset matching cdrom ovf schema content ovf_cdrom_by_label["files"]["dev/sr0"] = "No content match" self._check_via_dict( - ovf_cdrom_by_label, rc=RC_NOT_FOUND, policy_dmi="disabled" + ovf_cdrom_by_label, + str(tmp_path), + rc=RC_NOT_FOUND, + policy_dmi="disabled", ) # Add recognized labels @@ -1031,103 +1192,76 @@ def test_ovf_on_vmware_iso_found_by_cdrom_with_matching_fs_label(self): ] ) self._check_via_dict( - ovf_cdrom_by_label, rc=RC_FOUND, dslist=["OVF", DS_NONE] + ovf_cdrom_by_label, + str(tmp_path), + rc=RC_FOUND, + dslist=["OVF", DS_NONE], ) - def test_ovf_on_vmware_iso_found_by_cdrom_with_different_size(self): + def test_ovf_on_vmware_iso_found_by_cdrom_with_different_size( + self, tmp_path + ): """OVF is identified by well-known iso9660 labels.""" ovf_cdrom_with_size = copy.deepcopy(VALID_CFG["OVF"]) # Set cdrom size to 20480 (10MB in 512 byte units) ovf_cdrom_with_size["files"]["sys/class/block/sr0/size"] = "20480\n" self._check_via_dict( - ovf_cdrom_with_size, rc=RC_NOT_FOUND, policy_dmi="disabled" + ovf_cdrom_with_size, + str(tmp_path), + rc=RC_NOT_FOUND, + policy_dmi="disabled", ) # Set cdrom size to 204800 (100MB in 512 byte units) ovf_cdrom_with_size["files"]["sys/class/block/sr0/size"] = "204800\n" self._check_via_dict( - ovf_cdrom_with_size, rc=RC_NOT_FOUND, policy_dmi="disabled" + ovf_cdrom_with_size, + str(tmp_path), + rc=RC_NOT_FOUND, + policy_dmi="disabled", ) # Set cdrom size to 18432 (9MB in 512 byte units) ovf_cdrom_with_size["files"]["sys/class/block/sr0/size"] = "18432\n" self._check_via_dict( - ovf_cdrom_with_size, rc=RC_FOUND, dslist=["OVF", DS_NONE] + ovf_cdrom_with_size, + str(tmp_path), + rc=RC_FOUND, + dslist=["OVF", DS_NONE], ) # Set cdrom size to 2048 (1MB in 512 byte units) ovf_cdrom_with_size["files"]["sys/class/block/sr0/size"] = "2048\n" self._check_via_dict( - ovf_cdrom_with_size, rc=RC_FOUND, dslist=["OVF", DS_NONE] + ovf_cdrom_with_size, + str(tmp_path), + rc=RC_FOUND, + dslist=["OVF", DS_NONE], ) - def test_default_nocloud_as_vdb_iso9660(self): - """NoCloud is found with iso9660 filesystem on non-cdrom disk.""" - self._test_ds_found("NoCloud") - - def test_nocloud_upper(self): - """NoCloud is found with uppercase filesystem label.""" - self._test_ds_found("NoCloudUpper") - - def test_nocloud_seed_in_cfg(self): - """NoCloud seed definition can go in /etc/cloud/cloud.cfg[.d]""" - self._test_ds_found("NoCloud-cfg") - - def test_nocloud_fatboot(self): - """NoCloud fatboot label - LP: #184166.""" - self._test_ds_found("NoCloud-fatboot") - - def test_nocloud_seed(self): - """Nocloud seed directory.""" - self._test_ds_found("NoCloud-seed") - - def test_nocloud_seed_ubuntu_core_writable(self): - """Nocloud seed directory ubuntu core writable""" - self._test_ds_found("NoCloud-seed-ubuntu-core") - - def test_hetzner_found(self): - """Hetzner cloud is identified in sys_vendor.""" - self._test_ds_found("Hetzner") - - def test_cloudcix_found(self): - """CloudCIX cloud is identified in dmi product-name""" - self._test_ds_found("CloudCIX") - - def test_nwcs_found(self): - """NWCS is identified in sys_vendor.""" - self._test_ds_found("NWCS") - - def test_smartos_bhyve(self): - """SmartOS cloud identified by SmartDC in dmi.""" - self._test_ds_found("SmartOS-bhyve") - - def test_smartos_lxbrand(self): - """SmartOS cloud identified on lxbrand container.""" - self._test_ds_found("SmartOS-lxbrand") - - def test_smartos_lxbrand_env(self): - """SmartOS cloud identified on lxbrand container.""" - self._test_ds_found("SmartOS-lxbrand-env") - - def test_smartos_lxbrand_requires_socket(self): + def test_smartos_lxbrand_requires_socket(self, tmp_path): """SmartOS cloud should not be identified if no socket file.""" mycfg = copy.deepcopy(VALID_CFG["SmartOS-lxbrand"]) del mycfg["files"][ds_smartos.METADATA_SOCKFILE] - self._check_via_dict(mycfg, rc=RC_NOT_FOUND, policy_dmi="disabled") + self._check_via_dict( + mycfg, str(tmp_path), rc=RC_NOT_FOUND, policy_dmi="disabled" + ) - def test_smartos_lxbrand_requires_socket_env(self): + def test_smartos_lxbrand_requires_socket_env(self, tmp_path): """SmartOS cloud should not be identified if no socket file.""" mycfg = copy.deepcopy(VALID_CFG["SmartOS-lxbrand-env"]) del mycfg["files"][ds_smartos.METADATA_SOCKFILE] - self._check_via_dict(mycfg, rc=RC_NOT_FOUND, policy_dmi="disabled") + self._check_via_dict( + mycfg, str(tmp_path), rc=RC_NOT_FOUND, policy_dmi="disabled" + ) - def test_path_env_gets_set_from_main(self): + def test_path_env_gets_set_from_main(self, tmp_path): """PATH environment should always have some tokens when main is run. We explicitly call main as we want to ensure it updates PATH.""" cust = copy.deepcopy(VALID_CFG["NoCloud"]) - rootd = self.tmp_dir() + rootd = str(tmp_path) mpp = "main-printpath" pre = "MYPATH=" cust["files"][mpp] = ( @@ -1135,75 +1269,21 @@ def test_path_env_gets_set_from_main(self): ) ret = self._check_via_dict( cust, + rootd, RC_FOUND, func=".", args=[os.path.join(rootd, mpp)], - rootd=rootd, ) match = [ line for line in ret.stdout.splitlines() if line.startswith(pre) ][0] toks = match.replace(pre, "").split(":") expected = ["/sbin", "/bin", "/usr/sbin", "/usr/bin", "/mycust/path"] - self.assertEqual( - expected, - [p for p in expected if p in toks], - "path did not have expected tokens", - ) - - def test_zstack_is_ec2(self): - """EC2: chassis asset tag ends with 'zstack.io'""" - self._test_ds_found("Ec2-ZStack") - - def test_e24cloud_is_ec2(self): - """EC2: e24cloud identified by sys_vendor""" - self._test_ds_found("Ec2-E24Cloud") - - def test_e24cloud_not_active(self): - """EC2: bobrightbox.com in product_serial is not brightbox'""" - self._test_ds_not_found("Ec2-E24Cloud-negative") - - def test_outscale_is_ec2(self): - """EC2: outscale identified by sys_vendor and product_name""" - self._test_ds_found("Ec2-Outscale") + assert expected == [ + p for p in expected if p in toks + ], "path did not have expected tokens" - def test_outscale_not_active_sysvendor(self): - """EC2: outscale in sys_vendor is not outscale'""" - self._test_ds_not_found("Ec2-Outscale-negative-sysvendor") - - def test_outscale_not_active_productname(self): - """EC2: outscale in product_name is not outscale'""" - self._test_ds_not_found("Ec2-Outscale-negative-productname") - - def test_vmware_no_valid_transports(self): - """VMware: no valid transports""" - self._test_ds_not_found("VMware-NoValidTransports") - - def test_vmware_on_vmware_when_vmware_customization_is_enabled(self): - """VMware is identified when vmware customization is enabled.""" - self._test_ds_found("VMware-vmware-customization") - - def test_vmware_ovf_on_vmware_with_vmware_customization_and_ovf_schema( - self, - ): - """VMware and OVF are identified when: - 1. On VMware platform. - 2. VMware customization is enabled. - 3. iso9660 cdrom path contains ovf schema.""" - self._test_ds_found( - "VMware-OVF-on-vmware-with-vmware-customization-and-ovf-schema" - ) - - def test_ovf_not_on_vmware_with_vmware_customization_and_ovf_schema(self): - """OVF is identified when: - 1. Not on VMware platform. - 2. VMware customization is enabled. - 3. iso9660 cdrom path contains ovf schema.""" - self._test_ds_found( - "OVF-not-on-vmware-with-vmware-customization-and-ovf-schema" - ) - - def test_vmware_on_vmware_open_vm_tools_64(self): + def test_vmware_on_vmware_open_vm_tools_64(self, tmp_path): """VMware is identified when open-vm-tools installed in /usr/lib64.""" cust64 = copy.deepcopy(VALID_CFG["VMware-vmware-customization"]) p32 = "usr/lib/vmware-tools/plugins/vmsvc/libdeployPkgPlugin.so" @@ -1211,10 +1291,10 @@ def test_vmware_on_vmware_open_vm_tools_64(self): cust64["files"][open64] = cust64["files"][p32] del cust64["files"][p32] self._check_via_dict( - cust64, RC_FOUND, dslist=[cust64.get("ds"), DS_NONE] + cust64, str(tmp_path), RC_FOUND, dslist=[cust64.get("ds"), DS_NONE] ) - def test_vmware_on_vmware_open_vm_tools_x86_64_linux_gnu(self): + def test_vmware_on_vmware_open_vm_tools_x86_64_linux_gnu(self, tmp_path): """VMware is identified when open-vm-tools installed in /usr/lib/x86_64-linux-gnu.""" cust64 = copy.deepcopy(VALID_CFG["VMware-vmware-customization"]) @@ -1226,10 +1306,10 @@ def test_vmware_on_vmware_open_vm_tools_x86_64_linux_gnu(self): cust64["files"][x86] = cust64["files"][p32] del cust64["files"][p32] self._check_via_dict( - cust64, RC_FOUND, dslist=[cust64.get("ds"), DS_NONE] + cust64, str(tmp_path), RC_FOUND, dslist=[cust64.get("ds"), DS_NONE] ) - def test_vmware_on_vmware_open_vm_tools_aarch64_linux_gnu(self): + def test_vmware_on_vmware_open_vm_tools_aarch64_linux_gnu(self, tmp_path): """VMware is identified when open-vm-tools installed in /usr/lib/aarch64-linux-gnu.""" cust64 = copy.deepcopy(VALID_CFG["VMware-vmware-customization"]) @@ -1241,10 +1321,10 @@ def test_vmware_on_vmware_open_vm_tools_aarch64_linux_gnu(self): cust64["files"][aarch64] = cust64["files"][p32] del cust64["files"][p32] self._check_via_dict( - cust64, RC_FOUND, dslist=[cust64.get("ds"), DS_NONE] + cust64, str(tmp_path), RC_FOUND, dslist=[cust64.get("ds"), DS_NONE] ) - def test_vmware_on_vmware_open_vm_tools_i386_linux_gnu(self): + def test_vmware_on_vmware_open_vm_tools_i386_linux_gnu(self, tmp_path): """VMware is identified when open-vm-tools installed in /usr/lib/i386-linux-gnu.""" cust64 = copy.deepcopy(VALID_CFG["VMware-vmware-customization"]) @@ -1256,121 +1336,29 @@ def test_vmware_on_vmware_open_vm_tools_i386_linux_gnu(self): cust64["files"][i386] = cust64["files"][p32] del cust64["files"][p32] self._check_via_dict( - cust64, RC_FOUND, dslist=[cust64.get("ds"), DS_NONE] - ) - - def test_vmware_envvar_no_data(self): - """VMware: envvar transport no data""" - self._test_ds_not_found("VMware-EnvVar-NoData") - - def test_vmware_envvar_no_virt_id(self): - """VMware: envvar transport success if no virt id""" - self._test_ds_found("VMware-EnvVar-NoVirtID") - - def test_vmware_envvar_activated_by_metadata(self): - """VMware: envvar transport activated by metadata""" - self._test_ds_found("VMware-EnvVar-Metadata") - - def test_vmware_envvar_activated_by_userdata(self): - """VMware: envvar transport activated by userdata""" - self._test_ds_found("VMware-EnvVar-Userdata") - - def test_vmware_envvar_activated_by_vendordata(self): - """VMware: envvar transport activated by vendordata""" - self._test_ds_found("VMware-EnvVar-Vendordata") - - def test_vmware_guestinfo_no_data(self): - """VMware: guestinfo transport no data""" - self._test_ds_not_found("VMware-GuestInfo-NoData-Rpctool") - self._test_ds_not_found("VMware-GuestInfo-NoData-Vmtoolsd") - - def test_vmware_guestinfo_no_virt_id(self): - """VMware: guestinfo transport fails if no virt id""" - self._test_ds_not_found("VMware-GuestInfo-NoVirtID") - - def test_vmware_guestinfo_activated_by_metadata(self): - """VMware: guestinfo transport activated by metadata""" - self._test_ds_found("VMware-GuestInfo-Metadata") - - def test_vmware_guestinfo_activated_by_userdata(self): - """VMware: guestinfo transport activated by userdata""" - self._test_ds_found("VMware-GuestInfo-Userdata") - - def test_vmware_guestinfo_activated_by_vendordata(self): - """VMware: guestinfo transport activated by vendordata""" - self._test_ds_found("VMware-GuestInfo-Vendordata") - - def test_vmware_ovf_on_vmware_with_guestinfo_metadata_and_ovf_schema(self): - """VMware and OVF are identified when: - 1. On VMware platform. - 2. guestinfo transport activated by metadata - 3. iso9660 cdrom path contains ovf schema.""" - self._test_ds_found( - "VMware-OVF-on-vmware-with-guestinfo-metadata-and-ovf-schema" - ) - - def test_ovf_not_on_vmware_with_guestinfo_metadata_and_ovf_schema(self): - """OVF is identified when: - 1. Not on VMware platform. - 2. guestinfo transport activated by metadata - 3. iso9660 cdrom path contains ovf schema.""" - self._test_ds_found( - "OVF-not-on-vmware-with-guestinfo-metadata-and-ovf-schema" + cust64, str(tmp_path), RC_FOUND, dslist=[cust64.get("ds"), DS_NONE] ) +@pytest.mark.allow_subp_for("sh") class TestAkamai(DsIdentifyBase): - def test_found_by_sys_vendor(self): - """ds-identify finds Akamai by system-manufacturer dmi field""" - self._test_ds_found("Akamai") - - def test_found_by_sys_vendor_akamai(self): + def test_found_by_sys_vendor_akamai(self, tmp_path): """ ds-identify finds Akamai by system-manufacturer dmi field when set with name "Akamai" (expected in the future) """ cfg = copy.deepcopy(VALID_CFG["Akamai"]) cfg["mocks"][0]["RET"] = "Akamai" - self._check_via_dict(cfg, rc=RC_FOUND) + self._check_via_dict(cfg, str(tmp_path), rc=RC_FOUND) - def test_not_found(self): + def test_not_found(self, tmp_path): """ds-identify does not find Akamai by system-manufacturer field""" cfg = copy.deepcopy(VALID_CFG["Akamai"]) cfg["mocks"][0]["RET"] = "Other" - self._check_via_dict(cfg, rc=RC_NOT_FOUND) - - -class TestBSDNoSys(DsIdentifyBase): - """Test *BSD code paths - - FreeBSD doesn't have /sys so we use kenv(1) here. - OpenBSD uses sysctl(8). - Other BSD systems fallback to dmidecode(8). - BSDs also doesn't have systemd-detect-virt(8), so we use sysctl(8) to query - kern.vm_guest, and optionally map it""" - - def test_dmi_kenv(self): - """Test that kenv(1) works on systems which don't have /sys - - This will be used on FreeBSD systems. - """ - self._test_ds_found("Hetzner-kenv") - - def test_dmi_sysctl(self): - """Test that sysctl(8) works on systems which don't have /sys - - This will be used on OpenBSD systems. - """ - self._test_ds_found("Hetzner-sysctl") - - def test_dmi_dmidecode(self): - """Test that dmidecode(8) works on systems which don't have /sys - - This will be used on all other BSD systems. - """ - self._test_ds_found("Hetzner-dmidecode") + self._check_via_dict(cfg, str(tmp_path), rc=RC_NOT_FOUND) +@pytest.mark.allow_subp_for("sh") class TestIsIBMProvisioning(DsIdentifyBase): """Test the is_ibm_provisioning method in ds-identify.""" @@ -1379,19 +1367,23 @@ class TestIsIBMProvisioning(DsIdentifyBase): boot_ref = "/proc/1/environ" funcname = "is_ibm_provisioning" - def test_no_config(self): + def test_no_config(self, tmp_path): """No provisioning config means not provisioning.""" - ret = self.call(files={}, func=self.funcname) - self.assertEqual(shell_false, ret.rc) + ret = self.call(str(tmp_path), files={}, func=self.funcname) + assert shell_false == ret.rc - def test_config_only(self): + def test_config_only(self, tmp_path): """A provisioning config without a log means provisioning.""" - ret = self.call(files={self.prov_cfg: "key=value"}, func=self.funcname) - self.assertEqual(shell_true, ret.rc) + ret = self.call( + str(tmp_path), + files={self.prov_cfg: "key=value"}, + func=self.funcname, + ) + assert shell_true == ret.rc - def test_config_with_old_log(self): + def test_config_with_old_log(self, tmp_path): """A config with a log from previous boot is not provisioning.""" - rootd = self.tmp_dir() + rootd = str(tmp_path) data = { self.prov_cfg: ("key=value\nkey2=val2\n", -10), self.inst_log: ("log data\n", -30), @@ -1399,12 +1391,12 @@ def test_config_with_old_log(self): } populate_dir_with_ts(rootd, data) ret = self.call(rootd=rootd, func=self.funcname) - self.assertEqual(shell_false, ret.rc) - self.assertIn("from previous boot", ret.stderr) + assert shell_false == ret.rc + assert "from previous boot" in ret.stderr - def test_config_with_new_log(self): + def test_config_with_new_log(self, tmp_path): """A config with a log from this boot is provisioning.""" - rootd = self.tmp_dir() + rootd = str(tmp_path) data = { self.prov_cfg: ("key=value\nkey2=val2\n", -10), self.inst_log: ("log data\n", 30), @@ -1412,32 +1404,22 @@ def test_config_with_new_log(self): } populate_dir_with_ts(rootd, data) ret = self.call(rootd=rootd, func=self.funcname) - self.assertEqual(shell_true, ret.rc) - self.assertIn("from current boot", ret.stderr) + assert shell_true == ret.rc + assert "from current boot" in ret.stderr class TestOracle(DsIdentifyBase): - def test_found_by_chassis(self): - """Simple positive test of Oracle by chassis id.""" - self._test_ds_found("Oracle") - - def test_not_found(self): + @pytest.mark.allow_subp_for("sh") + def test_not_found(self, tmp_path): """Simple negative test of Oracle.""" mycfg = copy.deepcopy(VALID_CFG["Oracle"]) mycfg["files"][P_CHASSIS_ASSET_TAG] = "Not Oracle" - self._check_via_dict(mycfg, rc=RC_NOT_FOUND) + self._check_via_dict(mycfg, str(tmp_path), rc=RC_NOT_FOUND) +@pytest.mark.allow_subp_for("sh") class TestWSL(DsIdentifyBase): - def test_not_found_virt(self): - """Simple negative test for WSL due other virt.""" - self._test_ds_not_found("Not-WSL") - - def test_no_fs_mounts(self): - """Negative test by lack of host filesystem mount points.""" - self._test_ds_not_found("WSL-no-host-mounts") - - def test_no_userprofile(self): + def test_no_userprofile(self, tmp_path): """Negative test by failing to read the %USERPROFILE% environment variable. """ @@ -1449,12 +1431,12 @@ def test_no_userprofile(self): "RET": "\r\n", }, ) - self._check_via_dict(data, RC_NOT_FOUND) + self._check_via_dict(data, str(tmp_path), RC_NOT_FOUND) - def test_no_cloudinitdir_in_userprofile(self): + def test_no_cloudinitdir_in_userprofile(self, tmp_path): """Negative test by not finding %USERPROFILE%/.cloud-init.""" data = copy.deepcopy(VALID_CFG["WSL-supported"]) - userprofile = self.tmp_dir() + userprofile = str(tmp_path) data["mocks"].append( { "name": "WSL_profile_dir", @@ -1462,12 +1444,12 @@ def test_no_cloudinitdir_in_userprofile(self): "RET": userprofile, }, ) - self._check_via_dict(data, RC_NOT_FOUND) + self._check_via_dict(data, str(tmp_path), RC_NOT_FOUND) - def test_empty_cloudinitdir(self): + def test_empty_cloudinitdir(self, tmp_path): """Negative test by lack of host filesystem mount points.""" data = copy.deepcopy(VALID_CFG["WSL-supported"]) - userprofile = self.tmp_dir() + userprofile = str(tmp_path) data["mocks"].append( { "name": "WSL_profile_dir", @@ -1477,14 +1459,14 @@ def test_empty_cloudinitdir(self): ) cloudinitdir = os.path.join(userprofile, ".cloud-init") os.mkdir(cloudinitdir) - self._check_via_dict(data, RC_NOT_FOUND) + self._check_via_dict(data, str(tmp_path), RC_NOT_FOUND) - def test_found_fail_due_instance_name_parsing(self): + def test_found_fail_due_instance_name_parsing(self, tmp_path): """WSL datasource detection fail due parsing error even though the file exists. """ data = copy.deepcopy(VALID_CFG["WSL-supported-debian"]) - userprofile = self.tmp_dir() + userprofile = str(tmp_path) data["mocks"].append( { "name": "WSL_profile_dir", @@ -1503,13 +1485,13 @@ def test_found_fail_due_instance_name_parsing(self): os.mkdir(cloudinitdir) filename = os.path.join(cloudinitdir, "cant-findme.user-data") Path(filename).touch() - self._check_via_dict(data, RC_NOT_FOUND) + self._check_via_dict(data, str(tmp_path), RC_NOT_FOUND) Path(filename).unlink() - def test_found_via_userdata_version_codename(self): + def test_found_via_userdata_version_codename(self, tmp_path): """WSL datasource detected by VERSION_CODENAME when no VERSION_ID""" data = copy.deepcopy(VALID_CFG["WSL-supported-debian"]) - userprofile = self.tmp_dir() + userprofile = str(tmp_path) data["mocks"].append( { "name": "WSL_profile_dir", @@ -1521,15 +1503,17 @@ def test_found_via_userdata_version_codename(self): os.mkdir(cloudinitdir) filename = os.path.join(cloudinitdir, "debian-trixie.user-data") Path(filename).touch() - self._check_via_dict(data, RC_FOUND, dslist=[data.get("ds"), DS_NONE]) + self._check_via_dict( + data, str(tmp_path), RC_FOUND, dslist=[data.get("ds"), DS_NONE] + ) Path(filename).unlink() - def test_found_via_userdata(self): + def test_found_via_userdata(self, tmp_path): """ WSL datasource is found on applicable userdata files in cloudinitdir. """ data = copy.deepcopy(VALID_CFG["WSL-supported"]) - userprofile = self.tmp_dir() + userprofile = str(tmp_path) data["mocks"].append( { "name": "WSL_profile_dir", @@ -1564,16 +1548,19 @@ def test_found_via_userdata(self): os.path.join(cloudinitdir, "default.user-data"), ] - for filename in userdata_files: + for i, filename in enumerate(userdata_files): Path(filename).touch() self._check_via_dict( - data, RC_FOUND, dslist=[data.get("ds"), DS_NONE] + data, + str(tmp_path / str(i)), + RC_FOUND, + dslist=[data.get("ds"), DS_NONE], ) # Delete one by one Path(filename).unlink() # Until there is none, making the datasource no longer viable. - self._check_via_dict(data, RC_NOT_FOUND) + self._check_via_dict(data, str(tmp_path / "-final"), RC_NOT_FOUND) def blkid_out(disks=None): diff --git a/tests/unittests/test_features.py b/tests/unittests/test_features.py index c592f790..101f925a 100644 --- a/tests/unittests/test_features.py +++ b/tests/unittests/test_features.py @@ -11,8 +11,8 @@ class TestGetFeatures: def test_feature_without_override(self): # Since features are intended to be overridden downstream, mock them - # all here so new feature flags don't require a new change to this - # unit test. + # all here so that downstream changes to features do not require + # changes to this test. with mock.patch.multiple( "cloudinit.features", ERROR_ON_USER_DATA_FAILURE=True, @@ -23,6 +23,7 @@ def test_feature_without_override(self): NOCLOUD_SEED_URL_APPEND_FORWARD_SLASH=False, APT_DEB822_SOURCE_LIST_FILE=True, MANUAL_NETWORK_WAIT=False, + STRIP_INVALID_MTU=False, ): assert { "ERROR_ON_USER_DATA_FAILURE": True, @@ -33,4 +34,5 @@ def test_feature_without_override(self): "APT_DEB822_SOURCE_LIST_FILE": True, "DEPRECATION_INFO_BOUNDARY": "devel", "MANUAL_NETWORK_WAIT": False, + "STRIP_INVALID_MTU": False, } == features.get_features() diff --git a/tests/unittests/test_helpers.py b/tests/unittests/test_helpers.py index 3ef92bec..ec3abe19 100644 --- a/tests/unittests/test_helpers.py +++ b/tests/unittests/test_helpers.py @@ -7,7 +7,6 @@ from cloudinit import sources from tests.helpers import cloud_init_project_dir, get_top_level_dir -from tests.unittests.helpers import ResourceUsingTestCase class MyDataSource(sources.DataSource): @@ -17,24 +16,24 @@ def get_instance_id(self): return self._instance_id -class TestPaths(ResourceUsingTestCase): - def test_get_ipath_and_instance_id_with_slashes(self): +class TestPaths: + def test_get_ipath_and_instance_id_with_slashes(self, paths): myds = MyDataSource(sys_cfg={}, distro=None, paths={}) myds._instance_id = "/foo/bar" + paths.datasource = myds safe_iid = "_foo_bar" - mypaths = self.getCloudPaths(myds) - self.assertEqual( - os.path.join(mypaths.cloud_dir, "instances", safe_iid), - mypaths.get_ipath(), + assert ( + os.path.join(paths.cloud_dir, "instances", safe_iid) + == paths.get_ipath() ) - def test_get_ipath_and_empty_instance_id_returns_none(self): + def test_get_ipath_and_empty_instance_id_returns_none(self, paths): myds = MyDataSource(sys_cfg={}, distro=None, paths={}) myds._instance_id = None - mypaths = self.getCloudPaths(myds) + paths.datasource = myds - self.assertIsNone(mypaths.get_ipath()) + assert paths.get_ipath() is None class Testcloud_init_project_dir: diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index 6a71c703..4b1998fb 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -13,25 +13,32 @@ from cloudinit import lifecycle, util from cloudinit.analyze.dump import CLOUD_INIT_ASCTIME_FMT from cloudinit.log import loggers -from tests.unittests.helpers import CiTestCase - - -class TestCloudInitLogger(CiTestCase): - def setUp(self): - # set up a logger like cloud-init does in setup_logging, but instead - # of sys.stderr, we'll plug in a StringIO() object so we can see - # what gets logged - logging.Formatter.converter = time.gmtime - self.ci_logs = io.StringIO() - self.ci_root = logging.getLogger() - console = logging.StreamHandler(self.ci_logs) - console.setFormatter(logging.Formatter(loggers.DEFAULT_LOG_FORMAT)) - console.setLevel(logging.DEBUG) - self.ci_root.addHandler(console) - self.ci_root.setLevel(logging.DEBUG) - self.LOG = logging.getLogger("test_cloudinit_logger") - - def test_logger_uses_gmtime(self): + + +@pytest.fixture +def ci_logs(): + return io.StringIO() + + +@pytest.fixture +def log(ci_logs): + # set up a logger like cloud-init does in setup_logging, but instead + # of sys.stderr, we'll plug in a StringIO() object so we can see + # what gets logged + logging.Formatter.converter = time.gmtime + ci_root = logging.getLogger() + console = logging.StreamHandler(ci_logs) + console.setFormatter(logging.Formatter(loggers.DEFAULT_LOG_FORMAT)) + console.setLevel(logging.DEBUG) + ci_root.addHandler(console) + ci_root.setLevel(logging.DEBUG) + LOG = logging.getLogger("test_cloudinit_logger") + return LOG + + +class TestCloudInitLogger: + + def test_logger_uses_gmtime(self, log, ci_logs): """Test that log message have timestamp in UTC (gmtime)""" # Log a message, extract the timestamp from the log entry @@ -56,7 +63,7 @@ def remove_tz(_dt: datetime.datetime) -> datetime.datetime: datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(0, 0.5) ) - self.LOG.error("Test message") + log.error("Test message") utc_after = remove_tz( datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(0, 0.5) @@ -64,16 +71,16 @@ def remove_tz(_dt: datetime.datetime) -> datetime.datetime: # extract timestamp from log: # 2017-08-23 14:19:43,069 - test_log.py[ERROR]: Test message - logstr = self.ci_logs.getvalue().splitlines()[0] + logstr = ci_logs.getvalue().splitlines()[0] timestampstr = logstr.split(" - ")[0] parsed_dt = datetime.datetime.strptime( timestampstr, CLOUD_INIT_ASCTIME_FMT ) - self.assertLess(utc_before, parsed_dt) - self.assertLess(parsed_dt, utc_after) - self.assertLess(utc_before, utc_after) - self.assertGreater(utc_after, parsed_dt) + assert utc_before < parsed_dt + assert parsed_dt < utc_after + assert utc_before < utc_after + assert utc_after > parsed_dt class TestDeprecatedLogs: diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index efb71618..369f3b31 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -100,7 +100,7 @@ def make_dict(max_depth, seed=None): return _make_dict(0, max_depth, rand) -class TestSimpleRun(helpers.ResourceUsingTestCase): +class TestSimpleRun: def _load_merge_files(self): merge_root = helpers.resourceLocation("merge_sources") tests = [] @@ -138,7 +138,7 @@ def test_seed_runs(self): for test in test_dicts: c = _old_mergemanydict(*test) d = util.mergemanydict(test) - self.assertEqual(c, d) + assert c == d def test_merge_cc_samples(self): tests = self._load_merge_files() @@ -162,7 +162,7 @@ def test_merge_cc_samples(self): merged_buf, expected_merge, ) - self.assertEqual(expected_merge, merged_buf, msg=fail_msg) + assert expected_merge == merged_buf, fail_msg def test_compat_merges_dict(self): a = { @@ -174,7 +174,7 @@ def test_compat_merges_dict(self): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEqual(c, d) + assert c == d def test_compat_merges_dict2(self): a = { @@ -189,7 +189,7 @@ def test_compat_merges_dict2(self): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEqual(c, d) + assert c == d def test_compat_merges_list(self): a = {"b": [1, 2, 3]} @@ -197,7 +197,7 @@ def test_compat_merges_list(self): c = {"b": [6, 7]} e = _old_mergemanydict(a, b, c) f = util.mergemanydict([a, b, c]) - self.assertEqual(e, f) + assert e == f def test_compat_merges_str(self): a = {"b": "hi"} @@ -205,7 +205,7 @@ def test_compat_merges_str(self): c = {"b": "hallo"} e = _old_mergemanydict(a, b, c) f = util.mergemanydict([a, b, c]) - self.assertEqual(e, f) + assert e == f def test_compat_merge_sub_dict(self): a = { @@ -229,7 +229,7 @@ def test_compat_merge_sub_dict(self): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEqual(c, d) + assert c == d def test_compat_merge_sub_dict2(self): a = { @@ -245,7 +245,7 @@ def test_compat_merge_sub_dict2(self): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEqual(c, d) + assert c == d def test_compat_merge_sub_list(self): a = { @@ -261,7 +261,7 @@ def test_compat_merge_sub_list(self): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEqual(c, d) + assert c == d class TestMergingSchema: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 5e57beab..b60dedf4 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -3215,6 +3215,8 @@ def test_ipv4_and_ipv6_static_config(self, yaml_file, config, caplog): ("v6_and_v4", "yaml"), ("v1-dns", "yaml"), ("v2-mixed-routes", "yaml"), + ("v2-mixed-routes-reversed", "yaml"), + ("v2-mixed-routes-no-ipv6-addr", "yaml"), ("v2-dns", "yaml"), ("v2-dns-no-if-ips", "yaml"), ("v2-dns-no-dhcp", "yaml"), @@ -3372,6 +3374,110 @@ class TestNetplanNetRendering: """, id="default_generation", ), + # Asserts a netconf v1 with subnet metrics for dhcp4 and dhcp6 + # is properly rendered in Netplan v2 + pytest.param( + """ + version: 1 + config: + - type: physical + name: eth0 + mac_address: '00:11:22:33:44:55' + subnets: + - type: dhcp4 + metric: 100 + - type: dhcp6 + metric: 200 + """, + """ + network: + version: 2 + ethernets: + eth0: + dhcp4: true + dhcp4-overrides: + route-metric: 100 + dhcp6: true + dhcp6-overrides: + route-metric: 200 + match: + macaddress: 00:11:22:33:44:55 + set-name: eth0 + """, + id="subnet_metric_in_dhcp", + ), + # Asserts a netconf v1 with gateway and metric + # is properly rendered in Netplan v2 + pytest.param( + """ + version: 1 + config: + - type: physical + name: eth0 + mac_address: '00:11:22:33:44:55' + subnets: + - type: static + address: 192.168.1.10/24 + gateway: 192.168.1.1 + metric: 100 + """, + """ + network: + version: 2 + ethernets: + eth0: + addresses: + - 192.168.1.10/24 + match: + macaddress: 00:11:22:33:44:55 + routes: + - to: default + via: 192.168.1.1 + metric: 100 + set-name: eth0 + """, + id="gateway_with_metric", + ), + # Asserts a netconf v1 with static routes and metrics + # is properly rendered in Netplan v2 + pytest.param( + """ + version: 1 + config: + - type: physical + name: eth0 + mac_address: '00:11:22:33:44:55' + subnets: + - type: static + address: 192.168.1.10/24 + routes: + - destination: 10.0.0.0/8 + gateway: 192.168.1.254 + metric: 100 + - destination: 172.16.0.0/12 + gateway: 192.168.1.254 + metric: 200 + """, + """ + network: + version: 2 + ethernets: + eth0: + addresses: + - 192.168.1.10/24 + match: + macaddress: 00:11:22:33:44:55 + routes: + - to: 10.0.0.0/8 + via: 192.168.1.254 + metric: 100 + - to: 172.16.0.0/12 + via: 192.168.1.254 + metric: 200 + set-name: eth0 + """, + id="static_routes_with_metrics", + ), # Asserts a netconf v1 with a physical device and two gateways # does not produce deprecated keys, `gateway{46}`, in Netplan v2 pytest.param( @@ -4436,9 +4542,12 @@ def testsimple_convert_and_render(self): pytest.param( "v1-dns", "yaml", marks=pytest.mark.xfail(reason="GH-4219") ), - pytest.param( - "v2-dns", "yaml", marks=pytest.mark.xfail(reason="GH-4219") - ), + ("v2-dns", "yaml"), + ("v2-mixed-routes", "yaml"), + ("v2-mixed-routes-reversed", "yaml"), + ("v2-mixed-routes-no-ipv6-addr", "yaml"), + ("v2-dns-no-if-ips", "yaml"), + ("v2-dns-no-dhcp", "yaml"), ], ) def test_config(self, expected_name, yaml_version): @@ -5090,7 +5199,7 @@ def test_gi_excludes_any_without_mac_address(self, mocks): def test_gi_excludes_stolen_macs(self, mocks): ret = net.get_interfaces() mocks["interface_has_own_mac"].assert_has_calls( - [mock.call("enp0s1"), mock.call("bond1")], any_order=True + [mock.call("enp0s1")], any_order=True ) expected = [ ("enp0s2", "aa:aa:aa:aa:aa:02", "e1000", "0x5"), diff --git a/tests/unittests/test_net_activators.py b/tests/unittests/test_net_activators.py index 84876b73..4cbb6f10 100644 --- a/tests/unittests/test_net_activators.py +++ b/tests/unittests/test_net_activators.py @@ -83,7 +83,7 @@ def unavailable_mocks(): class TestSearchAndSelect: def test_empty_list(self, available_mocks): - resp = search_activator(priority=DEFAULT_PRIORITY, target=None) + resp = search_activator(priority=DEFAULT_PRIORITY) assert resp == NAME_TO_ACTIVATOR[DEFAULT_PRIORITY[0]] activator = select_activator() @@ -91,19 +91,12 @@ def test_empty_list(self, available_mocks): def test_priority(self, available_mocks): new_order = ["netplan", "network-manager"] - resp = search_activator(priority=new_order, target=None) + resp = search_activator(priority=new_order) assert resp == NetplanActivator activator = select_activator(priority=new_order) assert activator == NetplanActivator - def test_target(self, available_mocks): - search_activator(priority=DEFAULT_PRIORITY, target="/tmp") - assert "/tmp" == available_mocks.m_which.call_args[1]["target"] - - select_activator(target="/tmp") - assert "/tmp" == available_mocks.m_which.call_args[1]["target"] - @patch( "cloudinit.net.activators.IfUpDownActivator.available", return_value=False, @@ -111,7 +104,7 @@ def test_target(self, available_mocks): def test_first_not_available(self, m_available, available_mocks): # We've mocked out IfUpDownActivator as unavailable, so expect the # next in the list of default priorities - resp = search_activator(priority=DEFAULT_PRIORITY, target=None) + resp = search_activator(priority=DEFAULT_PRIORITY) assert resp == NAME_TO_ACTIVATOR[DEFAULT_PRIORITY[1]] resp = select_activator() @@ -119,12 +112,12 @@ def test_first_not_available(self, m_available, available_mocks): def test_priority_not_exist(self, available_mocks): with pytest.raises(ValueError): - search_activator(priority=["spam", "eggs"], target=None) + search_activator(priority=["spam", "eggs"]) with pytest.raises(ValueError): select_activator(priority=["spam", "eggs"]) def test_none_available(self, unavailable_mocks): - resp = search_activator(priority=DEFAULT_PRIORITY, target=None) + resp = search_activator(priority=DEFAULT_PRIORITY) assert resp is None with pytest.raises(NoActivatorException): @@ -132,18 +125,18 @@ def test_none_available(self, unavailable_mocks): IF_UP_DOWN_AVAILABLE_CALLS = [ - (("ifquery",), {"search": ["/sbin", "/usr/sbin"], "target": None}), - (("ifup",), {"search": ["/sbin", "/usr/sbin"], "target": None}), - (("ifdown",), {"search": ["/sbin", "/usr/sbin"], "target": None}), + (("ifquery",), {"search": ["/sbin", "/usr/sbin"]}), + (("ifup",), {"search": ["/sbin", "/usr/sbin"]}), + (("ifdown",), {"search": ["/sbin", "/usr/sbin"]}), ] NETPLAN_AVAILABLE_CALLS = [ - (("netplan",), {"search": ["/usr/sbin", "/sbin"], "target": None}), + (("netplan",), {"search": ["/usr/sbin", "/sbin"]}), ] NETWORKD_AVAILABLE_CALLS = [ - (("ip",), {"search": ["/usr/sbin", "/bin"], "target": None}), - (("systemctl",), {"search": ["/usr/sbin", "/bin"], "target": None}), + (("ip",), {"search": ["/usr/sbin", "/bin"]}), + (("systemctl",), {"search": ["/usr/sbin", "/bin"]}), ] diff --git a/tests/unittests/test_render_template.py b/tests/unittests/test_render_template.py index 0ed94648..7f8fc944 100644 --- a/tests/unittests/test_render_template.py +++ b/tests/unittests/test_render_template.py @@ -6,6 +6,7 @@ from cloudinit import subp, templater, util from tests.helpers import cloud_init_project_dir +from tests.unittests.helpers import skipUnlessJinjaVersionGreaterThan # TODO(Look to align with tools.render-template or cloudinit.distos.OSFAMILIES) DISTRO_VARIANTS = [ @@ -22,6 +23,7 @@ "netbsd", "openbsd", "photon", + "raspberry-pi-os", "rhel", "suse", "ubuntu", @@ -93,6 +95,7 @@ def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): "amazon": "ec2-user", "rhel": "cloud-user", "centos": "cloud-user", + "raspberry-pi-os": "pi", "unknown": "ubuntu", } default_user = system_cfg["system_info"]["default_user"]["name"] @@ -105,6 +108,7 @@ def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): ("netbsd", ["netbsd"]), ("openbsd", ["openbsd"]), ("ubuntu", ["netplan", "eni", "sysconfig"]), + ("raspberry-pi-os", ["netplan", "network-manager"]), ), ) def test_variant_sets_network_renderer_priority_in_cloud_cfg( @@ -121,3 +125,99 @@ def test_variant_sets_network_renderer_priority_in_cloud_cfg( system_cfg = util.load_yaml(stream.read()) assert renderers == system_cfg["system_info"]["network"]["renderers"] + + +EXPECTED_DEBIAN = """\ +deb testmirror testcodename main +deb-src testmirror testcodename main +deb testsecurity testcodename-security main +deb-src testsecurity testcodename-security main +deb testmirror testcodename-updates main +deb-src testmirror testcodename-updates main +deb testmirror testcodename-backports main +deb-src testmirror testcodename-backports main +""" + +EXPECTED_DEBIAN_DEB822 = """\ +Types: deb deb-src +URIs: testmirror +Suites: testcodename testcodename-updates testcodename-backports +Components: main +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg +Types: deb deb-src +URIs: testsecurity +Suites: testcodename-security +Components: main +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg +""" + +EXPECTED_UBUNTU = """\ +deb testmirror testcodename main restricted +deb testmirror testcodename-updates main restricted +deb testmirror testcodename universe +deb testmirror testcodename-updates universe +deb testmirror testcodename multiverse +deb testmirror testcodename-updates multiverse +deb testmirror testcodename-backports main restricted universe multiverse +deb testsecurity testcodename-security main restricted +deb testsecurity testcodename-security universe +deb testsecurity testcodename-security multiverse +""" + +EXPECTED_UBUNTU_DEB822 = """\ +Types: deb +URIs: testmirror +Suites: testcodename testcodename-updates testcodename-backports +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +Types: deb +URIs: testsecurity +Suites: testcodename-security +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +""" + + +class TestRenderSourcesList: + @pytest.mark.parametrize( + "template_path,expected", + [ + pytest.param( + "templates/sources.list.debian.tmpl", + EXPECTED_DEBIAN, + id="debian", + ), + pytest.param( + "templates/sources.list.debian.deb822.tmpl", + EXPECTED_DEBIAN_DEB822, + id="debian_822", + ), + pytest.param( + "templates/sources.list.ubuntu.tmpl", + EXPECTED_UBUNTU, + id="ubuntu", + ), + pytest.param( + "templates/sources.list.ubuntu.deb822.tmpl", + EXPECTED_UBUNTU_DEB822, + id="ubuntu_822", + ), + ], + ) + @skipUnlessJinjaVersionGreaterThan((3, 0, 0)) + def test_render_sources_list_templates( + self, tmpdir, template_path, expected + ): + params = { + "mirror": "testmirror", + "security": "testsecurity", + "codename": "testcodename", + } + template_path = cloud_init_project_dir(template_path) + rendered = templater.render_string(open(template_path).read(), params) + filtered = "\n".join( + line + for line in rendered.splitlines() + if line.strip() and not line.strip().startswith("#") + ) + assert filtered.strip() == expected.strip() diff --git a/tests/unittests/test_signal_handler.py b/tests/unittests/test_signal_handler.py index bb162799..98a98e49 100644 --- a/tests/unittests/test_signal_handler.py +++ b/tests/unittests/test_signal_handler.py @@ -1,51 +1,31 @@ """cloudinit.signal_handler tests""" -import inspect +import re import signal -from unittest.mock import Mock, patch +import sys -import pytest - -from cloudinit import signal_handler - -REENTRANT = "reentrant" +from cloudinit.signal_handler import _handle_exit class TestSignalHandler: - - @pytest.mark.parametrize( - "m_args", - [ - (signal.SIGINT, inspect.currentframe()), - (9, None), - (signal.SIGTERM, None), - (1, inspect.currentframe()), - ], - ) - @pytest.mark.parametrize( - "m_suspended", - [ - (REENTRANT, 0), - (True, 0), - (False, 1), - ], - ) - def test_suspend_signal(self, m_args, m_suspended): - """suspend_crash should prevent crashing (exit 1) on signal - - otherwise cloud-init should exit 1 - """ - sig, frame = m_args - suspended, rc = m_suspended - - with patch.object(signal_handler.sys, "exit", Mock()) as m_exit: - if suspended is True: - with signal_handler.suspend_crash(): - signal_handler._handle_exit(sig, frame) - elif suspended == REENTRANT: - with signal_handler.suspend_crash(): - with signal_handler.suspend_crash(): - signal_handler._handle_exit(sig, frame) - else: - signal_handler._handle_exit(sig, frame) - m_exit.assert_called_with(rc) + """Test signal_handler.py""" + + def test_handle_exit(self, mocker, caplog): + """Test handle_exit()""" + mocker.patch("cloudinit.signal_handler.sys.exit") + mocker.patch("cloudinit.log.log_util.write_to_console") + + frame = sys._getframe() + _handle_exit(signal.Signals.SIGHUP, frame) + + record = caplog.records[0] + assert record.levelname == "INFO" + assert re.match( + ( + r"Received signal SIGHUP resulting in exit. Cause:\n" + r" Filename:.*test_signal_handler.py\n" + r" Function: test_handle_exit\n" + r" Line number: \d+" + ), + record.message, + ) diff --git a/tests/unittests/test_ssh_util.py b/tests/unittests/test_ssh_util.py index cd78f75b..9f21b78f 100644 --- a/tests/unittests/test_ssh_util.py +++ b/tests/unittests/test_ssh_util.py @@ -561,6 +561,18 @@ def test_without_include(self, tmpdir): expected_conf_file = f"{mycfg}.d/50-cloud-init.conf" assert not os.path.isfile(expected_conf_file) + def test_without_sshd_config(self, tmpdir): + """In some cases /etc/ssh/sshd_config.d exists but /etc/ssh/sshd_config + doesn't. In this case we shouldn't create /etc/ssh/sshd_config but make + /etc/ssh/sshd_config.d/50-cloud-init.conf.""" + mycfg = tmpdir.join("sshd_config") + os.mkdir(os.path.join(tmpdir, "sshd_config.d")) + assert ssh_util.update_ssh_config({"key": "value"}, mycfg) + expected_conf_file = f"{mycfg}.d/50-cloud-init.conf" + assert os.path.isfile(expected_conf_file) + assert not os.path.isfile(mycfg) + assert "key value\n" == util.load_text_file(expected_conf_file) + @pytest.mark.parametrize( "cfg", ["Include {mycfg}.d/*.conf", "Include {mycfg}.d/*.conf # comment"], diff --git a/tests/unittests/test_upgrade.py b/tests/unittests/test_upgrade.py index 32a3d7c2..ac2545e2 100644 --- a/tests/unittests/test_upgrade.py +++ b/tests/unittests/test_upgrade.py @@ -156,6 +156,7 @@ class TestUpgrade: "Vultr": {"netcfg"}, "VMware": { "data_access_method", + "extra_hotplug_udev_rules", "rpctool", "rpctool_fn", }, diff --git a/tests/unittests/test_url_helper.py b/tests/unittests/test_url_helper.py index 10d9430f..e441f85c 100644 --- a/tests/unittests/test_url_helper.py +++ b/tests/unittests/test_url_helper.py @@ -478,7 +478,7 @@ def test_error_cb_false(self, mocker): ) assert m_request.call_count == 1 - def test_exception_503(self, mocker): + def test_exception_503(self, mocker, caplog): mocker.patch("time.sleep") retry_response = requests.Response() @@ -490,7 +490,12 @@ def test_exception_503(self, mocker): m_request = mocker.patch("requests.Session.request", autospec=True) m_request.side_effect = (retry_response, retry_response, good_response) - readurl("http://some/path") + with caplog.at_level(logging.WARNING): + readurl("http://some/path") + assert 2 == caplog.text.count( + "Endpoint returned a 503 error. HTTP endpoint is overloaded." + " Retrying URL (http://some/path)." + ), "Did not find expected logged 503 URL" assert m_request.call_count == 3 diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index ceb98b29..3bfaad6c 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -4,7 +4,6 @@ import base64 import errno -import io import json import logging import os @@ -2172,7 +2171,7 @@ def test_none_returns_default(self): ) -class TestMountinfoParsing(helpers.ResourceUsingTestCase): +class TestMountinfoParsing: def test_invalid_mountinfo(self): line = ( "20 1 252:1 / / rw,relatime - ext4 /dev/mapper/vg0-root" @@ -2185,49 +2184,49 @@ def test_invalid_mountinfo(self): expected = None else: expected = ("/dev/mapper/vg0-root", "ext4", "/") - self.assertEqual(expected, util.parse_mount_info("/", lines)) + assert expected == util.parse_mount_info("/", lines) def test_precise_ext4_root(self): lines = helpers.readResource("mountinfo_precise_ext4.txt").splitlines() expected = ("/dev/mapper/vg0-root", "ext4", "/") - self.assertEqual(expected, util.parse_mount_info("/", lines)) - self.assertEqual(expected, util.parse_mount_info("/usr", lines)) - self.assertEqual(expected, util.parse_mount_info("/usr/bin", lines)) + assert expected == util.parse_mount_info("/", lines) + assert expected == util.parse_mount_info("/usr", lines) + assert expected == util.parse_mount_info("/usr/bin", lines) expected = ("/dev/md0", "ext4", "/boot") - self.assertEqual(expected, util.parse_mount_info("/boot", lines)) - self.assertEqual(expected, util.parse_mount_info("/boot/grub", lines)) + assert expected == util.parse_mount_info("/boot", lines) + assert expected == util.parse_mount_info("/boot/grub", lines) expected = ("/dev/mapper/vg0-root", "ext4", "/") - self.assertEqual(expected, util.parse_mount_info("/home", lines)) - self.assertEqual(expected, util.parse_mount_info("/home/me", lines)) + assert expected == util.parse_mount_info("/home", lines) + assert expected == util.parse_mount_info("/home/me", lines) expected = ("tmpfs", "tmpfs", "/run") - self.assertEqual(expected, util.parse_mount_info("/run", lines)) + assert expected == util.parse_mount_info("/run", lines) expected = ("none", "tmpfs", "/run/lock") - self.assertEqual(expected, util.parse_mount_info("/run/lock", lines)) + assert expected == util.parse_mount_info("/run/lock", lines) def test_raring_btrfs_root(self): lines = helpers.readResource("mountinfo_raring_btrfs.txt").splitlines() expected = ("/dev/vda1", "btrfs", "/") - self.assertEqual(expected, util.parse_mount_info("/", lines)) - self.assertEqual(expected, util.parse_mount_info("/usr", lines)) - self.assertEqual(expected, util.parse_mount_info("/usr/bin", lines)) - self.assertEqual(expected, util.parse_mount_info("/boot", lines)) - self.assertEqual(expected, util.parse_mount_info("/boot/grub", lines)) + assert expected == util.parse_mount_info("/", lines) + assert expected == util.parse_mount_info("/usr", lines) + assert expected == util.parse_mount_info("/usr/bin", lines) + assert expected == util.parse_mount_info("/boot", lines) + assert expected == util.parse_mount_info("/boot/grub", lines) expected = ("/dev/vda1", "btrfs", "/home") - self.assertEqual(expected, util.parse_mount_info("/home", lines)) - self.assertEqual(expected, util.parse_mount_info("/home/me", lines)) + assert expected == util.parse_mount_info("/home", lines) + assert expected == util.parse_mount_info("/home/me", lines) expected = ("tmpfs", "tmpfs", "/run") - self.assertEqual(expected, util.parse_mount_info("/run", lines)) + assert expected == util.parse_mount_info("/run", lines) expected = ("none", "tmpfs", "/run/lock") - self.assertEqual(expected, util.parse_mount_info("/run/lock", lines)) + assert expected == util.parse_mount_info("/run/lock", lines) @mock.patch("cloudinit.subp.subp") def test_parse_mount_with_ext(self, mount_out): @@ -2237,16 +2236,16 @@ def test_parse_mount_with_ext(self, mount_out): ) # this one is valid and exists in mount_parse_ext.txt ret = util.parse_mount("/var") - self.assertEqual(("/dev/mapper/vg00-lv_var", "ext4", "/var"), ret) + assert ("/dev/mapper/vg00-lv_var", "ext4", "/var") == ret # another one that is valid and exists ret = util.parse_mount("/") - self.assertEqual(("/dev/mapper/vg00-lv_root", "ext4", "/"), ret) + assert ("/dev/mapper/vg00-lv_root", "ext4", "/") == ret # this one exists in mount_parse_ext.txt ret = util.parse_mount("/sys/kernel/debug") - self.assertEqual(("none", "debugfs", "/sys/kernel/debug"), ret) + assert ("none", "debugfs", "/sys/kernel/debug") == ret # this one does not exist in mount_parse_ext.txt ret = util.parse_mount("/var/tmp/cloud-init") - self.assertEqual(("/dev/mapper/vg00-lv_var", "ext4", "/var"), ret) + assert ("/dev/mapper/vg00-lv_var", "ext4", "/var") == ret @mock.patch("cloudinit.subp.subp") def test_parse_mount_with_zfs(self, mount_out): @@ -2256,13 +2255,13 @@ def test_parse_mount_with_zfs(self, mount_out): ) # this one is valid and exists in mount_parse_zfs.txt ret = util.parse_mount("/var") - self.assertEqual(("vmzroot/ROOT/freebsd/var", "zfs", "/var"), ret) + assert ("vmzroot/ROOT/freebsd/var", "zfs", "/var") == ret # this one is the root, valid and also exists in mount_parse_zfs.txt ret = util.parse_mount("/") - self.assertEqual(("vmzroot/ROOT/freebsd", "zfs", "/"), ret) + assert ("vmzroot/ROOT/freebsd", "zfs", "/") == ret # this one does not exist in mount_parse_ext.txt ret = util.parse_mount("/var/tmp/cloud-init") - self.assertEqual(("vmzroot/var/tmp", "zfs", "/var/tmp"), ret) + assert ("vmzroot/var/tmp", "zfs", "/var/tmp") == ret class TestIsX86(helpers.CiTestCase): @@ -2368,103 +2367,95 @@ def test_output_logs_parsed_when_teeing_files_and_rotated(self): ) -class TestMultiLog(helpers.FilesystemMockingTestCase): - def _createConsole(self, root): - os.mkdir(os.path.join(root, "dev")) - open(os.path.join(root, "dev", "console"), "a").close() +@pytest.mark.usefixtures("fake_filesystem") +class TestMultiLog: + def _createConsole(self): + os.mkdir("/dev") + open("/dev/console", "w").close() - def setUp(self): - super(TestMultiLog, self).setUp() - self.root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.root) - self.patchOS(self.root) - self.patchUtils(self.root) - self.patchOpen(self.root) - self.stdout = io.StringIO() - self.stderr = io.StringIO() - self.patchStdoutAndStderr(self.stdout, self.stderr) - - def test_stderr_used_by_default(self): + def test_stderr_used_by_default(self, capsys): logged_string = "test stderr output" log_util.multi_log(logged_string) - self.assertEqual(logged_string, self.stderr.getvalue()) + assert logged_string == capsys.readouterr().err - def test_stderr_not_used_if_false(self): + def test_stderr_not_used_if_false(self, capsys): log_util.multi_log("should not see this", stderr=False) - self.assertEqual("", self.stderr.getvalue()) + assert "" == capsys.readouterr().err def test_logs_go_to_console_by_default(self): - self._createConsole(self.root) + self._createConsole() logged_string = "something very important" log_util.multi_log(logged_string) with open("/dev/console") as f: - self.assertEqual(logged_string, f.read()) + assert logged_string == f.read() - def test_logs_dont_go_to_stdout_if_console_exists(self): - self._createConsole(self.root) + def test_logs_dont_go_to_stdout_if_console_exists(self, capsys): + self._createConsole() log_util.multi_log("something") - self.assertEqual("", self.stdout.getvalue()) + assert "" == capsys.readouterr().out - def test_logs_go_to_stdout_if_console_does_not_exist(self): + def test_logs_go_to_stdout_if_console_does_not_exist(self, capsys): logged_string = "something very important" log_util.multi_log(logged_string) - self.assertEqual(logged_string, self.stdout.getvalue()) + assert "something very important" == capsys.readouterr().out - def test_logs_dont_go_to_stdout_if_fallback_to_stdout_is_false(self): + def test_logs_dont_go_to_stdout_if_fallback_to_stdout_is_false( + self, capsys + ): log_util.multi_log("something", fallback_to_stdout=False) - self.assertEqual("", self.stdout.getvalue()) + assert "" == capsys.readouterr().out - @mock.patch( - "cloudinit.log.log_util.write_to_console", - mock.Mock(side_effect=OSError("Failed to write to console")), - ) def test_logs_go_to_stdout_if_writing_to_console_fails_and_fallback_true( - self, + self, mocker, capsys ): - self._createConsole(self.root) + mocker.patch( + "cloudinit.log.log_util.write_to_console", + side_effect=OSError("Failed to write to console"), + ) + self._createConsole() log_util.multi_log("something", fallback_to_stdout=True) - self.assertEqual( - "Failed to write to /dev/console\nsomething", - self.stdout.getvalue(), + assert ( + "Failed to write to /dev/console\nsomething" + == capsys.readouterr().out ) - @mock.patch( - "cloudinit.log.log_util.write_to_console", - mock.Mock(side_effect=OSError("Failed to write to console")), - ) def test_logs_go_nowhere_if_writing_to_console_fails_and_fallback_false( - self, + self, mocker, capsys ): - self._createConsole(self.root) - log_util.multi_log("something", fallback_to_stdout=False) - self.assertEqual( - "Failed to write to /dev/console\n", self.stdout.getvalue() + mocker.patch( + "cloudinit.log.log_util.write_to_console", + mock.Mock(side_effect=OSError("Failed to write to console")), ) + self._createConsole() + log_util.multi_log("something", fallback_to_stdout=False) + assert "Failed to write to /dev/console\n" == capsys.readouterr().out def test_logs_go_to_log_if_given(self): logger = mock.MagicMock() logged_string = "something very important" - log_util.multi_log(logged_string, log=logger) - self.assertEqual( - [((mock.ANY, logged_string), {})], logger.log.call_args_list - ) + log_util.multi_log(logged_string, log=logger, console=False) + assert [((mock.ANY, logged_string), {})] == logger.log.call_args_list def test_newlines_stripped_from_log_call(self): logger = mock.MagicMock() expected_string = "something very important" - log_util.multi_log("{0}\n".format(expected_string), log=logger) - self.assertEqual((mock.ANY, expected_string), logger.log.call_args[0]) + log_util.multi_log( + "{0}\n".format(expected_string), log=logger, console=False + ) + assert mock.ANY, expected_string == logger.log.call_args[0] def test_log_level_defaults_to_debug(self): logger = mock.MagicMock() - log_util.multi_log("message", log=logger) - self.assertEqual((logging.DEBUG, mock.ANY), logger.log.call_args[0]) + log_util.multi_log("message", log=logger, console=False) + assert logging.DEBUG, mock.ANY == logger.log.call_args[0] def test_given_log_level_used(self): logger = mock.MagicMock() log_level = mock.Mock() - log_util.multi_log("message", log=logger, log_level=log_level) - self.assertEqual((log_level, mock.ANY), logger.log.call_args[0]) + log_util.multi_log( + "message", log=logger, log_level=log_level, console=False + ) + assert log_level, mock.ANY == logger.log.call_args[0] class TestMessageFromString(helpers.TestCase): @@ -2779,35 +2770,28 @@ def test_pexec_error_multi_line_msgs(self): ) -class TestSystemIsSnappy(helpers.FilesystemMockingTestCase): +@pytest.mark.usefixtures("fake_filesystem") +class TestSystemIsSnappy: def test_id_in_os_release_quoted(self): """os-release containing ID="ubuntu-core" is snappy.""" orcontent = "\n".join(['ID="ubuntu-core"', ""]) - root_d = self.tmp_dir() - helpers.populate_dir(root_d, {"etc/os-release": orcontent}) - self.reRoot(root_d) - self.assertTrue(util.system_is_snappy()) + helpers.populate_dir("/", {"etc/os-release": orcontent}) + assert util.system_is_snappy() is True def test_id_in_os_release(self): """os-release containing ID=ubuntu-core is snappy.""" orcontent = "\n".join(["ID=ubuntu-core", ""]) - root_d = self.tmp_dir() - helpers.populate_dir(root_d, {"etc/os-release": orcontent}) - self.reRoot(root_d) - self.assertTrue(util.system_is_snappy()) + helpers.populate_dir("/", {"etc/os-release": orcontent}) + assert util.system_is_snappy() is True - @mock.patch(M_PATH + "get_cmdline") - def test_bad_content_in_os_release_no_effect(self, m_cmdline): + def test_bad_content_in_os_release_no_effect(self, mocker): """malformed os-release should not raise exception.""" - m_cmdline.return_value = "root=/dev/sda" + mocker.patch(M_PATH + "get_cmdline", return_value="root=/dev/sda") orcontent = "\n".join(["IDubuntu-core", ""]) - root_d = self.tmp_dir() - helpers.populate_dir(root_d, {"etc/os-release": orcontent}) - self.reRoot() - self.assertFalse(util.system_is_snappy()) + helpers.populate_dir("/", {"etc/os-release": orcontent}) + assert util.system_is_snappy() is False - @mock.patch(M_PATH + "get_cmdline") - def test_snap_core_in_cmdline_is_snappy(self, m_cmdline): + def test_snap_core_in_cmdline_is_snappy(self, mocker): """The string snap_core= in kernel cmdline indicates snappy.""" cmdline = ( "BOOT_IMAGE=(loop)/kernel.img root=LABEL=writable " @@ -2815,38 +2799,32 @@ def test_snap_core_in_cmdline_is_snappy(self, m_cmdline): "net.ifnames=0 init=/lib/systemd/systemd console=tty1 " "console=ttyS0 panic=-1" ) - m_cmdline.return_value = cmdline - self.assertTrue(util.system_is_snappy()) - self.assertTrue(m_cmdline.call_count > 0) + m_cmdline = mocker.patch(M_PATH + "get_cmdline", return_value=cmdline) + assert util.system_is_snappy() is True + assert m_cmdline.call_count > 0 - @mock.patch(M_PATH + "get_cmdline") - def test_nothing_found_is_not_snappy(self, m_cmdline): + def test_nothing_found_is_not_snappy(self, mocker): """If no positive identification, then not snappy.""" - m_cmdline.return_value = "root=/dev/sda" - self.reRoot() - self.assertFalse(util.system_is_snappy()) - self.assertTrue(m_cmdline.call_count > 0) + m_cmdline = mocker.patch( + M_PATH + "get_cmdline", return_value="root=/dev/sda" + ) + assert util.system_is_snappy() is False + assert m_cmdline.call_count > 0 - @mock.patch(M_PATH + "get_cmdline") - def test_channel_ini_with_snappy_is_snappy(self, m_cmdline): + def test_channel_ini_with_snappy_is_snappy(self, mocker): """A Channel.ini file with 'ubuntu-core' indicates snappy.""" - m_cmdline.return_value = "root=/dev/sda" - root_d = self.tmp_dir() + mocker.patch(M_PATH + "get_cmdline", return_value="root=/dev/sda") content = "\n".join(["[Foo]", "source = 'ubuntu-core'", ""]) - helpers.populate_dir(root_d, {"etc/system-image/channel.ini": content}) - self.reRoot(root_d) - self.assertTrue(util.system_is_snappy()) + helpers.populate_dir("/", {"etc/system-image/channel.ini": content}) + assert util.system_is_snappy() is True - @mock.patch(M_PATH + "get_cmdline") - def test_system_image_config_dir_is_snappy(self, m_cmdline): + def test_system_image_config_dir_is_snappy(self, mocker): """Existence of /etc/system-image/config.d indicates snappy.""" - m_cmdline.return_value = "root=/dev/sda" - root_d = self.tmp_dir() + mocker.patch(M_PATH + "get_cmdline", return_value="root=/dev/sda") helpers.populate_dir( - root_d, {"etc/system-image/config.d/my.file": "_unused"} + "/", {"etc/system-image/config.d/my.file": "_unused"} ) - self.reRoot(root_d) - self.assertTrue(util.system_is_snappy()) + assert util.system_is_snappy() is True class TestLoadShellContent(helpers.TestCase): diff --git a/tests/unittests/util.py b/tests/unittests/util.py index 02aa6b1a..72573075 100644 --- a/tests/unittests/util.py +++ b/tests/unittests/util.py @@ -72,6 +72,8 @@ class concreteCls(abclass): class MockDistro(distros.Distro): + fallback_interface = "eth9" + # MockDistro is here to test base Distro class implementations def __init__(self, name="testingdistro", cfg=None, paths=None): self._client = None diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers deleted file mode 100644 index 6e2740ff..00000000 --- a/tools/.github-cla-signers +++ /dev/null @@ -1,234 +0,0 @@ -a-dubs -aciba90 -acourdavAkamai -ader1990 -adobley -afbjorklund -ajmyyra -akhuettel -akutz -AlexBaranowski -alexsander-souza -AlexSv04047 -AliyevH -Aman306 -andgein -andrew-lee-metaswitch -andrewbogott -andrewlukoshko -andy191x -ani-sinha -antonyc -apollo13 -ashuntu -aswinrajamannar -bdrung -beantaxi -beezly -berolinux -bin456789 -bipinbachhao -BirknerAlex -blackhelicoptersdotnet -bmhughes -brianphaley -BrinKe-dev -bryanfraschetti -CalvoM -candlerb -CarlosNihelton -catmsred -cawamata -cclauss -chifac08 -chrislalos -ciprianbadescu -citrus-it -cjp256 -CodeBleu -Conan-Kudo -cpaelzer -cvstealth -dankenigsberg -dankm -dark2phoenix -david-caro -dbungert -ddstreet -ddstreetmicrosoft -ddymko -dermotbradley -dhalturin -dhensby -Dorthu -eaglegai -eandersson -eb3095 -ederst -edudobay -einsibjarni -emmanuelthome -eslerm -esposem -fionn -frantisekz -frikilax -frittentheke -GabrielNagy -garzdin -gglzf4 -giggsoff -gilbsgilbs -glyg -halfdime-code -hamalq -hamistao -hcartiaux -holmanb -impl -Indrranil -irishgordo -ITJamie -itsaviral2609 -ixjhuang -izzyleung -j5awry -jacobsalmela -jamesottinger -jberner12 -jcmoore3 -Jehops -jessealter -jf -jfroche -jgrassler -Jille -jinkkkang -JohnKepplers -johnsonshi -jordimassaguerpla -jqueuniet -jsf9k -jshen28 -jumpojoy -kadiron -kaiwalyakoparkar -kallioli -klausenbusk -KsenijaS -landon912 -ld9379435 -leavelet -licebmi -linitio -LKHN -lkundrak -LRitzdorf -lucasmoura -lucendio -lungj -magnetikonline -MaheshG11 -major -mal -mamercad -ManassehZhou -manuelisimo -MarkMielke -marlluslustosa -masihkhatibzadeh99 -mathmarchand -matthewruffell -maxnet -Mazorius -megian -metajiji -michaelrommel -mitechie -MjMoshiri -MostafaTarek124eru -mxwebdev -nazunalika -nelsonad-ops -netcho -nicolasbock -nishigori -nkukard -nmeyerhans -NoSuchCommand -ogayot -olivierlemasle -omBratteng -onitake -orndorffgrant -Oursin -outscale-mdr -philsphicas -phsm -phunyguy -pneigel-ca -qubidt -r00ta -RedKrieg -renanrodrigo -rhansen -riedel -rishitashaw -rmhsawyer -RomainDusi -rongz609 -s-makin -SadeghHayeri -sarahwzadara -sbraz -scorpion44 -SeanSith -shaardie -shaerpour -shell-skrimp -shi2wei3 -ShPakvel -simondeziel -slingamn -slyon -smoser -SRv6d -sshedi -sstallion -stappersg -stefanor -steverweber -t-8ch -taoyama -TheRealFalcon -thetoolsmith -thunderboltsid -timothegenzmer -tnt-dev -tobias-urdin -tomponline -tsanghan -tSU-RooT -tyb-truth -tylerschultz -us0310306 -vorlonofportland -vteratipally -Vultaire -waveform80 -WebSpider -weizhouapache -wideawakening -Wind-net -wmousa -wschoot -wynnfeng -xiachen-rh -xiaoge1001 -xnox -yangzz-97 -yawkat -zhan9san -zhuzaifangxuele -zimbatm -zykovd diff --git a/tools/.lp-to-git-user b/tools/.lp-to-git-user deleted file mode 100644 index 4df62976..00000000 --- a/tools/.lp-to-git-user +++ /dev/null @@ -1,37 +0,0 @@ -{ - "adam-collard": "sparkiegeek", - "adobrawy": "ad-m", - "afranceschini": "andreaf74", - "ahosmanmsft": "AOhassan", - "andreipoltavchenko": "pa-yourserveradmin-com", - "askon": "ask0n", - "b1sandmann": "B1Sandmann", - "bitfehler": "bitfehler", - "chad.smith": "blackboxsw", - "chcheng": "chengcheng-chcheng", - "d-info-e": "do3meli", - "daniel-thewatkins": "OddBloke", - "eric-lafontaine1": "elafontaine", - "fredlefebvre": "fred-lefebvre", - "goneri": "goneri", - "harald-jensas": "hjensas", - "i.galic": "igalic", - "kgarloff": "garloff", - "killermoehre": "killermoehre", - "larsks": "larsks", - "legovini": "paride", - "louis": "karibou", - "lp-markusschade": "asciiprod", - "madhuri-rai07": "madhuri-rai07", - "momousta": "Moustafa-Moustafa", - "otubo": "otubo", - "pengpengs": "PengpengSun", - "powersj": "powersj", - "raharper": "raharper", - "rjschwei": "rjschwei", - "tribaal": "chrisglass", - "trstringer": "trstringer", - "vlastimil-holer": "vholer", - "vtqanh": "anhvoms", - "xiaofengw": "xiaofengw-vmware" -} \ No newline at end of file diff --git a/tools/check-cla-signers b/tools/check-cla-signers deleted file mode 100755 index 5f8a9ade..00000000 --- a/tools/check-cla-signers +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -eu -set -o pipefail -export LC_ALL="C.UTF-8" - -CLA_SIGNERS_FILE="tools/.github-cla-signers" - -sort -f "${CLA_SIGNERS_FILE}" -o "${CLA_SIGNERS_FILE}" - -if [[ -n "$(git status --porcelain -- ${CLA_SIGNERS_FILE})" ]]; then - echo "Please make sure that ${CLA_SIGNERS_FILE} is in alphabetical order. Make sure to set LC_ALL=\"${LC_ALL}\"" - git --no-pager diff "${CLA_SIGNERS_FILE}" - exit 1 -fi diff --git a/tools/ds-identify b/tools/ds-identify index e6efa407..b531cfd0 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -951,11 +951,6 @@ dscheck_MAAS() { return ${DS_FOUND} ;; esac - - # check config files written by maas for installed system. - if check_config "MAAS"; then - return "${DS_FOUND}" - fi return ${DS_NOT_FOUND} } diff --git a/tools/migrate-lp-user-to-github b/tools/migrate-lp-user-to-github deleted file mode 100755 index c2d1dd39..00000000 --- a/tools/migrate-lp-user-to-github +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env python3 -"""Link your Launchpad user to GitHub, proposing branches to LP and GitHub""" - -from argparse import ArgumentParser -from subprocess import Popen, PIPE -import os -import sys - -try: - from launchpadlib.launchpad import Launchpad -except ImportError: - print( - "Missing python launchpadlib dependency to create branches for you." - "Install with: sudo apt-get install python3-launchpadlib" - ) - sys.exit(1) - -if "avoid-pep8-E402-import-not-top-of-file": - _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) - sys.path.insert(0, _tdir) - from cloudinit import util - - -DRYRUN = False -LP_TO_GIT_USER_FILE = ".lp-to-git-user" -MIGRATE_BRANCH_NAME = "migrate-lp-to-github" -GITHUB_PULL_URL = "https://github.com/canonical/cloud-init/compare/main...{github_user}:{branch}" -GH_UPSTREAM_URL = "https://github.com/canonical/cloud-init" - - -def error(message): - if isinstance(message, bytes): - message = message.decode("utf-8") - log("ERROR: {error}".format(error=message)) - sys.exit(1) - - -def log(message): - print(message) - - -def subp(cmd, skip=False): - prefix = "SKIPPED: " if skip else "$ " - log("{prefix}{command}".format(prefix=prefix, command=" ".join(cmd))) - if skip: - return - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - out, err = proc.communicate() - if proc.returncode: - error(err if err else out) - return out.decode("utf-8") - - -LP_GIT_PATH_TMPL = "git+ssh://{launchpad_user}@git.launchpad.net/" -LP_UPSTREAM_PATH_TMPL = LP_GIT_PATH_TMPL + "cloud-init" -LP_REMOTE_PATH_TMPL = LP_GIT_PATH_TMPL + "~{launchpad_user}/cloud-init" -GITHUB_REMOTE_PATH_TMPL = "git@github.com:{github_user}/cloud-init.git" - - -# Comment templates -COMMIT_MSG_TMPL = """\ -lp-to-git-users: adding {gh_username} - -Mapped from {lp_username} -""" -PUBLISH_DIR = "/tmp/cloud-init-lp-to-github-migration" - - -def get_parser(): - parser = ArgumentParser(description=__doc__) - parser.add_argument( - "--dryrun", - required=False, - default=False, - action="store_true", - help=( - "Run commands and review operation in dryrun mode, " - "making not changes." - ), - ) - parser.add_argument("launchpad_user", help="Your launchpad username.") - parser.add_argument("github_user", help="Your github username.") - parser.add_argument( - "--local-repo-dir", - required=False, - dest="repo_dir", - help=( - "The name of the local directory into which we clone." - " Default: {}".format(PUBLISH_DIR) - ), - ) - parser.add_argument( - "--upstream-branch", - required=False, - dest="upstream", - default="origin/main", - help=( - "The name of remote branch target into which we will merge." - " Default: origin/main" - ), - ) - parser.add_argument( - "-v", - "--verbose", - required=False, - default=False, - action="store_true", - help=("Print all actions."), - ) - return parser - - -def create_publish_branch(upstream, publish_branch): - """Create clean publish branch target in the current git repo.""" - branches = subp(["git", "branch"]) - upstream_remote, upstream_branch = upstream.split("/", 1) - subp(["git", "checkout", upstream_branch]) - subp(["git", "pull"]) - if publish_branch in branches: - subp(["git", "branch", "-D", publish_branch]) - subp(["git", "checkout", upstream, "-b", publish_branch]) - - -def add_lp_and_github_remotes(lp_user, gh_user): - """Add lp and github remotes if not present. - - @return Tuple with (lp_remote_name, gh_remote_name) - """ - lp_remote = LP_REMOTE_PATH_TMPL.format(launchpad_user=lp_user) - gh_remote = GITHUB_REMOTE_PATH_TMPL.format(github_user=gh_user) - remotes = subp(["git", "remote", "-v"]) - lp_remote_name = gh_remote_name = None - for remote in remotes.splitlines(): - if not remote: - continue - remote_name, remote_url, _operation = remote.split() - if lp_remote == remote_url: - lp_remote_name = remote_name - elif gh_remote == remote_url: - gh_remote_name = remote_name - if not lp_remote_name: - log( - "launchpad: Creating git remote launchpad-{} to point at your" - " LP repo".format(lp_user) - ) - lp_remote_name = "launchpad-{}".format(lp_user) - subp(["git", "remote", "add", lp_remote_name, lp_remote]) - try: - subp(["git", "fetch", lp_remote_name]) - except: - log("launchpad: Pushing to ensure LP repo exists") - subp(["git", "push", lp_remote_name, "main:main"]) - subp(["git", "fetch", lp_remote_name]) - if not gh_remote_name: - log( - "github: Creating git remote github-{} to point at your" - " GH repo".format(gh_user) - ) - gh_remote_name = "github-{}".format(gh_user) - subp(["git", "remote", "add", gh_remote_name, gh_remote]) - try: - subp(["git", "fetch", gh_remote_name]) - except: - log( - "ERROR: [github] Could not fetch remote '{remote}'." - "Please create a fork for your github user by clicking 'Fork'" - " from {gh_upstream}".format( - remote=gh_remote, gh_upstream=GH_UPSTREAM_URL - ) - ) - sys.exit(1) - return (lp_remote_name, gh_remote_name) - - -def create_migration_branch( - branch_name, upstream, lp_user, gh_user, commit_msg -): - """Create an LP to GitHub migration branch and add lp_user->gh_user.""" - log( - "Creating a migration branch: {} adding your users".format( - MIGRATE_BRANCH_NAME - ) - ) - create_publish_branch(upstream, MIGRATE_BRANCH_NAME) - lp_to_git_map = {} - lp_to_git_file = os.path.join(os.getcwd(), "tools", LP_TO_GIT_USER_FILE) - if os.path.exists(lp_to_git_file): - with open(lp_to_git_file) as stream: - lp_to_git_map = util.load_json(stream.read()) - if gh_user in lp_to_git_map.values(): - raise RuntimeError( - "github user '{}' already in {}".format(gh_user, lp_to_git_file) - ) - if lp_user in lp_to_git_map: - raise RuntimeError( - "launchpad user '{}' already in {}".format(lp_user, lp_to_git_file) - ) - lp_to_git_map[lp_user] = gh_user - with open(lp_to_git_file, "w") as stream: - stream.write(util.json_dumps(lp_to_git_map)) - subp(["git", "add", lp_to_git_file]) - commit_file = os.path.join(os.path.dirname(os.getcwd()), "commit.msg") - with open(commit_file, "wb") as stream: - stream.write(commit_msg.encode("utf-8")) - subp(["git", "commit", "--all", "-F", commit_file]) - - -def main(): - global DRYRUN - global VERBOSITY - parser = get_parser() - args = parser.parse_args() - DRYRUN = args.dryrun - VERBOSITY = 1 if args.verbose else 0 - repo_dir = args.repo_dir or PUBLISH_DIR - if not os.path.exists(repo_dir): - cleanup_repo_dir = True - subp( - [ - "git", - "clone", - LP_UPSTREAM_PATH_TMPL.format( - launchpad_user=args.launchpad_user - ), - repo_dir, - ] - ) - else: - cleanup_repo_dir = False - cwd = os.getcwd() - os.chdir(repo_dir) - log("Syncing main branch with upstream") - subp(["git", "checkout", "main"]) - subp(["git", "pull"]) - try: - lp_remote_name, gh_remote_name = add_lp_and_github_remotes( - args.launchpad_user, args.github_user - ) - commit_msg = COMMIT_MSG_TMPL.format( - gh_username=args.github_user, lp_username=args.launchpad_user - ) - create_migration_branch( - MIGRATE_BRANCH_NAME, - args.upstream, - args.launchpad_user, - args.github_user, - commit_msg, - ) - - for push_remote in (lp_remote_name, gh_remote_name): - subp(["git", "push", push_remote, MIGRATE_BRANCH_NAME, "--force"]) - except Exception as e: - error("Failed setting up migration branches: {0}".format(e)) - finally: - os.chdir(cwd) - if cleanup_repo_dir and os.path.exists(repo_dir): - util.del_dir(repo_dir) - # Make merge request on LP - log("[launchpad] Automatically creating merge proposal using launchpadlib") - lp = Launchpad.login_with( - "server-team github-migration tool", "production", version="devel" - ) - main = lp.git_repositories.getByPath(path="cloud-init").getRefByPath( - path="main" - ) - LP_BRANCH_PATH = "~{launchpad_user}/cloud-init/+git/cloud-init" - lp_git_repo = lp.git_repositories.getByPath( - path=LP_BRANCH_PATH.format(launchpad_user=args.launchpad_user) - ) - lp_user_migrate_branch = lp_git_repo.getRefByPath( - path="refs/heads/migrate-lp-to-github" - ) - lp_merge_url = ( - "https://code.launchpad.net/" - + LP_BRANCH_PATH.format(launchpad_user=args.launchpad_user) - + "/+ref/" - + MIGRATE_BRANCH_NAME - ) - try: - lp_user_migrate_branch.createMergeProposal( - commit_message=commit_msg, merge_target=main, needs_review=True - ) - except Exception: - log( - "[launchpad] active merge proposal already exists at:\n" - "{url}\n".format(url=lp_merge_url) - ) - else: - log( - "[launchpad] Merge proposal created at:\n{url}.\n".format( - url=lp_merge_url - ) - ) - log( - "To link your account to github open your browser and" - " click 'Create pull request' at the following URL:\n" - "{url}".format( - url=GITHUB_PULL_URL.format( - github_user=args.github_user, branch=MIGRATE_BRANCH_NAME - ) - ) - ) - if os.path.exists(repo_dir): - util.del_dir(repo_dir) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/read-dependencies b/tools/read-dependencies index 6e7bf461..934c88aa 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -26,6 +26,7 @@ DISTRO_PKG_TYPE_MAP = { "centos": "redhat", "eurolinux": "redhat", "miraclelinux": "redhat", + "fedora": "redhat", "rocky": "redhat", "redhat": "redhat", "debian": "debian", @@ -46,11 +47,35 @@ MAYBE_RELIABLE_YUM_INSTALL = [ """ error() { echo "$@" 1>&2; } configure_repos_for_proxy_use() { - grep -q "^proxy=" /etc/yum.conf || return 0 + if [ -f /etc/yum.conf ]; then + CFG_FILE=/etc/yum.conf + fi + if [ -f /etc/dnf/dnf.conf ]; then + CFG_FILE=/etc/dnf/dnf.conf + fi + if [ ! "${CFG_FILE}" ]; then + error "No yum.conf or dnf.conf" + return 0 + fi + grep -q "^proxy=" $CFG_FILE || return 0 error ":: http proxy in use => forcing the use of fixed URLs in /etc/yum.repos.d/*.repo" sed -i --regexp-extended '/^#baseurl=/s/#// ; /^(mirrorlist|metalink)=/s/^/#/' /etc/yum.repos.d/*.repo - sed -i 's/download\.fedoraproject\.org/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo - sed -i 's/download\.example/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo + sed -i 's/download\\.fedoraproject\\.org/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo + sed -i 's/download\\.example/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo + CENTOS_REPO="/etc/yum.repos.d/centos.repo" + if [ -f $CENTOS_REPO ]; then + grep -q baseurl $CENTOS_REPO + if [ $? -eq 1 ]; then + # CentOS 9 does not provide baseurl definitions + sed -i '/\[baseos\]/a baseurl=https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os' ${CENTOS_REPO} + sed -i '/\[appstream\]/a baseurl=https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os' ${CENTOS_REPO} + sed -i '/\[crb\]/a baseurl=https://mirror.stream.centos.org/9-stream/CRB/x86_64/os' ${CENTOS_REPO} + fi + CENTOS_EXTRAS_REPO="/etc/yum.repos.d/centos-addons.repo" + if [ -f $CENTOS_EXTRAS_REPO ]; then + sed -i '/\[extras-common\]/a baseurl=https://mirror.stream.centos.org/SIGs/9-stream/extras/x86_64/extras-common' ${CENTOS_EXTRAS_REPO} + fi + fi } configure_repos_for_proxy_use n=0; max=10; @@ -68,7 +93,7 @@ MAYBE_RELIABLE_YUM_INSTALL = [ error ":: running yum install --cacheonly --assumeyes $*" yum install --cacheonly --assumeyes "$@" configure_repos_for_proxy_use - """, + """, # noqa: E501 "reliable-yum-install", ] @@ -95,6 +120,7 @@ CI_SYSTEM_BASE_PKGS = { "common": ["make", "sudo", "tar"], "eurolinux": ["python3-tox"], "miraclelinux": ["python3-tox"], + "fedora": ["python3-tox"], "redhat": ["python3-tox"], "centos": ["python3-tox"], "ubuntu": ["devscripts", "python3-dev", "libssl-dev", "tox", "sbuild"], @@ -337,9 +363,18 @@ def pkg_install(pkg_list, distro, test_distro=False, dry_run=False): if distro in ["centos", "redhat", "rocky", "eurolinux"]: # CentOS and Redhat need epel-release to access oauthlib and jsonschema subprocess.check_call(install_cmd + ["epel-release"]) + subprocess.check_call( + ["dnf", "config-manager", "--set-enabled", "crb"] + ) + # Disable epel-cisco-openh264 if present, we don't need ffmeg codex + # and additional repos just puts unnecessary load on our proxies + subprocess.call( + ["dnf", "config-manager", "--set-disabled", "epel-cisco-openh264"] + ) if distro in [ "suse", "opensuse", + "fedora", "redhat", "rocky", "centos", diff --git a/tools/render-template b/tools/render-template index 78beeecb..4b5efcf3 100755 --- a/tools/render-template +++ b/tools/render-template @@ -34,6 +34,7 @@ def main(): "OpenCloudOS", "openmandriva", "photon", + "raspberry-pi-os", "rhel", "suse", "rocky", diff --git a/tools/run-container b/tools/run-container index 01b0fca8..5c5d3577 100755 --- a/tools/run-container +++ b/tools/run-container @@ -251,7 +251,7 @@ apt_install() { install_packages() { get_os_info || return case "$OS_NAME" in - centos|rocky*) yum_install "$@";; + centos|rocky*|fedora) yum_install "$@";; opensuse*) zypper_install "$@";; debian|ubuntu) apt_install "$@" -y;; *) error "Do not know how to install packages on ${OS_NAME}"; @@ -358,12 +358,27 @@ wait_for_boot() { { errorrc "wait inside $name failed."; return; } if [ -n "${http_proxy-}" ]; then - if [ "$OS_NAME" = "centos" ]; then + if [ "$OS_NAME" = "centos" -o "$OS_NAME" = "fedora" ]; then debug 1 "configuring proxy ${http_proxy}" inside "$name" sh -c "echo proxy=$http_proxy >> /etc/yum.conf" inside "$name" sh -c "sed -i --regexp-extended '/^#baseurl=/s/#// ; /^(mirrorlist|metalink)=/s/^/#/' /etc/yum.repos.d/*.repo" inside "$name" sh -c "sed -i 's/download\.fedoraproject\.org/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo" inside "$name" sh -c "sed -i 's/download\.example/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo" + if [ "$OS_NAME" = "centos" ]; then + CENTOS_REPO="/etc/yum.repos.d/centos.repo" + inside "$name" sh -c "grep -q baseurl $CENTOS_REPO" + if [ $? -eq 1 ]; then + # CentOS 9 does not provide baseurl definitions + inside "$name" sh -c "sed -i '/\[baseos\]/a baseurl=https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os' ${CENTOS_REPO}" + inside "$name" sh -c "sed -i '/\[appstream\]/a baseurl=https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os' ${CENTOS_REPO}" + inside "$name" sh -c "sed -i '/\[crb\]/a baseurl=https://mirror.stream.centos.org/9-stream/CRB/x86_64/os' ${CENTOS_REPO}" + CENTOS_EXTRAS_REPO="/etc/yum.repos.d/centos-addons.repo" + inside "$name" sh -c "sed -i '/\[extras-common\]/a baseurl=https://mirror.stream.centos.org/SIGs/9-stream/extras/x86_64/extras-common' ${CENTOS_EXTRAS_REPO}" + inside "$name" sh -c "dnf install -y 'dnf-command(config-manager)'" + inside "$name" sh -c "dnf config-manager --set-enabled crb" + inside "$name" sh -c "dnf config-manager --set-disabled epel-cisco-openh264" || true + fi + fi else debug 1 "do not know how to configure proxy on $OS_NAME" fi @@ -503,7 +518,7 @@ main() { local build_pkg="" build_srcpkg="" pkg_ext="" distflag="" case "$OS_NAME" in - centos|rocky*) distflag="--distro=redhat";; + centos|rocky*|fedora) distflag="--distro=redhat";; opensuse*) distflag="--distro=suse";; esac @@ -512,7 +527,7 @@ main() { build_pkg="./packages/bddeb -d" build_srcpkg="./packages/bddeb -S -d" pkg_ext=".deb";; - centos|opensuse*|rocky*) + centos|opensuse*|rocky*|fedora) build_pkg="./packages/brpm $distflag" build_srcpkg="./packages/brpm $distflag --srpm" pkg_ext=".rpm";; diff --git a/tools/run-lint b/tools/run-lint index 5dc338f2..2bd0ab17 100755 --- a/tools/run-lint +++ b/tools/run-lint @@ -13,7 +13,10 @@ else files=( "$@" ) fi -cmd=( "python3" -m "flake8" "${files[@]}" ) +if [ -z "$PYTHON" ]; then + PYTHON="python3" +fi +cmd=( "$PYTHON" -m "flake8" "${files[@]}" ) echo "Running: " "${cmd[@]}" 1>&2 exec "${cmd[@]}"