From b9773c54865ad4fbf156f20465aeaef90761279e Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 7 Sep 2018 18:17:33 -0500 Subject: [PATCH 01/28] capture work on dockerisation. --- Dockerfile | 14 +++++ Vagrantfile | 134 ++++++++++++++++++++++++++--------------------- bin/dockerrun.sh | 30 +++++++++++ bin/systemd.sh | 23 ++++++++ 4 files changed, 141 insertions(+), 60 deletions(-) create mode 100644 Dockerfile create mode 100644 bin/dockerrun.sh create mode 100644 bin/systemd.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..970ee7c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM library/debian:9 + +ENV container docker +ENV DEBIAN_FRONTEND "noninteractive" +ENV NOTVISIBLE "in users profile" + +ADD bin/dockerrun.sh /root/dockerrun.sh +ADD bin/systemd.sh /root/systemd.sh + +RUN /root/dockerrun.sh + +STOPSIGNAL SIGRTMIN+3 +CMD ["/usr/sbin/sshd", "-D"] +EXPOSE 22 diff --git a/Vagrantfile b/Vagrantfile index 2788103..5458049 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -28,77 +28,91 @@ Vagrant.configure("2") do |config| # Forward SSH agent from host into Vagrant machine config.ssh.forward_agent = true - # roughly tracking twlight VMs provisioned via wikimedia labs horizon - config.vm.box = "bento/debian-8" - #config.vm.box_version = "" + # Our provisioner expects /vagrant. + config.vm.synced_folder ".", "/vagrant" + # We need a little beef if we're pulling in production-scale data config.vm.provider :virtualbox do |v| + # roughly tracking twlight VMs provisioned via wikimedia labs horizon + v.box = "bento/debian-8" v.cpus = 4 v.memory = 4096 v.linked_clone = true end - # Our puppet provisioner expects /vagrant, and virtualbox is the only tested provider - if Vagrant.has_plugin?("vagrant-vbguest") - config.vm.synced_folder ".", "/vagrant", type: "virtualbox", mount_options: ['dmode=777', 'fmode=666'] - config.vbguest.auto_update = false + # Build container from Dockerfile + config.vm.provider "docker" do |d| + d.build_dir = "." + d.has_ssh = true + d.create_args = ["--privileged", "--cap-add", "SYS_ADMIN", "-v", "/run", "-v", "/tmp", "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro"] + end + + # Ensure systemd is running on all containers. + config.vm.provision "shell", + inline: "sudo /root/systemd.sh", + keep_color: "True", + run: "always" - # Allow the SSH agent to cross the sudo barrier. - # Handy if you use an SSH remote and want to run the git pull script (which require root). - config.vm.provision "shell", + # Allow the SSH agent to cross the sudo barrier. + # Handy if you use an SSH remote and want to run the git pull script (which require root). + config.vm.provision "shell", inline: "echo 'Defaults env_keep+=SSH_AUTH_SOCK' | sudo EDITOR='tee -a' visudo" - # Install puppet because we need it, chrony because its useful in Vagrant, - # and vim because the author of this Vagrantfile prefers it. - config.vm.provision "shell", - inline: "wget --quiet --timestamping --directory-prefix=/tmp \ - https://apt.puppetlabs.com/puppetlabs-release-pc1-jessie.deb && \ - dpkg -i /tmp/puppetlabs-release-pc1-jessie.deb && \ - apt update && apt install -y chrony puppet-agent vim" - - - # Add github's host key to our known hosts file - config.vm.provision "shell", - inline: "ssh-keyscan -t rsa github.com >> /etc/ssh/ssh_known_hosts" - - ## Handy method for fetching puppet module from github - #config.vm.provision "shell", - # inline: "wget --quiet --timestamping --directory-prefix=/vagrant/puppet/modules \ - # 'https://github.com/WikipediaLibrary/twlight_puppet/archive/"+ twlight_puppet_version +".tar.gz'" - - ## Install our twlight puppet module from github - #config.vm.provision "shell", - # inline: twlight_puppet_bin_path +"/puppet module install \ - # "+ twlight_puppet_options +" --target-dir /vagrant/puppet/modules \ - # /vagrant/puppet/modules/"+ twlight_puppet_version +".tar.gz" - - # Install our twlight puppet module from puppet forge - config.vm.provision "shell", - inline: twlight_puppet_bin_path +"/puppet module install \ - "+ twlight_puppet_options +" --target-dir /vagrant/puppet/modules \ - jsnshrmn/twlight --version "+ twlight_puppet_version +";" - - # Run the puppet provisioner - config.vm.provision "puppet" do |puppet| - puppet.working_directory = "/vagrant/puppet" - puppet.hiera_config_path = "puppet/hiera.yaml" - puppet.environment = "local" - puppet.environment_path = "puppet/environments" - puppet.module_path = "puppet/modules" - puppet.binary_path = twlight_puppet_bin_path - puppet.options = twlight_puppet_options - - # Run migration so any imported DB dump will work with current code. - config.vm.provision "shell", - inline: "sudo su www bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" - - # Allow vagrant user to write to project .git - config.vm.provision "shell", - inline: "usermod -a -G www vagrant && chmod -R g+w /var/www/html/TWLight" - - end + # Install puppet because we need it, chrony because its useful in Vagrant, + # and vim because the author of this Vagrantfile prefers it. + config.vm.provision "shell", + inline: "wget --quiet --timestamping --directory-prefix=/tmp \ + https://apt.puppetlabs.com/puppetlabs-release-pc1-jessie.deb && \ + dpkg -i /tmp/puppetlabs-release-pc1-jessie.deb && \ + apt update && apt install -y chrony puppet-agent vim" + + + # Add github's host key to our known hosts file + config.vm.provision "shell", + inline: "ssh-keyscan -t rsa github.com >> /etc/ssh/ssh_known_hosts" + + + # frontload some vagrant-specific systemd config. + config.vm.provision "shell", + inline: "mkdir -p /etc/systemd/system/mariadb.service.d; \ + printf '[Service]\ProtectHome=false\n' > /etc/systemd/system/mariadb.service.d/vagrant.conf; \ + systemctl daemon-reload" + + ## Handy method for fetching puppet module from github + #config.vm.provision "shell", + # inline: "wget --quiet --timestamping --directory-prefix=/vagrant/puppet/modules \ + # 'https://github.com/WikipediaLibrary/twlight_puppet/archive/"+ twlight_puppet_version +".tar.gz'" + + ## Install our twlight puppet module from github + #config.vm.provision "shell", + # inline: twlight_puppet_bin_path +"/puppet module install \ + # "+ twlight_puppet_options +" --target-dir /vagrant/puppet/modules \ + # /vagrant/puppet/modules/"+ twlight_puppet_version +".tar.gz" + + # Install our twlight puppet module from puppet forge + config.vm.provision "shell", + inline: twlight_puppet_bin_path +"/puppet module install \ + "+ twlight_puppet_options +" --target-dir /vagrant/puppet/modules \ + jsnshrmn/twlight --version "+ twlight_puppet_version +";" + + # Run the puppet provisioner + config.vm.provision "puppet" do |puppet| + puppet.working_directory = "/vagrant/puppet" + puppet.hiera_config_path = "puppet/hiera.yaml" + puppet.environment = "local" + puppet.environment_path = "puppet/environments" + puppet.module_path = "puppet/modules" + puppet.binary_path = twlight_puppet_bin_path + puppet.options = twlight_puppet_options + + # Run migration so any imported DB dump will work with current code. + config.vm.provision "shell", + inline: "sudo su www bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" + + # Allow vagrant user to write to project .git + config.vm.provision "shell", + inline: "usermod -a -G www vagrant && chmod -R g+w /var/www/html/TWLight" end - end diff --git a/bin/dockerrun.sh b/bin/dockerrun.sh new file mode 100644 index 0000000..f0e32d9 --- /dev/null +++ b/bin/dockerrun.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +echo "export VISIBLE=now" >> /etc/profile + +apt update && apt install -y dialog gnupg lsb-release openssh-server python-minimal sudo systemd wget + +mkdir -p /var/run/sshd + +mkdir -p /run/systemd + +# SSH login fix. Otherwise user is kicked off after login +sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# Configure insecure root user for Vagrant +echo 'root:vagrant' | chpasswd +sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config + +# Add vagrant user +adduser --quiet --disabled-password --shell /bin/bash --home /home/vagrant --gecos "User" vagrant +echo 'vagrant:vagrant' | chpasswd + +# Setup vagrant insecure key. +mkdir -p /home/vagrant/.ssh +echo 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key' >>/home/vagrant/.ssh/authorized_keys +chown -R 'vagrant:vagrant' /home/vagrant/.ssh +chmod 600 /home/vagrant/.ssh/authorized_keys +chmod 700 /home/vagrant/.ssh + +# Setup passwordless sudo for vagrant user. +echo 'vagrant ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers diff --git a/bin/systemd.sh b/bin/systemd.sh new file mode 100644 index 0000000..71c0bb7 --- /dev/null +++ b/bin/systemd.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# If systemd is working, exit. +if systemctl -a > /dev/null +then + exit 0 +else + # Fire up systemd if it's not happy + cd /root/ + nohup /bin/systemd --system --unit=basic.target > $(hostname -f).systemd.log 2>&1 & + + # Disable the ssh service, since we're running sshd in the foreground + # to keep our container running. + systemctl disable ssh + + # systemd will write a nologin file since it is in an interesting state. + # Wait for that file to show up so we can delete it. + while [ ! -f /run/nologin ] + do + sleep 1 + done + rm /run/nologin +fi From e0d7ac2d0de678db928734336013bbc3937be59c Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 7 Sep 2018 18:18:33 -0500 Subject: [PATCH 02/28] capture work on dockerisation. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 5458049..bab2319 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -76,7 +76,7 @@ Vagrant.configure("2") do |config| # frontload some vagrant-specific systemd config. config.vm.provision "shell", inline: "mkdir -p /etc/systemd/system/mariadb.service.d; \ - printf '[Service]\ProtectHome=false\n' > /etc/systemd/system/mariadb.service.d/vagrant.conf; \ + printf '[Service]\ProtectHome=false\nTimeoutStartSec=0\n' > /etc/systemd/system/mariadb.service.d/vagrant.conf; \ systemctl daemon-reload" ## Handy method for fetching puppet module from github From 6c58aca1ecd8315b4b14ce4d5f2ece26d9d21df4 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 7 Sep 2018 18:26:31 -0500 Subject: [PATCH 03/28] capture work on dockerisation. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index bab2319..614286c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -76,7 +76,7 @@ Vagrant.configure("2") do |config| # frontload some vagrant-specific systemd config. config.vm.provision "shell", inline: "mkdir -p /etc/systemd/system/mariadb.service.d; \ - printf '[Service]\ProtectHome=false\nTimeoutStartSec=0\n' > /etc/systemd/system/mariadb.service.d/vagrant.conf; \ + printf '[Service]\nProtectHome=false\nTimeoutStartSec=0\n' > /etc/systemd/system/mariadb.service.d/vagrant.conf; \ systemctl daemon-reload" ## Handy method for fetching puppet module from github From 5472b547c2b0cda41a7e10e65beade0edc2e4841 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Sat, 8 Sep 2018 15:02:38 -0500 Subject: [PATCH 04/28] Capture work on dockerisation. No longer need privileged flag since we're adding a capability flag, don't have to hack pam nologin since we're removing problematic user stuff from systemd as part of the build.. --- Vagrantfile | 4 ++-- bin/dockerrun.sh | 21 ++++++++++++++++++++- bin/systemd.sh | 21 +++++++-------------- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 614286c..d53153d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -45,7 +45,7 @@ Vagrant.configure("2") do |config| config.vm.provider "docker" do |d| d.build_dir = "." d.has_ssh = true - d.create_args = ["--privileged", "--cap-add", "SYS_ADMIN", "-v", "/run", "-v", "/tmp", "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro"] + d.create_args = ["--cap-add", "SYS_ADMIN", "-v", "/run", "-v", "/tmp", "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro"] end # Ensure systemd is running on all containers. @@ -76,7 +76,7 @@ Vagrant.configure("2") do |config| # frontload some vagrant-specific systemd config. config.vm.provision "shell", inline: "mkdir -p /etc/systemd/system/mariadb.service.d; \ - printf '[Service]\nProtectHome=false\nTimeoutStartSec=0\n' > /etc/systemd/system/mariadb.service.d/vagrant.conf; \ + printf '[Service]\nProtectHome=false\nType=simple\n' > /etc/systemd/system/mariadb.service.d/vagrant.conf; \ systemctl daemon-reload" ## Handy method for fetching puppet module from github diff --git a/bin/dockerrun.sh b/bin/dockerrun.sh index f0e32d9..fc3e36c 100644 --- a/bin/dockerrun.sh +++ b/bin/dockerrun.sh @@ -2,11 +2,30 @@ echo "export VISIBLE=now" >> /etc/profile +# Install packages. +# Basic vagrant workflow for faking real hosts needs openssh-server sudo and systemd. +# Puppet provisioner needs lsb-release. Puppet apt module needs gnupg for adding encrypted repos. +# TWLight vagrant shell provisioner expects wget. +# Ansbile needs python-minimal. @TODO try stripping that back out. apt update && apt install -y dialog gnupg lsb-release openssh-server python-minimal sudo systemd wget mkdir -p /var/run/sshd -mkdir -p /run/systemd +# systemd config. +mkdir -p /run/systemd +# Strip out bits that aren't going to work happily in this container. Largely cribbed from: +# https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/ +(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); +rm -f /lib/systemd/system/multi-user.target.wants/*; +rm -f /etc/systemd/system/*.wants/*; +rm -f /lib/systemd/system/local-fs.target.wants/*; +rm -f /lib/systemd/system/sockets.target.wants/*udev*; +rm -f /lib/systemd/system/sockets.target.wants/*initctl*; +rm -f /lib/systemd/system/basic.target.wants/*; +rm -f /lib/systemd/system/anaconda.target.wants/*; +# With these additions that @jsnshrmn found to be problematic, at least on Debian 9. +rm -f /lib/systemd/system/user\@.service +rm -r /lib/systemd/system/systemd-tmpfiles-setup.service # SSH login fix. Otherwise user is kicked off after login sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd diff --git a/bin/systemd.sh b/bin/systemd.sh index 71c0bb7..4b17101 100644 --- a/bin/systemd.sh +++ b/bin/systemd.sh @@ -1,23 +1,16 @@ #!/usr/bin/env bash -# If systemd is working, exit. -if systemctl -a > /dev/null +# If systemd is working: +if /bin/systemctl -a > /dev/null then + # exit. exit 0 +# If systemd isn't working: else - # Fire up systemd if it's not happy - cd /root/ - nohup /bin/systemd --system --unit=basic.target > $(hostname -f).systemd.log 2>&1 & + # Fire it up and detach. + /usr/bin/nohup /bin/systemd --system --unit=basic.target > /root/$(hostname -f).systemd.log 2>&1 & # Disable the ssh service, since we're running sshd in the foreground # to keep our container running. - systemctl disable ssh - - # systemd will write a nologin file since it is in an interesting state. - # Wait for that file to show up so we can delete it. - while [ ! -f /run/nologin ] - do - sleep 1 - done - rm /run/nologin + /bin/systemctl disable ssh fi From 3106a67c2bb1cc964957d185bcc217f481d05e14 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Sun, 9 Sep 2018 09:01:00 -0500 Subject: [PATCH 05/28] Capture work on dockerisation. Reorganized so that we can have ssh running as a service. --- Dockerfile | 5 +++-- Vagrantfile | 9 +++------ bin/dockerexec.sh | 23 +++++++++++++++++++++++ bin/dockerrun.sh | 6 +++--- bin/systemd.sh | 16 ---------------- 5 files changed, 32 insertions(+), 27 deletions(-) create mode 100644 bin/dockerexec.sh delete mode 100644 bin/systemd.sh diff --git a/Dockerfile b/Dockerfile index 970ee7c..bd3b948 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,11 @@ ENV DEBIAN_FRONTEND "noninteractive" ENV NOTVISIBLE "in users profile" ADD bin/dockerrun.sh /root/dockerrun.sh -ADD bin/systemd.sh /root/systemd.sh +ADD bin/dockerexec.sh /root/dockerexec.sh RUN /root/dockerrun.sh STOPSIGNAL SIGRTMIN+3 -CMD ["/usr/sbin/sshd", "-D"] + +CMD ["/root/dockerexec.sh", "-D"] EXPOSE 22 diff --git a/Vagrantfile b/Vagrantfile index d53153d..dcf9777 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -48,12 +48,6 @@ Vagrant.configure("2") do |config| d.create_args = ["--cap-add", "SYS_ADMIN", "-v", "/run", "-v", "/tmp", "-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro"] end - # Ensure systemd is running on all containers. - config.vm.provision "shell", - inline: "sudo /root/systemd.sh", - keep_color: "True", - run: "always" - # Allow the SSH agent to cross the sudo barrier. # Handy if you use an SSH remote and want to run the git pull script (which require root). config.vm.provision "shell", @@ -72,6 +66,9 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", inline: "ssh-keyscan -t rsa github.com >> /etc/ssh/ssh_known_hosts" + # Kill the SSHD process we used to bootstrap, and start a proper service. + config.vm.provision "shell", + inline: "/usr/bin/pkill sshd && /bin/systemctl start ssh" # frontload some vagrant-specific systemd config. config.vm.provision "shell", diff --git a/bin/dockerexec.sh b/bin/dockerexec.sh new file mode 100644 index 0000000..b0e6423 --- /dev/null +++ b/bin/dockerexec.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Fire up systemd and detach. +#/usr/bin/nohup /bin/systemd --system --unit=basic.target > /root/$(hostname -f).systemd.log 2>&1 & +#service dbus start +/bin/systemd --system --unit=basic.target & + +# Wait for dbus to come up. +#dbus=$(/bin/systemctl status dbus) +#until [ $dbus ] +#do +# echo "waiting for dbus." +# sleep 1 +#done + +# Start the ssh service. +#/bin/systemctl start ssh +#/bin/systemctl start ssh || /usr/sbin/sshd +/usr/sbin/sshd + +# Keep our container running. +trap : TERM INT +sleep infinity & wait diff --git a/bin/dockerrun.sh b/bin/dockerrun.sh index fc3e36c..6030587 100644 --- a/bin/dockerrun.sh +++ b/bin/dockerrun.sh @@ -3,11 +3,10 @@ echo "export VISIBLE=now" >> /etc/profile # Install packages. -# Basic vagrant workflow for faking real hosts needs openssh-server sudo and systemd. +# Basic vagrant workflow for faking real hosts needs locales openssh-server sudo and systemd. # Puppet provisioner needs lsb-release. Puppet apt module needs gnupg for adding encrypted repos. # TWLight vagrant shell provisioner expects wget. -# Ansbile needs python-minimal. @TODO try stripping that back out. -apt update && apt install -y dialog gnupg lsb-release openssh-server python-minimal sudo systemd wget +apt update && apt install -y dialog gnupg lsb-release locales openssh-server sudo systemd wget mkdir -p /var/run/sshd @@ -26,6 +25,7 @@ rm -f /lib/systemd/system/anaconda.target.wants/*; # With these additions that @jsnshrmn found to be problematic, at least on Debian 9. rm -f /lib/systemd/system/user\@.service rm -r /lib/systemd/system/systemd-tmpfiles-setup.service +(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-journald.service ] || rm -f $i; done); # SSH login fix. Otherwise user is kicked off after login sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd diff --git a/bin/systemd.sh b/bin/systemd.sh deleted file mode 100644 index 4b17101..0000000 --- a/bin/systemd.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# If systemd is working: -if /bin/systemctl -a > /dev/null -then - # exit. - exit 0 -# If systemd isn't working: -else - # Fire it up and detach. - /usr/bin/nohup /bin/systemd --system --unit=basic.target > /root/$(hostname -f).systemd.log 2>&1 & - - # Disable the ssh service, since we're running sshd in the foreground - # to keep our container running. - /bin/systemctl disable ssh -fi From 049b490c74ee36e71598f7efea400458064062cc Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Sun, 9 Sep 2018 15:46:32 -0500 Subject: [PATCH 06/28] capture work on getting docker provider working from WSL. --- README.md | 29 +++++++++-------------------- Vagrantfile | 35 ++++++++++++++++++----------------- bin/dockerrun.sh | 9 ++++++--- bin/wsl_docker_activate.sh | 28 ++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 40 deletions(-) create mode 100644 bin/wsl_docker_activate.sh diff --git a/README.md b/README.md index 00e863c..a1f8e45 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,7 @@ Those developing [Library Card Platform for The Wikipedia Library](https://githu ## Requirements * [Vagrant](https://www.vagrantup.com/downloads.html) -* [VirtualBox and VirtualBox Extension Pack](https://www.virtualbox.org/wiki/Downloads) -* vagrant-vbguest plugin (eg. vagrant plugin install vagrant-vbguest) +* [Docker](https://www.docker.com/get-started) * Browser configured to hit a local SOCKS proxy on a port of your choice, I use 2080 ## Optional @@ -20,30 +19,20 @@ Those developing [Library Card Platform for The Wikipedia Library](https://githu ## Notes for Linux users: -For a "just works" experience, I recommend fetching Vagrant and VirtualBox packages from the vendor websites rather than using your distribution's software repositories. Those likely include fairly old versions of the required packages, and you will find yourself having to carefully managing your Vagrant, VirtualBox, and base box updates to avoid breakage, if it's not broken out of the gate. +For a "just works" experience, I recommend fetching Vagrant and Docker packages from the vendor websites rather than using your distribution's software repositories. Those likely include fairly old versions of the required packages, and you will find yourself having to carefully managing your component updates to avoid breakage, if it's not broken out of the gate. ## Notes for Windows users: Some third-party endpoint security software, such as Dell Data Protection Encryption and several McAfee products, interfere with VirtualBox. You may need to temporarily disable these products or make different endpoint protection choices. -You'll need to add the following directory to your PATH environment variable after installing VirtualBox: +Vagrant's (early but generally working) support for Ubuntu via the Windows Subsystem for Linux is the recommended way to run this enviroment. You should be on Windows 10 Version 1709 or later and perform a store-based Ubuntu installation. See the [Vagrant and Windows Subsystem for Linux instructions](https://www.vagrantup.com/docs/other/wsl.html). Then: -``` -C:\Program Files\Oracle\VirtualBox -``` - -See [this example from Microsoft](https://msdn.microsoft.com/en-us/library/office/ee537574.aspx) for adding a path to the PATH environment variable. - -Vagrant's (early but generally working) support for Ubuntu via the Windows Subsystem for Linux is the recommended way to run this enviroment. You should be on Windows 10 Version 1709 or later and perform a store-based Ubuntu installation. See the [Vagrant and Windows Subsystem for Linux instructions](https://www.vagrantup.com/docs/other/wsl.html). You'll install VirtualBox on the Windows side, and then install exactly the same build of Vagrant in both Windows and Ubuntu. The Linux notes apply to the Ubuntu environment. Just download a fixed version of Vagrant (that matches the version you install in Windows) and install using dpkg as described in the instructions. Install any plugins in Ubuntu. - -On Vagrant 2.0.2 and earlier, [issue #9298](https://github.com/hashicorp/vagrant/issues/9298) means you'll need to create a symlink in the location of the deprecated lxrun installation that points to the new store-based installation. As pointed out in the reported issue, running the following powershell commands on the windows side will pull the information from the registry and create the appropriate symlink. - -``` -$WSLREGKEY="HKCU:\Software\Microsoft\Windows\CurrentVersion\Lxss" -$WSLDEFID=(Get-ItemProperty "$WSLREGKEY").DefaultDistribution -$WSLFSPATH=(Get-ItemProperty "$WSLREGKEY\$WSLDEFID").BasePath -New-Item -ItemType Junction -Path "$env:LOCALAPPDATA\lxss" -Value "$WSLFSPATH\rootfs" -``` + * Install Docker on the Windows side, and enable legacy mode (Expose daemon on tcp://localhost:2375 without TLS) + * Install exactly the same build of Vagrant in both Windows and Ubuntu. + * When you clone this repository in Ubuntu, make sure to do so in a location accessible to Windows, such as ``/mnt/c/Users/Public/v`` (``/mnt/c/`` corresponds to ``C:\``). This is required for the vagrant share to work properly. + * There are a number of environment variables that should be configured for WSL + Docker + Vagrant to work happily. As a convenience, you may just ``source bin/wsl_docker_activate.sh`` from within the project directory. + +The Linux notes apply to the Ubuntu environment. Just download a fixed version of Vagrant (that matches the version you install in Windows) and install using dpkg as described in the instructions. Install any plugins in Ubuntu. ## Usage diff --git a/Vagrantfile b/Vagrantfile index dcf9777..6649735 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,7 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -twlight_puppet_version = "0.5.2" +twlight_puppet_version = "0.5.3" #twlight_puppet_version = "master" # Put "--debug " in this string if you want to test the limits of your terminal @@ -23,10 +23,11 @@ Vagrant.configure("2") do |config| #else # # Else, throw a Vagrant Error. Cannot successfully startup on Windows without a GitHub SSH Key! # raise Vagrant::Errors::VagrantError, "\n\nERROR: GitHub SSH Key not found at ~/.ssh/github_rsa.\n\n" - end + #end # Forward SSH agent from host into Vagrant machine config.ssh.forward_agent = true + config.ssh.password = "vagrant" # Our provisioner expects /vagrant. config.vm.synced_folder ".", "/vagrant" @@ -94,22 +95,22 @@ Vagrant.configure("2") do |config| jsnshrmn/twlight --version "+ twlight_puppet_version +";" # Run the puppet provisioner - config.vm.provision "puppet" do |puppet| - puppet.working_directory = "/vagrant/puppet" - puppet.hiera_config_path = "puppet/hiera.yaml" - puppet.environment = "local" - puppet.environment_path = "puppet/environments" - puppet.module_path = "puppet/modules" - puppet.binary_path = twlight_puppet_bin_path - puppet.options = twlight_puppet_options - - # Run migration so any imported DB dump will work with current code. - config.vm.provision "shell", - inline: "sudo su www bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" + #config.vm.provision "puppet" do |puppet| + # puppet.working_directory = "/vagrant/puppet" + # puppet.hiera_config_path = "puppet/hiera.yaml" + # puppet.environment = "local" + # puppet.environment_path = "puppet/environments" + # puppet.module_path = "puppet/modules" + # puppet.binary_path = twlight_puppet_bin_path + # puppet.options = twlight_puppet_options + + ## Run migration so any imported DB dump will work with current code. + #config.vm.provision "shell", + # inline: "sudo su www bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" - # Allow vagrant user to write to project .git - config.vm.provision "shell", - inline: "usermod -a -G www vagrant && chmod -R g+w /var/www/html/TWLight" + ## Allow vagrant user to write to project .git + #config.vm.provision "shell", + # inline: "usermod -a -G www vagrant && chmod -R g+w /var/www/html/TWLight" end end diff --git a/bin/dockerrun.sh b/bin/dockerrun.sh index 6030587..d99488e 100644 --- a/bin/dockerrun.sh +++ b/bin/dockerrun.sh @@ -8,10 +8,10 @@ echo "export VISIBLE=now" >> /etc/profile # TWLight vagrant shell provisioner expects wget. apt update && apt install -y dialog gnupg lsb-release locales openssh-server sudo systemd wget -mkdir -p /var/run/sshd +# Lie if anyone anyone asks if we booted up with systemd. +# https://www.freedesktop.org/software/systemd/man/sd_booted.html +mkdir -p /run/systemd/system -# systemd config. -mkdir -p /run/systemd # Strip out bits that aren't going to work happily in this container. Largely cribbed from: # https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/ (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); @@ -27,6 +27,9 @@ rm -f /lib/systemd/system/user\@.service rm -r /lib/systemd/system/systemd-tmpfiles-setup.service (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-journald.service ] || rm -f $i; done); +# Prep us running SSH from a shell. +mkdir -p /var/run/sshd + # SSH login fix. Otherwise user is kicked off after login sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd diff --git a/bin/wsl_docker_activate.sh b/bin/wsl_docker_activate.sh new file mode 100644 index 0000000..ad646e2 --- /dev/null +++ b/bin/wsl_docker_activate.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Checks to see if this is a WSL environment; +# If true, it sets environment variables in an attempt to make things universially worky. +# Originally motivated by Windows + Hyper-V + Docker + Vagrant struggles. +# Logic for check pulled from https://stackoverflow.com/a/43618657 +# WSL-specific Vagrant information can be found here: +# https://www.vagrantup.com/docs/other/wsl.html +if grep -qE "(Microsoft|WSL)" /proc/version &> /dev/null ; then + + # Enable WSL vagrant + export VAGRANT_WSL_ENABLE_WINDOWS_ACCESS="1" + + # Put the dotfiles in the WSL home directory so that ssh key permissions + # may be set. + here=$(basename $(pwd)) + export VAGRANT_DOTFILE_PATH="~/.vagrant/.${here}" + + # Let WSL access Windows-side Docker if it's installed. + if [ -f '/mnt/c/Program Files/Docker Toolbox/docker.exe' ] ; then + export PATH="$PATH:/mnt/c/Program Files/Docker Toolbox" + export DOCKER_HOST=tcp://127.0.0.1:2375 + alias docker='docker.exe' + export VAGRANT_DEFAULT_PROVIDER="docker" + fi +else + echo "Not a WSL environment." +fi From caac354ef10c535661025bd0c3e4c40acb18a334 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Sun, 9 Sep 2018 15:48:54 -0500 Subject: [PATCH 07/28] capture work on getting docker provider working from WSL. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a1f8e45..01242c5 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Vagrant's (early but generally working) support for Ubuntu via the Windows Subsy * Install Docker on the Windows side, and enable legacy mode (Expose daemon on tcp://localhost:2375 without TLS) * Install exactly the same build of Vagrant in both Windows and Ubuntu. - * When you clone this repository in Ubuntu, make sure to do so in a location accessible to Windows, such as ``/mnt/c/Users/Public/v`` (``/mnt/c/`` corresponds to ``C:\``). This is required for the vagrant share to work properly. + * When you clone this repository in Ubuntu, make sure to do so in a location accessible to Windows, such as ``/mnt/c/Users/Username/v`` (``/mnt/c/`` corresponds to ``C:\``). This is required for the vagrant share to work properly. * There are a number of environment variables that should be configured for WSL + Docker + Vagrant to work happily. As a convenience, you may just ``source bin/wsl_docker_activate.sh`` from within the project directory. The Linux notes apply to the Ubuntu environment. Just download a fixed version of Vagrant (that matches the version you install in Windows) and install using dpkg as described in the instructions. Install any plugins in Ubuntu. From 14f86f7d685d50581657cf23be324d003c70f85f Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Sun, 9 Sep 2018 15:54:49 -0500 Subject: [PATCH 08/28] Run all the provisioning steps now that we've resolved some underlying provider issues. --- Vagrantfile | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 6649735..d39133b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,7 +2,6 @@ # vi: set ft=ruby : twlight_puppet_version = "0.5.3" -#twlight_puppet_version = "master" # Put "--debug " in this string if you want to test the limits of your terminal # emulator's buffer. @@ -23,7 +22,7 @@ Vagrant.configure("2") do |config| #else # # Else, throw a Vagrant Error. Cannot successfully startup on Windows without a GitHub SSH Key! # raise Vagrant::Errors::VagrantError, "\n\nERROR: GitHub SSH Key not found at ~/.ssh/github_rsa.\n\n" - #end + end # Forward SSH agent from host into Vagrant machine config.ssh.forward_agent = true @@ -95,22 +94,22 @@ Vagrant.configure("2") do |config| jsnshrmn/twlight --version "+ twlight_puppet_version +";" # Run the puppet provisioner - #config.vm.provision "puppet" do |puppet| - # puppet.working_directory = "/vagrant/puppet" - # puppet.hiera_config_path = "puppet/hiera.yaml" - # puppet.environment = "local" - # puppet.environment_path = "puppet/environments" - # puppet.module_path = "puppet/modules" - # puppet.binary_path = twlight_puppet_bin_path - # puppet.options = twlight_puppet_options - - ## Run migration so any imported DB dump will work with current code. - #config.vm.provision "shell", - # inline: "sudo su www bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" + config.vm.provision "puppet" do |puppet| + puppet.working_directory = "/vagrant/puppet" + puppet.hiera_config_path = "puppet/hiera.yaml" + puppet.environment = "local" + puppet.environment_path = "puppet/environments" + puppet.module_path = "puppet/modules" + puppet.binary_path = twlight_puppet_bin_path + puppet.options = twlight_puppet_options + + # Run migration so any imported DB dump will work with current code. + config.vm.provision "shell", + inline: "sudo su www bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" - ## Allow vagrant user to write to project .git - #config.vm.provision "shell", - # inline: "usermod -a -G www vagrant && chmod -R g+w /var/www/html/TWLight" + # Allow vagrant user to write to project .git + config.vm.provision "shell", + inline: "usermod -a -G www vagrant && chmod -R g+w /var/www/html/TWLight" end end From 9da8410efb2d84f6eddc11ab8275409f72830a1b Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Sun, 9 Sep 2018 17:01:36 -0500 Subject: [PATCH 09/28] Update path referenced in docs with WSL+Docker paths. --- docs/host_side_file_management.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/host_side_file_management.md b/docs/host_side_file_management.md index 43b689d..959519e 100644 --- a/docs/host_side_file_management.md +++ b/docs/host_side_file_management.md @@ -11,7 +11,7 @@ Host default UserKnownHostsFile /dev/null StrictHostKeyChecking no PasswordAuthentication no - IdentityFile /home/$USER/Projects/vagrant/twlight_vagrant/.vagrant/machines/default/virtualbox/private_key + IdentityFile /home/$USER/.vagrant/.twlight_vagrant/machines/twlight/docker/private_key IdentitiesOnly yes LogLevel FATAL ForwardAgent yes @@ -38,13 +38,13 @@ where $USER is your linux username and $somevalue is not consistent across insta To that path, add the path to the private key for the TWLight Vagrant machine that gets created upon "vagrant up." For example ``` -Projects\vagrant\twlight_vagrant\.vagrant\machines\default\virtualbox\private_key +.vagrant\.twlight_vagrant\machines\twlight\docker\private_key ``` In this case, the full path to your TWLight Vagrant machine's IdentityFile would be something like: ``` -%localappdata%\Packages\CanonicalGroupLimited.UbuntuonWindows_$somevalue\LocalState\rootfs\home\$USER\Projects\vagrant\twlight_vagrant\.vagrant\machines\default\virtualbox\private_key +%localappdata%\Packages\CanonicalGroupLimited.UbuntuonWindows_$somevalue\LocalState\rootfs\home\$USER\.vagrant\.twlight_vagrant\machines\twlight\docker\private_key ``` ## FileZilla example (Linux, MacOS, or Windows) From b8ad29d8f0cd4c34c6075b0eecb24b037179cc29 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Thu, 13 Sep 2018 19:52:07 -0500 Subject: [PATCH 10/28] actually need locales-all for lots of scripts to be able to set locale correctly. --- bin/dockerrun.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/dockerrun.sh b/bin/dockerrun.sh index d99488e..6806290 100644 --- a/bin/dockerrun.sh +++ b/bin/dockerrun.sh @@ -6,7 +6,7 @@ echo "export VISIBLE=now" >> /etc/profile # Basic vagrant workflow for faking real hosts needs locales openssh-server sudo and systemd. # Puppet provisioner needs lsb-release. Puppet apt module needs gnupg for adding encrypted repos. # TWLight vagrant shell provisioner expects wget. -apt update && apt install -y dialog gnupg lsb-release locales openssh-server sudo systemd wget +apt update && apt install -y dialog gnupg lsb-release locales locales-all openssh-server sudo systemd wget # Lie if anyone anyone asks if we booted up with systemd. # https://www.freedesktop.org/software/systemd/man/sd_booted.html From 799275afd7f681543fa219022112547d9ac54aec Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Mon, 15 Oct 2018 09:29:08 -0500 Subject: [PATCH 11/28] Set project directory permissions earlier in the process, so that perms are correct even if twlight commands fail. --- Vagrantfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index d39133b..d71572b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -103,13 +103,13 @@ Vagrant.configure("2") do |config| puppet.binary_path = twlight_puppet_bin_path puppet.options = twlight_puppet_options - # Run migration so any imported DB dump will work with current code. - config.vm.provision "shell", - inline: "sudo su www bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" - # Allow vagrant user to write to project .git config.vm.provision "shell", inline: "usermod -a -G www vagrant && chmod -R g+w /var/www/html/TWLight" + # Run migration so any imported DB dump will work with current code. + config.vm.provision "shell", + inline: "sudo su www bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" + end end From e657f0d98b9f012d7e699fd5d815ed6c297c3af0 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Mon, 15 Oct 2018 09:29:51 -0500 Subject: [PATCH 12/28] use ssh service instead of firing a one-off process. --- bin/dockerexec.sh | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/bin/dockerexec.sh b/bin/dockerexec.sh index b0e6423..12adc29 100644 --- a/bin/dockerexec.sh +++ b/bin/dockerexec.sh @@ -5,18 +5,14 @@ #service dbus start /bin/systemd --system --unit=basic.target & -# Wait for dbus to come up. -#dbus=$(/bin/systemctl status dbus) -#until [ $dbus ] -#do -# echo "waiting for dbus." -# sleep 1 -#done +# Generate SSH host keys +ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key +ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key +ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key +ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -# Start the ssh service. -#/bin/systemctl start ssh -#/bin/systemctl start ssh || /usr/sbin/sshd -/usr/sbin/sshd +# Fire up SSH +/bin/systemctl start sshd # Keep our container running. trap : TERM INT From 2dfa41c8ee4a0095b25794bc5ec1b5d3621c8011 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Mon, 15 Oct 2018 10:30:53 -0500 Subject: [PATCH 13/28] Allow host side clipboard to work as expected in vim. --- Vagrantfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index d71572b..be3f740 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -53,6 +53,10 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", inline: "echo 'Defaults env_keep+=SSH_AUTH_SOCK' | sudo EDITOR='tee -a' visudo" + # Allow host side clipboard to work as expected in vim. + config.vm.provision "shell", + inline: "echo 'set mouse=r' > ~/.vimrc" + # Install puppet because we need it, chrony because its useful in Vagrant, # and vim because the author of this Vagrantfile prefers it. config.vm.provision "shell", From 0e678db99eb0eae5b6d6677367414e9a544b43ab Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Mon, 15 Oct 2018 14:33:27 -0500 Subject: [PATCH 14/28] Allow host side clipboard to work as expected in vim. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index be3f740..623d1ed 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -55,7 +55,7 @@ Vagrant.configure("2") do |config| # Allow host side clipboard to work as expected in vim. config.vm.provision "shell", - inline: "echo 'set mouse=r' > ~/.vimrc" + inline: "echo 'set mouse=r' > /etc/vim/vimrc.local" # Install puppet because we need it, chrony because its useful in Vagrant, # and vim because the author of this Vagrantfile prefers it. From 6480428689d750705d133b8a86424da47bd10911 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 19 Oct 2018 10:49:31 -0500 Subject: [PATCH 15/28] unset environment conflicting environment vars that may be set by tools like minikube. --- bin/wsl_docker_activate.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/wsl_docker_activate.sh b/bin/wsl_docker_activate.sh index ad646e2..3af66d6 100644 --- a/bin/wsl_docker_activate.sh +++ b/bin/wsl_docker_activate.sh @@ -18,6 +18,10 @@ if grep -qE "(Microsoft|WSL)" /proc/version &> /dev/null ; then # Let WSL access Windows-side Docker if it's installed. if [ -f '/mnt/c/Program Files/Docker Toolbox/docker.exe' ] ; then + unset DOCKER_TLS_VERIFY + unset DOCKER_HOST + unset DOCKER_CERT_PATH + unset DOCKER_API_VERSION export PATH="$PATH:/mnt/c/Program Files/Docker Toolbox" export DOCKER_HOST=tcp://127.0.0.1:2375 alias docker='docker.exe' From 592cf4d4fcd3fa76cf3a8881dded4a962e420e22 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 19 Oct 2018 12:07:23 -0500 Subject: [PATCH 16/28] Let the docker image be more opinionated. Clean up Vagrantfile a bit. --- Dockerfile | 2 ++ Vagrantfile | 8 -------- etc/.vimrc | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 etc/.vimrc diff --git a/Dockerfile b/Dockerfile index bd3b948..2dfcea0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ ENV NOTVISIBLE "in users profile" ADD bin/dockerrun.sh /root/dockerrun.sh ADD bin/dockerexec.sh /root/dockerexec.sh +ADD etc/.vimrc /root/.vimrc +ADD etc/.vimrc /etc/skel/.vimrc RUN /root/dockerrun.sh diff --git a/Vagrantfile b/Vagrantfile index 623d1ed..f66fb02 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -53,10 +53,6 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", inline: "echo 'Defaults env_keep+=SSH_AUTH_SOCK' | sudo EDITOR='tee -a' visudo" - # Allow host side clipboard to work as expected in vim. - config.vm.provision "shell", - inline: "echo 'set mouse=r' > /etc/vim/vimrc.local" - # Install puppet because we need it, chrony because its useful in Vagrant, # and vim because the author of this Vagrantfile prefers it. config.vm.provision "shell", @@ -70,10 +66,6 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", inline: "ssh-keyscan -t rsa github.com >> /etc/ssh/ssh_known_hosts" - # Kill the SSHD process we used to bootstrap, and start a proper service. - config.vm.provision "shell", - inline: "/usr/bin/pkill sshd && /bin/systemctl start ssh" - # frontload some vagrant-specific systemd config. config.vm.provision "shell", inline: "mkdir -p /etc/systemd/system/mariadb.service.d; \ diff --git a/etc/.vimrc b/etc/.vimrc new file mode 100644 index 0000000..90c6e89 --- /dev/null +++ b/etc/.vimrc @@ -0,0 +1,14 @@ +" The following are commented out as they cause vim to behave a lot +" differently from regular Vi. They are highly recommended though. +syntax on +set showcmd " Show (partial) command in status line. +set showmatch " Show matching brackets. +set ignorecase " Do case insensitive matching +set smartcase " Do smart case matching +set incsearch " Incremental search +set autowrite " Automatically save before commands like :next and :make +set hidden " Hide buffers when they are abandoned +set mouse=r " Enable mouse usage (all modes) +set tabstop=4 softtabstop=0 expandtab shiftwidth=2 smarttab +set ruler +set background=dark From eac5ae793f4e00d754a8733f860e99332da02695 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Tue, 30 Oct 2018 15:53:29 -0500 Subject: [PATCH 17/28] Update bento box to debian jessie. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index f66fb02..2fb6762 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -35,7 +35,7 @@ Vagrant.configure("2") do |config| # We need a little beef if we're pulling in production-scale data config.vm.provider :virtualbox do |v| # roughly tracking twlight VMs provisioned via wikimedia labs horizon - v.box = "bento/debian-8" + v.box = "bento/debian-9" v.cpus = 4 v.memory = 4096 v.linked_clone = true From 8c342c26e25966127606a3e89e1485785f5cd192 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Tue, 30 Oct 2018 15:54:05 -0500 Subject: [PATCH 18/28] Add comments to docker activation script. --- bin/wsl_docker_activate.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/wsl_docker_activate.sh b/bin/wsl_docker_activate.sh index 3af66d6..df8c3b7 100644 --- a/bin/wsl_docker_activate.sh +++ b/bin/wsl_docker_activate.sh @@ -18,10 +18,13 @@ if grep -qE "(Microsoft|WSL)" /proc/version &> /dev/null ; then # Let WSL access Windows-side Docker if it's installed. if [ -f '/mnt/c/Program Files/Docker Toolbox/docker.exe' ] ; then + # Unset minikube variables that might make vagrant angry. unset DOCKER_TLS_VERIFY unset DOCKER_HOST unset DOCKER_CERT_PATH unset DOCKER_API_VERSION + + # Set variables to allow the docker service running in Windows to work in WSL. export PATH="$PATH:/mnt/c/Program Files/Docker Toolbox" export DOCKER_HOST=tcp://127.0.0.1:2375 alias docker='docker.exe' From 93927b475cb493beef9fcbe0860124198924b0f9 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Fri, 16 Nov 2018 10:00:03 -0600 Subject: [PATCH 19/28] update puppet provisioner --- Vagrantfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 2fb6762..5335023 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -57,8 +57,8 @@ Vagrant.configure("2") do |config| # and vim because the author of this Vagrantfile prefers it. config.vm.provision "shell", inline: "wget --quiet --timestamping --directory-prefix=/tmp \ - https://apt.puppetlabs.com/puppetlabs-release-pc1-jessie.deb && \ - dpkg -i /tmp/puppetlabs-release-pc1-jessie.deb && \ + https://apt.puppetlabs.com/puppetlabs-release-pc1-stretch.deb && \ + dpkg -i /tmp/puppetlabs-release-pc1-stretch.deb && \ apt update && apt install -y chrony puppet-agent vim" @@ -95,6 +95,7 @@ Vagrant.configure("2") do |config| puppet.hiera_config_path = "puppet/hiera.yaml" puppet.environment = "local" puppet.environment_path = "puppet/environments" + puppet.environment_variables = {"APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE" => "1" } puppet.module_path = "puppet/modules" puppet.binary_path = twlight_puppet_bin_path puppet.options = twlight_puppet_options From ad21e89c08f47a7e13845c5978d5141da2e84f2c Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Thu, 6 Dec 2018 14:00:13 -0600 Subject: [PATCH 20/28] update puppet provisioner. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 5335023..0ce7ba0 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,7 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -twlight_puppet_version = "0.5.3" +twlight_puppet_version = "0.5.8" # Put "--debug " in this string if you want to test the limits of your terminal # emulator's buffer. From 5b6cbb9cc19c7d3d3a32ba99be87915c9807956e Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Thu, 17 Jan 2019 06:05:17 -0600 Subject: [PATCH 21/28] update puppet provisioner. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 0ce7ba0..aeabb92 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,7 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -twlight_puppet_version = "0.5.8" +twlight_puppet_version = "0.5.9" # Put "--debug " in this string if you want to test the limits of your terminal # emulator's buffer. From 13ec806eb552b0fee94b8e7a6c801c34224c7a73 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Thu, 17 Jan 2019 06:08:02 -0600 Subject: [PATCH 22/28] update puppet provisioner. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index aeabb92..51e38f8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,7 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -twlight_puppet_version = "0.5.9" +twlight_puppet_version = "0.5.10" # Put "--debug " in this string if you want to test the limits of your terminal # emulator's buffer. From 64542873204f96ac9d892ea8fb291b22646154da Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Tue, 5 Feb 2019 11:20:29 -0600 Subject: [PATCH 23/28] cleanup bin/wsl_docker_activate to use WSL-side docker client only. --- bin/wsl_docker_activate.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/wsl_docker_activate.sh b/bin/wsl_docker_activate.sh index df8c3b7..8f4c1a3 100644 --- a/bin/wsl_docker_activate.sh +++ b/bin/wsl_docker_activate.sh @@ -15,9 +15,10 @@ if grep -qE "(Microsoft|WSL)" /proc/version &> /dev/null ; then # may be set. here=$(basename $(pwd)) export VAGRANT_DOTFILE_PATH="~/.vagrant/.${here}" - + + # @TODO: make this check less brittle. Currenly only works with default install location. # Let WSL access Windows-side Docker if it's installed. - if [ -f '/mnt/c/Program Files/Docker Toolbox/docker.exe' ] ; then + if [[ -f '/mnt/c/Program Files/Docker/Docker/DockerCli.exe' || -f '/mnt/c/Program Files/Docker Toolbox/docker.exe' ]] ; then # Unset minikube variables that might make vagrant angry. unset DOCKER_TLS_VERIFY unset DOCKER_HOST @@ -25,9 +26,7 @@ if grep -qE "(Microsoft|WSL)" /proc/version &> /dev/null ; then unset DOCKER_API_VERSION # Set variables to allow the docker service running in Windows to work in WSL. - export PATH="$PATH:/mnt/c/Program Files/Docker Toolbox" export DOCKER_HOST=tcp://127.0.0.1:2375 - alias docker='docker.exe' export VAGRANT_DEFAULT_PROVIDER="docker" fi else From 0a49b9303fa50fc681e43292d2f29af358f5786a Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Tue, 5 Feb 2019 11:21:14 -0600 Subject: [PATCH 24/28] twlight app user should be vagrant shell account. --- puppet/data/common.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/puppet/data/common.yaml b/puppet/data/common.yaml index 22c05f9..ff277a6 100644 --- a/puppet/data/common.yaml +++ b/puppet/data/common.yaml @@ -1,6 +1,7 @@ --- # These variables set config values for OS and package configuration as well as # the app. If you don't know what they are then you don't need to change them. +twlight::params::unixname: 'vagrant' twlight::params::mysqlroot_pw: 'vagrant' twlight::params::mysqltwlight_pw: 'vagrant' twlight::params::restore_file: '/vagrant/backup/twlight.tar.gz' From 46e803f6ec8e4e52417593b409ebc512ca549500 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Tue, 5 Feb 2019 11:22:26 -0600 Subject: [PATCH 25/28] Cleanp README.md based on clean Windows + WSL install. No need to install Vagrant on Windows. --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 01242c5..b789172 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,13 @@ Some third-party endpoint security software, such as Dell Data Protection Encryp Vagrant's (early but generally working) support for Ubuntu via the Windows Subsystem for Linux is the recommended way to run this enviroment. You should be on Windows 10 Version 1709 or later and perform a store-based Ubuntu installation. See the [Vagrant and Windows Subsystem for Linux instructions](https://www.vagrantup.com/docs/other/wsl.html). Then: - * Install Docker on the Windows side, and enable legacy mode (Expose daemon on tcp://localhost:2375 without TLS) - * Install exactly the same build of Vagrant in both Windows and Ubuntu. + * Install Docker on Windows and enable legacy mode (Expose daemon on tcp://localhost:2375 without TLS) + * Install Docker on Ubuntu. You can just use the Ubuntu-provided package, eg. `apt install docker.io` + * Install Vagrant on Ubuntu using dpkg as described in the Vagrant instructions. If you already have Vagrant on Windows, you'll need to keep the two at exactly the same build version. * When you clone this repository in Ubuntu, make sure to do so in a location accessible to Windows, such as ``/mnt/c/Users/Username/v`` (``/mnt/c/`` corresponds to ``C:\``). This is required for the vagrant share to work properly. * There are a number of environment variables that should be configured for WSL + Docker + Vagrant to work happily. As a convenience, you may just ``source bin/wsl_docker_activate.sh`` from within the project directory. -The Linux notes apply to the Ubuntu environment. Just download a fixed version of Vagrant (that matches the version you install in Windows) and install using dpkg as described in the instructions. Install any plugins in Ubuntu. +The Linux notes apply to the Ubuntu environment. ## Usage @@ -44,7 +45,7 @@ You might need to configure some of the settings for the [puppet module](https:/ ``` and configure any parameters you'd like to override, such as the git repository or revision. See the [parameters manifest in the puppet module](https://github.com/WikipediaLibrary/twlight_puppet/blob/master/manifests/params.pp). -If you have a tarball that you'd like to load on provision, place it +If you have a TWLight backup tarball that you'd like to load on provision, place it ``` ./backup/twlight.tar.gz From 572e9a4693b59a1574615677cc72e94bf6d9e60f Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Tue, 5 Feb 2019 11:22:51 -0600 Subject: [PATCH 26/28] use newer puppet module. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 51e38f8..3346528 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,7 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -twlight_puppet_version = "0.5.10" +twlight_puppet_version = "0.5.12" # Put "--debug " in this string if you want to test the limits of your terminal # emulator's buffer. From 15e30f67617cbc85293c9aa92fa32e655253f71f Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Wed, 6 Feb 2019 12:23:39 -0600 Subject: [PATCH 27/28] Drop vestigal references to www user. Updated puppet module version. --- Vagrantfile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 3346528..9b211bf 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,7 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -twlight_puppet_version = "0.5.12" +twlight_puppet_version = "0.5.13" # Put "--debug " in this string if you want to test the limits of your terminal # emulator's buffer. @@ -100,13 +100,8 @@ Vagrant.configure("2") do |config| puppet.binary_path = twlight_puppet_bin_path puppet.options = twlight_puppet_options - # Allow vagrant user to write to project .git - config.vm.provision "shell", - inline: "usermod -a -G www vagrant && chmod -R g+w /var/www/html/TWLight" - # Run migration so any imported DB dump will work with current code. config.vm.provision "shell", - inline: "sudo su www bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" - + inline: "sudo su vagrant bash -c '/var/www/html/TWLight/bin/./virtualenv_migrate.sh >>/var/www/html/TWLight/TWLight/logs/update.log 2>&1' || :" end end From 25d0f1a025cd5b07afd8343952f73f98ac0f9e89 Mon Sep 17 00:00:00 2001 From: Jason Sherman Date: Thu, 7 Feb 2019 09:48:19 -0600 Subject: [PATCH 28/28] Drop vestigal references to www user. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b789172..91bcb15 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ If you have a TWLight backup tarball that you'd like to load on provision, place Alternatively, scripts are included to create a superuser and generate example data. Before doing anything else, login to the platform as normal, then run ``` -sudo -u www /var/www/html/TWLight/bin/virtualenv_example_data.sh +/var/www/html/TWLight/bin/virtualenv_example_data.sh ``` The account you used to login will be made a superuser, giving you access to the Admin interface. The values in that file can be modified to generate more or less users, partners, and applications, but the file should only be run once. @@ -73,5 +73,5 @@ You can now work on the running app inside Vagrant and view the changes in your As you are making local changes, make sure to take advantage of the included test suite. To do so, run the following command within the vagrant machine: ``` -sudo su www /var/www/html/TWLight/bin/virtualenv_test.sh +/var/www/html/TWLight/bin/virtualenv_test.sh ```