galaxy.yml

---
- hosts: all
  name: apt update, python, pip
  become: true
  become_user: root
  tasks:
    - ansible.builtin.apt:
        name:
        - python3-pip
        - python-is-python3
        # this is not needed often and can fail the run since the machines have many repos
        # update_cache: yes
      when: ansible_os_family == 'Debian'

- hosts: production
  become: true
  become_user: root
  tasks:
    - name: Install production-specific dependencies
      package:
        name: ['ceph-common', 'xfsprogs']
    - name: Create the RBD mount directory
      file:
        path: "{{ rbd_mount_point }}"
        state: directory
        mode: '0755'
    - name: Copy ceph.conf
      ansible.builtin.copy:
        src: "{{ playbook_dir }}/files/ceph/ceph.{{ rbd_cluster_code }}.conf"
        dest: "/etc/ceph/ceph.conf"
        mode: '0644'
    - name: Copy ceph.keyring
      ansible.builtin.copy:
        content: |
          [client.{{ rbd_user }}]
          key = {{ rbd_key }}
        dest: "/etc/ceph/ceph.keyring"
        mode: '0600'
    - name: Copy rbdmap
      ansible.builtin.copy:
        content: |
          # RbdDevice             Parameters
          #poolname/imagename     id=client,keyring=/etc/ceph/ceph.client.keyring
          {{ rbd_pool }}/{{ rbd_image }}     id={{ rbd_user }},keyring=/etc/ceph/ceph.keyring
        dest: "/etc/ceph/rbdmap"
        mode: '0644'
    - name: Check RBD device list output
      ansible.builtin.command: "rbd device list --format json"
      register: rbd_devices
    - name: Set RBD device list output as variable
      set_fact:
        rbd_dev_data: "{{ rbd_devices.stdout | from_json }}"
    - name: Identify RBD device name
      set_fact:
        rbd_dev_name: "{{ rbd_dev_data | selectattr('name', 'equalto', rbd_image) | map(attribute='device') | first | default('/dev/rbd0') }}"
    - name: Check if RBD device exists
      stat:
        path: "{{ rbd_dev_name }}"
      register: rbd_dev
    - name: Restart rbdmap.service
      ansible.builtin.systemd:
        state: restarted
        name: rbdmap
      when: "not rbd_dev.stat.exists"
    - name: Check again if RBD device exists
      stat:
        path: "{{ rbd_dev_name }}"
      register: rbd_dev
      when: "not rbd_dev.stat.exists"
    - name: Create file system on RBD
      community.general.filesystem:
        fstype: xfs
        dev: "{{ rbd_dev_name }}"
        opts: -K
        state: present
    - name: Mount up RBD
      ansible.posix.mount:
        src: "{{ rbd_dev_name }}"
        path: "{{ rbd_mount_point }}"
        fstype: xfs
        opts: rw,_netdev
        state: mounted
    - name: Create RBD data directory
      file:
        owner: "{{ galaxy_user_name }}"
        group: "{{ galaxy_user_group_name }}"
        path: "{{ rbd_data_dir }}"
        state: directory
        mode: '0755'

- hosts: dbservers
  become: true
  become_user: root
  pre_tasks:
    - name: Install Dependencies
      package:
        name: ['acl', 'anacron']
  roles:
    - galaxyproject.postgresql
    - role: galaxyproject.postgresql_objects
      become: true
      become_user: postgres
  post_tasks:
    - name: Create pg_dump backup folder if needed
      file:
        path:  "{{ postgresql_backup_dir }}/pg_dump"
        state: directory
        owner: postgres
        group: "{{ galaxy_user_group_name }}"
        mode:  0750
    - name: Create initial empty pg_dump file if needed # otherwise the following logrotate conf will not work
      file:
        path:  "{{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql.gz"
        state: touch
        owner: postgres
        group: "{{ galaxy_user_group_name }}"
        mode:  0640
    - name: Set-up Galaxy DB backup using pg_dump and logrotate
      ansible.builtin.blockinfile:
        path: "/etc/logrotate.d/{{ postgresql_pg_dump_prefix }}"
        create: true
        block: |
          {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql.gz {
            daily
            missingok
            rotate 14
            nocompress
            nocreate
            postrotate
              sudo -u postgres pg_dump {{ galaxy_db_name }} --file={{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql --verbose 2>&1 | logger -t {{ postgresql_pg_dump_prefix }}
              echo "calling: postgres gzip {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql" && sudo -u postgres gzip {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql 2>&1 | logger -t {{ postgresql_pg_dump_prefix }}
              echo "calling: chown postgres:{{ galaxy_user_group_name }} {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql" && chown postgres:{{ galaxy_user_group_name }} {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql 2>&1 | logger -t {{ postgresql_pg_dump_prefix }}
              echo "calling: chmod 640 {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql" && chmod 640 {{ postgresql_backup_dir }}/pg_dump/{{ postgresql_pg_dump_prefix }}.sql 2>&1 | logger -t {{ postgresql_pg_dump_prefix }}
            endscript
          }

- hosts: noletsencrypt
  become: true
  become_user: root
  tasks:
    - name: Certificate
      copy:
        src: "{{ nginx_ssl_src_dir }}/{{ nginx_conf_ssl_certificate | basename }}"
        dest: "/etc/ssl/cert.pem"
        mode: "0644"
    - name: RabbitMQ key
      copy:
        content: "{{ sslkeys[nginx_conf_ssl_certificate_key | basename] }}"
        dest: "{{ rabbitmq_conf_ssl_certificate_key }}"
        owner: "999"
        mode: "0400"

- hosts: galaxyservers
  become: true
  become_user: root
  pre_tasks:
    - name: Admin ssh keys
      blockinfile:
        path: .ssh/authorized_keys
        block: "{{ admin_ssh_keys }}"
      when: admin_ssh_keys
    - name: Install Dependencies
      package:
        name: ['acl', 'bzip2', 'git', 'make', 'tar', 'python3-bioblend', 'python3-venv', 'python3-setuptools', 'python3-pip', 'python3-psycopg2', 'rsync', 'python3-passlib', 'nginx-full', 'logrotate', 'proftpd-mod-ldap', 'squashfs-tools', 'libldap2-dev', 'libsasl2-dev']#, 'krb5-user'] # krb5-user is neccessary only if you need to mount NFS storage
    - name: Install RHEL/CentOS/Rocky specific dependencies
      package:
        name: ['tmpwatch']
      when: ansible_os_family == 'RedHat'
    - name: Install Debian/Ubuntu specific dependencies
      package:
        name: ['tmpreaper']
      when: ansible_os_family == 'Debian'
    - name: Comment out SHOWWARNING line in tmpreaper config
      ansible.builtin.lineinfile:
        path: /etc/tmpreaper.conf
        regexp: "SHOWWARNING="
        state: absent
      when: ansible_os_family == 'Debian'
    - htpasswd:
        path: /etc/nginx/passwd
        name: admin        # Pick a username
        password: "{{ vault_reports_admin_password }}" # and a password
        owner: www-data    # nginx on centos
        group: root
        mode: 0640
      when: csnt_enable_reports
    - git:
        repo: 'https://github.com/usegalaxy-eu/libraries-training-repo'
        dest: /libraries/

    - name: stop galaxy to free mem for client build
      become: true
      become_user: root
      ansible.builtin.command: galaxyctl stop
      when: "inventory_hostname in ['galaxy-qa1.galaxy.cloud.e-infra.cz', 'galaxy-qa2.galaxy.cloud.e-infra.cz']"

    # this needs to be in the correct spot, not here, either via fork or enabling this configuration upstream

    # - name: client building optimization for small machines
    #   replace:
    #     path: "{{ galaxy_root }}/server/client/webpack.config.js"
    #     regexp: 'TerserPlugin\(\)'
    #     replace: 'TerserPlugin({parallel: false,})'
    #   when: "inventory_hostname in ['galaxy-qa1.galaxy.cloud.e-infra.cz', 'galaxy-qa2.galaxy.cloud.e-infra.cz']"

  roles:
    - galaxyproject.tusd
    - usegalaxy_eu.apptainer
    - galaxyproject.galaxy
    - usegalaxy_cz.llm_hub
    - usegalaxy_eu.tpv_auto_lint
    - geerlingguy.redis
    - role: usegalaxy_eu.flower
      when: enable_flower | bool
    - role: galaxyproject.tiaas2
      when: enable_tiaas | bool
    - galaxyproject.nginx
    - galaxyproject.proftpd
    - galaxyproject.rabbitmq
    - galaxyproject.gxadmin
    - galaxyproject.cvmfs
    - role: dj-wasabi.telegraf
      when: enable_telegraf | bool

  post_tasks:
    - name: Add Nginx warning about maintenance
      ansible.builtin.copy:
        src: "{{ playbook_dir }}/files/nginx/warning.html"
        dest: "/var/www/html/warning.html"
        owner: www-data
        group: www-data
        mode: '0644'

    - name: Configure ProFTPd LDAP option
      template:
        src: "{{ lookup('first_found', ['templates/'+inventory_hostname+'/config/proftpd_ldap.conf.j2', 'templates/galaxy/config/proftpd_ldap.conf.j2']) }}"
        dest: "{{ proftpd_config_include_dir }}/06_ldap.conf"
        backup: yes
      notify:
      - reload proftpd

    - name: Adds nginx's www-data user into galaxy user group
      ansible.builtin.user:
        name: www-data
        groups: "{{ galaxy_user_group_name }}"
        append: yes
    - name: Restart Nginx after changing its user
      ansible.builtin.systemd:
        state: restarted
        name: nginx

    # not detected automatically due to a bug: https://github.com/galaxyproject/galaxy/issues/20483
    - name: Install conditional dependencies
      pip:
        name: fs.onedatarestfs==21.2.5.2
        virtualenv: "{{ galaxy_venv_dir }}"
      when: "inventory_hostname in ['galaxy-qa1.galaxy.cloud.e-infra.cz','usegalaxy.cz']"

    # unnecessary after updating to release_25.1
    - name: Update TPV
      pip:
        name: total-perspective-vortex==3.1.3
        virtualenv: "{{ galaxy_venv_dir }}"

    # apply patch to adapt invenio integration to cesnet-invenio
    # atm it hardcodes resource type to be workflow
    - name: Copy patch
      ansible.builtin.copy:
        src: "{{ playbook_dir }}/files/{{inventory_hostname}}/patch/invenio-workflow.patch"
        dest: /tmp/invenio-workflow.patch
      when: "inventory_hostname in ['usegalaxy.cz']"
    - name: Apply patch
      ansible.builtin.shell:
        cmd: git apply /tmp/invenio-workflow.patch
        chdir: "{{ galaxy_root }}/server"
      when: "inventory_hostname in ['usegalaxy.cz']"

    # DEMON: cleanup on places which are not set in standard Galaxy cleanup process
    - name: Creates entry in crontab "tmpreaper call in tus upload dir"
      ansible.builtin.cron:
        name: "tmpreaper in tus upload dir"
        minute: "5"
        hour: "*"
        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 24h {{ galaxy_tus_upload_store }} >> {{ galaxy_mutable_data_dir }}/cleaning_tus_upload_dir.log"
      become: yes
      become_user: "{{ galaxy_user_name }}"
    - name: Creates entry in crontab "tmpreaper call in galaxy tmp dir"
      ansible.builtin.cron:
        name: "tmpreaper in galaxy temp dir"
        minute: "7"
        hour: "*"
        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 24h {{ galaxy_mutable_data_dir }}/tmp >> {{ galaxy_mutable_data_dir }}/cleaning_galaxy_temp_dir.log"
      become: yes
      become_user: "{{ galaxy_user_name }}"
#    - name: Creates entry in crontab "tmpreaper call in galaxy jobs dir"
#      ansible.builtin.cron:
#        name: "tmpreaper in galaxy jobs dir"
#        minute: "9"
#        hour: "*"
#        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 40d {{ galaxy_mutable_data_dir }}/jobs >> {{ galaxy_mutable_data_dir }}/cleaning_galaxy_jobs_dir.log"
#      become: yes
#      become_user: "{{ galaxy_user_name }}"


    # set up cron to cleanup Galaxy user data
    - name: Ensure the cleanup directory exists
      ansible.builtin.file:
        path: "{{ galaxy_root }}/cron/cleanup"
        state: directory
        owner: "{{ galaxy_user_name }}"
        group: "{{ galaxy_user_group_name }}"
        mode: '0755'
      tags: cron
    - name: Copy the cleanup script
      ansible.builtin.template:
        src: "{{ playbook_dir }}/extra_scripts/galaxy_user_data_cleanup.sh.j2"
        dest: "{{ galaxy_root }}/cron/cleanup/galaxy_user_data_cleanup.sh"
        owner: "{{ galaxy_user_name }}"
        group: "{{ galaxy_user_group_name }}"
      tags: cron
    - name: Setup gxadmin cleanup task
      ansible.builtin.cron:
        name: "Cleanup Old User Data"
        user: "{{ galaxy_user_name }}"
        minute: "1"
        hour: "0" # midnight daily
        job: "/bin/bash {{ galaxy_root }}/cron/cleanup/galaxy_user_data_cleanup.sh {{ csnt_galaxy_cleanup_delay }} 2>&1 | logger -t galaxy_user_data_cleanup"
      tags: cron

    - name: Create a file with key for CRON jobs
      ansible.builtin.copy:
        content: |
          {{ api_key }}
        dest: "/home/{{ galaxy_user_name }}/.galaxy_api_key"
        owner: "{{ galaxy_user_name }}"
        group: "{{ galaxy_user_group_name }}"
        mode: '0640'
      tags: cron
      when: "inventory_hostname in ['galaxy-qa1.galaxy.cloud.e-infra.cz', 'usegalaxy.cz', 'galaxy-umsa.grid.cesnet.cz', 'repeatexplorer-elixir.cerit-sc.cz']"
      no_log: true

    # set up cron to propagate einfra users to einfra galaxy group
    - name: Ensure the user_group directory exists
      ansible.builtin.file:
        path: "{{ galaxy_root }}/cron/user_group"
        state: directory
        owner: postgres
        group: postgres
        mode: '0755'
      when: "'usegalaxy.cz' in inventory_hostname"
      tags: cron
    - name: Copy user-group association script
      ansible.builtin.copy:
        src: "{{ playbook_dir }}/extra_scripts/add_group_users.py"
        dest: "{{ galaxy_root }}/cron/user_group/add_group_users.py"
        owner: postgres
        group: postgres
      when: "'usegalaxy.cz' in inventory_hostname"
      tags: cron
    - name: Add the user postgres to the galaxy user primary group, to access the Galaxy API key
      ansible.builtin.user:
        name: postgres
        groups: "{{ galaxy_user_group_name }}"
        append: yes
      when: "'usegalaxy.cz' in inventory_hostname"
      tags: cron
    - name: Setup user-group association rutine
      ansible.builtin.cron:
        name: "Add E-infra users to E-infra group"
        user: postgres
        minute: "0"
        hour: "*" # every hour
        job: "python3 {{ galaxy_root }}/cron/user_group/add_group_users.py {{ inventory_hostname }} $(cat /home/{{ galaxy_user_name }}/.galaxy_api_key)"
      when: "'usegalaxy.cz' in inventory_hostname"
      tags: cron

    # set up cron to shadow some shed_tools contents to minimize space consumption
    - name: Ensure the shadow cron script directory exists
      ansible.builtin.file:
        path: "{{ galaxy_root }}/cron/shadow"
        state: directory
        owner: "{{ galaxy_user_name }}"
        group: "{{ galaxy_user_group_name }}"
        mode: '0755'
      when: "shed_tools_shadowing"
      tags: cron
    - name: Copy shed_tools shadowing script
      ansible.builtin.copy:
        src: "{{ playbook_dir }}/extra_scripts/shed_tools_shadow_script.sh"
        dest: "{{ galaxy_root }}/cron/shadow/shed_tools_shadow_script.sh"
        owner: "{{ galaxy_user_name }}"
        group: "{{ galaxy_user_group_name }}"
        mode: '0755'
      when: "shed_tools_shadowing"
      tags: cron
    - name: Setup shed_tools shadowing rutine
      ansible.builtin.cron:
        name: "Shadow shed-tools directories"
        user: "{{ galaxy_user_name }}"
        minute: "0"
        hour: "2" # 2 am daily
        job: "{{ galaxy_root }}/cron/shadow/shed_tools_shadow_script.sh {{ galaxy_root }}/var {{ galaxy_mutable_data_dir }} 2>&1 | logger -t shed_tools_shadowing"
      when: "shed_tools_shadowing"
      tags: cron

    # set up cron to automate tool installation from https://github.com/CESNET/galaxy_tools repository
    - name: Ensure the tools cron script directory exists
      ansible.builtin.file:
        path: "{{ galaxy_root }}/cron/tools"
        state: directory
        owner: "{{ galaxy_user_name }}"
        group: "{{ galaxy_user_group_name }}"
        mode: '0755'
      tags: cron
      when: "inventory_hostname in ['galaxy-qa1.galaxy.cloud.e-infra.cz', 'usegalaxy.cz', 'galaxy-umsa.grid.cesnet.cz', 'repeatexplorer-elixir.cerit-sc.cz']"
    - name: Template tool installation cron script
      ansible.builtin.template:
        src: "{{ playbook_dir }}/templates/galaxy/cron/install_tools.sh.j2"
        dest: "{{ galaxy_root }}/cron/tools/install_tools.sh"
        owner: "{{ galaxy_user_name }}"
        group: "{{ galaxy_user_group_name }}"
        mode: '0755'
      tags: cron
      when: "inventory_hostname in ['galaxy-qa1.galaxy.cloud.e-infra.cz', 'usegalaxy.cz', 'galaxy-umsa.grid.cesnet.cz', 'repeatexplorer-elixir.cerit-sc.cz']"
    - name: Schedule tool installation script using cron
      ansible.builtin.cron:
        name: "Install tool updates"
        user: "{{ galaxy_user_name }}"
        minute: "0"
        hour: "3" # 3 am daily
        job: "/bin/bash -c 'URL={{ inventory_hostname }} API_KEY=$(cat /home/{{ galaxy_user_name }}/.galaxy_api_key) {{ galaxy_root }}/cron/tools/install_tools.sh 2>&1 | logger -t tool_install'"
      tags: cron
      when: "inventory_hostname in ['galaxy-qa1.galaxy.cloud.e-infra.cz', 'usegalaxy.cz', 'galaxy-umsa.grid.cesnet.cz', 'repeatexplorer-elixir.cerit-sc.cz']"

    # - name: Set-up pre-defined integrated_tool_panel config file on RE instance only
    #   ansible.builtin.copy:
    #     src: "{{ playbook_dir }}/files/{{ inventory_hostname }}/regalaxy_integrated_tool_panel.xml"
    #     dest: "{{ galaxy_mutable_config_dir }}/integrated_tool_panel.xml"
    #     owner: "{{ galaxy_user_name }}"
    #     group: "{{ galaxy_user_group_name }}"
    #     mode: '0644'
    #     backup: true
    #   when: "inventory_hostname in ['repeatexplorer-elixir.cerit-sc.cz', 'galaxy-re.grid.cesnet.cz']"

    - name: Add lines to logrotate config of ProFTPd
      lineinfile:
        path: /etc/logrotate.d/proftpd-core
        line: '{{ item }}'
        insertbefore: BOF
      with_items:
        - /var/log/proftpd/tls.log
        - /var/log/proftpd/ldap.log
        - /var/log/proftpd/sql.log

    - name: Add pulsar public ssh key
      ansible.posix.authorized_key:
        user: "{{ galaxy_user_name }}"
        state: present
        key: "{{ lookup('file','files/{{ inventory_hostname }}/pulsar.pub') }}"

    - name: Restart galaxy
      become: true
      become_user: root
      ansible.builtin.command: galaxyctl restart