HPCTrainingDock/getpackages4cache.sh at main · amd/HPCTrainingDock · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
#!/bin/bash

DISTRO=`cat /etc/os-release | grep '^NAME' | sed -e 's/NAME="//' -e 's/"$//' | tr '[:upper:]' '[:lower:]' `
DISTRO_VERSION=`cat /etc/os-release | grep '^VERSION_ID' | sed -e 's/VERSION_ID="//' -e 's/"$//' | tr '[:upper:]' '[:lower:]' `

SUDO="sudo"
if [  -f /.singularity.d/Singularity ]; then
   SUDO=""
fi

usage()
{
    echo "Usage:"
    echo "  --admin-user [ ADMIN_USER ] default autodetected"
    echo "  --amdgpu-gfxmodel-string [ AMDGPU_GFXMODEL_STRING ]"
    echo "  --identity-file [ IDENTITY_FILE ]"
    echo "  --port-number [ PORT_NUMBER ]"
    echo "  --rocm-version [ ROCM_VERSION ] default $ROCM_VERSION"
    echo "  --help: this usage information"
    exit 1
}

send-error()
{
    usage
    echo -e "\nError: ${@}"
    exit 1
}

reset-last()
{
   last() { send-error "Unsupported argument :: ${1}"; }
}

n=0
while [[ $# -gt 0 ]]
do
   case "${1}" in
      "--admin-user")
         shift
         ADMIN_USER=${1}
         reset-last
         ;;
      "--amdgpu-gfxmodel-string")
         shift
         AMDGPU_GFXMODEL_STRING=${1}
         reset-last
         ;;
      "--distro")
         shift
         DISTRO=${1}
         reset-last
         ;;
      "--distro-version")
         shift
         DISTRO_VERSION=${1}
         reset-last
         ;;
      "--identity-file")
          shift
          IDENTITY_FILE=${1}
          reset-last
          ;;
      "--port-number")
          shift
          PORT_NUMBER=${1}
          reset-last
          ;;
      "--rocm-version")
          shift
          ROCM_VERSION=${1}
          reset-last
          ;;
       "--*")
          send-error "Unsupported argument at position $((${n} + 1)) :: ${1}"
          ;;
      *)
         last ${1}
         ;;
   esac
   n=$((${n} + 1))
   shift
done

if [ "${IDENTITY_FILE}" != "" ]; then
   IDENTITY_FILE="-i ${IDENTITY_FILE}"
fi

ssh-copy-id ${IDENTITY_FILE} -p ${PORT_NUMBER} -o UpdateHostKeys=yes ${ADMIN_USER}@localhost

PACKAGE_LIST=`ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "ls /opt/rocmplus-${ROCM_VERSION}"`
# Remove trailing line break
PACKAGE_LIST=`echo ${PACKAGE_LIST} | tr -d '\r'`
echo "local rocmplus package.list"
echo ":${PACKAGE_LIST}:"

app=rocm-${ROCM_VERSION}
if [ ! -f "HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz" ]; then
   PACKAGE_SIZE=`ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "du -sk /opt/${app} | cut -f1 | tr -d $'\n'" `
   echo "Package size for ${app} is ${PACKAGE_SIZE}"
   if [ "${PACKAGE_SIZE}" -gt "100" ]; then
      echo "Retrieving ${app}"
      ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "cd /opt && tar -czpf /users/${ADMIN_USER}/${app}.tgz ${app}"
      rsync -avz -e "ssh -p ${PORT_NUMBER} ${IDENTITY_FILE}"  ${ADMIN_USER}@localhost:${app}.tgz HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz
   fi
fi

for app in ${PACKAGE_LIST}
do
   if [ ! -f "HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz" ]; then
      PACKAGE_SIZE=`ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "du -sk /opt/rocmplus-${ROCM_VERSION}/${app} | cut -f1 | tr -d $'\n'" `
      echo "Package size for ${app} is ${PACKAGE_SIZE}"
      if [ "${PACKAGE_SIZE}" -gt "100" ]; then
         echo "Retrieving ${app}"
         ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "cd /opt/rocmplus-${ROCM_VERSION} && tar -czf /users/${ADMIN_USER}/${app}.tgz ${app}"
         rsync -avz -e "ssh -p ${PORT_NUMBER} ${IDENTITY_FILE}" ${ADMIN_USER}@localhost:${app}.tgz HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}-rocm-${ROCM_VERSION}-${AMDGPU_GFXMODEL_STRING}/${app}.tgz
      fi
   fi
done

PACKAGE_LIST=`ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "ls /opt"`
# Remove trailing line break
PACKAGE_LIST=`echo ${PACKAGE_LIST} | tr -d '\r'`
echo "local linux package.list"
echo ":${PACKAGE_LIST}:"

for app in ${PACKAGE_LIST}
do
   if [[ "$app" == *rocm* ]]; then
      continue
   fi
   if [ ! -f "HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}/${app}.tgz" ]; then
      PACKAGE_SIZE=`ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "du -sk /opt/${app} | cut -f1 | tr -d $'\n'" `
      echo "Package size for ${app} is ${PACKAGE_SIZE}"
      if [ "${PACKAGE_SIZE}" -gt "100" ]; then
         echo "Retrieving ${app}"
         ssh -p ${PORT_NUMBER} ${IDENTITY_FILE} ${ADMIN_USER}@localhost -t "cd /opt && tar -czf /users/${ADMIN_USER}/${app}.tgz ${app}"
         rsync -avz -e "ssh -p ${PORT_NUMBER} ${IDENTITY_FILE}" ${ADMIN_USER}@localhost:${app}.tgz HPCTrainingDock/CacheFiles/${DISTRO}-${DISTRO_VERSION}/${app}.tgz
      fi
   fi
done