xdeploy

#!/usr/bin/env python
# -*-coding:utf-8-*-

# TODO 判断luiti package 好像是在本地的
# TODO compact with canceled, e.g. catch event, keep the log

import os
import getpass
import subprocess
import json
import pkg_resources
import datetime

today_str = datetime.datetime.now().strftime("%Y%m%d")


print """#1. Setup conf ... """
print """#1.1. read it ... """
current_dir = os.getcwd()
current_user = os.getenv("XDEPLOY_USER", getpass.getuser())
current_pid = os.getpid()
_guess_conf_file = os.path.join(current_dir, "xdeploy_" + current_user + ".json")
conf_file = os.getenv("XDEPLOY_CONF", _guess_conf_file)
example_conf = """ {
    "sync_projects": {
      "local_to_remote" : {
          "vox-hadoop" : ["/Users/mvj3/17zuoye/hg/vox-hadoop/",     "/home/hadoop/deploy/mvj3.20150414/"],
          "luiti"      : ["/Users/mvj3/github/17zuoye/luiti/",      "/home/hadoop/deploy/share.20150414/"],
          "etl_utils"  : ["/Users/mvj3/github/17zuoye/etl_utils/",  "/home/hadoop/deploy/share.20150414/"],
      },
      "remote_to_remote" : {
          "luiti"      : ["/home/hadoop/deploy/share.20150414/luiti", "/home/hadoop/deploy/share.20150414/"],
          "etl_utils"  : ["/home/hadoop/deploy/share.20150414/etl_utils", "/home/hadoop/deploy/share.20150414/"],
      },
      "projects_lazy_install_by_python": [
           "etl_utils",
           "etl_common",
           "table_dump",
           "table_tmp",
           "old_student_teacher_report"
      ],
      "exclude_rules": [
          "*.pyc",
          "build",
          "dist",
          "*.egg-info"
      ]
    },
    "remote_server": {
        "user"        : "hadoop",
        "host"        : "8.8.8.8",
        "execute_dir" : "/home/hadoop/deploy/mvj3.20150414/",
    },
    "scripts_before_run": [
    ],
    "launch": {
        "template": "cd %s; luiti run --task-name %s --date-value %sT00:00:00+08:00",
        "params_list": [
            ["old_student_teacher_report", "Redmine7518EnglishStudentReportWeek", "2015-03-09"],
            ["table_dump", "DumpSchoolCityRefFromDatabaseDay", "2015-03-09"]
        ],
        "params_index": 0
    }
}
"""
if not os.path.exists(conf_file):
    print """[warn] can't find %s ! Please create one, e.g. %s""" % (conf_file, example_conf)
conf = json.loads(file(conf_file).read())

print """#1.2. validate it ... """
assert isinstance(conf, dict)
assert "sync_projects" in conf, "See valid example in conf %s" % example_conf
assert "remote_server" in conf, "See valid example in conf %s" % example_conf
conf_file_msg = "Auto detected" if _guess_conf_file is conf_file else "Specified"
print "** %s config file is %s" % (conf_file_msg, conf_file)

print """#1.3. assign some variables ... """
remote_user_host = "%s@%s" % (conf["remote_server"]["user"], conf["remote_server"]["host"],)
conf["dep_packages_with_install_cmd"] = conf.get("dep_packages_with_install_cmd", dict())
execute_dir = conf["remote_server"]["execute_dir"]

# rsync 不要 -z 选项
rsync_command = "rsync -av "
exclude_rules = conf["sync_projects"].get("exclude_rules", list())
for rule1 in exclude_rules:
    rsync_command += (" --exclude %s " % rule1)
rsync_output_file = "/tmp/xdeploy_rsync_%s_%s" % (today_str, current_pid)
rsync_output_file_remote = "%s_2" % rsync_output_file


def wrap_remote_python_env(cmd):
    return """ssh %s "source ~/.bash_profile
              cd %s
              if [ -f ENV/bin/activate ];
                  then source ENV/bin/activate
              fi;
              %s"
            """ % (remote_user_host,
                   execute_dir,
                   cmd,)


def shell(cmd, wrap=None, return_output=False):
    # TODO replace with subprocess
    if wrap:
        cmd = wrap(cmd)
    print "[command]", cmd
    if return_output:
        return subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
    else:
        return os.system(cmd)


def rshell(cmd):
    return shell(cmd, wrap=wrap_remote_python_env)


def install_package_cmd(package_name):
    return """
             cd %s
             cd %s
             python setup.py install
            """ % (execute_dir, package_name,)

for pkg1 in conf["sync_projects"]["projects_lazy_install_by_python"]:
    conf["dep_packages_with_install_cmd"][pkg1] = {"match": "\n%s/" % pkg1,  # sync files is line by line.
                                                   "cmd": install_package_cmd(pkg1), }


print """#2. sync source code"""
for sync_opts_type in ["local_to_remote", "remote_to_remote"]:
    sync_opt2 = conf["sync_projects"][sync_opts_type]
    for project1 in sync_opt2:
        if sync_opts_type == "remote_to_remote":
            from_addr, to_addr = sync_opt2[project1]
        else:
            from_addr = sync_opt2[project1][0]
            to_addr = "%s:%s" % (remote_user_host, sync_opt2[project1][1])
        source_code_sync_command2 = """%s %s  %s | tee -a %s """ % \
            (rsync_command,
             from_addr,
             to_addr,
             rsync_output_file)
        if sync_opts_type == "remote_to_remote":
            rshell(source_code_sync_command2)
        else:
            shell(source_code_sync_command2)

# get remote changed content
shell("""scp %s:%s %s; cat %s >> %s; rm -f %s; """ % (remote_user_host, rsync_output_file, rsync_output_file_remote,
                                                      rsync_output_file_remote, rsync_output_file,
                                                      rsync_output_file_remote))
source_code_sync_result = file(rsync_output_file).read()

shell("echo rsync_output_file; cat %s" % rsync_output_file)
shell("rm -f %s" % rsync_output_file)

print """#3. prepare virtualenv"""
rshell("""
if [ ! -f ENV/bin/activate ]; then
    pip install virtualenv
    virtualenv ENV
fi;
""")


print """#4. install package lazily"""
for pkg1, install_cmd1 in conf["dep_packages_with_install_cmd"].iteritems():
    # compact with interrupt of sync code, and `rsync` will not detect changed code, and will not install updated packages.
    pkg1 = pkg1.split("/")[-1]  # if has dir
    need_install1 = False
    try:
        pkg_resources.require(pkg1)
    except:
        need_install1 = True
    if install_cmd1["match"] in source_code_sync_result:
        need_install1 = True

    if need_install1:
        print "[install packaqe]", pkg1, "is changed ..."
        rshell(install_cmd1["cmd"])
    else:
        print "[install packaqe]", pkg1, "is not changed."

print """#5. run some scripts. """
for script1 in conf.get("scripts_before_run", list()):
    rshell(script1)

print """#6. launch program """
task_opts = conf["launch"].get("params_list", list())
if not isinstance(task_opts, list):
    task_opts = [task_opts]
params_index = conf["launch"].get("params_index", 0)
task_opts = task_opts[params_index]
rshell(conf["launch"]["template"] % tuple(task_opts))

print """#7. clean tmp file """
clean_file_under_root_tmp = """find /tmp/ -maxdepth 1 -type f -mmin +30 | grep %s | xargs rm -f ;"""

rshell(clean_file_under_root_tmp % "xdeploy_")
# TODO when exit, clean