Dev master

INSTALL.PL

@@ -76,7 +76,7 @@ if( $ENV{box} )
 
     mysystem "rsync -a box/ box.i/";
     mysystem "rsync -a dan/node/ box.i/node/";
-    map { mysystem "rsync -a dan/tools/$_ box.i/tools/" } qw(range mcmd mssh expect vssh go rsync alias unalias sync config whois gateway git diskSpaceControl supervisor secure reborn check );
+    map { mysystem "rsync -a dan/tools/$_ box.i/tools/" } qw(range mcmd mssh expect vssh go rsync alias unalias sync config whois gateway git gitrsync diskSpaceControl supervisor secure reborn check fastmd5 os );
 
 
     my @sed = (

README.md

@@ -65,6 +65,21 @@ MYDan支持两种协议，ssh协议和MYDan自定义协议。推荐使用MYDan
 
     (等同于: curl -L http://install.mydan.org|bash)
 
+安装方式4:
+
+    (安装到/opt/mydan下)
+    需要安装的服务器不能上网,需要一个可以上网的机器先下载安装包,然后拷贝到服务器进行安装
+
+    安装步骤:
+        1. 新建一个目录,在目录中运行下载安装包命令:
+            curl -s https://raw.githubusercontent.com/MYDan/openapi/master/scripts/mydan/package.sh|bash
+            (也可以只下载某个版本 curl -s https://raw.githubusercontent.com/MYDan/openapi/master/scripts/mydan/package.sh|bash -s Linux:x86_64)
+
+            (等同于: curl -L http://package.mydan.org|bash)
+
+        2. 拷贝名如mydan.agent.20190524140060.Linux.x86_64 的文件到需要安装的服务器上
+        3. 在服务器上运行./mydan.agent.20190524140060.Linux.x86_64 进行安装
+
 ==============================推荐使用方式===================================
 
 第一步: 

bin/mydan

@@ -18,6 +18,7 @@ my @cmd =
     [ 'gateway',   "代理.\n", 'dan/tools/gateway', 'box/tools/gateway' ],
 
     [ 'rcall', "远程调用${name}的agent.", 'dan/tools/rcall' ],
+    [ 'lcall', "调用本地agent插件.", 'dan/tools/lcall' ],
     [ 'rtail',  'tail多个远程机器日志.', 'dan/tools/rtail' ],
     [ 'vssh',  '虚拟登陆多个机器.', 'dan/tools/vssh', 'box/tools/vssh' ],
     [ 'mssh',  '批量ssh命令.', 'dan/tools/mssh', 'box/tools/mssh' ],
@@ -33,13 +34,16 @@ my @cmd =
     [ 'config', "${name}配置.\n", 'dan/tools/config', 'box/tools/config' ],
 
     [ 'shell', "获取远程机器反弹shell.\n", 'dan/tools/shell' ],
+    [ 'shellv2', "获取远程机器反弹shell的v2版本.\n", 'dan/tools/shellv2' ],
 
     [ 'alias', "${name}内部使用的alias.", 'dan/tools/alias', 'box/tools/alias' ],
     [ 'unalias', "${name}内部使用的unalias.\n", 'dan/tools/unalias', 'box/tools/unalias' ],
 
+    [ 'fastmd5', "快速计算文件md5.", 'dan/tools/fastmd5', 'box/tools/fastmd5' ],
     [ 'sync', "同步${name}的配置", 'dan/tools/sync', 'box/tools/sync' ],
     [ 'xtar', '脚本和数据压缩工具.', 'dan/tools/xtar' ],
     [ 'git', 'git命令，添加了指定key功能.', 'dan/tools/git', 'box/tools/git' ],
+    [ 'gitrsync', '代码发布到远程机器.', 'dan/tools/gitrsync', 'box/tools/gitrsync' ],
     [ 'alarm', '设置超时闹钟运行命令.', 'dan/tools/alarm' ],
     [ 'bigest', '查找大文件.', 'dan/tools/bigest' ],
     [ 'expect', '自动应答.', 'dan/tools/expect' ],
@@ -51,12 +55,19 @@ my @cmd =
     [ 'deploy', "本地发布切连接小工具.\n", 'dan/tools/deploy' ],
     [ 'check', "检查服务的小工具.\n", 'dan/tools/check', 'box/tools/check' ],
     [ 'secure', "私密文件管理小工具.\n", 'dan/tools/secure', 'box/tools/secure' ],
+    [ 'udpfileserver', "udp文件服务.\n", 'dan/tools/udpfileserver' ],
 
     [ 'diskSpaceControl', "控制磁盘使用空间在某个百分比.\n", 'dan/tools/diskSpaceControl', 'box/tools/diskSpaceControl' ],
 
     [ 'node', '机器管理.', 'bin/mydan.node' ],
+    [ 'code', 'git和svn代码仓库管理.适用于管理员.', 'dan/code/bin/control' ],
+    [ 'checkuser', '检查用户状态.', 'dan/code/bin/checkuser' ],
 
     [ 'reborn', "重装系统.\n", 'dan/tools/reborn', 'box/tools/reborn' ],
+
+    [ 'os', "获取当前服务器的cpu/mem等信息.\n", 'dan/tools/os', 'box/tools/os' ],
+
+    [ 'antdencli', "AntDen 命令行工具.\n", 'dan/antden/bin/antdencli' ],
 );
 
 MYDan::Util::Command->new( $name => \@cmd,

dan/.config

@@ -51,6 +51,7 @@ collector:
 
 util:
   conf: $MYDanPATH/etc/util/conf
+  fastmd5: 5368709120
 
 monitor:
   conf: $ROOT/monitor/conf
@@ -62,6 +63,7 @@ monitorv2:
   code: $ROOT/monitorv2/code
   scripts: $ROOT/monitorv2/scripts
   logs: $MYDanPATH/var/logs/monitorv2
+  run: $MYDanPATH/var/run/monitorv2
 
 api:
   addr: http://127.0.0.1:5555 #dashboard_addr
@@ -80,6 +82,8 @@ tcpserver:
   rbuf: 104857600
   ReservedSpaceCount: 10
   ReservedSpaceSize: 2
+  rlimit: 20971520
+  wlimit: 20971520
 
 notify:
   level:

dan/agent/argv/cleandir

@@ -0,0 +1,17 @@
+#!# perl: agent/argv/cleandir
+use strict;
+use warnings;
+
+use Getopt::Long;
+
+return sub
+{
+    local $/;
+    my %o;
+
+    Getopt::Long::GetOptionsFromArray( \@_, \%o, qw( uuid=s expire ) );
+
+    $o{path} = \@_;
+
+    return \%o;
+};

dan/agent/argv/dump

@@ -3,8 +3,8 @@ use strict;
 use warnings;
 
 use Cwd;
-use Digest::MD5;
 use Getopt::Long;
+use MYDan::Util::FastMD5;
 
 return sub
 {
@@ -23,9 +23,7 @@ return sub
 	$o{chown} ||= ( getpwuid($uid) )[0];
     }
 
-    die "Can't open '$file': $!\n" unless open my  $fh, '+<', $file;
-    $o{md5} = Digest::MD5->new()->addfile( $fh )->hexdigest();
-    close $fh;
+    $o{md5} = MYDan::Util::FastMD5->hexdigest( $file );
 
     $ENV{MYDanExtractFile} = $file;
     $ENV{MYDanExtractFileAim} = $o{path};

dan/agent/argv/mrsync

@@ -1,4 +1,4 @@
-#!# perl: agent/argv/dump
+#!# perl: agent/argv/mrsync
 
 use strict;
 use warnings;
@@ -16,7 +16,7 @@ return sub
     else
     {
         Getopt::Long::GetOptionsFromArray( \@_, \%o, 
-            qw( src=s dst=s sp=s dp=s timeout=i max=i retry=i nice=i user=s sudo=s gave=i chown=s chmod=s cc ) 
+            qw( src=s dst=s sp=s dp=s timeout=i max=i retry=i nice=i user=s sudo=s gave=i chown=s chmod=s cc env ) 
         );
     }
 
@@ -37,7 +37,7 @@ return sub
     my %p = map{ $o{$_} => 1 }grep{ defined $o{$_} }qw( sp dp );
     my %n = map{ $_ => 1 }@{$o{src}}, @{$o{dst}};
 
-    $o{queryxdata} = MYDan::Agent::Mrsync::queryx( node => [ keys %n ], map{ $_ => $o{$_} }qw( user sudo ) );
+    $o{queryxdata} = MYDan::Agent::Mrsync::queryx( node => [ keys %n ], map{ $_ => $o{$_} }qw( user sudo env ) );
 
     map{ die "agent/argv/mrsync: $_ null" unless $o{$_} && @{$o{$_}} }qw( src dst );
     die "agent/argv/mrsync: sp undef" unless $o{sp};

dan/agent/argv/scriptsx

@@ -0,0 +1,23 @@
+#!# perl: agent/argv/scriptsx
+
+use strict;
+use warnings;
+
+use Cwd;
+use Digest::MD5;
+use Getopt::Long;
+
+return sub
+{
+    local $/;
+    my ( $file, %o, $data ) = shift;
+    die "file undef\n" unless $file;
+    die "file nofind: $file\n" unless -f $file;
+
+    Getopt::Long::GetOptionsFromArray( \@_, \%o, qw( scriptstype=s ) );
+
+    $o{type} = delete $o{scriptstype};
+    $o{argv} = join ' ', map{ "'$_'" }@_;
+    $o{md5} = Digest::MD5->new()->add( $o{cont} = `cat $file` )->hexdigest();
+    return [ \%o ];
+};

dan/agent/argv/unzipdir

@@ -0,0 +1,17 @@
+#!# perl: agent/argv/unzipdir
+use strict;
+use warnings;
+
+use Getopt::Long;
+
+return sub
+{
+    local $/;
+    my %o;
+
+    Getopt::Long::GetOptionsFromArray( \@_, \%o, qw( uuid=s chmod=s chown=s delete ) );
+
+    $o{path} = \@_;
+
+    return \%o;
+};

dan/agent/argv/zipdir

@@ -0,0 +1,20 @@
+#!# perl: agent/argv/zipdir
+use strict;
+use warnings;
+
+use Getopt::Long;
+
+return sub
+{
+    local $/;
+    my %o;
+
+    Getopt::Long::GetOptionsFromArray( \@_, \%o, qw( uuid=s makelist dirdetail) );
+
+    my $chars = [ "A" .. "Z", "a" .. "z", 0 .. 9 ];
+    $o{uuid} ||= join("", @$chars[ map { rand @$chars } ( 1 .. 8 ) ]);
+
+    $o{path} = \@_;
+
+    return \%o;
+};

dan/agent/code/antdencli

@@ -0,0 +1,42 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+use YAML::XS;
+use MYDan::API::AntDen;
+
+local $/ = undef;
+
+my ( %param, @r )= %{ YAML::XS::Load( <> ) };
+
+my $argv = $param{argv};
+my $api = MYDan::API::AntDen->new( addr => "http://127.0.0.1:3000" );
+my $idie = sub{ print shift; exit 1; };
+
+my %version = ( cli => '1.0.01' , image => '1.0.01' );
+for my $k ( keys %version )
+{
+    next unless $argv->{version} && $argv->{version}{$k};
+    my $v = $argv->{version}{$k};
+
+    $v =~ s/\D/./g;
+    next if $v ge $version{$k};
+
+    print "\n", '=' x 75, "\n";
+    print "Your current $k version is $v. Need an update to $version{$k}\n";
+    print "Get more help: https://antden.github.io\n";
+    print '=' x 75, "\n";
+    exit;
+}
+
+@r = eval{ $api->submitjob( %{$argv->{conf}}, owner => $param{user} ); } if $argv->{ctrl} eq 'submitjob';
+@r = eval{ $api->listjob( $param{user} ); } if $argv->{ctrl} eq 'listjob';
+@r = eval{ $api->info( $argv->{conf}{jobid}, $param{user} ); } if  $argv->{ctrl} eq 'info';
+@r = eval{ $api->stop( $argv->{conf}{jobid}, $param{user} ); } if $argv->{ctrl} eq 'stop';
+@r = eval{ $api->taskinfo( $argv->{conf}{taskid} ); } if $argv->{ctrl} eq 'taskinfo';
+@r = eval{ $api->resources( $param{user} ); } if $argv->{ctrl} eq 'resources';
+@r = eval{ $api->datasets( $param{user} ); } if $argv->{ctrl} eq 'datasets';
+
+&$idie( "err: $@" ) if $@;
+print YAML::XS::Dump @r;
+
+exit;

dan/agent/code/cleandir

@@ -0,0 +1,51 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+use YAML::XS;
+
+local $/ = undef;
+
+my %param = %{ YAML::XS::Load( <> ) };
+my ( $path, $uuid, $expire ) = map{ $param{argv}{$_}}qw( path uuid expire );
+
+$expire = 0 unless $expire && $expire =~ /^\d+$/;
+my $idie = sub{ print shift;exit 1; };
+
+&$idie( "uuid format error:$uuid" ) unless $uuid && $uuid =~ /^[a-zA-Z0-9]+$/;
+
+my $prefix = '.TEMP_MYDan_Grsync_';
+my ( $temp, $fail )= "${prefix}$uuid";
+
+for my $p ( @$path )
+{
+    next unless -e $p;
+    unless( chdir $p )
+    {
+        print "cd $p fail\n";
+        $fail = 1 ;
+        next;
+    }
+
+    if( system "rm -f $temp*" )
+    {
+        print "clean temp fail\n";
+        $fail = 1;
+        next;
+    }
+
+    next unless  $expire;
+    my $timeout = time - $expire;
+
+    for my $f ( grep{ -f $_ } glob "${prefix}*" )
+    {
+        my $mtime = ( stat $f )[9];
+        next if $mtime > $timeout;
+        unless( unlink  $f )
+        {
+            print "unlink $f fail: $!\n";
+            $fail = 1;
+        }
+    }
+}
+
+$fail ? exit( 1 ) : exit( 0 );

dan/agent/code/download

@@ -2,8 +2,60 @@
 use strict;
 use warnings;
 use YAML::XS;
+
+use AnyEvent;
+use AnyEvent::Handle;
+use AnyEvent::Socket;
+use Time::HiRes qw(time);
+use Sys::Hostname;
+
 use MYDan::Agent::Load;
-
+
+my $idie = sub{ print shift;exit 1; };
+my ( $cv, $pcb ) = ( AE::cv );
+if( my $rlog = $ENV{MYDan_rlog} )
+{
+    my ( $uuid, $server, $port ) = split /:/, $rlog;
+    my %argv = ( 
+        uuid => $uuid,
+        addr => $server || $ENV{TCPREMOTEIP}, 
+        port => $port || $ENV{TCPREMOTEPORT},
+    );
+
+    warn sprintf "download info: host:%s port:%s uuid:%s\n", map{ $argv{$_} ||'' }qw( addr port uuid );
+
+    my ( $index, %cv, $w, $hdl ) = ( 0 );
+
+    tcp_connect $argv{addr}, $argv{port}, sub {
+        my ( $fh ) = @_;
+        unless( $fh )
+        {
+            print "$argv{addr}:$argv{port} tcp_connect: $!";
+            $cv->send;
+            return;
+        }
+        $hdl = new AnyEvent::Handle(
+            fh => $fh,
+            rbuf_max => 10240000,
+            wbuf_max => 10240000,
+            autocork => 1,
+            on_error => sub{
+                warn 'tcp error';
+                undef $hdl;
+                $cv->send;
+            },
+            on_eof => sub{
+                warn 'tcp close';
+                undef $hdl;
+                $cv->send;
+            },
+        );
+        $hdl->push_write( $argv{uuid} . ':' . hostname . ':' );
+    },  sub{ return 5; };
+
+    $pcb = sub { $hdl->push_write( shift."\n" ) if $hdl; };
+}
+
 eval{
 
     local $/ = undef;
@@ -19,10 +71,9 @@ eval{
     MYDan::Agent::Load->new( 
         node => $param->{src}, 
         map{ $_ => $param->{$_} }qw( sp dp )
-    )->run( %$param, role => 'agent', query => $param->{load}||1 );
+    )->run( %$param, role => 'agent', pcb => $pcb, verbose => $pcb ? 1 : 0, query => $param->{load}||1 );
 };
 
 if($@) { print "load fail $@";exit 1; }
 
 exit 0;
-