fasts3 get s3://mybuck/logs/

.gitignore

@@ -0,0 +1,2 @@
+.idea/*
+fasts3

.goreleaser.yml

@@ -0,0 +1,45 @@
+project_name: fasts3
+release:
+  github:
+    owner: tuneinc
+    name: fasts3
+  name_template: '{{.Tag}}'
+brew:
+  description: "Fast s3 utility is a faster version of s3cmd's ls, get, and cp functions ideal for buckets containing millions of keys."
+  homepage: "https://github.com/tuneinc/fasts3"
+  github:
+    owner: tuneinc
+    name: homebrew-tap
+  commit_author:
+    name: goreleaserbot
+    email: goreleaser@carlosbecker.com
+  install: bin.install "fasts3"
+builds:
+- goos:
+  - linux
+  - darwin
+  goarch:
+  - amd64
+  - "386"
+  main: main.go
+  ldflags: -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}}
+  binary: fasts3
+archive:
+  format: tar.gz
+  name_template: '{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{
+    .Arm }}{{ end }}'
+snapshot:
+  name_template: SNAPSHOT-{{ .Commit }}
+checksum:
+  name_template: '{{ .ProjectName }}_{{ .Version }}_checksums.txt'
+changelog:
+  filters:
+    # commit messages matching the regexp listed here will be removed from
+    # the changelog
+    # Default is empty
+    exclude:
+      - '^\(docs\)'
+      - '^\(travis\)'
+      - '^\(coverage\)'
+      - '^\(tests?'
+      - '^Merge pull request'

.travis.yml

@@ -0,0 +1,28 @@
+language: go
+
+go:
+  - 1.9.x
+  - 1.10.x
+  - master
+
+stages:
+  - test
+  - name: deploy
+    if: tag IS present
+
+jobs:
+  include:
+    - stage: test
+      script: go build ./...
+    - stage: deploy
+      go: 1.10.x
+      script: curl -sL https://git.io/goreleaser | bash
+
+matrix:
+  allow_failures:
+    - go: master
+  fast_finish: true
+
+env:
+  global:
+    secure: "Iby/cnnH2C7SmMJoLBT1khL/jfiNxpoZ2axHyEHSeiaoixS75vkKtyHrrcs8A+71fKgMLOq4w5VGbNfHtrUWt6Ny4IFWiNk6RYPnFu72Bzm6bKFb5xS0yGPQ0Yo1GHGlNwHeBi9gI+MUQ4EDoKj4CGvVL60ELcZIaJjnNvXWRGU="

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 TUNE, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,157 +1,90 @@
+[![Build Status](https://travis-ci.org/tuneinc/fasts3.svg?branch=master)](https://travis-ci.org/tuneinc/fasts3) [![Go Report Card](https://goreportcard.com/badge/github.com/tuneinc/fasts3)](https://goreportcard.com/report/github.com/tuneinc/fasts3)
 ![FastS3](http://i.imgur.com/A42azaA.png)
 ---
 
-Fast s3 utility is a faster version of s3cmd's ls and del functions ideal for listing and deleting buckets containing millions of keys.
+Fast s3 utility is a faster version of s3cmd's ls, get, and cp functions ideal for buckets containing millions of keys.
 
-#Installation
+![autocomplete demo for zsh](autocomplete_demo.gif)
+
+# Installation
 
 ```bash
-go get github.com/TuneOSS/fasts3
-cd $GOPATH/src/github.com/TuneOSS/fasts3
-go build
+go get -u github.com/metaverse/fasts3
 ```
+This should install the binary under `$GOPATH/bin/`
 
-#Configuration
+# Configuration
 
-use `fasts3 init` command  which will create a template file in ~/.fs3cfg
+Use `aws configure` command from the aws cli tool (https://aws.amazon.com/cli/) which will create the necessary config files in `~/.aws/credentials`.
 
+add your region to your credentials file to support tab-completion of buckets:
 ```ini
 [default]
-access_key=<access_key>
-secret_key=<secret_key>
+aws_access_key_id=xxxx
+aws_secret_access_key=xxxx
+region=us-east-1
 ```
 
-fill in the template file with your s3 credentials
-
-alternatively you can set these environment variables:
+Alternatively you can set these environment variables which will take precedence over the credentials file:
 ```bash
 export AWS_ACCESS_KEY_ID=<access_key>
 export AWS_SECRET_ACCESS_KEY=<secret_key>
+export AWS_REGION=us-east-1
 ```
 
-#Usage
-
-```
-usage: fasts3 <command> [<flags>] [<args> ...]
-
-Multi-threaded s3 utility
-
-Flags:
-  --help  Show help.
-
-Commands:
-  help [<command>]
-    Show help for a command.
-
-  ls [<flags>] <s3uri>
-    List s3 prefixes.
-
-  del [<flags>] [<prefixes>]
-    Delete s3 keys
-
-  get [<flags>] [<prefixes>]
-    Fetch files from s3
-
-  stream [<flags>] [<prefixes>]
-    Stream s3 files to stdout
-
-  init
-    Initialize .fs3cfg file in home directory
-
-```
-
-#####ls
-```
-usage: fasts3 [<flags>] ls [<flags>] <s3uri>
-
-List s3 prefixes.
-
-Flags:
-  --help            Show help.
-  -r, --recursive   Get all keys for this prefix.
-  --search-depth=0  search depth to search for work.
-  -H, --human-readable  human readable key size.
-  -d, --with-date   include the last modified date.
-
-Args:
-  <s3uri>  paritial s3 uri to list, ex: s3://mary/had/a/little/lamb/
-
-```
-
-#####del
-```
-usage: fasts3 [<flags>] del [<flags>] [<prefixes>]
-
-Delete s3 keys
-
-Flags:
-  --help            Show help.
-  -r, --recursive   Delete all keys with prefix
-  --search-depth=0  search depth to search for work.
-
-Args:
-  [<prefixes>]  1 or more partial s3 uris to delete delimited by space
-
-```
-
-#####get
-```
-usage: fasts3 get [<flags>] [<prefixes>]
-
-Fetch files from s3
-
-Flags:
-  --search-depth=0  search depth to search for work.
-
-Args:
-  [<prefixes>]  list of prefixes or s3Uris to retrieve
+# Usage
+Use:
 ```
-
-#####stream
-```
-usage: fasts3 stream [<flags>] [<prefixes>]
-
-Stream s3 files to stdout
-
-Flags:
-  --search-depth=0  search depth to search for work.
-  --key-regex=KEY-REGEX
-                    regex filter for keys
-
-Args:
-  [<prefixes>]  list of prefixes or s3Uris to retrieve
+fasts3 --help
+fasts3 <cmd> --help
 ```
 
-####Using search depth to *go* faster
-Many times you know the structure of your s3 bucket, this can be used to optimize listings. Say you have a structure like so:
+### Using search depth to *go* faster
+Many times you know the structure of your s3 bucket, and this can be used to optimize listings. Say you have a structure like so:
 ```bash
 fasts3 ls s3://mybuck/logs/
 
-DIR s3://mybuck/logs/2010/
+DIR s3://mybuck/logs/2011/
 DIR s3://mybuck/logs/2012/
 DIR s3://mybuck/logs/2013/
 DIR s3://mybuck/logs/2014/
 DIR s3://mybuck/logs/2015/
 ```
 
-doing a `fasts3 ls -r s3://mybuck/logs/` will read all keys under `logs` sequentially. We can make this faster by adding a `--search-depth 1` flag to the command which gives each of the underlying directories it's own thread increasing throughput.
+Doing a `fasts3 ls -r s3://mybuck/logs/` will read all keys under `logs` sequentially. We can make this faster by adding a `--search-depth 1` flag to the command which gives each of the underlying directories its own thread, increasing throughput.
+
+### Concurrency
+The concurrency level of s3 command execution can be tweaked based on your usage needs. By default, `4*NumCPU` s3 commands will be executed concurrently, which is ideal based on our benchmarks. If you want to override this value, set `GOMAXPROCS` in your environment to set the concurrency level: `GOMAXPROCS=64 fasts3 ls -r s3://mybuck/logs/` will execute 64 s3 commands concurrently.
 
-####Examples
+### Examples
 ```bash
 # ls
 fasts3 ls s3://mybucket/ # lists top level directories and keys
 fasts3 ls -r s3://mybucket/ # lists all keys in the bucket
 fasts3 ls -r --search-depth 1 s3://mybucket/ # lists all keys in the bucket using the directories 1 level down to thread
+fasts3 ls -r s3://mybucket/ | awk '{s += $1}END{print s}' # sum sizes of all objects in the bucket
 
-# del
-fasts3 del -r s3://mybuck/logs/ # deletes all keys in the prefix
-fasts3 del s3://mybuck/logs/2015/01/12/api.log.201501122359.gz # deletes single key
-fasts3 del $(fasts3 ls s3://mybuck/logs/2015/01/12 | awk -F " " '/api.log/{print $2}') # delete all keys that have "api.log" in them
-
-#get
+# get
 fasts3 get s3://mybuck/logs/ # fetches all logs in the prefix
 
 # stream
 fasts3 stream s3://mybuck/logs/ # streams all logs under prefix to stdout
-fasts3 stream --key-filter ".*2015-01-01" s3://mybuck/logs/ # streams all logs with 2015-01-01 in the key name stdout
+fasts3 stream --key-regex ".*2015-01-01" s3://mybuck/logs/ # streams all logs with 2015-01-01 in the key name stdout
+
+# cp
+fasts3 cp -r s3://mybuck/logs/ s3://otherbuck/ # copies all subdirectories to another bucket
+fasts3 cp -r -f s3://mybuck/logs/ s3://otherbuck/all-logs/ # copies all source files into the same destination directory
+```
+
+### Completion
+Bash and ZSH completion are available.
+
+To install for bash:
+```
+source completion.sh
+```
+
+For zsh:
+```
+source completion.zsh
 ```

cmd/cp.go

@@ -0,0 +1,60 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/spf13/cobra"
+	"github.com/metaverse/fasts3/s3wrapper"
+)
+
+// cpCmd represents the cp command
+var cpCmd = &cobra.Command{
+	Use:   "cp <src> <dest>",
+	Short: "Copy files within S3",
+	Long:  ``,
+	Args:  validateS3URIs(cobra.ExactArgs(2)),
+	Run: func(cmd *cobra.Command, args []string) {
+		recursive, err := cmd.Flags().GetBool("recursive")
+		if err != nil {
+			log.Fatal(err)
+		}
+		flat, err := cmd.Flags().GetBool("flat")
+		if err != nil {
+			log.Fatal(err)
+		}
+		err = Cp(s3Client, args, recursive, delimiter, searchDepth, keyRegex, flat)
+		if err != nil {
+			log.Fatal(err)
+		}
+	},
+}
+
+// Cp copies files from one s3 location to another using svc, s3Uris is a list of source and dest s3 URIs, recurse tells
+// whether to list all keys under the source prefix,  delimiter tells the delimiter to use when listing, searchDepth determines
+// the number of prefixes to list before parallelizing list calls, keyRegex is a regex filter on keys, when flat is
+// true it only takes the last part of the prefix as the filename.
+func Cp(svc *s3.S3, s3Uris []string, recurse bool, delimiter string, searchDepth int, keyRegex string, flat bool) error {
+	listCh, err := Ls(svc, []string{s3Uris[0]}, recurse, delimiter, searchDepth, keyRegex)
+	if err != nil {
+		return err
+	}
+
+	wrap := s3wrapper.New(svc, maxParallel)
+
+	copiedFiles := wrap.CopyAll(listCh, s3Uris[0], s3Uris[1], delimiter, recurse, flat)
+	for file := range copiedFiles {
+		fmt.Printf("Copied %s -> %s%s%s\n", file.FullKey, strings.TrimRight(s3Uris[1], delimiter), delimiter, file.Key)
+	}
+
+	return nil
+}
+
+func init() {
+	rootCmd.AddCommand(cpCmd)
+
+	cpCmd.Flags().BoolP("recursive", "r", false, "Copy all keys for this prefix.")
+	cpCmd.Flags().BoolP("flat", "f", false, "Copy all source files into a flat destination folder (vs. corresponding subfolders)")
+}