ops.yaml

# Author: Nuno Aguiar
help:
  text   : |
    Performs a S3 operation over the provided server connection details. 
    Available operations:

      buckets     - List the available buckets.
                    ojob ojob.io/s3/ops ... op=buckets

      mkbucket    - Creates a new bucket.
                    ojob ojob.io/s3/ops ... op=mkbucket bucket=mynewbucket

      rmbucket    - Remove an existing a bucket.
                    ojob ojob.io/s3/ops ... op=rmbucket bucket=myoldbucket

      ls          - List objects in a bucket (optionally 'remote' prefix and full metadata (slower))
                    ojob ojob.io/s3/ops ... op=ls bucket=mybucket remote=my/folder/ full=true recursive=true

      put         - Put a file in a bucket.
                    ojob ojob.io/s3/ops ... op=put bucket=mybucket local=my/folder/file.txt remote=my/list/file.txt

      get         - Get an object from a bucket.
                    ojob ojob.io/s3/ops ... op=get bucket=mybucket remote=my/list/file.txt local=my/folder/file.txt

      mput        - Put several files in a bucket.
                    ojob ojob.io/s3/ops ... op=mput bucket=mybucket local=list remote=my/list 

      mget        - Get several files from a bucket.
                    ojob ojob.io/s3/ops ... op=mget bucket=mybucket remote=my/list local=list

      sput        - Snowball put several files in a bucket (requires snowball compatibility)
                    ojob ojob.io/s3/ops ... op=sput bucket=mybucket local=list remote=my/list                     

      cp          - Copies an object from a source bucket to a target bucket (can be the same using the bucket argument).
                    ojob ojob.io/s3/ops ... op=cp sourceBucket=original targetBucket=target source=my/file.txt target=archive/file.txt

      mv          - Moves an object from a source bucket to a target bucket (can be the same using the bucket argument).
                    ojob ojob.io/s3/ops ... op=cp sourceBucket=original targetBucket=target source=my/file.txt target=archive/file.txt

      rm          - Removes an object from a bucket.
                    ojob ojob.io/s3/ops ... op=rm bucket=mybucket remote=my/list/file.txt

      rmdir       - Removes objects based on prefix from a bucket.
                    ojob ojob.io/s3/ops ... op=rm bucket=mybucket remote=my/list/

      syncRemote  - Syncs files on a local folder with a remote prefix on a target bucket (deleting/overwriting remotely as needed) 
                    ojob ojob.io/s3/ops ... op=syncremote bucket=mybucket local=. remote=my/list go=true

      syncLocal   - Syncs files on to a local folder from a remote prefix on a target bucket (deleting/overwriting locally as needed)
                    ojob ojob.io/s3/ops ... op=synclocal bucket=mybucket local=. remote=my/list go=true

      sync        - Syncs files between a local folder and a remote prefix on a target bucket (overwrites locally and remotely as needed)
                    ojob ojob.io/s3/ops ... op=sync bucket=mybucket local=. remote=my/list go=true

      stat        - Retrieves the object metadata.
                    ojob ojob.io/s3/ops ... op=stat bucket=mybucket remote=my/lisT/file.txt

    Authentication:

      Use a $sec for an S3 object:
      ojob ojob.io/s3/ops secRepo=... secBucket=... secPass=... secKey=... 

      Set a new $sec for an S3 object:
      ojob ojob.io/sec/ops op=setobj secobj=s3.s3 secbucket=... seckey=... secrepo=... secpass=...

      Use directly (not secure):
      ojob ojob.io/s3/ops url=... accessKey=... secret=... region=...

  expects:
  - name     : op
    desc     : The operation to perform
    example  : help
    mandatory: true
    options  :
    - buckets
    - mkbucket
    - rmbucket
    - ls
    - put
    - get
    - mput
    - mget
    - sput
    - cp
    - mv
    - rm
    - rmdir
    - sync
    - syncLocal
    - syncRemote
    - stat

todo:
- S3

ojob:
  opacks      :
  - openaf: 20250725
  - S3
  - oJob-common
  loadLibs    :
  - s3.js
  catch       : logErr(exception);
  logToConsole: false   # to change when finished

jobs:
# -----------------
- name : S3 connect
  exec : | #js
    if (isDef(args.secEnv)) args.secEnv = toBoolean(args.secEnv)
    if (isDef(args.secKey)) args = $job("ojob sec get", args)
    if (isDef(args._args)) {
      args.url = args._args.aURL
      args.accessKey = args._args.aAccessKey
      args.secret = args._args.aSecret
      args.region = args._args.aRegion
      args.useVersion1 = args._args.useVersion1

      if (args.region == "") args.region = __
    }
    args.url = _$(args.url, "url").default("https://s3.amazonaws.com")
    args.accessKey = _$(args.accessKey, "accessKey").default(__)
    args.secret    = _$(args.secret, "secret").default(__)
    args.region    = _$(args.region, "region").default("us-east-1")

    global.s3 = new S3(args.url, args.accessKey, args.secret, args.region, args.useVersion1)

    $set("out", {})

# -------------------
- name: S3 disconnect
  exec: | #js
    if (isDef(global.s3.close)) global.s3.close()
    global.s3 = __

# ---------------------
- name: S3 list buckets
  from: S3 connect
  to  : S3 disconnect
  exec: | #js
    $set("out", global.s3.listBuckets())

# --------------
- name : S3 list
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket   : isString
      remote   : isString.default("")
      full     : toBoolean.isBoolean.default(false) 
      recursive: toBoolean.isBoolean.default(false) 
  exec : | #js
    var out = {}
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      out = global.s3.listObjects(args.bucket, args.remote, args.full, args.recursive)
      out = $from(out).sort("-isDirectory", "filename").select()
      out = out.map(r => {
        if (isDef(r.createdTime)) r.createdTime = (new Date(r.createdTime)).toISOString()
        if (isDef(r.lastModified)) r.lastModified = (new Date(r.lastModified)).toISOString()
        delete r.isFile
        delete r.isDirectory
        delete r.filename
        delete r.filepath
        delete r.contentType
        return r
      })
    } catch(e) {
      logErr("List error: " + e)
    }

    $set("out", out)

# -------------
- name : S3 get
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      remote: isString
      local : isString
  exec : | #js
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      if (!global.s3.objectExists(args.bucket, args.remote)) throw "Object '" + args.remote + "' in bucket '" + args.bucket + "' doesn't exist."
      global.s3.getObject(args.bucket, args.remote, args.local)
    } catch(e) {
      logErr("Get error: " + e);
    }

# --------------
- name : S3 mget
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      remote: isString
      local : isString
  exec : | #js
    var out = {}
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      if (!io.fileExists(args.local)) io.mkdir(args.local)
      if (io.fileInfo(args.local).isFile) throw "Path '" + args.local + "' not a folder."

      out.transfer = []
      var _m = args.remote.substring(0, args.remote.lastIndexOf("/") + 1), _f = []
      global.s3.listObjects(args.bucket, _m, __, true).forEach(file => {
        _f.push($do(() => {
          if (file.isFile && ow.format.string.wildcardTest(file.filepath, args.remote)) {
            var f = file.filepath.replace(_m, "")
            if (f.indexOf("/") > 0) io.mkdir(args.local + "/" + _m)
            global.s3.getObject(args.bucket, file.canonicalPath, args.local + "/" + f)
            out.transfer.push({ local: args.local + "/" + f, remote: file.canonicalPath })
          }
        }).catch(e2 => {
          logErr("Get error: " + e2)
        }))
      })
      $doWait($doAll(_f))

      $set("out", out)
    } catch(e) {
      logErr("MGet error: " + e)
    }

# --------------
- name : S3 copy
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      target: isString
      source: isString
  exec : | #js
    try {
      if (isDef(args.bucket) && !global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      if (isDef(args.sourceBucket) && !global.s3.bucketExists(args.sourceBucket)) throw "Bucket '" + args.sourceBucket + "' not found."
      if (isDef(args.targetBucket) && !global.s3.bucketExists(args.targetBucket)) throw "Bucket '" + args.targetBucket + "' not found."

      if (isUnDef(args.sourceBucket)) args.sourceBucket = args.bucket
      if (isUnDef(args.targetBucket)) args.targetBucket = args.bucket

      if (isUnDef(args.sourceBucket) && isUnDef(args.targetBucket)) throw "No bucket, sourceBucket or tbucket defined."

      global.s3.copyObject(args.sourceBucket, args.source, args.targetBucket, args.target)
    } catch(e) {
      logErr("Get error: " + e)
    }  

# --------------
- name : S3 move
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      target: isString
      source: isString
  exec : | #js
    try {
      if (isDef(args.bucket) && !global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      if (isDef(args.sourceBucket) && !global.s3.bucketExists(args.sourceBucket)) throw "Bucket '" + args.sourceBucket + "' not found."
      if (isDef(args.targetBucket) && !global.s3.bucketExists(args.targetBucket)) throw "Bucket '" + args.targetBucket + "' not found."

      if (isUnDef(args.sourceBucket)) args.sourceBucket = args.bucket
      if (isUnDef(args.targetBucket)) args.targetBucket = args.bucket

      if (isUnDef(args.sourceBucket) && isUnDef(args.targetBucket)) throw "No bucket, sourceBucket or tbucket defined."

      global.s3.copyObject(args.sourceBucket, args.source, args.targetBucket, args.target)
      global.s3.removeObject(args.sourceBucket, args.source)
    } catch(e) {
      logErr("Get error: " + e)
    }

# -------------
- name : S3 put
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      remote: isString.default("")
      local : isString
  exec : | #js
    try {
      if (!io.fileExists(args.local)) throw "File '" + args.local + "' doesn't exists."
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      global.s3.putObject(args.bucket, args.remote, args.local)
    } catch(e) {
      logErr("Get error: " + e)
    }

# --------------
- name : S3 mput
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      remote: isString.default("")
      local : isString
  exec : | #js
    var out = {}
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      var _m = args.local.substring(0, args.local.lastIndexOf("/") + 1)
      if (io.fileInfo(_m).isFile) throw "Path '" + args.local + "' not a folder."

      out.transfer = []
      var _f = []
      listFilesRecursive(_m).forEach(file => {
        _f.push($do(() => {
          if (file.isFile && ow.format.string.wildcardTest(file.filepath, args.local)) {
            var f = file.filepath.replace(_m, "")
            global.s3.putObject(args.bucket, args.remote + "/" + f, file.canonicalPath)
            out.transfer.push({ remote: args.remote + "/" + f, local: file.canonicalPath })
          }
        }).catch(e2 => {
          logErr("Put error: " + e2)
        }))
      })
      $doWait($doAll(_f))
    } catch(e) {
      logErr("MPut error: " + e)
    }

    $set("out", out)

# --------------
- name : S3 sput
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      remote: isString.default("")
      local : isString
  exec : | #js
    var out = {}
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      var _m = args.local.substring(0, args.local.lastIndexOf("/") + 1)
      if (io.fileInfo(_m).isFile) throw "Path '" + args.local + "' not a folder."

      out.transfer = []
      global.s3.putSnowballObjects(args.bucket, args.remote, args.local)
      out.transfer.push({ remote: args.remote, local: args.local })
    } catch(e) {
      logErr("SPut error: " + e)
    }

    $set("out", out)

# --------------
- name : S3 stat
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      remote: isString
  exec : | #js
    var out
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      if (!global.s3.objectExists(args.bucket, args.remote)) throw "Object '" + args.remote + "' in bucket '" + args.bucket + "' doesn't exist."
      out = global.s3.statObject(args.bucket, args.remote)
      out.createdTime = (new Date(out.createdTime)).toISOString()
    } catch(e) {
      logErr("Get error: " + e)
    }
    $set("out", out)

# ------------
- name : S3 rm
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      remote: isString
  exec : | #js
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      if (!global.s3.objectExists(args.bucket, args.remote)) throw "Object '" + args.remote + "' in bucket '" + args.bucket + "' doesn't exist."
      global.s3.removeObject(args.bucket, args.remote)
    } catch(e) {
      logErr("Get error: " + e)
    }

# ---------------
- name : S3 rmdir
  from : S3 connect
  to   : S3 disconnect
  check:
    in: 
      bucket: isString
      remote: isString.default("")
  exec : | #js
    var out
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      out = global.s3.listObjects(args.bucket, args.remote, __, true)
      out = out.map(r => {
        global.s3.removeObject(args.bucket, r.filepath)
        return r.filepath
      })
    } catch(e) {
      logErr("rmdir error: " + e)
    }
    $set("out", out)

# --------------------
- name : S3 make bucket
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
  exec : | #js
    try {
      if (global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' already exists."
      global.s3.makeBucket(args.bucket)
    } catch(e) {
      logErr("Make bucket: " + e)
    }

# -------------------
- name : S3 rm bucket
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
  exec : | #js
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      global.s3.removeBucket(args.bucket)
    } catch(e) {
      logErr("Remove bucket: " + e)
    }

# --------------
- name : S3 sync
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      local : isString
      remote: isString.default("")
      go    : toBoolean.isBoolean.default(false)
  exec : | #js
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      if (args.go) {
        global.s3.execActions(global.s3.syncActions(args.bucket, args.remote, args.local))
      } else {
        $set("out", global.s3.syncActions(args.bucket, args.remote, args.local))
      }
    } catch(e) {
      logErr("Sync error: " + e)
    }

# --------------------
- name : S3 sync local
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      local : isString
      remote: isString.default("")
      go    : toBoolean.isBoolean.default(false)
  exec : | #js
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      if (args.go) {
        global.s3.execActions(global.s3.squashLocalActions(args.bucket, args.remote, args.local))
      } else {
        $set("out", global.s3.squashLocalActions(args.bucket, args.remote, args.local))
      }
    } catch(e) {
      logErr("Sync error: " + e)
    }

# ---------------------
- name : S3 sync remote
  from : S3 connect
  to   : S3 disconnect
  check:
    in:
      bucket: isString
      local : isString
      remote: isString.default("")
      go    : toBoolean.isBoolean.default(false)
  exec : | #js
    try {
      if (!global.s3.bucketExists(args.bucket)) throw "Bucket '" + args.bucket + "' not found."
      if (args.go) {
        global.s3.execActions(global.s3.squashRemoteActions(args.bucket, args.remote, args.local))
      } else {
        $set("out", global.s3.squashRemoteActions(args.bucket, args.remote, args.local))
      }
    } catch(e) {
      logErr("Sync error: " + e)
    }

# -----------------------
- name : Set format table
  exec : | #js
    if (args.op.toLowerCase() == "lsbuckets" ||
        args.op.toLowerCase() == "buckets" || 
        args.op.toLowerCase() == "ls") args.__format = _$(args.__format).isString().default("table")

# -----------
- name : stop
  exec : exit(0)

# ---------
- name : S3
  check:
    in:
      bucket: isString.default(__)
      op    : isString.oneOf(["lsbuckets","buckets","ls","put","get","mput","mget","cp","mv","rm","rmdir","stat","rmbucket","mkbucket"])
  to   : 
  - ojob options
  - Set format table
  - ojob output
  args :
    __key      : out
    __optionOn : op
    __lowerCase: true
    __todos    :
      buckets   : S3 list buckets
      lsbuckets : S3 list buckets
      ls        : S3 list
      put       : S3 put
      get       : S3 get
      mput      : S3 mput
      sput      : S3 sput
      mget      : S3 mget
      cp        : S3 copy
      mv        : S3 move
      rm        : S3 rm
      rmdir     : S3 rmdir
      stat      : S3 stat
      rmbucket  : S3 rm bucket
      mkbucket  : S3 make bucket
      sync      : S3 sync
      synclocal : S3 sync local
      syncremote: S3 sync remote