Develop

.gitignore

@@ -10,3 +10,4 @@ data*
 *.pickle
 authorized_chats.txt
 log.txt
+accounts/*

Dockerfile

@@ -1,16 +1,20 @@
-FROM ubuntu:18.04
+FROM python:3-alpine
+
+WORKDIR /bot
+RUN chmod 777 /bot
+
+# install ca-certificates so that HTTPS works consistently
+RUN apk add --no-cache --update ca-certificates aria2 libmagic
+
+RUN apk add --no-cache --update --virtual .build-deps \
+      build-base \
+      libffi-dev \
+      openssl-dev
 
-WORKDIR /usr/src/app
-RUN chmod 777 /usr/src/app
-RUN apt -qq update
-RUN apt -qq install -y aria2 python3 python3-pip locales
 COPY requirements.txt .
 RUN pip3 install --no-cache-dir -r requirements.txt
+RUN apk del .build-deps
 COPY . .
-RUN chmod +x aria.sh
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+RUN chmod +x *.sh
 
-CMD ["bash","start.sh"]
+CMD ["sh", "./start.sh"]

README.md

@@ -84,3 +84,34 @@ sudo docker build . -t mirror-bot
 ```
 sudo docker run mirror-bot
 ```
+
+## Using service accounts for uploading to avoid user rate limit
+
+Many thanks to [AutoRClone](https://github.com/xyou365/AutoRclone) for the scripts
+### Generating service accounts
+Step 1. Generate service accounts [What is service account](https://cloud.google.com/iam/docs/service-accounts) [How to use service account in rclone](https://rclone.org/drive/#service-account-support).
+---------------------------------
+Let us create only the service accounts that we need. 
+**Warning:** abuse of this feature is not the aim of autorclone and we do **NOT** recommend that you make a lot of projects, just one project and 100 sa allow you plenty of use, its also possible that overabuse might get your projects banned by google. 
+
+```
+Note: 1 service account can copy around 750gb a day, 1 project makes 100 service accounts so thats 75tb a day, for most users this should easily suffice. 
+```
+
+`python3 gen_sa_accounts.py --quick-setup 1 --new-only`
+
+A folder named accounts will be created which will contain keys for the service accounts created
+```
+We highly recommend to zip this folder and store it somewhere safe, so that you do not have to create a new project everytime you want to deploy the bot
+```
+### Adding service accounts to Google Groups:
+We use Google Groups to manager our service accounts considering the  
+[Official limits to the members of Team Drive](https://support.google.com/a/answer/7338880?hl=en) (Limit for individuals and groups directly added as members: 600).
+
+1. Turn on the Directory API following [official steps](https://developers.google.com/admin-sdk/directory/v1/quickstart/python) (save the generated json file to folder `credentials`).
+
+2. Create group for your organization [in the Admin console](https://support.google.com/a/answer/33343?hl=en). After create a group, you will have an address for example`sa@yourdomain.com`.
+
+3. Run `python3 add_to_google_group.py -g sa@yourdomain.com`
+
+4. Now, add Google Groups (**Step 2**) to manager your service accounts, add the group address `sa@yourdomain.com` or `sa@googlegroups.com` to the Team drive or folder

add_to_google_group.py

@@ -0,0 +1,84 @@
+# auto rclone
+# Add service accounts to groups for your organization
+#
+# Author Telegram https://t.me/CodyDoby
+# Inbox  codyd@qq.com
+
+from __future__ import print_function
+
+import os
+import pickle
+
+import argparse
+import glob
+import googleapiclient.discovery
+import json
+import progress.bar
+import time
+from google.auth.transport.requests import Request
+from google_auth_oauthlib.flow import InstalledAppFlow
+
+stt = time.time()
+
+parse = argparse.ArgumentParser(
+    description='A tool to add service accounts to groups for your organization from a folder containing credential '
+                'files.')
+parse.add_argument('--path', '-p', default='accounts',
+                   help='Specify an alternative path to the service accounts folder.')
+parse.add_argument('--credentials', '-c', default='credentials/credentials.json',
+                   help='Specify the relative path for the controller file.')
+parsereq = parse.add_argument_group('required arguments')
+# service-account@googlegroups.com
+parsereq.add_argument('--groupaddr', '-g', help='The address of groups for your organization.', required=True)
+
+args = parse.parse_args()
+acc_dir = args.path
+gaddr = args.groupaddr
+credentials = glob.glob(args.credentials)
+
+creds = None
+if os.path.exists('credentials/token.pickle'):
+    with open('credentials/token.pickle', 'rb') as token:
+        creds = pickle.load(token)
+# If there are no (valid) credentials available, let the user log in.
+if not creds or not creds.valid:
+    if creds and creds.expired and creds.refresh_token:
+        creds.refresh(Request())
+    else:
+        flow = InstalledAppFlow.from_client_secrets_file(credentials[0], scopes=[
+            'https://www.googleapis.com/auth/admin.directory.group',
+            'https://www.googleapis.com/auth/admin.directory.group.member'
+        ])
+        # creds = flow.run_local_server(port=0)
+        creds = flow.run_console()
+    # Save the credentials for the next run
+    with open('credentials/token.pickle', 'wb') as token:
+        pickle.dump(creds, token)
+
+group = googleapiclient.discovery.build("admin", "directory_v1", credentials=creds)
+
+print(group.members())
+
+batch = group.new_batch_http_request()
+
+sa = glob.glob('%s/*.json' % acc_dir)
+
+# sa = sa[0:5]
+
+pbar = progress.bar.Bar("Readying accounts", max=len(sa))
+for i in sa:
+    ce = json.loads(open(i, 'r').read())['client_email']
+
+    body = {"email": ce, "role": "MEMBER"}
+    batch.add(group.members().insert(groupKey=gaddr, body=body))
+    # group.members().insert(groupKey=gaddr, body=body).execute()
+
+    pbar.next()
+pbar.finish()
+print('Adding...')
+batch.execute()
+
+print('Complete.')
+hours, rem = divmod((time.time() - stt), 3600)
+minutes, sec = divmod(rem, 60)
+print("Elapsed Time:\n{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), sec))

bot/__init__.py

@@ -1,11 +1,15 @@
 import logging
-import aria2p
-import threading
 import os
-from dotenv import load_dotenv
-import telegram.ext as tg
+import threading
 import time
 
+import aria2p
+import telegram.ext as tg
+from dotenv import load_dotenv
+import socket
+
+socket.setdefaulttimeout(600)
+
 botStartTime = time.time()
 if os.path.exists('log.txt'):
     with open('log.txt', 'r+') as f:
@@ -28,7 +32,7 @@ def getConfig(name: str):
 
 try:
     if bool(getConfig('_____REMOVE_THIS_LINE_____')):
-        logging.ERROR('The README.md file there to be read! Exiting now!')
+        logging.error('The README.md file there to be read! Exiting now!')
         exit()
 except KeyError:
     pass
@@ -80,13 +84,22 @@ def getConfig(name: str):
     INDEX_URL = None
 try:
     IS_TEAM_DRIVE = getConfig('IS_TEAM_DRIVE')
-    if IS_TEAM_DRIVE == 'True' or IS_TEAM_DRIVE == 'true':
+    if IS_TEAM_DRIVE.lower() == 'true':
         IS_TEAM_DRIVE = True
     else:
         IS_TEAM_DRIVE = False
-
 except KeyError:
     IS_TEAM_DRIVE = False
-updater = tg.Updater(token=BOT_TOKEN)
+
+try:
+    USE_SERVICE_ACCOUNTS = getConfig('USE_SERVICE_ACCOUNTS')
+    if USE_SERVICE_ACCOUNTS.lower() == 'true':
+        USE_SERVICE_ACCOUNTS = True
+    else:
+        USE_SERVICE_ACCOUNTS = False
+except KeyError:
+    USE_SERVICE_ACCOUNTS = False
+
+updater = tg.Updater(token=BOT_TOKEN,use_context=True)
 bot = updater.bot
 dispatcher = updater.dispatcher

bot/__main__.py

@@ -8,51 +8,60 @@
 import shutil
 from .helper.telegram_helper.filters import CustomFilters
 from bot.helper.telegram_helper.bot_commands import BotCommands
-from .modules import authorize, list, cancel_mirror, mirror_status, mirror
+from .modules import authorize, list, cancel_mirror, mirror_status, mirror, clone, watch
+
 
 @run_async
-def stats(bot,update):
+def stats(update,context):
     currentTime = get_readable_time((time.time() - botStartTime))
     total, used, free = shutil.disk_usage('.')
     total = get_readable_file_size(total)
     used = get_readable_file_size(used)
     free = get_readable_file_size(free)
     stats = f'Bot Uptime: {currentTime}\n' \
             f'Total disk space: {total}\n' \
-                        f'Used: {used}\n' \
-                        f'Free: {free}'
+            f'Used: {used}\n' \
+            f'Free: {free}'
     sendMessage(stats, bot, update)
 
 
-
 @run_async
-def start(bot,update):
+def start(update,context):
     sendMessage("This is a bot which can mirror all your links to Google drive!\n"
-                "Type /help to get a list of available commands", bot, update)
+                "Type /help to get a list of available commands", context.bot, update)
 
 
 @run_async
-def ping(bot,update):
+def ping(update,context):
     start_time = int(round(time.time() * 1000))
-    reply = sendMessage("Starting Ping", bot, update)
+    reply = sendMessage("Starting Ping", context.bot, update)
     end_time = int(round(time.time()*1000))
     editMessage(f'{end_time - start_time} ms',reply)
 
 
 @run_async
-def log(bot,update):
-    sendLogFile(bot, update)
+def log(update,context):
+    sendLogFile(context.bot, update)
 
 
 @run_async
-def bot_help(bot,update):
+def log(update,context):
+    sendLogFile(context.bot, update)
+
+
+@run_async
+def bot_help(update, context):
     help_string = f'''
 /{BotCommands.HelpCommand}: To get this message
 
 /{BotCommands.MirrorCommand} [download_url][magnet_link]: Start mirroring the link to google drive
 
 /{BotCommands.TarMirrorCommand} [download_url][magnet_link]: start mirroring and upload the archived (.tar) version of the download
 
+/{BotCommands.WatchCommand} [youtube-dl supported link]: Mirror through youtube-dl 
+
+/{BotCommands.TarWatchCommand} [youtube-dl supported link]: Mirror through youtube-dl and tar before uploading
+
 /{BotCommands.CancelMirror} : Reply to the message by which the download was initiated and that download will be cancelled
 
 /{BotCommands.StatusCommand}: Shows a status of all the downloads
@@ -66,7 +75,7 @@ def bot_help(bot,update):
 /{BotCommands.LogCommand}: Get a log file of the bot. Handy for getting crash reports
 
 '''
-    sendMessage(help_string, bot, update)
+    sendMessage(help_string, context.bot, update)
 
 
 def main():
@@ -78,7 +87,7 @@ def main():
     help_handler = CommandHandler(BotCommands.HelpCommand,
                                   bot_help, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user)
     stats_handler = CommandHandler(BotCommands.StatsCommand,
-                                  stats, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user)
+                                   stats, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user)
     log_handler = CommandHandler(BotCommands.LogCommand, log, filters=CustomFilters.owner_filter)
     dispatcher.add_handler(start_handler)
     dispatcher.add_handler(ping_handler)

bot/helper/ext_utils/bot_utils.py

@@ -54,6 +54,14 @@ def get_readable_file_size(size_in_bytes) -> str:
     except IndexError:
         return 'File too large'
 
+def getDownloadByGid(gid):
+    with download_dict_lock:
+        for dl in download_dict.values():
+            if dl.status() == MirrorStatus.STATUS_DOWNLOADING:
+                if dl.download().gid == gid:
+                    return dl
+    return None
+
 
 def get_progress_bar_string(status):
     completed = status.processed_bytes() / 8
@@ -90,7 +98,13 @@ def get_readable_message():
             if download.status() != MirrorStatus.STATUS_ARCHIVING:
                 msg += f"\n<code>{get_progress_bar_string(download)} {download.progress()}</code> of " \
                     f"{download.size()}" \
-                    f" at {download.speed()}, ETA: {download.eta()}\n\n"
+                    f" at {download.speed()}, ETA: {download.eta()} "
+            if download.status() == MirrorStatus.STATUS_DOWNLOADING:
+                if hasattr(download,'is_torrent'):
+                    msg += f"| P: {download.download().connections} " \
+                           f"| S: {download.download().num_seeders}"
+                msg += f"\nGID: <code>{download.download().gid}</code>"
+            msg += "\n\n"
         return msg
 
 

bot/helper/ext_utils/fs_utils.py

@@ -4,6 +4,7 @@
 import os
 import pathlib
 import mimetypes
+import tarfile
 
 
 def clean_download(path: str):
@@ -29,13 +30,25 @@ def exit_clean_up(signal, frame):
         LOGGER.warning("Force Exiting before the cleanup finishes!")
         sys.exit(1)
 
-
-def tar(orig_path: str):
-    path = pathlib.PurePath(orig_path)
-    base = path.name
-    root = pathlib.Path(path.parent.as_posix()).absolute().as_posix()
-    LOGGER.info(f'Tar: orig_path: {orig_path}, base: {base}, root: {root}')
-    return shutil.make_archive(orig_path, 'tar', root, base)
+def get_path_size(path):
+    if os.path.isfile(path):
+        return os.path.getsize(path)
+    total_size = 0
+    for root, dirs, files in os.walk(path):
+        for f in files:
+            abs_path = os.path.join(root, f)
+            total_size += os.path.getsize(abs_path)
+    return total_size
+
+
+def tar(org_path):
+    tar_path = org_path + ".tar"
+    path = pathlib.PurePath(org_path)
+    LOGGER.info(f'Tar: orig_path: {org_path}, tar_path: {tar_path}')
+    tar = tarfile.open(tar_path, "w")
+    tar.add(org_path,arcname=path.name)
+    tar.close()
+    return tar_path
 
 
 def get_mime_type(file_path):