Update 0.4.5

- reduced console spam in keyword mode
- fixed a crash if an invalid keyword file was specified
- made some console prints more accurate
- renamed some variables to improve code readability
- automated cleanup on program termination

Author: InitialPosition

scrape.py

@@ -2,7 +2,7 @@
 import threading
 from datetime import datetime
 from json import decoder
-from os import path, mkdir
+from os import path, mkdir, remove
 from os.path import isfile
 
 try:
@@ -21,25 +21,33 @@ def status(message):
     print("{0} {1}".format(datetime.now(), message))
 
 
+# clean any temp files created during runtime
+def cleanup():
+    if isfile("runfile"):
+        remove("runfile")
+
+
 # main loop
 def main():
-    status("Fetching latest pastes...")
+    status("Fetching latest paste IDs...")
 
     # fetch latest 100 paste IDs
     fetch_limit = 100
 
     current_request = requests.get("https://scrape.pastebin.com/api_scraping.php?limit={0}".format(fetch_limit))
-
     current_json = []
+
     try:
         current_json = current_request.json()
 
     except decoder.JSONDecodeError:
-        status(termcolor.colored("Unable to fetch latest pastes. Make sure your IP is whitelisted at "
+        status(termcolor.colored("Unable to fetch latest paste IDs. Make sure your IP is whitelisted at "
                                  "https://pastebin.com/doc_scraping_api", "red"))
+        cleanup()
+
         exit(0)
 
-    status("Pastes fetched. Processing...")
+    status("Paste IDs fetched. Processing...")
 
     # clean up fetched ids
     cleaned_json = []
@@ -55,26 +63,27 @@ def main():
                                          .format(entry["key"]))
 
             entry_content = entry_request.text
-            path_t_important = path.join("files", "{0}.txt".format(entry["key"]))
+            path_file = path.join("files", "{0}.txt".format(entry["key"]))
 
             paste_ids.append(entry["key"])
+
             # if we have a provided keyword list, check for keywords
             if keywords is not None:
                 for keyword in keywords:
                     if keyword.upper() in entry_content.upper():
-                        print(termcolor.colored(" [KEYWORD] Paste \'{0}\' contains keyword \'{1}\'".format(entry["key"]
-                                                                                                           , keyword)
-                                                , "green"))
+                        bar.suffix = "%(index)d/%(max)d " + termcolor.colored("[KEYWORD] Paste \'{0}\' contains "
+                                                                              "keyword \'{1}\'".format(entry["key"],
+                                                                                                       keyword),
+                                                                              "green")
 
-                        entry_file = open(path_t_important, "w+", encoding='utf-8')
-                        entry_file.write(entry_content)
-                        entry_file.close()
+                        with open(path_file, "w+", encoding='utf-8') as entry_file:
+                            entry_file.write(entry_content)
 
                         break
             else:
-                entry_file = open(path_t_important, "w+", encoding='utf-8')
-                entry_file.write(entry_content)
-                entry_file.close()
+                with open(path_file, "w+", encoding='utf-8') as entry_file:
+                    entry_file.write(entry_content)
+                bar.suffix = "%(index)d/%(max)d Saving paste \'{0}\'".format(entry["key"])
 
             bar.next()
 
@@ -104,38 +113,36 @@ def main():
 
     AUTHOR = "SYRAPT0R"
     COPYRIGHT = "2019-2020"
-    VERSION = "0.4.4"
-
-    status("STARTING PASTA SCRAPER {0}, (c) {1} {2}".format(VERSION, COPYRIGHT, AUTHOR))
-    print()
-
-    # make sure file directories exists
-    if not path.isdir("files"):
-        status(termcolor.colored("No file directory found, creating...", "yellow"))
-        mkdir("files")
+    VERSION = "0.4.5"
 
     # parse arguments
     keywords = None
 
     parser = argparse.ArgumentParser(description="A script to scrape pastebin.com with optional keyword search")
 
     parser.add_argument("-k", "--keywords", help="A file containing keywords for the search")
-    parser.add_argument("-i", "--infinite", help="Whether to run in infinite mode.", action="store_true")
+    parser.add_argument("-i", "--infinite", help="Whether to run in infinite mode (Default: false)",
+                        action="store_true", default=False)
 
     args = parser.parse_args()
 
-    # create non infinite file if needed
-    if args.infinite is False:
-        status("Creating run file...")
-        f = open("runfile", "w+", encoding='utf-8')
-        f.close()
-    else:
-        status("Running in infinite mode...")
+    status("STARTING PASTA SCRAPER {0}, (c) {1} {2}".format(VERSION, COPYRIGHT, AUTHOR))
+    print()
+
+    # make sure file directories exists
+    if not path.isdir("files"):
+        status(termcolor.colored("No file directory found, creating...", "yellow"))
+        mkdir("files")
 
     if args.keywords is not None:
-        f = open(args.keywords)
-        keywords = f.readlines()
-        f.close()
+        try:
+            with open(args.keywords, "r") as f:
+                keywords = f.readlines()
+
+        except IOError:
+            status(termcolor.colored("Unable to load specified keyword file. Aborting...", "red"))
+
+            exit(0)
 
         keywords = [keyword.strip() for keyword in keywords]
 
@@ -145,6 +152,15 @@ def main():
     paste_ids = []
     max_id_list_size = 200
 
+    # create non infinite file if needed
+    if args.infinite is False:
+        status("Creating run file...")
+
+        f = open("runfile", "w+")
+        f.close()
+    else:
+        status("Running in infinite mode...")
+
     # preparation done, enter main loop
     status("Entering main loop...")
     print()