main.py

from requests import head,get
from pandas import read_csv
from time import time,sleep
from datetime import datetime
from tkinter import *
from PIL import ImageTk,Image
import os
from sys import path,exit
import matplotlib.pyplot as plt
from numpy import array
from zipfile import ZipFile
from glob import glob
from filecmp import cmp,cmpfiles,clear_cache
from shutil import move,rmtree
from threading import Thread
from re import sub
from gc import collect
from random import choice
from datetime import datetime
from tzlocal import get_localzone
from pandastable import Table

################# Global Variables #################
sys_path = path[0]

websites_csv_path = os.path.join(sys_path,'websites.csv')

data_folder_path = os.path.join(sys_path,'Data')

temp_folder = os.path.join(data_folder_path,'temp')

################# Core Functions #################
class CoreUtils(object):
   def __init__(self):
      self.run = False
      self.stopped = True
      self.CheckAgain = int(time() + 1000)
      self.df_changed = False

   def set_run(self) -> None:
      self.run = True

   def stop_run(self) -> None:
      self.run = False

   def set_stopped(self) -> None:
      self.stopped = False

   def connected_to_internet(self,timeout: int = 5) -> bool:
      '''
      Does what it says, returns true if connected to internet, else returns false.
      '''
      options = ['http://www.google.com/','https://www.wikipedia.org/','https://github.com/']
      url = choice(options)
      try:
         _ = head(url,timeout=timeout)
         return True
      except:
         return False
      
   def ping(self,url) -> bool:
      '''
      Does what it says, returns true if connected to internet, else returns false.
      '''
      try:
         _ = head(url,timeout=5)
         return True
      except:
         return False

   def download_url_thread(self,url: str,save_path: str,type: str,index: int) -> None:
      '''
      Save path is just the folder you want to download it in.

      Type must be a string, with the type of file you're downloading.

      returns name of new file and its filepath. If downloaded is a zip, it will extract it and then
      return the path to the folder along with the name of the folder.
      '''
      if (self.check_internet_and_wait(check_now=True)):
         return
      
      dir_name = os.path.basename(save_path)
      num_ = len([i for i,j in enumerate(dir_name) if j == '_'])
      filename = '_'.join(dir_name.split(sep='_')[0:round(num_/2)])+'-'+str(index)+'.'+type
      filepath = os.path.join(save_path,filename)
      del dir_name,num_,filename

      try:
         r = get(url, stream=True)
      except:
         return
      with open(filepath, 'wb') as fd:
         for chunk in r.iter_content(chunk_size=1024):
            fd.write(chunk)

      try:
         if type == 'zip':
            with ZipFile(filepath,'r') as zObject:
               zObject.extractall(path=save_path)
               zObject.close()
               os.unlink(filepath)
            filepath = max(glob(os.path.join(save_path,'*/')),key=os.path.getmtime)
            try:
               del zObject
            except:
               pass
      except:
         return
      
      self.df.iloc[index,3] = int(time())
      self.df.iloc[index,4] = filepath
   
   def download_url(self,url: str,save_path: str,chunk_size = 1024,type: str ='csv') -> str:
      '''
      Save path is just the folder you want to download it in.

      Type must be a string, with the type of file you're downloading.

      returns name of new file and its filepath. If downloaded is a zip, it will extract it and then
      return the path to the folder along with the name of the folder.
      '''
      files_in_directory = len(next(os.walk(save_path),(None, None, []))[2])
      dir_name = os.path.basename(save_path)
      num_ = len([i for i,j in enumerate(dir_name) if j == '_'])
      filename = '_'.join(dir_name.split(sep='_')[0:round(num_/2)])+'-'+str(files_in_directory)+'.'+type
      filepath = os.path.join(save_path,filename)
      del files_in_directory,dir_name,num_,filename


      if not self.ping(url):
         raise ConnectionError
      try:
         r = get(url, stream=True)
      except:
         raise ConnectionError

      with open(filepath, 'wb') as fd:
         for chunk in r.iter_content(chunk_size=int((chunk_size))):
            fd.write(chunk)

      try:
         if type == 'zip':
            with ZipFile(filepath,'r') as zObject:
               zObject.extractall(path=save_path)
               zObject.close()
               os.unlink(filepath)
            filepath = max(glob(os.path.join(save_path,'*/')),key=os.path.getmtime)
            try:
               del zObject
            except:
               pass
      except:
         raise ValueError
      return filepath

   def clear_temp(self,dir: str = temp_folder) -> None:
      '''
      Clears all files and directories from temp folder, can be used on other folders.
      '''
      for filename in os.listdir(dir):
         file_path = os.path.join(dir, filename)
         try:
            if os.path.isfile(file_path) or os.path.islink(file_path):
               os.unlink(file_path)
            elif os.path.isdir(file_path):
               rmtree(file_path)
         except:
            pass
         del file_path
      del filename
      return

   def check_internet_and_wait(self,check_now = False) -> bool:
      '''
      This function basically just pings websites until it gets a response, if no response
      it will enter a loop where it checks again once a second, once it connectes it will
      return False. If the user clicks the stop button, it will return true if it is
      looping
      '''
      if (abs(self.CheckAgain-time()) >= 5) or (check_now):
         self.CheckAgain = time()
         connected = self.connected_to_internet(timeout=5)
         was_diconnected = False
         if not connected:
            start = time()
            was_diconnected = True
            print('Not connected to internet')
            while not connected:
               sleep(1)
               connected = self.connected_to_internet()
               if not self.run:
                  return True
            print('Connected to internet')
            end = time()
         if was_diconnected:
            print(f'Time disconnected: {end-start}s')
            del connected,was_diconnected,end,start
      return False

   def check_size(self,filepath: str,exp: int = 3,ceiling: int = 1) -> bool:
      '''
      Checks size of file before generating figures
      '''
      size = os.path.getsize(filepath)
      rel_size = size/(1024**exp)
      if (rel_size > ceiling):
         del size,rel_size
         return False
      elif (rel_size <= ceiling):
         del size,rel_size
         return True
      else:
         raise ValueError
      
   def empty_path_dl_manager(self) -> bool:
      '''
      Downloads any entries in websites.csv that doesn't have a file location
      '''
      paths = tuple(self.df.iloc[:,4])
      if 'empty' in paths:
         self.df_changed = True
         names = tuple(self.df.iloc[:,0])
         self.df.iloc[:,0] = [sub('[^0-9a-zA-Z._:/\\\]+','',name.replace(' ','_')).replace('__','_') for name in names]
         dl_folders = [os.path.join(data_folder_path,title) for title in self.df.iloc[:,0]]
         for folder in dl_folders:
            if not os.path.isdir(folder):
               os.mkdir(folder)

         current_idx = 0
         downloads = []
         while current_idx < len(paths):
            idx = 0
            while (len(downloads) < 5) and (idx+current_idx < len(paths)):
               if self.df.iloc[idx+current_idx,4] == 'empty':
                  downloads.append(Thread(target=self.download_url_thread,
                     args=(str(self.df.iloc[idx+current_idx,1]),
                        dl_folders[current_idx+idx],
                        str(self.df.iloc[idx+current_idx,2]),
                        idx+current_idx)))
                  idx += 1
               else:
                  idx += 1

            current_idx += idx

            for thread in downloads:
               try:
                  thread.start()
               except:
                  pass

            while len(downloads) > 0:
               try:
                  t = downloads.pop()
                  t.join()
               except:
                  pass
               
         del current_idx,downloads,t,thread,names,dl_folders
         self.df.to_csv(websites_csv_path,index=False)
         window.queue_autoprocess(ms=0,num=20)
         return True
      
   def main(self) -> None:
      '''
      So basically this runs on a thread and will only actually run after you press start on the GUI.

      This is the core loop that handles the data and determines where data goes. Reads websites.csv,
      so try not to mess anything up. Use edit_websites_csv.ipynb to add entries to the csv. If you want to
      remove an entry, you can just delete the line.
      '''
      sleep(0.1)
      print('Main thread started')

      while True:

         if (not self.run) or (self.check_internet_and_wait()):
            sleep(0.1)
            print('Exititng main thread')
            break

         # Check if websites.csv exists
         if not os.path.isfile(websites_csv_path):
            self.run = False
            window.on_stop()
            print('Necessary file "websites.csv" does not exist in current directory. Exiting main thread.')
            continue

         # Check if data folder exists
         if not os.path.isdir(data_folder_path):
            print('Directory "Data" does not exist in current directory. Creating Directory.')
            os.mkdir(path=data_folder_path)
            os.mkdir(path=temp_folder)

         # Read in the csv with websites and info
         try:
            self.df = read_csv(websites_csv_path,header=0)
            self.df = self.df.reset_index(drop=True)
            self.df['path'] = self.df['path'].fillna('empty')
            self.df['last_checked'] = self.df['last_checked'].fillna(time())
            self.df['title'] = self.df['title'].fillna('PLACEHOLDER_TITLE')
            self.df_changed = False
            self.stopped = False
         except:
            self.run = False
            window.on_stop()
            print('Failed to open websites.csv, exiting main thread')
            continue

         if self.empty_path_dl_manager():
            continue

         # Iterate over rows of websites.csv
         for idx,info in self.df.iterrows():

            name = info[0]
            url = info[1]
            dtype = info[2]
            last_checked = info[3]
            dpath = info[4]

            if not self.run:
               self.df_changed = False
               self.df.to_csv(websites_csv_path,index=False)
               del self.df
               break

            title = sub('[^0-9a-zA-Z._:/\\\]+','',name.replace(' ','_')).replace('__','_')
            if name != title:
               self.df_changed = True
               self.df.iloc[idx,0] = title

            dl_folder = os.path.join(data_folder_path,title)
            del title

            # Check if enough time has passed
            if (abs(round(time()) - last_checked) >= 604800):
               self.df.iloc[idx,3] = int(time())
               self.df_changed = True

               if (dpath!='empty') and ((os.path.isfile(dpath)) or (os.path.isdir(dpath))):
                  try:
                     # Download data to temp folder using url, return temp filepath
                     new_filepath = self.download_url(url=url,save_path=temp_folder,type=dtype)
                     old_filepath = dpath

                     # Check to see if files are the same
                     if (dtype == 'zip'):
                        try:
                           clear_cache()
                        except:
                           pass
                        dir_a = os.listdir(new_filepath).sort()
                        dir_b = os.listdir(old_filepath).sort()
                        if dir_a == dir_b:
                           same_files = cmpfiles(a=new_filepath,b=old_filepath,shallow=False,common=dir_a)[0]
                           if len(same_files) == len(dir_a):
                              same_file = True
                           else:
                              same_file = False
                           del dir_a,dir_b
                        else:
                           same_file = False
                     else:
                        same_file = cmp(f1=new_filepath,f2=old_filepath,shallow=False)

                     # If files are the same, clear temp folder
                     if same_file:
                        self.clear_temp()

                     # If files are different, move new file in temp to overwrite old file
                     # clear temp folder, delete old data folder, and process new data
                     elif not same_file:
                        # Check to see if old path is a file or directory
                        if os.path.isdir(old_filepath):
                           rmtree(old_filepath)
                        elif os.path.isfile(old_filepath):
                           os.unlink(old_filepath)
                        # Move file from temp to corresponding data folder
                        move(src=new_filepath,dst=old_filepath)
                        self.clear_temp()
                        # Delete old figures directory so that autoprocess can process it
                        filename = os.path.basename(old_filepath)
                        data_folder = os.path.join(dl_folder,(filename.split(sep='.')[0]+'_Data'))
                        rmtree(path=data_folder)
                        del same_file,data_folder,new_filepath,old_filepath
                        self.df_changed = True
                  except:
                     pass
                  try:
                     self.clear_temp()
                  except:
                     pass

         try:
            self.clear_temp()
         except:
            pass

         # 3. Check to see if user asked for entry to be deleted <- Not sure if this will get implemented

         # 4. Overwrite file
         try:
            if self.df_changed:
               self.df.to_csv(websites_csv_path,index=False)
         except:
            pass

         sleep(0.2)

      print("Exited main thread")
      self.stopped = True

   def autoprocess(self,num=5) -> None:
      '''
      Horrible Mess of a Function Held together by spit and duct tape

      Call this function to process any and all csv's in the data folder
      No inputs are required to make this work
      '''
      global window

      if (not self.run):
         window.queue_autoprocess()
         return
      
      all_dls = []
      all_dls = list(read_csv(filepath_or_buffer=websites_csv_path,usecols=['path']).iloc[:,0])

      if (len(all_dls) <= 0):
         del all_dls
         window.queue_autoprocess()
         return

      all_csv = []
      all_csv = [csv for csv in all_dls if '.csv' in csv]

      to_process = []
      for csv in all_csv:
         if not self.check_size(filepath=csv,exp=3,ceiling=0.5):
            continue
         dl_folder = csv[0:(len(csv)-(len(os.path.basename(csv))))]
         filename = os.path.basename(csv).split(sep='.')[0]
         data_folder = os.path.join(dl_folder,(filename.split(sep='.')[0] +'_Data'))
         a = len(dl_folder)
         b = len(temp_folder)
         if a >= b:
            upper = b
         else:
            upper = a
         if os.path.isdir(data_folder) or (dl_folder[0:upper] == temp_folder[0:upper]):
            del dl_folder,filename,data_folder,a,b,upper
            continue
         to_process.append((csv,data_folder,filename))
         del dl_folder
         if len(to_process) >= num:
            del filename,data_folder,a,b,upper
            break

      while len(to_process) > 0:
         vals = to_process.pop()

         if not self.run:
            window.queue_autoprocess()
            del to_process
            return

         csv_path = vals[0]
         data_folder = vals[1]
         filename = vals[2]
         histogram_path = os.path.join(data_folder,'Hist')
         plots_path = os.path.join(data_folder,'Plots')

         try:
            data = read_csv(csv_path,low_memory=False)
         except:
            os.mkdir(data_folder)
            failed_to_process = open(os.path.join(data_folder,'failed_to_process.txt'),'a')
            e = datetime.now()
            failed_to_process.write(f'{filename} failed to open on {str(e.year)}, {str(e.month)}, {str(e.day)}\n')
            failed_to_process.close()
            del data_folder,e,filename,csv_path,histogram_path,plots_path
            continue

         os.mkdir(data_folder)
         os.mkdir(histogram_path)
         os.mkdir(plots_path)

         for j,col in enumerate(data):
            if (data.dtypes[j] != 'object') and (data.dtypes[j] != 'bool'):
               try:
                  fig,ax = plt.subplots(nrows=1,ncols=1,figsize=(8,8))
                  plt.switch_backend('agg')
                  ax.hist(array(data[col]))
                  ax.set_xlabel(col)
                  ax.set_title('Autogenerated Histogram '+str(j+1))
                  ax.set_facecolor('#ADD8E6')
                  ax.set_axisbelow(True)
                  ax.yaxis.grid(color='white', linestyle='-')
                  savepath = os.path.join(histogram_path,(filename+'_Hist-'+str(j+1)+'.png'))
                  fig.savefig(savepath,format='png')
                  del savepath
                  plt.cla()
                  plt.clf()
                  plt.close('all')
               except:
                  plt.cla()
                  plt.clf()
                  plt.close('all')

               try:
                  fig,ax = plt.subplots(nrows=1,ncols=1,figsize=(8,8))
                  plt.switch_backend('agg')
                  ax.plot([idx+1 for idx,j in enumerate(data[col])],array(data[col]))
                  ax.set_xlabel(col)
                  ax.set_title('Autogenerated Plot '+str(j+1))
                  ax.set_facecolor('#ADD8E6')
                  ax.set_axisbelow(True)
                  ax.yaxis.grid(color='white', linestyle='-')
                  ax.xaxis.grid(color='white', linestyle='-')
                  savepath = os.path.join(plots_path,(filename+'_Plot-'+str(j+1)+'.png'))
                  fig.savefig(savepath,format='png')
                  del savepath
                  plt.cla()
                  plt.clf()
                  plt.close('all')
               except:
                  plt.cla()
                  plt.clf()
                  plt.close('all')

         try:
            del data,csv_path,filename,data,histogram_path,plots_path,fig,ax
         except:
            continue
      
      del all_csv
      collect()
      window.queue_autoprocess()
      return

################# GUI Window #################
class GUI(Tk):
   def __init__(self) -> Tk:
      super().__init__()
      self.process = CoreUtils()

      self.protocol("WM_DELETE_WINDOW",self.on_x)
      self.resizable(False, False)

      self.title('Transportation Data Manager')
      self.iconphoto(False,ImageTk.PhotoImage(file=os.path.join(sys_path,'Resources','road-210913_1280.jpg'),format='jpg'))

      self.frame = Frame(self)
      self.frame.pack()

      self.canvas = Canvas(self.frame, width=425, height=400, bg='#D3D3D3')
      self.canvas.pack()

      self.info_label = Text(self.canvas,wrap=WORD,width=40,height=3,padx=6,pady=5,highlightthickness=0)
      self.info_label.tag_configure('center',justify='center')
      self.info_label.insert('1.0','''This rudimentary GUI controls the script. New buttons and features may be added later if I can make it work''')
      self.info_label.tag_add('center',1.0,'end')
      self.info_label.place(relx=0.5, rely = 0.12,anchor=CENTER)
      self.info_label.config(state=DISABLED)

      self.start_label = Text(self.canvas,wrap=WORD,width=30,height=2,padx=6,pady=5,highlightthickness=0)
      self.start_label.tag_configure('center',justify='center')
      self.start_label.insert('1.0','When pressed, this button will start the loop')
      self.start_label.tag_add('center',1.0,'end')
      self.start_label.place(relx = 0.35, rely = 0.28,anchor=CENTER)
      self.start_label.config(state= DISABLED)

      self.start_button = Button(self.canvas,text="    Start    ",command=self.on_start,padx=6,pady=5,highlightthickness=0)
      self.start_button.place(relx=0.8,rely=0.28,anchor=CENTER)

      self.end_label = Text(self.canvas,wrap=WORD,width=30,height=2,padx=6,pady=5,highlightthickness=0)
      self.end_label.tag_configure('center',justify='center')
      self.end_label.insert('1.0','When pressed, this button will end the loop')
      self.end_label.tag_add('center',1.0,'end')
      self.end_label.place(relx = 0.35, rely = 0.41,anchor=CENTER)
      self.end_label.config(state=DISABLED)

      self.end_button = Button(self.canvas,text="    Stop    ",command=self.on_stop,padx=6,pady=5,highlightthickness=0)
      self.end_button.place(relx=0.8,rely=0.41,anchor=CENTER)

      self.monitor_label = Text(self.canvas,wrap=WORD,width=30,height=3,padx=6,pady=5,highlightthickness=0)
      self.monitor_label.tag_configure('center',justify='center')
      self.monitor_label.insert('1.0','This button will create a window with the current spreadsheet')
      self.monitor_label.tag_add('center',1.0,'end')
      self.monitor_label.place(relx = 0.35, rely = 0.56,anchor=CENTER)
      self.monitor_label.config(state=DISABLED)

      self.monitor_button = Button(self.canvas,text="Spreadsheet",command=self.create_monitor,padx=6,pady=5,highlightthickness=0)
      self.monitor_button.place(relx=0.8,rely=0.56,anchor=CENTER)

      self.add_label = Text(self.canvas,wrap=WORD,width=30,height=3,padx=6,pady=5,highlightthickness=0)
      self.add_label.tag_configure('center',justify='center')
      self.add_label.insert('1.0','This button will create a window that allows you to enter new websites to track')
      self.add_label.tag_add('center',1.0,'end')
      self.add_label.place(relx = 0.35, rely = 0.73,anchor=CENTER)
      self.add_label.config(state=DISABLED)

      self.add_button = Button(self.canvas,text="Add Entry",command=self.create_add_window,padx=6,pady=5,highlightthickness=0)
      self.add_button.place(relx=0.8,rely=0.73,anchor=CENTER)

      self.resized_img = Image.open(os.path.join(sys_path,'Resources','UT_logo.png')).resize((130,100),Image.LANCZOS);
      self.img = ImageTk.PhotoImage(self.resized_img)
      self.canvas.create_image(375,360,image=self.img)

      self.who_made_this = Text(self.canvas,wrap=WORD,width=35,height=3,padx=6,pady=5,highlightthickness=0)
      self.who_made_this.tag_configure('center',justify='center')
      self.who_made_this.insert('1.0','''This program was made by Collin Dobson for the UTORII SMaRT internship''')
      self.who_made_this.tag_add('center',1.0,'end')
      self.who_made_this.place(relx=0.399,rely = 0.9,anchor=CENTER)
      self.who_made_this.config(state=DISABLED)

      self.after(ms=10000,func=self.process.autoprocess)

   def create_add_window(self) -> None:
      '''
      Creates window that allows you to add entries to websites.csv
      '''
      def createWarning(self: GUI,relx: float,rely: float,text: str) -> None:
         bad_entry = Text(self.add_frame,wrap=WORD,width=15,height=1,padx=6,pady=5,highlightthickness=0,fg="#FF0000")
         bad_entry.tag_configure('center',justify='center')
         bad_entry.insert('1.0',text)
         bad_entry.tag_add('center',1.0,'end')
         bad_entry.place(relx=relx, rely=rely, anchor=CENTER)
         bad_entry.config(state=DISABLED)
         self.bad_labels.append(bad_entry)
         if (len(self.bad_labels) >= 4):
            del self.bad_labels[0]
         del bad_entry

      def destroyWarnings(self: GUI) -> None:
         '''
         Destroys warning popups
         '''
         for bad_label in self.bad_labels:
            try:
               bad_label.destroy()
            except:
               pass

      def getEntry(self: GUI) -> None:
         '''
         Extracts the text from the entries, creates popups if an input is invalid
         '''
         title = self.entry1.get()
         if len(title) <= 0:
            createWarning(self,0.3,0.85,'Title Too Short')
            self.entry1.delete(0,END)
            del title
            return
         
         destroyWarnings(self)

         link = self.entry2.get()
         if not self.process.ping(link):
            createWarning(self,0.5,0.85,'Invalid Link')
            self.entry2.delete(0,END)
            del title,link
            return
         
         destroyWarnings(self)
         
         type = self.entry3.get()
         if len(type) <= 0:
            createWarning(self,0.7,0.85,'Enter Filetype')
            self.entry3.delete(0,END)
            return
         
         destroyWarnings(self)

         final_entry = (title,link,type)

         with open(websites_csv_path,'a') as web:
            web.write(f'{final_entry[0]},{final_entry[1]},{final_entry[2]},{int(time())},empty\n')

         self.entry1.delete(0,END)
         self.entry2.delete(0,END)
         self.entry3.delete(0,END)
         
         del final_entry

         try:
            self.create_monitor.update_monitor(self)
         except:
            pass
      
      def delete_add_window(self: GUI) -> None:
         '''
         Deletes add_wwindow and reopens the main window
         '''
         destroyWarnings(self)
         self.add_win.destroy()
         self.add_button['state'] = 'normal'
         self.deiconify()  

      self.on_stop()
      self.withdraw()

      self.bad_labels = []

      self.add_win = Toplevel(master=self,bg='#D3D3D3')
      self.add_win.protocol("WM_DELETE_WINDOW",lambda: delete_add_window(self))
      self.add_win.iconphoto(False,ImageTk.PhotoImage(file=os.path.join(sys_path,'Resources','road-210913_1280.jpg'),format='jpg'))
      self.add_win.title('websites.csv')
      self.add_button['state'] = 'disabled'

      self.add_frame = Frame(self.add_win,height=300,width=900,bg='#D3D3D3')
      self.add_frame.pack()

      self.add_label1 = Text(self.add_frame,wrap=WORD,width=15,height=1,padx=6,pady=5,highlightthickness=0)
      self.add_label1.tag_configure('center',justify='center')
      self.add_label1.insert('1.0','''Enter Title''')
      self.add_label1.tag_add('center',1.0,'end')
      self.add_label1.place(relx=0.5, rely = 0.07,anchor=CENTER)
      self.add_label1.config(state=DISABLED)

      self.entry1 = Entry(self.add_frame,width=90)
      self.entry1.place(relx=0.5,rely=0.17,anchor=CENTER)

      self.add_label2 = Text(self.add_frame,wrap=WORD,width=15,height=1,padx=6,pady=5,highlightthickness=0)
      self.add_label2.tag_configure('center',justify='center')
      self.add_label2.insert('1.0','''Enter Link''')
      self.add_label2.tag_add('center',1.0,'end')
      self.add_label2.place(relx=0.5, rely = 0.27,anchor=CENTER)
      self.add_label2.config(state=DISABLED)

      self.entry2 = Entry(self.add_frame,width=90)
      self.entry2.place(relx=0.5, rely = 0.37,anchor=CENTER)

      self.add_label3 = Text(self.add_frame,wrap=WORD,width=15,height=1,padx=6,pady=5,highlightthickness=0)
      self.add_label3.tag_configure('center',justify='center')
      self.add_label3.insert('1.0','''Enter Filetype''')
      self.add_label3.tag_add('center',1.0,'end')
      self.add_label3.place(relx=0.5, rely = 0.47,anchor=CENTER)
      self.add_label3.config(state=DISABLED)

      self.entry3 = Entry(self.add_frame,width=90)
      self.entry3.place(relx=0.5, rely = 0.57,anchor=CENTER)

      self.add_entry_button = Button(self.add_frame,text="Add To websites.csv",command=lambda: getEntry(self),padx=6,pady=5,highlightthickness=0)
      self.add_entry_button.place(relx=0.5,rely=0.7,anchor=CENTER)

   def create_monitor(self) -> None:
      '''
      Creates spreadsheet window
      '''
      def update_monitor(self: GUI) -> None:
         '''
         Redraws the spreadsheet
         '''
         try:
            data = self.process.df.copy()
         except:
            data = read_csv(websites_csv_path)
         data = data.reset_index(drop=True)
         data['path'] = data['path'].fillna('empty')
         data['last_checked'] = data['last_checked'].fillna(time())
         data['title'] = data['title'].fillna('PLACEHOLDER_TITLE')
         tz = get_localzone()
         data.iloc[:,3] = [datetime.fromtimestamp(unix_timestamp, tz).strftime("%D %H:%M") for unix_timestamp in data.iloc[:,3]]
         self.pt.model.df = data
         self.pt.redraw()
         self.monitor.after(2500,lambda: update_monitor(self))
         del data

      def delete_monitor(self: GUI) -> None:
         '''
         Deletes the spreadsheet and toggles the spreadsheet button
         '''
         self.monitor.destroy()
         self.monitor_button['state'] = 'normal'

      self.monitor = Toplevel(master=self,bg='#D3D3D3')
      self.monitor.geometry('1000x400')
      self.monitor.protocol("WM_DELETE_WINDOW",lambda: delete_monitor(self))
      self.monitor.iconphoto(False,ImageTk.PhotoImage(file=os.path.join(sys_path,'Resources','road-210913_1280.jpg'),format='jpg'))
      self.monitor.title('websites.csv')
      self.monitor_button['state'] = 'disabled'

      self.f = Frame(self.monitor,height=1000,width=1600,bg='#D3D3D3')
      self.f.pack(fill=BOTH,expand=1)
      try:
         data = self.process.df.copy()
      except:
         data = read_csv(websites_csv_path)
      tz = get_localzone()
      data.iloc[:,3] = [datetime.fromtimestamp(unix_timestamp, tz).strftime("%D %H:%M") for unix_timestamp in data.iloc[:,3]]
      self.pt = Table(self.f,dataframe=data,showtoolbar=False,showstatusbar=False)
      self.pt.show()
      del data

      self.monitor.after(ms=5000,func=lambda: update_monitor(self))

   def switch(self) -> None:
      '''
      Toggles the buttons on the the GUI, because of the multithreading, make sure to not change this.
      '''
      if (self.start_button["state"] == "normal") and (self.end_button["state"] == "normal") and (self.process.run == False):
         self.start_button["state"] = "normal"
         self.end_button["state"] = "disabled"
      elif self.start_button["state"] == "normal":
         self.start_button["state"] = "disabled"
         self.end_button["state"] = "normal"
      elif (self.start_button["state"] != "normal"):
         self.start_button["state"] = "normal"
         self.end_button["state"] = "disabled"

   def on_start(self) -> None:
      '''
      This mess of a function starts the manin thread.
      '''
      if (not self.process.stopped):
         return

      self.process.set_run()
      self.process.set_stopped()
      self.main_thread = Thread(target=self.process.main).start()
      print('Starting main thread')
      self.switch()

   def on_stop(self) -> None:
      '''
      This function stops the main thread.
      '''
      if not self.process.run:
         self.switch()
         return
      print('Waiting for main thread to reach stopping point')
      self.process.stop_run()
      self.switch()

   def on_x(self) -> None:
      '''
      This is the behavior for when you close the window
      '''
      if self.process.run:
         print('Waiting for main thread to reach stopping point')
      self.process.stop_run()
      self.destroy()

   def queue_autoprocess(self,ms=604800000,num=5) -> None:
      '''
      Queues autoprocess to run after a specified time
      '''
      self.after(ms=ms,func=lambda: self.process.autoprocess(num=num))

window = GUI()
window.mainloop()

while True:
   sleep(0.1)
   if window.process.stopped:
      del window
      exit('Successfully exited program')