removeDuplicateLines.py

#!/usr/bin/env python3

import os
import sys
import argparse

PY3 = sys.version_info[0] == 3
if not PY3:
    print("This program has only been tested on Python 3.6+, you are using an unknown version of python.")
    sys.exit(0)

parser = argparse.ArgumentParser(description='remove duplicates in supplied wordlist')
parser.add_argument('wordlist', type=argparse.FileType('r'), help='The wordlist you would like to remove duplicates from')
parser.add_argument('wordlist_clean', type=argparse.FileType('w'), help='The file to store the cleaned wordlist')
args = parser.parse_args()
	
#from https://blog.georgechalhoub.com/2015/09/remove-duplicate-lines-from-file-using.html

lines_seen = set() # holds lines already seen
for line in args.wordlist:
    if line not in lines_seen: # not a duplicate
        args.wordlist_clean.write(line)
        lines_seen.add(line)
args.wordlist_clean.close()