GeeksForGeeks_article_extractor/main.py at master · yodebu/GeeksForGeeks_article_extractor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
#!/usr/bin/env python


'''
--------------------------------------------------
--------------------------------------------------
 Name:   GeeksForGeeks Article Extractor
 Purpose: To download and save articles filed under each and every tag mentioned in www.geeksforgeeks.org
 Author: Debapriya Das
 Dept of CSE, NIT Durgapur
 V1.0 - 06.02.2015 - basic implementation

--------------------------------------------------
--------------------------------------------------
'''


import urllib2
import os
import pdfkit
from bs4 import BeautifulSoup
from optparse import OptionParser

import crawler
from crawler import *


def parse_options():
  usage = "usage: prog [options] (arg1, arg2, ... argn)"
  parser = OptionParser(usage=usage)

  parser.add_option("-t", "--tag1", \
		      type="string", \
		      action="store", \
		      dest="inp_tag1", \
		      default = "", \
		      help="input tags for downloading from the website")


  parser.add_option("-u", "--tag2", \
		      type="string", \
		      action="store", \
		      dest="inp_tag2", \
		      default = "", \
		      help="input tags for downloading from the website")

  parser.add_option("-v", "--tag3", \
		      type="string", \
		      action="store", \
		      dest="inp_tag3", \
		      default = "", \
		      help="input tags for downloading from the website")

  parser.add_option("-l", "--location", \
		      type="string", \
		      action="store", \
		      dest="inp_location", \
		      default = "/home/yodebu/Desktop/GeeksForGeeks_article_extractor/", \
		      help="location where downloaded files willl be stored")

  opts, args = parser.parse_args()
  return opts, args


##-----------------------------------------------------

# main function

def main():
  # parse the input parameters
  opts, args = parse_options()
  AllTags = [opts.inp_tag1, opts.inp_tag2, opts.inp_tag3]
  path = opts.inp_location
  ExtractMainLinks(AllTags,path)


if __name__ == "__main__": main()