Scrape-n-Email/drudgeScraper.py at main · CGFixIT/Scrape-n-Email · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
#Ignore the name of this module; I am just being lazy - I originally started this by scraping druge but now just use the scrape function for RCP

from bs4 import BeautifulSoup
import requests, io, csv, sys
import rcpScraper, clistScraper

#This is a global variable and should really be it's own module for csv writing, but, again, I'm being lazy
def csvinit():
    with open('RCPlinks.csv', 'a', newline='') as f:
            writer = csv.writer(f)
            writer.writerow( ('HEADLINE', 'URL') )
            writer.writerow( ('', '') )
            f.close()

def writer(title, link):
    with open('RCPlinks.csv', 'a', newline='') as f:
        writer = csv.writer(f)
        writer.writerow( (title, link) )
        f.close()

def scrape():

    i = 0
    n = 15

    drudge = requests.get("http://www.drudgereport.com/")
    soup = BeautifulSoup(drudge.content, 'html.parser')

    outputFile = open('DRUDGEheadlines.txt', 'w+')

    outputFile.write('--------------\n--------------\nDrudge Headlines\n--------------\n--------------\n\n')
    try:
        for counter in range(1,n+1):
            for div in soup.findAll('div', {'id': 'app_topstories'}):
                a = div.findAll('a')[i]
                mainTitle = (a.text.strip())
                mainLink = (a.attrs['href'])
                outputFile.write(mainTitle + '\n' + mainLink + '\n\n')
                writer(mainTitle, mainLink)
                i += 1
    except IndexError:
        print("Error when attempting: Scraping 'Da News...")

    # outputFile.write('--------------\nLeft Headlines\n--------------\n')
    i=0
    try:
        for counter in range(1,n+1):
            for div in soup.findAll('div', {'id': 'app_col1'}):
                a = div.findAll('a')[i]
                leftTitle = (a.text.strip())
                leftLink = (a.attrs['href'])
                outputFile.write(leftTitle + '\n' + leftLink + '\n\n')
                writer(leftTitle, leftLink)
                i += 1
    except IndexError:
        print("Error when attempting: Scraping 'Da News...")

    # outputFile.write('--------------\nCenter Headlines\n--------------\n')
    i=0
    try:
        for counter in range(1,n+1):
            for div in soup.findAll('div', {'id': 'app_col2'}):
                a = div.findAll('a')[i]
                rightTitle = (a.text.strip())
                rightLink = (a.attrs['href'])
                outputFile.write(rightTitle + '\n' + rightLink + '\n\n')
                writer(rightTitle, rightLink)
                i += 1
    except IndexError:
        print("Error when attempting: Scraping 'Da News...")

    # outputFile.write('--------------\nRight Headlines\n--------------\n')
    i=0
    try:
        for counter in range(1,n+1):
            for div in soup.findAll('div', {'id': 'app_col3'}):
                a = div.findAll('a')[i]
                rightTitle = (a.text.strip())
                rightLink = (a.attrs['href'])
                outputFile.write(rightTitle + '\n' + rightLink + '\n\n')
                writer(rightTitle, rightLink)
                i += 1
    except IndexError:
        print("Error when attempting: Scraping 'Da News...")


    outputFile.close()

#Below only runs if program is ran independently; in main.py I only call the scrape() function
if __name__ == "__main__":
    csvinit()
    writer("~DRUDGE","  ~~~")
    scrape()
    writer("~SCIENCEDAILY","  ~~~")
    sciScraper.controller()
    writer("~REALCLEARPOLITICS","  ~~~")
    rcpScraper.scrape()
    exit()