rpggeekscraper/get_rpg_info.py at master · mdweisner/rpggeekscraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import csv
import time
import requests
from bs4 import BeautifulSoup


def get_val(tag, term):
	try:
		val = tag.find(term)['value'].encode('ascii', 'ignore')
	except:
        	val = 'NaN'
    	return val


base = 'http://www.rpggeek.com/xmlapi2/family?typ=rpg&id={}&stats=1'
with open('ids.txt') as f:
    ids = [line.strip() for line in f.readlines()]
split = 30
f = open('rpgs.csv', 'w')
writer = csv.writer(f)
writer.writerow(('id', 'type', 'name', 'yearpublished', 'rpggeekrank', 'users_rated', 'average_rating',
                 'bayes_average_rating', 'total_owners', 'total_traders', 'total_wanters',
                 'total_wishers', 'total_comments', 'total_weights', 'average_weight'))
for i in range(0, len(ids), split):
    url = base.format(','.join(ids[i:i+split]))
    print('Requesting {}'.format(url))
    req = requests.get(url)
    soup = BeautifulSoup(req.content, 'xml')
    items = soup.find_all('item')
    for item in items:
        gid = item['id']
        gtype = item['type']
        gname = get_val(item, 'name')
        gyear = get_val(item, 'yearpublished')
        grpggeekrank = get_val(item, 'rpggeekrank')
        usersrated = get_val(item.statistics.ratings, 'usersrated')
        avg = get_val(item.statistics.ratings, 'average')
        bayesavg = get_val(item.statistics.ratings, 'bayesaverage')
        owners = get_val(item.statistics.ratings, 'owned')
        traders = get_val(item.statistics.ratings, 'trading')
        wanters = get_val(item.statistics.ratings, 'wanting')
        wishers = get_val(item.statistics.ratings, 'wishing')
        numcomments = get_val(item.statistics.ratings, 'numcomments')
        numweights = get_val(item.statistics.ratings, 'numweights')
        avgweight = get_val(item.statistics.ratings, 'averageweight')
        # desc = item.description.text.encode('ascii', 'ignore')
        writer.writerow((gid, gtype, gname, gyear, grpggeekrank,
                         usersrated, avg, bayesavg, owners, traders, wanters, wishers, numcomments,
                         numweights, avgweight))
    time.sleep(2)
f.close()