TaskJAM/database.py at master · nyuad-hackathon-2020/TaskJAM · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
import pymongo
import random
from datetime import datetime
from datetime import timedelta
# import pandas as pd
import json
from bson import json_util

# user is a python dict
def add_user (collection, user):

	# Parse user json info
	ret_dict = {}
	try:
		ret_dict['msg'] =  "Succesfully registered user " + user['first_name'] + " " + user['last_name']
		ret_dict['status_code'] = 200
		# Insert to db
		collection.insert_one(user)
	except:
		ret_dict['msg'] = "Error registering user"
		ret_dict['status_code'] = 400
	return ret_dict

# task is a python dict
def add_task (collection, task):

	# Parse user json info
	ret_dict = {}
	try:
		ret_dict['msg'] =  "Succesfully registered task " + task['description'] + " for user " + task['username']
		ret_dict['status_code'] = 200
		# Insert to db
		collection.insert_one(task)
	except:
		ret_dict['msg'] = "Error adding task"
		ret_dict['status_code'] = 400
	return ret_dict

def make_match(collection, match):

	ret_dict = {}
	try:
		ret_dict['msg'] =  "Succesfully matched task " + match['description'] + " for user " + match['username']
		ret_dict['status_code'] = 200
	except:
		ret_dict['msg'] = "Error matching user"
		ret_dict['status_code'] = 400

	username = match['username']
	# Find match
	match_username = 'john_doe_123'

	ret_dict['matched_with'] = match_username

	return ret_dict

def init_datasets():

	first_df = pd.read_csv('names/first_names.csv', header=None)
	print(first_df.head())
	first_names = first_df.iloc[:,0].tolist()
	first_names = [x.strip().split(" ")[0] for x in first_names]

	last_df = pd.read_csv('names/last_names.csv', header=None, usecols=[0])
	print(last_df.head())
	last_names = last_df.iloc[:,0].tolist()
	last_names = [x.strip().split(" ")[0] for x in last_names]

	hobbies_df = pd.read_csv('names/hobbies.csv')
	print(hobbies_df.head())
	interests = hobbies_df.iloc[:,0].tolist()

	return first_names, last_names, interests

def generate_username(first_name, last_name, usernames):
	n = 0
	username = first_name.lower() + "_" + last_name.lower() + "_" + str(n)
	while (username in usernames):
		n += 1
		username = first_name.lower() + "_" + last_name.lower() + "_" + str(n)
	return username

def random_date_of_birth():
    """Generate a random datetime between `start` and `end`"""
    start = datetime.strptime('1/1/1960 1:30 PM', '%m/%d/%Y %I:%M %p')
    end = datetime.strptime('1/1/2006 1:30 PM', '%m/%d/%Y %I:%M %p')
    return start + timedelta(
        # Get a random amount of seconds between `start` and `end`
        seconds=random.randint(0, int((end - start).total_seconds())),
    )

# Create csv with mock user data
def populate_user_collection (collection, n_users = 100000):

	# https://github.com/smashew/NameDatabases/tree/master/NamesDatabases/
	# https://www.kaggle.com/muhadel/hobbies/version/3
	# Generate names dicts from the datasets above

	first_names, last_names, interests = init_datasets()
	usernames = set({})
	mail_hosts = ["hotmail", "gmail", "outlook", "yahoo"]
	languages = ["English", "Arabic", "Spanish"]

	# Final list of users with details, each user a dict
	users = []

	for i in range(n_users):

		new_user = {}

		new_user['first_name']  = random.choice(first_names)
		new_user['last_name'] = random.choice(last_names)

		new_user['username'] = generate_username(new_user['first_name'], new_user['last_name'], usernames)
		new_user['email'] = new_user['username'] + "@" + random.choice(mail_hosts) + ".com"

		new_user['languages'] = random.sample(languages, k = random.choice(range(len(languages))) + 1 )
		new_user['interests'] = random.sample(interests, k = random.choice(range(10)) + 1 )

		new_user['date_of_birth'] = random_date_of_birth()

		users.append(new_user)

		if (i%10000)==1 or i==n_users-1:
			collection.insert_many(users)
			users = []

	# users_df = pd.DataFrame(users)
	# print(users_df.head())
	# users_df.to_csv('names/taskjam_users.csv', index=False)

	# with open('names/taskjam_users.json', 'w') as f:
	# 	json.dump(users, f, default=str)

def populate_db_users (collection):

	# users_df = pd.read_csv('names/taskjam_users.csv')
	# users_df['json'] = users_df.apply(lambda x: x.to_json(), axis=1)
	# json_users = [json.loads(x) for x in users_df['json'].tolist()]
	# print((json_users[0]))

	with open('names/taskjam_users.json', 'r') as f:
		s = f.read()
		json_users = json.loads(s)
	for user in json_users:
		user['date_of_birth'] = datetime.strptime(user['date_of_birth'], '%Y-%m-%d %H:%M:%S')
		user['_id'] = user['username']
	# print(json_users[:2])
	collection.insert_many(json_users)


if __name__ == '__main__':


	# import pymongo

	# client = pymongo.MongoClient("mongodb+srv://oscargomezq:oscargomezq@cluster-taskjam-ahyms.mongodb.net/test?retryWrites=true&w=majority")
	# db = client["taskjam-db"]

	# populate_db_users(db.users_big)

	# populate_user_collection(db.users_big)

	pass