fix accomplishments page performance

.gitignore

@@ -1,6 +1,8 @@
 # ignore pycache
 /__pycache__/*
 /app/__pycache__/*
+/migrations/__pycache__/*
+/migrations/versions/__pycache__/*
 
 # ignore vscode
 /.vscode

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-    "python.pythonPath": "/anaconda3/bin/python"
+    "python.pythonPath": "/Users/maxalminasatriakahfi/anaconda3/bin/python"
 }

app/analytics.py

@@ -9,7 +9,7 @@
 import altair as alt
 import pandas as pd
 import numpy as np
-import string, urllib, re, pickle
+import string, urllib, re
 
 from sklearn.feature_extraction.text import TfidfVectorizer
 from pathlib import Path
@@ -46,55 +46,30 @@ def getuserdb():
 # ===================================================
 
 def get_response():
-    responses = pd.read_sql_query("SELECT response.workshop_id, response.satisfaction_score, response.comments, e.id as employee_id, w.workshop_name, w.workshop_start as timestamp\
-                              FROM response\
-                              LEFT JOIN workshop w ON w.id = response.workshop_id\
-                              LEFT JOIN employee e ON e.id = w.workshop_instructor", conn, parse_dates='timestamp')
-
-    responses.loc[:,'comments'] = responses['comments'].astype(str)
-    responses.loc[:,'satisfaction_score'] = responses['satisfaction_score'].astype(float)
-
+    responses = pd.read_sql_query("SELECT response.workshop_id, response.satisfaction_score, response.comments, e.id as employee_id, w.workshop_name, w.workshop_start as timestamp, response.sentiment\
+                            FROM response\
+                            LEFT JOIN workshop w ON w.id = response.workshop_id\
+                            LEFT JOIN employee e ON e.id = w.workshop_instructor", conn, parse_dates='timestamp')
     sixmonths = datetime.datetime.now() - datetime.timedelta(weeks=26)
     responses = responses[responses.timestamp >= sixmonths]
+
+    responses = responses[responses['comments']!='-']
+    responses = responses.sort_values(by='timestamp', ascending=False).reset_index(drop=True)
+
     return responses
 
 def get_sentiment_data(responses):
 
-    return responses[['workshop_id','workshop_name','employee_id','timestamp','comments']].copy()
+    return responses[['workshop_id','workshop_name','employee_id','timestamp','comments','sentiment']].copy()
 
 def get_reviews_data(responses):
 
-    return responses[['workshop_id', 'satisfaction_score', 'employee_id', 'timestamp']].copy()
-
-def process_sentiment(sentiment):
-    my_model = load(str(Path().absolute())+'/model/sentiment/model.joblib')
-    word_vector = load(str(Path().absolute())+'/model/sentiment/vector.joblib')
-
-    sentiment['comments'].replace(['','None'], np.nan, inplace=True)
-    sentiment.dropna(inplace=True)
-    sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: x.lower())
-    sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: x.translate(str.maketrans("","", string.punctuation)))
-    sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: x.translate(str.maketrans("","", string.digits)))
-    sentiment.loc[:,'comments'] = sentiment['comments'].apply(lambda x: re.sub(' +', ' ',x).strip())
-    # word_vector.fit(pd.read_csv(str(Path().absolute())+'/model/sentiment/train.csv'))
-
-    sentiment.loc[:,'score'] = ''
-    sentiment.loc[:,'score'] = my_model.predict(word_vector.transform(sentiment['comments']))
-
-    return sentiment
-
-def prereviews(reviews):
-    reviews.fillna(3, inplace=True)
-
-    return reviews
+    return responses[['workshop_id','satisfaction_score','employee_id','timestamp']].copy()
 
 response = get_response()
 
 sentiment = get_sentiment_data(response)
-sentiment = process_sentiment(sentiment)
-
 reviews = get_reviews_data(response)
-reviews = prereviews(reviews)
 
 domain = ['negative', 'positive']
 colors = ['#7dbbd2cc', '#bbc6cbe6']
@@ -103,45 +78,44 @@ def get_params():
     emp = getuserdb()
     dat = emp.loc[emp.this_user == True,:].copy()
     emp_id = dat.iloc[0]['workshop_instructor']
-    sixmonths = datetime.datetime.now() - datetime.timedelta(weeks=26)
 
-    return emp_id, sixmonths
+    return emp_id
 
 def get_overall_sentiment():
-    emp_id, sixmonths = get_params()
+    emp_id = get_params()
     person = sentiment[sentiment.employee_id==emp_id].copy()
-
-    return pd.crosstab([person['timestamp'],person['workshop_name'],person['workshop_id']],person['score']).apply(lambda x: round(x/x.sum()*100,2), axis=1).reset_index().melt(id_vars=['timestamp','workshop_name','workshop_id'])
+    monyear_percent = pd.crosstab([person['timestamp'],person['workshop_name'],person['workshop_id']],person['sentiment']).apply(lambda x: round(x/x.sum()*100,2), axis=1).reset_index().melt(id_vars=['timestamp','workshop_name','workshop_id'])
+
+    return monyear_percent.sort_values(by='timestamp', ascending=False).reset_index(drop=True)
 
 def get_overall_reviews(monyear_percent):
-    emp_id, sixmonths = get_params()
+    emp_id = get_params()
 
     filter_idx = monyear_percent.groupby(['workshop_id'])['value'].transform(max) == monyear_percent['value']
     al_reviews = monyear_percent[filter_idx].copy()
-    al_reviews.sort_values('score', ascending=False, inplace=True)
+    al_reviews.sort_values('sentiment', ascending=False, inplace=True)
     al_reviews.drop_duplicates(subset='workshop_id', keep="first", inplace=True)
 
-    person_reviews = reviews[(reviews.employee_id==emp_id) & (reviews.timestamp >= sixmonths)]
+    person_reviews = reviews[(reviews.employee_id==emp_id)]
 
-    sm_reviews = person_reviews[['satisfaction_score', 'workshop_id']].groupby(['workshop_id']).mean().round(1)
+    sm_reviews = person_reviews[['satisfaction_score', 'workshop_id', 'timestamp']].groupby(['workshop_id','timestamp']).mean().round(1)
     sm_reviews.reset_index(inplace=True)
 
     sm_reviews.loc[:,'workshop_name'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['workshop_name'])
-    sm_reviews.loc[:,'sentiment'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['score'])
+    sm_reviews.loc[:,'sentiment'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['sentiment'])
     sm_reviews.loc[:,'value'] = sm_reviews['workshop_id'].map(al_reviews.set_index('workshop_id')['value'])
 
-    return sm_reviews
+    return sm_reviews.sort_values(by='timestamp', ascending=False).reset_index(drop=True)
 
 @app.route('/data/person_sentiment')
 def vis_overall_sentiment():
     monyear_percent = get_overall_sentiment()
 
     chart = alt.Chart(monyear_percent).mark_bar().encode(
+        x=alt.X('workshop_name:N', axis=alt.Axis(title='Date'), sort=['timestamp:T']),
         y=alt.Y('value:Q', axis=alt.Axis(title='Percentage (%)')),
-        x=alt.X('workshop_name:N', axis=alt.Axis(title='Workshop Name')),
-        color=alt.Color('score', scale=alt.Scale(domain=domain, range=colors), legend=alt.Legend(title="Sentiment")),
-        order="timestamp:T",
-        tooltip=[alt.Tooltip('value:Q', title="Percentage"), alt.Tooltip('score:N', title="Sentiment")]
+        color=alt.Color('sentiment', scale=alt.Scale(domain=domain, range=colors), legend=alt.Legend(title="Sentiment")),
+        tooltip=[alt.Tooltip('timestamp:T', title="Workshop Name"), alt.Tooltip('value:Q', title="Percentage"), alt.Tooltip('sentiment:N', title="Sentiment")]
         ).properties(
             width=800, height=300
         ).configure_axis(grid=False)
@@ -641,9 +615,7 @@ def gettimenow():
         Response.workshop_id.in_(w.id for w in workshops), Response.comments != '').join(
             Workshop, isouter=True).order_by(
                 Workshop.workshop_start.desc()).paginate(
-                    per_page=30, page=1, error_out=True)
-
-    # comments = pd.read_sql_query("SELECT workshop_id, comments FROM response", conn)
+                    per_page=100, page=1, error_out=True)
 
     monyear_percent = get_overall_sentiment()
 

app/forms.py

@@ -3,6 +3,7 @@
 from wtforms.fields.html5 import IntegerRangeField
 from wtforms.validators import ValidationError, DataRequired, Email, EqualTo
 from app.users import User
+from joblib import load
 
 class LoginForm(FlaskForm):
     email = StringField('Work Email', validators=[DataRequired(), Email()])

app/models.py

@@ -69,3 +69,4 @@ class Response(db.Model):
     venue_score = db.Column(db.Integer)
     satisfaction_score = db.Column(db.Integer)
     comments = db.Column(db.Text)
+    sentiment = db.Column(db.String(10), nullable=True)

app/routes.py

@@ -6,6 +6,7 @@
 from app.models import Employee, Workshop, Response
 from app.forms import LoginForm, RegistrationForm, SurveyForm, ResetPasswordRequestForm, ResetPasswordForm
 from app.email import send_pw_reset_email
+from app.sentiment import preprocess_comments, predict_sentiment
 from datetime import datetime
 from sqlalchemy import func
 
@@ -142,7 +143,8 @@ def rate(workshop_id):
             timeliness=form.time.data,
             venue_score=form.venue.data,
             satisfaction_score=form.satisfaction.data,
-            comments=form.comments.data
+            comments=preprocess_comments(form.comments.data),
+            sentiment=predict_sentiment(preprocess_comments(form.comments.data))
         )
         db.session.add(response)
         db.session.commit()

app/sentiment.py

@@ -0,0 +1,18 @@
+from joblib import load
+from pathlib import Path
+import string, re
+
+def preprocess_comments(sentence):
+    sentence = sentence.lower()
+    sentence = sentence.translate(str.maketrans("","", string.punctuation))
+    sentence = sentence.translate(str.maketrans("","", string.digits))
+    sentence = re.sub(' +', ' ',sentence).strip()
+    if sentence == "": sentence = '-'
+
+    return sentence
+
+def predict_sentiment(sentence):
+    my_model = load(str(Path().absolute())+'/model/sentiment/model.joblib')
+    word_vector = load(str(Path().absolute())+'/model/sentiment/vector.joblib')
+
+    return str(my_model.predict(word_vector.transform([sentence]))[0])

app/templates/accomplishment.html

@@ -157,7 +157,6 @@ <h4><i class="material-icons md-light">rate_review</i> Most Recent Reviews</h4>
 
                     {% endif %}
                 {% endfor %}
-                </div>
             </div>
         </div>
     </div>

app/templates/sub/single_response.html

@@ -48,7 +48,7 @@ <h5 class="card-text">{{ review['sentiment'] }}</h5>
     {% endif %}
 
         {% for comments in personstats['qualitative'].items %}
-            {% if (comments['workshop_id'] == review['workshop_id'] and comments['comments'] != '')%}
+            {% if (comments['workshop_id'] == review['workshop_id'] and comments['comments'] != '-')%}
                 <hr/>
                 <p class="card-text">{{comments['comments']}}</p>
                 <ul>

config.py

@@ -11,7 +11,7 @@
 dburl = f'mysql+pymysql://{user}:{password}@{host}/{database}'
 # create conditional connection
 if(os.getenv('FLASK_ENV') == 'development'):
-    conn = sqlite3.connect('test.db', check_same_thread=False)
+    conn = sqlite3.connect('test.db')
 else:
     conn = pymysql.connect(
         host=host,

environment.yml

@@ -15,14 +15,14 @@ dependencies:
 - ipython_genutils=0.2.0=py36h241746c_0
 - jedi=0.13.1=py36_0
 - jinja2=2.10.1=py36_0
+- joblib=0.13.2=py36_0
 - jupyter_core=4.4.0=py36_0
 - libcxx=4.0.1=h579ed51_0
 - libcxxabi=4.0.1=hebd6815_0
 - libedit=3.1.20170329=hb402a30_2
 - libffi=3.2.1=h475c297_4
 - libgfortran=3.0.1=h93005f0_2
 - libsodium=1.0.16=h3efe00b_0
-- mkl=2019.0=118
 - ncurses=6.1=h0a44026_0
 - numpy=1.15.1=py36h6a91979_0
 - numpy-base=1.15.1=py36h8a80b8c_0
@@ -89,6 +89,7 @@ dependencies:
 - wheel=0.31.1=py36_1
 - wrapt=1.10.11=py36_0
 - wtforms=2.1=py36_0
+- autopep8=1.4.4=py_0
 - blas=1.0=mkl
 - bleach=3.1.0=py36_0
 - cryptography=2.4.2=py36ha12b0ac_0
@@ -97,7 +98,10 @@ dependencies:
 - libcurl=7.63.0=h051b688_1000
 - libiconv=1.15=hdd342a3_7
 - libssh2=1.8.0=ha12b0ac_4
+- llvm-openmp=4.0.1=hcfea43d_1
 - mistune=0.8.4=py36h1de35cc_0
+- mkl=2019.4=233
+- mkl-service=2.0.2=py36h1de35cc_0
 - nb_conda=2.2.1=py36_0
 - nb_conda_kernels=2.2.0=py36_0
 - nbconvert=5.3.1=py36_0
@@ -109,11 +113,13 @@ dependencies:
 - perl=5.26.2=h4e221da_0
 - pip=18.1=py36_0
 - prometheus_client=0.5.0=py36_0
+- pycodestyle=2.5.0=py36_0
 - python=3.6.8=haf84260_0
+- scikit-learn=0.21.2=py36h27c97d8_0
+- scipy=1.2.1=py36h1410ff5_0
 - send2trash=1.5.0=py36_0
 - sqlite=3.26.0=ha441bb4_0
 - terminado=0.8.1=py36_1
 - testpath=0.4.2=py36_0
 - webencodings=0.5.1=py36_1
-prefix: /anaconda3/envs/pedagogy
-
+prefix: /anaconda3/envs/pedagogy

migrations/env.py

@@ -1,8 +1,12 @@
 from __future__ import with_statement
-from alembic import context
-from sqlalchemy import engine_from_config, pool
-from logging.config import fileConfig
+
 import logging
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -18,8 +22,9 @@
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 from flask import current_app
-config.set_main_option('sqlalchemy.url',
-                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
+config.set_main_option(
+    'sqlalchemy.url', current_app.config.get(
+        'SQLALCHEMY_DATABASE_URI').replace('%', '%%'))
 target_metadata = current_app.extensions['migrate'].db.metadata
 
 # other values from the config, defined by the needs of env.py,
@@ -41,7 +46,9 @@ def run_migrations_offline():
 
     """
     url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url)
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True
+    )
 
     with context.begin_transaction():
         context.run_migrations()
@@ -65,21 +72,23 @@ def process_revision_directives(context, revision, directives):
                 directives[:] = []
                 logger.info('No changes in schema detected.')
 
-    engine = engine_from_config(config.get_section(config.config_ini_section),
-                                prefix='sqlalchemy.',
-                                poolclass=pool.NullPool)
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool,
+    )
 
-    connection = engine.connect()
-    context.configure(connection=connection,
-                      target_metadata=target_metadata,
-                      process_revision_directives=process_revision_directives,
-                      **current_app.extensions['migrate'].configure_args)
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            process_revision_directives=process_revision_directives,
+            **current_app.extensions['migrate'].configure_args
+        )
 
-    try:
         with context.begin_transaction():
             context.run_migrations()
-    finally:
-        connection.close()
+
 
 if context.is_offline_mode():
     run_migrations_offline()

migrations/versions/429782854307_updated_models.py

@@ -1,9 +1,7 @@
 """updated models
-
 Revision ID: 429782854307
 Revises: 80dbe21e8059
 Create Date: 2018-09-22 12:57:04.748750
-
 """
 from alembic import op
 import sqlalchemy as sa
@@ -58,4 +56,4 @@ def downgrade():
     op.drop_table('response')
     op.drop_table('assistants')
     op.drop_table('workshop')
-    # ### end Alembic commands ###
+    # ### end Alembic commands ###