Add embeddings

.gitignore

@@ -1 +1,3 @@
 .env
+_pycache_/
+.DS_Store

lsu_pilot/embedding/embed.py

@@ -0,0 +1,96 @@
+import pandas as pd
+import os
+import tiktoken
+from openai import OpenAI
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from dotenv import load_dotenv
+
+# Load environment variables from a .env file.
+load_dotenv()
+
+openai = OpenAI(api_key=os.environ['OPENAI_API_KEY'])
+
+DOMAIN = "developer.mozilla.org"
+
+
+def remove_newlines(series):
+  series = series.str.replace('\n', ' ')
+  series = series.str.replace('\\n', ' ')
+  series = series.str.replace('  ', ' ')
+  series = series.str.replace('  ', ' ')
+  return series
+
+
+# Create a list to store the text files
+texts = []
+
+# Get all the text files in the text directory
+for file in os.listdir("text/" + DOMAIN + "/"):
+
+  # Open the file and read the text
+  with open("text/" + DOMAIN + "/" + file, "r", encoding="UTF-8") as f:
+    text = f.read()
+    # we replace the last 4 characters to get rid of .txt, and replace _ with / to generate the URLs we scraped
+    filename = file[:-4].replace('_', '/')
+    """
+    There are a lot of contributor.txt files that got included in the scrape, this weeds them out. There are also a lot of auth required urls that have been scraped to weed out as well
+    """
+    if filename.endswith(".txt") or 'users/fxa/login' in filename:
+      continue
+
+    # then we replace underscores with / to get the actual links so we can cite contributions
+    texts.append((filename, text))
+
+# Create a dataframe from the list of texts
+df = pd.DataFrame(texts, columns=['fname', 'text'])
+
+# Set the text column to be the raw text with the newlines removed
+df['text'] = df.fname + ". " + remove_newlines(df.text)
+df.to_csv('processed/scraped.csv')
+
+# Load the cl100k_base tokenizer which is designed to work with the ada-002 model
+tokenizer = tiktoken.get_encoding("cl100k_base")
+
+df = pd.read_csv('processed/scraped.csv', index_col=0)
+df.columns = ['title', 'text']
+
+# Tokenize the text and save the number of tokens to a new column
+df['n_tokens'] = df.text.apply(lambda x: len(tokenizer.encode(x)))
+
+chunk_size = 1000  # Max number of tokens
+
+text_splitter = RecursiveCharacterTextSplitter(
+    # This could be replaced with a token counting function if needed
+    length_function=len,
+    chunk_size=chunk_size,
+    chunk_overlap=0,  # No overlap between chunks
+    add_start_index=False,  # We don't need start index in this case
+)
+
+shortened = []
+
+for row in df.iterrows():
+
+  # If the text is None, go to the next row
+  if row[1]['text'] is None:
+    continue
+
+  # If the number of tokens is greater than the max number of tokens, split the text into chunks
+  if row[1]['n_tokens'] > chunk_size:
+    # Split the text using LangChain's text splitter
+    chunks = text_splitter.create_documents([row[1]['text']])
+    # Append the content of each chunk to the 'shortened' list
+    for chunk in chunks:
+      shortened.append(chunk.page_content)
+
+  # Otherwise, add the text to the list of shortened texts
+  else:
+    shortened.append(row[1]['text'])
+
+df = pd.DataFrame(shortened, columns=['text'])
+df['n_tokens'] = df.text.apply(lambda x: len(tokenizer.encode(x)))
+
+df['embeddings'] = df.text.apply(lambda x: openai.embeddings.create(
+    input=x, model='text-embedding-ada-002').data[0].embedding)
+
+df.to_csv('processed/embeddings.csv')

lsu_pilot/main.py

@@ -0,0 +1,62 @@
+import os
+import logging
+from telegram import Update
+from telegram.ext import ApplicationBuilder, ContextTypes, CommandHandler
+from openai import OpenAI
+import pandas as pd
+import numpy as np
+from questions import answer_question
+from dotenv import load_dotenv
+
+# Load environment variables from a .env file.
+load_dotenv()
+
+openai = OpenAI(api_key=os.environ['OPENAI_API_KEY'])
+tg_bot_token = os.environ['TG_BOT_TOKEN']
+
+df = pd.read_csv('processed/embeddings.csv', index_col=0)
+df['embeddings'] = df['embeddings'].apply(eval).apply(np.array)
+
+messages = [{
+    "role": "system",
+    "content": "You are a helpful assistant that answers questions."
+}]
+
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    level=logging.INFO)
+
+
+async def chat(update: Update, context: ContextTypes.DEFAULT_TYPE):
+  messages.append({"role": "user", "content": update.message.text})
+  completion = openai.chat.completions.create(model="gpt-3.5-turbo",
+                                              messages=messages)
+  completion_answer = completion.choices[0].message
+  messages.append(completion_answer)
+
+  await context.bot.send_message(chat_id=update.effective_chat.id,
+                                 text=completion_answer.content)
+
+async def mozilla(update: Update, context: ContextTypes.DEFAULT_TYPE):
+      answer = answer_question(df, question=update.message.text, debug=True)
+      await context.bot.send_message(chat_id=update.effective_chat.id, text=answer)
+
+
+async def start(update: Update, context: ContextTypes.DEFAULT_TYPE):
+  await context.bot.send_message(chat_id=update.effective_chat.id,
+                                 text="I'm a bot, please talk to me!")
+
+
+if __name__ == '__main__':
+  application = ApplicationBuilder().token(tg_bot_token).build()
+
+  start_handler = CommandHandler('start', start)
+  chat_handler = CommandHandler('chat', chat)
+
+  mozilla_handler = CommandHandler('mozilla', mozilla)
+  application.add_handler(mozilla_handler)
+
+  application.add_handler(start_handler)
+  application.add_handler(chat_handler)
+
+  application.run_polling()

lsu_pilot/questions.py

@@ -0,0 +1,111 @@
+import numpy as np
+import pandas as pd
+from openai import OpenAI
+import os
+from typing import List
+from scipy import spatial
+from dotenv import load_dotenv
+
+# Load environment variables from a .env file.
+load_dotenv()
+
+
+def distances_from_embeddings(
+    query_embedding: List[float],
+    embeddings: List[List[float]],
+    distance_metric="cosine",
+) -> List[float]:
+  """Return the distances between a query embedding and a list of embeddings."""
+  distance_metrics = {
+      "cosine": spatial.distance.cosine,
+      "L1": spatial.distance.cityblock,
+      "L2": spatial.distance.euclidean,
+      "Linf": spatial.distance.chebyshev,
+  }
+  distances = [
+      distance_metrics[distance_metric](query_embedding, embedding)
+      for embedding in embeddings
+  ]
+  return distances
+
+
+openai = OpenAI(api_key=os.environ['OPENAI_API_KEY'])
+
+# NOTE unused:
+# df = pd.read_csv('processed/embeddings.csv', index_col=0)
+# df['embeddings'] = df['embeddings'].apply(eval).apply(np.array)
+
+
+def create_context(question, df, max_len=1800):
+  """
+    Create a context for a question by finding the most similar context from the dataframe
+    """
+  # Get the embeddings for the question
+  q_embeddings = openai.embeddings.create(
+      input=question, model='text-embedding-ada-002').data[0].embedding
+
+  # Get the distances from the embeddings
+  df['distances'] = distances_from_embeddings(q_embeddings,
+                                              df['embeddings'].values,
+                                              distance_metric='cosine')
+
+  returns = []
+  cur_len = 0
+
+  # Sort by distance and add the text to the context until the context is too long
+  for i, row in df.sort_values('distances', ascending=True).iterrows():
+    # Add the length of the text to the current length
+    cur_len += row['n_tokens'] + 4
+
+    # If the context is too long, break
+    if cur_len > max_len:
+      break
+
+    # Else add it to the text that is being returned
+    returns.append(row["text"])
+
+  # Return the context
+  return "\n\n###\n\n".join(returns)
+
+
+def answer_question(df,
+                    model="gpt-3.5-turbo-1106",
+                    question="What is the meaning of life?",
+                    max_len=1800,
+                    debug=False,
+                    max_tokens=150,
+                    stop_sequence=None):
+  """
+    Answer a question based on the most similar context from the dataframe texts
+    """
+  context = create_context(
+      question,
+      df,
+      max_len=max_len,
+  )
+  # If debug, print the raw model response
+  if debug:
+    print("Context:\n" + context)
+    print("\n\n")
+
+  try:
+    # Create a completions using the question and context
+    response = openai.chat.completions.create(
+        model=model,
+        messages=[{
+            "role":
+            "user",
+            "content":
+            f"Answer the question based on the context below, and if the question can't be answered based on the context, say \"I don't know.\" Try to site sources to the links in the context when possible.\n\nContext: {context}\n\n---\n\nQuestion: {question}\nAnswer:",
+        }],
+        temperature=0,
+        max_tokens=max_tokens,
+        top_p=1,
+        frequency_penalty=0,
+        presence_penalty=0,
+        stop=stop_sequence,
+    )
+    return response.choices[0].message.content
+  except Exception as e:
+    print(e)
+    return ""

lsu_pilot/text/developer.mozilla.org/developer.mozilla.org_en-US_docs_Glossary_Attribute.txt

@@ -0,0 +1,30 @@
+Attribute - MDN Web Docs Glossary: Definitions of Web-related terms | MDNSkip to main contentSkip to searchSkip to select languageMDN Web DocsOpen main menuReferencesReferencesOverview / Web TechnologyWeb technology reference for developersHTMLStructure of content on the webCSSCode used to describe document styleJavaScriptGeneral-purpose scripting languageHTTPProtocol for transmitting web resourcesWeb APIsInterfaces for building web applicationsWeb ExtensionsDeveloping extensions for web browsersWeb TechnologyWeb technology reference for developersGuidesGuidesOverview / MDN Learning AreaLearn web developmentMDN Learning AreaLearn web developmentHTMLLearn to structure web content with HTMLCSSLearn to style content using CSSJavaScriptLearn to run scripts in the browserAccessibilityLearn to make the web accessible to allPlusPlusOverviewA customized MDN experienceUpdatesAll browser compatibility updates at a glanceDocumentationLearn how to use MDN PlusFAQFrequently asked questions about MDN PlusBlog NewSearch MDNClear search inputSearchThemeLog inGet MDN PlusMDN Web Docs Glossary: Definitions of Web-related termsAttributeArticle ActionsEnglish (US)In this articleReflection of an attributeSee alsoIn this articleReflection of an attributeSee alsoAttributeAn attribute extends an HTML or XML element, changing its behavior or providing metadata.
+An attribute always has the form name="value" (the attribute's identifier followed by its associated value).
+You may see attributes without the equals sign or a value. That is a shorthand for providing the empty string in HTML, or the attribute's name in XML.
+<input required />
+<!-- is the same as… -->
+<input required="" />
+<!-- or -->
+<input required="required" />
+Reflection of an attribute
+  Attributes may be reflected into a particular property of the specific interface.
+  It means that the value of the attribute can be read by accessing the property,
+  and can be modified by setting the property to a different value.
+
+For example, the placeholder below is reflected into HTMLInputElement.placeholder.
+Considering the following HTML:
+<input placeholder="Original placeholder" />
+
+We can check the reflection between HTMLInputElement.placeholder and the attribute using:
+const input = document.querySelector("input");
+const attr = input.getAttributeNode("placeholder");
+console.log(attr.value);
+console.log(input.placeholder); // Prints the same value as `attr.value`
+
+// Changing placeholder value will also change the value of the reflected attribute.
+input.placeholder = "Modified placeholder";
+console.log(attr.value); // Prints `Modified placeholder`
+See also
+HTML attribute reference
+Information about HTML's global attributes
+Found a content problem with this page?Edit the page on GitHub.Report the content issue.View the source on GitHub.Want to get more involved? Learn how to contribute.This page was last modified on Feb 21, 2023 by MDN contributors.MDN logoYour blueprint for a better internet.MDN on TwitterMDN on GitHubMDNAboutBlogCareersAdvertise with usSupportProduct helpReport an issueOur communitiesMDN CommunityMDN ForumMDN ChatDevelopersWeb TechnologiesLearn Web DevelopmentMDN PlusHacks BlogMozilla logoWebsite Privacy NoticeCookiesLegalCommunity Participation GuidelinesVisit Mozilla Corporation’s not-for-profit parent, the Mozilla Foundation.Portions of this content are ©1998–2023 by individual mozilla.org contributors. Content available under a Creative Commons license.

lsu_pilot/text/developer.mozilla.org/developer.mozilla.org_en-US_docs_Glossary_BigInt.txt

@@ -0,0 +1,5 @@
+BigInt - MDN Web Docs Glossary: Definitions of Web-related terms | MDNSkip to main contentSkip to searchSkip to select languageMDN Web DocsOpen main menuReferencesReferencesOverview / Web TechnologyWeb technology reference for developersHTMLStructure of content on the webCSSCode used to describe document styleJavaScriptGeneral-purpose scripting languageHTTPProtocol for transmitting web resourcesWeb APIsInterfaces for building web applicationsWeb ExtensionsDeveloping extensions for web browsersWeb TechnologyWeb technology reference for developersGuidesGuidesOverview / MDN Learning AreaLearn web developmentMDN Learning AreaLearn web developmentHTMLLearn to structure web content with HTMLCSSLearn to style content using CSSJavaScriptLearn to run scripts in the browserAccessibilityLearn to make the web accessible to allPlusPlusOverviewA customized MDN experienceUpdatesAll browser compatibility updates at a glanceDocumentationLearn how to use MDN PlusFAQFrequently asked questions about MDN PlusBlog NewSearch MDNClear search inputSearchThemeLog inGet MDN PlusMDN Web Docs Glossary: Definitions of Web-related termsBigIntArticle ActionsEnglish (US)In this articleSee alsoIn this articleSee alsoBigIntIn JavaScript, BigInt is a numeric data type that can represent integers in the arbitrary precision format. In other programming languages different numeric types can exist, for examples: Integers, Floats, Doubles, or Bignums.See also
+Numeric types on Wikipedia
+The JavaScript type: BigInt
+The JavaScript global object BigInt
+Found a content problem with this page?Edit the page on GitHub.Report the content issue.View the source on GitHub.Want to get more involved? Learn how to contribute.This page was last modified on Feb 21, 2023 by MDN contributors.MDN logoYour blueprint for a better internet.MDN on TwitterMDN on GitHubMDNAboutBlogCareersAdvertise with usSupportProduct helpReport an issueOur communitiesMDN CommunityMDN ForumMDN ChatDevelopersWeb TechnologiesLearn Web DevelopmentMDN PlusHacks BlogMozilla logoWebsite Privacy NoticeCookiesLegalCommunity Participation GuidelinesVisit Mozilla Corporation’s not-for-profit parent, the Mozilla Foundation.Portions of this content are ©1998–2023 by individual mozilla.org contributors. Content available under a Creative Commons license.

lsu_pilot/text/developer.mozilla.org/developer.mozilla.org_en-US_docs_Glossary_BigInt_contributors.txt.txt

@@ -0,0 +1,9 @@
+# Contributors by commit history
+https://github.com/mdn/content/commits/main/files/en-us/glossary/bigint/index.md
+
+# Original Wiki contributors
+chrisdavidmills
+fscholz
+rwe2020
+milsyobtaf
+iulia

lsu_pilot/text/developer.mozilla.org/developer.mozilla.org_en-US_docs_Glossary_Browsing_context.txt

@@ -0,0 +1,5 @@
+Browsing context - MDN Web Docs Glossary: Definitions of Web-related terms | MDNSkip to main contentSkip to searchSkip to select languageMDN Web DocsOpen main menuReferencesReferencesOverview / Web TechnologyWeb technology reference for developersHTMLStructure of content on the webCSSCode used to describe document styleJavaScriptGeneral-purpose scripting languageHTTPProtocol for transmitting web resourcesWeb APIsInterfaces for building web applicationsWeb ExtensionsDeveloping extensions for web browsersWeb TechnologyWeb technology reference for developersGuidesGuidesOverview / MDN Learning AreaLearn web developmentMDN Learning AreaLearn web developmentHTMLLearn to structure web content with HTMLCSSLearn to style content using CSSJavaScriptLearn to run scripts in the browserAccessibilityLearn to make the web accessible to allPlusPlusOverviewA customized MDN experienceUpdatesAll browser compatibility updates at a glanceDocumentationLearn how to use MDN PlusFAQFrequently asked questions about MDN PlusBlog NewSearch MDNClear search inputSearchThemeLog inGet MDN PlusMDN Web Docs Glossary: Definitions of Web-related termsBrowsing contextArticle ActionsEnglish (US)In this articleSee alsoIn this articleSee alsoBrowsing contextA browsing context is an environment in which a browser displays a Document. In modern browsers, it usually is a tab, but can be a window or even only parts of a page, like a frame or an iframe.
+Each browsing context has an origin (that of the active document) and an ordered history of previously displayed documents.
+Communication between browsing contexts is severely constrained. Between browsing contexts of the same origin, a BroadcastChannel can be opened and used.See also
+See origin
+Found a content problem with this page?Edit the page on GitHub.Report the content issue.View the source on GitHub.Want to get more involved? Learn how to contribute.This page was last modified on Feb 22, 2023 by MDN contributors.MDN logoYour blueprint for a better internet.MDN on TwitterMDN on GitHubMDNAboutBlogCareersAdvertise with usSupportProduct helpReport an issueOur communitiesMDN CommunityMDN ForumMDN ChatDevelopersWeb TechnologiesLearn Web DevelopmentMDN PlusHacks BlogMozilla logoWebsite Privacy NoticeCookiesLegalCommunity Participation GuidelinesVisit Mozilla Corporation’s not-for-profit parent, the Mozilla Foundation.Portions of this content are ©1998–2023 by individual mozilla.org contributors. Content available under a Creative Commons license.