re-arranged data sets for pandas examples and added pyinstrument to useful packages list

Author: pybokeh

PyData/UsefulPackages.txt

@@ -33,6 +33,7 @@ pyflame: https://github.com/uber/pyflame   # Does not support Windows
 birdseye: https://github.com/alexmojaki/birdseye
 pyheat: https://github.com/csurfer/pyheat
 heartrate: https://github.com/alexmojaki/heartrate
+pyinstrument: https://github.com/joerick/pyinstrument
 https://www.markkeller.dev/2018-07-14-optimize_python/
 
 # Forecasting
@@ -71,8 +72,9 @@ bqplot: https://github.com/bloomberg/bqplot
 pdvega
 bokeh / HoloViews / hvplot / pandas-bokeh (pyviz stack)
 chartify: https://github.com/spotify/chartify
-folium
-geoplot
+folium: https://github.com/python-visualization/folium
+geoplot: https://github.com/ResidentMario/geoplot
+movingpandas: https://github.com/anitagraser/movingpandas
 plotnine: clone of R's ggplot2
 joypy: https://github.com/sbebo/joypy/blob/master/Joyplot.ipynb
 jmpy
@@ -86,6 +88,14 @@ Rain clouds viz: https://github.com/pog87/PtitPrince
 jupyterplot: https://lvwerra.github.io/jupyterplot/
 cyberpunk theme: https://github.com/dhaitz/mplcyberpunk/blob/master/README.md
 upsetplot: https://github.com/jnothman/UpSetPlot
+keplergl (PITA to install onto Windows): https://github.com/keplergl/kepler.gl/issues/557
+
+# geo visualization libraries:
+folium: https://github.com/python-visualization/folium
+geoplot: https://github.com/ResidentMario/geoplot
+ipyleaflet: https://github.com/jupyter-widgets/ipyleaflet/
+movingpandas: https://github.com/anitagraser/movingpandas
+keplergl: keplergl (PITA to install onto Windows): https://github.com/keplergl/kepler.gl/issues/557
 
 # Animation related
 celluloid: https://github.com/jwkvam/celluloid
@@ -204,14 +214,16 @@ https://github.com/RobinL/fuzzymatcher
 https://github.com/J535D165/recordlinkage
 scikit-neuralnetwork: https://github.com/aigamedev/scikit-neuralnetwork
 
-# Webscraping
+# HTTP / Webscraping
 beautifulsoup4
 lxml+XPath
 mechanicalsoup
 selenium (headless-browser framework for extracting javascript data)
 scrapy (OOP framework)
 newspaper3k: https://newspaper.readthedocs.io  # easily extract text from articles
-requests-html: requests-html: https://github.com/kennethreitz/requests-html
+requests-html: requests-html: https://github.com/kennethreitz/requests-htm
+HTTPX: https://github.com/encode/httpx
+autoscraper: https://github.com/alirezamika/autoscraper
 
 # Big Data
 boto3: Interface to S3

dask/coiled/Basic_Example.ipynb

@@ -0,0 +1,68 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dask.distributed import Client\n",
+    "import coiled\n",
+    "import dask.dataframe as dd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cluster = coiled.Cluster(n_workers=10)\n",
+    "client = Client(cluster)\n",
+    "print('Dashboard:', client.dashboard_link)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = dd.read_csv(\n",
+    "    \"s3://nyc-tlc/trip data/yellow_tripdata_2019-*.csv\",\n",
+    "    dtype={\n",
+    "        \"payment_type\": \"UInt8\",\n",
+    "        \"VendorID\": \"UInt8\",\n",
+    "        \"passenger_count\": \"UInt8\",\n",
+    "        \"RatecodeID\": \"UInt8\",\n",
+    "    },\n",
+    "    storage_options={\"anon\": True},\n",
+    "    blocksize=\"16 MiB\",\n",
+    ").persist()\n",
+    "\n",
+    "df.groupby(\"passenger_count\").tip_amount.mean().compute()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python (coiled_dev)",
+   "language": "python",
+   "name": "coiled_dev"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

hvplot/data/Generate_CPU_Data.py

@@ -0,0 +1,31 @@
+# pip install psutil
+import csv
+import psutil
+import time
+
+x_value = 0
+
+fieldnames = ["x_value", "cpu_perc"]
+
+
+with open('data.csv', 'w') as csv_file:
+    csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
+    csv_writer.writeheader()
+
+while True:
+
+    with open('data.csv', 'a') as csv_file:
+        csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
+        y_value = psutil.cpu_percent()
+
+        info = {
+            "x_value": x_value,
+            "cpu_perc": y_value
+        }
+
+        csv_writer.writerow(info)
+        print(x_value, y_value)
+
+        x_value += 1
+
+    time.sleep(1)