diff --git a/Pipfile b/Pipfile index 2288797..498a33d 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,7 @@ verify_ssl = true jupyter = "*" fastapi = "*" nbconvert = "*" +nbparameterise = "*" uvicorn = "*" pandas = "*" requests = "*" diff --git a/README.md b/README.md index 3f285c1..5398b23 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ pipenv shell ### Add your notebook(s) to "src/notebooks" -There's an example notebook already in there called `scrape.ipynb`. +There's an example notebook already in there called `hello.ipynb` and `scrape.ipynb`. ### Run locally @@ -35,12 +35,20 @@ chmod +x run.sh ./run.sh ``` -## Trigger a notebook. -With the server running, trigger notebook (relative path) like `notebooks/scrape.ipynb` +## Run a notebook. +With the server running, run notebook file like `scrape.ipynb` ```python import requests -r = requests.post("http://localhost:8000/trigger/notebooks/scrape.ipynb") +r = requests.post("http://localhost:8000/notebook/scrape.ipynb") +print(r.json()) +``` + +You can also change the variables in the notebook from the parameter. For example; + +```python +import requests +r = requests.post("http://localhost:8000/notebook/scrape.ipynb?start_year=2021&years_ago=7") print(r.json()) ``` @@ -74,4 +82,4 @@ heroku container:login docker build -t -f Dockerfile.web . heroku container:push web --recursive heroku container:release web -``` \ No newline at end of file +``` diff --git a/requirements.txt b/requirements.txt index 8eb145e..e34f91c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,6 +35,7 @@ MarkupSafe==1.1.1 mistune==0.8.4 nbconvert==5.6.1 nbformat==5.0.4 +nbparameterise==0.5 notebook==6.0.3 numpy==1.18.2 pandas==1.0.3 diff --git a/src/notebooks/hello.ipynb b/src/notebooks/hello.ipynb new file mode 100644 index 0000000..e9923f8 --- /dev/null +++ b/src/notebooks/hello.ipynb @@ -0,0 +1,38 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "to = \"World\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Hello, {to}!\")" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "36cf16204b8548560b1c020c4e8fb5b57f0e4c58016f52f2d4be01e192833930" + }, + "kernelspec": { + "display_name": "Python 3.9.4 64-bit", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/src/notebooks/scrape.ipynb b/src/notebooks/scrape.ipynb index b859a63..6ab308f 100644 --- a/src/notebooks/scrape.ipynb +++ b/src/notebooks/scrape.ipynb @@ -1,28 +1,15 @@ { - "nbformat": 4, - "nbformat_minor": 2, - "metadata": { - "language_info": { - "name": "python", - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "version": "3.6.8-final" - }, - "orig_nbformat": 2, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "npconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": 3, - "kernelspec": { - "name": "python36864bitjupyterapipipenv100023b4210342f0b1028e92ab62ef71", - "display_name": "Python 3.6.8 64-bit ('jupyter-api': pipenv)" - } - }, "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "start_year = 2014\n", + "years_ago = 0" + ] + }, { "cell_type": "code", "execution_count": 10, @@ -131,13 +118,15 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "Finished 2020\n" + "output_type": "stream", + "text": [ + "Finished 2020\n" + ] } ], "source": [ - "run()" + "run(int(start_year), int(years_ago))" ] }, { @@ -147,5 +136,27 @@ "outputs": [], "source": [] } - ] + ], + "metadata": { + "file_extension": ".py", + "interpreter": { + "hash": "36cf16204b8548560b1c020c4e8fb5b57f0e4c58016f52f2d4be01e192833930" + }, + "kernelspec": { + "display_name": "Python 3.9.4 64-bit", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + }, + "mimetype": "text/x-python", + "name": "python", + "npconvert_exporter": "python", + "orig_nbformat": 2, + "pygments_lexer": "ipython3", + "version": 3 + }, + "nbformat": 4, + "nbformat_minor": 2 } \ No newline at end of file diff --git a/src/server.py b/src/server.py index 86e4975..768dac9 100644 --- a/src/server.py +++ b/src/server.py @@ -1,6 +1,8 @@ -from fastapi import FastAPI +from fastapi import FastAPI, Request from pprint import pprint + app = FastAPI() + import inspect import os @@ -8,17 +10,17 @@ filename = inspect.getframeinfo(inspect.currentframe()).filename BASE_DIR = os.path.dirname(os.path.abspath(filename)) +NOTEBOOKS_DIR = BASE_DIR + '/notebooks' @app.get("/") async def read_root(): return {"Hello": "World"} -# @app.get("/trigger/{filepath:path}") -@app.post("/trigger/{filepath:path}") -async def read_item(filepath): - filepath = os.path.join(BASE_DIR, filepath) - ep = trigger(notebook_filename=filepath) - # data = vars(ep) +@app.get("/notebook/{filepath:path}") +async def read_item(filepath, request: Request): + filepath = os.path.join(NOTEBOOKS_DIR, filepath) + ep = trigger(notebook_filename=filepath, params=request.query_params) + cells = ep[0]['cells'] all_outputs = [] for i, cell in enumerate(cells): diff --git a/src/trigger.py b/src/trigger.py index acf0066..01799cb 100644 --- a/src/trigger.py +++ b/src/trigger.py @@ -1,9 +1,16 @@ import nbformat from nbconvert.preprocessors import ExecutePreprocessor +from nbparameterise import ( + extract_parameters, replace_definitions, parameter_values +) -def trigger(notebook_filename='chp-traffic.ipynb'): +def trigger(notebook_filename='hello.ipynb', params={}): with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) + + orig_parameters = extract_parameters(nb) + new_nb = replace_definitions(nb, parameter_values(orig_parameters, **params)) + ep = ExecutePreprocessor(timeout=600, kernel_name='python3') - r = ep.preprocess(nb) + r = ep.preprocess(new_nb) return r