From bbbc9aeb269101b36cfd3e886f8d09fd48b6ec16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nurullah=20Sevin=C3=A7tekin?= Date: Thu, 7 Oct 2021 20:55:59 +0300 Subject: [PATCH 1/5] Notebook file is prepared to work with the parameter. --- Pipfile | 1 + requirements.txt | 1 + src/notebooks/scrape.ipynb | 65 ++++++++++++++++++++++---------------- src/server.py | 12 +++---- src/trigger.py | 11 +++++-- 5 files changed, 55 insertions(+), 35 deletions(-) diff --git a/Pipfile b/Pipfile index 2288797..498a33d 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,7 @@ verify_ssl = true jupyter = "*" fastapi = "*" nbconvert = "*" +nbparameterise = "*" uvicorn = "*" pandas = "*" requests = "*" diff --git a/requirements.txt b/requirements.txt index 8eb145e..e34f91c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,6 +35,7 @@ MarkupSafe==1.1.1 mistune==0.8.4 nbconvert==5.6.1 nbformat==5.0.4 +nbparameterise==0.5 notebook==6.0.3 numpy==1.18.2 pandas==1.0.3 diff --git a/src/notebooks/scrape.ipynb b/src/notebooks/scrape.ipynb index b859a63..6ab308f 100644 --- a/src/notebooks/scrape.ipynb +++ b/src/notebooks/scrape.ipynb @@ -1,28 +1,15 @@ { - "nbformat": 4, - "nbformat_minor": 2, - "metadata": { - "language_info": { - "name": "python", - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "version": "3.6.8-final" - }, - "orig_nbformat": 2, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "npconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": 3, - "kernelspec": { - "name": "python36864bitjupyterapipipenv100023b4210342f0b1028e92ab62ef71", - "display_name": "Python 3.6.8 64-bit ('jupyter-api': pipenv)" - } - }, "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "start_year = 2014\n", + "years_ago = 0" + ] + }, { "cell_type": "code", "execution_count": 10, @@ -131,13 +118,15 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", - "text": "Finished 2020\n" + "output_type": "stream", + "text": [ + "Finished 2020\n" + ] } ], "source": [ - "run()" + "run(int(start_year), int(years_ago))" ] }, { @@ -147,5 +136,27 @@ "outputs": [], "source": [] } - ] + ], + "metadata": { + "file_extension": ".py", + "interpreter": { + "hash": "36cf16204b8548560b1c020c4e8fb5b57f0e4c58016f52f2d4be01e192833930" + }, + "kernelspec": { + "display_name": "Python 3.9.4 64-bit", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + }, + "mimetype": "text/x-python", + "name": "python", + "npconvert_exporter": "python", + "orig_nbformat": 2, + "pygments_lexer": "ipython3", + "version": 3 + }, + "nbformat": 4, + "nbformat_minor": 2 } \ No newline at end of file diff --git a/src/server.py b/src/server.py index 86e4975..adfbbe1 100644 --- a/src/server.py +++ b/src/server.py @@ -1,4 +1,4 @@ -from fastapi import FastAPI +from fastapi import FastAPI, Request from pprint import pprint app = FastAPI() import inspect @@ -8,17 +8,17 @@ filename = inspect.getframeinfo(inspect.currentframe()).filename BASE_DIR = os.path.dirname(os.path.abspath(filename)) +NOTEBOOKS_DIR = BASE_DIR + '/notebooks' @app.get("/") async def read_root(): return {"Hello": "World"} -# @app.get("/trigger/{filepath:path}") @app.post("/trigger/{filepath:path}") -async def read_item(filepath): - filepath = os.path.join(BASE_DIR, filepath) - ep = trigger(notebook_filename=filepath) - # data = vars(ep) +async def read_item(filepath, request: Request): + filepath = os.path.join(NOTEBOOKS_DIR, filepath) + ep = trigger(notebook_filename=filepath, params=request.query_params) + cells = ep[0]['cells'] all_outputs = [] for i, cell in enumerate(cells): diff --git a/src/trigger.py b/src/trigger.py index acf0066..837fa05 100644 --- a/src/trigger.py +++ b/src/trigger.py @@ -1,9 +1,16 @@ import nbformat from nbconvert.preprocessors import ExecutePreprocessor +from nbparameterise import ( + extract_parameters, replace_definitions, parameter_values +) -def trigger(notebook_filename='chp-traffic.ipynb'): +def trigger(notebook_filename='chp-traffic.ipynb', params={}): with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) + + orig_parameters = extract_parameters(nb) + new_nb = replace_definitions(nb, parameter_values(orig_parameters, **params)) + ep = ExecutePreprocessor(timeout=600, kernel_name='python3') - r = ep.preprocess(nb) + r = ep.preprocess(new_nb) return r From 41c87fd0c57e2e7a7481dc0991c4bf20aa884d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nurullah=20Sevin=C3=A7tekin?= Date: Thu, 7 Oct 2021 21:02:22 +0300 Subject: [PATCH 2/5] Hello notebook made default. --- src/notebooks/hello.ipynb | 38 ++++++++++++++++++++++++++++++++++++++ src/trigger.py | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 src/notebooks/hello.ipynb diff --git a/src/notebooks/hello.ipynb b/src/notebooks/hello.ipynb new file mode 100644 index 0000000..e9923f8 --- /dev/null +++ b/src/notebooks/hello.ipynb @@ -0,0 +1,38 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "to = \"World\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Hello, {to}!\")" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "36cf16204b8548560b1c020c4e8fb5b57f0e4c58016f52f2d4be01e192833930" + }, + "kernelspec": { + "display_name": "Python 3.9.4 64-bit", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/src/trigger.py b/src/trigger.py index 837fa05..01799cb 100644 --- a/src/trigger.py +++ b/src/trigger.py @@ -4,7 +4,7 @@ extract_parameters, replace_definitions, parameter_values ) -def trigger(notebook_filename='chp-traffic.ipynb', params={}): +def trigger(notebook_filename='hello.ipynb', params={}): with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) From 9052ce1b27a84373be6d09f25960087559be21d9 Mon Sep 17 00:00:00 2001 From: Nurullah Sevinctekin Date: Thu, 7 Oct 2021 21:04:39 +0300 Subject: [PATCH 3/5] Readme updated. --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3f285c1..7e3b808 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ pipenv shell ### Add your notebook(s) to "src/notebooks" -There's an example notebook already in there called `scrape.ipynb`. +There's an example notebook already in there called `hello.ipynb` and `scrape.ipynb`. ### Run locally @@ -44,6 +44,14 @@ r = requests.post("http://localhost:8000/trigger/notebooks/scrape.ipynb") print(r.json()) ``` +You can also change the variables in the notebook from the parameter. For example; + +```python +import requests +r = requests.post("http://localhost:8000/notebook/scrape.ipynb?start_year=2021&years_ago=7") +print(r.json()) +``` + ## Deployment @@ -74,4 +82,4 @@ heroku container:login docker build -t -f Dockerfile.web . heroku container:push web --recursive heroku container:release web -``` \ No newline at end of file +``` From 36d08c04fb2c61d1892e6c499461fb0e08f63239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nurullah=20Sevin=C3=A7tekin?= Date: Fri, 8 Oct 2021 16:18:02 +0300 Subject: [PATCH 4/5] Changed trigger endpoint to notebook. --- README.md | 6 +++--- src/server.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7e3b808..5398b23 100644 --- a/README.md +++ b/README.md @@ -35,12 +35,12 @@ chmod +x run.sh ./run.sh ``` -## Trigger a notebook. -With the server running, trigger notebook (relative path) like `notebooks/scrape.ipynb` +## Run a notebook. +With the server running, run notebook file like `scrape.ipynb` ```python import requests -r = requests.post("http://localhost:8000/trigger/notebooks/scrape.ipynb") +r = requests.post("http://localhost:8000/notebook/scrape.ipynb") print(r.json()) ``` diff --git a/src/server.py b/src/server.py index adfbbe1..3a67418 100644 --- a/src/server.py +++ b/src/server.py @@ -14,7 +14,7 @@ async def read_root(): return {"Hello": "World"} -@app.post("/trigger/{filepath:path}") +@app.post("/notebook/{filepath:path}") async def read_item(filepath, request: Request): filepath = os.path.join(NOTEBOOKS_DIR, filepath) ep = trigger(notebook_filename=filepath, params=request.query_params) From 70ef3f9657702db3204210d2bf13efa54307fabe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nurullah=20Sevin=C3=A7tekin?= Date: Thu, 11 Aug 2022 16:45:26 +0300 Subject: [PATCH 5/5] Notebook method changed. --- src/server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server.py b/src/server.py index 3a67418..768dac9 100644 --- a/src/server.py +++ b/src/server.py @@ -1,6 +1,8 @@ from fastapi import FastAPI, Request from pprint import pprint + app = FastAPI() + import inspect import os @@ -14,7 +16,7 @@ async def read_root(): return {"Hello": "World"} -@app.post("/notebook/{filepath:path}") +@app.get("/notebook/{filepath:path}") async def read_item(filepath, request: Request): filepath = os.path.join(NOTEBOOKS_DIR, filepath) ep = trigger(notebook_filename=filepath, params=request.query_params)