Use data science 2 image (us-east-1), simplify dependency mgmt, avoid kernel restart

02_data_exploration_and_feature_eng/02_data_exploration_and_feature_eng.ipynb

@@ -19,28 +19,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "import sagemaker\n",
-    "import sys\n",
-    "import IPython\n",
-    "\n",
-    "# Let's make sure we have the required version of the SM PySDK.\n",
-    "required_version = '2.49.2'\n",
-    "\n",
-    "def versiontuple(v):\n",
-    "    return tuple(map(int, (v.split(\".\"))))\n",
-    "\n",
-    "if versiontuple(sagemaker.__version__) < versiontuple(required_version):\n",
-    "    !{sys.executable} -m pip install -U sagemaker=={required_version}\n",
-    "\n",
-    "!{sys.executable} -m pip install -U numpy\n",
-    "!{sys.executable} -m pip install -U pandas\n",
-    "\n",
-    "IPython.Application.instance().kernel.do_shutdown(True)"
+    "from notebook_utilities import check_dependencies\n",
+    "check_dependencies() # Check SageMaker, Numpy, Pandas versions and install upgrades if needed"
    ]
   },
   {
@@ -50,9 +33,10 @@
    "outputs": [],
    "source": [
     "import sagemaker, pandas, numpy\n",
-    "print(sagemaker.__version__)\n",
-    "print(pandas.__version__)\n",
-    "print(numpy.__version__)"
+    "\n",
+    "print('sagemaker version:', sagemaker.__version__)\n",
+    "print('pandas version:', pandas.__version__)\n",
+    "print('numpy version:', numpy.__version__)\n"
    ]
   },
   {
@@ -487,9 +471,9 @@
  "metadata": {
   "instance_type": "ml.t3.medium",
   "kernelspec": {
-   "display_name": "Python 3 (Data Science)",
+   "display_name": "Python 3 (Data Science 2.0)",
    "language": "python",
-   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0"
+   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/sagemaker-data-science-38"
   },
   "language_info": {
    "codemirror_mode": {
@@ -501,7 +485,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.8.13"
   }
  },
  "nbformat": 4,

02_data_exploration_and_feature_eng/02_data_exploration_and_feature_eng_athena.ipynb

@@ -24,24 +24,8 @@
    },
    "outputs": [],
    "source": [
-    "import sagemaker\n",
-    "import sys\n",
-    "import IPython\n",
-    "\n",
-    "# Let's make sure we have the required version of the SM PySDK.\n",
-    "required_version = '2.49.2'\n",
-    "\n",
-    "def versiontuple(v):\n",
-    "    return tuple(map(int, (v.split(\".\"))))\n",
-    "\n",
-    "if versiontuple(sagemaker.__version__) < versiontuple(required_version):\n",
-    "    !{sys.executable} -m pip install -U sagemaker=={required_version}\n",
-    "\n",
-    "!{sys.executable} -m pip install -U numpy\n",
-    "!{sys.executable} -m pip install -U pandas\n",
-    "!{sys.executable} -m pip install botocore==1.22.10 # Downgrade botocore. Required for PyAthena to work.\n",
-    " \n",
-    "IPython.Application.instance().kernel.do_shutdown(True)"
+    "from notebook_utilities import check_dependencies\n",
+    "check_dependencies() # Check SageMaker, Numpy, Pandas versions and install upgrades if needed"
    ]
   },
   {

02_data_exploration_and_feature_eng/notebook_utilities.py

@@ -30,4 +30,48 @@ def cleanup_glue_resources():
     if job_found:
          glue_client.delete_job(JobName = 'endtoendml-job')
 
-    print("Cleanup completed.")
+    print("Cleanup completed.")
+
+def check_dependencies():
+
+    import sagemaker
+    import numpy
+    import pandas
+
+    import sys
+    import os
+    import IPython
+
+    def versiontuple(v):
+        return tuple(map(int, (v.split("."))))
+
+    kernel_restart_required = False
+
+    required_version = '2.90.0'
+    if (versiontuple(sagemaker.__version__) < versiontuple(required_version)): 
+        print('This workshop was tested with sagemaker version {} but you are using {}. Installing required version...'.format(required_version, sagemaker.__version__) )
+        stream = os.popen('{} -m pip install -U sagemaker=={}'.format(sys.executable, required_version))
+        output = stream.read()
+        print(output)
+        kernel_restart_required = True
+
+    required_version = '1.3.4'
+    if (versiontuple(pandas.__version__) < versiontuple(required_version) ):
+        print('This workshop was tested with pandas version {} but you are using {}. Installing required version...'.format(required_version, pandas.__version__))
+        stream = os.popen('{} -m pip install -U pandas=={}'.format(sys.executable, required_version))
+        output = stream.read()
+        print(output)
+        kernel_restart_required = True
+
+    required_version = '1.21.6' 
+    if (versiontuple(numpy.__version__) < versiontuple(required_version)):
+        print('This workshop was tested with numpy version {} but you are using {}. Installing required version...'.format(required_version, numpy.__version__ ) )
+        stream = os.popen('{} -m pip install -U numpy=={}'.format(sys.executable, required_version))
+        output = stream.read()
+        print(output)
+        kernel_restart_required = True
+
+    if kernel_restart_required:
+        print("Restarting kernel after installing new dependencies...")
+        IPython.Application.instance().kernel.do_shutdown(True)     
+        print('WARNING: Kernel restarting...please wait 30 seconds before moving to the next cell!')

03_train_model/03_train_model.ipynb

@@ -20,24 +20,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "import sagemaker\n",
-    "import sys\n",
-    "import IPython\n",
-    "\n",
-    "# Let's make sure we have the required version of the SM PySDK.\n",
-    "required_version = '2.49.2'\n",
-    "\n",
-    "def versiontuple(v):\n",
-    "    return tuple(map(int, (v.split(\".\"))))\n",
-    "\n",
-    "if versiontuple(sagemaker.__version__) < versiontuple(required_version):\n",
-    "    !{sys.executable} -m pip install -U sagemaker=={required_version}\n",
-    "    IPython.Application.instance().kernel.do_shutdown(True)"
+    "from notebook_utilities import check_dependencies\n",
+    "check_dependencies() # Check SageMaker version and install upgrades if needed"
    ]
   },
   {
@@ -214,9 +201,9 @@
  "metadata": {
   "instance_type": "ml.t3.medium",
   "kernelspec": {
-   "display_name": "Python 3 (Data Science)",
+   "display_name": "Python 3 (Data Science 2.0)",
    "language": "python",
-   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0"
+   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/sagemaker-data-science-38"
   },
   "language_info": {
    "codemirror_mode": {
@@ -228,7 +215,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.8.13"
   }
  },
  "nbformat": 4,

03_train_model/03_train_model_debug.ipynb

@@ -25,19 +25,8 @@
    },
    "outputs": [],
    "source": [
-    "import sagemaker\n",
-    "import sys\n",
-    "import IPython\n",
-    "\n",
-    "# Let's make sure we have the required version of the SM PySDK.\n",
-    "required_version = '2.49.2'\n",
-    "\n",
-    "def versiontuple(v):\n",
-    "    return tuple(map(int, (v.split(\".\"))))\n",
-    "\n",
-    "if versiontuple(sagemaker.__version__) < versiontuple(required_version):\n",
-    "    !{sys.executable} -m pip install -U sagemaker=={required_version}\n",
-    "    IPython.Application.instance().kernel.do_shutdown(True)"
+    "from notebook_utilities import check_dependencies\n",
+    "check_dependencies() # Check SageMaker version and install upgrades if needed"
    ]
   },
   {
@@ -633,9 +622,9 @@
  "metadata": {
   "instance_type": "ml.t3.medium",
   "kernelspec": {
-   "display_name": "Python 3 (Data Science)",
+   "display_name": "Python 3 (Data Science 2.0)",
    "language": "python",
-   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0"
+   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/sagemaker-data-science-38"
   },
   "language_info": {
    "codemirror_mode": {

03_train_model/notebook_utilities.py

@@ -0,0 +1,25 @@
+def check_dependencies():
+
+    import sagemaker
+
+    import sys
+    import os
+    import IPython
+
+    def versiontuple(v):
+        return tuple(map(int, (v.split("."))))
+
+    kernel_restart_required = False
+
+    required_version = '2.90.0'
+    if (versiontuple(sagemaker.__version__) < versiontuple(required_version)): 
+        print('This workshop was tested with sagemaker version {} but you are using {}. Installing required version...'.format(required_version, sagemaker.__version__) )
+        stream = os.popen('{} -m pip install -U sagemaker=={}'.format(sys.executable, required_version))
+        output = stream.read()
+        print(output)
+        kernel_restart_required = True
+
+    if kernel_restart_required:
+        print("Restarting kernel after installing new dependencies...")
+        IPython.Application.instance().kernel.do_shutdown(True)     
+        print('WARNING: Kernel restarting...please wait 30 seconds before moving to the next cell!')

04_deploy_model/04_deploy_model.ipynb

@@ -22,24 +22,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "import sagemaker\n",
-    "import sys\n",
-    "import IPython\n",
-    "\n",
-    "# Let's make sure we have the required version of the SM PySDK.\n",
-    "required_version = '2.49.2'\n",
-    "\n",
-    "def versiontuple(v):\n",
-    "    return tuple(map(int, (v.split(\".\"))))\n",
-    "\n",
-    "if versiontuple(sagemaker.__version__) < versiontuple(required_version):\n",
-    "    !{sys.executable} -m pip install -U sagemaker=={required_version}\n",
-    "    IPython.Application.instance().kernel.do_shutdown(True)"
+    "from notebook_utilities import check_dependencies\n",
+    "check_dependencies() # Check SageMaker version and install upgrades if needed"
    ]
   },
   {
@@ -346,9 +333,9 @@
  "metadata": {
   "instance_type": "ml.t3.medium",
   "kernelspec": {
-   "display_name": "Python 3 (Data Science)",
+   "display_name": "Python 3 (Data Science 2.0)",
    "language": "python",
-   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0"
+   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/sagemaker-data-science-38"
   },
   "language_info": {
    "codemirror_mode": {
@@ -360,7 +347,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.8.13"
   }
  },
  "nbformat": 4,

04_deploy_model/04_deploy_model_monitor.ipynb

@@ -27,19 +27,8 @@
    },
    "outputs": [],
    "source": [
-    "import sagemaker\n",
-    "import sys\n",
-    "import IPython\n",
-    "\n",
-    "# Let's make sure we have the required version of the SM PySDK.\n",
-    "required_version = '2.49.2'\n",
-    "\n",
-    "def versiontuple(v):\n",
-    "    return tuple(map(int, (v.split(\".\"))))\n",
-    "\n",
-    "if versiontuple(sagemaker.__version__) < versiontuple(required_version):\n",
-    "    !{sys.executable} -m pip install -U sagemaker=={required_version}\n",
-    "    IPython.Application.instance().kernel.do_shutdown(True)"
+    "from notebook_utilities import check_dependencies\n",
+    "check_dependencies() # Check SageMaker version and install upgrades if needed"
    ]
   },
   {
@@ -920,9 +909,9 @@
  "metadata": {
   "instance_type": "ml.t3.medium",
   "kernelspec": {
-   "display_name": "Python 3 (Data Science)",
+   "display_name": "Python 3 (Data Science 2.0)",
    "language": "python",
-   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0"
+   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/sagemaker-data-science-38"
   },
   "language_info": {
    "codemirror_mode": {

04_deploy_model/notebook_utilities.py

@@ -14,3 +14,29 @@ def get_training_job_s3_model_artifacts(job_name):
     response = client.describe_training_job(TrainingJobName=job_name)
     s3_model_artifacts = response['ModelArtifacts']['S3ModelArtifacts']
     return s3_model_artifacts
+
+def check_dependencies():
+
+    import sagemaker
+
+    import sys
+    import os
+    import IPython
+
+    def versiontuple(v):
+        return tuple(map(int, (v.split("."))))
+
+    kernel_restart_required = False
+
+    required_version = '2.90.0'
+    if (versiontuple(sagemaker.__version__) < versiontuple(required_version)): 
+        print('This workshop was tested with sagemaker version {} but you are using {}. Installing required version...'.format(required_version, sagemaker.__version__) )
+        stream = os.popen('{} -m pip install -U sagemaker=={}'.format(sys.executable, required_version))
+        output = stream.read()
+        print(output)
+        kernel_restart_required = True
+
+    if kernel_restart_required:
+        print("Restarting kernel after installing new dependencies...")
+        IPython.Application.instance().kernel.do_shutdown(True)     
+        print('WARNING: Kernel restarting...please wait 30 seconds before moving to the next cell!')