Updated to 1.6.8: side-load modules, Skia for better x-platform images, logging improvements

Author: ChrisMaunder

.gitignore

@@ -13,6 +13,9 @@
 /src/AnalysisLayer/ObjectDetectionNet/assets/
 /src/AnalysisLayer/Vision/assets
 
+# Generate Wix files.
+*Files.wxs
+
 /src/API/Server/FrontEnd/installconfig.json
 
 # User-specific files
@@ -375,6 +378,12 @@ FodyWeavers.xsd
 
 *.DS_Store
 
-/demos/Python/Vision/detected
+/Installers/zlib123dllx64.zip
+/Installers/cudnn-windows-x86_64-8.5.0.96_cuda11-archive.zip
+/src/AnalysisLayer/Vision/datastore/faceembedding.db
 /src/AnalysisLayer/ObjectDetectionYolo/assets
+/src/AnalysisLayer/ObjectDetectionYolo/custom
+/src/AnalysisLayer/CustomObjectDetection/assets
 /src/AnalysisLayer/ObjectDetectionYolo/custom-models
+/demos/Python/Vision/detected
+/src/modules/ALPR/paddleocr

CodeProject.AI.sln

@@ -63,7 +63,7 @@ if /i "%cleanBuild%" == "true" (
     call utils.bat WriteLine "Cleaning Build                                                      " "White" "Blue"
     call utils.bat WriteLine 
 
-    call :CleanSubDirs ".\..\..\src"   "bin" "\AnalysisLayer\bin\"
+    call :CleanSubDirs ".\..\..\src"   "bin" "\AnalysisLayer\bin"
     call :CleanSubDirs ".\..\..\src"   "obj"
     call :CleanSubDirs ".\..\Windows"  "bin"
     call :CleanSubDirs ".\..\Windows"  "obj"

Installers/Dev/clean.bat

@@ -59,6 +59,9 @@ set downloadDir=downloads
 :: The name of the dir holding the backend analysis services
 set analysisLayerDir=AnalysisLayer
 
+:: The name of the dir holding the downloaded/sideloaded backend analysis services
+set sideloadModulesDir=modules
+
 :: Absolute paths :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 :: The absolute path to the root directory of CodeProject.AI
 set installBasePath=%cd%
@@ -69,6 +72,7 @@ cd %installBasePath%
 
 :: The location of directories relative to the root of the solution directory
 set analysisLayerPath=%absoluteRootDir%\%srcDir%\%analysisLayerDir%
+set sideloadModulesPath=%absoluteRootDir%\%srcDir%\%sideloadModulesDir%
 set downloadPath=%absoluteRootDir%\Installers\%downloadDir%
 
 
@@ -143,7 +147,29 @@ for /f "delims=" %%D in ('dir /a:d /b "%analysisLayerPath%"') do (
         if exist "!modulePath!\install.dev.bat" (
 
             REM Pad right to 60 chars
-            set announcement=Processing !moduleDir! !spaces!
+            set announcement=Processing pre-installed module !moduleDir! !spaces!
+            set announcement=!announcement:~0,70!
+
+            call utils.bat WriteLine
+            call utils.bat WriteLine "!announcement!" "White" "Blue"
+            call utils.bat WriteLine
+
+            call "!modulePath!\install.dev.bat"
+        )
+    )
+)
+
+:: Walk through the sideloaded / downloaded modules directory and call the setup script in each dir
+:: TODO: This should be just a simple for /d %%D in ("%sideloadModulesPath%") do (
+for /f "delims=" %%D in ('dir /a:d /b "%sideloadModulesPath%"') do (
+    set moduleDir=%%~nxD
+    set modulePath=!sideloadModulesPath!\!moduleDir!
+
+    if /i "!moduleDir!" NEQ "bin" (
+        if exist "!modulePath!\install.dev.bat" (
+
+            REM Pad right to 60 chars
+            set announcement=Processing side-loaded module !moduleDir! !spaces!
             set announcement=!announcement:~0,70!
 
             call utils.bat WriteLine
@@ -155,6 +181,19 @@ for /f "delims=" %%D in ('dir /a:d /b "%analysisLayerPath%"') do (
     )
 )
 
+
+:: Demos
+set announcement=Processing Demos !spaces!
+set announcement=!announcement:~0,70!
+call utils.bat WriteLine
+call utils.bat WriteLine "!announcement!" "White" "Blue"
+call utils.bat WriteLine
+
+set moduleDir=demos
+set modulePath=%absoluteRootDir%\!moduleDir!
+call "!modulePath!\install.dev.bat"
+
+
 call utils.bat WriteLine
 call utils.bat WriteLine "Modules installed" "Green"
 

Installers/Dev/setup.dev.bat

@@ -93,6 +93,9 @@ downloadDir='downloads'
 # The name of the dir holding the backend analysis services
 analysisLayerDir='AnalysisLayer'
 
+:: The name of the dir holding the downloaded/sideloaded backend analysis services
+sideloadModulesDir="modules"
+
 # Absolute paths :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 # The absolute path to the root directory of CodeProject.AI
 currentDir="$(pwd)"
@@ -102,6 +105,7 @@ cd $currentDir
 
 # The location of directories relative to the root of the solution directory
 analysisLayerPath="${absoluteRootDir}/${srcDir}/${analysisLayerDir}"
+sideloadModulesPath="${absoluteRootDir}/${srcDir}/${sideloadModulesDir}"
 downloadPath="${absoluteRootDir}/Installers/${downloadDir}"
 
 # Set Flags
@@ -200,7 +204,36 @@ for d in ${analysisLayerPath}/*/ ; do
        if [ -f "${modulePath}/install.dev.sh" ]; then
 
             # Pad right to 70 chars
-            announcement=$(printf %-70s "Processing ${moduleDir}")
+            announcement=$(printf %-70s "Processing pre-installed module ${moduleDir}")
+
+            writeLine
+            writeLine "${announcement}" "White" "Blue"
+            writeLine
+
+            correctLineEndings "${modulePath}/install.dev.sh"
+            source "${modulePath}/install.dev.sh"
+        fi
+    fi
+done
+
+# Walk through the sideloaded / downloaded modules directory and call the setup script in each dir
+for d in ${analysisLayerPath}/*/ ; do
+
+    moduleDir="$(basename $d)"
+    modulePath=$d
+
+    if [ "${modulePath: -1}" == "/" ]; then
+        modulePath="${modulePath:0:${#modulePath}-1}"
+    fi
+
+    # dirname=${moduleDir,,} # requires bash 4.X, which isn't on macOS by default
+    dirname=$(echo $moduleDir | tr '[:upper:]' '[:lower:]')
+    if [ "${dirname}" != 'bin' ]; then
+
+       if [ -f "${modulePath}/install.dev.sh" ]; then
+
+            # Pad right to 70 chars
+            announcement=$(printf %-70s "Processing side-loaded module ${moduleDir}")
 
             writeLine
             writeLine "${announcement}" "White" "Blue"
@@ -212,6 +245,17 @@ for d in ${analysisLayerPath}/*/ ; do
     fi
 done
 
+# demos
+moduleDir="demos"
+modulePath="${absoluteRootDir}/${moduleDir}"
+announcement=$(printf %-70s "Processing demos")
+writeLine
+writeLine "${announcement}" "White" "Blue"
+writeLine
+correctLineEndings "${modulePath}/install.dev.sh"
+source "${modulePath}/install.dev.sh"
+
+
 # libfontconfig1 is required for SkiaSharp, libgdplus is required for System.Drawing
 if [ "${verbosity}" == "quiet" ]; then
     write "Installing supporting image libraries..."

Installers/Dev/setup.dev.sh

@@ -355,7 +355,7 @@ shift & goto :%~1
         if not exist "!installPath!" (
 
             rem Params are:     S3 storage bucket |    fileToGet   | downloadToDir | dirToSaveTo  | message
-            call :DownloadAndExtract "%storageUrl%" "!pythonName!.zip" "!baseDir!"   "!pythonName!" "Downloading Python !pythonVersion! interpreter..."
+            call :DownloadAndExtract "%storageUrl%" "!pythonName!.zip" "!baseDir!"  "!pythonName!" "Downloading Python !pythonVersion! interpreter..."
             if exist "!downloadPath!\!platform!\!pythonName!" (
                 robocopy /e "!downloadPath!\!platform!\!pythonName! " "!installPath! " !roboCopyFlags! > NUL
             )
@@ -473,9 +473,9 @@ shift & goto :%~1
     REM python compatible version.
     call :Write "Ensuring Python package manager (pip) is installed..."
     if /i "%verbosity%" == "quiet" (
-        !pythonPath! -m ensurepip !pipFlags!  >nul 2>nul 
+        !pythonPath! -m ensurepip >nul 2>nul 
     ) else (
-        !pythonPath! -m ensurepip !pipFlags!
+        !pythonPath! -m ensurepip
     )
     call :WriteLine "Done" %color_success%
 

Installers/Dev/utils.bat

@@ -22,7 +22,7 @@ function Color () {
     local foreground=$1
     local background=$2
 
-    if [ "$foreground" == "" ];  then foreground='Default'; fi
+    if [ "$foreground" == "" ]; then foreground='Default'; fi
     if [ "$background" == "" ]; then background="$color_background"; fi
 
     if [ "$foreground" == 'Contrast' ]; then
@@ -1014,7 +1014,7 @@ function needRosettaAndiBrew () {
 
 function quit () {
 
-    if [ "${useColor}" == "true" ] && "${darkmode}" == "true" ]; then
+    if [ "${useColor}" == "true" ] && [ "${darkmode}" == "true" ]; then
         # this resets the terminal, but also clears the screen which isn't great
         # tput reset
         echo

Installers/Dev/utils.sh

@@ -0,0 +1,9 @@
+{
+  "commands": {
+    "BuildDockerBase": {
+      "fileName": "cmd.exe",
+      "workingDirectory": "./Installers/Docker",
+      "arguments": "/c BuildDockerBase.bat"
+    }
+  }
+}