From 346f2e75f6a70268a71b75a232235e62b6dcb950 Mon Sep 17 00:00:00 2001 From: Antoine <36314396+antoinepetty@users.noreply.github.com> Date: Wed, 9 Jun 2021 12:00:10 +0100 Subject: [PATCH 01/11] Change incoming Arduino data keys to match new IDs --- surface/constants/athena.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/surface/constants/athena.py b/surface/constants/athena.py index 2700a18..6c19390 100644 --- a/surface/constants/athena.py +++ b/surface/constants/athena.py @@ -23,10 +23,10 @@ } # Data received from Raspberry Pi DATA_RECEIVED = { - "A_O": False, - "A_I": False, - "S_O": 0, - "S_I": 0 + "A_A": False, + "A_B": False, + "S_A": 0, + "S_B": 0 } # Data that will be sent to Raspberry Pi DATA_TRANSMISSION = { From 54c8e2a490cf86474435d3375dfca61cab5b559c Mon Sep 17 00:00:00 2001 From: Antoine <36314396+antoinepetty@users.noreply.github.com> Date: Sun, 13 Jun 2021 14:08:38 +0100 Subject: [PATCH 02/11] messy add to readme to add static ip stuff --- README.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/README.md b/README.md index 9184b81..e5ab817 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,63 @@ # surface Surface code 2020-21 + +Setting the static ip + +On the Ubuntu Desktop +Note: This is for Ubuntu desktop. The interface for Mate may be different + +On the computer, which is connected to the Internet, click the network icon in the panel and go to "Edit Connections..." at the bottom of the menu. + +Edit Connection... + +Double click your Wired Connection (Leave your wireless connection untouched, the one connected to Internet and the one you want to share, as I understand). + +Network Connections Dialog + +On the "IPv4 Settings tab", select Method: "Shared to other computers" + +Editing Wired Connection + +Reconnect by clicking on the Wired Network, so it gets a new IP address. (The two computers must be connected by an ethernet cable for this step, so connect them now if you haven't already.) + +Click on "Connection Information" in the network menu and write down the IP address and network mask (in my case it was assigned 10.42.0.1/255.255.255.0 but I do not know if that will always be the case). + +Connection Information + +On the Raspberry Pi +Assign static IP to the Ethernet connection + +In Pi the WiFi device is called wlan but the ethernet device name is hard to guess. To find the device names use the command: + +$ ip link show + +The output will show your Ethernet device in Pi enxb827eb3d64cc + +Next we need to find the current IP addresses assigned to enxb827eb3d64cc: + +$ ip -4 addr show dev enxb827eb3d64cc | grep inet + +I get something like this, yours may be different: + +inet 10.42.0.211/24 brd 10.42.0.255 scope global enxb827eb3d64cc +You can keep the assigned IP address or choose a different one in the same subnet. Add the following lines at the end of /etc/dhcpcd.conf by: + +$ sudo nano /etc/dhcpcd.conf + +With the following content to make the assigned IP address static: + +# Custom static IP address for enxb827eb3d64cc +interface enxb827eb3d64cc +static ip_address=10.42.0.211/24 +static routers=10.42.0.255 +static domain_name_servers=10.42.0.255 +Change 10.42.0.211 above to 10.42.0.x where x is a number between 2 and 254 if you want to assign a different IP address. + +Reboot Pi to make the new IP address take effect: + +$ sudo reboot now + +Now you should be able to ssh from the desktop to the Pi with the following command: + +$ ssh pi@10.42.0.211 +Hope this helps \ No newline at end of file From fa46f145ad5da7ce27c80a86108efd1cd487281f Mon Sep 17 00:00:00 2001 From: TheCodeSummoner Date: Sat, 12 Jun 2021 19:38:54 +0200 Subject: [PATCH 03/11] Added control values management for gripper, cord, and micro ROV thruster. --- surface/constants/athena.py | 9 ++++++ surface/constants/control.py | 4 +++ surface/control/converter.py | 39 +++++++++++++++++++++++++- surface/control/manual.py | 42 +++++++++++++++++++++++++++- surface/control/model.py | 53 +++++++++++++++++++++++++++++++++++- 5 files changed, 144 insertions(+), 3 deletions(-) diff --git a/surface/constants/athena.py b/surface/constants/athena.py index 6c19390..ac45be4 100644 --- a/surface/constants/athena.py +++ b/surface/constants/athena.py @@ -50,18 +50,27 @@ CONTROL_MANAGER_NAME + "-sway": CONTROL_NORM_IDLE, CONTROL_MANAGER_NAME + "-surge": CONTROL_NORM_IDLE, CONTROL_MANAGER_NAME + "-heave": CONTROL_NORM_IDLE, + CONTROL_MANAGER_NAME + "-cord": CONTROL_NORM_IDLE, + CONTROL_MANAGER_NAME + "-gripper": CONTROL_NORM_IDLE, + CONTROL_MANAGER_NAME + "-micro": CONTROL_NORM_IDLE, CONTROL_MANUAL_NAME + "-yaw": CONTROL_NORM_IDLE, CONTROL_MANUAL_NAME + "-pitch": CONTROL_NORM_IDLE, CONTROL_MANUAL_NAME + "-roll": CONTROL_NORM_IDLE, CONTROL_MANUAL_NAME + "-sway": CONTROL_NORM_IDLE, CONTROL_MANUAL_NAME + "-surge": CONTROL_NORM_IDLE, CONTROL_MANUAL_NAME + "-heave": CONTROL_NORM_IDLE, + CONTROL_MANUAL_NAME + "-cord": CONTROL_NORM_IDLE, + CONTROL_MANUAL_NAME + "-gripper": CONTROL_NORM_IDLE, + CONTROL_MANUAL_NAME + "-micro": CONTROL_NORM_IDLE, CONTROL_AUTONOMOUS_NAME + "-yaw": CONTROL_NORM_IDLE, CONTROL_AUTONOMOUS_NAME + "-pitch": CONTROL_NORM_IDLE, CONTROL_AUTONOMOUS_NAME + "-roll": CONTROL_NORM_IDLE, CONTROL_AUTONOMOUS_NAME + "-sway": CONTROL_NORM_IDLE, CONTROL_AUTONOMOUS_NAME + "-surge": CONTROL_NORM_IDLE, CONTROL_AUTONOMOUS_NAME + "-heave": CONTROL_NORM_IDLE, + CONTROL_AUTONOMOUS_NAME + "-cord": CONTROL_NORM_IDLE, + CONTROL_AUTONOMOUS_NAME + "-gripper": CONTROL_NORM_IDLE, + CONTROL_AUTONOMOUS_NAME + "-micro": CONTROL_NORM_IDLE, RK_CONTROL_DRIVING_MODE: DrivingMode.MANUAL.value } # Other, un-classified data diff --git a/surface/constants/control.py b/surface/constants/control.py index 5f9c863..5a014a7 100644 --- a/surface/constants/control.py +++ b/surface/constants/control.py @@ -23,8 +23,12 @@ THRUSTER_MAX = 1900 THRUSTER_IDLE = 1500 THRUSTER_MIN = 1100 +GRIPPER_MAX = 1900 GRIPPER_IDLE = 1500 +GRIPPER_MIN = 1100 +CORD_MAX = 1900 CORD_IDLE = 1500 +CORD_MIN = 1100 # Joystick control boundary - any values below are considered 0 DEAD_ZONE = 1025 diff --git a/surface/control/converter.py b/surface/control/converter.py index 249f7ec..25f7e72 100644 --- a/surface/control/converter.py +++ b/surface/control/converter.py @@ -3,7 +3,8 @@ """ from typing import Dict from .common import normalise -from ..constants.control import CONTROL_NORM_IDLE, CONTROL_NORM_MAX, CONTROL_NORM_MIN, THRUSTER_MAX, THRUSTER_MIN +from ..constants.control import CONTROL_NORM_IDLE, CONTROL_NORM_MAX, CONTROL_NORM_MIN +from ..constants.control import THRUSTER_MAX, THRUSTER_MIN, GRIPPER_MAX, GRIPPER_MIN, CORD_MAX, CORD_MIN class Converter: @@ -25,6 +26,9 @@ def convert(motions: Dict[str, float]) -> Dict[str, int]: "T_VFS": Converter._thruster_vfs(motions), "T_VAP": Converter._thruster_vap(motions), "T_VAS": Converter._thruster_vas(motions), + "T_M": Converter._thruster_micro(motions), + "M_C": Converter._cord(motions), + "M_G": Converter._gripper(motions), } @staticmethod @@ -227,6 +231,39 @@ def _thruster_vas(motions: Dict[str, float]) -> int: return Converter._to_thruster_value(value) + @staticmethod + def _thruster_micro(motions: Dict[str, float]) -> int: + """ + Hierarchical control for micro ROV thruster. + """ + micro = motions["micro"] + + value = micro if micro else CONTROL_NORM_IDLE + + return Converter._to_thruster_value(value) + + @staticmethod + def _cord(motions: Dict[str, float]) -> int: + """ + Hierarchical control for cord control. + """ + cord = motions["cord"] + + value = cord if cord else CONTROL_NORM_IDLE + + return int(normalise(value, CONTROL_NORM_MIN, CONTROL_NORM_MAX, CORD_MIN, CORD_MAX)) + + @staticmethod + def _gripper(motions: Dict[str, float]) -> int: + """ + Hierarchical control for gripper control. + """ + gripper = motions["gripper"] + + value = gripper if gripper else CONTROL_NORM_IDLE + + return int(normalise(value, CONTROL_NORM_MIN, CONTROL_NORM_MAX, GRIPPER_MIN, GRIPPER_MAX)) + @staticmethod def _to_thruster_value(value: float) -> int: """ diff --git a/surface/control/manual.py b/surface/control/manual.py index 6307108..969f0e6 100644 --- a/surface/control/manual.py +++ b/surface/control/manual.py @@ -190,6 +190,7 @@ def hat_x(self, value: int): Set state of joystick's horizontal hat. """ self._hat_x = value + self._update_cord() @property def hat_y(self) -> int: @@ -204,6 +205,7 @@ def hat_y(self, value: int): Set state of joystick's vertical hat. """ self._hat_y = value + self._update_micro_thruster() @property def button_a(self) -> bool: @@ -218,6 +220,7 @@ def button_a(self, value: bool): Set state of joystick's button A. """ self._button_a = value + self._update_gripper() @property def button_b(self) -> bool: @@ -262,6 +265,7 @@ def button_y(self, value: bool): Set state of joystick's button Y. """ self._button_y = value + self._update_gripper() @property def button_lb(self) -> bool: @@ -402,6 +406,39 @@ def _update_heave(self): else: self.heave = CONTROL_NORM_IDLE + def _update_cord(self): + """ + Cord is determined by the horizontal hat value. + """ + if self.hat_x > 0: + self.cord = CONTROL_NORM_MAX + elif self.hat_x < 0: + self.cord = CONTROL_NORM_MIN + else: + self.cord = CONTROL_NORM_IDLE + + def _update_gripper(self): + """ + Gripper is determined by the buttons Y and A. + """ + if self.button_y: + self.gripper = CONTROL_NORM_MAX + elif self.button_a: + self.gripper = CONTROL_NORM_MIN + else: + self.gripper = CONTROL_NORM_IDLE + + def _update_micro_thruster(self): + """ + Micro ROV thruster is determined by the vertical hat value. + """ + if self.hat_y < 0: + self.micro = CONTROL_NORM_MAX + elif self.hat_y > 0: + self.micro = CONTROL_NORM_MIN + else: + self.micro = CONTROL_NORM_IDLE + def _update_mode(self): """ Mode can be switched between manual and assisted only. @@ -417,7 +454,10 @@ def _update_mode(self): "roll": CONTROL_NORM_IDLE, "sway": CONTROL_NORM_IDLE, "surge": CONTROL_NORM_IDLE, - "heave": CONTROL_NORM_IDLE + "heave": CONTROL_NORM_IDLE, + "cord": CONTROL_NORM_IDLE, + "gripper": CONTROL_NORM_IDLE, + "micro": CONTROL_NORM_IDLE, } elif self.button_select: self.mode = DrivingMode.ASSISTED diff --git a/surface/control/model.py b/surface/control/model.py index 007c512..2458c99 100644 --- a/surface/control/model.py +++ b/surface/control/model.py @@ -28,6 +28,9 @@ def __init__(self, name: str): self._sway = Motion("sway") self._surge = Motion("surge") self._heave = Motion("heave") + self._cord = Motion("cord") + self._gripper = Motion("gripper") + self._micro = Motion("micro") @property def yaw(self) -> float: @@ -113,6 +116,48 @@ def heave(self, value: float): """ self._heave.value = value + @property + def cord(self) -> float: + """ + Get current cord value. + """ + return self._cord.value + + @cord.setter + def cord(self, value: float): + """ + Set current cord value. + """ + self._cord.value = value + + @property + def gripper(self) -> float: + """ + Get current gripper value. + """ + return self._gripper.value + + @gripper.setter + def gripper(self, value: float): + """ + Set current gripper value. + """ + self._gripper.value = value + + @property + def micro(self) -> float: + """ + Get current micro ROV thruster value. + """ + return self._micro.value + + @micro.setter + def micro(self, value: float): + """ + Set current micro ROV thruster value. + """ + self._micro.value = value + @property def mode(self) -> DrivingMode: """ @@ -139,7 +184,10 @@ def motions(self) -> Dict[str, float]: "roll": self.roll, "sway": self.sway, "surge": self.surge, - "heave": self.heave + "heave": self.heave, + "cord": self.cord, + "gripper": self.gripper, + "micro": self.micro } @motions.setter @@ -155,6 +203,9 @@ def motions(self, values: dict): self.sway = values["sway"] self.surge = values["surge"] self.heave = values["heave"] + self.cord = values["cord"] + self.gripper = values["gripper"] + self.micro = values["micro"] @property def keys(self) -> Set[str]: From 778b0adb88b00e161ae301d1bc7fea1abe6705d8 Mon Sep 17 00:00:00 2001 From: Antoine <36314396+antoinepetty@users.noreply.github.com> Date: Sat, 12 Jun 2021 21:11:40 +0100 Subject: [PATCH 04/11] Update the min-max of gripper and cord --- surface/constants/control.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/surface/constants/control.py b/surface/constants/control.py index 5a014a7..7a0de7f 100644 --- a/surface/constants/control.py +++ b/surface/constants/control.py @@ -23,12 +23,12 @@ THRUSTER_MAX = 1900 THRUSTER_IDLE = 1500 THRUSTER_MIN = 1100 -GRIPPER_MAX = 1900 +GRIPPER_MAX = 1600 GRIPPER_IDLE = 1500 -GRIPPER_MIN = 1100 -CORD_MAX = 1900 +GRIPPER_MIN = 1400 +CORD_MAX = 1600 CORD_IDLE = 1500 -CORD_MIN = 1100 +CORD_MIN = 1400 # Joystick control boundary - any values below are considered 0 DEAD_ZONE = 1025 From c5437e41764e1ca69cad1dea152c2dac4ad33199 Mon Sep 17 00:00:00 2001 From: TheCodeSummoner Date: Sat, 10 Jul 2021 17:26:21 +0200 Subject: [PATCH 05/11] Fixed hardware trigger max value (1023 instead of 255) --- surface/constants/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surface/constants/control.py b/surface/constants/control.py index 7a0de7f..efc7fc3 100644 --- a/surface/constants/control.py +++ b/surface/constants/control.py @@ -36,7 +36,7 @@ # Hardware and the expected min/max values HARDWARE_AXIS_MAX = 32767 HARDWARE_AXIS_MIN = -32768 -HARDWARE_TRIGGER_MAX = 255 +HARDWARE_TRIGGER_MAX = 1023 HARDWARE_TRIGGER_MIN = 0 INTENDED_AXIS_MAX = CONTROL_NORM_MAX INTENDED_AXIS_MIN = CONTROL_NORM_MIN From e464d99aee1ecf668eb3b4bec34751f75bd8c564 Mon Sep 17 00:00:00 2001 From: TheCodeSummoner Date: Sat, 10 Jul 2021 17:36:14 +0200 Subject: [PATCH 06/11] Added a working version of the main execution file --- surface/__main__.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/surface/__main__.py b/surface/__main__.py index 386de6a..56e1562 100644 --- a/surface/__main__.py +++ b/surface/__main__.py @@ -1,3 +1,29 @@ """ Surface control station execution file. """ +import os +import sys +import time + +# Make sure a local raspberry-pi package can be found and overrides any installed versions +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) + +# pylint: disable = wrong-import-position +from surface.networking import Connection +from surface.control import ControlManager, ManualController +from surface.enums import ConnectionStatus + +if __name__ == '__main__': + control_manager_pid = ControlManager().start() + manual_controller_pid = ManualController().start() + connection = Connection() + + while True: + status = connection.status + + if status == ConnectionStatus.DISCONNECTED: + connection.connect() + elif status == ConnectionStatus.IDLE: + connection.reconnect() + + time.sleep(1) From e8d77699067879bbb58c2fee373f7598a84a3107 Mon Sep 17 00:00:00 2001 From: TheCodeSummoner Date: Sat, 10 Jul 2021 17:45:22 +0200 Subject: [PATCH 07/11] Better file hierarchy (created an assets folder) --- setup.py | 9 +++++---- surface/__init__.py | 19 +++++++++++++++++++ surface/{res => assets}/log-config.json | 0 surface/{ => assets}/log/.keep | 0 surface/{res => assets}/metadata.json | 0 surface/constants/common.py | 6 +++--- 6 files changed, 27 insertions(+), 7 deletions(-) rename surface/{res => assets}/log-config.json (100%) rename surface/{ => assets}/log/.keep (100%) rename surface/{res => assets}/metadata.json (100%) diff --git a/setup.py b/setup.py index d70c0a5..67e78cf 100644 --- a/setup.py +++ b/setup.py @@ -6,15 +6,16 @@ import setuptools # Fetch the root folder to specify absolute paths to the "include" files -ROOT = os.path.normpath(os.path.dirname(__file__)) +ASSETS_DIR = os.path.join(os.path.normpath(os.path.dirname(__file__)), "surface", "assets") # Specify which files should be added to the installation PACKAGE_DATA = [ - os.path.join(ROOT, "surface", "res", "metadata.json"), - os.path.join(ROOT, "surface", "log", ".keep") + os.path.join(ASSETS_DIR, "metadata.json"), + os.path.join(ASSETS_DIR, "log-config.json"), + os.path.join(ASSETS_DIR, "log", ".keep") ] -with open(os.path.join(ROOT, "surface", "res", "metadata.json")) as f: +with open(os.path.join(ASSETS_DIR, "metadata.json")) as f: metadata = json.load(f) setuptools.setup( diff --git a/surface/__init__.py b/surface/__init__.py index 0158319..c4b9732 100644 --- a/surface/__init__.py +++ b/surface/__init__.py @@ -1,8 +1,21 @@ """ Surface control station package. """ +import os +import json from . import control, networking, vision, athena from .exceptions import SurfaceException +from .constants.common import ASSETS_DIR + +with open(os.path.join(ASSETS_DIR, "metadata.json")) as f: + metadata = json.load(f) + +__title__ = metadata["__title__"], +__description__ = metadata["__description__"], +__version__ = metadata["__version__"], +__lead__ = metadata["__lead__"], +__email__ = metadata["__email__"], +__url__ = metadata["__url__"] __all__ = [ "control", @@ -10,4 +23,10 @@ "vision", "athena", "SurfaceException", + "__title__", + "__description__", + "__version__", + "__lead__", + "__email__", + "__url__", ] diff --git a/surface/res/log-config.json b/surface/assets/log-config.json similarity index 100% rename from surface/res/log-config.json rename to surface/assets/log-config.json diff --git a/surface/log/.keep b/surface/assets/log/.keep similarity index 100% rename from surface/log/.keep rename to surface/assets/log/.keep diff --git a/surface/res/metadata.json b/surface/assets/metadata.json similarity index 100% rename from surface/res/metadata.json rename to surface/assets/metadata.json diff --git a/surface/constants/common.py b/surface/constants/common.py index 7129886..d200459 100644 --- a/surface/constants/common.py +++ b/surface/constants/common.py @@ -7,13 +7,13 @@ # Declare paths to relevant folders - tests folder shouldn't be known here ROOT_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) SURFACE_DIR = os.path.join(ROOT_DIR, "surface") -RES_DIR = os.path.join(SURFACE_DIR, "res") +ASSETS_DIR = os.path.join(SURFACE_DIR, "assets") LOG_DIR = os.path.join(SURFACE_DIR, "log") # Load the environment variables from the root folder and/or the resources folder dotenv.load_dotenv(dotenv_path=os.path.join(ROOT_DIR, ".env")) -dotenv.load_dotenv(dotenv_path=os.path.join(RES_DIR, ".env")) +dotenv.load_dotenv(dotenv_path=os.path.join(ASSETS_DIR, ".env")) # Declare logging config - use .env file to override the defaults -LOG_CONFIG_PATH = os.getenv("LOG_CONFIG_PATH", os.path.join(RES_DIR, "log-config.json")) +LOG_CONFIG_PATH = os.getenv("LOG_CONFIG_PATH", os.path.join(ASSETS_DIR, "log-config.json")) LOGGER_NAME = os.getenv("LOGGER_NAME", "surface") From 6d5603e60a9a157fa075ccba951430fabbedaf11 Mon Sep 17 00:00:00 2001 From: TheCodeSummoner Date: Sun, 11 Jul 2021 15:27:55 +0200 Subject: [PATCH 08/11] Fixed linting errors and less significant code issues --- surface/__init__.py | 10 +++++----- surface/constants/common.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/surface/__init__.py b/surface/__init__.py index c4b9732..84afdab 100644 --- a/surface/__init__.py +++ b/surface/__init__.py @@ -10,11 +10,11 @@ with open(os.path.join(ASSETS_DIR, "metadata.json")) as f: metadata = json.load(f) -__title__ = metadata["__title__"], -__description__ = metadata["__description__"], -__version__ = metadata["__version__"], -__lead__ = metadata["__lead__"], -__email__ = metadata["__email__"], +__title__ = metadata["__title__"] +__description__ = metadata["__description__"] +__version__ = metadata["__version__"] +__lead__ = metadata["__lead__"] +__email__ = metadata["__email__"] __url__ = metadata["__url__"] __all__ = [ diff --git a/surface/constants/common.py b/surface/constants/common.py index d200459..d69fe7a 100644 --- a/surface/constants/common.py +++ b/surface/constants/common.py @@ -8,11 +8,11 @@ ROOT_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) SURFACE_DIR = os.path.join(ROOT_DIR, "surface") ASSETS_DIR = os.path.join(SURFACE_DIR, "assets") -LOG_DIR = os.path.join(SURFACE_DIR, "log") +LOG_DIR = os.path.join(ASSETS_DIR, "log") # Load the environment variables from the root folder and/or the resources folder -dotenv.load_dotenv(dotenv_path=os.path.join(ROOT_DIR, ".env")) dotenv.load_dotenv(dotenv_path=os.path.join(ASSETS_DIR, ".env")) +dotenv.load_dotenv(dotenv_path=os.path.join(ROOT_DIR, ".env")) # Declare logging config - use .env file to override the defaults LOG_CONFIG_PATH = os.getenv("LOG_CONFIG_PATH", os.path.join(ASSETS_DIR, "log-config.json")) From 3b74f716f878ceb7d1a43e684cafb004708e2f2d Mon Sep 17 00:00:00 2001 From: Mateusz Radzikowski Date: Mon, 12 Jul 2021 16:20:34 +0200 Subject: [PATCH 09/11] Add constants for colours of changes for reef coral task --- surface/constants/vision.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/surface/constants/vision.py b/surface/constants/vision.py index 23b9f27..649c0d1 100644 --- a/surface/constants/vision.py +++ b/surface/constants/vision.py @@ -1,3 +1,16 @@ """ Computer vision constants. """ + +# Colors for borders of found changes in coral reef task +# Bleaching changes +RED = (0, 0, 255) + +# Growth changes +GREEN = (0, 255, 0) + +# Recovery changes +BLUE = (255, 0, 0) + +# Death changes +YELLOW = (0, 255, 255) From 3192db0efb6ca5ac0fb4604bd63b46206ed70ce5 Mon Sep 17 00:00:00 2001 From: Mateusz Radzikowski Date: Mon, 12 Jul 2021 16:24:52 +0200 Subject: [PATCH 10/11] Add algorithm for solving reef coral task Module is divided into preprocessing part and the one with the main logic and functionality. determine_coral_reef_changes is used to combine all logic and preprocessing parts. The key to solve it was to get white and pink parts of the coral reef and then work on them (using bitwise operations). We needed to align two pictures to have similar perspective (align_photos function). --- surface/vision/coral_reef.py | 230 +++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 surface/vision/coral_reef.py diff --git a/surface/vision/coral_reef.py b/surface/vision/coral_reef.py new file mode 100644 index 0000000..6a7062d --- /dev/null +++ b/surface/vision/coral_reef.py @@ -0,0 +1,230 @@ +""" +Coral reef task. + +Module storing an implementation of the coral reef task. +""" +import cv2 +import numpy as np +import imutils +from ..constants.vision import RED, GREEN, BLUE, YELLOW + + +# Preprocessing part of the module +def align_photos(img_before: np.ndarray, img_now: np.ndarray, img_before_color: np.ndarray) -> np.ndarray: + """ + Align photo from before and now to have the same perspective. + + :param img_before: image given as before converted into grayscale + :param img_now: image taken now converted into grayscale + :param img_before_color: original image given as before + :return: transformed (aligned) image in color + """ + # Create ORB detector with 5000 features + # Find key points and descriptors. The first arg is the image, second arg is the mask, that is not required. + kp1, first_descriptor = cv2.ORB_create(5000).detectAndCompute(img_before, None) + kp2, second_descriptor = cv2.ORB_create(5000).detectAndCompute(img_now, None) + + # Match features between the two images. We create a Brute Force matcher with hamming distance as measurement mode. + # Match the two sets of descriptors. + matches = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True).match(first_descriptor, second_descriptor) + + # Sort matches on the basis of their Hamming distance. + matches.sort(key=lambda x: x.distance) + + # Take the top 90 % matches forward. + matches = matches[:int(len(matches) * 90)] + + # Define empty matrices of shape no_of_matches * 2. + empty_first = np.zeros((len(matches), 2)) + empty_second = np.zeros((len(matches), 2)) + + for idx, match in enumerate(matches): + empty_first[idx, :] = kp1[match.queryIdx].pt + empty_second[idx, :] = kp2[match.trainIdx].pt + + # Find the homography matrix. + homography, _ = cv2.findHomography(empty_first, empty_second, cv2.RANSAC) + + # Use this matrix to transform the colored image wrt the reference image. + transformed_img = cv2.warpPerspective(img_before_color, homography, (img_now.shape[1], img_now.shape[0])) + + return transformed_img + + +def get_white_masked_photo(img_bgr: np.ndarray) -> np.ndarray: + """ + Get masked white parts of coral reef from the photo. + + :param img_bgr: Image as numpy array + :return: Masked white parts of image + """ + # Limits for white color used in inRange function + lower_white = np.array([180, 180, 80]) + upper_white = np.array([255, 255, 255]) + + # Kept bgr image for masking the photo + mask = cv2.inRange(img_bgr, lower_white, upper_white) + + # Postprocessing of masked image, closing small holes inside the object + kernel = np.ones((3, 3), np.uint8) + closing = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel=kernel, iterations=3) + + return closing + + +def get_pink_masked_photo(img_bgr: np.ndarray) -> np.ndarray: + """ + Get masked pink parts of coral reef from the photo. + + :param img_bgr: Image as numpy array + :return: Masked pink parts of image + """ + # Limits for pink color used in inRange function + lower_pink = np.array([100, 50, 130]) + upper_pink = np.array([240, 200, 230]) + + # Used preprocessing with converting into hsv, because for pink colors it gives better results + img_hsv = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2HSV) + + mask = cv2.inRange(img_hsv, lower_pink, upper_pink) + + # Postprocessing of masked image, closing small holes inside the object + kernel = np.ones((3, 3), np.uint8) + closing = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel=kernel, iterations=5) + + return closing + + +def find_contours(image_masked: np.ndarray) -> list: + """ + Retrieve information (points, width and height) of specific changes and stores them in list. + + :param image_masked: Masked image + :return: List of bounding boxes. + """ + contours = cv2.findContours(image_masked, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + contours = imutils.grab_contours(contours) + array_of_bounding_boxes = [] + for contour in contours: + area = cv2.contourArea(contour) + area_min = 800 + area_max = 4000 + if area_min <= area <= area_max: + (x_point, y_point, width, height) = cv2.boundingRect(contour) + array_of_bounding_boxes.append((x_point, y_point, width, height)) + return array_of_bounding_boxes + + +# Main functionality and logic of the task +def draw_bounding_boxes(boxes: list, colour: tuple, image_now: np.ndarray) -> np.ndarray: + """ + Draw bounding boxes on image. + + :param boxes: List of bounding boxes + :param colour: Tuple of colour that is used to colour border of bounding box + :param image_now: Image taken underwater + :return: Image taken now with bounding boxes on it + """ + for box in boxes: + x_point = box[0] + y_point = box[1] + width = box[2] + height = box[3] + cv2.rectangle(image_now, (x_point, y_point), (x_point + width, y_point + height), colour, 2) + return image_now + + +def get_bounding_boxes(recovery_mask: np.ndarray, growth_mask: np.ndarray, + death_mask: np.ndarray, bleaching_mask: np.ndarray, image_now: np.ndarray) -> np.ndarray: + """ + Get any changes produced from image before and now. + + :param recovery_mask: Masked photo of recovery changes. + :param growth_mask: Masked photo of growth changes. + :param death_mask: Masked photo of death changes. + :param bleaching_mask: Masked photo of bleaching changes. + :param image_now: + :return: + """ + # Counter for keeping track of found changes, (only two types changes for image) + counter = 0 + + # Finding the bounding boxes for changes + recovery_bounding_boxes = find_contours(recovery_mask) + growth_bounding_boxes = find_contours(growth_mask) + death_bounding_boxes = find_contours(death_mask) + bleaching_bounding_boxes = find_contours(bleaching_mask) + + if recovery_bounding_boxes: + counter += 1 + image_now = draw_bounding_boxes(recovery_bounding_boxes, BLUE, image_now) + + if growth_bounding_boxes: + counter += 1 + image_now = draw_bounding_boxes(growth_bounding_boxes, GREEN, image_now) + + if death_bounding_boxes and counter < 2: + counter += 1 + image_now = draw_bounding_boxes(death_bounding_boxes, YELLOW, image_now) + + if bleaching_bounding_boxes and counter < 2: + counter += 1 + image_now = draw_bounding_boxes(bleaching_bounding_boxes, RED, image_now) + return image_now + + +def determine_coral_reef_change() -> np.ndarray: + """ + Process images and find the changes of the coral reef between two images. + + :return: Image with outlined changes + """ + # Loading before and now images + image_before = cv2.imread("photos/before.png") + image_now = cv2.imread("photos/now.png") + + image_before = cv2.resize(image_before, (480, 360)) + image_now = cv2.resize(image_now, (480, 360)) + + # Preprocessing images with bilateral filter to preserve edges + image_before = cv2.bilateralFilter(image_before, 3, 75, 75) + image_now = cv2.bilateralFilter(image_now, 3, 75, 75) + + # Converting images into grayscale + gray_before = cv2.cvtColor(image_before, cv2.COLOR_BGR2GRAY) + gray_now = cv2.cvtColor(image_now, cv2.COLOR_BGR2GRAY) + + transformed_image = align_photos(gray_before, gray_now, image_before) + + # Get masked white parts of image now and before (already aligned) + img_masked_white = get_white_masked_photo(image_now) + img_masked_white_before_aligned = get_white_masked_photo(transformed_image) + + # Get masked pink parts of image now and before (already aligned) + pink_masked_now = get_pink_masked_photo(image_now) + pink_masked_aligned = get_pink_masked_photo(transformed_image) + + # Get masked pink and white parts of before and now images + pink_and_white_now = cv2.bitwise_or(img_masked_white, pink_masked_now) + pink_and_white_before = cv2.bitwise_or(img_masked_white_before_aligned, pink_masked_aligned) + + # Recovery changes are when white parts from before become pink (BITWISE AND OPERATION) + recovery_changes = cv2.bitwise_and(pink_masked_now, img_masked_white_before_aligned) + + # Bleaching changes are when pink parts from before become white (BITWISE AND OPERATION) + bleaching_changes = cv2.bitwise_and(pink_masked_aligned, img_masked_white) + + # Growth changes are when there are only new pink parts that did not belong to before image and postprocessing + growth_changes = pink_masked_now - pink_masked_aligned - recovery_changes + bleaching_changes + growth_changes = cv2.morphologyEx(growth_changes, cv2.MORPH_OPEN, np.ones((7, 7), np.uint8), 10) + + # Missing parts of coral reef and postprocessing + death_changes = pink_and_white_before - pink_and_white_now + death_changes = cv2.morphologyEx(death_changes, cv2.MORPH_OPEN, np.ones((7, 7), np.uint8), 10) + + return get_bounding_boxes(recovery_changes, growth_changes, death_changes, bleaching_changes, image_now) + + +if __name__ == "__main__": + cv2.imshow("CHANGES", determine_coral_reef_change()) + cv2.waitKey(0) From e71259d295fcf6b4839fb424068ee306b35b0fc8 Mon Sep 17 00:00:00 2001 From: Mateusz Radzikowski Date: Mon, 12 Jul 2021 16:32:51 +0200 Subject: [PATCH 11/11] Delete imutils package that is not necessary for finding contours --- surface/vision/coral_reef.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/surface/vision/coral_reef.py b/surface/vision/coral_reef.py index 6a7062d..ad4d82f 100644 --- a/surface/vision/coral_reef.py +++ b/surface/vision/coral_reef.py @@ -5,7 +5,6 @@ """ import cv2 import numpy as np -import imutils from ..constants.vision import RED, GREEN, BLUE, YELLOW @@ -102,8 +101,7 @@ def find_contours(image_masked: np.ndarray) -> list: :param image_masked: Masked image :return: List of bounding boxes. """ - contours = cv2.findContours(image_masked, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - contours = imutils.grab_contours(contours) + contours, _ = cv2.findContours(image_masked, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) array_of_bounding_boxes = [] for contour in contours: area = cv2.contourArea(contour)