From 8bcef6693f2b027c4b43f9bccb6acbd0e9b96937 Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Tue, 6 Jul 2021 08:31:18 -0700 Subject: [PATCH 1/6] CommandDef: Boolean value example added WIP Adding boolean values to CommandDefintion configuration examples and tests. Signed-off-by: Josh Roppo --- satellite/example_commands.json | 4 +++- satellite/satellite.py | 10 ++++++++++ tests/satellite/test_sat.py | 17 +++++++++++++++-- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/satellite/example_commands.json b/satellite/example_commands.json index 30f35d4..9ea6d3d 100644 --- a/satellite/example_commands.json +++ b/satellite/example_commands.json @@ -37,7 +37,9 @@ "display_name": "6. Update File List", "description": "Gets the latest list of downloadable files from the spacecraft.", "tags": ["files", "operations"], - "fields": [] + "fields": [ + {"name": "show_hidden", "type": "boolean", "default": false } + ] }, "uplink_file": { "display_name": "Uplink File", diff --git a/satellite/satellite.py b/satellite/satellite.py index c76a4be..4d361ad 100644 --- a/satellite/satellite.py +++ b/satellite/satellite.py @@ -105,8 +105,18 @@ def process_command(self, bytes, gateway): "metadata": {"type": "image", "lat": (randint(-89, 89) + .0001*randint(0, 9999)), "lng": (randint(-179, 179) + .0001*randint(0, 9999))} }) + if self.command.fields['show_hidden']: + for i in range(1, randint(1, 3)): + self.file_list.append({ + "name": f'.thumb-{(len(self.file_list)+1):04d}.png', + "size": randint(200, 300), + "timestamp": int(time.time() * 1000) + i*10, + "metadata": {"type": "image", "lat": (randint(-89, 89) + .0001*randint(0, 9999)), "lng": (randint(-179, 179) + .0001*randint(0, 9999))} + }) + self.check_cancelled(id=command.id) logger.info("Files: {}".format(self.file_list)) + r = Timer(0.1, gateway.update_file_list, kwargs={"system":self.name, "files":self.file_list}) r.start() time.sleep(10) diff --git a/tests/satellite/test_sat.py b/tests/satellite/test_sat.py index d9fa06a..78bea51 100644 --- a/tests/satellite/test_sat.py +++ b/tests/satellite/test_sat.py @@ -1,5 +1,18 @@ import sys -from satellite import satellite +import json + +from majortom_gateway import command +from satellite import Satellite def test_stub(): - assert True \ No newline at end of file + assert True + +def test_command_processing(): + # To make it easier to interact with this Gateway, we are going to configure a bunch of commands for a satellite + # called "Example FlatSat". Please see the associated json file to see the list of commands. + # logger.debug("Setting up Example Flatsat satellite and associated commands") + with open('../../satellite/example_commands.json','r') as f: + command_bytes = f.read() + s = Satellite() + s.process_command(command_bytes) + From 673993d32b47d62d09294c3765c2026b3894ea14 Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Wed, 7 Jul 2021 07:07:34 -0700 Subject: [PATCH 2/6] Package structure update Tests were picking up the import cycle between Gateway<->satellite packages. Over-separated functionality into separate packages. WIP Signed-off-by: Josh Roppo --- commands/__init__.py | 1 + {gateway => commands}/statuses.py | 0 demo/__init__.py | 2 + demo/demo_sat.py | 4 +- gateway/__init__.py | 1 + gateway/gateway.py | 6 +-- run.py | 4 +- satellite/__init__.py | 2 + satellite/satellite.py | 12 +++-- test_requirements.txt | 6 +++ tests/__init__.py | 0 tests/satellite/test_sat.py | 18 ------- tests/test_sat.py | 83 +++++++++++++++++++++++++++++++ transform/__init__.py | 1 + {gateway => transform}/stubs.py | 0 15 files changed, 111 insertions(+), 29 deletions(-) create mode 100644 commands/__init__.py rename {gateway => commands}/statuses.py (100%) create mode 100644 test_requirements.txt create mode 100644 tests/__init__.py delete mode 100644 tests/satellite/test_sat.py create mode 100644 tests/test_sat.py create mode 100644 transform/__init__.py rename {gateway => transform}/stubs.py (100%) diff --git a/commands/__init__.py b/commands/__init__.py new file mode 100644 index 0000000..c1cae88 --- /dev/null +++ b/commands/__init__.py @@ -0,0 +1 @@ +from .statuses import CommandStatus \ No newline at end of file diff --git a/gateway/statuses.py b/commands/statuses.py similarity index 100% rename from gateway/statuses.py rename to commands/statuses.py diff --git a/demo/__init__.py b/demo/__init__.py index a7ede05..188fc47 100644 --- a/demo/__init__.py +++ b/demo/__init__.py @@ -1 +1,3 @@ +from .demo_sat import DemoSat, CommandCancelledError +from .demo_telemetry import DemoTelemetry name = "demo" diff --git a/demo/demo_sat.py b/demo/demo_sat.py index 97b4d21..f21a7e9 100644 --- a/demo/demo_sat.py +++ b/demo/demo_sat.py @@ -48,7 +48,9 @@ def __init__(self, name="Space Oddity"): "display_name": "Update File List", "description": "Downlinks the latest file list from the spacecraft.", "tags": ["files", "operations"], - "fields": [] + "fields": [ + {"name": "show_hidden", "type": "boolean", "default": False } + ] }, "uplink_file": { "display_name": "Uplink File", diff --git a/gateway/__init__.py b/gateway/__init__.py index e69de29..f3a0ead 100644 --- a/gateway/__init__.py +++ b/gateway/__init__.py @@ -0,0 +1 @@ +from .gateway import Gateway, CommandStatus \ No newline at end of file diff --git a/gateway/gateway.py b/gateway/gateway.py index 8b71b5c..303b53e 100644 --- a/gateway/gateway.py +++ b/gateway/gateway.py @@ -2,9 +2,9 @@ import time from asgiref.sync import async_to_sync from random import randint -from . import stubs -from .statuses import CommandStatus -from satellite.satellite import Satellite +from transform import stubs +from commands import CommandStatus +from satellite import Satellite logger = logging.getLogger(__name__) diff --git a/run.py b/run.py index 573b90b..676376c 100644 --- a/run.py +++ b/run.py @@ -2,8 +2,8 @@ import asyncio import argparse import json -from gateway.gateway import Gateway -from demo.demo_sat import DemoSat +from gateway import Gateway +from demo import DemoSat from majortom_gateway import GatewayAPI logger = logging.getLogger(__name__) diff --git a/satellite/__init__.py b/satellite/__init__.py index e69de29..8e47211 100644 --- a/satellite/__init__.py +++ b/satellite/__init__.py @@ -0,0 +1,2 @@ +from .satellite import Satellite, CommandCancelledError +from .telemetry import FakeTelemetry \ No newline at end of file diff --git a/satellite/satellite.py b/satellite/satellite.py index 4d361ad..a8e1608 100644 --- a/satellite/satellite.py +++ b/satellite/satellite.py @@ -4,13 +4,15 @@ It takes the place of a simulator, flatsat, engineering model, or real satellite. ''' import time -from threading import Timer -from gateway import stubs -from gateway.statuses import CommandStatus -from satellite.telemetry import FakeTelemetry -from random import randint import logging +from random import randint +from threading import Timer + +from transform import stubs +from commands import CommandStatus +from .telemetry import FakeTelemetry + logger = logging.getLogger(__name__) class CommandCancelledError(RuntimeError): diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 0000000..e2e7817 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,6 @@ +mock>=4.0.3 +pytest-asyncio>=0.14.0 +pytest-only>=1.2.2 +pytest-watch>=4.2.0 +pytest>=6.2.2 +virtualenv diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/satellite/test_sat.py b/tests/satellite/test_sat.py deleted file mode 100644 index 78bea51..0000000 --- a/tests/satellite/test_sat.py +++ /dev/null @@ -1,18 +0,0 @@ -import sys -import json - -from majortom_gateway import command -from satellite import Satellite - -def test_stub(): - assert True - -def test_command_processing(): - # To make it easier to interact with this Gateway, we are going to configure a bunch of commands for a satellite - # called "Example FlatSat". Please see the associated json file to see the list of commands. - # logger.debug("Setting up Example Flatsat satellite and associated commands") - with open('../../satellite/example_commands.json','r') as f: - command_bytes = f.read() - s = Satellite() - s.process_command(command_bytes) - diff --git a/tests/test_sat.py b/tests/test_sat.py new file mode 100644 index 0000000..6e21b6f --- /dev/null +++ b/tests/test_sat.py @@ -0,0 +1,83 @@ +import sys +import json +import unittest +from unittest.mock import call +import pytest +import asyncio + +try: + # Python 3.8+ + from unittest.mock import AsyncMock +except ImportError: + # Python 3.6+ + from mock import AsyncMock + +from majortom_gateway.command import Command +from majortom_gateway import gateway_api +from satellite import Satellite +from transform import stubs +from gateway import Gateway + +class TypeMatcher: + def __init__(self, expected_type): + self.expected_type = expected_type + + def __eq__(self, other): + return isinstance(other, self.expected_type) + +@pytest.fixture +def callback_mock(): + future = asyncio.Future() + future.set_result(None) + fn = AsyncMock(return_value=future) + return fn + + +#@pytest.mark.asyncio examplej test +#async def test_calls_transit_callback(callback_mock): +# gw = GatewayAPI("host", "gateway_token", transit_callback=callback_mock) +# # ToDo: Update this example message +# message = { +# "type": "transit", +# } +# +# res = await gw.handle_message(json.dumps(message)) +# +# # The transit callback is given the raw message +# callback_mock.assert_called_once_with(message) + + +class TestCommandDefinitions(unittest.TestCase): + def setUp(self): + self.gw_api = gateway_api.GatewayAPI("host", "gateway_token", + command_callback=callback_mock, transit_callback=callback_mock) + self.gateway = Gateway(api=self.gw_api) + + def test_process_command(self): + system = "Example Flatsat" # Match 'system' identifier with Gateway CommandDef + json_bytes = """{ + "type": "command", + "id": "update_file_list", + "system": "Example FlatSat", + "display_name": "Update File List", + "tags": ["files", "operations"], + "fields": [ + {"name": "show_hidden", "type": "boolean", "value": true } + ] + }""" + jl = json.loads(json_bytes) + cmd = Command(jl) + + cmd_bytes = stubs.translate_command_to_binary(cmd) + print("CMD: #{cmd_bytes}", cmd_bytes) + + + with open('satellite/example_commands.json','r') as f: + command_defs = json.loads(f.read()) + asyncio.ensure_future(self.gw_api.update_command_definitions( + system="Example FlatSat", + definitions=command_defs["definitions"])) + + resp = "" + self.gateway.satellite_response(cmd_bytes, resp) + print("command resp: {}", resp) \ No newline at end of file diff --git a/transform/__init__.py b/transform/__init__.py new file mode 100644 index 0000000..adbaf5c --- /dev/null +++ b/transform/__init__.py @@ -0,0 +1 @@ +from .stubs import * \ No newline at end of file diff --git a/gateway/stubs.py b/transform/stubs.py similarity index 100% rename from gateway/stubs.py rename to transform/stubs.py From e1d15e56d641bba868d7f46ee680d22b8011c3ac Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Wed, 7 Jul 2021 13:32:35 -0700 Subject: [PATCH 3/6] Gateway uses boolean value to return hidden files The `show_hidden` field is now accepted on the "update_file_list" command. When True, the files returned will include 'hidden' files using the unix `.filename` syntax. Signed-off-by: Josh Roppo --- requirements.txt | 4 ++-- satellite/satellite.py | 41 +++++++++++++++++++++++++---------------- tests/test_sat.py | 7 +++++-- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/requirements.txt b/requirements.txt index caae78e..e3da46d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ websockets requests -majortom_gateway>=0.0.10 +majortom_gateway>=0.1.0 mock>=4.0.3 pytest-asyncio pytest-only>=1.2.2 pytest-watch>=4.2.0 pytest>=6.2.2 -asgiref \ No newline at end of file +asgiref diff --git a/satellite/satellite.py b/satellite/satellite.py index a8e1608..828319c 100644 --- a/satellite/satellite.py +++ b/satellite/satellite.py @@ -5,9 +5,12 @@ ''' import time import logging +import asyncio +import majortom_gateway from random import randint from threading import Timer +from distutils.util import strtobool from transform import stubs from commands import CommandStatus @@ -75,7 +78,7 @@ def process_command(self, bytes, gateway): duration = command.fields['duration'] mode = command.fields['mode'] if errors: - gateway.fail_command(command.id, errors) + gateway.api.fail_command(command.id, errors) else: msg = f"Started Telemetry Beacon in mode: {command.fields['mode']} for {command.fields['duration']} seconds." gateway.set_command_status(command.id, CommandStatus.COMPLETED, payload=msg) @@ -99,6 +102,8 @@ def process_command(self, bytes, gateway): Certain commands are 'known' to Major Tom, and can appear in more convenient places in the UI because of that. See the documentation for a full list of such commands. """ + # Handle potential API returning 'true' or 'false' instead of proper boolean value. + show_hidden = bool(strtobool(command.fields['show_hidden'])) for i in range(1, randint(2, 4)): self.file_list.append({ "name": f'Payload-Image-{(len(self.file_list)+1):04d}.png', @@ -107,26 +112,30 @@ def process_command(self, bytes, gateway): "metadata": {"type": "image", "lat": (randint(-89, 89) + .0001*randint(0, 9999)), "lng": (randint(-179, 179) + .0001*randint(0, 9999))} }) - if self.command.fields['show_hidden']: - for i in range(1, randint(1, 3)): - self.file_list.append({ - "name": f'.thumb-{(len(self.file_list)+1):04d}.png', - "size": randint(200, 300), - "timestamp": int(time.time() * 1000) + i*10, - "metadata": {"type": "image", "lat": (randint(-89, 89) + .0001*randint(0, 9999)), "lng": (randint(-179, 179) + .0001*randint(0, 9999))} - }) + if show_hidden: + for i in range(1, randint(1, 3)): + self.file_list.append({ + "name": f'.thumb-{(len(self.file_list)+1):04d}.png', + "size": randint(200, 300), + "timestamp": int(time.time() * 1000) + i*10, + "metadata": {"type": "image", "lat": (randint(-89, 89) + .0001*randint(0, 9999)), "lng": (randint(-179, 179) + .0001*randint(0, 9999))} + }) self.check_cancelled(id=command.id) logger.info("Files: {}".format(self.file_list)) - r = Timer(0.1, gateway.update_file_list, kwargs={"system":self.name, "files":self.file_list}) - r.start() - time.sleep(10) + binary = stubs.translate_command_to_binary(command) + packetized = stubs.packetize(binary) + encrypted = stubs.encrypt(packetized) + + self.check_cancelled(id=command.id) + asyncio.run(gateway.api.update_file_list(system=self.name, files=self.file_list)) + self.check_cancelled(id=command.id) - gateway.set_command_status( - command_id=command.id, - status=CommandStatus.COMPLETED, - payload="Updated Remote File List") + asyncio.run(gateway.api.complete_command( + command_id=command.id, + output="Updated Remote File List", + )) elif command.type == "error": """ Simulates a command erroring out. """ diff --git a/tests/test_sat.py b/tests/test_sat.py index 6e21b6f..a6fd8ee 100644 --- a/tests/test_sat.py +++ b/tests/test_sat.py @@ -78,6 +78,9 @@ def test_process_command(self): system="Example FlatSat", definitions=command_defs["definitions"])) + self.gateway.command_callback(cmd, self.gw_api) + #... + # Profit? resp = "" - self.gateway.satellite_response(cmd_bytes, resp) - print("command resp: {}", resp) \ No newline at end of file + #self.gateway.satellite_response(cmd_bytes, resp) + #print("command resp: {}", resp) \ No newline at end of file From 3116a07f055f305759da5d87720471a0d3aff0ea Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Wed, 7 Jul 2021 16:00:11 -0700 Subject: [PATCH 4/6] Organize code by usefulness and demonstration Code which has value to use as base for further functionality resides in the `gateway` package. The `satellite` package only contains demonstration functionality of what a satellite might provide. Signed-off-by: Josh Roppo --- commands/__init__.py | 1 - gateway/gateway.py | 9 ++++----- {commands => gateway}/statuses.py | 0 satellite/__init__.py | 3 ++- satellite/satellite.py | 6 +++--- {transform => satellite}/stubs.py | 0 transform/__init__.py | 1 - 7 files changed, 9 insertions(+), 11 deletions(-) delete mode 100644 commands/__init__.py rename {commands => gateway}/statuses.py (100%) rename {transform => satellite}/stubs.py (100%) delete mode 100644 transform/__init__.py diff --git a/commands/__init__.py b/commands/__init__.py deleted file mode 100644 index c1cae88..0000000 --- a/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .statuses import CommandStatus \ No newline at end of file diff --git a/gateway/gateway.py b/gateway/gateway.py index 303b53e..2ffbcd9 100644 --- a/gateway/gateway.py +++ b/gateway/gateway.py @@ -1,13 +1,12 @@ import logging import time -from asgiref.sync import async_to_sync from random import randint -from transform import stubs -from commands import CommandStatus -from satellite import Satellite +from asgiref.sync import async_to_sync -logger = logging.getLogger(__name__) +from .statuses import CommandStatus +from satellite import Satellite, stubs +logger = logging.getLogger(__name__) class Gateway: def __init__(self, *args, **kwargs): diff --git a/commands/statuses.py b/gateway/statuses.py similarity index 100% rename from commands/statuses.py rename to gateway/statuses.py diff --git a/satellite/__init__.py b/satellite/__init__.py index 8e47211..3ba3a0a 100644 --- a/satellite/__init__.py +++ b/satellite/__init__.py @@ -1,2 +1,3 @@ from .satellite import Satellite, CommandCancelledError -from .telemetry import FakeTelemetry \ No newline at end of file +from .telemetry import FakeTelemetry +from . import stubs diff --git a/satellite/satellite.py b/satellite/satellite.py index 828319c..04d5124 100644 --- a/satellite/satellite.py +++ b/satellite/satellite.py @@ -12,8 +12,7 @@ from threading import Timer from distutils.util import strtobool -from transform import stubs -from commands import CommandStatus +from . import stubs from .telemetry import FakeTelemetry logger = logging.getLogger(__name__) @@ -81,7 +80,8 @@ def process_command(self, bytes, gateway): gateway.api.fail_command(command.id, errors) else: msg = f"Started Telemetry Beacon in mode: {command.fields['mode']} for {command.fields['duration']} seconds." - gateway.set_command_status(command.id, CommandStatus.COMPLETED, payload=msg) + command_completed = "completed" + gateway.set_command_status(command.id, command_completed, payload=msg) timeout = time.time() + duration while time.time() < timeout: self.check_cancelled(id=command.id) diff --git a/transform/stubs.py b/satellite/stubs.py similarity index 100% rename from transform/stubs.py rename to satellite/stubs.py diff --git a/transform/__init__.py b/transform/__init__.py deleted file mode 100644 index adbaf5c..0000000 --- a/transform/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .stubs import * \ No newline at end of file From 6cf37da584ae426b0c954a765253d36419265d3f Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Wed, 7 Jul 2021 16:50:18 -0700 Subject: [PATCH 5/6] test_sat fixes to asyncio and "true" boolean field Signed-off-by: Josh Roppo --- satellite/satellite.py | 4 ++-- tests/test_sat.py | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/satellite/satellite.py b/satellite/satellite.py index 04d5124..44d9f76 100644 --- a/satellite/satellite.py +++ b/satellite/satellite.py @@ -145,7 +145,7 @@ def process_command(self, bytes, gateway): else: # We'd want to generate an error if the command wasn't found. - logger.warn(f"Satellite does not recognize command {command.type}") + logger.warning(f"Satellite does not recognize command {command.type}") errors = [f"Command {command.type} not found on Satellite."] gateway.fail_command(command.id, errors=errors) @@ -158,4 +158,4 @@ def validate(self, command): if type(command.fields['duration']) != type(int()): return [f"Duration type is invalid. Must be an int. Type: {type(command.fields['duration'])}"] - return None \ No newline at end of file + return None diff --git a/tests/test_sat.py b/tests/test_sat.py index a6fd8ee..b201e8c 100644 --- a/tests/test_sat.py +++ b/tests/test_sat.py @@ -14,8 +14,7 @@ from majortom_gateway.command import Command from majortom_gateway import gateway_api -from satellite import Satellite -from transform import stubs +from satellite import Satellite, stubs from gateway import Gateway class TypeMatcher: @@ -33,7 +32,7 @@ def callback_mock(): return fn -#@pytest.mark.asyncio examplej test +#@pytest.mark.asyncio example test #async def test_calls_transit_callback(callback_mock): # gw = GatewayAPI("host", "gateway_token", transit_callback=callback_mock) # # ToDo: Update this example message @@ -56,13 +55,13 @@ def setUp(self): def test_process_command(self): system = "Example Flatsat" # Match 'system' identifier with Gateway CommandDef json_bytes = """{ - "type": "command", - "id": "update_file_list", + "type": "update_file_list", + "id": 18, "system": "Example FlatSat", "display_name": "Update File List", "tags": ["files", "operations"], "fields": [ - {"name": "show_hidden", "type": "boolean", "value": true } + {"name": "show_hidden", "type": "boolean", "value": "true" } ] }""" jl = json.loads(json_bytes) @@ -74,7 +73,7 @@ def test_process_command(self): with open('satellite/example_commands.json','r') as f: command_defs = json.loads(f.read()) - asyncio.ensure_future(self.gw_api.update_command_definitions( + asyncio.run(self.gw_api.update_command_definitions( system="Example FlatSat", definitions=command_defs["definitions"])) @@ -83,4 +82,4 @@ def test_process_command(self): # Profit? resp = "" #self.gateway.satellite_response(cmd_bytes, resp) - #print("command resp: {}", resp) \ No newline at end of file + #print("command resp: {}", resp) From 3bf6185893b9a75dc77e957f122b5a273b2c4393 Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Thu, 8 Jul 2021 15:30:39 -0700 Subject: [PATCH 6/6] Additional update_file_list params Enables testing the Radio-boolean fields with other options. Signed-off-by: Josh Roppo --- satellite/example_commands.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/satellite/example_commands.json b/satellite/example_commands.json index 9ea6d3d..4ad942f 100644 --- a/satellite/example_commands.json +++ b/satellite/example_commands.json @@ -38,6 +38,9 @@ "description": "Gets the latest list of downloadable files from the spacecraft.", "tags": ["files", "operations"], "fields": [ + {"name": "sort_by", "type": "string", "range": ["none", "size", "time", "type"], "default": "none"}, + {"name": "filter_regex", "type": "string", "default": ""}, + {"name": "exclude_regex", "type": "string", "default": ""}, {"name": "show_hidden", "type": "boolean", "default": false } ] },