From 4901c8e42bc11907548e2506e72f87310b9c1e9e Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Sat, 6 Nov 2021 23:04:13 -0400 Subject: [PATCH 1/8] first draft of RockBLOCK interface --- packetraven/connections/internet.py | 93 +++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/packetraven/connections/internet.py b/packetraven/connections/internet.py index 11068ec3..a99dd14f 100644 --- a/packetraven/connections/internet.py +++ b/packetraven/connections/internet.py @@ -110,6 +110,99 @@ def __repr__(self): return f'{self.__class__.__name__}({repr(self.callsigns)}, {repr("****")})' +class RockBLOCK(PacketSource, NetworkConnection): + interval = timedelta(seconds=10) + + def __init__(self, imei: str, username: str = None, password: str = None): + """ + connect to RockBLOCK API + + :param imei: IMEI of RockBLOCK + :param username: RockBLOCK username + :param password: RockBLOCK password + """ + + url = 'https://rockblock.rock7.com/rockblock/MT' + super().__init__(url) + + if username is None or username == '': + configuration = read_configuration(CREDENTIALS_FILENAME) + + if 'RockBLOCK' in configuration: + username = configuration['RockBLOCK']['username'] + else: + raise ConnectionError(f'no RockBLOCK username specified') + if password is None or password == '': + configuration = read_configuration(CREDENTIALS_FILENAME) + + if 'RockBLOCK' in configuration: + password = configuration['RockBLOCK']['password'] + else: + raise ConnectionError(f'no RockBLOCK password specified') + + if not self.connected: + raise ConnectionError(f'no network connection') + + self.imei = imei + self.username = username + self.password = password + + self.__last_access_time = None + + @property + def packets(self) -> [LocationPacket]: + if self.__last_access_time is not None and self.interval is not None: + interval = datetime.now() - self.__last_access_time + if interval < self.interval: + raise TimeIntervalError( + f'interval {interval} less than minimum interval {self.interval}' + ) + + query = { + 'imei': self.imei, + 'username': self.username, + 'password': self.password, + } + + query = '&'.join(f'{key}={value}' for key, value in query.items()) + + response = requests.get(f'{self.location}?{query}') + + status, code, data = response.text.split(',') + if status == 'OK': + # TODO test this with RockBLOCK data + packets = data.split(',') + + if len(packets) > 0: + # respond promptly with normal response code if received data + post_query = { + 'username': self.username, + 'password': self.password, + } + requests.post( + f'{self.location}?{post_query}', headers={'Accept': 'text/plain'} + ) + + # TODO write packet parsing code here + else: + if code == '10': + raise ConnectionError(data) + elif code in ['11', '12', '13']: + raise PermissionError(data) + elif code in ['14', '15']: + raise ValueError(data) + elif code == '16': + packets = [] + else: + raise SystemError(data) + + self.__last_access_time = datetime.now() + return packets + + def close(self): + pass + + class PacketDatabaseTable(PostGresTable, PacketSource, PacketSink): __default_fields = { 'time': datetime, From b66b9d32d63995c0c6511030261d6017ce37f90f Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Sat, 6 Nov 2021 23:06:09 -0400 Subject: [PATCH 2/8] formatting --- README.md | 8 ++++---- packetraven/__main__.py | 4 +++- packetraven/predicts.py | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3532f11a..b5728aac 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,15 @@ pip install packetraven virtualenv packetraven_env ``` 2. activate your new virtual environment - - On Linux: + - On Linux: ```bash source packetraven_env/bin/activate ``` - - On Windows native command prompt (`cmd`): + - On Windows native command prompt (`cmd`): ```cmd .\packetraven_env\Scripts\activate.bat ``` - - On Windows PowerShell: + - On Windows PowerShell: ```cmd .\packetraven_env\Scripts\activate.ps1 ``` @@ -39,7 +39,7 @@ pip install packetraven ```bash pip install packetraven ``` - + # Usage ## Command-line Options diff --git a/packetraven/__main__.py b/packetraven/__main__.py index 86492b2b..04e12b58 100644 --- a/packetraven/__main__.py +++ b/packetraven/__main__.py @@ -247,7 +247,9 @@ def main(): elif start_date is None and end_date is not None: filter_message += f' sent before {end_date:%Y-%m-%d %H:%M:%S}' elif start_date is not None and end_date is not None: - filter_message += f' sent between {start_date:%Y-%m-%d %H:%M:%S} and {end_date:%Y-%m-%d %H:%M:%S}' + filter_message += ( + f' sent between {start_date:%Y-%m-%d %H:%M:%S} and {end_date:%Y-%m-%d %H:%M:%S}' + ) if callsigns is not None: filter_message += f' from {len(callsigns)} callsigns: {callsigns}' LOGGER.info(filter_message) diff --git a/packetraven/predicts.py b/packetraven/predicts.py index 637fed8d..0f52d0d1 100644 --- a/packetraven/predicts.py +++ b/packetraven/predicts.py @@ -479,8 +479,8 @@ def get_predictions( prediction_float_end_time = None else: prediction_float_end_time = None - descent_only = ( - packet_track.falling or numpy.any(packet_track.ascent_rates[-2:] < 0) + descent_only = packet_track.falling or numpy.any( + packet_track.ascent_rates[-2:] < 0 ) try: From 5ee844446eca6d1219ccac1c5d13e01892c5fd10 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Sun, 7 Nov 2021 08:35:21 -0500 Subject: [PATCH 3/8] add send function --- packetraven/connections/internet.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/packetraven/connections/internet.py b/packetraven/connections/internet.py index a99dd14f..d9028dc5 100644 --- a/packetraven/connections/internet.py +++ b/packetraven/connections/internet.py @@ -110,7 +110,7 @@ def __repr__(self): return f'{self.__class__.__name__}({repr(self.callsigns)}, {repr("****")})' -class RockBLOCK(PacketSource, NetworkConnection): +class RockBLOCK(PacketSource, PacketSink, NetworkConnection): interval = timedelta(seconds=10) def __init__(self, imei: str, username: str = None, password: str = None): @@ -149,6 +149,12 @@ def __init__(self, imei: str, username: str = None, password: str = None): self.__last_access_time = None + def query(self, **query) -> str: + query['username'] = self.username + query['password'] = self.password + query = '&'.join(f'{key}={value}' for key, value in query.items()) + return f'{self.location}?{query}' + @property def packets(self) -> [LocationPacket]: if self.__last_access_time is not None and self.interval is not None: @@ -158,15 +164,7 @@ def packets(self) -> [LocationPacket]: f'interval {interval} less than minimum interval {self.interval}' ) - query = { - 'imei': self.imei, - 'username': self.username, - 'password': self.password, - } - - query = '&'.join(f'{key}={value}' for key, value in query.items()) - - response = requests.get(f'{self.location}?{query}') + response = requests.get(self.query()) status, code, data = response.text.split(',') if status == 'OK': @@ -202,6 +200,10 @@ def packets(self) -> [LocationPacket]: def close(self): pass + def send(self, packets: [LocationPacket]): + # TODO convert list of packets into hex-encoded byte array of data + requests.post(self.query(imei=self.imei), headers={'Accept': 'text/plain'}) + class PacketDatabaseTable(PostGresTable, PacketSource, PacketSink): __default_fields = { From 21c1b45b056f7c2691e8bdc96d059d9f272189ca Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Thu, 11 Nov 2021 22:50:41 -0500 Subject: [PATCH 4/8] comment out internet interface for now, add CSV interface for use with `rockblock-tools` parser package (Flask app) --- packetraven/connections/file.py | 42 ++++++- packetraven/connections/internet.py | 186 ++++++++++++++-------------- 2 files changed, 134 insertions(+), 94 deletions(-) diff --git a/packetraven/connections/file.py b/packetraven/connections/file.py index d2823120..70f06502 100644 --- a/packetraven/connections/file.py +++ b/packetraven/connections/file.py @@ -1,10 +1,11 @@ -from datetime import datetime +from datetime import datetime, timedelta from os import PathLike from pathlib import Path from urllib.parse import urlparse from dateutil.parser import parse as parse_date import geojson +import pandas import requests from packetraven.connections.base import ( @@ -89,6 +90,45 @@ def __repr__(self): return f'{self.__class__.__name__}({repr(self.location)}, {repr(self.callsigns)})' +class RockBLOCKtoolsCSV(PacketSource): + interval = timedelta(seconds=10) + + def __init__(self, csv_filename: PathLike = None): + """ + watch a CSV file being written to by a listening instance of `rockblock-tools` + ``` + rockblock listen csv localhost 80 + ``` + + :param csv_filename: path to CSV file + """ + + if not isinstance(csv_filename, Path): + csv_filename = Path(csv_filename) + + super().__init__(csv_filename) + self.__last_access_time = None + + @property + def packets(self) -> [LocationPacket]: + records = pandas.read_csv(self.location) + + return [ + LocationPacket( + time=record['Transmit Time'], + x=record['Iridium Longitude'], + y=record['Iridium Latitude'], + device=record['Device Type'], + momsn=record['MOMSN'], + imei=record['IMEI'], + serial=record['Serial'], + data=record['Data'], + cep=record['Iridum CEP'], + ) + for record in records.iterrows() + ] + + class PacketGeoJSON(PacketSource): def __init__(self, filename: PathLike = None): """ diff --git a/packetraven/connections/internet.py b/packetraven/connections/internet.py index d9028dc5..44b02b08 100644 --- a/packetraven/connections/internet.py +++ b/packetraven/connections/internet.py @@ -110,99 +110,99 @@ def __repr__(self): return f'{self.__class__.__name__}({repr(self.callsigns)}, {repr("****")})' -class RockBLOCK(PacketSource, PacketSink, NetworkConnection): - interval = timedelta(seconds=10) - - def __init__(self, imei: str, username: str = None, password: str = None): - """ - connect to RockBLOCK API - - :param imei: IMEI of RockBLOCK - :param username: RockBLOCK username - :param password: RockBLOCK password - """ - - url = 'https://rockblock.rock7.com/rockblock/MT' - super().__init__(url) - - if username is None or username == '': - configuration = read_configuration(CREDENTIALS_FILENAME) - - if 'RockBLOCK' in configuration: - username = configuration['RockBLOCK']['username'] - else: - raise ConnectionError(f'no RockBLOCK username specified') - if password is None or password == '': - configuration = read_configuration(CREDENTIALS_FILENAME) - - if 'RockBLOCK' in configuration: - password = configuration['RockBLOCK']['password'] - else: - raise ConnectionError(f'no RockBLOCK password specified') - - if not self.connected: - raise ConnectionError(f'no network connection') - - self.imei = imei - self.username = username - self.password = password - - self.__last_access_time = None - - def query(self, **query) -> str: - query['username'] = self.username - query['password'] = self.password - query = '&'.join(f'{key}={value}' for key, value in query.items()) - return f'{self.location}?{query}' - - @property - def packets(self) -> [LocationPacket]: - if self.__last_access_time is not None and self.interval is not None: - interval = datetime.now() - self.__last_access_time - if interval < self.interval: - raise TimeIntervalError( - f'interval {interval} less than minimum interval {self.interval}' - ) - - response = requests.get(self.query()) - - status, code, data = response.text.split(',') - if status == 'OK': - # TODO test this with RockBLOCK data - packets = data.split(',') - - if len(packets) > 0: - # respond promptly with normal response code if received data - post_query = { - 'username': self.username, - 'password': self.password, - } - requests.post( - f'{self.location}?{post_query}', headers={'Accept': 'text/plain'} - ) - - # TODO write packet parsing code here - else: - if code == '10': - raise ConnectionError(data) - elif code in ['11', '12', '13']: - raise PermissionError(data) - elif code in ['14', '15']: - raise ValueError(data) - elif code == '16': - packets = [] - else: - raise SystemError(data) - - self.__last_access_time = datetime.now() - return packets - - def close(self): - pass - - def send(self, packets: [LocationPacket]): - # TODO convert list of packets into hex-encoded byte array of data - requests.post(self.query(imei=self.imei), headers={'Accept': 'text/plain'}) +# class RockBLOCK(PacketSource, PacketSink, NetworkConnection): +# interval = timedelta(seconds=10) +# +# def __init__(self, imei: str, username: str = None, password: str = None): +# """ +# connect to RockBLOCK API +# +# :param imei: IMEI of RockBLOCK +# :param username: RockBLOCK username +# :param password: RockBLOCK password +# """ +# +# url = 'https://rockblock.rock7.com/rockblock/MT' +# super().__init__(url) +# +# if username is None or username == '': +# configuration = read_configuration(CREDENTIALS_FILENAME) +# +# if 'RockBLOCK' in configuration: +# username = configuration['RockBLOCK']['username'] +# else: +# raise ConnectionError(f'no RockBLOCK username specified') +# if password is None or password == '': +# configuration = read_configuration(CREDENTIALS_FILENAME) +# +# if 'RockBLOCK' in configuration: +# password = configuration['RockBLOCK']['password'] +# else: +# raise ConnectionError(f'no RockBLOCK password specified') +# +# if not self.connected: +# raise ConnectionError(f'no network connection') +# +# self.imei = imei +# self.username = username +# self.password = password +# +# self.__last_access_time = None +# +# def query(self, **query) -> str: +# query['username'] = self.username +# query['password'] = self.password +# query = '&'.join(f'{key}={value}' for key, value in query.items()) +# return f'{self.location}?{query}' +# +# @property +# def packets(self) -> [LocationPacket]: +# if self.__last_access_time is not None and self.interval is not None: +# interval = datetime.now() - self.__last_access_time +# if interval < self.interval: +# raise TimeIntervalError( +# f'interval {interval} less than minimum interval {self.interval}' +# ) +# +# response = requests.get(self.query()) +# +# status, code, data = response.text.split(',') +# if status == 'OK': +# # TODO test this with RockBLOCK data +# packets = data.split(',') +# +# if len(packets) > 0: +# # respond promptly with normal response code if received data +# post_query = { +# 'username': self.username, +# 'password': self.password, +# } +# requests.post( +# f'{self.location}?{post_query}', headers={'Accept': 'text/plain'} +# ) +# +# # TODO write packet parsing code here +# else: +# if code == '10': +# raise ConnectionError(data) +# elif code in ['11', '12', '13']: +# raise PermissionError(data) +# elif code in ['14', '15']: +# raise ValueError(data) +# elif code == '16': +# packets = [] +# else: +# raise SystemError(data) +# +# self.__last_access_time = datetime.now() +# return packets +# +# def close(self): +# pass +# +# def send(self, packets: [LocationPacket]): +# # TODO convert list of packets into hex-encoded byte array of data +# requests.post(self.query(imei=self.imei), headers={'Accept': 'text/plain'}) class PacketDatabaseTable(PostGresTable, PacketSource, PacketSink): From 873642cbfd4528f69451dc52c6eec98ceccefd50 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Sat, 13 Nov 2021 21:55:42 -0500 Subject: [PATCH 5/8] implement Flask app into RockBLOCK interface --- packetraven/connections/file.py | 217 +++++++++++++++------------- packetraven/connections/internet.py | 180 +++++++++++------------ packetraven/packets/base.py | 22 +++ setup.py | 1 + 4 files changed, 230 insertions(+), 190 deletions(-) diff --git a/packetraven/connections/file.py b/packetraven/connections/file.py index 70f06502..f10acb4d 100644 --- a/packetraven/connections/file.py +++ b/packetraven/connections/file.py @@ -1,6 +1,8 @@ -from datetime import datetime, timedelta +from datetime import datetime +from io import BytesIO, StringIO from os import PathLike from pathlib import Path +from typing import Union from urllib.parse import urlparse from dateutil.parser import parse as parse_date @@ -15,9 +17,49 @@ TimeIntervalError, ) from packetraven.packets import APRSPacket, LocationPacket +from packetraven.packets.base import IridiumPacket -class RawAPRSTextFile(APRSPacketSource): +class WatchedFile: + def __init__(self, file: Union[PathLike, StringIO, BytesIO]): + if not isinstance(file, (StringIO, BytesIO)): + if not urlparse(str(file)).scheme in ['http', 'https', 'ftp', 'sftp']: + if not isinstance(file, Path): + if isinstance(file, str): + file = file.strip('"') + file = Path(file) + file = str(file) + + self.file = file + self.__parsed_lines = None + + def new_lines(self) -> [str]: + new_lines = [] + + if Path(self.file).exists(): + file_connection = open(Path(self.file).expanduser().resolve()) + lines = file_connection.readlines() + else: + file_connection = requests.get(self.file, stream=True) + lines = file_connection.iter_lines() + + for line in lines: + if len(line) > 0: + if isinstance(line, bytes): + line = line.decode() + if line not in self.__parsed_lines: + self.__parsed_lines.append(line) + new_lines.append(line) + + file_connection.close() + + return new_lines + + def close(self): + pass + + +class RawAPRSTextFile(APRSPacketSource, WatchedFile): def __init__(self, filename: PathLike = None, callsigns: str = None): """ read APRS packets from a given text file where each line consists of the time sent (`YYYY-MM-DDTHH:MM:SS`) followed by @@ -27,14 +69,9 @@ def __init__(self, filename: PathLike = None, callsigns: str = None): :param callsigns: list of callsigns to return from source """ - if not urlparse(str(filename)).scheme in ['http', 'https', 'ftp', 'sftp']: - if not isinstance(filename, Path): - if isinstance(filename, str): - filename = filename.strip('"') - filename = Path(filename) - filename = str(filename) + WatchedFile.__init__(self, filename) + APRSPacketSource.__init__(self, self.file, callsigns) - super().__init__(filename, callsigns) self.__last_access_time = None self.__parsed_lines = [] @@ -47,35 +84,23 @@ def packets(self) -> [APRSPacket]: f'interval {interval} less than minimum interval {self.interval}' ) - if Path(self.location).exists(): - file_connection = open(Path(self.location).expanduser().resolve()) - lines = file_connection.readlines() - else: - file_connection = requests.get(self.location, stream=True) - lines = file_connection.iter_lines() + new_lines = self.new_lines() packets = [] - for line in lines: - if len(line) > 0: - if isinstance(line, bytes): - line = line.decode() - if line not in self.__parsed_lines: - self.__parsed_lines.append(line) - try: - packet_time, raw_aprs = line.split(': ', 1) - packet_time = parse_date(packet_time) - except: - raw_aprs = line - packet_time = datetime.now() - raw_aprs = raw_aprs.strip() - try: - packets.append( - APRSPacket.from_frame(raw_aprs, packet_time, source=self.location) - ) - except Exception as error: - LOGGER.error(f'{error.__class__.__name__} - {error}') - - file_connection.close() + for line in new_lines: + try: + packet_time, raw_aprs = line.split(': ', 1) + packet_time = parse_date(packet_time) + except: + raw_aprs = line + packet_time = datetime.now() + raw_aprs = raw_aprs.strip() + try: + packets.append( + APRSPacket.from_frame(raw_aprs, packet_time, source=self.location) + ) + except Exception as error: + LOGGER.error(f'{error.__class__.__name__} - {error}') if self.callsigns is not None: packets = [packet for packet in packets if packet.from_callsign in self.callsigns] @@ -90,9 +115,7 @@ def __repr__(self): return f'{self.__class__.__name__}({repr(self.location)}, {repr(self.callsigns)})' -class RockBLOCKtoolsCSV(PacketSource): - interval = timedelta(seconds=10) - +class RockBLOCKtoolsCSV(PacketSource, WatchedFile): def __init__(self, csv_filename: PathLike = None): """ watch a CSV file being written to by a listening instance of `rockblock-tools` @@ -111,25 +134,33 @@ def __init__(self, csv_filename: PathLike = None): @property def packets(self) -> [LocationPacket]: - records = pandas.read_csv(self.location) - - return [ - LocationPacket( - time=record['Transmit Time'], - x=record['Iridium Longitude'], - y=record['Iridium Latitude'], - device=record['Device Type'], - momsn=record['MOMSN'], - imei=record['IMEI'], - serial=record['Serial'], - data=record['Data'], - cep=record['Iridum CEP'], + new_lines = self.new_lines() + + packets = [] + for line in new_lines: + line = [entry.strip() for entry in line.split(',')] + packet = IridiumPacket( + time=line[3], + x=float(line[1]), + y=float(line[0]), + device_type=line[2], + momsn=line[4], + imei=line[5], + serial=line[6], + data=line[7], + cep=line[8], ) - for record in records.iterrows() - ] + packets.append(packet) + self.__last_access_time = datetime.now() + + return packets + + def close(self): + pass -class PacketGeoJSON(PacketSource): + +class PacketGeoJSON(PacketSource, WatchedFile): def __init__(self, filename: PathLike = None): """ read location packets from a given GeoJSON file @@ -137,14 +168,9 @@ def __init__(self, filename: PathLike = None): :param filename: path to GeoJSON file """ - if not urlparse(str(filename)).scheme in ['http', 'https', 'ftp', 'sftp']: - if not isinstance(filename, Path): - if isinstance(filename, str): - filename = filename.strip('"') - filename = Path(filename) - filename = str(filename) + WatchedFile.__init__(self, filename) + PacketSource.__init__(self, self.file) - super().__init__(filename) self.__last_access_time = None @property @@ -156,42 +182,39 @@ def packets(self) -> [LocationPacket]: f'interval {interval} less than minimum interval {self.interval}' ) - if Path(self.location).exists(): - with open(Path(self.location).expanduser().resolve()) as file_connection: - features = geojson.load(file_connection) - else: - response = requests.get(self.location, stream=True) - features = geojson.loads(response.text) - packets = [] - for feature in features['features']: - if feature['geometry']['type'] == 'Point': - properties = feature['properties'] - time = parse_date(properties['time']) - del properties['time'] - - if 'from' in properties: - from_callsign = properties['from'] - to_callsign = properties['to'] - del properties['from'], properties['to'] - - packet = APRSPacket( - from_callsign, - to_callsign, - time, - *feature['geometry']['coordinates'], - source=self.location, - **properties, - ) - else: - packet = LocationPacket( - time, - *feature['geometry']['coordinates'], - source=self.location, - **properties, - ) - - packets.append(packet) + new_lines = self.new_lines() + if len(new_lines) > 0: + features = geojson.loads(new_lines) + + for feature in features['features']: + if feature['geometry']['type'] == 'Point': + properties = feature['properties'] + time = parse_date(properties['time']) + del properties['time'] + + if 'from' in properties: + from_callsign = properties['from'] + to_callsign = properties['to'] + del properties['from'], properties['to'] + + packet = APRSPacket( + from_callsign, + to_callsign, + time, + *feature['geometry']['coordinates'], + source=self.location, + **properties, + ) + else: + packet = LocationPacket( + time, + *feature['geometry']['coordinates'], + source=self.location, + **properties, + ) + + packets.append(packet) self.__last_access_time = datetime.now() diff --git a/packetraven/connections/internet.py b/packetraven/connections/internet.py index 44b02b08..79f4c84f 100644 --- a/packetraven/connections/internet.py +++ b/packetraven/connections/internet.py @@ -1,9 +1,15 @@ +from argparse import Namespace from datetime import datetime, timedelta +from io import StringIO from time import sleep from typing import Any, Sequence import aprslib +from flask import request import requests +import rockblock_tools +from rockblock_tools import listen +from rockblock_tools.formatter import CSVFormatter from shapely.geometry import Point from tablecrow import PostGresTable from tablecrow.utilities import split_hostname_port @@ -18,6 +24,7 @@ PacketSource, TimeIntervalError, ) +from packetraven.connections.file import RockBLOCKtoolsCSV from packetraven.packets import APRSPacket, LocationPacket from packetraven.packets.parsing import InvalidPacketError from packetraven.utilities import read_configuration @@ -110,99 +117,86 @@ def __repr__(self): return f'{self.__class__.__name__}({repr(self.callsigns)}, {repr("****")})' -# class RockBLOCK(PacketSource, PacketSink, NetworkConnection): -# interval = timedelta(seconds=10) -# -# def __init__(self, imei: str, username: str = None, password: str = None): -# """ -# connect to RockBLOCK API -# -# :param imei: IMEI of RockBLOCK -# :param username: RockBLOCK username -# :param password: RockBLOCK password -# """ -# -# url = 'https://rockblock.rock7.com/rockblock/MT' -# super().__init__(url) -# -# if username is None or username == '': -# configuration = read_configuration(CREDENTIALS_FILENAME) -# -# if 'RockBLOCK' in configuration: -# username = configuration['RockBLOCK']['username'] -# else: -# raise ConnectionError(f'no RockBLOCK username specified') -# if password is None or password == '': -# configuration = read_configuration(CREDENTIALS_FILENAME) -# -# if 'RockBLOCK' in configuration: -# password = configuration['RockBLOCK']['password'] -# else: -# raise ConnectionError(f'no RockBLOCK password specified') -# -# if not self.connected: -# raise ConnectionError(f'no network connection') -# -# self.imei = imei -# self.username = username -# self.password = password -# -# self.__last_access_time = None -# -# def query(self, **query) -> str: -# query['username'] = self.username -# query['password'] = self.password -# query = '&'.join(f'{key}={value}' for key, value in query.items()) -# return f'{self.location}?{query}' -# -# @property -# def packets(self) -> [LocationPacket]: -# if self.__last_access_time is not None and self.interval is not None: -# interval = datetime.now() - self.__last_access_time -# if interval < self.interval: -# raise TimeIntervalError( -# f'interval {interval} less than minimum interval {self.interval}' -# ) -# -# response = requests.get(self.query()) -# -# status, code, data = response.text.split(',') -# if status == 'OK': -# # TODO test this with RockBLOCK data -# packets = data.split(',') -# -# if len(packets) > 0: -# # respond promptly with normal response code if received data -# post_query = { -# 'username': self.username, -# 'password': self.password, -# } -# requests.post( -# f'{self.location}?{post_query}', headers={'Accept': 'text/plain'} -# ) -# -# # TODO write packet parsing code here -# else: -# if code == '10': -# raise ConnectionError(data) -# elif code in ['11', '12', '13']: -# raise PermissionError(data) -# elif code in ['14', '15']: -# raise ValueError(data) -# elif code == '16': -# packets = [] -# else: -# raise SystemError(data) -# -# self.__last_access_time = datetime.now() -# return packets -# -# def close(self): -# pass -# -# def send(self, packets: [LocationPacket]): -# # TODO convert list of packets into hex-encoded byte array of data -# requests.post(self.query(imei=self.imei), headers={'Accept': 'text/plain'}) +class RockBLOCK(PacketSource, NetworkConnection): + def __init__(self, imei: str = None, username: str = None, password: str = None): + """ + connect to RockBLOCK API + + :param imei: IMEI of RockBLOCK + :param username: RockBLOCK username + :param password: RockBLOCK password + """ + + url = 'https://rockblock.rock7.com' + NetworkConnection.__init__(self, url) + + configuration = read_configuration(CREDENTIALS_FILENAME) + if imei is None or imei == '': + if 'RockBLOCK' in configuration: + imei = configuration['RockBLOCK']['imei'] + if username is None or username == '': + if 'RockBLOCK' in configuration: + username = configuration['RockBLOCK']['username'] + if password is None or password == '': + if 'RockBLOCK' in configuration: + password = configuration['RockBLOCK']['password'] + + if not self.connected: + raise ConnectionError(f'no network connection') + + self.imei = imei + self.username = username + self.password = password + + self.__last_access_time = None + self.__parsed_lines = [] + + self.__csv_stream = StringIO() + formatter_options = Namespace() + setattr(formatter_options, 'data_format', 'raw') + setattr(formatter_options, 'csv_file', self.__csv_stream) + self.__csv_formatter = CSVFormatter(formatter_options) + self.__csv_parser = RockBLOCKtoolsCSV(self.__csv_stream) + + def start_listening(self, hostname: str = None, port: int = None): + """ + :param hostname: hostname on which to listen to POST requests from RockBLOCK + :param port: port on which to listen (must be available) + """ + + if hostname is None: + hostname = 'localhost' + if port is None: + port = 80 + + listen(hostname, port, self.__csv_formatter) + + @property + def packets(self) -> [LocationPacket]: + return self.__csv_parser.packets + + def close(self): + shutdown_function = request.environ.get('werkzeug.server.shutdown') + if shutdown_function is None: + raise RuntimeError('Not running with the Werkzeug Server') + shutdown_function() + + self.__csv_formatter.close() + self.__csv_stream.close() + + def send_message(self, message: str): + """ + :param message: message data + """ + + if self.imei is None: + raise ValueError('RockBLOCK IMEI not provided') + if self.username is None: + raise ValueError('RockBLOCK username not provided') + if self.password is None: + raise ValueError('RockBLOCK password not provided') + + rockblock_tools.send(self.imei, self.username, self.password, message) class PacketDatabaseTable(PostGresTable, PacketSource, PacketSink): diff --git a/packetraven/packets/base.py b/packetraven/packets/base.py index 54616910..4f3b0a84 100644 --- a/packetraven/packets/base.py +++ b/packetraven/packets/base.py @@ -316,3 +316,25 @@ def __repr__(self) -> str: f'{self.__class__.__name__}(from_callsign={repr(self.from_callsign)}, to_callsign={repr(self.to_callsign)}, time={repr(self.time)}, ' f'{coordinate_string}, crs={self.crs.__class__.__name__}.from_epsg({repr(self.crs.to_epsg())}), {attribute_string})' ) + + +class IridiumPacket(LocationPacket): + def __init__( + self, + time: datetime, + x: float, + y: float, + device_type: str, + momsn, + imei: str, + serial: str, + data: str, + cep: str, + ): + super().__init__(time, x, y) + self.device_type = device_type + self.momsn = momsn + self.imei = imei + self.serial = serial + self.data = data + self.cep = cep diff --git a/setup.py b/setup.py index 17b746f5..0496e977 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ 'psycopg2-binary': [], 'pyproj': [], 'requests': [], + 'rockblock-tools': [], 'shapely': [], 'sshtunnel': [], 'tablecrow>=1.3.9': [], From f0e017293c8f375f102f7451d278917eedaca03b Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Sun, 14 Nov 2021 08:20:57 -0500 Subject: [PATCH 6/8] add rockblock parameters --- packetraven/__main__.py | 54 +++++++++++++++++++++++++++-- packetraven/connections/internet.py | 7 +++- packetraven/gui/base.py | 42 ++++++++++++++++------ 3 files changed, 90 insertions(+), 13 deletions(-) diff --git a/packetraven/__main__.py b/packetraven/__main__.py index 04e12b58..98c74a47 100644 --- a/packetraven/__main__.py +++ b/packetraven/__main__.py @@ -21,10 +21,11 @@ TimeIntervalError, ) from packetraven.connections.base import PacketSource +from packetraven.connections.internet import RockBLOCK from packetraven.packets import APRSPacket from packetraven.packets.tracks import APRSTrack, LocationPacketTrack from packetraven.packets.writer import write_packet_tracks -from packetraven.predicts import get_predictions, PredictionAPIURL, PredictionError +from packetraven.predicts import PredictionAPIURL, PredictionError, get_predictions from packetraven.utilities import get_logger, read_configuration, repository_root LOGGER = get_logger('packetraven', log_format='%(asctime)s | %(levelname)-8s | %(message)s') @@ -42,7 +43,16 @@ def main(): args_parser.add_argument( '--tnc', help='comma-separated list of serial ports / text files of a TNC parsing APRS packets from analog audio to ASCII' - ' (set to `auto` to use the first open serial port)', + ' (set to `auto` to use the first open serial port)', + ) + args_parser.add_argument( + '--rockblock', + default='localhost:80', + help='listen to incoming POST requests (defaults to `localhost:80`)' + ) + args_parser.add_argument( + '--imei', + help='IMEI of RockBLOCK modem' ) args_parser.add_argument( '--database', help='PostGres database table `user@hostname:port/database/table`' @@ -110,6 +120,22 @@ def main(): if args.tnc is not None: kwargs['tnc'] = [tnc.strip() for tnc in args.tnc.split(',')] + if args.rockblock is not None: + listen_hostname = parse_hostname(args.rockblock) + + hostname = listen_hostname['hostname'] + port = listen_hostname['port'] + username = listen_hostname['username'] + password = listen_hostname['password'] + + if port is not None: + hostname = f'{hostname}:{port}' + + kwargs['rockblock_hostname'] = hostname + kwargs['rockblock_imei'] = args.imei + kwargs['rockblock_username'] = username + kwargs['rockblock_password'] = password + if args.database is not None: database = parse_hostname(args.database) @@ -297,6 +323,30 @@ def main(): except ConnectionError as error: LOGGER.warning(f'{error.__class__.__name__} - {error}') + if 'rockblock_hostname' in kwargs: + rockblock_kwargs = { + key: kwargs[key] + for key in [ + 'rockblock_hostname', + 'rockblock_imei', + 'rockblock_username', + 'rockblock_password', + ] + if key in kwargs + } + + try: + rockblock = RockBLOCK( + imei=None, + username=rockblock_kwargs['rockblock_username'], + password=rockblock_kwargs['rockblock_password'], + ) + rockblock.start_listening(rockblock_kwargs['rockblock_hostname']) + LOGGER.info(f'connected to {rockblock.location}') + connections.append(rockblock) + except ConnectionError: + pass + if 'database_hostname' in kwargs: database_kwargs = { key: kwargs[key] diff --git a/packetraven/connections/internet.py b/packetraven/connections/internet.py index 79f4c84f..7d4744f5 100644 --- a/packetraven/connections/internet.py +++ b/packetraven/connections/internet.py @@ -12,7 +12,7 @@ from rockblock_tools.formatter import CSVFormatter from shapely.geometry import Point from tablecrow import PostGresTable -from tablecrow.utilities import split_hostname_port +from tablecrow.utilities import parse_hostname, split_hostname_port from packetraven.connections.base import ( APRSPacketSink, @@ -166,6 +166,11 @@ def start_listening(self, hostname: str = None, port: int = None): if hostname is None: hostname = 'localhost' + else: + connection_info = parse_hostname(hostname) + hostname = connection_info['hostname'] + if port is None: + port = connection_info['port'] if port is None: port = 80 diff --git a/packetraven/gui/base.py b/packetraven/gui/base.py index 7e6e97d5..b43531ad 100644 --- a/packetraven/gui/base.py +++ b/packetraven/gui/base.py @@ -16,12 +16,12 @@ from packetraven.__main__ import DEFAULT_INTERVAL_SECONDS, LOGGER, retrieve_packets from packetraven.connections.base import available_serial_ports, next_open_serial_port from packetraven.connections.file import PacketGeoJSON -from packetraven.connections.internet import APRSis +from packetraven.connections.internet import APRSis, RockBLOCK from packetraven.gui.plotting import LivePlot from packetraven.packets import APRSPacket, LocationPacket from packetraven.packets.tracks import LocationPacketTrack, PredictedTrajectory from packetraven.packets.writer import write_packet_tracks -from packetraven.predicts import get_predictions, PredictionError +from packetraven.predicts import PredictionError, get_predictions from packetraven.utilities import get_logger @@ -48,6 +48,12 @@ def __init__( self.__configuration = { 'aprs_fi': {'aprs_fi_key': None}, 'tnc': {'tnc': None}, + 'rockblock': { + 'rockblock_hostname': None, + 'rockblock_imei': None, + 'rockblock_username': None, + 'rockblock_password': None, + }, 'database': { 'database_hostname': None, 'database_database': None, @@ -548,7 +554,7 @@ def toggle(self): connection.location for connection in self.__connections if isinstance(connection, SerialTNC) - or isinstance(connection, RawAPRSTextFile) + or isinstance(connection, RawAPRSTextFile) ] api_key = self.__configuration['aprs_fi']['aprs_fi_key'] @@ -567,6 +573,22 @@ def toggle(self): except Exception as error: connection_errors.append(f'aprs.fi - {error}') + if ( + 'rockblock' in self.__configuration + and self.__configuration['rockblock']['rockblock_hostname'] is not None + ): + try: + rockblock = RockBLOCK( + imei=None, + username=self.__configuration['rockblock']['rockblock_username'], + password=self.__configuration['rockblock']['rockblock_password'], + ) + rockblock.start_listening(self.__configuration['rockblock']['rockblock_hostname']) + LOGGER.info(f'connected to {rockblock.location}') + self.__connections.append(rockblock) + except ConnectionError as error: + connection_errors.append(f'rockblock - {error}') + if ( 'database' in self.__configuration and self.__configuration['database']['database_hostname'] is not None @@ -1074,7 +1096,7 @@ def __add_callsign_window(self, callsign: str) -> teek.Window: sticky='w', row=self.__elements[f'{callsign}.distance_overground'].grid_info()['row'], column=self.__elements[f'{callsign}.distance_overground'].grid_info()['column'] - + 3, + + 3, ) separator = teek.Separator(window, orient='vertical') @@ -1198,9 +1220,9 @@ def __add_sources_window(self) -> teek.Window: sticky='w', row=self.__elements[f'sources.source_{index}_location'].grid_info()['row'], column=self.__elements[f'sources.source_{index}_location'].grid_info()[ - 'column' - ] - + 1, + 'column' + ] + + 1, ) self.__replace_text( self.__elements[f'sources.source_{index}_location'], connection.location, @@ -1242,9 +1264,9 @@ async def __update_sources_window(self, sources: {str: [LocationPacket]}): f'sources.source_{index}_location' ].grid_info()['row'], column=self.__elements[ - f'sources.source_{index}_location' - ].grid_info()['column'] - + 1, + f'sources.source_{index}_location' + ].grid_info()['column'] + + 1, ) self.__replace_text( self.__elements[f'sources.source_{index}_location'], From 4707eb92d97352a31c47dd08ed27a192ea05099a Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Sun, 14 Nov 2021 08:46:40 -0500 Subject: [PATCH 7/8] add ability to watch CSV from `rockblock-tools` --- packetraven/__main__.py | 45 +++++++++++++++-------------- packetraven/connections/internet.py | 7 +++-- packetraven/gui/base.py | 38 +++++++++++++++++------- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/packetraven/__main__.py b/packetraven/__main__.py index 98c74a47..b35cf6f9 100644 --- a/packetraven/__main__.py +++ b/packetraven/__main__.py @@ -25,7 +25,7 @@ from packetraven.packets import APRSPacket from packetraven.packets.tracks import APRSTrack, LocationPacketTrack from packetraven.packets.writer import write_packet_tracks -from packetraven.predicts import PredictionAPIURL, PredictionError, get_predictions +from packetraven.predicts import get_predictions, PredictionAPIURL, PredictionError from packetraven.utilities import get_logger, read_configuration, repository_root LOGGER = get_logger('packetraven', log_format='%(asctime)s | %(levelname)-8s | %(message)s') @@ -43,17 +43,14 @@ def main(): args_parser.add_argument( '--tnc', help='comma-separated list of serial ports / text files of a TNC parsing APRS packets from analog audio to ASCII' - ' (set to `auto` to use the first open serial port)', + ' (set to `auto` to use the first open serial port)', ) args_parser.add_argument( '--rockblock', default='localhost:80', - help='listen to incoming POST requests (defaults to `localhost:80`)' - ) - args_parser.add_argument( - '--imei', - help='IMEI of RockBLOCK modem' + help='listen to incoming POST requests (defaults to `localhost:80`)', ) + args_parser.add_argument('--imei', help='IMEI of RockBLOCK modem') args_parser.add_argument( '--database', help='PostGres database table `user@hostname:port/database/table`' ) @@ -121,20 +118,26 @@ def main(): kwargs['tnc'] = [tnc.strip() for tnc in args.tnc.split(',')] if args.rockblock is not None: - listen_hostname = parse_hostname(args.rockblock) - - hostname = listen_hostname['hostname'] - port = listen_hostname['port'] - username = listen_hostname['username'] - password = listen_hostname['password'] - - if port is not None: - hostname = f'{hostname}:{port}' - - kwargs['rockblock_hostname'] = hostname - kwargs['rockblock_imei'] = args.imei - kwargs['rockblock_username'] = username - kwargs['rockblock_password'] = password + try: + if Path(args.rockblock).exists(): + kwargs['rockblock_csv'] = args.rockblock + else: + raise FileNotFoundError + except FileNotFoundError: + listen_hostname = parse_hostname(args.rockblock) + + hostname = listen_hostname['hostname'] + port = listen_hostname['port'] + username = listen_hostname['username'] + password = listen_hostname['password'] + + if port is not None: + hostname = f'{hostname}:{port}' + + kwargs['rockblock_hostname'] = hostname + kwargs['rockblock_imei'] = args.imei + kwargs['rockblock_username'] = username + kwargs['rockblock_password'] = password if args.database is not None: database = parse_hostname(args.database) diff --git a/packetraven/connections/internet.py b/packetraven/connections/internet.py index 7d4744f5..0f79a09c 100644 --- a/packetraven/connections/internet.py +++ b/packetraven/connections/internet.py @@ -1,6 +1,7 @@ from argparse import Namespace from datetime import datetime, timedelta from io import StringIO +from tempfile import NamedTemporaryFile, TemporaryFile from time import sleep from typing import Any, Sequence @@ -151,12 +152,12 @@ def __init__(self, imei: str = None, username: str = None, password: str = None) self.__last_access_time = None self.__parsed_lines = [] - self.__csv_stream = StringIO() + self.__csv_stream = NamedTemporaryFile() formatter_options = Namespace() setattr(formatter_options, 'data_format', 'raw') - setattr(formatter_options, 'csv_file', self.__csv_stream) + setattr(formatter_options, 'csv_file', self.__csv_stream.name) self.__csv_formatter = CSVFormatter(formatter_options) - self.__csv_parser = RockBLOCKtoolsCSV(self.__csv_stream) + self.__csv_parser = RockBLOCKtoolsCSV(self.__csv_stream.name) def start_listening(self, hostname: str = None, port: int = None): """ diff --git a/packetraven/gui/base.py b/packetraven/gui/base.py index b43531ad..96ecf853 100644 --- a/packetraven/gui/base.py +++ b/packetraven/gui/base.py @@ -15,13 +15,13 @@ from packetraven import APRSDatabaseTable, APRSfi, RawAPRSTextFile, SerialTNC from packetraven.__main__ import DEFAULT_INTERVAL_SECONDS, LOGGER, retrieve_packets from packetraven.connections.base import available_serial_ports, next_open_serial_port -from packetraven.connections.file import PacketGeoJSON +from packetraven.connections.file import PacketGeoJSON, RockBLOCKtoolsCSV from packetraven.connections.internet import APRSis, RockBLOCK from packetraven.gui.plotting import LivePlot from packetraven.packets import APRSPacket, LocationPacket from packetraven.packets.tracks import LocationPacketTrack, PredictedTrajectory from packetraven.packets.writer import write_packet_tracks -from packetraven.predicts import PredictionError, get_predictions +from packetraven.predicts import get_predictions, PredictionError from packetraven.utilities import get_logger @@ -49,6 +49,7 @@ def __init__( 'aprs_fi': {'aprs_fi_key': None}, 'tnc': {'tnc': None}, 'rockblock': { + 'rockblock_csv': None, 'rockblock_hostname': None, 'rockblock_imei': None, 'rockblock_username': None, @@ -554,7 +555,7 @@ def toggle(self): connection.location for connection in self.__connections if isinstance(connection, SerialTNC) - or isinstance(connection, RawAPRSTextFile) + or isinstance(connection, RawAPRSTextFile) ] api_key = self.__configuration['aprs_fi']['aprs_fi_key'] @@ -573,6 +574,19 @@ def toggle(self): except Exception as error: connection_errors.append(f'aprs.fi - {error}') + if ( + 'rockblock' in self.__configuration + and self.__configuration['rockblock']['rockblock_csv'] is not None + ): + try: + rockblock = RockBLOCKtoolsCSV( + self.__configuration['rockblock']['rockblock_csv'] + ) + LOGGER.info(f'connected to {rockblock.location}') + self.__connections.append(rockblock) + except ConnectionError as error: + connection_errors.append(f'rockblock - {error}') + if ( 'rockblock' in self.__configuration and self.__configuration['rockblock']['rockblock_hostname'] is not None @@ -583,7 +597,9 @@ def toggle(self): username=self.__configuration['rockblock']['rockblock_username'], password=self.__configuration['rockblock']['rockblock_password'], ) - rockblock.start_listening(self.__configuration['rockblock']['rockblock_hostname']) + rockblock.start_listening( + self.__configuration['rockblock']['rockblock_hostname'] + ) LOGGER.info(f'connected to {rockblock.location}') self.__connections.append(rockblock) except ConnectionError as error: @@ -1096,7 +1112,7 @@ def __add_callsign_window(self, callsign: str) -> teek.Window: sticky='w', row=self.__elements[f'{callsign}.distance_overground'].grid_info()['row'], column=self.__elements[f'{callsign}.distance_overground'].grid_info()['column'] - + 3, + + 3, ) separator = teek.Separator(window, orient='vertical') @@ -1220,9 +1236,9 @@ def __add_sources_window(self) -> teek.Window: sticky='w', row=self.__elements[f'sources.source_{index}_location'].grid_info()['row'], column=self.__elements[f'sources.source_{index}_location'].grid_info()[ - 'column' - ] - + 1, + 'column' + ] + + 1, ) self.__replace_text( self.__elements[f'sources.source_{index}_location'], connection.location, @@ -1264,9 +1280,9 @@ async def __update_sources_window(self, sources: {str: [LocationPacket]}): f'sources.source_{index}_location' ].grid_info()['row'], column=self.__elements[ - f'sources.source_{index}_location' - ].grid_info()['column'] - + 1, + f'sources.source_{index}_location' + ].grid_info()['column'] + + 1, ) self.__replace_text( self.__elements[f'sources.source_{index}_location'], From ad10340b0accaa427010bcc48c99788b920f1eb8 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Sun, 14 Nov 2021 09:18:01 -0500 Subject: [PATCH 8/8] fix rockblock defaults --- packetraven/__main__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packetraven/__main__.py b/packetraven/__main__.py index b35cf6f9..6f1c1881 100644 --- a/packetraven/__main__.py +++ b/packetraven/__main__.py @@ -46,11 +46,9 @@ def main(): ' (set to `auto` to use the first open serial port)', ) args_parser.add_argument( - '--rockblock', - default='localhost:80', - help='listen to incoming POST requests (defaults to `localhost:80`)', + '--rockblock', help='listen to incoming POST requests (defaults to `localhost:80`)', ) - args_parser.add_argument('--imei', help='IMEI of RockBLOCK modem') + args_parser.add_argument('--rockblock-imei', help='IMEI of RockBLOCK modem') args_parser.add_argument( '--database', help='PostGres database table `user@hostname:port/database/table`' ) @@ -135,7 +133,7 @@ def main(): hostname = f'{hostname}:{port}' kwargs['rockblock_hostname'] = hostname - kwargs['rockblock_imei'] = args.imei + kwargs['rockblock_imei'] = args.rockblock_imei kwargs['rockblock_username'] = username kwargs['rockblock_password'] = password