diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e26976d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +venv/ +venv2/ +.vscode/ +results/ \ No newline at end of file diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 3e3652b..3d085c0 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -3,3 +3,5 @@ A big thank you to everyone who is helping out with this project. Below is a list of contributors: Rongkuan Ma - University of Sheffield + +Hammond Pearce - NYU \ No newline at end of file diff --git a/INSTALL.rst b/INSTALL.rst index 7cbda42..241a1c4 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,11 +1,11 @@ -``ICSREF`` is developed in Python 2.7. To install it on a fresh Ubuntu 16.04 LTS system you can follow these steps: +``ICSREF`` is developed in Python 3+. To install it on a fresh Ubuntu 20.04 LTS system you can follow these steps: Install the system dependencies: -------------------------------- .. code-block:: none - sudo apt install git python-pip libcapstone3 python-dev libffi-dev build-essential virtualenvwrapper graphviz libgraphviz-dev graphviz-dev pkg-config + sudo apt install git python3-pip libcapstone3 python3-dev libffi-dev build-essential graphviz libgraphviz-dev graphviz-dev pkg-config Install radare2 --------------- @@ -27,16 +27,18 @@ Create a virtual environment (in a fresh shell) .. code-block:: none - mkvirtualenv icsref + virtualenv venv + source venv/bin/activate + Install the python package dependencies --------------------------------------- -To ensure stability, local wheels of the dependencies are included in the ``wheelhouse`` directory: - .. code-block:: none - pip install --no-index --find-links=wheelhouse -r requirements.txt + pip install -r requirements_full.txt + OR (sometimes the above fails) + pip install networkx r2pipe dill ujson cmd2 angr pygraphviz pymodbus testtools six==1.14.0 Create bash alias ----------------- diff --git a/icsref/PRG_analysis.py b/icsref/PRG_analysis.py index 8fb0a70..17096dc 100644 --- a/icsref/PRG_analysis.py +++ b/icsref/PRG_analysis.py @@ -15,6 +15,7 @@ import dill import struct import logging +import binascii from difflib import SequenceMatcher from glob import glob @@ -164,14 +165,14 @@ def __find_blocks(self): """ # Matches the prologue - prologue = '\x0d\xc0\xa0\xe1\x00\x58\x2d\xe9\x0c\xb0\xa0\xe1' + prologue = b'\x0d\xc0\xa0\xe1\x00\x58\x2d\xe9\x0c\xb0\xa0\xe1' beginnings = self.__allindices(self.hexdump, prologue) # Matches the epilogue - epilogue = '\x00\xa8\x1b\xe9' + epilogue = b'\x00\xa8\x1b\xe9' endings = self.__allindices(self.hexdump, epilogue) endings = [i+4 for i in endings] - return zip(beginnings, endings) + return list(zip(beginnings, endings)) def __find_functions(self): """ @@ -182,6 +183,7 @@ def __find_functions(self): # Set r2 architecture configuration - Processor specific r2.cmd('e asm.arch=arm; e asm.bits=32; e cfg.bigendian=false') # Instantiate Functions + #functions = list(self.FunctionBoundaries) for i in range(len(self.FunctionBoundaries)): # Code disassembly # Start: MOV, STMFD, MOV @@ -218,18 +220,19 @@ def __find_dynlibs(self): """ offset = self.dynlib_end # Reverse find 0xFFFF (offset for the beginning of strings) - dynlib_offset = self.hexdump.rfind('\xff\xff',0,offset) + 2 + dynlib_offset = self.hexdump.rfind(b'\xff\xff',0,offset) + 2 dynlibs = {} # Match printable ASCII characters - dynlib = re.search('[ -~]*', self.hexdump[dynlib_offset:]).group(0) + dynlib = re.search(b'[ -~]*', self.hexdump[dynlib_offset:]).group(0) # Find the offsets to dynamic libs while dynlib: dynlib_offset += len(dynlib) + 1 - temp = self.hexdump[dynlib_offset:dynlib_offset+2].encode('hex') + temp = binascii.hexlify(self.hexdump[dynlib_offset:dynlib_offset+2]) + temp = temp.decode('ascii') #this might be all kinds of weird jump_offset = int(''.join([m[2:4]+m[0:2] for m in [temp[i:i+4] for i in range(0,len(temp),4)]]),16) * 4 + 8 dynlibs[jump_offset] = dynlib dynlib_offset += 2 - dynlib = re.search('[ -~]*', self.hexdump[dynlib_offset:]).group(0) + dynlib = re.search(b'[ -~]*', self.hexdump[dynlib_offset:]).group(0) return dynlibs def __find_statlibs(self): @@ -237,9 +240,9 @@ def __find_statlibs(self): stop_offset = self.FunctionBoundaries[-1][1]-8 funs = [x for x, _ in self.FunctionBoundaries] # Change 0x2000 location of writing address in OUTRO with 0x10000000 to not overwrite code - code_start = '\x00\x20\x00\x00' - with open('temphexdump.bin', 'w') as f: - hexdump_mod = self.hexdump.replace(code_start, '\x00\x00\x00\x10') + code_start = b'\x00\x20\x00\x00' + with open('temphexdump.bin', 'wb') as f: + hexdump_mod = self.hexdump.replace(code_start, b'\x00\x00\x00\x10') f.write(hexdump_mod) proj = angr.Project('temphexdump.bin', load_options={'main_opts': {'backend': 'blob', 'custom_base_addr': 0, 'custom_arch':'ARMEL', 'custom_entry_point':0x50}, 'auto_load_libs':False}) state = proj.factory.entry_state() @@ -307,7 +310,7 @@ def __save_object(self): raise dat_f = os.path.join(path, '{}_init_analysis.dat'.format(self.name)) - with open(dat_f, 'w') as f: + with open(dat_f, 'wb') as f: dill.dump(self, f) def __allindices(self, file_bytes, sub, offset=0): @@ -317,7 +320,7 @@ def __allindices(self, file_bytes, sub, offset=0): :param: file_bytes: bytes to perform the search on :param: sub: substring to search for in file_bytes """ - + i = file_bytes.find(sub, offset) listindex=[] while i >= 0: @@ -330,7 +333,7 @@ def __strings(self): Finds consecutive <= 4-byte ASCII character strings """ strings = {} - p=re.compile('([ -~]{4,})') + p=re.compile(b'([ -~]{4,})') for m in p.finditer(self.hexdump): strings[m.start()] = m.group() return strings @@ -379,7 +382,7 @@ def __init__(self, path, start, stop, hexdump, disasm): if len(op) < 6: op_str += line[43:].split(' ')[0] # Function opcodes SHA256 hash - self.hash = hashlib.sha256(op_str).hexdigest() + self.hash = hashlib.sha256(op_str.encode()).hexdigest() # Initialize list of calls from function. Gets populated later self.calls = {} diff --git a/requirements.txt b/requirements.txt index e6dc4f5..c4ef713 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ -networkx==1.9 -r2pipe==0.9.5 -dill==0.2.7.1 -ujson==1.35 -cmd2==0.7.9 -angr==7.7.9.14 -pygraphviz==1.3.1 -pymodbus==1.4.0 -testtools==2.3.0 \ No newline at end of file +networkx +r2pipe +dill +ujson +cmd2 +angr +pygraphviz +pymodbus +testtools +six==1.14.0 \ No newline at end of file diff --git a/requirements_full.txt b/requirements_full.txt new file mode 100644 index 0000000..fad1b7b --- /dev/null +++ b/requirements_full.txt @@ -0,0 +1,75 @@ +ailment==8.20.7.27 +angr==8.20.7.27 +appdirs==1.4.3 +archinfo==8.20.7.27 +attrs==20.2.0 +bitstring==3.1.7 +CacheControl==0.12.6 +cachetools==4.1.1 +capstone==4.0.2 +certifi==2019.11.28 +cffi==1.14.2 +chardet==3.0.4 +claripy==8.20.7.27 +cle==8.20.7.27 +cmd2==1.3.9 +colorama==0.4.3 +contextlib2==0.6.0 +CppHeaderParser==2.7.4 +decorator==4.4.2 +dill==0.3.2 +distlib==0.3.0 +distro==1.4.0 +dpkt==1.9.3 +extras==1.0.0 +fixtures==3.0.0 +future==0.18.2 +gitdb==4.0.5 +GitPython==3.1.8 +html5lib==1.0.1 +idna==2.8 +ipaddr==2.2.0 +itanium-demangler==1.0 +linecache2==1.0.0 +lockfile==0.12.2 +msgpack==0.6.2 +mulpyplexer==0.8 +networkx==2.5 +packaging==20.3 +pbr==5.5.0 +pefile==2019.4.18 +pep517==0.8.2 +plumbum==1.6.9 +ply==3.11 +progress==1.5 +progressbar2==3.53.1 +protobuf==3.13.0 +psutil==5.7.2 +pycparser==2.20 +pyelftools==0.26 +pygraphviz==1.6 +pymodbus==2.3.0 +pyparsing==2.4.6 +pyperclip==1.8.0 +pyserial==3.4 +PySMT==0.9.0 +python-mimeparse==1.6.0 +python-utils==2.4.0 +pytoml==0.1.21 +pyvex==8.20.7.27 +r2pipe==1.4.2 +requests==2.22.0 +retrying==1.3.3 +rpyc==4.1.5 +six==1.14.0 +smmap==3.0.4 +sortedcontainers==2.2.2 +testtools==2.4.0 +traceback2==1.4.0 +ujson==3.2.0 +unicorn==1.0.2rc4 +unittest2==1.1.0 +urllib3==1.25.8 +wcwidth==0.2.5 +webencodings==0.5.1 +z3-solver==4.8.8.0