From d27ccbd7833172851d443c466760a6c6b3a3239f Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Mon, 22 Aug 2022 18:51:41 +0200 Subject: [PATCH 01/15] Add a concept of ambigous KOD table If the cracking heuristics can not completely deduct a KOD table, allow for unresolved entries with the value of -1 To fix up some of these entries, the user can provide a new fix for the table via the -f option. To further guide the user, print out preliminarily decoded struc records and some stats about duplicated and missing KOD mappings. --- crodump/crodump.py | 70 +++++++++++++++++++++++++++++++++++++++++-- crodump/hexdump.py | 19 +++++++++++- crodump/koddecoder.py | 12 ++++++-- 3 files changed, 95 insertions(+), 6 deletions(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index 7153f02..ed7c251 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -1,5 +1,6 @@ from .kodump import kod_hexdump -from .hexdump import unhex, tohex +from .koddecoder import INITIAL_KOD +from .hexdump import unhex, tohex, asambigoushex, asasc, as1251 from .readers import ByteReader from .Database import Database from .Datamodel import TableDefinition @@ -130,11 +131,73 @@ def strucrack(kod, args): for ofs, byte in enumerate(data): xref[(ofs+i+1)%256][byte] += 1 - KOD = [0] * 256 + KOD = [-1] * 256 for i, xx in enumerate(xref): k, v = max(enumerate(xx), key=lambda kv: kv[1]) + +# Display the confidence, matches under 3 usually are unreliable +# print("%02x :: %02x :: %d" % (i, k, v)) KOD[k] = i +# Test deducted KOD against the default one, for debugging purposes +# if KOD[k] != INITIAL_KOD[k]: +# print("# KOD[%02x] == %02x, should be %02x" % (i, KOD[i], INITIAL_KOD[i])) +# KOD[k] = -1 + + for fix in args.fix or []: + # read format cciioo for hex char, KOD index and step offset + if len(fix) == 6 and fix[1] != "=": + c, i, o = unhex(fix) + + # read format ciioo for ascii + if len(fix) == 6 and fix[1] == "=": + c, = as1251(fix[0]) + i, o = unhex(fix[2:]) + + if len(fix) == 5: + c, = as1251(fix[0]) + i, o = unhex(fix[1:]) + + KOD[i] = (c + o) % 256 + # print("%02x %02x %02x" % (c, i, o)) + + import crodump.koddecoder + kod = crodump.koddecoder.new(KOD) + + # Dump partially decoded stru records for the user to try to spot patterns + for i, data in enumerate(table.enumrecords()): + if not data: continue + candidate = kod.try_decode(i + 1, data) + p_hex = asambigoushex(candidate) + p_able = asasc(candidate) + data_chunks = [data[j:j+24] for j in range(0, len(data), 24)] + text_chunks = [p_able[j:j+24] for j in range(0, len(p_able), 24)] + hex_chunks = [p_hex[j:j+48] for j in range(0, len(p_hex), 48)] + for ofs, chunk in enumerate(text_chunks): + kh = " ".join([ "%c=%02x%02x" % (chunk[o], b, (24 * ofs + i + 1 + o) % 256) for o, b in enumerate(data_chunks[ofs])]) + print ("%05d %-24s : %-48s : %s" % (24 * ofs, chunk, hex_chunks[ofs], kh)) + print() + + # Show duplicates that may arise by the user forcing KOD entries from command line + duplicates = [(o, v) for o, v in enumerate(KOD) if KOD.count(v) > 1 and v >= 0] + if len(duplicates): + print("duplicates found: " + ", ".join(["[%02x=>%02x]" % (o, v) for o, v in duplicates])) + + # If the KOD is not completely resolved, show the missing mappings + unset_count = KOD.count(-1) + if unset_count > 0: + if not args.silent: + unset_fields = ", ".join(["%02x" % o for o, v in enumerate(KOD) if v == -1]) + unused_values = ", ".join(["%02x" % v for v in sorted(set(range(0,256)).difference(set(KOD)))]) + print("Missing mappings: [%s] => [%s]\n" % (unset_fields, unused_values )) + print("ambigous result when cracking. %d fields unsolved." % unset_count ) + print("KOD estimate:") + print(asambigoushex(KOD)) + + print("\nIf you can provide clues for unresolved KOD entries by looking at the output, pass them via") + print("crodump strucrack -f B=f103 -f Ф2305 -f 011725") + return [0 if _ < 0 else _ for _ in KOD] + if not args.silent: print(tohex(bytes(KOD))) @@ -226,7 +289,7 @@ def main(): p.add_argument("--find1d", action="store_true", help="Find records with 0x1d in it") p.add_argument("--stats", action="store_true", help="calc table stats from the first byte of each record",) p.add_argument("--index", action="store_true", help="dump CroIndex") - p.add_argument("--stru", action="store_true", help="dump CroIndex") + p.add_argument("--stru", action="store_true", help="dump CroStru") p.add_argument("--bank", action="store_true", help="dump CroBank") p.add_argument("--sys", action="store_true", help="dump CroSys") p.add_argument("dbdir", type=str) @@ -247,6 +310,7 @@ def main(): p = subparsers.add_parser("strucrack", help="Crack v4 KOD encrypion, bypassing the need for the database password.") p.add_argument("--sys", action="store_true", help="Use CroSys for cracking") p.add_argument("--silent", action="store_true", help="no output") + p.add_argument("--fix", "-f", action="append", dest="fix", help="force KOD entries after identification") p.add_argument("dbdir", type=str) p.set_defaults(handler=strucrack) diff --git a/crodump/hexdump.py b/crodump/hexdump.py index eb82eba..74e35fb 100644 --- a/crodump/hexdump.py +++ b/crodump/hexdump.py @@ -22,6 +22,23 @@ def ashex(line): """ return " ".join("%02x" % _ for _ in line) +def asambigoushex(line): + """ + convert an array to a list of 2-digit hex values with potentially unset values of -1 + """ + return "".join("%02x" % _ if _ >= 0 else "??" for _ in line) + +def as1251(b): + """ + convert a unicode character to a CP-1251 byte + This will help parse cyrillic user entries from command line. + """ + try: + c = str(b).encode("cp1251") + return bytes(c) + except: + pass + return bytes(".") def aschr(b): """ @@ -45,7 +62,7 @@ def asasc(line): """ convert a CP-1251 encoded byte-array to a line of unicode characters. """ - return "".join(aschr(_) for _ in line) + return "".join(aschr(_) if _ >= 0 else '?' for _ in line) def hexdump(ofs, data, args): diff --git a/crodump/koddecoder.py b/crodump/koddecoder.py index 56f8457..ce3d6ef 100644 --- a/crodump/koddecoder.py +++ b/crodump/koddecoder.py @@ -30,9 +30,10 @@ def __init__(self, initial=INITIAL_KOD): self.kod = [_ for _ in initial] # calculate the inverse table. - self.inv = [0 for _ in initial] + self.inv = [-1 for _ in initial] for i, x in enumerate(self.kod): - self.inv[x] = i + if x >= 0: + self.inv[x] = i def decode(self, o, data): """ @@ -41,6 +42,13 @@ def decode(self, o, data): """ return bytes((self.kod[b] - i - o) % 256 for i, b in enumerate(data)) + def try_decode(self, o, data): + """ + decode : shift, a[0]..a[n-1] -> b[0]..b[n-1] + b[i] = KOD[a[i]]- (i+shift) + """ + return [(self.kod[b] - i - o) % 256 if self.kod[b] >= 0 else -1 for i, b in enumerate(data)] + def encode(self, o, data): """ encode : shift, b[0]..b[n-1] -> a[0]..a[n-1] From 173c6c46b47bfbdc8981db43873a12d74373fa14 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Tue, 23 Aug 2022 10:57:50 +0200 Subject: [PATCH 02/15] Invert syntax for user provided cod fixes and make width dynamic --- crodump/crodump.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index ed7c251..e32704b 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -145,18 +145,15 @@ def strucrack(kod, args): # KOD[k] = -1 for fix in args.fix or []: - # read format cciioo for hex char, KOD index and step offset - if len(fix) == 6 and fix[1] != "=": - c, i, o = unhex(fix) + if len(fix) != 6: + print("Invalid Fix format. Use xxyy=C or xxyycc") + continue - # read format ciioo for ascii - if len(fix) == 6 and fix[1] == "=": - c, = as1251(fix[0]) - i, o = unhex(fix[2:]) - - if len(fix) == 5: - c, = as1251(fix[0]) - i, o = unhex(fix[1:]) + if (fix[4] != "="): + i, o, c = unhex(fix) + else: + i, o = unhex(fix[0:4]) + c, = as1251(fix[5:]) KOD[i] = (c + o) % 256 # print("%02x %02x %02x" % (c, i, o)) @@ -165,17 +162,19 @@ def strucrack(kod, args): kod = crodump.koddecoder.new(KOD) # Dump partially decoded stru records for the user to try to spot patterns + w = args.width + display_string = "%%05d %%-%ds : %%-%ds : %%s" % (w, 2 * w) for i, data in enumerate(table.enumrecords()): if not data: continue candidate = kod.try_decode(i + 1, data) p_hex = asambigoushex(candidate) p_able = asasc(candidate) - data_chunks = [data[j:j+24] for j in range(0, len(data), 24)] - text_chunks = [p_able[j:j+24] for j in range(0, len(p_able), 24)] - hex_chunks = [p_hex[j:j+48] for j in range(0, len(p_hex), 48)] + data_chunks = [data[j:j+w] for j in range(0, len(data), w)] + text_chunks = [p_able[j:j+w] for j in range(0, len(p_able), w)] + hex_chunks = [p_hex[j:j+2*w] for j in range(0, len(p_hex), 2*w)] for ofs, chunk in enumerate(text_chunks): - kh = " ".join([ "%c=%02x%02x" % (chunk[o], b, (24 * ofs + i + 1 + o) % 256) for o, b in enumerate(data_chunks[ofs])]) - print ("%05d %-24s : %-48s : %s" % (24 * ofs, chunk, hex_chunks[ofs], kh)) + kh = " ".join([ "%02x%02x=%c" % (b, (w * ofs + i + 1 + o) % 256, chunk[o]) for o, b in enumerate(data_chunks[ofs])]) + print (display_string % (w * ofs, chunk, hex_chunks[ofs], kh)) print() # Show duplicates that may arise by the user forcing KOD entries from command line @@ -195,7 +194,7 @@ def strucrack(kod, args): print(asambigoushex(KOD)) print("\nIf you can provide clues for unresolved KOD entries by looking at the output, pass them via") - print("crodump strucrack -f B=f103 -f Ф2305 -f 011725") + print("crodump strucrack -f f103=B -f f10325") return [0 if _ < 0 else _ for _ in KOD] if not args.silent: @@ -311,6 +310,8 @@ def main(): p.add_argument("--sys", action="store_true", help="Use CroSys for cracking") p.add_argument("--silent", action="store_true", help="no output") p.add_argument("--fix", "-f", action="append", dest="fix", help="force KOD entries after identification") + p.add_argument("--width", "-w", type=int, help="max number of decoded characters on screen", default=24) + p.add_argument("dbdir", type=str) p.set_defaults(handler=strucrack) From 5404833d647686f22fcce8de57869f73e55696e2 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Sat, 27 Aug 2022 02:24:18 +0200 Subject: [PATCH 03/15] Add a concept of confidence in KOD entries and options to output them and decoded candidates in color --- crodump/crodump.py | 77 +++++++++++++++++++++++++++++++++---------- crodump/hexdump.py | 12 ++++--- crodump/koddecoder.py | 34 ++++++++++++++++--- 3 files changed, 96 insertions(+), 27 deletions(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index e32704b..eb37689 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -1,6 +1,6 @@ from .kodump import kod_hexdump -from .koddecoder import INITIAL_KOD -from .hexdump import unhex, tohex, asambigoushex, asasc, as1251 +from .koddecoder import INITIAL_KOD, match_with_mismatches +from .hexdump import unhex, tohex, asambigoushex, asasc, aschr, as1251 from .readers import ByteReader from .Database import Database from .Datamodel import TableDefinition @@ -104,6 +104,19 @@ def destruct(kod, args): elif args.type == 3: destruct_sys_definition(args, data) +def color_code(c, confidence, force): + from sys import stdout + is_a_tty = hasattr(stdout, 'isatty') and stdout.isatty() + if not force and not is_a_tty: + return c + + if confidence == 0: + return "\033[31m" + c + "\033[0m" + if confidence == 255: + return "\033[32m" + c + "\033[0m" + if confidence > 3: + return "\033[93m" + c + "\033[0m" + return "\033[35m" + c + "\033[0m" def strucrack(kod, args): """ @@ -131,13 +144,15 @@ def strucrack(kod, args): for ofs, byte in enumerate(data): xref[(ofs+i+1)%256][byte] += 1 - KOD = [-1] * 256 + KOD = [0] * 256 + KOD_CONFIDENCE = [0] * 256 for i, xx in enumerate(xref): k, v = max(enumerate(xx), key=lambda kv: kv[1]) # Display the confidence, matches under 3 usually are unreliable # print("%02x :: %02x :: %d" % (i, k, v)) KOD[k] = i + KOD_CONFIDENCE[k] = v # Test deducted KOD against the default one, for debugging purposes # if KOD[k] != INITIAL_KOD[k]: @@ -156,25 +171,50 @@ def strucrack(kod, args): c, = as1251(fix[5:]) KOD[i] = (c + o) % 256 + KOD_CONFIDENCE[i] = 255 # print("%02x %02x %02x" % (c, i, o)) import crodump.koddecoder - kod = crodump.koddecoder.new(KOD) + kod = crodump.koddecoder.new(KOD, KOD_CONFIDENCE) + + known_strings = [ + (b'\x08BankName', 5), + (b'\x0f' + as1251("Системный номер"), 6) + ] + + force_color = args.color # Dump partially decoded stru records for the user to try to spot patterns w = args.width display_string = "%%05d %%-%ds : %%-%ds : %%s" % (w, 2 * w) for i, data in enumerate(table.enumrecords()): if not data: continue - candidate = kod.try_decode(i + 1, data) - p_hex = asambigoushex(candidate) - p_able = asasc(candidate) - data_chunks = [data[j:j+w] for j in range(0, len(data), w)] - text_chunks = [p_able[j:j+w] for j in range(0, len(p_able), w)] - hex_chunks = [p_hex[j:j+2*w] for j in range(0, len(p_hex), 2*w)] - for ofs, chunk in enumerate(text_chunks): - kh = " ".join([ "%02x%02x=%c" % (b, (w * ofs + i + 1 + o) % 256, chunk[o]) for o, b in enumerate(data_chunks[ofs])]) - print (display_string % (w * ofs, chunk, hex_chunks[ofs], kh)) + candidate, candidate_confidence = kod.try_decode(i + 1, data) + + for s, maxsubs in known_strings: + incomplete_matches = match_with_mismatches(candidate, candidate_confidence, s, maxsubs) + # print(sisnm) + for ofix in incomplete_matches: + do = ofix[0] + print("Found %s which looks a lot like %s " % (asasc(candidate[do:do+len(s)]), asasc(s)) ) + print("Add the following switches to your command line to fix the decoder box:\n ", end='') + for o, c in enumerate(s): + print("-f %02x%02x%02x " % (data[do + o], (do + i + 1 + o) % 256, c), end='') + print("\n") + + candidate_chunks = [candidate[j:j+w] for j in range(0, len(candidate), w)] + for ofs, chunk in enumerate(candidate_chunks): + confidence = candidate_confidence[ofs * w:ofs * w + w] + text = asasc(chunk, confidence) + hexed = asambigoushex(chunk, confidence) + + colored = "".join(color_code(c, confidence[o], force_color) for o, c in enumerate(text)) + colored_hexed = "".join(color_code(c, confidence[o>>1], force_color) for o, c in enumerate(hexed)) + fix_helper = " ".join("%02x%02x=%s" % (b, (w * ofs + i + 1 + o) % 256, color_code(text[o], confidence[o], force_color)) for o, b in enumerate(data[ofs * w:ofs * w + w])) + + padding = " " * (w - len(chunk)) + + print (display_string % (w * ofs, colored + padding, colored_hexed + padding * 2, fix_helper)) print() # Show duplicates that may arise by the user forcing KOD entries from command line @@ -183,19 +223,19 @@ def strucrack(kod, args): print("duplicates found: " + ", ".join(["[%02x=>%02x]" % (o, v) for o, v in duplicates])) # If the KOD is not completely resolved, show the missing mappings - unset_count = KOD.count(-1) + unset_count = KOD_CONFIDENCE.count(0) if unset_count > 0: if not args.silent: - unset_fields = ", ".join(["%02x" % o for o, v in enumerate(KOD) if v == -1]) + unset_fields = ", ".join(["%02x" % o for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] == 0]) unused_values = ", ".join(["%02x" % v for v in sorted(set(range(0,256)).difference(set(KOD)))]) print("Missing mappings: [%s] => [%s]\n" % (unset_fields, unused_values )) print("ambigous result when cracking. %d fields unsolved." % unset_count ) print("KOD estimate:") - print(asambigoushex(KOD)) + print("".join(color_code("%02x" % c if KOD_CONFIDENCE[o] > 0 else "??", KOD_CONFIDENCE[o], force_color) for o, c in enumerate(KOD) )) print("\nIf you can provide clues for unresolved KOD entries by looking at the output, pass them via") - print("crodump strucrack -f f103=B -f f10325") - return [0 if _ < 0 else _ for _ in KOD] + print("crodump strucrack -f f103=B -f f10342") + return [0 if KOD_CONFIDENCE[o] == 0 else _ for o, _ in enumerate(KOD)] if not args.silent: print(tohex(bytes(KOD))) @@ -309,6 +349,7 @@ def main(): p = subparsers.add_parser("strucrack", help="Crack v4 KOD encrypion, bypassing the need for the database password.") p.add_argument("--sys", action="store_true", help="Use CroSys for cracking") p.add_argument("--silent", action="store_true", help="no output") + p.add_argument("--color", action="store_true", help="force color output even on non-ttys") p.add_argument("--fix", "-f", action="append", dest="fix", help="force KOD entries after identification") p.add_argument("--width", "-w", type=int, help="max number of decoded characters on screen", default=24) diff --git a/crodump/hexdump.py b/crodump/hexdump.py index 74e35fb..f8b2f35 100644 --- a/crodump/hexdump.py +++ b/crodump/hexdump.py @@ -22,11 +22,11 @@ def ashex(line): """ return " ".join("%02x" % _ for _ in line) -def asambigoushex(line): +def asambigoushex(line, confidence): """ convert an array to a list of 2-digit hex values with potentially unset values of -1 """ - return "".join("%02x" % _ if _ >= 0 else "??" for _ in line) + return "".join("%02x" % _ if confidence[o] > 0 else "??" for o, _ in enumerate(line)) def as1251(b): """ @@ -58,12 +58,14 @@ def aschr(b): return "." -def asasc(line): +def asasc(line, confidence=None): """ convert a CP-1251 encoded byte-array to a line of unicode characters. """ - return "".join(aschr(_) if _ >= 0 else '?' for _ in line) - + if confidence == None: + return "".join(aschr(_) for _ in line) + else: + return "".join(aschr(_) if confidence[o] > 0 else "?" for o, _ in enumerate(line)) def hexdump(ofs, data, args): """ diff --git a/crodump/koddecoder.py b/crodump/koddecoder.py index ce3d6ef..789b24d 100644 --- a/crodump/koddecoder.py +++ b/crodump/koddecoder.py @@ -26,13 +26,14 @@ class KODcoding: class handing KOD encoding and decoding, optionally with a user specified KOD table. """ - def __init__(self, initial=INITIAL_KOD): + def __init__(self, initial=INITIAL_KOD, confidence=[255] * 256): self.kod = [_ for _ in initial] + self.confidence = confidence # calculate the inverse table. - self.inv = [-1 for _ in initial] + self.inv = [0 for _ in initial] for i, x in enumerate(self.kod): - if x >= 0: + if confidence[i]: self.inv[x] = i def decode(self, o, data): @@ -47,7 +48,10 @@ def try_decode(self, o, data): decode : shift, a[0]..a[n-1] -> b[0]..b[n-1] b[i] = KOD[a[i]]- (i+shift) """ - return [(self.kod[b] - i - o) % 256 if self.kod[b] >= 0 else -1 for i, b in enumerate(data)] + return ( + [(self.kod[b] - i - o) % 256 if self.confidence[b] > 0 else 0 for i, b in enumerate(data)], + [self.confidence[b] for b in data] + ) def encode(self, o, data): """ @@ -63,5 +67,27 @@ def new(*args): """ return KODcoding(*args) +def match_with_mismatches(data, confidence, string, maxsubs=None): + """ + find all occurences of string in data with at least one and allowing a + maximum of maxsubs substitutions + """ + + # default for maximum of substitutions is to have at least two matching chars + maxsubs = maxsubs if maxsubs is not None else max( 2, len(string) - 2) + + # if string cant fit into data, return no matches + if len(string) > len(data): + return [] + + matches = [] + for offs in range(0, len(data) - len(string)): + matching = 0 + for o, c in enumerate(string): + if data[offs + o] == c and confidence[offs + o] > 0: + matching += 1 + if matching != len(string) and matching >= maxsubs: + matches.append((offs, matching)) + return sorted(matches, key=lambda x: x[1]) From c9554a8b1830158aad8248f528d87b1c226b29e2 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Sat, 27 Aug 2022 11:45:12 +0200 Subject: [PATCH 04/15] Tweak color scheme --- crodump/crodump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index eb37689..b1be2dd 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -116,7 +116,7 @@ def color_code(c, confidence, force): return "\033[32m" + c + "\033[0m" if confidence > 3: return "\033[93m" + c + "\033[0m" - return "\033[35m" + c + "\033[0m" + return "\033[94m" + c + "\033[0m" def strucrack(kod, args): """ From 2b8783f1f13d85d0b275ed44e6fa1a4350b3b267 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Tue, 30 Aug 2022 21:15:03 +0200 Subject: [PATCH 05/15] Do padding manually --- crodump/crodump.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index b1be2dd..194bd09 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -186,9 +186,11 @@ def strucrack(kod, args): # Dump partially decoded stru records for the user to try to spot patterns w = args.width - display_string = "%%05d %%-%ds : %%-%ds : %%s" % (w, 2 * w) for i, data in enumerate(table.enumrecords()): if not data: continue + + print("Processing record number %d" % i ) + candidate, candidate_confidence = kod.try_decode(i + 1, data) for s, maxsubs in known_strings: @@ -212,24 +214,29 @@ def strucrack(kod, args): colored_hexed = "".join(color_code(c, confidence[o>>1], force_color) for o, c in enumerate(hexed)) fix_helper = " ".join("%02x%02x=%s" % (b, (w * ofs + i + 1 + o) % 256, color_code(text[o], confidence[o], force_color)) for o, b in enumerate(data[ofs * w:ofs * w + w])) + + # Can't use left padding in format string, because we have color escape codes, + # so do manual padding padding = " " * (w - len(chunk)) - print (display_string % (w * ofs, colored + padding, colored_hexed + padding * 2, fix_helper)) + print ("%05d %s : %s : %s" % (w * ofs, colored + padding, colored_hexed + padding * 2, fix_helper)) print() # Show duplicates that may arise by the user forcing KOD entries from command line - duplicates = [(o, v) for o, v in enumerate(KOD) if KOD.count(v) > 1 and v >= 0] + kod_set = [v for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] > 0] + duplicates = [(o, v) for o, v in enumerate(KOD) if kod_set.count(v) > 1 and KOD_CONFIDENCE[o] > 0] + duplicates = sorted(duplicates, key=lambda x: x[1]) if len(duplicates): - print("duplicates found: " + ", ".join(["[%02x=>%02x]" % (o, v) for o, v in duplicates])) + print("\nDuplicates found:\n" + ", ".join(color_code("[%02x=>%02x]" % (o, v), KOD_CONFIDENCE[o], force_color) for o, v in duplicates)) # If the KOD is not completely resolved, show the missing mappings unset_count = KOD_CONFIDENCE.count(0) if unset_count > 0: if not args.silent: unset_fields = ", ".join(["%02x" % o for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] == 0]) - unused_values = ", ".join(["%02x" % v for v in sorted(set(range(0,256)).difference(set(KOD)))]) - print("Missing mappings: [%s] => [%s]\n" % (unset_fields, unused_values )) - print("ambigous result when cracking. %d fields unsolved." % unset_count ) + unused_values = ", ".join(["%02x" % v for v in sorted(set(range(0,256)).difference(set(kod_set)))]) + print("\nAmbigous result when cracking. %d fields unsolved. Missing mappings:" % unset_count ) + print("[%s] => [%s]\n" % (unset_fields, unused_values )) print("KOD estimate:") print("".join(color_code("%02x" % c if KOD_CONFIDENCE[o] > 0 else "??", KOD_CONFIDENCE[o], force_color) for o, c in enumerate(KOD) )) From 0ab81d653e2148db0ce7a34ed9f591d4d94d5e99 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Wed, 14 Sep 2022 01:55:31 +0200 Subject: [PATCH 06/15] Auto resolve a single missing KOD entry --- crodump/crodump.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index 194bd09..eb1ca75 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -5,7 +5,6 @@ from .Database import Database from .Datamodel import TableDefinition - def destruct_sys3_def(rd): # todo pass @@ -174,6 +173,17 @@ def strucrack(kod, args): KOD_CONFIDENCE[i] = 255 # print("%02x %02x %02x" % (c, i, o)) + kod_set = set([v for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] > 0]) + unset_entries = [o for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] == 0] + unused_values = [v for v in sorted(set(range(0,256)).difference(kod_set))] + + # if there's only one mapping missing in KOD and only one value not used, we + # just assume those to belong together with a low confidence + if len(unset_entries) == 1 and len(unused_values) == 1: + entry = unset_entries[0] + KOD[entry] = unused_values[0] + KOD_CONFIDENCE[entry] = 1 + import crodump.koddecoder kod = crodump.koddecoder.new(KOD, KOD_CONFIDENCE) @@ -233,10 +243,10 @@ def strucrack(kod, args): unset_count = KOD_CONFIDENCE.count(0) if unset_count > 0: if not args.silent: - unset_fields = ", ".join(["%02x" % o for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] == 0]) + unset_entries = ", ".join(["%02x" % o for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] == 0]) unused_values = ", ".join(["%02x" % v for v in sorted(set(range(0,256)).difference(set(kod_set)))]) - print("\nAmbigous result when cracking. %d fields unsolved. Missing mappings:" % unset_count ) - print("[%s] => [%s]\n" % (unset_fields, unused_values )) + print("\nAmbigous result when cracking. %d entries unsolved. Missing mappings:" % unset_count ) + print("[%s] => [%s]\n" % (unset_entries, unused_values )) print("KOD estimate:") print("".join(color_code("%02x" % c if KOD_CONFIDENCE[o] > 0 else "??", KOD_CONFIDENCE[o], force_color) for o, c in enumerate(KOD) )) @@ -358,6 +368,7 @@ def main(): p.add_argument("--silent", action="store_true", help="no output") p.add_argument("--color", action="store_true", help="force color output even on non-ttys") p.add_argument("--fix", "-f", action="append", dest="fix", help="force KOD entries after identification") + p.add_argument("--text", "-t", action="append", dest="text", help="add fixed to decoder box by providing whole strings for a position in a record") p.add_argument("--width", "-w", type=int, help="max number of decoded characters on screen", default=24) p.add_argument("dbdir", type=str) From 35eedb5cea56c14f761a497fc00673f41809cbf6 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Wed, 19 Jul 2023 23:48:34 +0200 Subject: [PATCH 07/15] Fix strucrack when called from croconvert --- crodump/croconvert.py | 26 +++++++++++++------------- crodump/crodump.py | 4 +++- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/crodump/croconvert.py b/crodump/croconvert.py index 0d99e0f..34934fc 100644 --- a/crodump/croconvert.py +++ b/crodump/croconvert.py @@ -59,6 +59,9 @@ def csv_output(kod, args): filereferences.extend([field for field in record.fields if field.typ == 6]) + if args.nofiles: + return + # Write all files from the file table. This is useful for unreferenced files for table in db.enumerate_tables(files=True): filedir = "Files-" + table.abbrev @@ -94,6 +97,7 @@ def main(): parser.add_argument("--strucrack", action="store_true", help="infer the KOD sbox from CroStru.dat") parser.add_argument("--dbcrack", action="store_true", help="infer the KOD sbox from CroIndex.dat+CroBank.dat") parser.add_argument("--nokod", "-n", action="store_true", help="don't KOD decode") + parser.add_argument("--nofiles", "-F", action="store_true", help="don't export files with .csv export") parser.add_argument("dbdir", type=str) args = parser.parse_args() @@ -104,25 +108,21 @@ def main(): kod = crodump.koddecoder.new(list(unhex(args.kod))) elif args.nokod: kod = None - elif args.strucrack: - class Cls: pass - cargs = Cls() - cargs.dbdir = args.dbdir - cargs.sys = False - cargs.silent = True - cracked = strucrack(None, cargs) - if not cracked: - return - kod = crodump.koddecoder.new(cracked) - elif args.dbcrack: + elif args.strucrack or args.dbcrack: class Cls: pass cargs = Cls() cargs.dbdir = args.dbdir cargs.sys = False cargs.silent = True - cracked = dbcrack(None, cargs) + cargs.fix = [] + cargs.color = False + cargs.width = 24 + cargs.noninteractive = True + cracked = strucrack(None, cargs) if args.strucrack else dbcrack(None, cargs) if not cracked: - return + exit( + "Can't automatically crack the database password. Try using crodump strucrack and pass the database key (KOD) using --kod" + ) kod = crodump.koddecoder.new(cracked) else: kod = crodump.koddecoder.new() diff --git a/crodump/crodump.py b/crodump/crodump.py index eb1ca75..fa25819 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -242,6 +242,8 @@ def strucrack(kod, args): # If the KOD is not completely resolved, show the missing mappings unset_count = KOD_CONFIDENCE.count(0) if unset_count > 0: + if args.noninteractive: + return if not args.silent: unset_entries = ", ".join(["%02x" % o for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] == 0]) unused_values = ", ".join(["%02x" % v for v in sorted(set(range(0,256)).difference(set(kod_set)))]) @@ -368,7 +370,7 @@ def main(): p.add_argument("--silent", action="store_true", help="no output") p.add_argument("--color", action="store_true", help="force color output even on non-ttys") p.add_argument("--fix", "-f", action="append", dest="fix", help="force KOD entries after identification") - p.add_argument("--text", "-t", action="append", dest="text", help="add fixed to decoder box by providing whole strings for a position in a record") + p.add_argument("--text", "-t", action="append", dest="text", help="add fixed bytes to decoder box by providing whole strings for a position in a record") p.add_argument("--width", "-w", type=int, help="max number of decoded characters on screen", default=24) p.add_argument("dbdir", type=str) From 56ab843f167404d56026c9355b666346ace13577 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Wed, 19 Jul 2023 23:53:59 +0200 Subject: [PATCH 08/15] Add an explainer what to do with the KOD --- crodump/crodump.py | 1 + 1 file changed, 1 insertion(+) diff --git a/crodump/crodump.py b/crodump/crodump.py index fa25819..5ff5d2f 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -257,6 +257,7 @@ def strucrack(kod, args): return [0 if KOD_CONFIDENCE[o] == 0 else _ for o, _ in enumerate(KOD)] if not args.silent: + print("Use the following database key to decrypt the database with crodump or croconvert with the -f option:") print(tohex(bytes(KOD))) return KOD From 450d0dc575fbcb4ed09dd041f97f021b85618cbf Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Thu, 20 Jul 2023 00:00:48 +0200 Subject: [PATCH 09/15] Make it clearer that the strucrack subcommand for the crodump binary is the way forward --- crodump/Database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crodump/Database.py b/crodump/Database.py index 0e2dcd6..78234df 100644 --- a/crodump/Database.py +++ b/crodump/Database.py @@ -172,7 +172,7 @@ def enumerate_tables(self, files=False): dbdef = self.decode_db_definition(dbinfo[1:]) except Exception as e: print("ERROR decoding db definition: %s" % e) - print("This could possibly mean that you need to try with the --strucrack option") + print("This could possibly mean that you need to try crodump strucrack to deduct the database key first") return for k, v in dbdef.items(): From e4b2dcb0905b7f813d80a3314f5da133745fe0fb Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Thu, 20 Jul 2023 00:21:07 +0200 Subject: [PATCH 10/15] Fix: option for KOD is --kod, not -f --- crodump/Database.py | 1 + crodump/crodump.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crodump/Database.py b/crodump/Database.py index 78234df..cca79be 100644 --- a/crodump/Database.py +++ b/crodump/Database.py @@ -201,6 +201,7 @@ def enumerate_records(self, table): print("Record %d too short: -- %s" % (i+1, ashex(data)), file=stderr) except Exception as e: print("Record %d broken: ERROR '%s' -- %s" % (i+1, e, ashex(data)), file=stderr) + del data def enumerate_files(self, table): """ diff --git a/crodump/crodump.py b/crodump/crodump.py index 5ff5d2f..361c5b1 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -257,7 +257,7 @@ def strucrack(kod, args): return [0 if KOD_CONFIDENCE[o] == 0 else _ for o, _ in enumerate(KOD)] if not args.silent: - print("Use the following database key to decrypt the database with crodump or croconvert with the -f option:") + print("Use the following database key to decrypt the database with crodump or croconvert with the --kod option:") print(tohex(bytes(KOD))) return KOD From 02c7b258e3284433ced4e83ab4e3a3ef5ef07822 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Sat, 22 Jul 2023 03:28:23 +0200 Subject: [PATCH 11/15] Add a negative confidence in case a manual match has overridden an automatic one --- crodump/crodump.py | 20 +++++++++++++++----- crodump/koddecoder.py | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index 361c5b1..5ed65f2 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -109,6 +109,8 @@ def color_code(c, confidence, force): if not force and not is_a_tty: return c + if confidence < 0: + return "\033[96m" + c + "\033[0m" if confidence == 0: return "\033[31m" + c + "\033[0m" if confidence == 255: @@ -184,6 +186,15 @@ def strucrack(kod, args): KOD[entry] = unused_values[0] KOD_CONFIDENCE[entry] = 1 + # Show duplicates that may arise by the user forcing KOD entries from command line + kod_set = [v for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] > 0] + duplicates = [(o, v) for o, v in enumerate(KOD) if kod_set.count(v) > 1 and KOD_CONFIDENCE[o] > 0] + duplicates = sorted(duplicates, key=lambda x: x[1]) + + for o, v in duplicates: + if KOD_CONFIDENCE[o] < 255: + KOD_CONFIDENCE[o] = -1 + import crodump.koddecoder kod = crodump.koddecoder.new(KOD, KOD_CONFIDENCE) @@ -232,12 +243,8 @@ def strucrack(kod, args): print ("%05d %s : %s : %s" % (w * ofs, colored + padding, colored_hexed + padding * 2, fix_helper)) print() - # Show duplicates that may arise by the user forcing KOD entries from command line - kod_set = [v for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] > 0] - duplicates = [(o, v) for o, v in enumerate(KOD) if kod_set.count(v) > 1 and KOD_CONFIDENCE[o] > 0] - duplicates = sorted(duplicates, key=lambda x: x[1]) if len(duplicates): - print("\nDuplicates found:\n" + ", ".join(color_code("[%02x=>%02x]" % (o, v), KOD_CONFIDENCE[o], force_color) for o, v in duplicates)) + print("\nDuplicates found:\n" + ", ".join(color_code("[%02x=>%02x (%d)]" % (o, v, KOD_CONFIDENCE[o]), KOD_CONFIDENCE[o], force_color) for o, v in duplicates)) # If the KOD is not completely resolved, show the missing mappings unset_count = KOD_CONFIDENCE.count(0) @@ -369,6 +376,7 @@ def main(): p = subparsers.add_parser("strucrack", help="Crack v4 KOD encrypion, bypassing the need for the database password.") p.add_argument("--sys", action="store_true", help="Use CroSys for cracking") p.add_argument("--silent", action="store_true", help="no output") + p.add_argument("--noninteractive", action="store_true", help="Stop if automatic cracking fails") p.add_argument("--color", action="store_true", help="force color output even on non-ttys") p.add_argument("--fix", "-f", action="append", dest="fix", help="force KOD entries after identification") p.add_argument("--text", "-t", action="append", dest="text", help="add fixed bytes to decoder box by providing whole strings for a position in a record") @@ -397,6 +405,7 @@ class Cls: pass cargs.dbdir = args.dbdir cargs.sys = False cargs.silent = True + cargs.noninteractive = False cracked = strucrack(None, cargs) if not cracked: return @@ -407,6 +416,7 @@ class Cls: pass cargs.dbdir = args.dbdir cargs.sys = False cargs.silent = True + cargs.noninteractive = False cracked = dbcrack(None, cargs) if not cracked: return diff --git a/crodump/koddecoder.py b/crodump/koddecoder.py index 789b24d..b3fa124 100644 --- a/crodump/koddecoder.py +++ b/crodump/koddecoder.py @@ -49,7 +49,7 @@ def try_decode(self, o, data): b[i] = KOD[a[i]]- (i+shift) """ return ( - [(self.kod[b] - i - o) % 256 if self.confidence[b] > 0 else 0 for i, b in enumerate(data)], + [(self.kod[b] - i - o) % 256 if self.confidence[b] != 0 else 0 for i, b in enumerate(data)], [self.confidence[b] for b in data] ) From ca86fa457bd2bfe8ee6740217068950fd36bbad4 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Sat, 22 Jul 2023 03:29:50 +0200 Subject: [PATCH 12/15] Make debug print more useful by showing the resulting KOD index, not only its components --- crodump/crodump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index 5ed65f2..58360f3 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -173,7 +173,7 @@ def strucrack(kod, args): KOD[i] = (c + o) % 256 KOD_CONFIDENCE[i] = 255 - # print("%02x %02x %02x" % (c, i, o)) + # print("%02x %02x %02x" % ((c + o) % 256, i, o)) kod_set = set([v for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] > 0]) unset_entries = [o for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] == 0] From e9f5b1fd241c7fea76f8c45189956bd559b4f063 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Sat, 22 Jul 2023 16:21:55 +0200 Subject: [PATCH 13/15] Add the helper for known plain text --- crodump/crodump.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index 99abb13..7e256f4 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -1,6 +1,6 @@ from .kodump import kod_hexdump from .koddecoder import INITIAL_KOD, match_with_mismatches -from .hexdump import unhex, tohex, asambigoushex, asasc, aschr, as1251 +from .hexdump import unhex, tohex, asambigoushex, asasc, aschr, as1251, ashex from .readers import ByteReader from .Database import Database from .Datamodel import TableDefinition @@ -175,6 +175,17 @@ def strucrack(kod, args): KOD_CONFIDENCE[i] = 255 # print("%02x %02x %02x" % ((c + o) % 256, i, o)) + # For chunks of text where record and offset is known, set the KOD + for fix in args.text: + record, line, offset, text = fix.split(':', 4) + data = table.readrec(int(record)+1) + dataoff = int(line) + int(offset) + o = int(record) + 1 + int(line) + int(offset) + for i, c in enumerate(text): + d = data[dataoff + i] + KOD[d] = (int.from_bytes(as1251(c), "little") + o + i) % 256 + KOD_CONFIDENCE[d] = 255 + kod_set = set([v for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] > 0]) unset_entries = [o for o, v in enumerate(KOD) if KOD_CONFIDENCE[o] == 0] unused_values = [v for v in sorted(set(range(0,256)).difference(kod_set))] @@ -199,8 +210,10 @@ def strucrack(kod, args): kod = crodump.koddecoder.new(KOD, KOD_CONFIDENCE) known_strings = [ - (b'\x08BankName', 5), - (b'\x0f' + as1251("Системный номер"), 6) + (b'USERINFO', 4, b'\x08USERINFO', -1), + (b'Version', 4, b'\x07Version', -1), + (b'\x08BankName', 5, b'\x08BankName', 0), + (as1251("Системный номер"), 6, b'\x00\x00\x00\x00\x00\x00\x0f' + as1251("Системный номер") + b'\x01\x00\x00\x00\x00', -7) ] force_color = args.color @@ -214,15 +227,15 @@ def strucrack(kod, args): candidate, candidate_confidence = kod.try_decode(i + 1, data) - for s, maxsubs in known_strings: + for s, maxsubs, deststring, destoffset in known_strings: incomplete_matches = match_with_mismatches(candidate, candidate_confidence, s, maxsubs) # print(sisnm) for ofix in incomplete_matches: do = ofix[0] print("Found %s which looks a lot like %s " % (asasc(candidate[do:do+len(s)]), asasc(s)) ) print("Add the following switches to your command line to fix the decoder box:\n ", end='') - for o, c in enumerate(s): - print("-f %02x%02x%02x " % (data[do + o], (do + i + 1 + o) % 256, c), end='') + for o, c in enumerate(deststring): + print("-f %02x%02x%02x " % (data[do + o + destoffset], (do + i + 1 + o + destoffset) % 256, c), end='') print("\n") candidate_chunks = [candidate[j:j+w] for j in range(0, len(candidate), w)] From 50e7db0388212e418e90daaba5ceaf0b534cd1e3 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Fri, 4 Aug 2023 02:25:15 +0200 Subject: [PATCH 14/15] Prevent empty text args array from throwing --- crodump/crodump.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crodump/crodump.py b/crodump/crodump.py index 7e256f4..9d2c31c 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -176,7 +176,7 @@ def strucrack(kod, args): # print("%02x %02x %02x" % ((c + o) % 256, i, o)) # For chunks of text where record and offset is known, set the KOD - for fix in args.text: + for fix in args.text or []: record, line, offset, text = fix.split(':', 4) data = table.readrec(int(record)+1) dataoff = int(line) + int(offset) @@ -393,7 +393,7 @@ def main(): p.add_argument("--noninteractive", action="store_true", help="Stop if automatic cracking fails") p.add_argument("--color", action="store_true", help="force color output even on non-ttys") p.add_argument("--fix", "-f", action="append", dest="fix", help="force KOD entries after identification") - p.add_argument("--text", "-t", action="append", dest="text", help="add fixed bytes to decoder box by providing whole strings for a position in a record") + p.add_argument("--text", "-t", action="append", dest="text", help="add fixed bytes to decoder box by providing whole strings for a position in a record, format is record:line:offset:plaintext") p.add_argument("--width", "-w", type=int, help="max number of decoded characters on screen", default=24) p.add_argument("dbdir", type=str) From 88257890b9f4aeba57619b91b2c5fdf7f013dde1 Mon Sep 17 00:00:00 2001 From: Borys Kabakov Date: Tue, 3 Oct 2023 00:12:18 +0300 Subject: [PATCH 15/15] fix crodump.py (#22) --- crodump/crodump.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crodump/crodump.py b/crodump/crodump.py index 9d2c31c..55b2a1f 100644 --- a/crodump/crodump.py +++ b/crodump/crodump.py @@ -420,6 +420,10 @@ class Cls: pass cargs.sys = False cargs.silent = True cargs.noninteractive = False + # add all keys we forgot to add + for k, v in args.__dict__.items(): + if not cargs.__dict__.get(k): + cargs.__dict__.update({k: v}) cracked = strucrack(None, cargs) if not cracked: return