Skip to content

fix bug in discordChat that caused multiplied duplicate records #1503

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JamesHabben wants to merge 1 commit into
base: main
Choose a base branch
from
+53 −43
Open

fix bug in discordChat that caused multiplied duplicate records #1503

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 53 additions & 43 deletions scripts/artifacts/discordChats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
__artifacts_v2__ = {
"discordChats": { # This should match the function name exactly
"discordChats": {
"name": "Discord - Chats",
"description": "Parses Discord chat messages from FSCacheData and \'a\' database",
"author": "Original Unknown, John Hyla & @stark4n6",
Expand All @@ -8,8 +8,11 @@
"requirements": "none",
"category": "Discord",
"notes": "",
"paths": ('*/activation_record.plist', '*/com.hammerandchisel.discord/fsCachedData/*', '*/Library/Caches/kv-storage/@account*/a*', '*/Library/Caches/com.hackemist.SDImageCache/default/*'),
"output_types": "standard", # or ["html", "tsv", "timeline", "lava"]
"paths": ('*/activation_record.plist',
'*/com.hammerandchisel.discord/fsCachedData/*',
'*/Library/Caches/kv-storage/@account*/a*',
'*/Library/Caches/com.hackemist.SDImageCache/default/*'),
"output_types": "standard",
"artifact_icon": "message-circle"
}
}
Expand All @@ -21,10 +24,12 @@
import os
import re

from scripts.ilapfuncs import artifact_processor, logfunc, get_resolution_for_model_id, get_file_path, get_sqlite_db_records, check_in_media
from scripts.ilapfuncs import artifact_processor, logfunc, get_resolution_for_model_id, get_file_path, \
get_sqlite_db_records, check_in_media

@artifact_processor
def discordChats(context):
""" see artifact info """
files_found = context.get_files_found()

def reduceSize(width: int, height: int, max_width: int, max_height: int) -> (int, int):
Expand All @@ -45,31 +50,30 @@ def reduceSize(width: int, height: int, max_width: int, max_height: int) -> (int
new_width = width
new_height = height
return (new_width, new_height)

def process_json(jsonfinal):
timestamp = editedtimestamp = username = botuser = content = attachments = userid = channelid = emdeddedauthor = authorurl = authoriconurl = embededurl = embededdescript = footertext = footericonurl = pathedtail = ''

if 'author' in jsonfinal:
username = (jsonfinal['author']['username'])
userid = (jsonfinal['author']['id'])
username = jsonfinal['author']['username']
userid = jsonfinal['author']['id']
if 'bot' in jsonfinal['author']:
botuser = (jsonfinal['author']['bot'])
botuser = jsonfinal['author']['bot']
else:
botuser = ''

if 'timestamp' in jsonfinal:
timestamp = str(jsonfinal['timestamp']).replace('T',' ')

if 'edited_timestamp' in jsonfinal:
editedtimestamp = str(jsonfinal['edited_timestamp']).replace('T',' ')

if 'content' in jsonfinal:
content = jsonfinal['content']

if 'channel_id' in jsonfinal:
channelid = jsonfinal['channel_id']



if 'attachments' in jsonfinal:
attachmentsData = jsonfinal['attachments']
attachmentsArray = []
Expand Down Expand Up @@ -114,62 +118,65 @@ def process_json(jsonfinal):

#Check if a file by this name was found
if any(proxy_url_md5 in string for string in files_found):
#If Yes, generate thumbnail
#attachmentsArray.append(media_to_html(proxy_url_md5, files_found, report_folder))
# If Yes, generate thumbnail
# attachmentsArray.append(media_to_html(proxy_url_md5, files_found, report_folder))
attachment_file = check_in_media(proxy_url_md5)
attachmentsArray.append([attachment_file, proxy_url_md5])
else:
#If no, show the URL, but also show the filename we think should exist in case it can be located elsewhere
# If no, show the URL, but also show the filename we think should exist
# in case it can be located elsewhere
attachmentsArray.append([None, a.get('proxy_url') + f' ({proxy_url_md5})'])
else:
#Resolution was not found, just show the URL
attachmentsArray.append([None, a.get('proxy_url')])

if 'embeds' in jsonfinal:
if len(jsonfinal['embeds']) > 0:
y = 0
lenembeds = (len(jsonfinal['embeds']))
lenembeds = len(jsonfinal['embeds'])
while y < lenembeds:
if 'url' in jsonfinal['embeds'][y]:
embededurl = (jsonfinal['embeds'][y]['url'])
embededurl = jsonfinal['embeds'][y]['url']
else:
embededurl = ''

if 'description' in jsonfinal['embeds'][y]:
embededdescript = (jsonfinal['embeds'][y]['description'])
embededdescript = jsonfinal['embeds'][y]['description']
else:
embededdescript = ''

if 'author' in jsonfinal['embeds'][y]:
emdeddedauthor = (jsonfinal['embeds'][y]['author']['name'])
emdeddedauthor = jsonfinal['embeds'][y]['author']['name']
if 'url' in jsonfinal['embeds'][y]['author']:
authorurl = (jsonfinal['embeds'][y]['author']['url'])
authorurl = jsonfinal['embeds'][y]['author']['url']
else:
authorurl = ''
authorurl = ''
if 'icon_url' in jsonfinal['embeds'][y]['author']:
authoriconurl =(jsonfinal['embeds'][y]['author']['icon_url'])
authoriconurl = jsonfinal['embeds'][y]['author']['icon_url']
else:
authoriconurl = ''
else:
emdeddedauthor = ''

if 'footer' in jsonfinal['embeds']:
footertext = (jsonfinal['embeds'][y]['footer']['text'])
footericonurl = (jsonfinal['embeds'][y]['footer']['icon_url'])
footertext = jsonfinal['embeds'][y]['footer']['text']
footericonurl = jsonfinal['embeds'][y]['footer']['icon_url']
else:
footertext = ''
footericonurl = ''

y = y + 1
_, pathedtail = os.path.split(pathed)

_, pathedtail = os.path.split(pathed)

if timestamp == '':
pass
else:
if len(attachmentsArray) > 0:
for attach in attachmentsArray:
data_list.append((timestamp, editedtimestamp, username, botuser, content, attach[0], attach[1], userid, channelid, emdeddedauthor, authorurl, authoriconurl, embededurl, embededdescript, footertext, footericonurl, pathedtail))
data_list.append((timestamp, editedtimestamp, username, botuser, content, attach[0],\
attach[1], userid, channelid, emdeddedauthor, authorurl, authoriconurl, embededurl,\
embededdescript, footertext, footericonurl, pathedtail))
else:
data_list.append(
(timestamp, editedtimestamp, username, botuser, content, None, None, userid, channelid,
Expand Down Expand Up @@ -207,20 +214,22 @@ def process_json(jsonfinal):

data_list = []

source_path = get_file_path(files_found, "a")

for file_found in files_found:
file_found = str(file_found)
pathed = file_found
source_path = get_file_path(files_found, "a")


try:
if not file_found.endswith('activation_record.plist') and os.path.isfile(file_found) and file_found != source_path and 'com.hackemist.SDImageCache' not in file_found:
if not file_found.endswith('activation_record.plist') and os.path.isfile(file_found) \
and file_found != source_path and 'com.hackemist.SDImageCache' not in file_found:
with open(file_found, "r", encoding="utf-8") as f_in:
for jsondata in f_in:
jsonfinal = json.loads(jsondata)
if isinstance(jsonfinal, list):
for json_record in jsonfinal:
process_json(json_record)
elif source_path:
elif file_found == source_path:
query = '''select data from messages0'''

db_records = get_sqlite_db_records(source_path, query)
Expand All @@ -232,11 +241,12 @@ def process_json(jsonfinal):
if 'message' in json_load:
jsonfinal = json.loads(json.dumps(json_load['message']))
process_json(jsonfinal)

except ValueError as e:
logfunc(f"Error parsing JSON from {file_found}: {str(e)}")

data_headers = (('Timestamp', 'datetime'), ('Edited Timestamp', 'datetime'), 'Username', 'Bot?', 'Content', ('Attachment', 'media'), 'Attachment Link',
'User ID', 'Channel ID', 'Embedded Author', 'Author URL', 'Author Icon URL', 'Embedded URL', 'Embedded Script',
data_headers = (('Timestamp', 'datetime'), ('Edited Timestamp', 'datetime'), 'Username', 'Bot?', 'Content',
('Attachment', 'media'), 'Attachment Link', 'User ID', 'Channel ID', 'Embedded Author',
'Author URL', 'Author Icon URL', 'Embedded URL', 'Embedded Script',
'Footer Text', 'Footer Icon URL', 'Source File')
return data_headers, data_list, 'See source file(s) below:'
return data_headers, data_list, 'See source file(s) below:'
Loading