From fe77f0b59bc20a8c9d61f71cd4eea8a540fabdc1 Mon Sep 17 00:00:00 2001 From: varthe Date: Sat, 21 Dec 2024 20:17:32 +0000 Subject: [PATCH 1/6] fail when file name is too long --- blackhole.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/blackhole.py b/blackhole.py index ec7cb88..0380459 100644 --- a/blackhole.py +++ b/blackhole.py @@ -277,7 +277,16 @@ async def is_accessible(path, timeout=10): executor.shutdown(wait=False) time.sleep(.1) # Wait before processing the file in case it isn't fully written yet. - os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) + try: + os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) + except OSError as e: + if e.errno == 36: # File name too long + print(f"Error handling paths for {file.fileInfo.filenameWithoutExt}. Paths may be too long. Blacklisting.") + os.remove(file.fileInfo.filePath) + await fail(arr=arr, filename=file.fileInfo.filenameWithoutExt) + return False + raise + with open(file.fileInfo.filePathProcessing, 'rb' if file.torrentInfo.isDotTorrentFile else 'r') as f: fileData = f.read() @@ -315,21 +324,37 @@ async def is_accessible(path, timeout=10): discordError(f"Error processing {file.fileInfo.filenameWithoutExt}", e) -async def fail(torrent: TorrentBase, arr: Arr): +async def fail(torrent: TorrentBase = None, arr: Arr = None, filename: str = None): _print = globals()['print'] def print(*values: object): - _print(f"[{torrent.__class__.__name__}] [{torrent.file.fileInfo.filenameWithoutExt}]", *values) + source = torrent.file.fileInfo.filenameWithoutExt if torrent else filename + _print(f"[{torrent.__class__.__name__ if torrent else 'ValidationError'}] [{source}]", *values) print(f"Failing") - torrentHash = torrent.getHash() - history = await asyncio.to_thread(arr.getHistory, blackhole['historyPageSize']) - items = [item for item in history if (item.torrentInfoHash and item.torrentInfoHash.casefold() == torrentHash.casefold()) or cleanFileName(item.sourceTitle.casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()] + torrentHash = torrent.getHash() if torrent else None + cleanName = cleanFileName(filename.casefold()) if filename else None + + history = await asyncio.to_thread(arr.getHistory, blackhole['historyPageSize']) if arr else [] + + def matchesTorrent(item): + if torrentHash and item.torrentInfoHash and item.torrentInfoHash.casefold() == torrentHash.casefold(): + return True + if torrent and cleanFileName(item.sourceTitle.casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold(): + return True + return False + + def matchesFilename(item): + if cleanName and cleanFileName(item.sourceTitle.casefold()) == cleanName: + return True + return False + + items = [item for item in history if matchesTorrent(item) or matchesFilename(item)] if not items: message = "No history items found to mark as failed. Arr will not attempt to grab an alternative." print(message) - discordError(message, torrent.file.fileInfo.filenameWithoutExt) + discordError(message, torrent.file.fileInfo.filenameWithoutExt if torrent else filename) else: # TODO: See if we can fail without blacklisting as cached items constantly changes failTasks = [asyncio.to_thread(arr.failHistoryItem, item.id) for item in items] From 3e858856cb03ab86cdb6b4240d84eb45f4160430 Mon Sep 17 00:00:00 2001 From: varthe Date: Sun, 22 Dec 2024 08:20:46 +0000 Subject: [PATCH 2/6] extra logs --- blackhole.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/blackhole.py b/blackhole.py index 0380459..7d9eb75 100644 --- a/blackhole.py +++ b/blackhole.py @@ -53,7 +53,8 @@ def __init__(self, filename, isRadarr) -> None: isTorrentOrMagnet = isDotTorrentFile or filename.casefold().endswith('.magnet') filenameWithoutExt, ext = os.path.splitext(filename) filePath = os.path.join(baseBath, filename) - filePathProcessing = os.path.join(baseBath, 'processing', f"{filenameWithoutExt}_{uniqueId}{ext}") + filePathProcessing = trimProcessingFilepath(os.path.join(baseBath, 'processing'), filenameWithoutExt, f"_{uniqueId}{ext}") + #filePathProcessing = os.path.join(baseBath, 'processing', f"{filenameWithoutExt}_{uniqueId}{ext}") folderPathCompleted = os.path.join(baseBath, 'completed', filenameWithoutExt) self.fileInfo = self.FileInfo(filename, filenameWithoutExt, filePath, filePathProcessing, folderPathCompleted) @@ -72,6 +73,17 @@ def getPath(isRadarr, create=False): return finalPath +def trimProcessingFilepath(basepath, filename, ext): + MAX_PATH_LENGTH = 255 + path = os.path.join(basepath, filename + ext) + if len(path) <= MAX_PATH_LENGTH: + return path + + remainingLength = MAX_PATH_LENGTH - len(basepath) - len(ext) - 1 + return os.path.join(basepath, filename[:remainingLength] + ext) + + + # From Radarr Radarr/src/NzbDrone.Core/Organizer/FileNameBuilder.cs def cleanFileName(name): result = name @@ -277,15 +289,15 @@ async def is_accessible(path, timeout=10): executor.shutdown(wait=False) time.sleep(.1) # Wait before processing the file in case it isn't fully written yet. - try: - os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) - except OSError as e: - if e.errno == 36: # File name too long - print(f"Error handling paths for {file.fileInfo.filenameWithoutExt}. Paths may be too long. Blacklisting.") - os.remove(file.fileInfo.filePath) - await fail(arr=arr, filename=file.fileInfo.filenameWithoutExt) - return False - raise + # try: + os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) + # except OSError as e: + # if e.errno == 36: # File name too long + # print(f"Error handling paths for {file.fileInfo.filenameWithoutExt}. Paths may be too long. Blacklisting.") + # os.remove(file.fileInfo.filePath) + # await fail(arr=arr, filename=file.fileInfo.filenameWithoutExt) + # return False + # raise with open(file.fileInfo.filePathProcessing, 'rb' if file.torrentInfo.isDotTorrentFile else 'r') as f: From dac4c3cb1aff3df763874d0cd7c472024c19fbf7 Mon Sep 17 00:00:00 2001 From: varthe Date: Sun, 22 Dec 2024 08:32:44 +0000 Subject: [PATCH 3/6] Revert "fail when file name is too long" This reverts commit fe77f0b5 --- blackhole.py | 49 +++++++++++++++---------------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/blackhole.py b/blackhole.py index 7d9eb75..ad8bf0d 100644 --- a/blackhole.py +++ b/blackhole.py @@ -289,16 +289,7 @@ async def is_accessible(path, timeout=10): executor.shutdown(wait=False) time.sleep(.1) # Wait before processing the file in case it isn't fully written yet. - # try: os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) - # except OSError as e: - # if e.errno == 36: # File name too long - # print(f"Error handling paths for {file.fileInfo.filenameWithoutExt}. Paths may be too long. Blacklisting.") - # os.remove(file.fileInfo.filePath) - # await fail(arr=arr, filename=file.fileInfo.filenameWithoutExt) - # return False - # raise - with open(file.fileInfo.filePathProcessing, 'rb' if file.torrentInfo.isDotTorrentFile else 'r') as f: fileData = f.read() @@ -336,48 +327,38 @@ async def is_accessible(path, timeout=10): discordError(f"Error processing {file.fileInfo.filenameWithoutExt}", e) -async def fail(torrent: TorrentBase = None, arr: Arr = None, filename: str = None): + +async def fail(torrent: TorrentBase, arr: Arr): _print = globals()['print'] def print(*values: object): - source = torrent.file.fileInfo.filenameWithoutExt if torrent else filename - _print(f"[{torrent.__class__.__name__ if torrent else 'ValidationError'}] [{source}]", *values) + _print(f"[{torrent.__class__.__name__}] [{torrent.file.fileInfo.filenameWithoutExt}]", *values) print(f"Failing") - - torrentHash = torrent.getHash() if torrent else None - cleanName = cleanFileName(filename.casefold()) if filename else None - - history = await asyncio.to_thread(arr.getHistory, blackhole['historyPageSize']) if arr else [] - - def matchesTorrent(item): - if torrentHash and item.torrentInfoHash and item.torrentInfoHash.casefold() == torrentHash.casefold(): - return True - if torrent and cleanFileName(item.sourceTitle.casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold(): - return True - return False - - def matchesFilename(item): - if cleanName and cleanFileName(item.sourceTitle.casefold()) == cleanName: - return True - return False - items = [item for item in history if matchesTorrent(item) or matchesFilename(item)] + torrentHash = torrent.getHash() + history = await asyncio.to_thread(arr.getHistory, blackhole['historyPageSize']) + items = [item for item in history if + (item.torrentInfoHash and item.torrentInfoHash.casefold() == torrentHash.casefold()) or cleanFileName( + item.sourceTitle.casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()] if not items: message = "No history items found to mark as failed. Arr will not attempt to grab an alternative." print(message) - discordError(message, torrent.file.fileInfo.filenameWithoutExt if torrent else filename) + discordError(message, torrent.file.fileInfo.filenameWithoutExt) else: # TODO: See if we can fail without blacklisting as cached items constantly changes failTasks = [asyncio.to_thread(arr.failHistoryItem, item.id) for item in items] await asyncio.gather(*failTasks) print(f"Failed") - + + def getFiles(isRadarr): print('getFiles') - files = (TorrentFileInfo(filename, isRadarr) for filename in os.listdir(getPath(isRadarr)) if filename not in ['processing', 'completed']) + files = (TorrentFileInfo(filename, isRadarr) for filename in os.listdir(getPath(isRadarr)) if + filename not in ['processing', 'completed']) return [file for file in files if file.torrentInfo.isTorrentOrMagnet] + async def on_created(isRadarr): print("Enter 'on_created'") try: @@ -390,7 +371,7 @@ async def on_created(isRadarr): futures: list[asyncio.Future] = [] firstGo = True - + # Consider switching to a queue while firstGo or not all(future.done() for future in futures): files = getFiles(isRadarr) From 5080364acd199602710b9bc11769acc8291375a2 Mon Sep 17 00:00:00 2001 From: varthe Date: Sun, 22 Dec 2024 08:33:53 +0000 Subject: [PATCH 4/6] removed comment --- blackhole.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/blackhole.py b/blackhole.py index ad8bf0d..cc76ce1 100644 --- a/blackhole.py +++ b/blackhole.py @@ -54,7 +54,6 @@ def __init__(self, filename, isRadarr) -> None: filenameWithoutExt, ext = os.path.splitext(filename) filePath = os.path.join(baseBath, filename) filePathProcessing = trimProcessingFilepath(os.path.join(baseBath, 'processing'), filenameWithoutExt, f"_{uniqueId}{ext}") - #filePathProcessing = os.path.join(baseBath, 'processing', f"{filenameWithoutExt}_{uniqueId}{ext}") folderPathCompleted = os.path.join(baseBath, 'completed', filenameWithoutExt) self.fileInfo = self.FileInfo(filename, filenameWithoutExt, filePath, filePathProcessing, folderPathCompleted) @@ -82,8 +81,6 @@ def trimProcessingFilepath(basepath, filename, ext): remainingLength = MAX_PATH_LENGTH - len(basepath) - len(ext) - 1 return os.path.join(basepath, filename[:remainingLength] + ext) - - # From Radarr Radarr/src/NzbDrone.Core/Organizer/FileNameBuilder.cs def cleanFileName(name): result = name From aa9849c63f2fcc8538c6e87aaeea793243435891 Mon Sep 17 00:00:00 2001 From: varthe Date: Sun, 22 Dec 2024 08:36:24 +0000 Subject: [PATCH 5/6] reverted formatting changes --- blackhole.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/blackhole.py b/blackhole.py index cc76ce1..d0be416 100644 --- a/blackhole.py +++ b/blackhole.py @@ -324,7 +324,6 @@ async def is_accessible(path, timeout=10): discordError(f"Error processing {file.fileInfo.filenameWithoutExt}", e) - async def fail(torrent: TorrentBase, arr: Arr): _print = globals()['print'] @@ -335,9 +334,7 @@ def print(*values: object): torrentHash = torrent.getHash() history = await asyncio.to_thread(arr.getHistory, blackhole['historyPageSize']) - items = [item for item in history if - (item.torrentInfoHash and item.torrentInfoHash.casefold() == torrentHash.casefold()) or cleanFileName( - item.sourceTitle.casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()] + items = [item for item in history if(item.torrentInfoHash and item.torrentInfoHash.casefold() == torrentHash.casefold()) or cleanFileName(item.sourceTitle.casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()] if not items: message = "No history items found to mark as failed. Arr will not attempt to grab an alternative." print(message) @@ -348,14 +345,11 @@ def print(*values: object): await asyncio.gather(*failTasks) print(f"Failed") - def getFiles(isRadarr): print('getFiles') - files = (TorrentFileInfo(filename, isRadarr) for filename in os.listdir(getPath(isRadarr)) if - filename not in ['processing', 'completed']) + files = (TorrentFileInfo(filename, isRadarr) for filename in os.listdir(getPath(isRadarr)) if filename not in ['processing', 'completed']) return [file for file in files if file.torrentInfo.isTorrentOrMagnet] - async def on_created(isRadarr): print("Enter 'on_created'") try: From 603f438eec65268e6a4110f82b677dbc26f1fcbc Mon Sep 17 00:00:00 2001 From: varthe Date: Sun, 22 Dec 2024 08:36:55 +0000 Subject: [PATCH 6/6] fixed whitespace --- blackhole.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blackhole.py b/blackhole.py index d0be416..1778eca 100644 --- a/blackhole.py +++ b/blackhole.py @@ -334,7 +334,7 @@ def print(*values: object): torrentHash = torrent.getHash() history = await asyncio.to_thread(arr.getHistory, blackhole['historyPageSize']) - items = [item for item in history if(item.torrentInfoHash and item.torrentInfoHash.casefold() == torrentHash.casefold()) or cleanFileName(item.sourceTitle.casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()] + items = [item for item in history if (item.torrentInfoHash and item.torrentInfoHash.casefold() == torrentHash.casefold()) or cleanFileName(item.sourceTitle.casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()] if not items: message = "No history items found to mark as failed. Arr will not attempt to grab an alternative." print(message)