diff --git a/hydrus/client/ClientConstants.py b/hydrus/client/ClientConstants.py index b9cef2a88..ba0cec4c1 100644 --- a/hydrus/client/ClientConstants.py +++ b/hydrus/client/ClientConstants.py @@ -378,6 +378,9 @@ SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_GREEN_RED = 25 SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_BLUE_YELLOW = 26 SORT_FILES_BY_AVERAGE_COLOUR_HUE = 27 +SORT_FILES_BY_SERIATION = 28 +SORT_FILES_BY_SERIATION_TAGS = 29 +SORT_FILES_BY_SERIATION_VISUAL_TAGS = 30 AVERAGE_COLOUR_FILE_SORTS = { SORT_FILES_BY_AVERAGE_COLOUR_LIGHTNESS, @@ -415,7 +418,10 @@ SORT_FILES_BY_AVERAGE_COLOUR_CHROMATIC_MAGNITUDE, SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_GREEN_RED, SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_BLUE_YELLOW, - SORT_FILES_BY_AVERAGE_COLOUR_HUE + SORT_FILES_BY_AVERAGE_COLOUR_HUE, + SORT_FILES_BY_SERIATION, + SORT_FILES_BY_SERIATION_TAGS, + SORT_FILES_BY_SERIATION_VISUAL_TAGS } system_sort_type_submetatype_string_lookup = { @@ -437,6 +443,9 @@ SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_GREEN_RED : 'average colour', SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_BLUE_YELLOW : 'average colour', SORT_FILES_BY_AVERAGE_COLOUR_HUE : 'average colour', + SORT_FILES_BY_SERIATION : 'file', + SORT_FILES_BY_SERIATION_TAGS : 'tags', + SORT_FILES_BY_SERIATION_VISUAL_TAGS : 'file', SORT_FILES_BY_MIME : 'file', SORT_FILES_BY_HAS_AUDIO : 'file', SORT_FILES_BY_RANDOM : None, @@ -461,6 +470,9 @@ SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_GREEN_RED : 'balance - green-red', SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_BLUE_YELLOW : 'balance - blue-yellow', SORT_FILES_BY_AVERAGE_COLOUR_HUE : 'hue', + SORT_FILES_BY_SERIATION : 'seriation (blurhash)', + SORT_FILES_BY_SERIATION_TAGS : 'seriation (tags)', + SORT_FILES_BY_SERIATION_VISUAL_TAGS : 'seriation (visual+tags)', SORT_FILES_BY_RATIO : 'resolution ratio', SORT_FILES_BY_WIDTH : 'width', SORT_FILES_BY_APPROX_BITRATE : 'approximate bitrate', diff --git a/hydrus/client/ClientOptions.py b/hydrus/client/ClientOptions.py index d6ef68347..fa9bf87c7 100644 --- a/hydrus/client/ClientOptions.py +++ b/hydrus/client/ClientOptions.py @@ -246,6 +246,9 @@ def _InitialiseDefaults( self ): 'remember_last_advanced_file_deletion_reason' : True, 'remember_last_advanced_file_deletion_special_action' : False, 'do_macos_debug_dialog_menus' : False, + 'allow_missing_file_locations' : False, + 'search_hide_files_in_missing_file_locations' : False, + 'search_hide_files_with_missing_paths' : False, 'save_default_tag_service_tab_on_change' : True, 'force_animation_scanbar_show' : False, 'call_mouse_buttons_primary_secondary' : False, @@ -283,6 +286,7 @@ def _InitialiseDefaults( self ): 'media_viewer_recenter_media_on_window_resize': True, 'allow_blurhash_fallback' : True, 'fade_thumbnails' : True, + 'thumbnail_masonry' : False, 'slideshow_always_play_duration_media_once_through' : False, 'enable_truncated_images_pil' : True, 'do_icc_profile_normalisation' : True, @@ -316,6 +320,7 @@ def _InitialiseDefaults( self ): 'mpv_loop_playlist_instead_of_file' : False, 'draw_thumbnail_rating_background' : True, 'draw_thumbnail_numerical_ratings_collapsed_always' : False, + 'draw_thumbnail_header' : True, 'show_destination_page_when_dnd_url' : True, 'confirm_non_empty_downloader_page_close' : True, 'confirm_all_page_closes' : False, @@ -497,6 +502,7 @@ def _InitialiseDefaults( self ): 'thumbnail_border' : 1, 'thumbnail_margin' : 2, 'thumbnail_dpr_percent' : 100, + 'thumbnail_zoom_percent' : 100, 'file_maintenance_idle_throttle_files' : 1, 'file_maintenance_idle_throttle_time_delta' : 2, 'file_maintenance_active_throttle_files' : 1, @@ -553,6 +559,11 @@ def _InitialiseDefaults( self ): 'tag_list_tag_display_type_sidebar' : ClientTags.TAG_DISPLAY_SELECTION_LIST, 'tag_list_tag_display_type_media_viewer_hover' : ClientTags.TAG_DISPLAY_SINGLE_MEDIA, 'command_palette_num_chars_for_results_threshold' : 0, + 'seriation_visual_bin_size' : 32, + 'seriation_max_visual_candidates' : 200, + 'seriation_max_tag_candidates' : 200, + 'seriation_max_tag_keys' : 5, + 'seriation_max_hybrid_candidates' : 250, } self._dictionary[ 'floats' ] = { @@ -568,6 +579,8 @@ def _InitialiseDefaults( self ): 'dialog_rating_icon_size_px' : ClientGUIPainterShapes.SIZE.width(), 'dialog_rating_incdec_width_px' : ClientGUIPainterShapes.SIZE.width() * 2, #deprecated 'dialog_rating_incdec_height_px' : ClientGUIPainterShapes.SIZE.height(), + 'seriation_visual_weight' : 0.7, + 'seriation_tag_weight' : 0.3, } # diff --git a/hydrus/client/db/ClientDBFilesSearch.py b/hydrus/client/db/ClientDBFilesSearch.py index 60b132658..37409fa55 100644 --- a/hydrus/client/db/ClientDBFilesSearch.py +++ b/hydrus/client/db/ClientDBFilesSearch.py @@ -1,4 +1,6 @@ +import collections import collections.abc +import os import random import sqlite3 import typing @@ -7,6 +9,7 @@ from hydrus.core import HydrusData from hydrus.core import HydrusExceptions from hydrus.core import HydrusLists +from hydrus.core import HydrusRustStorage from hydrus.core import HydrusTags from hydrus.client import ClientConstants as CC @@ -291,6 +294,104 @@ def __init__( super().__init__( 'client file search using tags', cursor ) + def _TryRustGetHashIdsFromTagIds( + self, + tag_display_type: int, + file_service_key: bytes, + tag_context: ClientSearchTagContext.TagContext, + tag_ids: collections.abc.Collection[ int ], + hash_ids, + do_hash_table_join: bool + ): + + if tag_display_type != ClientTags.TAG_DISPLAY_STORAGE: + + return None + + + file_service_id = self.modules_services.GetServiceId( file_service_key ) + + if file_service_id != self.modules_services.combined_file_service_id: + + return None + + + if len( tag_ids ) == 0: + + return set() + + + if tag_context.service_key == CC.COMBINED_TAG_SERVICE_KEY: + + tag_service_ids = list( self.modules_services.GetServiceIds( HC.REAL_TAG_SERVICES ) ) + + else: + + tag_service_ids = [ self.modules_services.GetServiceId( tag_context.service_key ) ] + + + statuses = [] + + if tag_context.include_current_tags: + + statuses.append( HC.CONTENT_STATUS_CURRENT ) + + + if tag_context.include_pending_tags: + + statuses.append( HC.CONTENT_STATUS_PENDING ) + + + if len( statuses ) == 0: + + return set() + + + for tag_service_id in tag_service_ids: + + ( current_mappings_table_name, _deleted_mappings_table_name, pending_mappings_table_name, _petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( tag_service_id ) + + if HC.CONTENT_STATUS_CURRENT in statuses: + + if not HydrusRustStorage.EnsureMappings( HydrusRustStorage.GetMappingsStore(), self._c, tag_service_id, HC.CONTENT_STATUS_CURRENT, current_mappings_table_name ): + + return None + + + + if HC.CONTENT_STATUS_PENDING in statuses: + + if not HydrusRustStorage.EnsureMappings( HydrusRustStorage.GetMappingsStore(), self._c, tag_service_id, HC.CONTENT_STATUS_PENDING, pending_mappings_table_name ): + + return None + + + + + filter_hash_ids = None + + if do_hash_table_join and hash_ids is not None: + + filter_hash_ids = set( hash_ids ) + + + result_hash_ids = set() + + for status in statuses: + + rust_results = HydrusRustStorage.GetHashIdsForTagIds( tag_service_ids, status, tag_ids, hash_ids_filter = filter_hash_ids ) + + if rust_results is None: + + return None + + + result_hash_ids.update( rust_results ) + + + return result_hash_ids + + def GetHashIdsAndNonZeroTagCounts( self, tag_display_type: int, location_context: ClientLocation.LocationContext, tag_context: ClientSearchTagContext.TagContext, hash_ids, namespace_wildcard = '*', job_status = None ): if namespace_wildcard == '*': @@ -306,6 +407,60 @@ def GetHashIdsAndNonZeroTagCounts( self, tag_display_type: int, location_context ( file_service_keys, file_location_is_cross_referenced ) = location_context.GetCoveringCurrentFileServiceKeys() + if HydrusRustStorage.IsPrimary() and tag_display_type == ClientTags.TAG_DISPLAY_STORAGE and self.modules_services.combined_file_service_id in { self.modules_services.GetServiceId( file_service_key ) for file_service_key in file_service_keys }: + + if tag_context.service_key == CC.COMBINED_TAG_SERVICE_KEY: + + tag_service_ids = self.modules_services.GetServiceIds( HC.REAL_TAG_SERVICES ) + + else: + + tag_service_ids = [ self.modules_services.GetServiceId( tag_context.service_key ) ] + + + statuses = [] + + if tag_context.include_current_tags: + + statuses.append( HC.CONTENT_STATUS_CURRENT ) + + + if tag_context.include_pending_tags: + + statuses.append( HC.CONTENT_STATUS_PENDING ) + + + counts_by_hash_id = collections.Counter() + + for tag_service_id in tag_service_ids: + + for status in statuses: + + tags_by_hash = HydrusRustStorage.GetTagIdsForHashes( tag_service_id, status, hash_ids ) + + if tags_by_hash is None: + + continue + + + for ( hash_id, tag_ids ) in tags_by_hash.items(): + + counts_by_hash_id[ hash_id ] += len( tag_ids ) + + + + + results = [ ( hash_id, count ) for ( hash_id, count ) in counts_by_hash_id.items() if count > 0 ] + + if not file_location_is_cross_referenced: + + filtered_hash_ids = self.modules_files_storage.FilterHashIds( location_context, set( counts_by_hash_id.keys() ) ) + + results = [ ( hash_id, count ) for ( hash_id, count ) in results if hash_id in filtered_hash_ids ] + + + return results + mapping_and_tag_table_names = set() for file_service_key in file_service_keys: @@ -501,12 +656,55 @@ def GetHashIdsFromTagAdvanced( self, tag: str, tag_display_type: int, statuses: tag_id = self.modules_tags.GetTagId( tag ) + if hash_ids is not None and not isinstance( hash_ids, set ): + + hash_ids = set( hash_ids ) + for file_service_id in file_service_ids: for tag_service_id in tag_service_ids: ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( tag_service_id ) + if HydrusRustStorage.IsPrimary() and file_service_id == self.modules_services.combined_file_service_id: + + for status in statuses: + + if tag_display_type == ClientTags.TAG_DISPLAY_DISPLAY_ACTUAL: + + ideal_tag_id = self.modules_tag_siblings.GetIdealTagId( ClientTags.TAG_DISPLAY_DISPLAY_ACTUAL, tag_service_id, tag_id ) + + search_tag_ids = self.modules_tag_siblings.GetChainMembersFromIdeal( ClientTags.TAG_DISPLAY_DISPLAY_ACTUAL, tag_service_id, ideal_tag_id ) + + else: + + search_tag_ids = ( tag_id, ) + + + if len( search_tag_ids ) == 0: + + continue + + + rust_hash_ids = HydrusRustStorage.GetHashIdsForTagIds( ( tag_service_id, ), status, search_tag_ids ) + + if rust_hash_ids is None: + + continue + + + if hash_ids is not None: + + result_hash_ids.update( rust_hash_ids.intersection( hash_ids ) ) + + else: + + result_hash_ids.update( rust_hash_ids ) + + + + continue + if file_service_id == self.modules_services.combined_file_service_id: statuses_to_table_names = { @@ -639,6 +837,13 @@ def GetHashIdsFromTagIds( self, tag_display_type: int, file_service_key: bytes, + rust_results = self._TryRustGetHashIdsFromTagIds( tag_display_type, file_service_key, tag_context, tag_ids, hash_ids, do_hash_table_join ) + + if rust_results is not None: + + return rust_results + + result_hash_ids = set() table_names = self.modules_tag_search.GetMappingTables( tag_display_type, file_service_key, tag_context ) @@ -997,6 +1202,112 @@ def GetHashIdsThatHaveTagsComplexLocation( self, tag_display_type: int, location def GetHashIdsThatHaveTagsSimpleLocation( self, tag_display_type: int, file_service_key: bytes, tag_context: ClientSearchTagContext.TagContext, namespace_ids_table_name = None, hash_ids_table_name = None, job_status = None ): + file_service_id = self.modules_services.GetServiceId( file_service_key ) + + if HydrusRustStorage.IsPrimary() and tag_display_type == ClientTags.TAG_DISPLAY_STORAGE and file_service_id == self.modules_services.combined_file_service_id: + + if tag_context.service_key == CC.COMBINED_TAG_SERVICE_KEY: + + tag_service_ids = list( self.modules_services.GetServiceIds( HC.REAL_TAG_SERVICES ) ) + + else: + + tag_service_ids = [ self.modules_services.GetServiceId( tag_context.service_key ) ] + + + statuses = [] + + if tag_context.include_current_tags: + + statuses.append( HC.CONTENT_STATUS_CURRENT ) + + + if tag_context.include_pending_tags: + + statuses.append( HC.CONTENT_STATUS_PENDING ) + + + if len( statuses ) == 0: + + return set() + + + filter_hash_ids = None + + if hash_ids_table_name is not None: + + filter_hash_ids = self._STI( self._Execute( 'SELECT hash_id FROM {};'.format( hash_ids_table_name ) ) ) + + if len( filter_hash_ids ) == 0: + + return set() + + + + result_hash_ids = set() + + if namespace_ids_table_name is None: + + for tag_service_id in tag_service_ids: + + for status in statuses: + + rust_hash_ids = HydrusRustStorage.GetHashIdsWithMappings( tag_service_id, status ) + + if rust_hash_ids is None: + + continue + + + result_hash_ids.update( rust_hash_ids ) + + if job_status is not None and job_status.IsCancelled(): + + return set() + + + + + if filter_hash_ids is not None: + + result_hash_ids.intersection_update( filter_hash_ids ) + + + return result_hash_ids + + + for tag_service_id in tag_service_ids: + + tags_table_name = self.modules_tag_search.GetTagsTableName( file_service_id, tag_service_id ) + + tag_ids = self._STL( self._Execute( 'SELECT DISTINCT tag_id FROM {} CROSS JOIN {} USING ( namespace_id );'.format( tags_table_name, namespace_ids_table_name ) ) ) + + if len( tag_ids ) == 0: + + continue + + + for status in statuses: + + rust_hash_ids = HydrusRustStorage.GetHashIdsForTagIds( ( tag_service_id, ), status, tag_ids, hash_ids_filter = filter_hash_ids ) + + if rust_hash_ids is None: + + continue + + + result_hash_ids.update( rust_hash_ids ) + + if job_status is not None and job_status.IsCancelled(): + + return set() + + + + + return result_hash_ids + + mapping_and_tag_table_names = self.modules_tag_search.GetMappingAndTagTables( tag_display_type, file_service_key, tag_context ) if hash_ids_table_name is None: @@ -2976,6 +3287,118 @@ def GetHashIdsFromQuery( return [] + # + + if CG.client_controller.new_options.GetBoolean( 'search_hide_files_in_missing_file_locations' ): + + if query_hash_ids is not None and len( query_hash_ids ) > 0: + + client_files_manager = CG.client_controller.client_files_manager + + if client_files_manager is not None: + + ( missing_prefixes, prefix_hex_length ) = client_files_manager.GetMissingFilePrefixesAndHexLength() + + if len( missing_prefixes ) > 0: + + file_location_is_all_local = self.modules_services.LocationContextIsCoveredByHydrusLocalFileStorage( location_context ) + + if file_location_is_all_local: + + hash_ids_to_check = query_hash_ids + + else: + + hash_ids_to_check = self.modules_files_storage.FilterHashIdsToStatus( + self.modules_services.hydrus_local_file_storage_service_id, + query_hash_ids, + HC.CONTENT_STATUS_CURRENT + ) + + + if len( hash_ids_to_check ) > 0: + + hash_ids_to_hashes = self.modules_hashes.GetHashIdsToHashes( hash_ids = hash_ids_to_check ) + + missing_hash_ids = { + hash_id + for ( hash_id, file_hash ) in hash_ids_to_hashes.items() + if f'f{file_hash.hex()[ : prefix_hex_length ]}' in missing_prefixes + } + + if len( missing_hash_ids ) > 0: + + query_hash_ids.difference_update( missing_hash_ids ) + + + + + + + + if CG.client_controller.new_options.GetBoolean( 'search_hide_files_with_missing_paths' ): + + if query_hash_ids is not None and len( query_hash_ids ) > 0: + + client_files_manager = CG.client_controller.client_files_manager + + if client_files_manager is not None: + + file_location_is_all_local = self.modules_services.LocationContextIsCoveredByHydrusLocalFileStorage( location_context ) + + if file_location_is_all_local: + + hash_ids_to_check = query_hash_ids + + else: + + hash_ids_to_check = self.modules_files_storage.FilterHashIdsToStatus( + self.modules_services.hydrus_local_file_storage_service_id, + query_hash_ids, + HC.CONTENT_STATUS_CURRENT + ) + + + if len( hash_ids_to_check ) > 0: + + with self._MakeTemporaryIntegerTable( hash_ids_to_check, 'hash_id' ) as temp_hash_ids_table_name: + + hash_ids_to_mimes = dict( self._Execute( 'SELECT hash_id, mime FROM {} CROSS JOIN files_info USING ( hash_id );'.format( temp_hash_ids_table_name ) ) ) + + + if len( hash_ids_to_mimes ) > 0: + + hash_ids_to_hashes = self.modules_hashes.GetHashIdsToHashes( hash_ids = set( hash_ids_to_mimes.keys() ) ) + + missing_hash_ids = set() + + for ( hash_id, file_hash ) in hash_ids_to_hashes.items(): + + mime = hash_ids_to_mimes.get( hash_id ) + + if mime is None: + + continue + + + expected_path = client_files_manager.GetFilePath( file_hash, mime, check_file_exists = False ) + + if not os.path.exists( expected_path ): + + missing_hash_ids.add( hash_id ) + + + + if len( missing_hash_ids ) > 0: + + query_hash_ids.difference_update( missing_hash_ids ) + + + + + + + # query_hash_ids = list( query_hash_ids ) diff --git a/hydrus/client/files/ClientFilesMaintenance.py b/hydrus/client/files/ClientFilesMaintenance.py index c9a8c5f72..7ab64b04b 100644 --- a/hydrus/client/files/ClientFilesMaintenance.py +++ b/hydrus/client/files/ClientFilesMaintenance.py @@ -27,6 +27,7 @@ from hydrus.client import ClientThreading from hydrus.client.metadata import ClientContentUpdates from hydrus.client.metadata import ClientTags +from hydrus.client.metadata import ClientTikTok REGENERATE_FILE_DATA_JOB_FILE_METADATA = 0 REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL = 1 @@ -52,6 +53,7 @@ REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_PRESENCE_DELETE_RECORD = 21 REGENERATE_FILE_DATA_JOB_BLURHASH = 22 REGENERATE_FILE_DATA_JOB_FILE_HAS_TRANSPARENCY = 23 +REGENERATE_FILE_DATA_JOB_TIKTOK_TAG_ENRICHMENT = 24 regen_file_enum_to_str_lookup = { REGENERATE_FILE_DATA_JOB_FILE_METADATA : 'regenerate file metadata', @@ -77,7 +79,8 @@ REGENERATE_FILE_DATA_JOB_FILE_HAS_HUMAN_READABLE_EMBEDDED_METADATA : 'determine if the file has non-EXIF embedded metadata', REGENERATE_FILE_DATA_JOB_FILE_HAS_ICC_PROFILE : 'determine if the file has an icc profile', REGENERATE_FILE_DATA_JOB_PIXEL_HASH : 'regenerate pixel hashes', - REGENERATE_FILE_DATA_JOB_BLURHASH: 'regenerate blurhash' + REGENERATE_FILE_DATA_JOB_BLURHASH: 'regenerate blurhash', + REGENERATE_FILE_DATA_JOB_TIKTOK_TAG_ENRICHMENT : 'add tiktok site/creator tags from known URLs' } # wrapped in triple quotes so I don't have to backslash escape so much @@ -145,7 +148,8 @@ REGENERATE_FILE_DATA_JOB_FILE_HAS_HUMAN_READABLE_EMBEDDED_METADATA : '''This loads the file to see if it has non-EXIF human-readable embedded metadata, which can be shown in the media viewer and searched with "system:image has human-readable embedded metadata".''', REGENERATE_FILE_DATA_JOB_FILE_HAS_ICC_PROFILE : '''This loads the file to see if it has an ICC profile, which is used in "system:has icc profile" search.''', REGENERATE_FILE_DATA_JOB_PIXEL_HASH : '''This generates a fast unique identifier for the pixels in a still image, which is used in duplicate pixel searches.''', - REGENERATE_FILE_DATA_JOB_BLURHASH : '''This generates a very small version of the file's thumbnail that can be used as a placeholder while the thumbnail loads.''' + REGENERATE_FILE_DATA_JOB_BLURHASH : '''This generates a very small version of the file's thumbnail that can be used as a placeholder while the thumbnail loads.''', + REGENERATE_FILE_DATA_JOB_TIKTOK_TAG_ENRICHMENT : '''This looks at a file's known URLs and adds downloader tags like "site:tiktok" and "creator:" if it detects TikTok URLs. No network requests are made.''' } NORMALISED_BIG_JOB_WEIGHT = 100 @@ -174,7 +178,8 @@ REGENERATE_FILE_DATA_JOB_FILE_HAS_HUMAN_READABLE_EMBEDDED_METADATA : 25, REGENERATE_FILE_DATA_JOB_FILE_HAS_ICC_PROFILE : 25, REGENERATE_FILE_DATA_JOB_PIXEL_HASH : 100, - REGENERATE_FILE_DATA_JOB_BLURHASH: 15 + REGENERATE_FILE_DATA_JOB_BLURHASH: 15, + REGENERATE_FILE_DATA_JOB_TIKTOK_TAG_ENRICHMENT : 1 } regen_file_enum_to_overruled_jobs = { @@ -201,7 +206,8 @@ REGENERATE_FILE_DATA_JOB_FILE_HAS_HUMAN_READABLE_EMBEDDED_METADATA : [], REGENERATE_FILE_DATA_JOB_FILE_HAS_ICC_PROFILE : [], REGENERATE_FILE_DATA_JOB_PIXEL_HASH : [], - REGENERATE_FILE_DATA_JOB_BLURHASH: [] + REGENERATE_FILE_DATA_JOB_BLURHASH: [], + REGENERATE_FILE_DATA_JOB_TIKTOK_TAG_ENRICHMENT : [] } ALL_REGEN_JOBS_IN_RUN_ORDER = [ @@ -214,6 +220,7 @@ REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_DATA_REMOVE_RECORD, REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_DATA_SILENT_DELETE, REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_PRESENCE_LOG_ONLY, + REGENERATE_FILE_DATA_JOB_TIKTOK_TAG_ENRICHMENT, REGENERATE_FILE_DATA_JOB_FILE_METADATA, REGENERATE_FILE_DATA_JOB_REFIT_THUMBNAIL, REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL, @@ -241,6 +248,7 @@ REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_DATA_REMOVE_RECORD, REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_DATA_SILENT_DELETE, REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_PRESENCE_LOG_ONLY, + REGENERATE_FILE_DATA_JOB_TIKTOK_TAG_ENRICHMENT, REGENERATE_FILE_DATA_JOB_FILE_METADATA, REGENERATE_FILE_DATA_JOB_REFIT_THUMBNAIL, REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL, @@ -1045,6 +1053,56 @@ def _RegenBlurhash( self, media_result ): + def _EnrichTikTokTags( self, media_result ): + + urls = media_result.GetLocationsManager().GetURLs() + + if len( urls ) == 0: + + return None + + + tags_to_add = ClientTikTok.GetTikTokTagsFromURLs( urls ) + + description_text = ClientTikTok.GetTikTokDescriptionTextFromNotes( media_result.GetNotesManager() ) + + if description_text is not None: + + tags_to_add.update( ClientTikTok.GetTikTokHashtagTagsFromText( description_text ) ) + + if len( tags_to_add ) == 0: + + return None + + + tags_manager = media_result.GetTagsManager() + existing_tags = tags_manager.GetCurrentAndPending( CC.DEFAULT_LOCAL_DOWNLOADER_TAG_SERVICE_KEY, ClientTags.TAG_DISPLAY_STORAGE ) + + tags_to_add.difference_update( existing_tags ) + + if len( tags_to_add ) == 0: + + return None + + + hash = media_result.GetHash() + + content_updates = [ + ClientContentUpdates.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( tag, { hash } ) ) + for tag in tags_to_add + ] + + content_update_package = ClientContentUpdates.ContentUpdatePackage.STATICCreateFromContentUpdates( + CC.DEFAULT_LOCAL_DOWNLOADER_TAG_SERVICE_KEY, + content_updates + ) + + CG.client_controller.WriteSynchronous( 'content_updates', content_update_package ) + + return len( tags_to_add ) + + + def _RegenSimilarFilesMetadata( self, media_result ): @@ -1212,6 +1270,10 @@ def _RunJob( self, media_results_to_job_types, job_status, job_done_hook = None additional_data = self._RegenBlurhash( media_result ) + elif job_type == REGENERATE_FILE_DATA_JOB_TIKTOK_TAG_ENRICHMENT: + + additional_data = self._EnrichTikTokTags( media_result ) + elif job_type in ( REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_PRESENCE_REMOVE_RECORD, REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_PRESENCE_DELETE_RECORD, diff --git a/hydrus/client/files/ClientFilesManager.py b/hydrus/client/files/ClientFilesManager.py index 457040921..a445353c6 100644 --- a/hydrus/client/files/ClientFilesManager.py +++ b/hydrus/client/files/ClientFilesManager.py @@ -734,6 +734,10 @@ def _LookForFilePath( self, hash ): for subfolder in subfolders: if not subfolder.PathExists(): + if CG.client_controller.new_options.GetBoolean( 'allow_missing_file_locations' ) and not subfolder.base_location.PathExists(): + + continue + raise HydrusExceptions.DirectoryMissingException( f'The directory {subfolder.path} was not found! Reconnect the missing location or shut down the client immediately!' ) @@ -877,12 +881,17 @@ def _ReinitSubfolders( self ): def _ReinitMissingLocations( self ): self._missing_subfolders = set() + allow_missing_locations = CG.client_controller.new_options.GetBoolean( 'allow_missing_file_locations' ) for subfolders in self._prefix_umbrellas_to_client_files_subfolders.values(): for subfolder in subfolders: if not subfolder.PathExists(): + if allow_missing_locations and not subfolder.base_location.PathExists(): + + continue + self._missing_subfolders.add( subfolder ) @@ -1392,6 +1401,44 @@ def GetMissingSubfolders( self ): return self._missing_subfolders + def GetMissingFilePrefixesAndHexLength( self ): + + with self._master_locations_rwlock.read: + + file_subfolders = [ + subfolder + for subfolders in self._prefix_umbrellas_to_client_files_subfolders.values() + for subfolder in subfolders + if subfolder.IsForFiles() + ] + prefix_hex_length = self._shortest_prefix + + + if len( file_subfolders ) == 0: + + return ( set(), prefix_hex_length ) + + + base_locations = { subfolder.base_location for subfolder in file_subfolders } + missing_base_locations = { base_location for base_location in base_locations if not base_location.PathExists() } + + missing_prefixes = set() + + for subfolder in file_subfolders: + + if subfolder.base_location in missing_base_locations: + + missing_prefixes.add( subfolder.prefix ) + + elif not subfolder.PathExists(): + + missing_prefixes.add( subfolder.prefix ) + + + + return ( missing_prefixes, prefix_hex_length ) + + def GetAllSubfolders( self ) -> collections.abc.Collection[ ClientFilesPhysical.FilesStorageSubfolder ]: with self._master_locations_rwlock.read: diff --git a/hydrus/client/gui/canvas/ClientGUICanvas.py b/hydrus/client/gui/canvas/ClientGUICanvas.py index 3e4f48ffa..1f9abaf6d 100644 --- a/hydrus/client/gui/canvas/ClientGUICanvas.py +++ b/hydrus/client/gui/canvas/ClientGUICanvas.py @@ -332,6 +332,7 @@ def __init__( self, parent, location_context: ClientLocation.LocationContext ): super().__init__( parent ) self.setObjectName( 'HydrusMediaViewer' ) + self.setAttribute( QC.Qt.WidgetAttribute.WA_OpaquePaintEvent, True ) self.setSizePolicy( QW.QSizePolicy.Policy.Expanding, QW.QSizePolicy.Policy.Expanding ) @@ -857,6 +858,7 @@ def paintEvent( self, event ): try: painter = QG.QPainter( self ) + painter.setClipRegion( event.region() ) self._DrawBackgroundBitmap( painter ) diff --git a/hydrus/client/gui/canvas/ClientGUICanvasMedia.py b/hydrus/client/gui/canvas/ClientGUICanvasMedia.py index 2134e5b26..25209fad4 100644 --- a/hydrus/client/gui/canvas/ClientGUICanvasMedia.py +++ b/hydrus/client/gui/canvas/ClientGUICanvasMedia.py @@ -36,6 +36,7 @@ from hydrus.client.gui import ClientGUIShortcuts from hydrus.client.gui import QtInit from hydrus.client.gui import QtPorting as QP +from hydrus.client.gui.canvas import ClientGUICanvasMediaLayout from hydrus.client.gui.canvas import ClientGUIMPV from hydrus.client.gui.media import ClientGUIMediaControls from hydrus.client.gui.media import ClientGUIMediaVolume @@ -296,7 +297,6 @@ def CalculateMediaContainerSize( media, device_pixel_ratio: float, zoom, show_ac return QC.QSize( media_width, media_height ) - def CalculateMediaSize( media, zoom ): if media.GetMime() in HC.AUDIO or not media.HasUsefulResolution(): @@ -382,6 +382,8 @@ def __init__( self, parent, canvas_type, background_colour_generator ): super().__init__( parent ) + self.setAttribute( QC.Qt.WidgetAttribute.WA_OpaquePaintEvent, True ) + self._canvas_type = canvas_type self._background_colour_generator = background_colour_generator @@ -720,6 +722,7 @@ def paintEvent( self, event ): painter = QG.QPainter( self ) + painter.setClipRegion( event.region() ) if self._canvas_qt_pixmap is None: @@ -990,6 +993,8 @@ def __init__( self, parent ): super().__init__( parent ) + self.setAttribute( QC.Qt.WidgetAttribute.WA_OpaquePaintEvent, True ) + self._qss_colours = { 'hab_border' : QG.QColor( 0, 0, 0 ), 'hab_background' : QG.QColor( 240, 240, 240 ), @@ -1328,6 +1333,7 @@ def paintEvent( self, event ): try: painter = QG.QPainter( self ) + painter.setClipRegion( event.region() ) if self._CurrentMediaWindowIsBad() or self._next_draw_info is None: @@ -1658,6 +1664,12 @@ def _GetMaxZoomDimension( self ): + def _ControlsBarUsesReservedSpace( self ): + + # QVideoWidget can sit above other widgets, so reserve space for the scrubber bar. + return isinstance( self._media_window, QtMediaPlayer ) + + def _MakeMediaWindow( self ): old_media_window = self._media_window @@ -1853,6 +1865,7 @@ def _ShowHideControlBar( self ): is_near = False show_small_instead_of_hiding = None force_show = False + controls_bar_visibility_changed = False if not ShouldHaveAnimationBar( self._media, self._show_action ): @@ -1897,6 +1910,7 @@ def _ShowHideControlBar( self ): self._animation_bar.setGubbinsVisible( True ) self._animation_bar.repaint() # this is probably not needed + controls_bar_visibility_changed = True do_layout = True @@ -1930,6 +1944,12 @@ def _ShowHideControlBar( self ): self._controls_bar.layout() # this is probably not needed + controls_bar_visibility_changed = True + + if controls_bar_visibility_changed and self._ControlsBarUsesReservedSpace(): + + self._SizeAndPositionChildren() + @@ -1940,13 +1960,22 @@ def _SizeAndPositionChildren( self ): self._embed_button.setFixedSize( self.size() ) self._embed_button.move( QC.QPoint( 0, 0 ) ) + controls_bar_rect = self.GetIdealControlsBarRect( full_size = self._controls_bar_show_full ) + if self._media_window is not None: - self._media_window.setFixedSize( self.size() ) - self._media_window.move( QC.QPoint( 0, 0 ) ) + media_rect = QC.QRect( QC.QPoint( 0, 0 ), self.size() ) - - controls_bar_rect = self.GetIdealControlsBarRect( full_size = self._controls_bar_show_full ) + media_height = ClientGUICanvasMediaLayout.CalculateMediaHeightForControlsBar( + media_rect.height(), + controls_bar_rect.height(), + self._ControlsBarUsesReservedSpace(), + self._controls_bar.isHidden() + ) + + media_rect.setHeight( media_height ) + self._media_window.setFixedSize( media_rect.size() ) + self._media_window.move( media_rect.topLeft() ) if controls_bar_rect.size() != self._controls_bar.size(): @@ -2580,7 +2609,14 @@ def SizeSelfToMedia( self ): return - media_size = self._media_window.size() + if self._ControlsBarUsesReservedSpace() and not self._controls_bar.isHidden(): + + media_size = self.size() + + else: + + media_size = self._media_window.size() + media_width = media_size.width() media_height = media_size.height() @@ -3075,6 +3111,8 @@ def __init__( self, parent, background_colour_generator ): super().__init__( parent ) + self.setAttribute( QC.Qt.WidgetAttribute.WA_OpaquePaintEvent, True ) + self._background_colour_generator = background_colour_generator self._media = None @@ -3161,6 +3199,7 @@ def paintEvent( self, event ): try: painter = QG.QPainter( self ) + painter.setClipRegion( event.region() ) self._Redraw( painter ) @@ -3939,6 +3978,7 @@ def paintEvent( self, event ): painter = QG.QPainter( self ) + painter.setClipRegion( event.region() ) if self._image_renderer is None or not self._image_renderer.IsReady(): diff --git a/hydrus/client/gui/canvas/ClientGUICanvasMediaLayout.py b/hydrus/client/gui/canvas/ClientGUICanvasMediaLayout.py new file mode 100644 index 000000000..696f732f7 --- /dev/null +++ b/hydrus/client/gui/canvas/ClientGUICanvasMediaLayout.py @@ -0,0 +1,8 @@ +def CalculateMediaHeightForControlsBar( container_height: int, controls_bar_height: int, reserve_space: bool, controls_bar_hidden: bool ) -> int: + + if reserve_space and not controls_bar_hidden: + + return max( 1, container_height - controls_bar_height ) + + + return container_height diff --git a/hydrus/client/gui/pages/ClientGUIMediaResultsPanel.py b/hydrus/client/gui/pages/ClientGUIMediaResultsPanel.py index cf42984b6..74d33dc50 100644 --- a/hydrus/client/gui/pages/ClientGUIMediaResultsPanel.py +++ b/hydrus/client/gui/pages/ClientGUIMediaResultsPanel.py @@ -10,6 +10,7 @@ from hydrus.core import HydrusConstants as HC from hydrus.core import HydrusData from hydrus.core import HydrusExceptions +from hydrus.core import HydrusLists from hydrus.core import HydrusNumbers from hydrus.core import HydrusPaths from hydrus.core import HydrusTime @@ -131,6 +132,8 @@ def __init__( self, parent, page_key, page_manager: ClientGUIPageManager.PageMan CG.client_controller.sub( self, 'SelectByTags', 'select_files_with_tags' ) self._had_changes_to_tag_presentation_while_hidden = False + self._sort_generation = 0 + self._seriation_sort_in_progress = False self._my_shortcut_handler = ClientGUIShortcuts.ShortcutsHandler( self, self, [ 'media', 'thumbnails' ] ) @@ -1988,6 +1991,161 @@ def Collect( self, media_collect = None ): self.Sort() + def _IsSeriationSort( self, media_sort ): + + if media_sort is None: + + return False + + + ( sort_metatype, sort_data ) = media_sort.sort_type + + if sort_metatype != 'system': + + return False + + + return sort_data in ( + CC.SORT_FILES_BY_SERIATION, + CC.SORT_FILES_BY_SERIATION_TAGS, + CC.SORT_FILES_BY_SERIATION_VISUAL_TAGS + ) + + + def _SortFinished( self ): + + pass + + + def _QTApplySeriationSort( self, sort_generation, ordered_medias, exception ): + + if not QP.isValid( self ): + + return + + + if sort_generation != self._sort_generation: + + return + + + self._seriation_sort_in_progress = False + + if exception is not None: + + HydrusData.ShowException( exception ) + + return + + + if ordered_medias is None: + + return + + + order_lookup = { media : i for ( i, media ) in enumerate( ordered_medias ) } + max_index = len( order_lookup ) + + self._sorted_media.sort( key = lambda m: order_lookup.get( m, max_index ), reverse = False ) + + self._RecalcHashes() + + self._SortFinished() + + + def _THREADSeriationSort( self, sort_generation, media_sort, medias_snapshot ): + + ordered_medias = None + exception = None + + try: + + ( sort_metatype, sort_data ) = media_sort.sort_type + reverse = media_sort.sort_order == CC.SORT_DESC + + if sort_data == CC.SORT_FILES_BY_SERIATION: + + ordered_medias = ClientMedia.GetSeriationOrderingForMediasByBlurhash( medias_snapshot, reverse ) + + elif sort_data == CC.SORT_FILES_BY_SERIATION_TAGS: + + ordered_medias = ClientMedia.GetSeriationOrderingForMediasByTags( medias_snapshot, reverse, media_sort.tag_context ) + + elif sort_data == CC.SORT_FILES_BY_SERIATION_VISUAL_TAGS: + + ordered_medias = ClientMedia.GetSeriationOrderingForMediasByVisualAndTags( medias_snapshot, reverse, media_sort.tag_context ) + + else: + + ordered_medias = medias_snapshot + + + except Exception as e: + + exception = e + + + CG.client_controller.CallAfterQtSafe( + self, + self._QTApplySeriationSort, + sort_generation, + ordered_medias, + exception + ) + + + def Sort( self, media_sort = None, secondary_sort = None ): + + self._sort_generation += 1 + sort_generation = self._sort_generation + + if media_sort is None: + + media_sort = self._media_sort + + + if secondary_sort is None: + + if self._secondary_media_sort is None: + + secondary_sort = CG.client_controller.new_options.GetFallbackSort() + + else: + + secondary_sort = self._secondary_media_sort + + + + if not self._IsSeriationSort( media_sort ): + + self._seriation_sort_in_progress = False + + ClientMedia.ListeningMediaList.Sort( self, media_sort = media_sort, secondary_sort = secondary_sort ) + + self._SortFinished() + + return + + + self._seriation_sort_in_progress = True + + self._media_sort = media_sort + self._secondary_media_sort = secondary_sort + + for media in self._collected_media: + + media.Sort( media_sort = media_sort, secondary_sort = secondary_sort ) + + + medias_snapshot = HydrusLists.FastIndexUniqueList( list( self._sorted_media ) ) + + self._secondary_media_sort.Sort( self._location_context, self._tag_context, medias_snapshot ) + + medias_snapshot = list( medias_snapshot ) + + CG.client_controller.CallToThread( self._THREADSeriationSort, sort_generation, media_sort, medias_snapshot ) + + def GetColour( self, colour_type ): if CG.client_controller.new_options.GetBoolean( 'override_stylesheet_colours' ): @@ -2791,6 +2949,7 @@ def __init__( self, parent: "MediaResultsPanel" ): super().__init__( parent ) + self.setAttribute( QC.Qt.WidgetAttribute.WA_OpaquePaintEvent, True ) self._parent = parent @@ -2799,6 +2958,7 @@ def paintEvent( self, event ): try: painter = QG.QPainter( self ) + painter.setClipRegion( event.region() ) bg_colour = self._parent.GetColour( CC.COLOUR_THUMBGRID_BACKGROUND ) diff --git a/hydrus/client/gui/pages/ClientGUIMediaResultsPanelThumbnails.py b/hydrus/client/gui/pages/ClientGUIMediaResultsPanelThumbnails.py index 7ace9310b..ea6b80b08 100644 --- a/hydrus/client/gui/pages/ClientGUIMediaResultsPanelThumbnails.py +++ b/hydrus/client/gui/pages/ClientGUIMediaResultsPanelThumbnails.py @@ -1,3 +1,5 @@ +import bisect +import collections import random import typing @@ -12,6 +14,7 @@ from hydrus.core import HydrusLists from hydrus.core import HydrusNumbers from hydrus.core import HydrusTime +from hydrus.core.files.images import HydrusImageHandling from hydrus.client import ClientApplicationCommand as CAC from hydrus.client import ClientConstants as CC @@ -146,12 +149,21 @@ def __init__( self, parent, page_key, page_manager: ClientGUIPageManager.PageMan self._drag_prefire_event_count = 0 self._hashes_to_thumbnails_waiting_to_be_drawn: dict[ bytes, ThumbnailWaitingToBeDrawn ] = {} self._hashes_faded = set() + self._masonry_dirty = True + self._masonry_positions = [] + self._masonry_column_entries = [] + self._masonry_column_starts = [] + self._masonry_page_to_indices = {} + self._masonry_index_to_pages = [] + self._masonry_virtual_height = 0 super().__init__( parent, page_key, page_manager, media_results ) self._my_current_drag_object = None self._last_device_pixel_ratio = self.devicePixelRatio() + self._thumbnail_layout_options = self._GetThumbnailLayoutOptions() + self._thumbnail_render_options = self._GetThumbnailRenderOptions() ( thumbnail_span_width, thumbnail_span_height ) = self._GetThumbnailSpanDimensions() @@ -177,6 +189,7 @@ def __init__( self, parent, page_key, page_manager: ClientGUIPageManager.PageMan CG.client_controller.sub( self, 'ThumbnailsReset', 'notify_complete_thumbnail_reset' ) CG.client_controller.sub( self, 'RedrawAllThumbnails', 'refresh_all_tag_presentation_gui' ) CG.client_controller.sub( self, 'WaterfallThumbnails', 'waterfall_thumbnails' ) + CG.client_controller.sub( self, 'NotifyNewOptions', 'notify_new_options' ) def _CalculateVisiblePageIndices( self ): @@ -189,7 +202,7 @@ def _CalculateVisiblePageIndices( self ): ( thumbnail_span_width, thumbnail_span_height ) = self._GetThumbnailSpanDimensions() - page_height = self._num_rows_per_canvas_page * thumbnail_span_height + page_height = self._GetPageHeight() first_visible_page_index = earliest_y // page_height @@ -317,6 +330,12 @@ def _DrawCanvasPage( self, page_index, canvas_page ): thumbnail_margin = CG.client_controller.new_options.GetInteger( 'thumbnail_margin' ) + if self._UsingMasonryLayout(): + + self._EnsureMasonryLayout() + + page_y_start = page_index * self._GetPageHeight() + for ( thumbnail_index, thumbnail ) in page_thumbnails: display_media = thumbnail.GetDisplayMedia() @@ -332,13 +351,32 @@ def _DrawCanvasPage( self, page_index, canvas_page ): self._StopFading( hash ) - thumbnail_col = thumbnail_index % self._num_columns - - thumbnail_row = thumbnail_index // self._num_columns - - x = thumbnail_col * thumbnail_span_width + thumbnail_margin - - y = ( thumbnail_row - ( page_index * self._num_rows_per_canvas_page ) ) * thumbnail_span_height + thumbnail_margin + if self._UsingMasonryLayout(): + + if thumbnail_index >= len( self._masonry_positions ): + + continue + + + rect = self._masonry_positions[ thumbnail_index ] + + if rect is None: + + continue + + + x = rect.x() + y = rect.y() - page_y_start + + else: + + thumbnail_col = thumbnail_index % self._num_columns + + thumbnail_row = thumbnail_index // self._num_columns + + x = thumbnail_col * thumbnail_span_width + thumbnail_margin + + y = ( thumbnail_row - ( page_index * self._num_rows_per_canvas_page ) ) * thumbnail_span_height + thumbnail_margin painter.drawImage( x, y, thumbnail.GetQtImage( thumbnail, self, self.devicePixelRatio() ) ) @@ -430,11 +468,209 @@ def _GenerateMediaSingleton( self, media_result ): return ThumbnailMediaSingleton( media_result ) + def _GetThumbnailLayoutOptions( self ): + + new_options = CG.client_controller.new_options + + return ( + tuple( HC.options[ 'thumbnail_dimensions' ] ), + new_options.GetInteger( 'thumbnail_border' ), + new_options.GetInteger( 'thumbnail_margin' ), + new_options.GetInteger( 'thumbnail_zoom_percent' ), + new_options.GetBoolean( 'thumbnail_masonry' ), + new_options.GetInteger( 'thumbnail_scale_type' ) + ) + + + def _GetThumbnailRenderOptions( self ): + + new_options = CG.client_controller.new_options + + return ( + new_options.GetBoolean( 'draw_thumbnail_header' ), + ) + + + def _UsingMasonryLayout( self ): + + return CG.client_controller.new_options.GetBoolean( 'thumbnail_masonry' ) + + + def _GetThumbnailDimensions( self ): + + ( width, height ) = HC.options[ 'thumbnail_dimensions' ] + + zoom_percent = CG.client_controller.new_options.GetInteger( 'thumbnail_zoom_percent' ) + + if zoom_percent != 100: + + width = max( 1, int( round( width * ( zoom_percent / 100 ) ) ) ) + height = max( 1, int( round( height * ( zoom_percent / 100 ) ) ) ) + + + return ( width, height ) + + + def GetThumbnailInnerDimensions( self, media: ClientMedia.Media ): + + ( base_width, base_height ) = self._GetThumbnailDimensions() + + if not self._UsingMasonryLayout(): + + return ( base_width, base_height ) + + + ( media_width, media_height ) = media.GetResolution() + + if media_width is None or media_height is None or media_width <= 0 or media_height <= 0: + + return ( base_width, base_height ) + + + new_options = CG.client_controller.new_options + + thumbnail_scale_type = new_options.GetInteger( 'thumbnail_scale_type' ) + + ( thumb_width, thumb_height ) = HydrusImageHandling.GetThumbnailResolution( + ( media_width, media_height ), + ( base_width, base_height ), + thumbnail_scale_type, + 100 + ) + + if thumb_width <= 0 or thumb_height <= 0: + + return ( base_width, base_height ) + + + ratio = thumb_height / thumb_width + + target_height = max( 1, int( round( base_width * ratio ) ) ) + + return ( base_width, target_height ) + + + def _GetPageHeight( self ): + + ( thumbnail_span_width, thumbnail_span_height ) = self._GetThumbnailSpanDimensions() + + return max( 1, self._num_rows_per_canvas_page * thumbnail_span_height ) + + + def _InvalidateMasonryLayout( self ): + + self._masonry_dirty = True + self._masonry_positions = [] + self._masonry_column_entries = [] + self._masonry_column_starts = [] + self._masonry_page_to_indices = {} + self._masonry_index_to_pages = [] + self._masonry_virtual_height = 0 + + + def _EnsureMasonryLayout( self ): + + if not self._UsingMasonryLayout(): + + return + + + if not self._masonry_dirty: + + return + + + self._masonry_dirty = False + + num_media = len( self._sorted_media ) + + self._masonry_positions = [ None ] * num_media + self._masonry_page_to_indices = collections.defaultdict( list ) + self._masonry_index_to_pages = [ [] for i in range( num_media ) ] + + if num_media == 0: + + return + + + new_options = CG.client_controller.new_options + + thumbnail_margin = new_options.GetInteger( 'thumbnail_margin' ) + thumbnail_border = new_options.GetInteger( 'thumbnail_border' ) + + ( thumbnail_span_width, thumbnail_span_height ) = self._GetThumbnailSpanDimensions() + + outer_width = thumbnail_span_width - ( thumbnail_margin * 2 ) + + self._masonry_column_entries = [ [] for i in range( self._num_columns ) ] + self._masonry_column_starts = [ [] for i in range( self._num_columns ) ] + + column_heights = [ 0 ] * self._num_columns + + page_height = self._GetPageHeight() + + for ( index, thumbnail ) in enumerate( self._sorted_media ): + + ( inner_width, inner_height ) = self.GetThumbnailInnerDimensions( thumbnail ) + + outer_height = inner_height + ( thumbnail_border * 2 ) + + column_index = column_heights.index( min( column_heights ) ) + + x = column_index * thumbnail_span_width + thumbnail_margin + y = column_heights[ column_index ] + thumbnail_margin + + rect = QC.QRect( x, y, outer_width, outer_height ) + + self._masonry_positions[ index ] = rect + + self._masonry_column_entries[ column_index ].append( ( y, y + outer_height, index ) ) + self._masonry_column_starts[ column_index ].append( y ) + + column_heights[ column_index ] += outer_height + ( thumbnail_margin * 2 ) + + start_page = y // page_height + end_page = ( y + outer_height ) // page_height + + for page_index in range( start_page, end_page + 1 ): + + self._masonry_page_to_indices[ page_index ].append( index ) + self._masonry_index_to_pages[ index ].append( page_index ) + + + + if len( column_heights ) > 0: + + self._masonry_virtual_height = max( column_heights ) + + else: + + self._masonry_virtual_height = 0 + + def _GetMediaCoordinates( self, media ): try: index = self._sorted_media.index( media ) except: return ( -1, -1 ) + if self._UsingMasonryLayout(): + + self._EnsureMasonryLayout() + + if index >= len( self._masonry_positions ): + + return ( -1, -1 ) + + + rect = self._masonry_positions[ index ] + + if rect is None: + + return ( -1, -1 ) + + + return ( rect.x(), rect.y() ) + row = index // self._num_columns column = index % self._num_columns @@ -449,11 +685,33 @@ def _GetMediaCoordinates( self, media ): def _GetPageIndexFromThumbnailIndex( self, thumbnail_index ): - thumbnails_per_page = self._num_columns * self._num_rows_per_canvas_page - - page_index = thumbnail_index // thumbnails_per_page - - return page_index + if self._UsingMasonryLayout(): + + self._EnsureMasonryLayout() + + if thumbnail_index >= len( self._masonry_positions ): + + return 0 + + + rect = self._masonry_positions[ thumbnail_index ] + + if rect is None: + + return 0 + + + page_height = self._GetPageHeight() + + return rect.y() // page_height + + else: + + thumbnails_per_page = self._num_columns * self._num_rows_per_canvas_page + + page_index = thumbnail_index // thumbnails_per_page + + return page_index def _GetThumbnailSpanDimensions( self ): @@ -461,7 +719,7 @@ def _GetThumbnailSpanDimensions( self ): thumbnail_border = CG.client_controller.new_options.GetInteger( 'thumbnail_border' ) thumbnail_margin = CG.client_controller.new_options.GetInteger( 'thumbnail_margin' ) - return ClientData.AddPaddingToDimensions( HC.options[ 'thumbnail_dimensions' ], ( thumbnail_border + thumbnail_margin ) * 2 ) + return ClientData.AddPaddingToDimensions( self._GetThumbnailDimensions(), ( thumbnail_border + thumbnail_margin ) * 2 ) def _GetThumbnailUnderMouse( self, mouse_event ): @@ -473,11 +731,65 @@ def _GetThumbnailUnderMouse( self, mouse_event ): ( t_span_x, t_span_y ) = self._GetThumbnailSpanDimensions() + thumbnail_margin = CG.client_controller.new_options.GetInteger( 'thumbnail_margin' ) + + if self._UsingMasonryLayout(): + + self._EnsureMasonryLayout() + + x_mod = x % t_span_x + + if x_mod <= thumbnail_margin or x_mod > t_span_x - thumbnail_margin: + + return None + + + column_index = x // t_span_x + + if column_index >= self._num_columns: + + return None + + + if column_index >= len( self._masonry_column_entries ): + + return None + + + entries = self._masonry_column_entries[ column_index ] + + if len( entries ) == 0: + + return None + + + starts = self._masonry_column_starts[ column_index ] + + entry_index = bisect.bisect_right( starts, y ) - 1 + + if entry_index < 0: + + return None + + + ( start_y, end_y, thumbnail_index ) = entries[ entry_index ] + + if y < start_y or y > end_y: + + return None + + + if thumbnail_index >= len( self._sorted_media ): + + return None + + + return self._sorted_media[ thumbnail_index ] + + x_mod = x % t_span_x y_mod = y % t_span_y - thumbnail_margin = CG.client_controller.new_options.GetInteger( 'thumbnail_margin' ) - if x_mod <= thumbnail_margin or y_mod <= thumbnail_margin or x_mod > t_span_x - thumbnail_margin or y_mod > t_span_y - thumbnail_margin: return None @@ -508,22 +820,32 @@ def _GetThumbnailUnderMouse( self, mouse_event ): def _GetThumbnailsFromPageIndex( self, page_index ): - num_thumbnails_per_page = self._num_columns * self._num_rows_per_canvas_page - - start_index = num_thumbnails_per_page * page_index - - if start_index <= len( self._sorted_media ): + if self._UsingMasonryLayout(): + + self._EnsureMasonryLayout() - end_index = min( len( self._sorted_media ), start_index + num_thumbnails_per_page ) + indices = self._masonry_page_to_indices.get( page_index, [] ) - thumbnails = [ ( index, self._sorted_media[ index ] ) for index in range( start_index, end_index ) ] + return [ ( index, self._sorted_media[ index ] ) for index in indices if index < len( self._sorted_media ) ] else: - thumbnails = [] + num_thumbnails_per_page = self._num_columns * self._num_rows_per_canvas_page - - return thumbnails + start_index = num_thumbnails_per_page * page_index + + if start_index <= len( self._sorted_media ): + + end_index = min( len( self._sorted_media ), start_index + num_thumbnails_per_page ) + + thumbnails = [ ( index, self._sorted_media[ index ] ) for index in range( start_index, end_index ) ] + + else: + + thumbnails = [] + + + return thumbnails def _MediaIsInCleanPage( self, thumbnail ): @@ -537,20 +859,76 @@ def _MediaIsInCleanPage( self, thumbnail ): return False - if self._GetPageIndexFromThumbnailIndex( index ) in self._clean_canvas_pages: + if self._UsingMasonryLayout(): - return True + self._EnsureMasonryLayout() - else: + if index < len( self._masonry_index_to_pages ): + + for page_index in self._masonry_index_to_pages[ index ]: + + if page_index in self._clean_canvas_pages: + + return True + + + return False + else: + + if self._GetPageIndexFromThumbnailIndex( index ) in self._clean_canvas_pages: + + return True + + else: + + return False + def _MediaIsVisible( self, media ): if media is not None: + if self._UsingMasonryLayout(): + + try: + + index = self._sorted_media.index( media ) + + except HydrusExceptions.DataMissing: + + return False + + + self._EnsureMasonryLayout() + + if index >= len( self._masonry_positions ): + + return False + + + rect = self._masonry_positions[ index ] + + if rect is None: + + return False + + + visible_rect = QP.ScrollAreaVisibleRect( self ) + + visible_rect_y = visible_rect.y() + + visible_rect_height = visible_rect.height() + + bottom_edge_below_top_of_view = visible_rect_y < rect.y() + rect.height() + top_edge_above_bottom_of_view = rect.y() < visible_rect_y + visible_rect_height + + return bottom_edge_below_top_of_view and top_edge_above_bottom_of_view + + ( x, y ) = self._GetMediaCoordinates( media ) visible_rect = QP.ScrollAreaVisibleRect( self ) @@ -632,6 +1010,10 @@ def _NotifyThumbnailsHaveMoved( self ): self._DirtyAllPages() + if self._UsingMasonryLayout(): + + self._InvalidateMasonryLayout() + self.widget().update() @@ -646,19 +1028,28 @@ def _RecalculateVirtualSize( self, called_from_resize_event = False ): ( thumbnail_span_width, thumbnail_span_height ) = self._GetThumbnailSpanDimensions() - num_media = len( self._sorted_media ) - - num_rows = max( 1, num_media // self._num_columns ) - - if num_media % self._num_columns > 0: + if self._UsingMasonryLayout(): + + self._EnsureMasonryLayout() - num_rows += 1 + virtual_height = self._masonry_virtual_height + + else: + + num_media = len( self._sorted_media ) + + num_rows = max( 1, num_media // self._num_columns ) + + if num_media % self._num_columns > 0: + + num_rows += 1 + + + virtual_height = num_rows * thumbnail_span_height virtual_width = my_width - virtual_height = num_rows * thumbnail_span_height - yUnit = self.verticalScrollBar().singleStep() excess = virtual_height % yUnit @@ -783,6 +1174,11 @@ def _ReinitialisePageCacheIfNeeded( self ): self._DeleteAllDirtyPages() + if self._UsingMasonryLayout(): + + self._InvalidateMasonryLayout() + + self.widget().update() @@ -801,6 +1197,10 @@ def _RemoveMediaDirectly( self, singleton_media, collected_media ): self._EndShiftSelect() + if self._UsingMasonryLayout(): + + self._InvalidateMasonryLayout() + self._RecalculateVirtualSize() self._DirtyAllPages() @@ -856,6 +1256,41 @@ def _ScrollToMedia( self, media ): percent_visible = new_options.GetInteger( 'thumbnail_visibility_scroll_percent' ) / 100 + if self._UsingMasonryLayout(): + + try: + + index = self._sorted_media.index( media ) + + except HydrusExceptions.DataMissing: + + return + + + self._EnsureMasonryLayout() + + if index >= len( self._masonry_positions ): + + return + + + rect = self._masonry_positions[ index ] + + if rect is None: + + return + + + if rect.y() < visible_rect_y: + + self.ensureVisible( 0, rect.y(), 0, 0 ) + + elif rect.y() > visible_rect_y + visible_rect_height - ( rect.height() * percent_visible ): + + self.ensureVisible( 0, rect.y() + rect.height() ) + + return + if y < visible_rect_y: self.ensureVisible( 0, y, 0, 0 ) @@ -906,6 +1341,10 @@ def AddMediaResults( self, page_key, media_results ): if len( thumbnails ) > 0: + if self._UsingMasonryLayout(): + + self._InvalidateMasonryLayout() + self._RecalculateVirtualSize() CG.client_controller.thumbnails_cache.Waterfall( self._page_key, thumbnails ) @@ -1091,6 +1530,34 @@ def NewThumbnails( self, hashes ): + def NotifyNewOptions( self ): + + new_layout_options = self._GetThumbnailLayoutOptions() + new_render_options = self._GetThumbnailRenderOptions() + + if new_layout_options != self._thumbnail_layout_options: + + self._thumbnail_layout_options = new_layout_options + self._thumbnail_render_options = new_render_options + + self.ThumbnailsReset() + + elif new_render_options != self._thumbnail_render_options: + + self._thumbnail_render_options = new_render_options + + self.RedrawAllThumbnails() + + else: + + ( thumbnail_span_width, thumbnail_span_height ) = self._GetThumbnailSpanDimensions() + + thumbnail_scroll_rate = float( CG.client_controller.new_options.GetString( 'thumbnail_scroll_rate' ) ) + + self.verticalScrollBar().setSingleStep( int( round( thumbnail_span_height * thumbnail_scroll_rate ) ) ) + + + def NotifyFilesNeedRedraw( self, hashes ): affected_media = self._GetMedia( hashes ) @@ -1864,12 +2331,15 @@ def ShowMenu( self, do_not_show_just_return = False ): + def _SortFinished( self ): + + self._NotifyThumbnailsHaveMoved() + + def Sort( self, media_sort = None ): super().Sort( media_sort ) - self._NotifyThumbnailsHaveMoved() - def ThumbnailsReset( self ): @@ -1882,6 +2352,10 @@ def ThumbnailsReset( self ): self._hashes_to_thumbnails_waiting_to_be_drawn = {} self._hashes_faded = set() + if self._UsingMasonryLayout(): + + self._InvalidateMasonryLayout() + self._ReinitialisePageCacheIfNeeded() self._RecalculateVirtualSize() @@ -1901,7 +2375,7 @@ def TIMERAnimationUpdate( self ): page_indices_to_painters = {} - page_height = self._num_rows_per_canvas_page * thumbnail_span_height + page_height = self._GetPageHeight() for hash in HydrusLists.IterateListRandomlyAndFast( hashes ): @@ -1922,44 +2396,106 @@ def TIMERAnimationUpdate( self ): expected_thumbnail = None - page_index = self._GetPageIndexFromThumbnailIndex( thumbnail_index ) - if expected_thumbnail != thumbnail_draw_object.thumbnail: delete_entry = True - elif page_index not in self._clean_canvas_pages: - - delete_entry = True - else: - thumbnail_col = thumbnail_index % self._num_columns - - thumbnail_row = thumbnail_index // self._num_columns - - x = thumbnail_col * thumbnail_span_width + thumbnail_margin - - y = ( thumbnail_row - ( page_index * self._num_rows_per_canvas_page ) ) * thumbnail_span_height + thumbnail_margin - - if page_index not in page_indices_to_painters: + if self._UsingMasonryLayout(): - canvas_page = self._clean_canvas_pages[ page_index ] + self._EnsureMasonryLayout() - painter = QG.QPainter( canvas_page ) + if thumbnail_index >= len( self._masonry_positions ): + + delete_entry = True + + else: + + rect = self._masonry_positions[ thumbnail_index ] + + if rect is None: + + delete_entry = True + + else: + + pages_for_thumb = [] + + if thumbnail_index < len( self._masonry_index_to_pages ): + + pages_for_thumb = self._masonry_index_to_pages[ thumbnail_index ] + + + clean_pages = [ page_index for page_index in pages_for_thumb if page_index in self._clean_canvas_pages ] + + if len( clean_pages ) == 0: + + delete_entry = True + + else: + + for page_index in clean_pages: + + if page_index not in page_indices_to_painters: + + canvas_page = self._clean_canvas_pages[ page_index ] + + painter = QG.QPainter( canvas_page ) + + page_indices_to_painters[ page_index ] = painter + + + painter = page_indices_to_painters[ page_index ] + + y = rect.y() - ( page_index * page_height ) + + thumbnail_draw_object.DrawToPainter( rect.x(), y, painter ) + + page_virtual_y = page_height * page_index + + self.widget().update( QC.QRect( rect.x(), page_virtual_y + y, rect.width(), rect.height() ) ) + + + + - page_indices_to_painters[ page_index ] = painter + else: - - painter = page_indices_to_painters[ page_index ] - - thumbnail_draw_object.DrawToPainter( x, y, painter ) - - # - - page_virtual_y = page_height * page_index - - self.widget().update( QC.QRect( x, page_virtual_y + y, thumbnail_span_width - thumbnail_margin, thumbnail_span_height - thumbnail_margin ) ) + page_index = self._GetPageIndexFromThumbnailIndex( thumbnail_index ) + + if page_index not in self._clean_canvas_pages: + + delete_entry = True + + else: + + thumbnail_col = thumbnail_index % self._num_columns + + thumbnail_row = thumbnail_index // self._num_columns + + x = thumbnail_col * thumbnail_span_width + thumbnail_margin + + y = ( thumbnail_row - ( page_index * self._num_rows_per_canvas_page ) ) * thumbnail_span_height + thumbnail_margin + + if page_index not in page_indices_to_painters: + + canvas_page = self._clean_canvas_pages[ page_index ] + + painter = QG.QPainter( canvas_page ) + + page_indices_to_painters[ page_index ] = painter + + + painter = page_indices_to_painters[ page_index ] + + thumbnail_draw_object.DrawToPainter( x, y, painter ) + + # + + page_virtual_y = page_height * page_index + + self.widget().update( QC.QRect( x, page_virtual_y + y, thumbnail_span_width - thumbnail_margin, thumbnail_span_height - thumbnail_margin ) ) @@ -1996,6 +2532,7 @@ def __init__( self, parent: "MediaResultsPanelThumbnails" ): super().__init__( parent ) self.setMouseTracking( True ) + self.setAttribute( QC.Qt.WidgetAttribute.WA_OpaquePaintEvent, True ) self._parent = parent @@ -2035,10 +2572,11 @@ def paintEvent( self, event ): painter = QG.QPainter( self ) + painter.setClipRegion( event.region() ) ( thumbnail_span_width, thumbnail_span_height ) = self._parent._GetThumbnailSpanDimensions() - page_height = self._parent._num_rows_per_canvas_page * thumbnail_span_height + page_height = self._parent._GetPageHeight() page_indices_to_display = self._parent._CalculateVisiblePageIndices() @@ -2229,7 +2767,9 @@ def GetQtImage( self, media: ClientMedia.Media, media_panel: ClientGUIMediaResul thumbnail_border = CG.client_controller.new_options.GetInteger( 'thumbnail_border' ) - ( width, height ) = ClientData.AddPaddingToDimensions( HC.options[ 'thumbnail_dimensions' ], thumbnail_border * 2 ) + inner_dimensions = media_panel.GetThumbnailInnerDimensions( media ) + + ( width, height ) = ClientData.AddPaddingToDimensions( inner_dimensions, thumbnail_border * 2 ) qt_image_width = int( width * device_pixel_ratio ) @@ -2344,19 +2884,34 @@ def GetQtImage( self, media: ClientMedia.Media, media_panel: ClientGUIMediaResul thumbnail_dpr_percent = new_options.GetInteger( 'thumbnail_dpr_percent' ) + thumbnail_dpr = thumbnail_dpr_percent / 100 if thumbnail_dpr_percent != 100: - thumbnail_dpr = thumbnail_dpr_percent / 100 - raw_thumbnail_qt_image.setDevicePixelRatio( thumbnail_dpr ) - # qt_image.deviceIndepedentSize isn't supported in Qt5 lmao - device_independent_thumb_size = raw_thumbnail_qt_image.size() / thumbnail_dpr + + # qt_image.deviceIndepedentSize isn't supported in Qt5 lmao + device_independent_thumb_size = raw_thumbnail_qt_image.size() / thumbnail_dpr + + device_independent_thumb_width = device_independent_thumb_size.width() + device_independent_thumb_height = device_independent_thumb_size.height() + + if int( round( device_independent_thumb_width ) ) != inner_dimensions[ 0 ] or int( round( device_independent_thumb_height ) ) != inner_dimensions[ 1 ]: - else: + scaled_width = max( 1, int( round( inner_dimensions[ 0 ] * thumbnail_dpr ) ) ) + scaled_height = max( 1, int( round( inner_dimensions[ 1 ] * thumbnail_dpr ) ) ) - device_independent_thumb_size = raw_thumbnail_qt_image.size() + raw_thumbnail_qt_image = raw_thumbnail_qt_image.scaled( + scaled_width, + scaled_height, + QC.Qt.AspectRatioMode.KeepAspectRatio, + QC.Qt.TransformationMode.SmoothTransformation + ) + + raw_thumbnail_qt_image.setDevicePixelRatio( thumbnail_dpr ) + + device_independent_thumb_size = raw_thumbnail_qt_image.size() / thumbnail_dpr x_offset = ( width - device_independent_thumb_size.width() ) // 2 @@ -2391,9 +2946,11 @@ def GetQtImage( self, media: ClientMedia.Media, media_panel: ClientGUIMediaResul self._last_lower_summary = lower_summary - if len( upper_summary ) > 0 or len( lower_summary ) > 0: + draw_thumbnail_header = new_options.GetBoolean( 'draw_thumbnail_header' ) + + if ( draw_thumbnail_header and len( upper_summary ) > 0 ) or len( lower_summary ) > 0: - if len( upper_summary ) > 0: + if draw_thumbnail_header and len( upper_summary ) > 0: text_colour_with_alpha = upper_tag_summary_generator.GetTextColour() diff --git a/hydrus/client/gui/panels/options/FileSearchPanel.py b/hydrus/client/gui/panels/options/FileSearchPanel.py index 10309de1a..e895fb33a 100644 --- a/hydrus/client/gui/panels/options/FileSearchPanel.py +++ b/hydrus/client/gui/panels/options/FileSearchPanel.py @@ -56,6 +56,14 @@ def __init__( self, parent, new_options ): tt = 'This is a fairly advanced option. It only fires if the sort is simple enough for the database to do the limited sort. Some people like it, some do not. If you turn it on and _do_ want to sort a limited set by a different sort, hit "searching immediately" to pause search updates.' self._refresh_search_page_on_system_limited_sort_changed.setToolTip( ClientGUIFunctions.WrapToolTip( tt ) ) + self._hide_missing_file_locations_in_search = QW.QCheckBox( misc_panel ) + tt = 'If some file storage locations are missing (e.g. an offline NAS), hide those files from search results. Nothing is deleted or changed.' + self._hide_missing_file_locations_in_search.setToolTip( ClientGUIFunctions.WrapToolTip( tt ) ) + + self._hide_missing_file_paths_in_search = QW.QCheckBox( misc_panel ) + tt = 'Hide files from search results if their expected file path is missing. This may be slow for large result sets.' + self._hide_missing_file_paths_in_search.setToolTip( ClientGUIFunctions.WrapToolTip( tt ) ) + # self._default_tag_service_search_page.addItem( 'all known tags', CC.COMBINED_TAG_SERVICE_KEY ) @@ -81,6 +89,9 @@ def __init__( self, parent, new_options ): self._refresh_search_page_on_system_limited_sort_changed.setChecked( self._new_options.GetBoolean( 'refresh_search_page_on_system_limited_sort_changed' ) ) + self._hide_missing_file_locations_in_search.setChecked( self._new_options.GetBoolean( 'search_hide_files_in_missing_file_locations' ) ) + self._hide_missing_file_paths_in_search.setChecked( self._new_options.GetBoolean( 'search_hide_files_with_missing_paths' ) ) + # message = 'This tag autocomplete appears in file search pages and other places where you use tags and system predicates to search for files.' @@ -108,6 +119,8 @@ def __init__( self, parent, new_options ): rows.append( ( 'Implicit system:limit for all searches: ', self._forced_search_limit ) ) rows.append( ( 'If explicit system:limit, then refresh search when file sort changes: ', self._refresh_search_page_on_system_limited_sort_changed ) ) + rows.append( ( 'Hide files in missing file locations in search results: ', self._hide_missing_file_locations_in_search ) ) + rows.append( ( 'Hide files with missing file paths in search results: ', self._hide_missing_file_paths_in_search ) ) gridbox = ClientGUICommon.WrapInGrid( misc_panel, rows ) @@ -141,5 +154,7 @@ def UpdateOptions( self ): self._new_options.SetNoneableInteger( 'forced_search_limit', self._forced_search_limit.GetValue() ) self._new_options.SetBoolean( 'refresh_search_page_on_system_limited_sort_changed', self._refresh_search_page_on_system_limited_sort_changed.isChecked() ) + self._new_options.SetBoolean( 'search_hide_files_in_missing_file_locations', self._hide_missing_file_locations_in_search.isChecked() ) + self._new_options.SetBoolean( 'search_hide_files_with_missing_paths', self._hide_missing_file_paths_in_search.isChecked() ) diff --git a/hydrus/client/gui/panels/options/FileSortCollectPanel.py b/hydrus/client/gui/panels/options/FileSortCollectPanel.py index 779844b59..81fb82104 100644 --- a/hydrus/client/gui/panels/options/FileSortCollectPanel.py +++ b/hydrus/client/gui/panels/options/FileSortCollectPanel.py @@ -2,6 +2,7 @@ from hydrus.client import ClientConstants as CC from hydrus.client import ClientGlobals as CG +from hydrus.client.gui import ClientGUIFunctions from hydrus.client.gui import QtPorting as QP from hydrus.client.gui.lists import ClientGUIListBoxes from hydrus.client.gui.metadata import ClientGUITagsEditNamespaceSort @@ -72,6 +73,53 @@ def __init__( self, parent, new_options ): # + seriation_box = ClientGUICommon.StaticBox( self._file_sort_panel, 'seriation' ) + + seriation_text = 'Controls for the seriation sort types (visual, tags, or blended).' + + self._seriation_visual_weight = ClientGUICommon.BetterDoubleSpinBox( seriation_box, min = 0.0, max = 1.0 ) + self._seriation_visual_weight.setSingleStep( 0.05 ) + + self._seriation_tag_weight = ClientGUICommon.BetterDoubleSpinBox( seriation_box, min = 0.0, max = 1.0 ) + self._seriation_tag_weight.setSingleStep( 0.05 ) + + self._seriation_visual_bin_size = ClientGUICommon.BetterSpinBox( seriation_box, min = 1, max = 255 ) + self._seriation_max_visual_candidates = ClientGUICommon.BetterSpinBox( seriation_box, min = 10, max = 10000 ) + self._seriation_max_tag_candidates = ClientGUICommon.BetterSpinBox( seriation_box, min = 10, max = 10000 ) + self._seriation_max_tag_keys = ClientGUICommon.BetterSpinBox( seriation_box, min = 1, max = 50 ) + self._seriation_max_hybrid_candidates = ClientGUICommon.BetterSpinBox( seriation_box, min = 10, max = 10000 ) + + self._seriation_visual_weight.setToolTip( ClientGUIFunctions.WrapToolTip( 'Weight for visual similarity (blurhash). If both weights are zero, defaults apply.' ) ) + self._seriation_tag_weight.setToolTip( ClientGUIFunctions.WrapToolTip( 'Weight for tag similarity (Jaccard distance). If both weights are zero, defaults apply.' ) ) + self._seriation_visual_bin_size.setToolTip( ClientGUIFunctions.WrapToolTip( 'Colour bin size used to shortlist candidates for visual seriation. Smaller is more precise but slower.' ) ) + self._seriation_max_visual_candidates.setToolTip( ClientGUIFunctions.WrapToolTip( 'Max candidates considered per item for visual seriation.' ) ) + self._seriation_max_tag_candidates.setToolTip( ClientGUIFunctions.WrapToolTip( 'Max candidates considered per item for tag seriation.' ) ) + self._seriation_max_tag_keys.setToolTip( ClientGUIFunctions.WrapToolTip( 'How many rarest tags to use when building tag candidates.' ) ) + self._seriation_max_hybrid_candidates.setToolTip( ClientGUIFunctions.WrapToolTip( 'Max candidates considered per item for visual+tag seriation.' ) ) + + self._seriation_visual_weight.setValue( self._new_options.GetFloat( 'seriation_visual_weight' ) ) + self._seriation_tag_weight.setValue( self._new_options.GetFloat( 'seriation_tag_weight' ) ) + self._seriation_visual_bin_size.setValue( self._new_options.GetInteger( 'seriation_visual_bin_size' ) ) + self._seriation_max_visual_candidates.setValue( self._new_options.GetInteger( 'seriation_max_visual_candidates' ) ) + self._seriation_max_tag_candidates.setValue( self._new_options.GetInteger( 'seriation_max_tag_candidates' ) ) + self._seriation_max_tag_keys.setValue( self._new_options.GetInteger( 'seriation_max_tag_keys' ) ) + self._seriation_max_hybrid_candidates.setValue( self._new_options.GetInteger( 'seriation_max_hybrid_candidates' ) ) + + seriation_rows = [] + + seriation_rows.append( ( 'Visual weight (0-1): ', self._seriation_visual_weight ) ) + seriation_rows.append( ( 'Tag weight (0-1): ', self._seriation_tag_weight ) ) + seriation_rows.append( ( 'Visual bin size: ', self._seriation_visual_bin_size ) ) + seriation_rows.append( ( 'Max visual candidates: ', self._seriation_max_visual_candidates ) ) + seriation_rows.append( ( 'Max tag candidates: ', self._seriation_max_tag_candidates ) ) + seriation_rows.append( ( 'Max tag keys: ', self._seriation_max_tag_keys ) ) + seriation_rows.append( ( 'Max hybrid candidates: ', self._seriation_max_hybrid_candidates ) ) + + seriation_box.Add( ClientGUICommon.BetterStaticText( seriation_box, seriation_text ), CC.FLAGS_EXPAND_PERPENDICULAR ) + seriation_box.Add( ClientGUICommon.WrapInGrid( seriation_box, seriation_rows ), CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) + + # + rows = [] rows.append( ( 'Default file sort: ', self._default_media_sort ) ) @@ -83,6 +131,7 @@ def __init__( self, parent, new_options ): self._file_sort_panel.Add( gridbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) self._file_sort_panel.Add( namespace_file_sorting_box, CC.FLAGS_EXPAND_BOTH_WAYS ) + self._file_sort_panel.Add( seriation_box, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR ) vbox = QP.VBoxLayout() @@ -116,6 +165,13 @@ def UpdateOptions( self ): self._new_options.SetFallbackSort( self._fallback_media_sort.GetSort() ) self._new_options.SetBoolean( 'save_page_sort_on_change', self._save_page_sort_on_change.isChecked() ) self._new_options.SetDefaultCollect( self._default_media_collect.GetValue() ) + self._new_options.SetFloat( 'seriation_visual_weight', self._seriation_visual_weight.value() ) + self._new_options.SetFloat( 'seriation_tag_weight', self._seriation_tag_weight.value() ) + self._new_options.SetInteger( 'seriation_visual_bin_size', self._seriation_visual_bin_size.value() ) + self._new_options.SetInteger( 'seriation_max_visual_candidates', self._seriation_max_visual_candidates.value() ) + self._new_options.SetInteger( 'seriation_max_tag_candidates', self._seriation_max_tag_candidates.value() ) + self._new_options.SetInteger( 'seriation_max_tag_keys', self._seriation_max_tag_keys.value() ) + self._new_options.SetInteger( 'seriation_max_hybrid_candidates', self._seriation_max_hybrid_candidates.value() ) namespace_sorts = [ ClientMedia.MediaSort( sort_type = ( 'namespaces', sort_data ) ) for sort_data in self._namespace_file_sort_by.GetData() ] diff --git a/hydrus/client/gui/panels/options/SystemPanel.py b/hydrus/client/gui/panels/options/SystemPanel.py index c36da72d4..82e80d6f9 100644 --- a/hydrus/client/gui/panels/options/SystemPanel.py +++ b/hydrus/client/gui/panels/options/SystemPanel.py @@ -29,6 +29,9 @@ def __init__( self, parent, new_options ): self._file_system_waits_on_wakeup = QW.QCheckBox( sleep_panel ) self._file_system_waits_on_wakeup.setToolTip( ClientGUIFunctions.WrapToolTip( 'This is useful if your hydrus is stored on a NAS that takes a few seconds to get going after your machine resumes from sleep.' ) ) + + self._allow_missing_file_locations = QW.QCheckBox( sleep_panel ) + self._allow_missing_file_locations.setToolTip( ClientGUIFunctions.WrapToolTip( 'Allow file locations to be temporarily unavailable (e.g. removable drives or a NAS). This avoids repair prompts on boot, but files in the missing location will be unavailable until it returns.' ) ) # @@ -37,6 +40,7 @@ def __init__( self, parent, new_options ): self._wake_delay_period.setValue( self._new_options.GetInteger( 'wake_delay_period' ) ) self._file_system_waits_on_wakeup.setChecked( self._new_options.GetBoolean( 'file_system_waits_on_wakeup' ) ) + self._allow_missing_file_locations.setChecked( self._new_options.GetBoolean( 'allow_missing_file_locations' ) ) # @@ -45,6 +49,7 @@ def __init__( self, parent, new_options ): rows.append( ( 'Allow wake-from-system-sleep detection:', self._do_sleep_check ) ) rows.append( ( 'After a wake from system sleep, wait this many seconds before allowing new network access:', self._wake_delay_period ) ) rows.append( ( 'Include the file system in this wait: ', self._file_system_waits_on_wakeup ) ) + rows.append( ( 'Allow missing file locations (e.g. removable/NAS):', self._allow_missing_file_locations ) ) gridbox = ClientGUICommon.WrapInGrid( sleep_panel, rows ) @@ -65,5 +70,6 @@ def UpdateOptions( self ): self._new_options.SetBoolean( 'do_sleep_check', self._do_sleep_check.isChecked() ) self._new_options.SetInteger( 'wake_delay_period', self._wake_delay_period.value() ) self._new_options.SetBoolean( 'file_system_waits_on_wakeup', self._file_system_waits_on_wakeup.isChecked() ) + self._new_options.SetBoolean( 'allow_missing_file_locations', self._allow_missing_file_locations.isChecked() ) diff --git a/hydrus/client/gui/panels/options/ThumbnailsPanel.py b/hydrus/client/gui/panels/options/ThumbnailsPanel.py index a7d26aeea..223bd0857 100644 --- a/hydrus/client/gui/panels/options/ThumbnailsPanel.py +++ b/hydrus/client/gui/panels/options/ThumbnailsPanel.py @@ -39,16 +39,30 @@ def __init__( self, parent, new_options ): tt += '\n' * 2 tt += 'I believe the UI scale on the monitor this dialog opened on was {}'.format( HydrusNumbers.FloatToPercentage( self.devicePixelRatio() ) ) self._thumbnail_dpr_percentage.setToolTip( ClientGUIFunctions.WrapToolTip( tt ) ) + + self._thumbnail_zoom_percent = QP.LabelledSlider( thumbnail_appearance_box ) + self._thumbnail_zoom_percent.SetRange( 50, 300 ) + self._thumbnail_zoom_percent.SetInterval( 10 ) + tt = 'Scale the displayed thumbnail size without regenerating them. Useful for quick zoom adjustments while browsing.' + self._thumbnail_zoom_percent.setToolTip( ClientGUIFunctions.WrapToolTip( tt ) ) self._video_thumbnail_percentage_in = ClientGUICommon.BetterSpinBox( thumbnail_appearance_box, min=0, max=100 ) self._fade_thumbnails = QW.QCheckBox( thumbnail_appearance_box ) tt = 'Whenever thumbnails change (appearing on a page, selecting, an icon or tag banner changes), they normally fade from the old to the new. If you would rather they change instantly, in one frame, uncheck this.' self._fade_thumbnails.setToolTip( ClientGUIFunctions.WrapToolTip( tt ) ) - + self._allow_blurhash_fallback = QW.QCheckBox( thumbnail_appearance_box ) tt = 'If hydrus does not have a thumbnail for a file (e.g. you are looking at a deleted file, or one unexpectedly missing), but it does know its blurhash, it will generate a blurry thumbnail based off that blurhash. Turning this behaviour off here will make it always show the default "hydrus" thumbnail.' self._allow_blurhash_fallback.setToolTip( ClientGUIFunctions.WrapToolTip( tt ) ) + + self._thumbnail_masonry = QW.QCheckBox( thumbnail_appearance_box ) + tt = 'Lay out thumbnails in a masonry column layout with variable heights.' + self._thumbnail_masonry.setToolTip( ClientGUIFunctions.WrapToolTip( tt ) ) + + self._draw_thumbnail_header = QW.QCheckBox( thumbnail_appearance_box ) + tt = 'Show the top tag summary header on thumbnails.' + self._draw_thumbnail_header.setToolTip( ClientGUIFunctions.WrapToolTip( tt ) ) # @@ -86,10 +100,13 @@ def __init__( self, parent, new_options ): self._thumbnail_scale_type.SetValue( self._new_options.GetInteger( 'thumbnail_scale_type' ) ) self._thumbnail_dpr_percentage.setValue( self._new_options.GetInteger( 'thumbnail_dpr_percent' ) ) + self._thumbnail_zoom_percent.SetValue( self._new_options.GetInteger( 'thumbnail_zoom_percent' ) ) self._video_thumbnail_percentage_in.setValue( self._new_options.GetInteger( 'video_thumbnail_percentage_in' ) ) self._allow_blurhash_fallback.setChecked( self._new_options.GetBoolean( 'allow_blurhash_fallback' ) ) + self._thumbnail_masonry.setChecked( self._new_options.GetBoolean( 'thumbnail_masonry' ) ) + self._draw_thumbnail_header.setChecked( self._new_options.GetBoolean( 'draw_thumbnail_header' ) ) self._fade_thumbnails.setChecked( self._new_options.GetBoolean( 'fade_thumbnails' ) ) @@ -121,9 +138,12 @@ def __init__( self, parent, new_options ): rows.append( ( 'Thumbnail margin: ', self._thumbnail_margin ) ) rows.append( ( 'Thumbnail scaling: ', self._thumbnail_scale_type ) ) rows.append( ( 'Thumbnail UI-scale supersampling %: ', self._thumbnail_dpr_percentage ) ) + rows.append( ( 'Thumbnail zoom %: ', self._thumbnail_zoom_percent ) ) rows.append( ( 'Generate video thumbnails this % in: ', self._video_thumbnail_percentage_in ) ) rows.append( ( 'Fade thumbnails: ', self._fade_thumbnails ) ) rows.append( ( 'Use blurhash missing thumbnail fallback: ', self._allow_blurhash_fallback ) ) + rows.append( ( 'Use masonry thumbnail layout: ', self._thumbnail_masonry ) ) + rows.append( ( 'Show thumbnail header: ', self._draw_thumbnail_header ) ) gridbox = ClientGUICommon.WrapInGrid( thumbnail_appearance_box, rows ) @@ -186,6 +206,7 @@ def UpdateOptions( self ): self._new_options.SetInteger( 'thumbnail_scale_type', self._thumbnail_scale_type.GetValue() ) self._new_options.SetInteger( 'thumbnail_dpr_percent', self._thumbnail_dpr_percentage.value() ) + self._new_options.SetInteger( 'thumbnail_zoom_percent', self._thumbnail_zoom_percent.GetValue() ) self._new_options.SetInteger( 'video_thumbnail_percentage_in', self._video_thumbnail_percentage_in.value() ) @@ -195,6 +216,8 @@ def UpdateOptions( self ): self._new_options.SetBoolean( 'focus_preview_on_shift_click_only_static', self._focus_preview_on_shift_click_only_static.isChecked() ) self._new_options.SetBoolean( 'allow_blurhash_fallback', self._allow_blurhash_fallback.isChecked() ) + self._new_options.SetBoolean( 'thumbnail_masonry', self._thumbnail_masonry.isChecked() ) + self._new_options.SetBoolean( 'draw_thumbnail_header', self._draw_thumbnail_header.isChecked() ) self._new_options.SetBoolean( 'fade_thumbnails', self._fade_thumbnails.isChecked() ) diff --git a/hydrus/client/importing/ClientImportFileSeeds.py b/hydrus/client/importing/ClientImportFileSeeds.py index 1e3a8f42b..36b6253cc 100644 --- a/hydrus/client/importing/ClientImportFileSeeds.py +++ b/hydrus/client/importing/ClientImportFileSeeds.py @@ -34,6 +34,7 @@ from hydrus.client.importing.options import TagImportOptions from hydrus.client.metadata import ClientContentUpdates from hydrus.client.metadata import ClientTags +from hydrus.client.metadata import ClientTikTok from hydrus.client.networking import ClientNetworkingFunctions from hydrus.client.parsing import ClientParsing from hydrus.client.parsing import ClientParsingResults @@ -684,10 +685,21 @@ def AddParsedPost( self, parsed_post: ClientParsingResults.ParsedPost, file_impo tags = parsed_post.GetTags() - self._tags.update( tags ) - names_and_notes = parsed_post.GetNamesAndNotes() + tiktok_tags = ClientTikTok.GetTikTokTagsFromParsedMetadata( + parsed_post.GetURLs( ClientTikTok.TIKTOK_URL_TYPES ), + names_and_notes, + tags + ) + + if len( tiktok_tags ) > 0: + + tags.update( tiktok_tags ) + + + self._tags.update( tags ) + self._names_and_notes_dict.update( names_and_notes ) source_timestamp = parsed_post.GetTimestamp( HC.TIMESTAMP_TYPE_MODIFIED_DOMAIN ) diff --git a/hydrus/client/media/ClientMedia.py b/hydrus/client/media/ClientMedia.py index 093d4717c..0ebdad42b 100644 --- a/hydrus/client/media/ClientMedia.py +++ b/hydrus/client/media/ClientMedia.py @@ -92,6 +92,608 @@ def GetBlurhashToSortableCall( sort_data: int ): return sort_data_to_blurhash_to_sortable_calls.get( sort_data, HydrusBlurhash.ConvertBlurhashToSortableLightness ) +SERIATION_VISUAL_WEIGHT = 0.7 +SERIATION_TAG_WEIGHT = 0.3 +SERIATION_VISUAL_BIN_SIZE = 32 +SERIATION_VISUAL_JACCARD_BUCKET_SIZE = 4 +SERIATION_MAX_VISUAL_CANDIDATES = 200 +SERIATION_MAX_TAG_CANDIDATES = 200 +SERIATION_MAX_TAG_KEYS = 5 +SERIATION_MAX_HYBRID_CANDIDATES = 250 + + +def _GetSeriationInteger( name, default_value ): + + try: + + value = CG.client_controller.new_options.GetInteger( name ) + + return max( 1, int( value ) ) + + except Exception: + + return default_value + + + +def _GetSeriationFloat( name, default_value ): + + try: + + value = CG.client_controller.new_options.GetFloat( name ) + + return max( 0.0, float( value ) ) + + except Exception: + + return default_value + + + +def _ColourDistance( c1, c2 ): + + ( r1, g1, b1 ) = c1 + ( r2, g2, b2 ) = c2 + + return ( r1 - r2 ) ** 2 + ( g1 - g2 ) ** 2 + ( b1 - b2 ) ** 2 + + +def _JaccardDistance( s1, s2 ): + + intersection = len( s1.intersection( s2 ) ) + union = len( s1 ) + len( s2 ) - intersection + + if union == 0: + + return 0.0 + + + return 1.0 - ( intersection / union ) + + +def _TagDistance( t1, t2 ): + + return _JaccardDistance( t1, t2 ) + + +def _BuildVisualJaccardFeature( numpy_image, bucket_size: int ): + + flat = numpy_image.reshape( -1 ) + + return frozenset( ( i, int( v ) // bucket_size ) for ( i, v ) in enumerate( flat ) ) + + +def _TrimCandidates( candidates, score_fn, max_candidates ): + + if len( candidates ) <= max_candidates: + + return set( candidates ) + + + trimmed = sorted( candidates, key = lambda i: ( score_fn( i ), i ) ) + + return set( trimmed[ : max_candidates ] ) + + +def _BuildVisualCandidateSets( average_colours, bin_size: int, max_candidates: int ): + + bins = collections.defaultdict( list ) + + for ( i, colour ) in enumerate( average_colours ): + + if colour is None: + + continue + + + ( r, g, b ) = colour + key = ( int( r ) // bin_size, int( g ) // bin_size, int( b ) // bin_size ) + + bins[ key ].append( i ) + + + candidates_by_index = [ set() for _ in average_colours ] + + for ( i, colour ) in enumerate( average_colours ): + + if colour is None: + + continue + + + ( r, g, b ) = colour + key = ( int( r ) // bin_size, int( g ) // bin_size, int( b ) // bin_size ) + + for dx in ( -1, 0, 1 ): + + for dy in ( -1, 0, 1 ): + + for dz in ( -1, 0, 1 ): + + neighbour_key = ( key[0] + dx, key[1] + dy, key[2] + dz ) + + candidates_by_index[ i ].update( bins.get( neighbour_key, [] ) ) + + + + + candidates_by_index[ i ].discard( i ) + + if len( candidates_by_index[ i ] ) > max_candidates: + + candidates_by_index[ i ] = _TrimCandidates( + candidates_by_index[ i ], + lambda j, c = colour: _ColourDistance( average_colours[ j ], c ), + max_candidates + ) + + + + return candidates_by_index + + +def _BuildTagCandidateSets( tag_sets, max_candidates: int, max_tags_per_item: int ): + + tag_counts = collections.Counter() + tag_to_indices = collections.defaultdict( list ) + + for ( i, tags ) in enumerate( tag_sets ): + + if tags is None or len( tags ) == 0: + + continue + + + tag_counts.update( tags ) + + for tag in tags: + + tag_to_indices[ tag ].append( i ) + + + + candidates_by_index = [ set() for _ in tag_sets ] + + for ( i, tags ) in enumerate( tag_sets ): + + if tags is None or len( tags ) == 0: + + continue + + + sorted_tags = sorted( tags, key = lambda t: ( tag_counts[ t ], t ) ) + + for tag in sorted_tags[ : max_tags_per_item ]: + + candidates_by_index[ i ].update( tag_to_indices[ tag ] ) + + if len( candidates_by_index[ i ] ) >= max_candidates: + + break + + + + candidates_by_index[ i ].discard( i ) + + if len( candidates_by_index[ i ] ) > max_candidates: + + candidates_by_index[ i ] = _TrimCandidates( + candidates_by_index[ i ], + lambda j, t = tags: _TagDistance( t, tag_sets[ j ] ), + max_candidates + ) + + + + return ( candidates_by_index, tag_counts ) + + +def _PickFallbackIndex( unvisited, fallback_indices ): + + for i in fallback_indices: + + if i in unvisited: + + return i + + + + return next( iter( unvisited ) ) + + +def _GetGreedySeriationOrderWithCandidates( count: int, distance, start_index: int, candidates_by_index, fallback_indices ): + + unvisited = set( range( count ) ) + order = [] + current_index = start_index + + # Greedy nearest-neighbor path across the feature space, limited to candidate sets. + while len( unvisited ) > 0: + + if current_index not in unvisited: + + current_index = _PickFallbackIndex( unvisited, fallback_indices ) + + + order.append( current_index ) + unvisited.discard( current_index ) + + if len( unvisited ) == 0: + + break + + + best_index = None + best_distance = None + + for i in candidates_by_index[ current_index ]: + + if i not in unvisited: + + continue + + + value = distance( i, current_index ) + + if best_distance is None or value < best_distance: + + best_distance = value + best_index = i + + + + if best_index is None: + + current_index = _PickFallbackIndex( unvisited, fallback_indices ) + + else: + + current_index = best_index + + + + return order + + +def GetSeriationOrderingForMediasByBlurhash( medias: list[ "Media" ], reverse: bool ): + + medias_with_features = [] + medias_without_features = [] + visual_features = [] + average_colours = [] + visual_bin_size = _GetSeriationInteger( 'seriation_visual_bin_size', SERIATION_VISUAL_BIN_SIZE ) + max_visual_candidates = _GetSeriationInteger( 'seriation_max_visual_candidates', SERIATION_MAX_VISUAL_CANDIDATES ) + + for media in medias: + + display_media = media.GetDisplayMedia() + + if display_media is None: + + medias_without_features.append( media ) + + continue + + + blurhash = display_media.GetMediaResult().GetFileInfoManager().blurhash + + if blurhash is None: + + medias_without_features.append( media ) + + continue + + + try: + + average_colour = HydrusBlurhash.GetAverageColourFromBlurhash( blurhash ) + numpy_image = HydrusBlurhash.GetNumpyFromBlurhash( blurhash, 8, 8 ) + feature = _BuildVisualJaccardFeature( numpy_image, SERIATION_VISUAL_JACCARD_BUCKET_SIZE ) + + except Exception: + + medias_without_features.append( media ) + + continue + + + if len( feature ) == 0: + + medias_without_features.append( media ) + + continue + + + medias_with_features.append( media ) + visual_features.append( feature ) + average_colours.append( average_colour ) + + + if len( medias_with_features ) <= 1: + + ordered_medias = list( medias_with_features ) + ordered_medias.extend( medias_without_features ) + + if reverse: + + ordered_medias.reverse() + + + return ordered_medias + + + count = len( medias_with_features ) + candidate_sets = _BuildVisualCandidateSets( average_colours, visual_bin_size, max_visual_candidates ) + def distance( i, j ): + + return _JaccardDistance( visual_features[ i ], visual_features[ j ] ) + + + ( rs, gs, bs ) = ( 0, 0, 0 ) + + for ( r, g, b ) in average_colours: + + rs += r + gs += g + bs += b + + + centroid = ( rs / count, gs / count, bs / count ) + + start_index = min( range( count ), key = lambda i: _ColourDistance( average_colours[ i ], centroid ) ) + fallback_indices = sorted( range( count ), key = lambda i: _ColourDistance( average_colours[ i ], centroid ) ) + + order = _GetGreedySeriationOrderWithCandidates( count, distance, start_index, candidate_sets, fallback_indices ) + ordered_medias = [ medias_with_features[ i ] for i in order ] + ordered_medias.extend( medias_without_features ) + + if reverse: + + ordered_medias.reverse() + + + return ordered_medias + + +def GetSeriationOrderingForMediasByTags( medias: list[ "Media" ], reverse: bool, tag_context: ClientSearchTagContext.TagContext ): + + medias_with_features = [] + medias_without_features = [] + tag_sets = [] + + for media in medias: + + tags_manager = media.GetTagsManager() + tags = set( tags_manager.GetCurrentAndPending( tag_context.service_key, ClientTags.TAG_DISPLAY_DISPLAY_ACTUAL ) ) + + if len( tags ) == 0: + + medias_without_features.append( media ) + + continue + + + medias_with_features.append( media ) + tag_sets.append( tags ) + + + if len( medias_with_features ) <= 1: + + ordered_medias = list( medias_with_features ) + ordered_medias.extend( medias_without_features ) + + if reverse: + + ordered_medias.reverse() + + + return ordered_medias + + + max_tag_candidates = _GetSeriationInteger( 'seriation_max_tag_candidates', SERIATION_MAX_TAG_CANDIDATES ) + max_tag_keys = _GetSeriationInteger( 'seriation_max_tag_keys', SERIATION_MAX_TAG_KEYS ) + ( candidate_sets, tag_counts ) = _BuildTagCandidateSets( tag_sets, max_tag_candidates, max_tag_keys ) + count = len( medias_with_features ) + + def distance( i, j ): + + return _TagDistance( tag_sets[ i ], tag_sets[ j ] ) + + + start_index = max( range( count ), key = lambda i: sum( tag_counts[ tag ] for tag in tag_sets[ i ] ) ) + fallback_indices = sorted( range( count ), key = lambda i: -sum( tag_counts[ tag ] for tag in tag_sets[ i ] ) ) + + order = _GetGreedySeriationOrderWithCandidates( count, distance, start_index, candidate_sets, fallback_indices ) + ordered_medias = [ medias_with_features[ i ] for i in order ] + ordered_medias.extend( medias_without_features ) + + if reverse: + + ordered_medias.reverse() + + + return ordered_medias + + +def GetSeriationOrderingForMediasByVisualAndTags( medias: list[ "Media" ], reverse: bool, tag_context: ClientSearchTagContext.TagContext ): + + medias_with_features = [] + medias_without_features = [] + visual_features = [] + average_colours = [] + tag_sets = [] + + for media in medias: + + tags_manager = media.GetTagsManager() + tags = set( tags_manager.GetCurrentAndPending( tag_context.service_key, ClientTags.TAG_DISPLAY_DISPLAY_ACTUAL ) ) + + display_media = media.GetDisplayMedia() + blurhash = None + average_colour = None + visual_feature = None + + if display_media is not None: + + blurhash = display_media.GetMediaResult().GetFileInfoManager().blurhash + + + if blurhash is not None: + + try: + + average_colour = HydrusBlurhash.GetAverageColourFromBlurhash( blurhash ) + numpy_image = HydrusBlurhash.GetNumpyFromBlurhash( blurhash, 8, 8 ) + visual_feature = _BuildVisualJaccardFeature( numpy_image, SERIATION_VISUAL_JACCARD_BUCKET_SIZE ) + + except Exception: + + average_colour = None + visual_feature = None + + + + if visual_feature is None and len( tags ) == 0: + + medias_without_features.append( media ) + + continue + + + medias_with_features.append( media ) + visual_features.append( visual_feature ) + average_colours.append( average_colour ) + tag_sets.append( tags ) + + + if len( medias_with_features ) <= 1: + + ordered_medias = list( medias_with_features ) + ordered_medias.extend( medias_without_features ) + + if reverse: + + ordered_medias.reverse() + + + return ordered_medias + + + count = len( medias_with_features ) + max_tag_candidates = _GetSeriationInteger( 'seriation_max_tag_candidates', SERIATION_MAX_TAG_CANDIDATES ) + max_tag_keys = _GetSeriationInteger( 'seriation_max_tag_keys', SERIATION_MAX_TAG_KEYS ) + visual_bin_size = _GetSeriationInteger( 'seriation_visual_bin_size', SERIATION_VISUAL_BIN_SIZE ) + max_visual_candidates = _GetSeriationInteger( 'seriation_max_visual_candidates', SERIATION_MAX_VISUAL_CANDIDATES ) + max_hybrid_candidates = _GetSeriationInteger( 'seriation_max_hybrid_candidates', SERIATION_MAX_HYBRID_CANDIDATES ) + visual_weight = _GetSeriationFloat( 'seriation_visual_weight', SERIATION_VISUAL_WEIGHT ) + tag_weight = _GetSeriationFloat( 'seriation_tag_weight', SERIATION_TAG_WEIGHT ) + + if visual_weight == 0.0 and tag_weight == 0.0: + + visual_weight = SERIATION_VISUAL_WEIGHT + tag_weight = SERIATION_TAG_WEIGHT + + + ( tag_candidates, tag_counts ) = _BuildTagCandidateSets( tag_sets, max_tag_candidates, max_tag_keys ) + visual_candidates = _BuildVisualCandidateSets( average_colours, visual_bin_size, max_visual_candidates ) + + candidates_by_index = [] + + for i in range( count ): + + combined = set() + combined.update( visual_candidates[ i ] ) + combined.update( tag_candidates[ i ] ) + candidates_by_index.append( combined ) + + + def visual_distance( f1, f2 ): + + return _JaccardDistance( f1, f2 ) + + + def distance( i, j ): + + total = 0.0 + weight_sum = 0.0 + + if visual_features[ i ] is not None and visual_features[ j ] is not None: + + total += visual_weight * visual_distance( visual_features[ i ], visual_features[ j ] ) + weight_sum += visual_weight + + + if len( tag_sets[ i ] ) > 0 and len( tag_sets[ j ] ) > 0: + + total += tag_weight * _TagDistance( tag_sets[ i ], tag_sets[ j ] ) + weight_sum += tag_weight + + + if weight_sum == 0.0: + + return 1.0 + + + return total / weight_sum + + + for i in range( count ): + + if len( candidates_by_index[ i ] ) > max_hybrid_candidates: + + candidates_by_index[ i ] = _TrimCandidates( + candidates_by_index[ i ], + lambda j, idx = i: distance( idx, j ), + max_hybrid_candidates + ) + + + centroid = None + colours = [ c for c in average_colours if c is not None ] + + if len( colours ) > 0: + + ( rs, gs, bs ) = ( 0, 0, 0 ) + + for ( r, g, b ) in colours: + + rs += r + gs += g + bs += b + + + colour_count = len( colours ) + centroid = ( rs / colour_count, gs / colour_count, bs / colour_count ) + + + if centroid is not None: + + start_index = min( + range( count ), + key = lambda i: _ColourDistance( average_colours[ i ], centroid ) if average_colours[ i ] is not None else float( 'inf' ) + ) + fallback_indices = sorted( + range( count ), + key = lambda i: _ColourDistance( average_colours[ i ], centroid ) if average_colours[ i ] is not None else float( 'inf' ) + ) + + else: + + start_index = max( range( count ), key = lambda i: sum( tag_counts[ tag ] for tag in tag_sets[ i ] ) ) + fallback_indices = sorted( range( count ), key = lambda i: -sum( tag_counts[ tag ] for tag in tag_sets[ i ] ) ) + + + order = _GetGreedySeriationOrderWithCandidates( count, distance, start_index, candidates_by_index, fallback_indices ) + ordered_medias = [ medias_with_features[ i ] for i in order ] + ordered_medias.extend( medias_without_features ) + + if reverse: + + ordered_medias.reverse() + + + return ordered_medias + def GetLocalFileServiceKeys( flat_medias: collections.abc.Collection[ "MediaSingleton" ] ): local_media_file_service_keys = set( CG.client_controller.services_manager.GetServiceKeys( ( HC.LOCAL_FILE_DOMAIN, ) ) ) @@ -2832,6 +3434,9 @@ def GetSortOrderStrings( self ): sort_string_lookup[ CC.SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_GREEN_RED ] = ( 'greens first', 'reds first', CC.SORT_ASC ) sort_string_lookup[ CC.SORT_FILES_BY_AVERAGE_COLOUR_CHROMATICITY_BLUE_YELLOW ] = ( 'blues first', 'yellows first', CC.SORT_ASC ) sort_string_lookup[ CC.SORT_FILES_BY_AVERAGE_COLOUR_HUE ] = ( 'rainbow - red first', 'rainbow - purple first', CC.SORT_ASC ) + sort_string_lookup[ CC.SORT_FILES_BY_SERIATION ] = ( 'forward', 'reverse', CC.SORT_ASC ) + sort_string_lookup[ CC.SORT_FILES_BY_SERIATION_TAGS ] = ( 'forward', 'reverse', CC.SORT_ASC ) + sort_string_lookup[ CC.SORT_FILES_BY_SERIATION_VISUAL_TAGS ] = ( 'forward', 'reverse', CC.SORT_ASC ) sort_string_lookup[ CC.SORT_FILES_BY_WIDTH ] = ( 'slimmest first', 'widest first', CC.SORT_ASC ) sort_string_lookup[ CC.SORT_FILES_BY_HEIGHT ] = ( 'shortest first', 'tallest first', CC.SORT_ASC ) sort_string_lookup[ CC.SORT_FILES_BY_RATIO ] = ( 'tallest first', 'widest first', CC.SORT_ASC ) @@ -2900,6 +3505,30 @@ def Sort( self, location_context: ClientLocation.LocationContext, tag_context: C media_results_list.random_sort() + elif sort_data == CC.SORT_FILES_BY_SERIATION: + + ordered_medias = GetSeriationOrderingForMediasByBlurhash( list( media_results_list ), self.sort_order == CC.SORT_DESC ) + order_lookup = { media : i for ( i, media ) in enumerate( ordered_medias ) } + max_index = len( order_lookup ) + + media_results_list.sort( key = lambda m: order_lookup.get( m, max_index ), reverse = False ) + + elif sort_data == CC.SORT_FILES_BY_SERIATION_TAGS: + + ordered_medias = GetSeriationOrderingForMediasByTags( list( media_results_list ), self.sort_order == CC.SORT_DESC, self.tag_context ) + order_lookup = { media : i for ( i, media ) in enumerate( ordered_medias ) } + max_index = len( order_lookup ) + + media_results_list.sort( key = lambda m: order_lookup.get( m, max_index ), reverse = False ) + + elif sort_data == CC.SORT_FILES_BY_SERIATION_VISUAL_TAGS: + + ordered_medias = GetSeriationOrderingForMediasByVisualAndTags( list( media_results_list ), self.sort_order == CC.SORT_DESC, self.tag_context ) + order_lookup = { media : i for ( i, media ) in enumerate( ordered_medias ) } + max_index = len( order_lookup ) + + media_results_list.sort( key = lambda m: order_lookup.get( m, max_index ), reverse = False ) + else: ( sort_key, reverse ) = self.GetSortKeyAndReverse( location_context ) diff --git a/hydrus/client/metadata/ClientTikTok.py b/hydrus/client/metadata/ClientTikTok.py new file mode 100644 index 000000000..9d4b99519 --- /dev/null +++ b/hydrus/client/metadata/ClientTikTok.py @@ -0,0 +1,174 @@ +import re +import urllib.parse + +from hydrus.core import HydrusConstants as HC +from hydrus.core import HydrusTags + +_TIKTOK_HANDLE_RE = re.compile( r'/@([^/?#]+)' ) +_TIKTOK_HASHTAG_RE = re.compile( r'(? bool: + + if not url: + + return False + + + try: + + parsed = urllib.parse.urlparse( url ) + + except Exception: + + return False + + + netloc = parsed.netloc.lower() + + if not netloc: + + return False + + + if ':' in netloc: + + netloc = netloc.split( ':', 1 )[0] + + + return netloc.endswith( 'tiktok.com' ) + + +def GetTikTokTagsFromURLs( urls ): + + raw_tags = set() + + for url in urls: + + if not _LooksLikeTikTokURL( url ): + + continue + + + raw_tags.add( 'site:tiktok' ) + + match = _TIKTOK_HANDLE_RE.search( url ) + + if match is not None: + + handle = match.group( 1 ) + + if handle.startswith( '@' ): + + handle = handle[1:] + + + if handle: + + raw_tags.add( f'creator:{handle}' ) + + + + + return HydrusTags.CleanTags( raw_tags ) + + +def GetTikTokDescriptionTextFromNotes( notes_manager ): + + return GetTikTokDescriptionTextFromNamesAndNotes( notes_manager.GetNamesToNotes() ) + + +def GetTikTokDescriptionTextFromNamesAndNotes( names_and_notes ): + + if isinstance( names_and_notes, dict ): + + names_to_notes = dict( names_and_notes ) + + else: + + names_to_notes = { name : note for ( name, note ) in names_and_notes } + + + for ( name, note ) in names_to_notes.items(): + + if name is None: + + continue + + + lowered_name = name.strip().lower() + + if lowered_name in _TIKTOK_DESCRIPTION_NOTE_NAMES: + + return note + + + + if len( names_to_notes ) == 1: + + return next( iter( names_to_notes.values() ) ) + + + return None + + +def GetTikTokHashtagTagsFromText( text ): + + if not text: + + return set() + + + raw_tags = set() + + for match in _TIKTOK_HASHTAG_RE.finditer( text ): + + hashtag = match.group( 1 ) + + if hashtag: + + raw_tags.add( f'tiktok:{hashtag}' ) + + + + return HydrusTags.CleanTags( raw_tags ) + + +def GetTikTokTagsFromParsedMetadata( urls, names_and_notes, existing_tags = None ): + + tags = set() + + tags.update( GetTikTokTagsFromURLs( urls ) ) + + looks_like_tiktok = len( tags ) > 0 + + if existing_tags is not None and 'site:tiktok' in existing_tags: + + looks_like_tiktok = True + + + if looks_like_tiktok: + + description_text = GetTikTokDescriptionTextFromNamesAndNotes( names_and_notes ) + + if description_text is not None: + + tags.update( GetTikTokHashtagTagsFromText( description_text ) ) + + + + return tags + diff --git a/hydrus/core/HydrusGlobals.py b/hydrus/core/HydrusGlobals.py index 9e03eafb1..05a73b689 100644 --- a/hydrus/core/HydrusGlobals.py +++ b/hydrus/core/HydrusGlobals.py @@ -43,12 +43,12 @@ boot_with_network_traffic_paused_command_line = False -db_profile_min_job_time_ms = 16 -callto_profile_min_job_time_ms = 10 +db_profile_min_job_time_ms = 1 +callto_profile_min_job_time_ms = 1 server_profile_min_job_time_ms = 10 -menu_profile_min_job_time_ms = 16 -pubsub_profile_min_job_time_ms = 5 -ui_timer_profile_min_job_time_ms = 5 +menu_profile_min_job_time_ms = 1 +pubsub_profile_min_job_time_ms = 1 +ui_timer_profile_min_job_time_ms = 1 macos_antiflicker_test = False diff --git a/hydrus/core/HydrusLists.py b/hydrus/core/HydrusLists.py index 5b455e809..b2939c202 100644 --- a/hydrus/core/HydrusLists.py +++ b/hydrus/core/HydrusLists.py @@ -14,23 +14,8 @@ def DedupeList( xs: collections.abc.Iterable ): return list( xs ) - xs_seen = set() - - xs_return = [] - - for x in xs: - - if x in xs_seen: - - continue - - - xs_return.append( x ) - - xs_seen.add( x ) - - - return xs_return + # dict preserves insertion order, so this removes duplicates while keeping the first occurrence + return list( dict.fromkeys( xs ) ) class FastIndexUniqueList( collections.abc.MutableSequence ): diff --git a/hydrus/core/files/images/HydrusBlurhash.py b/hydrus/core/files/images/HydrusBlurhash.py index 83bb2a3cf..6a9f7a54f 100644 --- a/hydrus/core/files/images/HydrusBlurhash.py +++ b/hydrus/core/files/images/HydrusBlurhash.py @@ -1,3 +1,5 @@ +import functools + import numpy import cv2 @@ -5,6 +7,9 @@ from hydrus.core.files.images import HydrusImageHandling +_BLURHASH_DECODE_CACHE_MAX = 256 +_BLURHASH_DECODE_CACHE_MAX_DIMENSION = 256 + # pretty grunky but it seems to all work and this is low level so I'll endure it def rgb_to_hsl( r, g, b ): @@ -202,6 +207,16 @@ def GetBlurhashFromNumPy( numpy_image: numpy.ndarray ) -> str: def GetNumpyFromBlurhash( blurhash, width, height ) -> numpy.ndarray: # this thing is super slow, they recommend even in the documentation to render small and scale up + if width > _BLURHASH_DECODE_CACHE_MAX_DIMENSION or height > _BLURHASH_DECODE_CACHE_MAX_DIMENSION: + + return _GetNumpyFromBlurhashUncached( blurhash, width, height ) + + + return _GetNumpyFromBlurhashCached( blurhash, width, height ) + + +def _GetNumpyFromBlurhashUncached( blurhash, width, height ) -> numpy.ndarray: + if width > 32 or height > 32: numpy_image = numpy.array( external_blurhash.blurhash_decode( blurhash, 32, 32 ), dtype = 'uint8' ) @@ -214,4 +229,13 @@ def GetNumpyFromBlurhash( blurhash, width, height ) -> numpy.ndarray: return numpy_image + + +@functools.lru_cache( maxsize = _BLURHASH_DECODE_CACHE_MAX ) +def _GetNumpyFromBlurhashCached( blurhash, width, height ) -> numpy.ndarray: + + numpy_image = _GetNumpyFromBlurhashUncached( blurhash, width, height ) + numpy_image.setflags( write = False ) + + return numpy_image diff --git a/hydrus/test/TestClientFilesMaintenance.py b/hydrus/test/TestClientFilesMaintenance.py new file mode 100644 index 000000000..a78e3ae2b --- /dev/null +++ b/hydrus/test/TestClientFilesMaintenance.py @@ -0,0 +1,106 @@ +import unittest + +from hydrus.client.metadata import ClientTikTok + + +class _DummyNotesManager: + + def __init__( self, names_to_notes ): + + self._names_to_notes = dict( names_to_notes ) + + + def GetNamesToNotes( self ): + + return dict( self._names_to_notes ) + + +class TestClientFilesMaintenanceTikTokTags( unittest.TestCase ): + + def test_tiktok_tags_from_urls( self ): + + urls = [ + 'https://www.tiktok.com/@ExampleUser/video/12345?is_from_webapp=1', + 'https://m.tiktok.com/@some_user', + 'https://vm.tiktok.com/ZSAbc/' + ] + + tags = ClientTikTok.GetTikTokTagsFromURLs( urls ) + + self.assertEqual( tags, { 'site:tiktok', 'creator:exampleuser', 'creator:some_user' } ) + + + def test_non_tiktok_urls_are_ignored( self ): + + urls = [ 'https://example.com/@user/video/1' ] + + tags = ClientTikTok.GetTikTokTagsFromURLs( urls ) + + self.assertEqual( tags, set() ) + + + def test_tiktok_description_note_lookup( self ): + + notes_manager = _DummyNotesManager( + { + 'Description' : 'hello #One', + 'other' : '#two' + } + ) + + description_text = ClientTikTok.GetTikTokDescriptionTextFromNotes( notes_manager ) + + self.assertEqual( description_text, 'hello #One' ) + + + def test_tiktok_description_note_fallback_single_note( self ): + + notes_manager = _DummyNotesManager( { 'misc' : 'hi #tag' } ) + + description_text = ClientTikTok.GetTikTokDescriptionTextFromNotes( notes_manager ) + + self.assertEqual( description_text, 'hi #tag' ) + + + def test_tiktok_hashtag_tags_from_text( self ): + + text = 'hello #One and #two_2 then #3rd.' + + tags = ClientTikTok.GetTikTokHashtagTagsFromText( text ) + + self.assertEqual( tags, { 'tiktok:one', 'tiktok:two_2', 'tiktok:3rd' } ) + + + def test_tiktok_tags_from_parsed_metadata( self ): + + urls = [ 'https://www.tiktok.com/@ExampleUser/video/12345' ] + names_and_notes = [ ( 'description', 'hello #Tag' ) ] + + tags = ClientTikTok.GetTikTokTagsFromParsedMetadata( urls, names_and_notes ) + + self.assertEqual( tags, { 'site:tiktok', 'creator:exampleuser', 'tiktok:tag' } ) + + + def test_non_tiktok_parsed_metadata_is_ignored( self ): + + urls = [ 'https://example.com/@user/video/1' ] + names_and_notes = [ ( 'description', 'hello #Tag' ) ] + + tags = ClientTikTok.GetTikTokTagsFromParsedMetadata( urls, names_and_notes ) + + self.assertEqual( tags, set() ) + + + def test_tiktok_parsed_metadata_with_existing_site_tag( self ): + + urls = [] + names_and_notes = [ ( 'description', 'hello #Tag' ) ] + + tags = ClientTikTok.GetTikTokTagsFromParsedMetadata( + urls, + names_and_notes, + existing_tags = { 'site:tiktok' } + ) + + self.assertEqual( tags, { 'tiktok:tag' } ) + diff --git a/hydrus/test/TestClientMediaSort.py b/hydrus/test/TestClientMediaSort.py new file mode 100644 index 000000000..080102db0 --- /dev/null +++ b/hydrus/test/TestClientMediaSort.py @@ -0,0 +1,163 @@ +import unittest +from unittest import mock + +import numpy + +from hydrus.client import ClientConstants as CC +from hydrus.client.media import ClientMedia +from hydrus.client.search import ClientSearchTagContext + + +class _DummyTagsManager: + + def __init__( self, tags ): + + self._tags = set( tags ) + + + def GetCurrentAndPending( self, service_key, tag_display_type ): + + return set( self._tags ) + + + +class _DummyFileInfoManager: + + def __init__( self, blurhash ): + + self.blurhash = blurhash + + + +class _DummyMediaResult: + + def __init__( self, blurhash ): + + self._file_info_manager = _DummyFileInfoManager( blurhash ) + + + def GetFileInfoManager( self ): + + return self._file_info_manager + + + +class _DummyMedia: + + def __init__( self, blurhash = None, tags = None ): + + self._media_result = _DummyMediaResult( blurhash ) + self._tags_manager = _DummyTagsManager( tags or set() ) + + + def GetDisplayMedia( self ): + + return self + + + def GetMediaResult( self ): + + return self._media_result + + + def GetTagsManager( self ): + + return self._tags_manager + + + +class TestClientMediaSeriation( unittest.TestCase ): + + def test_seriation_blurhash_greedy_order( self ): + + m1 = _DummyMedia( blurhash = 'bh1' ) + m2 = _DummyMedia( blurhash = 'bh2' ) + m3 = _DummyMedia( blurhash = 'bh3' ) + m4 = _DummyMedia( blurhash = None ) + + mapping = { + 'bh1' : numpy.array( [[[ 0, 0, 0 ]]], dtype = numpy.float32 ), + 'bh2' : numpy.array( [[[ 1, 1, 1 ]]], dtype = numpy.float32 ), + 'bh3' : numpy.array( [[[ 4, 4, 4 ]]], dtype = numpy.float32 ) + } + colour_mapping = { + 'bh1' : ( 0, 0, 0 ), + 'bh2' : ( 1, 1, 1 ), + 'bh3' : ( 4, 4, 4 ) + } + + def fake_blurhash( blurhash, width, height ): + + return mapping[ blurhash ] + + + def fake_average_colour( blurhash ): + + return colour_mapping[ blurhash ] + + + with mock.patch( 'hydrus.client.media.ClientMedia.HydrusBlurhash.GetNumpyFromBlurhash', side_effect = fake_blurhash ), mock.patch( 'hydrus.client.media.ClientMedia.HydrusBlurhash.GetAverageColourFromBlurhash', side_effect = fake_average_colour ): + + ordered = ClientMedia.GetSeriationOrderingForMediasByBlurhash( [ m1, m2, m3, m4 ], reverse = False ) + self.assertEqual( ordered, [ m2, m1, m3, m4 ] ) + + ordered_reverse = ClientMedia.GetSeriationOrderingForMediasByBlurhash( [ m1, m2, m3, m4 ], reverse = True ) + self.assertEqual( ordered_reverse, [ m4, m3, m1, m2 ] ) + + + + def test_seriation_tags_greedy_order( self ): + + m0 = _DummyMedia( tags = { 'a', 'b' } ) + m1 = _DummyMedia( tags = { 'a', 'b', 'c' } ) + m2 = _DummyMedia( tags = { 'c' } ) + m3 = _DummyMedia( tags = { 'd' } ) + m4 = _DummyMedia( tags = set() ) + + tag_context = ClientSearchTagContext.TagContext( service_key = CC.COMBINED_TAG_SERVICE_KEY ) + ordered = ClientMedia.GetSeriationOrderingForMediasByTags( [ m0, m1, m2, m3, m4 ], reverse = False, tag_context = tag_context ) + self.assertEqual( ordered, [ m1, m0, m2, m3, m4 ] ) + + ordered_reverse = ClientMedia.GetSeriationOrderingForMediasByTags( [ m0, m1, m2, m3, m4 ], reverse = True, tag_context = tag_context ) + self.assertEqual( ordered_reverse, [ m4, m3, m2, m0, m1 ] ) + + + def test_seriation_visual_and_tags_greedy_order( self ): + + m0 = _DummyMedia( blurhash = 'bh1', tags = { 'a' } ) + m1 = _DummyMedia( blurhash = 'bh2', tags = { 'a', 'b' } ) + m2 = _DummyMedia( blurhash = 'bh3', tags = { 'c' } ) + m3 = _DummyMedia( tags = { 'c' } ) + + mapping = { + 'bh1' : numpy.array( [[[ 0, 0, 0 ]]], dtype = numpy.float32 ), + 'bh2' : numpy.array( [[[ 1, 1, 1 ]]], dtype = numpy.float32 ), + 'bh3' : numpy.array( [[[ 4, 4, 4 ]]], dtype = numpy.float32 ) + } + colour_mapping = { + 'bh1' : ( 0, 0, 0 ), + 'bh2' : ( 1, 1, 1 ), + 'bh3' : ( 4, 4, 4 ) + } + + def fake_blurhash( blurhash, width, height ): + + return mapping[ blurhash ] + + + def fake_average_colour( blurhash ): + + return colour_mapping[ blurhash ] + + + tag_context = ClientSearchTagContext.TagContext( service_key = CC.COMBINED_TAG_SERVICE_KEY ) + + with mock.patch( 'hydrus.client.media.ClientMedia.HydrusBlurhash.GetNumpyFromBlurhash', side_effect = fake_blurhash ), mock.patch( 'hydrus.client.media.ClientMedia.HydrusBlurhash.GetAverageColourFromBlurhash', side_effect = fake_average_colour ): + + ordered = ClientMedia.GetSeriationOrderingForMediasByVisualAndTags( [ m0, m1, m2, m3 ], reverse = False, tag_context = tag_context ) + self.assertEqual( ordered, [ m1, m0, m2, m3 ] ) + + ordered_reverse = ClientMedia.GetSeriationOrderingForMediasByVisualAndTags( [ m0, m1, m2, m3 ], reverse = True, tag_context = tag_context ) + self.assertEqual( ordered_reverse, [ m3, m2, m0, m1 ] ) + + diff --git a/hydrus_client.sh b/hydrus_client.sh index 6df1bdfd5..153c86be4 100755 --- a/hydrus_client.sh +++ b/hydrus_client.sh @@ -1,5 +1,18 @@ #!/bin/bash +if [ "$(uname)" = "Darwin" ]; then + if sysctl -n hw.optional.arm64 >/dev/null 2>&1; then + if [ "$(sysctl -n hw.optional.arm64)" = "1" ] && [ "$(uname -m)" = "x86_64" ]; then + if command -v arch >/dev/null 2>&1; then + echo "Detected Rosetta shell on Apple Silicon. Relaunching under arm64..." + exec arch -arm64 /bin/bash "$0" "$@" + else + echo "Warning: running under Rosetta; performance may suffer." + fi + fi + fi +fi + pushd "$(dirname "$0")" || exit 1 if [ ! -d "venv" ]; then @@ -14,6 +27,13 @@ if ! source venv/bin/activate; then exit 1 fi +if [ "$(uname)" = "Darwin" ]; then + py_arch="$(python -c 'import platform; print(platform.machine())' 2>/dev/null)" + if [ "$py_arch" = "x86_64" ]; then + echo "Warning: venv python is x86_64; install an arm64 Python/venv for best performance." + fi +fi + # You can copy this file to 'hydrus_client-user.sh' and add in your own launch parameters here if you like. A git pull won't overwrite that filename. # Just tack new hardcoded params on like this: # diff --git a/static/default/parsers/tiktok file page parser.png b/static/default/parsers/tiktok file page parser.png new file mode 100644 index 000000000..e29c89e83 Binary files /dev/null and b/static/default/parsers/tiktok file page parser.png differ diff --git a/static/default/url_classes/tiktok file page.png b/static/default/url_classes/tiktok file page.png new file mode 100644 index 000000000..deafc57f0 Binary files /dev/null and b/static/default/url_classes/tiktok file page.png differ diff --git a/tests/test_media_controls_bar_layout.py b/tests/test_media_controls_bar_layout.py new file mode 100644 index 000000000..3f91a2bbc --- /dev/null +++ b/tests/test_media_controls_bar_layout.py @@ -0,0 +1,29 @@ +import unittest + +from hydrus.client.gui.canvas import ClientGUICanvasMediaLayout + + +class TestMediaControlsBarLayout(unittest.TestCase): + def test_reserved_space_reduces_height(self): + height = ClientGUICanvasMediaLayout.CalculateMediaHeightForControlsBar( + 600, 40, True, False + ) + self.assertEqual(height, 560) + + def test_hidden_controls_bar_does_not_reserve_space(self): + height = ClientGUICanvasMediaLayout.CalculateMediaHeightForControlsBar( + 600, 40, True, True + ) + self.assertEqual(height, 600) + + def test_no_reserved_space_keeps_height(self): + height = ClientGUICanvasMediaLayout.CalculateMediaHeightForControlsBar( + 600, 40, False, False + ) + self.assertEqual(height, 600) + + def test_reserved_space_has_minimum_height(self): + height = ClientGUICanvasMediaLayout.CalculateMediaHeightForControlsBar( + 10, 50, True, False + ) + self.assertEqual(height, 1)