combine.py

from PIL import Image
import numpy as np
from glob import glob
import time
from os import path, remove
import rapidjson as json
from tqdm import tqdm
import re
import asyncio

async def getfps(extensions):
    fps = []
    for ext in extensions:
        fps += glob('Assets/**/**.' + ext, recursive=True)
    return fps

class reSub:
    
    def __init__(self, dict):
        self.dict = dict
        self.regex = re.compile("(%s)" % "|".join(map(re.escape, dict.keys())))
        
    async def replace(self, text):
        return self.regex.subn(lambda mo: self.dict[mo.string[mo.start():mo.end()]], text)

async def getGuid(fp):
    metaPath = fp + '.meta'
    with open(metaPath, 'r') as f:
        guid = f.read().split('\n')[1].split(':')[1].strip()
        f.close()
        return guid

async def getImage(fp):
    img = Image.open(fp)
    img = np.array(img)
    imgHash = hash(img.data.tobytes())
    
    return img, str(imgHash)

async def getImageReplacements():
    uniqueImages = {}
    guidSubs = {}

    deleteImages = []
    imageSet = set()

    i = 0

    fps = await getfps(['png'])

    for fp in tqdm(fps, total=len(fps)):
        img, imgHash = await getImage(fp)
        guid = await getGuid(fp)
        imageSet.add(imgHash)
        if imgHash not in uniqueImages:
            uniqueImages[imgHash] = {'path': fp, 'guid': guid, 'guids': [guid]}
        else:
            guidSubs[guid] = uniqueImages[imgHash]['guid']
            uniqueImages[imgHash]['guids'].append(guid)
            deleteImages.append(fp)

    return guidSubs, deleteImages

async def overwriteGUIDs(fp, reSubs):
    with open(fp, 'r') as f:
        data = f.read()
        f.close()
        
    replaced, count = await reSubs.replace(data)
    
    if count > 0:

        with open(fp, 'w') as f:
            f.write(replaced)
            
    return count

async def replaceGUIDs(guidSubs):
    
    fps = await getfps(['mat'])
    reSubs = reSub(guidSubs)
    
    proceses = []
        
    for fp in tqdm(fps):
        proceses.append(overwriteGUIDs(fp, reSubs))
           
    responses = [await p for p in tqdm(asyncio.as_completed(proceses), total=len(proceses))]
    replacements = sum(responses)
                
    return replacements

async def deleteImages(deleteImages):
    for fp in tqdm(deleteImages):
        metaPath = fp + '.meta'
        if path.exists(fp):
            remove(fp)
        if path.exists(metaPath):
            remove(metaPath)

async def main():
    
    start = time.time()
    
    print('Getting image replacements...')
    guidSubs, deleteFps = await getImageReplacements()
    
    print('Replacing GUIDs...')
    replacements = await replaceGUIDs(guidSubs)
    
    print(f'Replaced {replacements} GUIDs')
    print('Deleting unneeded images and metadata...')
    await deleteImages(deleteFps)
    
    
    
if __name__ == '__main__':
    asyncio.run(main())