upstream-layout.diff

diff --git a/commands/preprocess.py b/commands/preprocess.py
index 35aa7c3..65ebf39 100644
--- a/commands/preprocess.py
+++ b/commands/preprocess.py
@@ -28,17 +28,75 @@ import urllib.parse
 from lxml import etree
 
 
+LOADER_ALIASES = (
+        'startup_scripts.js',
+        'site_scripts.js',
+        'site_modules.css',
+        'skin_scripts.js',
+        'ext.css')
+
+
 def rmtree_if_exists(dir):
     if os.path.isdir(dir):
         shutil.rmtree(dir)
 
 
+def copy_if_exists(src, dst):
+    if os.path.exists(src):
+        os.makedirs(os.path.dirname(dst), exist_ok=True)
+        shutil.copy(src, dst)
+        return True
+    return False
+
+
 def move_dir_contents_to_dir(srcdir, dstdir):
     for fn in os.listdir(srcdir):
         shutil.move(os.path.join(srcdir, fn),
                     os.path.join(dstdir, fn))
 
 
+def move_tree_contents_to_dir(srcdir, dstdir):
+    if not os.path.isdir(srcdir):
+        return
+    os.makedirs(dstdir, exist_ok=True)
+    for fn in os.listdir(srcdir):
+        src = os.path.join(srcdir, fn)
+        dst = os.path.join(dstdir, fn)
+        if os.path.isdir(src) and os.path.isdir(dst):
+            move_tree_contents_to_dir(src, dst)
+            shutil.rmtree(src)
+        else:
+            shutil.move(src, dst)
+
+
+def copy_tree_contents_to_dir(srcdir, dstdir):
+    if not os.path.isdir(srcdir):
+        return
+    os.makedirs(dstdir, exist_ok=True)
+    for fn in os.listdir(srcdir):
+        src = os.path.join(srcdir, fn)
+        dst = os.path.join(dstdir, fn)
+        if os.path.isdir(src):
+            if os.path.isdir(dst):
+                copy_tree_contents_to_dir(src, dst)
+            else:
+                shutil.copytree(src, dst)
+        else:
+            shutil.copy2(src, dst)
+
+
+def normalize_cppreference2_skin(data_path):
+    copy_tree_contents_to_dir(
+            os.path.join(data_path, 'skins', 'Cppreference2',
+                         'resources', 'images'),
+            os.path.join(data_path, 'skins', 'cppreference2', 'images'))
+    copy_if_exists(
+            os.path.join('skins', 'cppreference2', 'images',
+                         'navbar-inv-tab.png'),
+            os.path.join(data_path, 'skins', 'cppreference2', 'images',
+                         'navbar-inv-tab.png'))
+
+
 def rearrange_archive(root):
     # rearrange the archive. {root} here is output/reference
 
@@ -56,33 +114,72 @@ def rearrange_archive(root):
 
     data_path = os.path.join(root, 'common')
     rmtree_if_exists(data_path)
-    shutil.move(os.path.join(root, 'upload.cppreference.com/mwiki'), data_path)
-    shutil.rmtree(os.path.join(root, 'upload.cppreference.com'), ignore_errors=True)
+    os.makedirs(data_path)
+
+    upload_root = os.path.join(root, 'upload.cppreference.com')
+    upload_mwiki = os.path.join(upload_root, 'mwiki')
+    upload_images = os.path.join(upload_root, 'images')
+    if os.path.isdir(upload_mwiki):
+        move_dir_contents_to_dir(upload_mwiki, data_path)
+    if os.path.isdir(upload_images):
+        move_tree_contents_to_dir(upload_images,
+                                  os.path.join(data_path, 'images'))
+    shutil.rmtree(upload_root, ignore_errors=True)
 
     for lang in ["zh"]:
         path = os.path.join(root, lang + ".cppreference.com/")
-        src_html_path = path + "w/"
-        src_data_path = path + "mwiki/"
+        src_html_path = os.path.join(path, "w")
+        src_data_path = os.path.join(path, "mwiki")
         html_path = os.path.join(root, lang)
 
         if os.path.isdir(src_html_path):
             shutil.move(src_html_path, html_path)
+        elif os.path.isdir(path):
+            os.makedirs(html_path)
+            for fn in os.listdir(path):
+                if fn in ('extensions', 'mwiki', 'resources',
+                          'rest.php', 'skins'):
+                    continue
+                if fn.startswith('load.php') or fn in LOADER_ALIASES:
+                    continue
+                if fn.startswith('DejaVu') or fn == 'favicon.ico':
+                    continue
+                shutil.move(os.path.join(path, fn),
+                            os.path.join(html_path, fn))
 
         if os.path.isdir(src_data_path):
             # the skin files should be the same for all languages thus we
             # can merge everything
             move_dir_contents_to_dir(src_data_path, data_path)
 
-        # also copy the custom fonts
-        shutil.copy(os.path.join(path, 'DejaVuSansMonoCondensed60.ttf'),
-                    data_path)
-        shutil.copy(os.path.join(path, 'DejaVuSansMonoCondensed75.ttf'),
-                    data_path)
-        # and the favicon
-        shutil.copy(os.path.join(path, 'favicon.ico'), data_path)
+        if os.path.isdir(path):
+            for dirname in ('extensions', 'resources', 'rest.php', 'skins'):
+                move_tree_contents_to_dir(
+                    os.path.join(path, dirname),
+                    os.path.join(data_path, dirname))
+            for fn in fnmatch.filter(os.listdir(path), 'load.php*'):
+                shutil.move(os.path.join(path, fn),
+                            os.path.join(data_path, fn))
+            for fn in LOADER_ALIASES:
+                copy_if_exists(os.path.join(path, fn),
+                               os.path.join(data_path, fn))
+
+        normalize_cppreference2_skin(data_path)
+
+        for font in (
+                'DejaVuSans.ttf',
+                'DejaVuSans-Bold.ttf',
+                'DejaVuSansMono.ttf',
+                'DejaVuSansMono-Bold.ttf',
+                'DejaVuSansMonoCondensed60.ttf',
+                'DejaVuSansMonoCondensed75.ttf'):
+            copy_if_exists(os.path.join(path, font),
+                           os.path.join(data_path, font))
+        copy_if_exists(os.path.join(path, 'favicon.ico'),
+                       os.path.join(data_path, 'favicon.ico'))
 
         # remove what's left
-        # shutil.rmtree(path)
+        shutil.rmtree(path, ignore_errors=True)
 
     # remove the XML source file
     for fn in fnmatch.filter(os.listdir(root), 'cppreference-export*.xml'):
@@ -92,48 +189,98 @@ def rearrange_archive(root):
 def convert_loader_name(fn):
     # Converts complex URL to resources supplied by MediaWiki loader to a
     # simplified name
-    if "modules=site&only=scripts" in fn:
+    parsed = urllib.parse.urlparse(fn)
+    basename = os.path.basename(parsed.path)
+    query = parsed.query
+
+    if not query:
+        if '?' in basename:
+            basename, query = basename.split('?', 1)
+        elif '@' in basename:
+            basename, query = basename.split('@', 1)
+
+    if basename != 'load.php' or not query:
+        msg = 'Loader file {0} does not match any known files'.format(fn)
+        raise Exception(msg)
+
+    params = urllib.parse.parse_qs(query, keep_blank_values=True)
+    modules = urllib.parse.unquote(params.get('modules', ['loader'])[0])
+    lang = params.get('lang', ['common'])[0]
+    only = params.get('only', [''])[0]
+    ext = '.css' if only == 'styles' else '.js' if only == 'scripts' else ''
+
+    if modules == 'site' and only == 'scripts':
         return "site_scripts.js"
-    if "modules=site&only=styles" in fn:
+    if modules in ('site', 'site.styles') and only == 'styles':
         return "site_modules.css"
-    if "modules=startup&only=scripts" in fn:
+    if modules == 'startup' and only == 'scripts':
         return "startup_scripts.js"
-    if re.search("modules=skins.*&only=scripts", fn):
+    if modules == 'skins.cppreference2' and only == 'scripts':
         return "skin_scripts.js"
-    if re.search("modules=.*ext.*&only=styles", fn):
+    if modules in ('ext.pygments|skins.cppreference2',
+                   'ext.pygments|skins.cppreference2.styles') and \
+            only == 'styles':
         return "ext.css"
-    msg = 'Loader file {0} does not match any known files'.format(fn)
-    raise Exception(msg)
+
+    if 'lang' not in params:
+        if modules.startswith('skins.') and only == 'scripts':
+            return "skin_scripts.js"
+        if 'ext' in modules and only == 'styles':
+            return "ext.css"
+
+    def sanitize(value):
+        value = urllib.parse.unquote(value)
+        value = value.replace('|', '_')
+        value = value.replace('.styles', '')
+        value = value.replace('.scripts', '')
+        value = re.sub(r'[^0-9A-Za-z]+', '_', value)
+        value = re.sub(r'_+', '_', value).strip('_').lower()
+        return value or 'loader'
+
+    parts = ['loader', sanitize(lang), sanitize(modules)]
+    if only not in ('styles', 'scripts'):
+        parts.append(sanitize(only))
+    return '_'.join(parts) + ext
 
 
 def build_rename_map(root):
-    # Returns a rename map: a map from old to new file name
+    # Returns a rename map keyed by source path and, where unambiguous, source
+    # filename. Canonical names already present in the tree keep their plain
+    # target name and duplicate loader variants are suffixed.
     loader = re.compile(r'load\.php[\?@].*')
+    image_hash = re.compile(
+            r'(.+\.(?:png|jpe?g|gif|svg|webp|ico))(?:@|%40)[^/\\]+$',
+            re.IGNORECASE)
     query = re.compile(r'[\?@].*')
-    result = dict()
+    result = {}
 
-    # find files with invalid names -> rename all occurrences
-    for fn in set(fn for _, _, filenames in os.walk(root) for fn in filenames):
+    def candidate_name(fn):
         if loader.match(fn):
-            result[fn] = convert_loader_name(fn)
-
-        elif any((c in fn) for c in '?*"'):
+            return convert_loader_name(fn)
+        match = image_hash.match(fn)
+        if match:
+            return match.group(1)
+        if any((c in fn) for c in '?*"'):
             new_fn = query.sub('', fn)
             new_fn = new_fn.replace('"', '_q_')
             new_fn = new_fn.replace('*', '_star_')
-            result[fn] = new_fn
+            return new_fn
+        return fn
 
-    # find files that conflict on case-insensitive filesystems
     for dir, _, filenames in os.walk(root):
         seen = dict()
-        for fn in (result.get(s, s) for s in filenames):
-            low = fn.lower()
+        candidates = [(fn, candidate_name(fn)) for fn in filenames]
+        candidates.sort(key=lambda item: (0 if item[0] == item[1] else 1,
+                                          item[0].lower()))
+        for old_fn, new_fn in candidates:
+            low = new_fn.lower()
             num = seen.setdefault(low, 0)
             if num > 0:
-                name, ext = os.path.splitext(fn)
-                # add file with its path -> only rename that occurrence
-                result[os.path.join(dir, fn)] = \
-                    "{}.{}{}".format(name, num + 1, ext)
+                name, ext = os.path.splitext(new_fn)
+                new_fn = "{}.{}{}".format(name, num + 1, ext)
+            if new_fn != old_fn:
+                result[os.path.join(dir, old_fn)] = new_fn
+                result.setdefault(old_fn, new_fn)
             seen[low] += 1
 
     return result
@@ -144,15 +291,7 @@ def rename_files(root, rename_map):
                         for dir, _, filenames in os.walk(root)
                         for fn in filenames):
         src_path = os.path.join(dir, old_fn)
-
-        new_fn = rename_map.get(old_fn)
-        if new_fn:
-            # look for case conflict of the renamed file
-            new_path = os.path.join(dir, new_fn)
-            new_fn = rename_map.get(new_path, new_fn)
-        else:
-            # original filename unchanged, look for case conflict
-            new_fn = rename_map.get(src_path)
+        new_fn = rename_map.get(src_path, rename_map.get(old_fn))
         if new_fn:
             dst_path = os.path.join(dir, new_fn)
             print("Renaming {0}\n      to {1}".format(src_path, dst_path))
@@ -171,12 +310,24 @@ def find_html_files(root):
 def is_loader_link(target):
     if re.match(r'https?://[a-z]+\.cppreference\.com/mwiki/load\.php', target):
         return True
+    if re.match(r'https?://[a-z]+\.cppreference\.com/load\.php', target):
+        return True
+    basename = os.path.basename(urllib.parse.urlparse(target).path)
+    if basename == 'load.php' and ('?' in target or '@' in target):
+        return True
+    if basename.startswith('load.php@'):
+        return True
     return False
 
 
-def transform_loader_link(target, file, root):
-    # Absolute loader.php links need to be made relative
-    abstarget = os.path.join(root, "common", convert_loader_name(target))
+def transform_loader_link(target, file, root, rename_map=None):
+    # Loader assets are moved to common regardless of their original depth.
+    filename = convert_loader_name(target)
+    abstarget = os.path.join(root, "common", filename)
+    if rename_map:
+        filename = rename_map.get(
+                abstarget, rename_map.get(filename, filename))
+        abstarget = os.path.join(root, "common", filename)
     return os.path.relpath(abstarget, os.path.dirname(file))
 
 
@@ -212,26 +363,100 @@ def is_external_link(target):
     return url.scheme != '' or url.netloc != ''
 
 
-def trasform_relative_link(rename_map, target, file):
+def transform_external_asset_link(target, file, root):
+    parsed = urllib.parse.urlparse(target)
+    local_hosts = ('zh.cppreference.com', 'upload.cppreference.com',
+                   'cdn.jsdelivr.net')
+    if parsed.scheme in ('http', 'https') and \
+            parsed.netloc == 'upload.cppreference.com':
+        path = parsed.path.lstrip('/')
+        if path.startswith('mwiki/'):
+            path = path[len('mwiki/'):]
+        elif path.startswith('images/'):
+            pass
+        else:
+            path = os.path.join('images', path)
+        abstarget = os.path.join(root, 'common', path)
+        return os.path.relpath(abstarget, os.path.dirname(file))
+    if parsed.scheme in ('http', 'https') and parsed.netloc in local_hosts:
+        abstarget = os.path.join(root, parsed.netloc, parsed.path.lstrip('/'))
+        return os.path.relpath(abstarget, os.path.dirname(file))
+    return target
+
+
+def common_relative_path(root, file, *parts):
+    abstarget = os.path.join(root, 'common', *parts)
+    return os.path.relpath(abstarget, os.path.dirname(file))
+
+
+def trasform_relative_link(rename_map, target, file, root=None):
+    if root is None:
+        root = os.path.dirname(os.path.dirname(file))
+
     # urlparse returns (scheme, host, path, params, query, fragment)
     _, _, path, params, _, fragment = urllib.parse.urlparse(target)
     assert params == ''
 
+    raw_path = path
     path = urllib.parse.unquote(path)
-    path = path.replace('../../upload.cppreference.com/mwiki/', '../common/')
-    path = path.replace('../mwiki/', '../common/')
+    is_image = re.search(
+            r'\.(?:png|jpe?g|gif|svg|webp|ico)'
+            r'((?:@|%40)[^/\\?#]*)?$',
+            path, re.IGNORECASE) is not None
+    if is_image:
+        fragment = ''
+
+    def move_common_path(value):
+        match = re.match(
+                r"^(?:\.\./)+upload.cppreference.com/mwiki/(.+)$",
+                value)
+        if match:
+            return common_relative_path(root, file, match.group(1))
+        match = re.match(
+                r"^(?:\.\./)+upload.cppreference.com/images/(.+)$",
+                value)
+        if match:
+            return common_relative_path(root, file, "images", match.group(1))
+        match = re.match(r"^(?:\.\./)+mwiki/(.+)$", value)
+        if match:
+            return common_relative_path(root, file, match.group(1))
+        match = re.match(
+                r"^(?:\.\./)+(extensions|resources|rest.php|skins)/(.+)$",
+                value)
+        if match:
+            return common_relative_path(
+                    root, file, match.group(1), match.group(2))
+        return value
+
+    path = re.sub(r'(\.(?:png|jpe?g|gif|svg|webp|ico))'
+                  r'(?:@|%40)[^/\\?#]*$', r'\1',
+                  path, flags=re.IGNORECASE)
+    path = re.sub(r'(\.(?:png|jpe?g|gif|svg|webp|ico))#.*$',
+                  r'\1', path, flags=re.IGNORECASE)
+
+    if path.startswith('/common/'):
+        path = common_relative_path(root, file, path[len('/common/'):])
+    elif re.match(r'^(?:\.\./)+common/', path):
+        common_path = re.sub(r'^(?:\.\./)+common/', '', path)
+        path = common_relative_path(root, file, common_path)
+    elif re.search(r'(^|/)common/images/', path):
+        image_path = re.sub(r'^.*?(?:^|/)common/images/', '', path)
+        path = common_relative_path(root, file, 'images', image_path)
+
+    path = move_common_path(path)
+    raw_path = move_common_path(raw_path)
 
     dir, fn = os.path.split(path)
-    new_fn = rename_map.get(fn)
-    if new_fn:
-        # look for case conflict of the renamed file
-        abstarget = os.path.normpath(
-                os.path.join(os.path.dirname(file), dir, new_fn))
-        new_fn = rename_map.get(abstarget, new_fn)
-    else:
-        # original filename unchanged, look for case conflict
-        abstarget = os.path.normpath(os.path.join(os.path.dirname(file), path))
-        new_fn = rename_map.get(abstarget)
+    _, raw_fn = os.path.split(raw_path)
+    abstarget = os.path.normpath(os.path.join(os.path.dirname(file), path))
+    raw_abstarget = os.path.normpath(
+            os.path.join(os.path.dirname(file), raw_path))
+    new_fn = rename_map.get(abstarget)
+    new_fn = rename_map.get(raw_abstarget, new_fn)
+    new_fn = rename_map.get(fn, new_fn)
+    new_fn = rename_map.get(raw_fn, new_fn)
+    if new_fn == 'loader_zh_ext_pygments_skins_cppreference2.css':
+        new_fn = 'ext.css'
     if new_fn:
         path = os.path.join(dir, new_fn)
 
@@ -245,15 +470,18 @@ def trasform_relative_link(rename_map, target, file):
 # root is the path to the root of the archive.
 def transform_link(rename_map, target, file, root):
     if is_loader_link(target):
-        return transform_loader_link(target, file, root)
+        return transform_loader_link(target, file, root, rename_map)
 
     if is_ranges_placeholder(target):
         return transform_ranges_placeholder(target, file, root)
 
     if is_external_link(target):
+        transformed = transform_external_asset_link(target, file, root)
+        if transformed != target:
+            return trasform_relative_link(rename_map, transformed, file, root)
         return target
 
-    return trasform_relative_link(rename_map, target, file)
+    return trasform_relative_link(rename_map, target, file, root)
 
 
 def has_class(el, *classes_to_check):
@@ -314,14 +542,23 @@ def remove_see_also(html):
 def remove_google_analytics(html):
     for el in html.xpath('/html/body/script'):
         if el.get('src') is not None:
-            if 'google-analytics.com/ga.js' in el.get('src'):
+            if 'google-analytics.com/ga.js' in el.get('src') or \
+                    'googletagmanager.com' in el.get('src'):
                 el.getparent().remove(el)
         elif el.text is not None:
             if 'google-analytics.com/ga.js' in el.text or \
-                    'pageTracker' in el.text:
+                    'pageTracker' in el.text or \
+                    'googletagmanager.com' in el.text or \
+                    "gtag('config'" in el.text:
                 el.getparent().remove(el)
 
 
+def remove_cloudflare_insights(html):
+    for el in html.xpath('//script[@src]'):
+        if 'static.cloudflareinsights.com' in el.get('src'):
+            el.getparent().remove(el)
+
+
 # remove ads
 def remove_ads(html):
     # Carbon Ads
@@ -384,8 +621,7 @@ def remove_unused_external(html):
         if el.get('rel') in ('alternate', 'search', 'edit', 'EditURI'):
             el.getparent().remove(el)
         elif el.get('rel') == 'shortcut icon':
-            (head, tail) = os.path.split(el.get('href'))
-            el.set('href', os.path.join(head, 'common', tail))
+            el.set('href', '/common/favicon.ico')
 
 
 def preprocess_html_file(root, fn, rename_map):
@@ -396,6 +632,7 @@ def preprocess_html_file(root, fn, rename_map):
     remove_unused_external(html)
     remove_noprint(html, keep_footer=True)
     remove_google_analytics(html)
+    remove_cloudflare_insights(html)
     remove_ads(html)
     remove_fileinfo(html)
 
@@ -421,13 +658,39 @@ def preprocess_css_file(fn):
 
     # note that query string is not used in css files
 
+    text = text.replace('https://zh.cppreference.com/', '')
+    text = text.replace('https://upload.cppreference.com/mwiki/images/',
+                        'images/')
+    text = text.replace('https://upload.cppreference.com/images/',
+                        'images/')
+    text = text.replace('https://zh.cppreference.com/mwiki/skins/',
+                        'skins/')
+    text = text.replace('https://en.cppreference.com/mwiki/skins/',
+                        'skins/')
+    text = text.replace('https://upload.wikimedia.org/wikipedia/commons/'
+                        '5/5d/Checker-16x16.png',
+                        'skins/common/images/Checker-16x16.png')
     text = text.replace('../DejaVuSansMonoCondensed60.ttf',
                         'DejaVuSansMonoCondensed60.ttf')
     text = text.replace('../DejaVuSansMonoCondensed75.ttf',
                         'DejaVuSansMonoCondensed75.ttf')
 
+    text = text.replace('../upload.cppreference.com/mwiki/images/',
+                        'images/')
+    text = text.replace('../upload.cppreference.com/images/',
+                        'images/')
     text = text.replace('../../upload.cppreference.com/mwiki/images/',
                         'images/')
+    text = text.replace('../../upload.cppreference.com/images/',
+                        'images/')
+    text = text.replace('mwiki/skins/', 'skins/')
+    text = text.replace('Cppreference2/resources/images/',
+                        'cppreference2/images/')
+    text = re.sub(r'(\.(?:png|jpe?g|gif|svg|webp|ico))'
+                  r'(?:@|%40)[^"\'\s)]+',
+                  r'\1', text, flags=re.IGNORECASE)
+    text = re.sub(r'(\.(?:png|jpe?g|gif|svg|webp|ico))#[^"\'\s)]+',
+                  r'\1', text, flags=re.IGNORECASE)
 
     # QT Help viewer doesn't understand nth-child
     text = text.replace('nth-child(1)', 'first-child')
@@ -442,6 +705,9 @@ def preprocess_startup_script(fn):
         text = f.read()
 
     text = re.sub(r'document\.write\([^)]+\);', '', text)
+    text = re.sub(r'mw\.loader\.addSource\(\{.*?\}\);', '',
+                  text, flags=re.DOTALL)
+    text = re.sub(r'mw\.loader\.load\([^;]*\);', '', text)
 
     with open(fn, "w", encoding='utf-8') as f:
         f.write(text)
diff --git a/preprocess.py b/preprocess.py
index c2513aa..2f1c7e1 100755
--- a/preprocess.py
+++ b/preprocess.py
@@ -20,7 +20,6 @@
 import argparse
 import concurrent.futures
 import os
-import shutil
 
 from commands import preprocess
 
@@ -72,12 +71,16 @@ def main():
 
     # clean the css files
 
-    for fn in [os.path.join(root, 'common/site_modules.css'),
-               os.path.join(root, 'common/ext.css')]:
-        preprocess.preprocess_css_file(fn)
+    common = os.path.join(root, 'common')
+    for filename in os.listdir(common):
+        if filename.endswith('.css'):
+            preprocess.preprocess_css_file(os.path.join(common, filename))
 
-    preprocess.preprocess_startup_script(
-        os.path.join(root, 'common/startup_scripts.js'))
+    for filename in os.listdir(common):
+        if filename.startswith(('startup_scripts', 'loader_')) and \
+                filename.endswith('.js'):
+            preprocess.preprocess_startup_script(
+                    os.path.join(common, filename))
 
 
 if __name__ == "__main__":