improve handling of different languages (#112)

Signed-off-by: Matthew Peveler <matt.peveler@gmail.com>

Author: MasterOdin

README.md

@@ -27,7 +27,7 @@ A `Python` command line client for [tldr](https://github.com/tldr-pages/tldr).
 
 ## Configuration
 
-You can configure the behaviour and output of the `tldr` client by setting environment variables. For example, in the `.bashrc` file:
+You can configure the behavior and output of the `tldr` client by setting environment variables. For example, in the `.bashrc` file:
 
     export TLDR_COLOR_NAME="cyan"
     export TLDR_COLOR_DESCRIPTION="white"
@@ -38,6 +38,7 @@ You can configure the behaviour and output of the `tldr` client by setting envir
     export TLDR_CACHE_MAX_AGE=720
     export TLDR_PAGES_SOURCE_LOCATION="https://raw.githubusercontent.com/tldr-pages/tldr/master/pages"
     export TLDR_DOWNLOAD_CACHE_LOCATION="https://tldr-pages.github.io/assets/tldr.zip"
+    export TLDR_LANGUAGE="en"
 
 ### Cache
 
@@ -82,7 +83,18 @@ Any of the values of above may be omitted. For example, you can do similar thing
 * `TLDR_COLOR_PARAMETER="red on_yellow underline"` for underlined red text on yellow background
 * `TLDR_COLOR_NAME="bold underline"` for default system font and background colors with underline and bolded effects
 
-## Remote source
+### Language 
+
+The language that tldr will use is dependent on a number of factors. If you specify a language via the
+`--language` flag, tldr will attempt to use that language and only that language. Otherwise, it will
+default to language set using either `TLDR_LANGUAGE` before falling back to `LANG` (ignoring the value `C`).
+If neither are set, then tldr will always attempt to get the `en` page. Finally, if `LANGUAGES` is set, it uses
+this as the priority list to try languages in, with the exception that it will attempt `TLDR_LANGUAGE` and `LANG`
+first, and if neither are set, will use `en` last (assuming it does not already appear somewhere in `LANGUAGES`).
+All language values should be set to a value that follows [RFC 1766](https://tools.ietf.org/html/rfc1766.html),
+with the special exception of `C` which is ignored.
+
+### Remote source
 
 If you wish to use your own instance of the tldr pages instead of the default repository, you
 can either use the `--source` flag when using tldr or by specifying the following environment variables:
@@ -91,4 +103,4 @@ can either use the `--source` flag when using tldr or by specifying the followin
   * defaults to `https://raw.githubusercontent.com/tldr-pages/tldr/master/pages`
   * it can also point to local directory using `file:///path/to/directory`
 * `TLDR_DOWNLOAD_CACHE_LOCATION` to control where to pull a zip of all pages from
-  * defaults to `https://tldr-pages.github.io/assets/tldr.zip`
+  * defaults to `https://tldr-pages.github.io/assets/tldr.zip`

tests/test_tldr.py

@@ -1,5 +1,6 @@
 import sys
 import io
+import types
 import unittest
 import tldr
 
@@ -10,6 +11,8 @@ def test_whole_page(self):
             with open("tests/data/gem_rendered", "rb") as f_rendered:
                 old_stdout = sys.stdout
                 sys.stdout = io.StringIO()
+                sys.stdout.buffer = types.SimpleNamespace()
+                sys.stdout.buffer.write = lambda x: sys.stdout.write(x.decode('utf-8'))
                 tldr.output(f_original)
 
                 sys.stdout.seek(0)

tldr.py

@@ -3,7 +3,6 @@
 import sys
 import os
 import re
-import locale
 from argparse import ArgumentParser
 from zipfile import ZipFile
 from datetime import datetime
@@ -27,9 +26,18 @@
     'TLDR_DOWNLOAD_CACHE_LOCATION',
     'https://tldr-pages.github.io/assets/tldr.zip'
 )
-DEFAULT_LANG = os.environ.get('LANG', locale.getlocale()[0]).split("_")[0]
+
+DEFAULT_LANG = os.environ.get(
+    'TLDR_LANGUAGE',
+    os.environ.get('LANG', None)
+).split('_')[0]
+
+if DEFAULT_LANG == 'C':
+    DEFAULT_LANG = None
+
 USE_CACHE = int(os.environ.get('TLDR_CACHE_ENABLED', '1')) > 0
 MAX_CACHE_AGE = int(os.environ.get('TLDR_CACHE_MAX_AGE', 24))
+
 URLOPEN_CONTEXT = None
 if int(os.environ.get('TLDR_ALLOW_INSECURE', '0')) == 1:
     URLOPEN_CONTEXT = ssl.create_default_context()
@@ -148,20 +156,40 @@ def get_platform_list():
     return platforms
 
 
-def get_page(command, remote=None, platform=None, language=None):
-    if platform is None:
-        platform = get_platform_list()
-    for _platform in platform:
-        if _platform is None:
-            continue
+def get_language_list():
+    languages = os.environ.get('LANGUAGES', '').split(':')
+    languages = list(map(
+        lambda x: x.split('_')[0],
+        filter(lambda x: x != 'C', languages)
+    ))
+    if DEFAULT_LANG is not None:
         try:
-            return get_page_for_platform(command, _platform, remote, language)
-        except HTTPError as err:
-            if err.code != 404:
-                raise
-        except URLError:
-            if not PAGES_SOURCE_LOCATION.startswith('file://'):
-                raise
+            languages.remove(DEFAULT_LANG)
+        except ValueError:
+            pass
+        languages.insert(0, DEFAULT_LANG)
+    else:
+        languages.append('en')
+    return languages
+
+
+def get_page(command, remote=None, platforms=None, languages=None):
+    if platforms is None:
+        platforms = get_platform_list()
+    if languages is None:
+        languages = get_language_list()
+    for platform in platforms:
+        for language in languages:
+            if platform is None:
+                continue
+            try:
+                return get_page_for_platform(command, platform, remote, language)
+            except HTTPError as err:
+                if err.code != 404:
+                    raise
+            except URLError:
+                if not PAGES_SOURCE_LOCATION.startswith('file://'):
+                    raise
 
     return False
 
@@ -218,22 +246,22 @@ def output(page):
         for line in page:
             line = line.rstrip().decode('utf-8')
             if len(line) == 0:
-                pass
+                continue
             elif line[0] == '#':
                 line = ' ' * LEADING_SPACES_NUM + \
                     colored(line.replace('# ', ''), *colors_of('name')) + '\n'
-                print(line)
+                sys.stdout.buffer.write(line.encode('utf-8'))
             elif line[0] == '>':
                 line = ' ' * (LEADING_SPACES_NUM - 1) + \
                     colored(
                         line.replace('>', '').replace('<', ''),
                         *colors_of('description')
                     )
-                print(line)
+                sys.stdout.buffer.write(line.encode('utf-8'))
             elif line[0] == '-':
                 line = '\n' + ' ' * LEADING_SPACES_NUM + \
                     colored(line, *colors_of('example'))
-                print(line)
+                sys.stdout.buffer.write(line.encode('utf-8'))
             elif line[0] == '`':
                 line = line[1:-1]  # need to actually parse ``
                 elements = [' ' * 2 * LEADING_SPACES_NUM]
@@ -245,11 +273,14 @@ def output(page):
                     if not replaced:
                         item = colored(item, *colors_of('command'))
                     elements.append(item)
-                print(''.join(elements))
+                sys.stdout.buffer.write(''.join(elements).encode('utf-8'))
+            print()
         print()
 
 
 def update_cache(language=None):
+    if language is None:
+        language = DEFAULT_LANG if DEFAULT_LANG is not None else 'en'
     try:
         pages_dir = "pages"
         if language and language != 'en':
@@ -322,7 +353,8 @@ def main():
                         )
 
     parser.add_argument('-L', '--language',
-                        default=DEFAULT_LANG,
+                        nargs=1,
+                        default=None,
                         type=str,
                         help='Override the default language')
 
@@ -354,9 +386,9 @@ def main():
             command = '-'.join(rest.command)
             result = get_page(
                 command,
-                platform=options.platform,
-                remote=options.source,
-                language=options.language
+                options.source,
+                options.platform,
+                options.language
             )
             if not result:
                 print((