Minipedia/encoder.py at main · khorneflaeks/Minipedia · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
from bitstring import BitArray
import json
import requests
import re
import unidecode

charlist = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',' ','.',',','(',')'," "]

pages=[[],[],[]]

symboldict = {'@' : 'at','!' : '.','&' : 'and','=' : 'equals','+' : 'plus', '-' : 'minus', '0' : 'zero', '1' : 'one', '2' : 'two', '3' : 'three', '4' : 'four', '5' : 'five', '6' : 'six', '7' : 'seven', '8' : 'eight', '9' : 'nine'}
words = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"]

#Turns 8bit binary into 5bit
def bitify(inputs):
    binry = ""
    testval = 0
    for x in inputs.lower():
        match x:
            case " ":
                binry = binry + "00000"
            case ".":
                binry = binry + "11011"
            case ",":
                binry = binry + "11100"
            case "(":
                binry = binry + "11101"
            case ")":
                binry = binry + "11110"
            case "#":
                binry = binry + "11111"
            case _:
                binry = binry + "{0:b}".format(ord(x) - 96).zfill(5)
    return binry

#Writes input to file with filename
def writefile(binarys, filename):
    binary_file = open(filename+".5bf", 'ab')
    b = BitArray(bin=binarys)
    b.tofile(binary_file)
    binary_file.close()

def overwritefile(binarys, filename):
    binary_file = open(filename+".5bf", 'wb')
    b = BitArray(bin=binarys)
    b.tofile(binary_file)
    binary_file.close()

#Downloads webpage off wikipedia, set link for now
def wikifind(webname):
    x = requests.get('https://en.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=1&explaintext=1&titles=' + webname)
    wikitext = x.json()
    for k,item in wikitext["query"]["pages"].items():
        return item['extract']

#Grab all pages starting from forward. gaplimit must be a given as a string.
def wikigraball(forward, gaplimit):
    x = requests.get('https://en.wikipedia.org/w/api.php?action=query&format=json&prop=extracts&generator=allpages&exintro=1&explaintext=1&&gapcontinue='+forward+'&gapfilterredir=nonredirects&gaplimit='+gaplimit)
    wikitext = x.json()
    for k,item in wikitext["query"]["pages"].items():
        pages[0].append(item['title'])
        pages[1].append(item['extract'])
        pages[2].append(wikitext['continue']['continue'])
    return pages

def addtitle(title):
    text = bitify(remover(symboltoplaintext(accenttochar(title.lower()))) + "#" + remover(symboltoplaintext(accenttochar(capitals(wikifind(title))))) + "##")
    return text

def addmultititle(title,inputtext):
    text = bitify(remover(symboltoplaintext(accenttochar(title.lower()))) + "#" + remover(symboltoplaintext(accenttochar(capitals(inputtext)))) + "##")
    return text

#Removes all letters not part of the 5 bits
def remover(text):
    text = ''.join([i for i in text if i in charlist])
    return text

#Turns all characters into lowercase
def capitals(text1):
    return re.sub(r"([A-Z])", r"#\1", text1).lower()

#Takes a string with accented text in it and turns the accented characters into ascii text using unidecode
def accenttochar(text2):
    return unidecode.unidecode(text2)

def symboltoplaintext(text3):
    for word, initial in symboldict.items():
        text3 = text3.replace(word.lower(), initial)
    return text3