Skip to content

Add and Enhance Finnish Language Support with cp1257 Charset and Improved Input Handling #86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Syed-Shahrukh-OSSRevival merged 11 commits into from
Aug 1, 2025
Merged

Add and Enhance Finnish Language Support with cp1257 Charset and Improved Input Handling #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
402d10b
Add Finnish language support and encoding data
Egor-OSSRevival Jul 21, 2025
b4fa0d0
Add Finnish language support to Makefile configurations
Egor-OSSRevival Jul 21, 2025
5f90ab7
Improve input handling in normalize.pl to skip malformed lines and co…
Egor-OSSRevival Jul 25, 2025
fd9acfc
Enhance Finnish language support by adding cp1257 charset handling an…
Egor-OSSRevival Jul 25, 2025
943abd6
Refactor header inclusions and formatting in internal.h, lang.c, loca…
Egor-OSSRevival Jul 28, 2025
6c7e496
Add Finnish language support to setup script and include sample text …
Egor-OSSRevival Jul 29, 2025
f539245
Update Finnish test files with new sample text for cp1257, iso88594, …
Egor-OSSRevival Jul 29, 2025
66123e2
Update Finnish test files with new sample text for cp1257, iso88594, …
Egor-OSSRevival Jul 29, 2025
1a963bc
Add Finnish encoding support to test-guess-short.expected
Egor-OSSRevival Jul 29, 2025
f110467
Fix encoding test files by removing extraneous characters and ensurin…
Egor-OSSRevival Jul 29, 2025
1d05dcc
Add Finnish encoding support to test-guess-stdin.msys2.expected
Egor-OSSRevival Jul 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DEVELOP.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ Specifically, for multibyte encodings:
existing languages in `data/*` and read `data/README`.
* `lib/internal.h`:
* Add new `ENCA_LANGUAGE_....`
* `src/lang.c`:
* `lib/lang.c`:
* Add a new `LANGUAGE_LIST[]` entry pointing to the `ENCA_LANGUAGE_....`


Expand Down
4 changes: 4 additions & 0 deletions data/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ noinst_HEADERS = \
croatian/croatian.h \
czech/czech.h \
estonian/estonian.h \
finnish/finnish.h \
hungarian/hungarian.h \
latvian/latvian.h \
lithuanian/lithuanian.h \
Expand Down Expand Up @@ -46,6 +47,7 @@ noinst_SCRPITS = \
croatian/doit.sh \
czech/doit.sh \
estonian/doit.sh \
finnish/doit.sh \
hungarian/doit.sh \
latvian/doit.sh \
lithuanian/doit.sh \
Expand All @@ -61,6 +63,7 @@ BASES = \
croatian/cp1250.base \
czech/iso88592.base \
estonian/iso88594.base \
finnish/iso88594.base \
hungarian/iso88592.base \
russian/koi8r.base \
latvian/cp1257.base \
Expand All @@ -76,6 +79,7 @@ RAWCOUNTS = \
croatian/rawcounts.cp1250 \
czech/rawcounts.iso88592 \
estonian/rawcounts.iso88594 \
finnish/rawcounts.iso88594 \
hungarian/rawcounts.iso88592 \
latvian/rawcounts.cp1257 \
lithuanian/rawcounts.cp1257 \
Expand Down
4 changes: 4 additions & 0 deletions data/Makefile.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ noinst_HEADERS = \
croatian/croatian.h \
czech/czech.h \
estonian/estonian.h \
finnish/finnish.h \
hungarian/hungarian.h \
latvian/latvian.h \
lithuanian/lithuanian.h \
Expand Down Expand Up @@ -396,6 +397,7 @@ noinst_SCRPITS = \
croatian/doit.sh \
czech/doit.sh \
estonian/doit.sh \
finnish/doit.sh \
hungarian/doit.sh \
latvian/doit.sh \
lithuanian/doit.sh \
Expand All @@ -411,6 +413,7 @@ BASES = \
croatian/cp1250.base \
czech/iso88592.base \
estonian/iso88594.base \
finnish/iso88594.base \
hungarian/iso88592.base \
russian/koi8r.base \
latvian/cp1257.base \
Expand All @@ -426,6 +429,7 @@ RAWCOUNTS = \
croatian/rawcounts.cp1250 \
czech/rawcounts.iso88592 \
estonian/rawcounts.iso88594 \
finnish/rawcounts.iso88594 \
hungarian/rawcounts.iso88592 \
latvian/rawcounts.cp1257 \
lithuanian/rawcounts.cp1257 \
Expand Down
2 changes: 2 additions & 0 deletions data/finnish/doit.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#! /bin/bash
../doit.sh iso88594 cp1257
132 changes: 132 additions & 0 deletions data/finnish/finnish.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/***** THIS IS A GENERATED FILE. DO NOT TOUCH! *****/
/* THIS IS A GENERATED TABLE, see data/basetoc.c. */
static const unsigned short int RAW_ISO88594[] = {
0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x08 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x10 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x18 */
0, 4, 21, 0, 0, 2, 1, 2, /* 0x20 */
24, 24, 0, 0, 326, 128, 0, 7, /* 0x28 */
122, 105, 76, 33, 28, 34, 25, 25, /* 0x30 */
28, 49, 39, 2, 0, 0, 0, 8, /* 0x38 */
0, 58, 21, 18, 14, 52, 14, 11, /* 0x40 */
69, 31, 46, 118, 61, 83, 42, 37, /* 0x48 */
74, 0, 39, 133, 96, 23, 66, 9, /* 0x50 */
1, 23, 1, 0, 0, 0, 0, 1, /* 0x58 */
0, 5399, 37, 32, 403, 3579, 32, 84, /* 0x60 */
765, 4735, 859, 2254, 2539, 1338, 3767, 2470, /* 0x68 */
768, 1, 1096, 3362, 4242, 2326, 1031, 11, /* 0x70 */
4, 800, 6, 0, 0, 0, 0, 0, /* 0x78 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x88 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x90 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x98 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xa0 */
0, 0, 0, 0, 0, 6, 0, 0, /* 0xa8 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xb0 */
0, 1, 0, 0, 0, 0, 0, 0, /* 0xb8 */
0, 0, 0, 0, 2, 0, 0, 0, /* 0xc0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xc8 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xd0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xd8 */
0, 0, 0, 0, 1808, 0, 0, 0, /* 0xe0 */
0, 1, 0, 0, 0, 0, 0, 0, /* 0xe8 */
0, 0, 0, 0, 0, 0, 255, 0, /* 0xf0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xf8 */
};

/* THIS IS A GENERATED TABLE, see data/basetoc.c. */
static const unsigned short int RAW_CP1257[] = {
0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x08 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x10 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x18 */
0, 4, 21, 0, 0, 2, 1, 2, /* 0x20 */
24, 24, 0, 0, 326, 128, 0, 7, /* 0x28 */
122, 105, 76, 33, 28, 34, 25, 25, /* 0x30 */
28, 49, 39, 2, 0, 0, 0, 8, /* 0x38 */
0, 58, 21, 18, 14, 52, 14, 11, /* 0x40 */
69, 31, 46, 118, 61, 83, 42, 37, /* 0x48 */
74, 0, 39, 133, 96, 23, 66, 9, /* 0x50 */
1, 23, 1, 0, 0, 0, 0, 1, /* 0x58 */
0, 5399, 37, 32, 403, 3579, 32, 84, /* 0x60 */
765, 4735, 859, 2254, 2539, 1338, 3767, 2470, /* 0x68 */
768, 1, 1096, 3362, 4242, 2326, 1031, 11, /* 0x70 */
4, 800, 6, 0, 0, 0, 0, 0, /* 0x78 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x88 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x90 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x98 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xa0 */
0, 0, 0, 0, 0, 6, 0, 0, /* 0xa8 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xb0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xb8 */
0, 0, 0, 0, 2, 0, 0, 0, /* 0xc0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xc8 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xd0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xd8 */
0, 0, 0, 0, 1808, 0, 0, 0, /* 0xe0 */
0, 1, 0, 0, 0, 0, 0, 0, /* 0xe8 */
1, 0, 0, 0, 0, 0, 255, 0, /* 0xf0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xf8 */
};

/* THIS IS A GENERATED TABLE, see data/totals.pl. */
static const unsigned short int SIGNIFICANT[] = {
0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x08 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x10 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x18 */
0, 8, 42, 0, 0, 4, 2, 4, /* 0x20 */
48, 48, 0, 0, 652, 256, 0, 14, /* 0x28 */
244, 210, 152, 66, 56, 68, 50, 50, /* 0x30 */
56, 98, 78, 4, 0, 0, 0, 16, /* 0x38 */
0, 116, 42, 36, 28, 104, 28, 22, /* 0x40 */
138, 62, 92, 236, 122, 166, 84, 74, /* 0x48 */
148, 0, 78, 266, 192, 46, 132, 18, /* 0x50 */
2, 46, 2, 0, 0, 0, 0, 2, /* 0x58 */
0, 10798, 74, 64, 806, 7158, 64, 168, /* 0x60 */
1530, 9470, 1718, 4508, 5078, 2676, 7534, 4940, /* 0x68 */
1536, 2, 2192, 6724, 8484, 4652, 2062, 22, /* 0x70 */
8, 1600, 12, 0, 0, 0, 0, 0, /* 0x78 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x88 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x90 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0x98 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xa0 */
0, 0, 0, 0, 0, 12, 0, 0, /* 0xa8 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xb0 */
0, 1, 0, 0, 0, 0, 0, 0, /* 0xb8 */
0, 0, 0, 0, 4, 0, 0, 0, /* 0xc0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xc8 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xd0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xd8 */
0, 0, 0, 0, 3616, 0, 0, 0, /* 0xe0 */
0, 2, 0, 0, 0, 0, 0, 0, /* 0xe8 */
1, 0, 0, 0, 0, 0, 510, 0, /* 0xf0 */
0, 0, 0, 0, 0, 0, 0, 0, /* 0xf8 */
};

/* THIS IS A GENERATED VALUE, see data/totals.pl */
#define WEIGHT_SUM 46267

/* THIS IS A GENERATED TABLE, see data/totals.pl */
static const char *const CHARSET_NAMES[] = {
"iso88594",
"cp1257",
};

/* THIS IS A GENERATED TABLE, see data/totals.pl */
static const unsigned short int *const CHARSET_WEIGHTS[] = {
RAW_ISO88594,
RAW_CP1257,
};

/* THIS IS A GENERATED VALUE, see data/totals.pl */
#define CHARSET_LETTERS NULL

/* THIS IS A GENERATED VALUE, see data/totals.pl */
#define CHARSET_PAIRS NULL

/* THIS IS A GENERATED VALUE, see data/totals.pl */
#define NCHARSETS 2
155 changes: 155 additions & 0 deletions data/finnish/iso88594.base
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
. 0
. 0
. 0
5999
! 4
" 21
# 0
$ 0
% 2
& 1
' 2
( 24
) 24
* 0
+ 0
, 326
- 128
. 558
/ 7
0 122
1 105
2 76
3 33
4 28
5 34
6 25
7 25
8 28
9 49
: 39
; 2
< 0
= 0
> 0
? 8
@ 0
A 58
B 21
C 18
D 14
E 52
F 14
G 11
H 69
I 31
J 46
K 118
L 61
M 83
N 42
O 37
P 74
Q 0
R 39
S 133
T 96
U 23
V 66
W 9
X 1
Y 23
Z 1
[ 0
\ 0
] 0
^ 0
_ 1
` 0
a 5399
b 37
c 32
d 403
e 3579
f 32
g 84
h 765
i 4735
j 859
k 2254
l 2539
m 1338
n 3767
o 2470
p 768
q 1
r 1096
s 3362
t 4242
u 2326
v 1031
w 11
x 4
y 800
z 6
{ 0
| 0
} 0
~ 0
. 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 6
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 1
� 0
� 0
� 0
� 0
� 2
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 1808
� 0
� 0
� 0
� 0
� 1
� 0
� 0
� 0
� 0
� 0
� 0
� 0
� 255
� 0
� 0
� 0
� 0
� 0
Loading