rockbox/tools/genlang
Tomer Shalev 6d80565b1b RTL support in menus
git-svn-id: svn://svn.rockbox.org/rockbox/trunk@22945 a1c6a512-1295-4272-9138-f99709370657
2009-10-05 11:43:38 +00:00

759 lines
21 KiB
Perl
Executable file

#!/usr/bin/perl -s
# __________ __ ___.
# Open \______ \ ____ ____ | | _\_ |__ _______ ___
# Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
# Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
# Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
# \/ \/ \/ \/ \/
# $Id$
#
# Copyright (C) 2006 - 2008 by Daniel Stenberg
#
# See apps/language.c (TODO: Use common include for both)
# Cookie and binary version for the binary lang file
my $LANGUAGE_COOKIE = 0x1a;
my $LANGUAGE_VERSION = 0x05;
my $LANGUAGE_FLAG_RTL = 0x01;
# A note for future users and readers: The original v1 language system allowed
# the build to create and use a different language than english built-in. We
# removed that feature from our build-system, but the build scripts still had
# the ability. But, starting now, this ability is no longer provided since I
# figured it was boring and unnecessary to write support for now since we
# don't use it anymore.
my %user2num =
('core' => 1);
if(!$ARGV[0]) {
print <<MOO
Usage: genlang [options] <langv2 file>
-p=<prefix>
Make the tool create a [prefix].c and [prefix].h file.
-b=<outfile>
Make the tool create a binary language (.lng) file named [outfile].
The use of this option requires that you also use -e, -t and -i.
-u
Update language file. Given the translated file and the most recent english
file, you\'ll get an updated version sent to stdout. Suitable action to do
when you intend to update a translation.
-e=<english lang file>
Point out the english (original source) file, to use that as master
language template. Used in combination with -b, -u or -s.
-s
Sort the Update language file in the same order as the strings in the
English file.
-t=<target>
Specify which target you want the translations/phrases for. Required when
-b or -p is used.
The target can in fact be specified as numerous different strings,
separated with colons. This will make genlang to use all the specified
strings when searching for a matching phrase.
-i=<target id>
The target id number, needed for -b.
-o
Voice mode output. Outputs all id: and voice: lines for the given target!
-v
Enables verbose (debug) output.
MOO
;
exit;
}
# How update works:
#
# 1) scan the english file, keep the whole <phrase> for each phrase.
# 2) read the translated file, for each end of phrase, compare:
# A) all source strings, if there's any change there should be a comment about
# it output
# B) the desc fields
#
# 3) output the phrase with the comments from above
# 4) check which phrases that the translated version didn't have, and spit out
# the english version of those
#
my $prefix = $p;
my $binary = $b;
my $update = $u;
my $sortfile = $s;
my $english = $e;
my $voiceout = $o;
my $check = ($binary?1:0) + ($prefix?1:0) + ($update?1:0) + ($voiceout?1:0) + ($sortfile?1:0);
if($check > 1) {
print STDERR "Please use only one of -p, -u, -o, -b and -s\n";
exit;
}
if(!$check) {
print STDERR "Please use at least one of -p, -u, -o, -b and -s\n";
exit;
}
if(($binary || $update || $voiceout || $sortfile) && !$english) {
print STDERR "Please use -e too when you use -b, -o, -u or -s\n";
exit;
}
my $target_id = $i;
if($binary && !$target_id) {
print STDERR "Please specify a target id number (with -i)!\n";
exit;
}
my $target = $t;
if(!$target && !$update && !$sortfile) {
print STDERR "Please specify a target (with -t)!\n";
exit;
}
my $verbose=$v;
my %id; # string to num hash
my @idnum; # num to string array
my %allphrases; # For sorting - an array of the <phrase> elements
my %source; # id string to source phrase hash
my %dest; # id string to dest phrase hash
my %voice; # id string to voice phrase hash
my $input = $ARGV[0];
my @m;
my $m="blank";
sub trim {
my ($string) = @_;
$string =~ s/^\s+//;
$string =~ s/\s+$//;
return $string;
}
sub match {
my ($string, $pattern)=@_;
$pattern =~ s/\*/.*/g;
$pattern =~ s/\?/./g;
return ($string =~ /^$pattern\z/);
}
sub blank {
# nothing to do
}
my %head;
sub header {
my ($full, $n, $v)=@_;
$head{$n}=$v;
}
my %phrase;
sub phrase {
my ($full, $n, $v)=@_;
$phrase{$n}=$v;
}
my %options;
sub options {
my ($full, $n, $v)=@_;
$options{$n}=$v;
}
sub parsetarget {
my ($debug, $strref, $full, $n, $v)=@_;
my $string;
my @all= split(" *, *", $n);
my $test;
for $test (@all) {
# print "TEST ($debug) $target for $test\n";
for my $part (split(":", $target)) {
if(match($part, $test)) {
$string = $v;
# print "MATCH: $test => $v\n";
$$strref = $string;
return $string;
}
}
}
}
my $src;
sub source {
parsetarget("src", \$src, @_);
}
my $dest;
sub dest {
parsetarget("dest", \$dest, @_);
}
my $voice;
sub voice {
parsetarget("voice", \$voice, @_);
}
my %idmap;
my %english;
if($english) {
# For the cases where the english file needs to be scanned/read, we do
# it before we read the translated file. For -b it isn't necessary, but for
# -u it is convenient.
my $idnum=0; # start with a true number
my $vidnum=0x8000; # first voice id
open(ENG, "<$english") || die "Error: can't open $english";
my @phrase;
my $id;
my $maybeid;
my $user;
my $maybeuser;
my $withindest;
my $numphrases = 0;
while(<ENG>) {
# get rid of DOS newlines
$_ =~ s/\r//g;
if($_ =~ /^ *\<phrase\>/) {
# this is the start of a phrase
}
elsif($_ =~ /^ *\<\/phrase\>/) {
# if id is something, when we count and store this phrase
if($id) {
# voice-only entries get a difference range
if($id =~ /^VOICE_/) {
# Assign an ID number to this entry
$idmap{$id}=$vidnum;
$vidnum++;
}
else {
# Assign an ID number to this entry
$idmap{$id}=$idnum;
$idnum++;
# print STDERR "DEST: bumped idnum to $idnum\n";
}
# this is the end of a phrase, add it to the english hash
$english{$id}=join("", @phrase);
}
undef @phrase;
$id="";
}
elsif($_ ne "\n") {
# gather everything related to this phrase
push @phrase, $_;
if($_ =~ /^ *\<dest\>/i) {
$withindest=1;
$deststr="";
}
elsif($withindest && ($_ =~ /^ *\<\/dest\>/i)) {
$withindest=0;
if($update || ($deststr && ($deststr !~ /^none\z/i))) {
# we unconditionally always use all IDs when the "update"
# feature is used
$id = $maybeid;
$user = $user2num{$maybeuser};
# print "DEST: use this id $id\n";
}
else {
# print "skip $maybeid for $name\n";
}
}
elsif($withindest && ($_ =~ / *([^:]+): *(.*)/)) {
my ($name, $val)=($1, $2);
$dest=""; # in case it is left untouched for when the
# model name isn't "our"
dest($_, $name, $val);
if($dest) {
# Store the current dest string. If this target matches
# multiple strings, it will get updated several times.
$deststr = $dest;
}
}
}
if($_ =~ /^ *id: ([^ \t\n]+)/i) {
$maybeid=$1;
$sortorder{$maybeid}=$numphrases++;
}
if($_ =~ /^ *user: ([^ \t\n]+)/i) {
$maybeuser=$1;
}
}
close(ENG);
}
# a function that compares the english phrase with the translated one.
# compare source strings and desc
# Then output the updated version!
sub compare {
my ($idstr, $engref, $locref)=@_;
my ($edesc, $ldesc);
my ($esource, $lsource);
my $mode=0;
for my $l (@$engref) {
if($l =~ /^ *#/) {
# comment
next;
}
if($l =~ /^ *desc: (.*)/) {
$edesc=$1;
}
elsif($l =~ / *\<source\>/i) {
$mode=1;
}
elsif($mode) {
if($l =~ / *\<\/source\>/i) {
last;
}
$esource .= "$l\n";
}
}
my @show;
my @source;
$mode = 0;
for my $l (@$locref) {
if($l =~ /^ *desc: (.*)/) {
$ldesc=$1;
if(trim($edesc) ne trim($ldesc)) {
$l = "### The 'desc' field differs from the english!\n### the previously used desc is commented below:\n### desc: $ldesc\n desc: $edesc\n";
}
push @show, $l;
}
elsif($l =~ / *\<source\>/i) {
$mode=1;
push @show, $l;
}
elsif($mode) {
if($l =~ / *\<\/source\>/i) {
$mode = 0;
print @show;
if(trim($esource) ne trim($lsource)) {
print "### The <source> section differs from the english!\n",
"### the previously used one is commented below:\n";
for(split("\n", $lsource)) {
print "### $_\n";
}
print $esource;
}
else {
print $lsource;
}
undef @show; # start over
push @show, $l;
}
else {
$lsource .= "$l";
}
}
else {
push @show, $l;
}
}
print @show;
}
my $idcount; # counter for lang ID numbers
my $voiceid=0x8000; # counter for voice-only ID numbers
#
# Now start the scanning of the selected language string
#
open(LANG, "<$input") || die "Error: couldn't read language file named $input\n";
my @phrase;
my $header = 1;
my $langoptions = 0;
while(<LANG>) {
$line++;
# get rid of DOS newlines
$_ =~ s/\r//g;
if($_ =~ /^( *\#|[ \t\n\r]*\z)/) {
# comment or empty line - output it if it's part of the header
if ($header and ($update || $sortfile)) {
print($_);
}
next;
}
$header = 0;
my $ll = $_;
# print "M: $m\n";
push @phrase, $ll;
# this is an XML-lookalike tag
if (/^(<|[^\"<]+<)([^>]*)>/) {
my $part = $2;
# print "P: $part\n";
if($part =~ /^\//) {
# this was a closing tag
if($part eq "/phrase") {
# closing the phrase
my $idstr = $phrase{'id'};
my $idnum;
if($binary && !$english{$idstr}) {
# $idstr doesn't exist for english, skip it\n";
}
elsif($dest =~ /^none\z/i) {
# "none" as dest (without quotes) means that this entire
# phrase is to be ignored
}
elsif($sortfile) {
$allphrases{$idstr}=join('',@phrase);
}
elsif(!$update) {
# we don't do the fully detailed analysis when we "update"
# since we don't do it for a particular target etc
# allow the keyword 'deprecated' to be used on dest and
# voice strings to mark that as deprecated. It will then
# be replaced with "".
$dest =~ s/^deprecate(|d)\z/\"\"/i;
$voice =~ s/^deprecate(|d)\z/\"\"/i;
# basic syntax error alerts, if there are no quotes we
# will assume an empty string was intended
if($dest !~ /^\"/) {
print STDERR "$input:$line:1: warning: dest before line lacks quotes ($dest)!\n";
$dest='""';
}
if($src !~ /^\"/) {
print STDERR "$input:$line:1: warning: source before line lacks quotes ($src)!\n";
$src='""';
}
if($voice !~ /^\"/ and $voice !~ /^none\z/i) {
print STDERR "$input:$line:1: warning: voice before line lacks quotes ($voice)!\n";
$voice='""';
}
if($dest eq '""' && $phrase{'desc'} !~ /deprecated/i && $idstr !~ /^VOICE/) {
print STDERR "$input:$line:1: warning: empty dest before line in non-deprecated phrase!\n";
}
my $user = $user2num{trim($phrase{'user'})};
if(!$user) {
print STDERR "$input:$line:1: warning: unknown user!\n";
$user = 1;
}
# Use the ID name to figure out which id number range we
# should use for this phrase. Voice-only strings are
# separated.
if($idstr =~ /^VOICE/) {
$idnum = $voiceid++;
}
else {
$idnum = $idcount++;
}
$id{$idstr} = $idnum;
$idnum[$idnum]=$idstr;
$source{$idstr}=$src;
$dest{$idstr}=$dest;
$voice{$idstr}=$voice;
if($verbose) {
print "id: $phrase{id} ($idnum)\n";
print "source: $src\n";
print "dest: $dest\n";
print "voice: $voice\n";
print "user: $user\n";
}
undef $src;
undef $dest;
undef $voice;
undef $user;
undef %phrase;
}
if($update) {
my $e = $english{$idstr};
if($e) {
# compare original english with this!
my @eng = split("\n", $english{$idstr});
compare($idstr, \@eng, \@phrase);
$english{$idstr}=""; # clear it
}
else {
print "### $idstr: The phrase is not used. Skipped\n";
}
}
} # end of </phrase>
elsif($part eq "/options") {
# closing the options
if ($options{'rtl'}) {
$langoptions |= $LANGUAGE_FLAG_RTL;
}
} # end of </options>
undef @phrase;
# starts with a slash, this _ends_ this section
$m = pop @m; # get back old value, the previous level's tag
next;
} # end of tag close
# This is an opening (sub) tag
push @m, $m; # store old value
$m = $part;
next;
}
if(/^ *([^:]+): *(.*)/) {
my ($name, $val)=($1, $2);
&$m($_, $name, $val);
}
}
close(LANG);
if($update) {
my $any=0;
for(keys %english) {
if($english{$_}) {
print "###\n",
"### This phrase below was not present in the translated file\n",
"<phrase>\n";
print $english{$_};
print "</phrase>\n";
}
}
}
if ($sortfile) {
for(sort { $sortorder{$a} <=> $sortorder{$b} } keys %allphrases) {
print $allphrases{$_};
}
}
if($prefix) {
# We create a .c and .h file
open(HFILE, ">$prefix/lang.h") ||
die "Error: couldn't create file $prefix/lang.h\n";
open(CFILE, ">$prefix/lang_core.c") ||
die "Error: couldn't create file $prefix/lang_core.c\n";
# get header file name
$headername = "$prefix/lang.h";
$headername =~ s/(.*\/)*//;
print HFILE <<MOO
/* This file was automatically generated using genlang */
/*
* The str() macro/functions is how to access strings that might be
* translated. Use it like str(MACRO) and expect a string to be
* returned!
*/
#define str(x) language_strings[x]
/* this is the array for holding the string pointers.
It will be initialized at runtime. */
extern unsigned char *language_strings[];
/* this contains the concatenation of all strings, separated by \\0 chars */
extern const unsigned char language_builtin[];
/* The enum below contains all available strings */
enum \{
MOO
;
print CFILE <<MOO
/* This file was automaticly generated using genlang, the strings come
from "$input" */
#include "$headername"
unsigned char *language_strings[LANG_LAST_INDEX_IN_ARRAY];
const unsigned char language_builtin[] =
MOO
;
# Output the ID names for the enum in the header file
my $i;
for $i (1 .. $idcount) {
my $name=$idnum[$i - 1]; # get the ID name
$name =~ s/\"//g; # cut off the quotes
printf HFILE (" %s, /* %d */\n", $name, $i-1);
}
# Output separation marker for last string ID and the upcoming voice IDs
print HFILE <<MOO
LANG_LAST_INDEX_IN_ARRAY, /* this is not a string, this is a marker */
/* --- below this follows voice-only strings --- */
VOICEONLY_DELIMITER = 0x8000,
MOO
;
# Output the ID names for the enum in the header file
for $i (0x8000 .. ($voiceid-1)) {
my $name=$idnum[$i]; # get the ID name
$name =~ s/\"//g; # cut off the quotes
printf HFILE (" %s,\n", $name);
}
# Output end of enum
print HFILE "\n};\n/* end of generated enum list */\n";
# Output the target phrases for the source file
for $i (1 .. $idcount) {
my $name=$idnum[$i - 1]; # get the ID
my $dest = $dest{$name}; # get the destination phrase
$dest =~ s:\"$:\\0\":; # insert a \0 before the second quote
if(!$dest) {
# this is just to be on the safe side
$dest = '"\0"';
}
printf CFILE (" %s\n", $dest);
}
# Output end of string chunk
print CFILE <<MOO
;
/* end of generated string list */
MOO
;
close(HFILE);
close(CFILE);
} # end of the c/h file generation
elsif($binary) {
# Creation of a binary lang file was requested
# We must first scan the english file to get the correct order of the id
# numbers used there, as that is what sets the id order for all language
# files. The english file is scanned before the translated file was
# scanned.
open(OUTF, ">$binary") or die "Error: Can't create $binary";
binmode OUTF;
printf OUTF ("%c%c%c%c", $LANGUAGE_COOKIE, $LANGUAGE_VERSION, $target_id,
$langoptions); # magic lang file header
# loop over the target phrases
for $i (1 .. $idcount) {
my $name=$idnum[$i - 1]; # get the ID
my $dest = $dest{$name}; # get the destination phrase
if($dest) {
$dest =~ s/^\"(.*)\"\s*$/$1/g; # cut off quotes
# Now, make sure we get the number from the english sort order:
$idnum = $idmap{$name};
printf OUTF ("%c%c%s\x00", ($idnum>>8), ($idnum&0xff), $dest);
}
}
}
elsif($voiceout) {
# voice output requested, display id: and voice: strings in a v1-like
# fashion
my @engl;
# This loops over the strings in the translated language file order
my @ids = ((0 .. ($idcount-1)));
push @ids, (0x8000 .. ($voiceid-1));
#for my $id (@ids) {
# print "$id\n";
#}
for $i (@ids) {
my $name=$idnum[$i]; # get the ID
my $dest = $voice{$name}; # get the destination voice string
if($dest) {
$dest =~ s/^\"(.*)\"\s*$/$1/g; # cut off quotes
# Now, make sure we get the number from the english sort order:
$idnum = $idmap{$name};
if(length($idnum)) {
$engl[$idnum] = $i;
#print "Input index $i output index $idnum\n";
}
else {
# not used, mark it so
$engl[$i] = -1
}
}
}
for my $i (@ids) {
my $o = $engl[$i];
if(($o < 0) || !length($o)) {
print "#$i\nid: NOT_USED_$i\nvoice: \"\"\n";
next;
}
my $name=$idnum[$o]; # get the ID
my $dest = $voice{$name}; # get the destination voice string
print "#$i ($o)\nid: $name\nvoice: $dest\n";
}
}
if($verbose) {
printf("%d ID strings scanned\n", $idcount);
print "* head *\n";
for(keys %head) {
printf "$_: %s\n", $head{$_};
}
}