From 02103a2fa701954e42c8081fccf75eea26f52ce8 Mon Sep 17 00:00:00 2001 From: Dominik Riebeling Date: Sat, 2 Aug 2008 20:39:03 +0000 Subject: [PATCH] Unify opening of utf-8 files (FS#6203). This also adds ignoring the BOM in several places it has been missing (as FS#6071). git-svn-id: svn://svn.rockbox.org/rockbox/trunk@18185 a1c6a512-1295-4272-9138-f99709370657 --- apps/cuesheet.c | 2 +- apps/gui/wps_parser.c | 18 +--------------- apps/misc.c | 31 ++++++++++++++++++++++++++- apps/misc.h | 6 ++++++ apps/playlist.c | 46 ++++++++++++---------------------------- apps/recorder/keyboard.c | 2 +- apps/recorder/radio.c | 2 +- apps/settings.c | 2 +- 8 files changed, 55 insertions(+), 54 deletions(-) diff --git a/apps/cuesheet.c b/apps/cuesheet.c index 853a2a0f95..5583264ab9 100644 --- a/apps/cuesheet.c +++ b/apps/cuesheet.c @@ -147,7 +147,7 @@ bool parse_cuesheet(char *file, struct cuesheet *cue) char *s; DEBUGF("cue parse\n"); - int fd = open(file,O_RDONLY); + int fd = open_utf8(file,O_RDONLY); if (fd < 0) { /* couln't open the file */ diff --git a/apps/gui/wps_parser.c b/apps/gui/wps_parser.c index 88a237cb0f..714c419af1 100644 --- a/apps/gui/wps_parser.c +++ b/apps/gui/wps_parser.c @@ -1527,19 +1527,6 @@ static bool load_wps_bitmaps(struct wps_data *wps_data, char *bmpdir) #endif /* HAVE_LCD_BITMAP */ -/* Skip leading UTF-8 BOM, if present. */ -static char *skip_utf8_bom(char *buf) -{ - unsigned char *s = (unsigned char *)buf; - - if(s[0] == 0xef && s[1] == 0xbb && s[2] == 0xbf) - { - buf += 3; - } - - return buf; -} - /* to setup up the wps-data from a format-buffer (isfile = false) from a (wps-)file (isfile = true)*/ bool wps_data_load(struct wps_data *wps_data, @@ -1604,7 +1591,7 @@ bool wps_data_load(struct wps_data *wps_data, #endif #endif /* __PCTOOL__ */ - int fd = open(buf, O_RDONLY); + int fd = open_utf8(buf, O_RDONLY); if (fd < 0) return false; @@ -1639,9 +1626,6 @@ bool wps_data_load(struct wps_data *wps_data, memset(bmp_names, 0, sizeof(bmp_names)); #endif - /* Skip leading UTF-8 BOM, if present. */ - wps_buffer = skip_utf8_bom(wps_buffer); - /* parse the WPS source */ if (!wps_parse(wps_data, wps_buffer)) { wps_reset(wps_data); diff --git a/apps/misc.c b/apps/misc.c index 60955b9820..f1f5c4aa12 100644 --- a/apps/misc.c +++ b/apps/misc.c @@ -1154,7 +1154,7 @@ char *strip_extension(char* buffer, int buffer_size, const char *filename) { char *dot = strrchr(filename, '.'); int len; - + if (buffer_size <= 0) { return NULL; @@ -1180,6 +1180,35 @@ char *strip_extension(char* buffer, int buffer_size, const char *filename) } #endif /* !defined(__PCTOOL__) */ +/** Open a UTF-8 file and set file descriptor to first byte after BOM. + * If no BOM is present this behaves like open(). + * If the file is opened for writing and O_TRUNC is set, write a BOM to + * the opened file and leave the file pointer set after the BOM. + */ +int open_utf8(const char* pathname, int flags) +{ + int fd; + unsigned char bom[BOM_SIZE]; + + fd = open(pathname, flags); + if(fd < 0) + return fd; + + if(flags & (O_TRUNC | O_WRONLY)) + { + write(fd, BOM, BOM_SIZE); + } + else + { + read(fd, bom, BOM_SIZE); + /* check for BOM */ + if(memcmp(bom, BOM, BOM_SIZE)) + lseek(fd, 0, SEEK_SET); + } + return fd; +} + + #ifdef HAVE_LCD_COLOR /* * Helper function to convert a string of 6 hex digits to a native colour diff --git a/apps/misc.h b/apps/misc.h index 160d74473f..b5547ec38f 100644 --- a/apps/misc.h +++ b/apps/misc.h @@ -23,6 +23,10 @@ #include #include + +#define BOM "\xef\xbb\xbf" +#define BOM_SIZE 3 + /* Format a large-range value for output, using the appropriate unit so that * the displayed value is in the range 1 <= display < 1000 (1024 for "binary" * units) if possible, and 3 significant digits are shown. If a buffer is @@ -106,6 +110,8 @@ extern int show_logo(void); int get_replaygain_mode(bool have_track_gain, bool have_album_gain); #endif +int open_utf8(const char* pathname, int flags); + #ifdef BOOTFILE #if !defined(USB_NONE) && !defined(USB_IPODSTYLE) void check_bootfile(bool do_rolo); diff --git a/apps/playlist.c b/apps/playlist.c index f6d536fa25..b42ecb5681 100644 --- a/apps/playlist.c +++ b/apps/playlist.c @@ -200,9 +200,6 @@ static long playlist_stack[(DEFAULT_STACK_SIZE + 0x800)/sizeof(long)]; static const char playlist_thread_name[] = "playlist cachectrl"; #endif -#define BOM "\xef\xbb\xbf" -#define BOM_SIZE 3 - /* Check if the filename suggests M3U or M3U8 format. */ static bool is_m3u8(const char* filename) { @@ -212,11 +209,6 @@ static bool is_m3u8(const char* filename) return !(len > 4 && strcasecmp(&filename[len - 4], ".m3u") == 0); } -/* Check if a strings starts with an UTF-8 byte-order mark. */ -static bool is_utf8_bom(const char* str, int len) -{ - return len >= BOM_SIZE && memcmp(str, BOM, BOM_SIZE) == 0; -} /* Convert a filename in an M3U playlist to UTF-8. * @@ -538,9 +530,11 @@ static int add_indices_to_playlist(struct playlist_info* playlist, int result = 0; if(-1 == playlist->fd) - playlist->fd = open(playlist->filename, O_RDONLY); + playlist->fd = open_utf8(playlist->filename, O_RDONLY); if(playlist->fd < 0) return -1; /* failure */ + if(lseek(playlist->fd, 0, SEEK_CUR) > 0) + playlist->utf8 = true; /* Override any earlier indication. */ gui_syncsplash(0, ID2P(LANG_WAIT)); @@ -568,14 +562,6 @@ static int add_indices_to_playlist(struct playlist_info* playlist, p = (unsigned char *)buffer; - /* utf8 BOM at beginning of file? */ - if(i == 0 && is_utf8_bom(p, nread)) { - nread -= BOM_SIZE; - p += BOM_SIZE; - i += BOM_SIZE; - playlist->utf8 = true; /* Override any earlier indication. */ - } - for(count=0; count < nread; count++,p++) { /* Are we on a new line? */ @@ -2972,7 +2958,7 @@ int playlist_insert_playlist(struct playlist_info* playlist, const char *filenam return -1; } - fd = open(filename, O_RDONLY); + fd = open_utf8(filename, O_RDONLY); if (fd < 0) { gui_syncsplash(HZ*2, ID2P(LANG_PLAYLIST_ACCESS_ERROR)); @@ -3010,12 +2996,6 @@ int playlist_insert_playlist(struct playlist_info* playlist, const char *filenam if (action_userabort(TIMEOUT_NOBLOCK)) break; - if (count == 0 && is_utf8_bom(temp_buf, max)) - { - max -= BOM_SIZE; - memmove(temp_buf, temp_buf + BOM_SIZE, max); - } - if (temp_buf[0] != '#' && temp_buf[0] != '\0') { int insert_pos; @@ -3413,7 +3393,15 @@ int playlist_save(struct playlist_info* playlist, char *filename) overwrite_current = true; } - fd = open(path, O_CREAT|O_WRONLY|O_TRUNC); + if (is_m3u8(path)) + { + fd = open_utf8(path, O_CREAT|O_WRONLY|O_TRUNC); + } + else + { + /* some applications require a BOM to read the file properly */ + fd = open(path, O_CREAT|O_WRONLY|O_TRUNC); + } if (fd < 0) { gui_syncsplash(HZ*2, ID2P(LANG_PLAYLIST_ACCESS_ERROR)); @@ -3423,12 +3411,6 @@ int playlist_save(struct playlist_info* playlist, char *filename) display_playlist_count(count, ID2P(LANG_PLAYLIST_SAVE_COUNT), false); cpu_boost(true); - - if (is_m3u8(path)) - { - /* some applications require a BOM to read the file properly */ - write(fd, BOM, BOM_SIZE); - } index = playlist->first_index; for (i=0; iamount; i++) @@ -3496,7 +3478,7 @@ int playlist_save(struct playlist_info* playlist, char *filename) { if (rename(path, playlist->filename) >= 0) { - playlist->fd = open(playlist->filename, O_RDONLY); + playlist->fd = open_utf8(playlist->filename, O_RDONLY); if (playlist->fd >= 0) { index = playlist->first_index; diff --git a/apps/recorder/keyboard.c b/apps/recorder/keyboard.c index a15b4cc346..621ec4e1c3 100644 --- a/apps/recorder/keyboard.c +++ b/apps/recorder/keyboard.c @@ -141,7 +141,7 @@ int load_kbd(unsigned char* filename) return 0; } - fd = open(filename, O_RDONLY|O_BINARY); + fd = open_utf8(filename, O_RDONLY|O_BINARY); if (fd < 0) return 1; diff --git a/apps/recorder/radio.c b/apps/recorder/radio.c index bd91defe09..145972939e 100644 --- a/apps/recorder/radio.c +++ b/apps/recorder/radio.c @@ -1112,7 +1112,7 @@ void radio_load_presets(char *filename) snprintf(filepreset, sizeof(filepreset), "%s/%s.fmr", FMPRESET_PATH, filename); - fd = open(filepreset, O_RDONLY); + fd = open_utf8(filepreset, O_RDONLY); if(fd >= 0) { while(!done && num_presets < MAX_PRESETS) diff --git a/apps/settings.c b/apps/settings.c index 4c181593db..a9257aaf53 100644 --- a/apps/settings.c +++ b/apps/settings.c @@ -265,7 +265,7 @@ bool settings_load_config(const char* file, bool apply) char* name; char* value; int i; - fd = open(file, O_RDONLY); + fd = open_utf8(file, O_RDONLY); if (fd < 0) return false;