Unify opening of utf-8 files (FS#6203). This also adds ignoring the BOM in several places it has been missing (as FS#6071).

git-svn-id: svn://svn.rockbox.org/rockbox/trunk@18185 a1c6a512-1295-4272-9138-f99709370657

apps/cuesheet.c

@@ -147,7 +147,7 @@ bool parse_cuesheet(char *file, struct cuesheet *cue)
     char *s;
 
     DEBUGF("cue parse\n");
-    int fd = open(file,O_RDONLY);
+    int fd = open_utf8(file,O_RDONLY);
     if (fd < 0)
     {
         /* couln't open the file */

apps/gui/wps_parser.c

@@ -1527,19 +1527,6 @@ static bool load_wps_bitmaps(struct wps_data *wps_data, char *bmpdir)
 
 #endif /* HAVE_LCD_BITMAP */
 
-/* Skip leading UTF-8 BOM, if present. */
-static char *skip_utf8_bom(char *buf)
-{
-    unsigned char *s = (unsigned char *)buf;
-
-    if(s[0] == 0xef && s[1] == 0xbb && s[2] == 0xbf)
-    {
-        buf += 3;
-    }
-
-    return buf;
-}
-
 /* to setup up the wps-data from a format-buffer (isfile = false)
    from a (wps-)file (isfile = true)*/
 bool wps_data_load(struct wps_data *wps_data,
@@ -1604,7 +1591,7 @@ bool wps_data_load(struct wps_data *wps_data,
 #endif
 #endif /* __PCTOOL__ */
 
-        int fd = open(buf, O_RDONLY);
+        int fd = open_utf8(buf, O_RDONLY);
 
         if (fd < 0)
             return false;
@@ -1639,9 +1626,6 @@ bool wps_data_load(struct wps_data *wps_data,
         memset(bmp_names, 0, sizeof(bmp_names));
 #endif
 
-        /* Skip leading UTF-8 BOM, if present. */
-        wps_buffer = skip_utf8_bom(wps_buffer);
-
         /* parse the WPS source */
         if (!wps_parse(wps_data, wps_buffer)) {
             wps_reset(wps_data);

apps/misc.c

@@ -1180,6 +1180,35 @@ char *strip_extension(char* buffer, int buffer_size, const char *filename)
 }
 #endif /* !defined(__PCTOOL__) */
 
+/** Open a UTF-8 file and set file descriptor to first byte after BOM.
+ *  If no BOM is present this behaves like open().
+ *  If the file is opened for writing and O_TRUNC is set, write a BOM to
+ *  the opened file and leave the file pointer set after the BOM.
+ */
+int open_utf8(const char* pathname, int flags)
+{
+    int fd;
+    unsigned char bom[BOM_SIZE];
+
+    fd = open(pathname, flags);
+    if(fd < 0)
+        return fd;
+
+    if(flags & (O_TRUNC | O_WRONLY))
+    {
+        write(fd, BOM, BOM_SIZE);
+    }
+    else
+    {
+        read(fd, bom, BOM_SIZE);
+        /* check for BOM */
+        if(memcmp(bom, BOM, BOM_SIZE))
+	    lseek(fd, 0, SEEK_SET);
+    }
+    return fd;
+}
+
+
 #ifdef HAVE_LCD_COLOR
 /*
  * Helper function to convert a string of 6 hex digits to a native colour

apps/misc.h

@@ -23,6 +23,10 @@
 
 #include <stdbool.h>
 #include <inttypes.h>
+
+#define BOM "\xef\xbb\xbf"
+#define BOM_SIZE 3
+
 /* Format a large-range value for output, using the appropriate unit so that
  * the displayed value is in the range 1 <= display < 1000 (1024 for "binary"
  * units) if possible, and 3 significant digits are shown. If a buffer is
@@ -106,6 +110,8 @@ extern int show_logo(void);
 int get_replaygain_mode(bool have_track_gain, bool have_album_gain);
 #endif
 
+int open_utf8(const char* pathname, int flags);
+
 #ifdef BOOTFILE
 #if !defined(USB_NONE) && !defined(USB_IPODSTYLE)
 void check_bootfile(bool do_rolo);

apps/playlist.c

@@ -200,9 +200,6 @@ static long playlist_stack[(DEFAULT_STACK_SIZE + 0x800)/sizeof(long)];
 static const char playlist_thread_name[] = "playlist cachectrl";
 #endif
 
-#define BOM "\xef\xbb\xbf"
-#define BOM_SIZE 3
-
 /* Check if the filename suggests M3U or M3U8 format. */
 static bool is_m3u8(const char* filename)
 {
@@ -212,11 +209,6 @@ static bool is_m3u8(const char* filename)
     return !(len > 4 && strcasecmp(&filename[len - 4], ".m3u") == 0);
 }
 
-/* Check if a strings starts with an UTF-8 byte-order mark. */
-static bool is_utf8_bom(const char* str, int len)
-{
-    return len >= BOM_SIZE && memcmp(str, BOM, BOM_SIZE) == 0;
-}
 
 /* Convert a filename in an M3U playlist to UTF-8. 
  *
@@ -538,9 +530,11 @@ static int add_indices_to_playlist(struct playlist_info* playlist,
     int result = 0;
 
     if(-1 == playlist->fd)
-        playlist->fd = open(playlist->filename, O_RDONLY);
+        playlist->fd = open_utf8(playlist->filename, O_RDONLY);
     if(playlist->fd < 0)
         return -1; /* failure */
+    if(lseek(playlist->fd, 0, SEEK_CUR) > 0)
+        playlist->utf8 = true; /* Override any earlier indication. */
     
     gui_syncsplash(0, ID2P(LANG_WAIT));
 
@@ -568,14 +562,6 @@ static int add_indices_to_playlist(struct playlist_info* playlist,
         
         p = (unsigned char *)buffer;
 
-        /* utf8 BOM at beginning of file? */
-        if(i == 0 && is_utf8_bom(p, nread)) {
-            nread -= BOM_SIZE;
-            p += BOM_SIZE;
-            i += BOM_SIZE;
-            playlist->utf8 = true;  /* Override any earlier indication. */
-        }
-
         for(count=0; count < nread; count++,p++) {
 
             /* Are we on a new line? */
@@ -2972,7 +2958,7 @@ int playlist_insert_playlist(struct playlist_info* playlist, const char *filenam
         return -1;
     }
 
-    fd = open(filename, O_RDONLY);
+    fd = open_utf8(filename, O_RDONLY);
     if (fd < 0)
     {
         gui_syncsplash(HZ*2, ID2P(LANG_PLAYLIST_ACCESS_ERROR));
@@ -3010,12 +2996,6 @@ int playlist_insert_playlist(struct playlist_info* playlist, const char *filenam
         if (action_userabort(TIMEOUT_NOBLOCK))
             break;
     
-        if (count == 0 && is_utf8_bom(temp_buf, max))
-        {
-            max -= BOM_SIZE;
-            memmove(temp_buf, temp_buf + BOM_SIZE, max);
-        }
-
         if (temp_buf[0] != '#' && temp_buf[0] != '\0')
         {
             int insert_pos;
@@ -3413,7 +3393,15 @@ int playlist_save(struct playlist_info* playlist, char *filename)
         overwrite_current = true;
     }
 
-    fd = open(path, O_CREAT|O_WRONLY|O_TRUNC);
+    if (is_m3u8(path))
+    {
+        fd = open_utf8(path, O_CREAT|O_WRONLY|O_TRUNC);
+    }
+    else
+    {
+        /* some applications require a BOM to read the file properly */
+        fd = open(path, O_CREAT|O_WRONLY|O_TRUNC);
+    }
     if (fd < 0)
     {
         gui_syncsplash(HZ*2, ID2P(LANG_PLAYLIST_ACCESS_ERROR));
@@ -3424,12 +3412,6 @@ int playlist_save(struct playlist_info* playlist, char *filename)
 
     cpu_boost(true);
 
-    if (is_m3u8(path))
-    {
-        /* some applications require a BOM to read the file properly */
-        write(fd, BOM, BOM_SIZE);
-    }
-
     index = playlist->first_index;
     for (i=0; i<playlist->amount; i++)
     {
@@ -3496,7 +3478,7 @@ int playlist_save(struct playlist_info* playlist, char *filename)
         {
             if (rename(path, playlist->filename) >= 0)
             {
-                playlist->fd = open(playlist->filename, O_RDONLY);
+                playlist->fd = open_utf8(playlist->filename, O_RDONLY);
                 if (playlist->fd >= 0)
                 {
                     index = playlist->first_index;

apps/recorder/keyboard.c

@@ -141,7 +141,7 @@ int load_kbd(unsigned char* filename)
         return 0;
     }
 
-    fd = open(filename, O_RDONLY|O_BINARY);
+    fd = open_utf8(filename, O_RDONLY|O_BINARY);
     if (fd < 0)
         return 1;
 

apps/recorder/radio.c

@@ -1112,7 +1112,7 @@ void radio_load_presets(char *filename)
         snprintf(filepreset, sizeof(filepreset), "%s/%s.fmr",
             FMPRESET_PATH, filename);
     
-    fd = open(filepreset, O_RDONLY);
+    fd = open_utf8(filepreset, O_RDONLY);
     if(fd >= 0)
     {
         while(!done && num_presets < MAX_PRESETS)

apps/settings.c

@@ -265,7 +265,7 @@ bool settings_load_config(const char* file, bool apply)
     char* name;
     char* value;
     int i;
-    fd = open(file, O_RDONLY);
+    fd = open_utf8(file, O_RDONLY);
     if (fd < 0)
         return false;