rockbox/apps/tagcache.c

/***************************************************************************
 *             __________               __   ___.
 *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
 *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
 *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
 *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
 *                     \/            \/     \/    \/            \/
 * $Id$
 *
 * Copyright (C) 2005 by Miika Pekkarinen
 *
 * All files in this archive are subject to the GNU General Public License.
 * See the file COPYING in the source tree root for full license agreement.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 ****************************************************************************/

#include <stdio.h>
#include "thread.h"
#include "kernel.h"
#include "system.h"
#include "logf.h"
#include "string.h"
#include "usb.h"
#include "dircache.h"
#include "metadata.h"
#include "id3.h"
#include "settings.h"
#include "splash.h"
#include "lang.h"
#include "tagcache.h"
#include "buffer.h"
#include "atoi.h"
#include "crc32.h"

/* Tag Cache thread. */
static struct event_queue tagcache_queue;
static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
static const char tagcache_thread_name[] = "tagcache";

/* Previous path when scanning directory tree recursively. */
static char curpath[MAX_PATH*2];
static long curpath_size = sizeof(curpath);

/* Used when removing duplicates. */
static char *tempbuf;     /* Allocated when needed. */
static long tempbufidx;   /* Current location in buffer. */
static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
static long tempbuf_left; /* Buffer space left. */
static long tempbuf_pos;

/* Tags we want to get sorted (loaded to the tempbuf). */
static const int sorted_tags[] = { tag_artist, tag_album, tag_genre, tag_composer, tag_title };

/* Uniqued tags (we can use these tags with filters and conditional clauses). */
static const int unique_tags[] = { tag_artist, tag_album, tag_genre, tag_composer };

/* Numeric tags (we can use these tags with conditional clauses). */
static const int numeric_tags[] = { tag_year, tag_tracknumber, tag_length, tag_bitrate,
    tag_playcount, tag_playtime, tag_lastplayed, tag_virt_autoscore };

/* Status information of the tagcache. */
static struct tagcache_stat stat;

/* Queue commands. */
enum tagcache_queue {
    Q_STOP_SCAN = 0,
    Q_START_SCAN,
    Q_UPDATE,
    Q_REBUILD,
};


/* Tag database structures. */

/* Variable-length tag entry in tag files. */
struct tagfile_entry {
    short tag_length;
    short idx_id;
    char tag_data[0];
};

/* Fixed-size tag entry in master db index. */
struct index_entry {
    long tag_seek[TAG_COUNT];
    long flag;
};

/* Header is the same in every file. */
struct tagcache_header {
    long magic;
    long datasize;
    long entry_count;
};

#ifdef HAVE_TC_RAMCACHE
/* Header is created when loading database to ram. */
struct ramcache_header {
    struct tagcache_header h;
    struct index_entry *indices;
    char *tags[TAG_COUNT];
    int entry_count[TAG_COUNT];
};

static struct ramcache_header *hdr;
#endif

/** 
 * Full tag entries stored in a temporary file waiting 
 * for commit to the cache. */
struct temp_file_entry {
    long tag_offset[TAG_COUNT];
    short tag_length[TAG_COUNT];

    long data_length;
};

struct tempbuf_id_list {
    long id;
    struct tempbuf_id_list *next;
};

struct tempbuf_searchidx {
    long idx_id;
    char *str;
    int seek;
    struct tempbuf_id_list idlist;
};

#define LOOKUP_BUF_DEPTH  (TAGFILE_MAX_ENTRIES*2 \
    * (TAGFILE_ENTRY_AVG_LENGTH/TAGFILE_ENTRY_CHUNK_LENGTH))

struct tempbuf_searchidx **lookup;

/* Used when building the temporary file. */
static int cachefd = -1, filenametag_fd;
static int total_entry_count = 0;
static int data_size = 0;
static int processed_dir_count;

bool tagcache_is_numeric_tag(int type)
{
    int i;

    for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
    {
        if (type == numeric_tags[i])
            return true;
    }

    return false;
}

bool tagcache_is_unique_tag(int type)
{
    int i;

    for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
    {
        if (type == unique_tags[i])
            return true;
    }

    return false;
}

bool tagcache_is_sorted_tag(int type)
{
    int i;

    for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
    {
        if (type == sorted_tags[i])
            return true;
    }

    return false;
}

#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
static long find_entry_ram(const char *filename,
                           const struct dircache_entry *dc)
{
    static long last_pos = 0;
    int counter = 0;
    int i;
    
    /* Check if we tagcache is loaded into ram. */
    if (!stat.ramcache)
        return -1;

    if (dc == NULL)
        dc = dircache_get_entry_ptr(filename);
    
    if (dc == NULL)
    {
        logf("tagcache: file not found.");
        return -1;
    }

    try_again:
            
    if (last_pos > 0)
        i = last_pos;
    else
        i = 0;
    
    for (; i < hdr->h.entry_count; i++)
    {
        if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
        {
            last_pos = MAX(0, i - 3);
            return i;
        }
        
        if (++counter == 100)
        {
            yield();
            counter = 0;
        }
    }

    if (last_pos > 0)
    {
        last_pos = 0;
        goto try_again;
    }

    return -1;
}
#endif

static long find_entry_disk(const char *filename)
{
    static long last_pos = -1;
    long pos_history[POS_HISTORY_COUNT];
    long pos_history_idx = 0;
    bool found = false;
    struct tagfile_entry tfe;
    int fd;
    char buf[MAX_PATH];
    int i;
    int pos = -1;

    fd = filenametag_fd;
    if (fd < 0)
    {
        last_pos = -1;
        if ( (fd = open(TAGCACHE_FILE_MASTER, O_RDONLY)) < 0)
            return -1;
    }
    
    check_again:
    
    if (last_pos > 0)
        lseek(fd, last_pos, SEEK_SET);
    else
        lseek(fd, sizeof(struct tagcache_header), SEEK_SET);

    while (true)
    {
        pos = lseek(fd, 0, SEEK_CUR);
        for (i = pos_history_idx-1; i >= 0; i--)
            pos_history[i+1] = pos_history[i];
        pos_history[0] = pos;

        if (read(fd, &tfe, sizeof(struct tagfile_entry)) !=
            sizeof(struct tagfile_entry))
        {
            break ;
        }
        
        if (tfe.tag_length >= (long)sizeof(buf))
        {
            logf("too long tag #1");
            close(fd);
            last_pos = -1;
            return -2;
        }
        
        if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
        {
            logf("read error #2");
            close(fd);
            last_pos = -1;
            return -3;
        }
        
        if (!strcasecmp(filename, buf))
        {
            last_pos = pos_history[pos_history_idx];
            found = true;
            break ;
        }
        
        if (pos_history_idx < POS_HISTORY_COUNT - 1)
            pos_history_idx++;
    }
    
    /* Not found? */
    if (!found)
    {
        if (last_pos > 0)
        {
            last_pos = -1;
            logf("seek again");
            goto check_again;
        }
        
        if (fd != filenametag_fd)
            close(fd);
        return -4;
    }
    
    if (fd != filenametag_fd)
        close(fd);
        
    return tfe.idx_id;
}

bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
{
    long idx_id = -1;
    
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
    if (stat.ramcache && dircache_is_enabled())
        idx_id = find_entry_ram(filename, NULL);
#endif
    
    if (idx_id < 0)
        idx_id = find_entry_disk(filename);
    
    if (idx_id < 0)
        return false;
    
    if (!tagcache_search(tcs, tag_filename))
        return false;
    
    tcs->entry_count = 0;
    tcs->idx_id = idx_id;
    
    return true;
}

static bool tagcache_get_index(const struct tagcache_search *tcs,
                               int idxid, struct index_entry *idx)
{
#ifdef HAVE_TC_RAMCACHE
    if (tcs->ramsearch)
    {
        if (hdr->indices[idxid].flag & FLAG_DELETED)
            return false;
        
        memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
        return true;
    }
#endif
    
    lseek(tcs->masterfd, idxid * sizeof(struct index_entry) 
          + sizeof(struct tagcache_header), SEEK_SET);
    if (read(tcs->masterfd, idx, sizeof(struct index_entry)) !=
        sizeof(struct index_entry))
    {
        logf("read error #3");
        return false;
    }
    
    return true;
}

static long check_virtual_tags(int tag, const struct index_entry *idx)
{
    long data = 0;
    
    switch (tag) 
    {
        case tag_virt_autoscore:
            if (idx->tag_seek[tag_length] == 0 
                || idx->tag_seek[tag_playcount] == 0)
            {
                data = 0;
            }
            else
            {
                data = 100 * idx->tag_seek[tag_playtime]
                    / idx->tag_seek[tag_length]
                    / idx->tag_seek[tag_playcount];
            }
            break;
        
        default:
            data = idx->tag_seek[tag];
    }
    
    return data;
}

long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
{
    struct index_entry idx;
    
    if (!tagcache_is_numeric_tag(tag))
        return -1;
    
    if (!tagcache_get_index(tcs, tcs->idx_id, &idx))
        return -2;
    
    return check_virtual_tags(tag, &idx);
}

static bool check_against_clause(long numeric, const char *str,
                                 const struct tagcache_search_clause *clause)
{
    switch (clause->type)
    {
        case clause_is:
            if (clause->numeric)
                return numeric == clause->numeric_data;
            else
                return !strcasecmp(clause->str, str);
        
        case clause_gt:
            return numeric > clause->numeric_data;
        case clause_gteq:
            return numeric >= clause->numeric_data;
        case clause_lt:
            return numeric < clause->numeric_data;
        case clause_lteq:
            return numeric <= clause->numeric_data;
        
        case clause_contains:
            return (strcasestr(str, clause->str) != NULL);
        case clause_begins_with:
            return (strcasestr(str, clause->str) == str);
        case clause_ends_with: /* Not supported yet */
            return false;
    }
    
    return false;
}

static bool build_lookup_list(struct tagcache_search *tcs)
{
    struct index_entry entry;
    int i;
    
    tcs->seek_list_count = 0;
    
#ifdef HAVE_TC_RAMCACHE
    if (tcs->ramsearch)
    {
        int j;

        for (i = tcs->seek_pos; i < hdr->h.entry_count; i++)
        {
            if (tcs->seek_list_count == SEEK_LIST_SIZE)
                break ;
            
            /* Skip deleted files. */
            if (hdr->indices[i].flag & FLAG_DELETED)
                continue;
            
            /* Go through all filters.. */
            for (j = 0; j < tcs->filter_count; j++)
            {
                if (hdr->indices[i].tag_seek[tcs->filter_tag[j]] !=
                    tcs->filter_seek[j])
                    break ;
            }
            
            if (j < tcs->filter_count)
                continue ;

            /* Go through all conditional clauses. */
            for (j = 0; j < tcs->clause_count; j++)
            {
                struct index_entry *idx = &hdr->indices[i];
                int seek;
                char buf[256];
                char *str = NULL;
                struct tagfile_entry *entry;
                
                seek = check_virtual_tags(tcs->clause[j]->tag, idx);
                
                if (!tagcache_is_numeric_tag(tcs->clause[j]->tag))
                {
                    if (tcs->clause[j]->tag == tag_filename)
                    {
                        tagcache_retrieve(tcs, i, buf, sizeof buf);
                        str = buf;
                    }
                    else
                    {
                        entry = (struct tagfile_entry *)&hdr->tags[tcs->clause[j]->tag][seek];
                        str = entry->tag_data;
                    }
                }
                
                
                if (!check_against_clause(seek, str, tcs->clause[j]))
                    break ;
            }
                                          
            if (j < tcs->clause_count)
                continue ;
            
            /* Add to the seek list if not already there. */
            for (j = 0; j < tcs->seek_list_count; j++)
            {
                if (tcs->seek_list[j] == hdr->indices[i].tag_seek[tcs->type])
                    break ;
            }

            /* Lets add it. */
            if (j == tcs->seek_list_count)
            {
                tcs->seek_list[tcs->seek_list_count] =
                        hdr->indices[i].tag_seek[tcs->type];
                tcs->seek_flags[tcs->seek_list_count] =
                        hdr->indices[i].flag;
                tcs->seek_list_count++;
            }
        }
        
        tcs->seek_pos = i;

        return tcs->seek_list_count > 0;
    }
#endif
    
    lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
            sizeof(struct tagcache_header), SEEK_SET);
    
    while (read(tcs->masterfd, &entry, sizeof(struct index_entry)) ==
           sizeof(struct index_entry))
    {
        /* Check if entry has been deleted. */
        if (entry.flag & FLAG_DELETED)
            continue;
        
        if (tcs->seek_list_count == SEEK_LIST_SIZE)
            break ;

        /* Go through all filters.. */
        for (i = 0; i < tcs->filter_count; i++)
        {
            if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
                break ;
        }
        
        tcs->seek_pos++;
        
        if (i < tcs->filter_count)
            continue ;
        
        /* Check for conditions. */
        for (i = 0; i < tcs->clause_count; i++)
        {
            struct tagfile_entry tfe;
            int seek;
            char str[256];
            
            seek = check_virtual_tags(tcs->clause[i]->tag, &entry);
                
            memset(str, 0, sizeof str);
            if (!tagcache_is_numeric_tag(tcs->clause[i]->tag))
            {
                int fd = tcs->idxfd[tcs->clause[i]->tag];
                lseek(fd, seek, SEEK_SET);
                read(fd, &tfe, sizeof(struct tagfile_entry));
                if (tfe.tag_length >= (int)sizeof(str))
                {
                    logf("Too long tag read!");
                    break ;
                }

                read(fd, str, tfe.tag_length);
                
                /* Check if entry has been deleted. */
                if (str[0] == '\0')
                    break;
            }
            
            if (!check_against_clause(seek, str, tcs->clause[i]))
                break ;
        }
        
        if (i < tcs->clause_count)
            continue ;
            
        /* Add to the seek list if not already there. */
        for (i = 0; i < tcs->seek_list_count; i++)
        {
            if (tcs->seek_list[i] == entry.tag_seek[tcs->type])
                break ;
        }

        /* Lets add it. */
        if (i == tcs->seek_list_count)
        {
            tcs->seek_list[tcs->seek_list_count] =
                    entry.tag_seek[tcs->type];
            tcs->seek_flags[tcs->seek_list_count] = entry.flag;
            tcs->seek_list_count++;
        }
        
    }

    return tcs->seek_list_count > 0;
}

        
bool tagcache_search(struct tagcache_search *tcs, int tag)
{
    struct tagcache_header h;
    char buf[MAX_PATH];
    int i;

    if (tcs->valid)
        tagcache_search_finish(tcs);
    
    memset(tcs, 0, sizeof(struct tagcache_search));
    if (stat.commit_step > 0)
        return false;
    
    tcs->position = sizeof(struct tagcache_header);
    tcs->type = tag;
    tcs->seek_pos = 0;
    tcs->seek_list_count = 0;
    tcs->filter_count = 0;
    tcs->valid = true;
    tcs->masterfd = -1;

    for (i = 0; i < TAG_COUNT; i++)
        tcs->idxfd[i] = -1;

#ifndef HAVE_TC_RAMCACHE
    tcs->ramsearch = false;
#else
    tcs->ramsearch = stat.ramcache;
    if (tcs->ramsearch)
    {
        tcs->entry_count = hdr->entry_count[tcs->type];
    }
    else
#endif
    {
        if (tagcache_is_numeric_tag(tcs->type))
            return true;
        
        snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tcs->type);
        tcs->idxfd[tcs->type] = open(buf, O_RDONLY);
        if (tcs->idxfd[tcs->type] < 0)
        {
            logf("failed to open index");
            return false;
        }

        /* Check the header. */
        if (read(tcs->idxfd[tcs->type], &h, sizeof(struct tagcache_header)) !=
            sizeof(struct tagcache_header) || h.magic != TAGCACHE_MAGIC)
        {
            logf("incorrect header");
            return false;
        }
        
        tcs->masterfd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
        
        if (tcs->masterfd < 0)
        {
            logf("open fail");
            return false;
        }
        
        if (read(tcs->masterfd, &h, sizeof(struct tagcache_header)) !=
            sizeof(struct tagcache_header) || h.magic != TAGCACHE_MAGIC)
        {
            logf("header error");
            close(tcs->masterfd);
            tcs->masterfd = -1;
            return false;
        }
    }

    return true;
}

bool tagcache_search_add_filter(struct tagcache_search *tcs,
                                int tag, int seek)
{
    if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
        return false;

    if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
        return false;
    
    tcs->filter_tag[tcs->filter_count] = tag;
    tcs->filter_seek[tcs->filter_count] = seek;
    tcs->filter_count++;

    return true;
}

bool tagcache_search_add_clause(struct tagcache_search *tcs,
                                struct tagcache_search_clause *clause)
{
    if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
    {
        logf("Too many clauses");
        return false;
    }

    if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
    {
        char buf[MAX_PATH];

        snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);        
        tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
    }
    
    tcs->clause[tcs->clause_count] = clause;
    tcs->clause_count++;
    
    return true;
}

static bool open_files(struct tagcache_search *tcs)
{
    if (tcs->idxfd[tcs->type] < 0)
    {
        char fn[MAX_PATH];

        snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tcs->type);
        tcs->idxfd[tcs->type] = open(fn, O_RDONLY);
    }
    
    if (tcs->idxfd[tcs->type] < 0)
    {
        logf("File not open!");
        return false;
    }
    
    return true;
}

#define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
                                   ? (tcs->seek_flags[tcs->seek_list_count] \
                                      & FLAG_DIRCACHE) : 1)

static bool get_next(struct tagcache_search *tcs)
{
    static char buf[MAX_PATH];
    struct tagfile_entry entry;

    if (!tcs->valid)
        return false;
        
    if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
#ifdef HAVE_TC_RAMCACHE
        && !tcs->ramsearch
#endif
        )
        return false;
    
    /* Searching not supported for numeric tags yet. */
    if (tagcache_is_numeric_tag(tcs->type))
        return false;
    
    /* Relative fetch. */
    if (tcs->filter_count > 0 || tcs->clause_count > 0)
    {
        /* Check for end of list. */
        if (tcs->seek_list_count == 0)
        {
            /* Try to fetch more. */
            if (!build_lookup_list(tcs))
                return false;
        }
        
        tcs->seek_list_count--;
        
        /* Seek stream to the correct position and continue to direct fetch. */
        if (!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
        {
            if (!open_files(tcs))
                return false;
        
            lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
        }
        else
            tcs->position = tcs->seek_list[tcs->seek_list_count];
    }
    
    /* Direct fetch. */
#ifdef HAVE_TC_RAMCACHE
    if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
    {
        struct tagfile_entry *ep;
        
        if (tcs->entry_count == 0)
        {
            tcs->valid = false;
            return false;
        }
        tcs->entry_count--;
        tcs->result_seek = tcs->position;
        
# ifdef HAVE_DIRCACHE
        if (tcs->type == tag_filename)
        {
            dircache_copy_path((struct dircache_entry *)tcs->position,
                               buf, sizeof buf);
            tcs->result = buf;
            tcs->result_len = strlen(buf) + 1;
            
            return true;
        }
# endif
        
        ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
        tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
        tcs->result = ep->tag_data;
        tcs->result_len = strlen(tcs->result) + 1;
        tcs->idx_id = ep->idx_id;
        
        if (!tagcache_is_unique_tag(tcs->type))
            tcs->result_seek = tcs->idx_id;

        return true;
    }
    else
#endif
    {
        if (!open_files(tcs))
            return false;
        
        tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
        if (read(tcs->idxfd[tcs->type], &entry, sizeof(struct tagfile_entry)) !=
            sizeof(struct tagfile_entry))
        {
            /* End of data. */
            tcs->valid = false;
            return false;
        }
    }
    
    if (entry.tag_length > (long)sizeof(buf))
    {
        tcs->valid = false;
        logf("too long tag #2");
        return false;
    }
    
    if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
    {
        tcs->valid = false;
        logf("read error");
        return false;
    }
    
    tcs->result = buf;
    tcs->result_len = strlen(tcs->result) + 1;
    tcs->idx_id = entry.idx_id;
    if (!tagcache_is_unique_tag(tcs->type))
        tcs->result_seek = tcs->idx_id;

    return true;
}

bool tagcache_get_next(struct tagcache_search *tcs)
{
    while (get_next(tcs))
    {
        if (tcs->result_len > 1)
            return true;
    }
    
    return false;
}

bool tagcache_retrieve(struct tagcache_search *tcs, int idxid, 
                       char *buf, long size)
{
    struct tagfile_entry tfe;
    struct index_entry idx;
    long seek;
    
    if (!tagcache_get_index(tcs, idxid, &idx))
        return false;
    
    seek = idx.tag_seek[tcs->type];
    if (seek < 0)
    {
        logf("Retrieve failed");
        return false;
    }
    
#ifdef HAVE_TC_RAMCACHE
    if (tcs->ramsearch)
    {
        struct tagfile_entry *ep;
        
# ifdef HAVE_DIRCACHE
        if (tcs->type == tag_filename && hdr->indices[idxid].flag & FLAG_DIRCACHE)
        {
            dircache_copy_path((struct dircache_entry *)seek,
                               buf, size);
            return true;
        }
        else
# endif
        if (tcs->type != tag_filename)
        {
            ep = (struct tagfile_entry *)&hdr->tags[tcs->type][seek];
            strncpy(buf, ep->tag_data, size-1);
            
            return true;
        }
    }
#endif
    
    if (!open_files(tcs))
        return false;
    
    lseek(tcs->idxfd[tcs->type], seek, SEEK_SET);
    if (read(tcs->idxfd[tcs->type], &tfe, sizeof(struct tagfile_entry)) !=
        sizeof(struct tagfile_entry))
    {
        logf("read error");
        return false;
    }
    
    if (tfe.tag_length >= size)
    {
        logf("too small buffer");
        return false;
    }
    
    if (read(tcs->idxfd[tcs->type], buf, tfe.tag_length) !=
        tfe.tag_length)
    {
        logf("read error #2");
        return false;
    }
    
    buf[tfe.tag_length] = '\0';
    
    return true;
}

#if 0

void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
{
    struct tagentry *entry;
    
    if (tcs->type != tag_title)
        return ;

    /* We will need reserve buffer for this. */
    if (tcs->ramcache)
    {
        struct tagfile_entry *ep;

        ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
        tcs->seek_list[tcs->seek_list_count];
    }

    entry = find_entry_ram();
    
}
#endif

void tagcache_search_finish(struct tagcache_search *tcs)
{
    int i;
    
    if (tcs->masterfd >= 0)
    {
        close(tcs->masterfd);
        tcs->masterfd = -1;
    }

    for (i = 0; i < TAG_COUNT; i++)
    {
        if (tcs->idxfd[i] >= 0)
        {
            close(tcs->idxfd[i]);
            tcs->idxfd[i] = -1;
        }
    }
    
    tcs->ramsearch = false;
    tcs->valid = false;
}

#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
{
    return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
}

static long get_tag_numeric(const struct index_entry *entry, int tag)
{
    return entry->tag_seek[tag];
}

bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
{
    struct index_entry *entry;
    int idx_id;
    
    /* Find the corresponding entry in tagcache. */
    idx_id = find_entry_ram(filename, NULL);
    if (idx_id < 0 || !stat.ramcache)
        return false;
    
    entry = &hdr->indices[idx_id];
    
    id3->title = get_tag(entry, tag_title)->tag_data;
    id3->artist = get_tag(entry, tag_artist)->tag_data;
    id3->album = get_tag(entry, tag_album)->tag_data;
    id3->genre_string = get_tag(entry, tag_genre)->tag_data;
    id3->composer = get_tag(entry, tag_composer)->tag_data;
    id3->year = get_tag_numeric(entry, tag_year);
    id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
    id3->bitrate = get_tag_numeric(entry, tag_bitrate);
    if (id3->bitrate == 0)
        id3->bitrate = 1;

    return true;
}
#endif

static inline void write_item(const char *item)
{
    int len = strlen(item) + 1;
    
    data_size += len;
    write(cachefd, item, len);
}

inline void check_if_empty(char **tag)
{
    if (tag == NULL || *tag == NULL || *tag[0] == '\0')
        *tag = "<Untagged>";
}

#define CRC_BUF_LEN 8

#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
static void add_tagcache(const char *path, const struct dircache_entry *dc)
#else
static void add_tagcache(const char *path)
#endif
{
    struct track_info track;
    struct temp_file_entry entry;
    bool ret;
    int fd;
    char tracknumfix[3];
    char *genrestr;
    //uint32_t crcbuf[CRC_BUF_LEN];

    if (cachefd < 0)
        return ;

    /* Check if the file is supported. */
    if (probe_file_format(path) == AFMT_UNKNOWN)
        return ;
    
    /* Check if the file is already cached. */
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
    if (stat.ramcache && dircache_is_enabled())
    {
        if (find_entry_ram(path, dc) >= 0)
            return ;
    }
    else
#endif
    {
        if (find_entry_disk(path) >= 0)
            return ;
    }
    
    fd = open(path, O_RDONLY);
    if (fd < 0)
    {
        logf("open fail: %s", path);
        return ;
    }

    memset(&track, 0, sizeof(struct track_info));
    memset(&entry, 0, sizeof(struct temp_file_entry));
    memset(&tracknumfix, 0, sizeof(tracknumfix));
    ret = get_metadata(&track, fd, path, false);
    close(fd);

    if (!ret)
        return ;

    genrestr = id3_get_genre(&track.id3);
    
    check_if_empty(&track.id3.title);
    check_if_empty(&track.id3.artist);
    check_if_empty(&track.id3.album);
    check_if_empty(&genrestr);
    check_if_empty(&track.id3.composer);
    
    entry.tag_length[tag_filename] = strlen(path) + 1;
    entry.tag_length[tag_title] = strlen(track.id3.title) + 1;
    entry.tag_length[tag_artist] = strlen(track.id3.artist) + 1;
    entry.tag_length[tag_album] = strlen(track.id3.album) + 1;
    entry.tag_length[tag_genre] = strlen(genrestr) + 1;
    entry.tag_length[tag_composer] = strlen(track.id3.composer) + 1;
    
    entry.tag_offset[tag_filename] = 0;
    entry.tag_offset[tag_title] = entry.tag_offset[tag_filename] + entry.tag_length[tag_filename];
    entry.tag_offset[tag_artist] = entry.tag_offset[tag_title] + entry.tag_length[tag_title];
    entry.tag_offset[tag_album] = entry.tag_offset[tag_artist] + entry.tag_length[tag_artist];
    entry.tag_offset[tag_genre] = entry.tag_offset[tag_album] + entry.tag_length[tag_album];
    entry.tag_offset[tag_composer] = entry.tag_offset[tag_genre] + entry.tag_length[tag_genre];
    entry.data_length = entry.tag_offset[tag_composer] + entry.tag_length[tag_composer];
    
    /* Numeric tags */
    entry.tag_offset[tag_year] = track.id3.year;
    entry.tag_offset[tag_tracknumber] = track.id3.tracknum;
    entry.tag_offset[tag_length] = track.id3.length;
    entry.tag_offset[tag_bitrate] = track.id3.bitrate;
    
    if (entry.tag_offset[tag_tracknumber] <= 0)
    {
        const char *p = strrchr(path, '.');
        
        if (p == NULL)
            p = &path[strlen(path)-1];
        
        while (*p != '/')
        {
            if (isdigit(*p) && isdigit(*(p-1)))
            {
                tracknumfix[1] = *p--;
                tracknumfix[0] = *p;
                break;
            }
            p--;
        }
        
        if (tracknumfix[0] != '\0')
            entry.tag_offset[tag_tracknumber] = atoi(tracknumfix);
        else
            entry.tag_offset[tag_tracknumber] = -1;
    }
    
    write(cachefd, &entry, sizeof(struct temp_file_entry));
    write_item(path);
    write_item(track.id3.title);
    write_item(track.id3.artist);
    write_item(track.id3.album);
    write_item(genrestr);
    write_item(track.id3.composer);
    total_entry_count++;    
}

static void remove_files(void)
{
    int i;
    char buf[MAX_PATH];
    
    remove(TAGCACHE_FILE_MASTER);
    for (i = 0; i < TAG_COUNT; i++)
    {
        if (tagcache_is_numeric_tag(i))
            continue;
        
        snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
        remove(buf);
    }
}

static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
{
    struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
    int len = strlen(str)+1;
    int i;
    unsigned crc32;
    unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
    char buf[MAX_PATH];
    
    for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
        buf[i] = tolower(str[i]);
    buf[i] = '\0';
    
    crc32 = crc_32(buf, i, 0xffffffff);
    
    if (unique)
    {
        /* Check if the crc does not exist -> entry does not exist for sure. */
        for (i = 0; i < tempbufidx; i++)
        {
            if (crcbuf[-i] != crc32)
                continue;
            
            if (!strcasecmp(str, index[i].str))
            {
                if (id >= 0 && id < LOOKUP_BUF_DEPTH)
                    lookup[id] = &index[i];
                
                return true;
            }
        }
    }
    
    /* Insert to CRC buffer. */
    crcbuf[-tempbufidx] = crc32;
    tempbuf_left -= 4;
    
    /* Insert it to the buffer. */
    tempbuf_left -= len;
    if (tempbuf_left - 4 < 0 || tempbufidx >= TAGFILE_MAX_ENTRIES-1)
        return false;
    
    if (id >= 0 && id < LOOKUP_BUF_DEPTH)
    {
        lookup[id] = &index[tempbufidx];
        index[tempbufidx].idlist.id = id;
    }
    else
        index[tempbufidx].idlist.id = -1;
    
    index[tempbufidx].idlist.next = NULL;
    index[tempbufidx].idx_id = idx_id;
    index[tempbufidx].seek = -1;
    index[tempbufidx].str = &tempbuf[tempbuf_pos];
    memcpy(index[tempbufidx].str, str, len);
    tempbuf_pos += len;
    tempbufidx++;
    
    return true;
}

static int compare(const void *p1, const void *p2)
{
    struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
    struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
    
    /*
    if (!strncasecmp("the ", e1, 4))
        e1 = &e1[4];
    if (!strncasecmp("the ", e2, 4))
        e2 = &e2[4];
    */
    
    return strncasecmp(e1->str, e2->str, MAX_PATH);
}

static int tempbuf_sort(int fd)
{
    struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
    struct tagfile_entry fe;
    int i;
    int length;
    
    /* Generate reverse lookup entries. */
    for (i = 0; i < LOOKUP_BUF_DEPTH; i++)
    {
        struct tempbuf_id_list *idlist;
        
        if (!lookup[i])
            continue;
        
        if (lookup[i]->idlist.id == i)
            continue;
        
        idlist = &lookup[i]->idlist;
        while (idlist->next != NULL)
            idlist = idlist->next;
        
        tempbuf_left -= sizeof(struct tempbuf_id_list);
        if (tempbuf_left - 4 < 0)
            return -1;
        
        idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
        if (tempbuf_pos & 0x03)
        {
            tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
            tempbuf_left -= 3;
            idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
        }
        tempbuf_pos += sizeof(struct tempbuf_id_list);
        
        idlist = idlist->next;
        idlist->id = i;
        idlist->next = NULL;
    }
    
    qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
    memset(lookup, 0, LOOKUP_BUF_DEPTH * sizeof(struct tempbuf_searchidx **));
    
    for (i = 0; i < tempbufidx; i++)
    {
        struct tempbuf_id_list *idlist = &index[i].idlist;
        
        /* Fix the lookup list. */
        while (idlist != NULL)
        {
            if (idlist->id >= 0)
                lookup[idlist->id] = &index[i];
            idlist = idlist->next;
        }
        
        index[i].seek = lseek(fd, 0, SEEK_CUR);
        length = strlen(index[i].str) + 1;
        fe.tag_length = length;
        fe.idx_id = index[i].idx_id;
        
        /* Check the chunk alignment. */
        if ((fe.tag_length + sizeof(struct tagfile_entry)) 
            % TAGFILE_ENTRY_CHUNK_LENGTH)
        {
            fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH - 
                ((fe.tag_length + sizeof(struct tagfile_entry)) 
                 % TAGFILE_ENTRY_CHUNK_LENGTH);
        }
        
#ifdef TAGCACHE_STRICT_ALIGN
        /* Make sure the entry is long aligned. */
        if (index[i].seek & 0x03)
        {
            logf("tempbuf_sort: alignment error!");
            return -3;
        }
#endif
        
        if (write(fd, &fe, sizeof(struct tagfile_entry)) !=
            sizeof(struct tagfile_entry))
        {
            logf("tempbuf_sort: write error #1");
            return -1;
        }
        
        if (write(fd, index[i].str, length) != length)
        {
            logf("tempbuf_sort: write error #2");
            return -2;
        }
        
        /* Write some padding. */
        if (fe.tag_length - length > 0)
            write(fd, "XXXXXXXX", fe.tag_length - length);
    }

    return i;
}
    
inline static struct tempbuf_searchidx* tempbuf_locate(int id)
{
    if (id < 0 || id >= LOOKUP_BUF_DEPTH)
        return NULL;
    
    return lookup[id];
}


inline static int tempbuf_find_location(int id)
{
    struct tempbuf_searchidx *entry;

    entry = tempbuf_locate(id);
    if (entry == NULL)
        return -1;

    return entry->seek;
}

static bool build_numeric_index(int index_type, struct tagcache_header *h, int tmpfd)
{
    struct tagcache_header tch;
    struct index_entry idx;
    int masterfd;
    int masterfd_pos;
    long *databuf = (long *)tempbuf;
    int max_entries;
    int i;
    
    max_entries = tempbuf_size / sizeof(long);
    
    if (h->entry_count >= max_entries)
    {
        logf("not enough space!");
        return false;
    }
        
    logf("Building numeric index: %d", index_type);
    
    /* Walk through the temporary file. */
    lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
    for (i = 0; i < h->entry_count; i++)
    {
        struct temp_file_entry entry;
            
        if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
            sizeof(struct temp_file_entry))
        {
            logf("read fail #1");
            return false;
        }

        /* Insert data in buffer. */
        databuf[i] = (long)entry.tag_offset[index_type];

        /* Skip to next. */
        lseek(tmpfd, entry.data_length, SEEK_CUR);
    }
    
    /* Update the entries in index. */
    masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);

    if (masterfd < 0)
    {
        logf("No master file found!");
        return false;
    }
    
    if (read(masterfd, &tch, sizeof(struct tagcache_header)) !=
        sizeof(struct tagcache_header) || tch.magic != TAGCACHE_MAGIC)
    {
        logf("header error");
        close(masterfd);
        return false;
    }

    masterfd_pos = lseek(masterfd, tch.entry_count * sizeof(struct index_entry),
                         SEEK_CUR);
    if (masterfd_pos == filesize(masterfd))
    {
        logf("we can't append!");
        close(masterfd);
        return false;
    }

    for (i = 0; i < h->entry_count; i++)
    {
        int loc = lseek(masterfd, 0, SEEK_CUR);
        
        if (read(masterfd, &idx, sizeof(struct index_entry)) !=
            sizeof(struct index_entry))
        {
            logf("read fail #2");
            close(masterfd);
            return false;
        }
        
        idx.tag_seek[index_type] = databuf[i];
        
        /* Write back the updated index. */
        lseek(masterfd, loc, SEEK_SET);
        if (write(masterfd, &idx, sizeof(struct index_entry)) !=
            sizeof(struct index_entry))
        {
            logf("write fail");
            close(masterfd);
            return false;
        }
    }
    
    close(masterfd);
    
    return true;
}

/**
 * Return values:
 *     > 0   success
 *    == 0   temporary failure
 *     < 0   fatal error
 */
static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
{
    int i;
    struct tagcache_header tch;
    struct index_entry idxbuf[IDX_BUF_DEPTH];
    int idxbuf_pos;
    char buf[MAX_PATH];
    int fd = -1, masterfd;
    bool error = false;
    int init;
    int masterfd_pos;
    
    logf("Building index: %d", index_type);
    
    tempbufidx = 0;
    tempbuf_pos = TAGFILE_MAX_ENTRIES * sizeof(struct tempbuf_searchidx);
    tempbuf_pos += LOOKUP_BUF_DEPTH * sizeof(void **);
    tempbuf_left = tempbuf_size - tempbuf_pos - 8;
    if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * TAGFILE_MAX_ENTRIES < 0)
    {
        logf("Buffer way too small!");
        return 0;
    }

    lookup = (struct tempbuf_searchidx **)
        (tempbuf + sizeof(struct tempbuf_searchidx)*TAGFILE_MAX_ENTRIES);
    memset(lookup, 0, LOOKUP_BUF_DEPTH * sizeof(void **));
    
    /* Open the index file, which contains the tag names. */
    snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
    fd = open(buf, O_RDWR);

    if (fd >= 0)
    {
        /* Read the header. */
        if (read(fd, &tch, sizeof(struct tagcache_header)) !=
            sizeof(struct tagcache_header) || tch.magic != TAGCACHE_MAGIC)
        {
            logf("header error");
            close(fd);
            return -2;
        }

        /**
         * If tag file contains unique tags (sorted index), we will load
         * it entirely into memory so we can resort it later for use with
         * chunked browsing.
         */
        if (tagcache_is_sorted_tag(index_type))
        {
            logf("loading tags...");
            for (i = 0; i < tch.entry_count; i++)
            {
                struct tagfile_entry entry;
                int loc = lseek(fd, 0, SEEK_CUR);
                
                if (read(fd, &entry, sizeof(struct tagfile_entry))
                    != sizeof(struct tagfile_entry))
                {
                    logf("read error");
                    close(fd);
                    return -2;
                }
                
                if (entry.tag_length >= (int)sizeof(buf))
                {
                    logf("too long tag #3");
                    close(fd);
                    return -2;
                }
                
                if (read(fd, buf, entry.tag_length) != entry.tag_length)
                {
                    logf("read error #2");
                    close(fd);
                    return -2;
                }

                /* Skip deleted entries. */
                if (buf[0] == '\0')
                    continue;
                
                /**
                 * Save the tag and tag id in the memory buffer. Tag id
                 * is saved so we can later reindex the master lookup
                 * table when the index gets resorted.
                 */
                tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH 
                               + TAGFILE_MAX_ENTRIES, entry.idx_id,
                               tagcache_is_unique_tag(index_type));
                yield();
            }
            logf("done");
        }
        else
            tempbufidx = tch.entry_count;
    }
    else
    {
        /**
         * Creating new index file to store the tags. No need to preload
         * anything whether the index type is sorted or not.
         */
        fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
        if (fd < 0)
        {
            logf("%s open fail", buf);
            return -2;
        }
        
        tch.magic = TAGCACHE_MAGIC;
        tch.entry_count = 0;
        tch.datasize = 0;
        
        if (write(fd, &tch, sizeof(struct tagcache_header)) !=
            sizeof(struct tagcache_header))
        {
            logf("header write failed");
            close(fd);
            return -2;
        }
    }

    /* Loading the tag lookup file as "master file". */
    logf("Loading index file");
    masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);

    if (masterfd < 0)
    {
        logf("Creating new index");
        masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);

        if (masterfd < 0)
        {
            logf("Failure to create index file");
            close(fd);
            return -2;
        }

        /* Write the header (write real values later). */
        tch = *h;
        tch.entry_count = 0;
        tch.datasize = 0;
        write(masterfd, &tch, sizeof(struct tagcache_header));
        init = true;
        masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
    }
    else
    {
        /**
         * Master file already exists so we need to process the current
         * file first.
         */
        init = false;

        if (read(masterfd, &tch, sizeof(struct tagcache_header)) !=
            sizeof(struct tagcache_header) || tch.magic != TAGCACHE_MAGIC)
        {
            logf("header error");
            close(fd);
            close(masterfd);
            return -2;
        }

        /**
         * If we reach end of the master file, we need to expand it to
         * hold new tags. If the current index is not sorted, we can
         * simply append new data to end of the file.
         * However, if the index is sorted, we need to update all tag
         * pointers in the master file for the current index.
         */
        masterfd_pos = lseek(masterfd, tch.entry_count * sizeof(struct index_entry),
            SEEK_CUR);
        if (masterfd_pos == filesize(masterfd))
        {
            logf("appending...");
            init = true;
        }
    }

    /**
     * Load new unique tags in memory to be sorted later and added
     * to the master lookup file.
     */
    if (tagcache_is_sorted_tag(index_type))
    {
        lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
        /* h is the header of the temporary file containing new tags. */
        logf("inserting new tags...");
        for (i = 0; i < h->entry_count; i++)
        {
            struct temp_file_entry entry;
            
            if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
                sizeof(struct temp_file_entry))
            {
                logf("read fail #1");
                error = true;
                goto error_exit;
            }

            /* Read data. */
            if (entry.tag_length[index_type] >= (long)sizeof(buf))
            {
                logf("too long entry!");
                error = true;
                goto error_exit;
            }

            lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
            if (read(tmpfd, buf, entry.tag_length[index_type]) !=
                entry.tag_length[index_type])
            {
                logf("read fail #3");
                error = true;
                goto error_exit;
            }
            
            if (tagcache_is_unique_tag(index_type))
                error = !tempbuf_insert(buf, i, -1, true);
            else
                error = !tempbuf_insert(buf, i, tch.entry_count + i, false);
            
            if (error)
            {
                logf("insert error");
                goto error_exit;
            }
            
            /* Skip to next. */
            lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
                    entry.tag_length[index_type], SEEK_CUR);
            yield();
        }
        logf("done");

        /* Sort the buffer data and write it to the index file. */
        lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
        i = tempbuf_sort(fd);
        if (i < 0)
            goto error_exit;
        logf("sorted %d tags", i);
        
        /**
         * Now update all indexes in the master lookup file.
         */
        logf("updating indices...");
        lseek(masterfd, sizeof(struct tagcache_header), SEEK_SET);
        for (i = 0; i < tch.entry_count; i += idxbuf_pos)
        {
            int j;
            int loc = lseek(masterfd, 0, SEEK_CUR);
            
            idxbuf_pos = MIN(tch.entry_count - i, IDX_BUF_DEPTH);
            
            if (read(masterfd, idxbuf, sizeof(struct index_entry)*idxbuf_pos) !=
                (int)sizeof(struct index_entry)*idxbuf_pos)
            {
                logf("read fail #2");
                error = true;
                goto error_exit ;
            }
            lseek(masterfd, loc, SEEK_SET);
            
            for (j = 0; j < idxbuf_pos; j++)
            {
                if (idxbuf[j].flag & FLAG_DELETED)
                {
                    int k;
                    
                    idxbuf_pos--;
                    for (k = j; k < idxbuf_pos; k++)
                        idxbuf[k] = idxbuf[k+1];
                    
                    j--;
                    continue;
                }
                
                idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
                    idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
                    + TAGFILE_MAX_ENTRIES);
                
                if (idxbuf[j].tag_seek[index_type] < 0)
                {
                    logf("update error: %d/%d", i+j, tch.entry_count);
                    error = true;
                    goto error_exit;
                }
                
                yield();
            }
            
            /* Write back the updated index. */
            if (write(masterfd, idxbuf, sizeof(struct index_entry)*idxbuf_pos) !=
                (int)sizeof(struct index_entry)*idxbuf_pos)
            {
                logf("write fail");
                error = true;
                goto error_exit;
            }
        }
        logf("done");
    }

    /**
     * Walk through the temporary file containing the new tags.
     */
    // build_normal_index(h, tmpfd, masterfd, idx);
    logf("updating new indices...");
    lseek(masterfd, masterfd_pos, SEEK_SET);
    lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
    lseek(fd, 0, SEEK_END);
    for (i = 0; i < h->entry_count; i += idxbuf_pos)
    {
        int j;
        
        idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
        if (init)
        {
            memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
        }
        else
        {
            int loc = lseek(masterfd, 0, SEEK_CUR);
            
            if (read(masterfd, idxbuf, sizeof(struct index_entry)*idxbuf_pos) !=
                (int)sizeof(struct index_entry)*idxbuf_pos)
            {
                logf("read fail #2");
                error = true;
                break ;
            }
            lseek(masterfd, loc, SEEK_SET);
        }

        /* Read entry headers. */
        for (j = 0; j < idxbuf_pos; j++)
        {
            if (!tagcache_is_sorted_tag(index_type))
            {
                struct temp_file_entry entry;
                struct tagfile_entry fe;
                
                if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
                    sizeof(struct temp_file_entry))
                {
                    logf("read fail #1");
                    error = true;
                    break ;
                }
                
                /* Read data. */
                if (entry.tag_length[index_type] >= (int)sizeof(buf))
                {
                    logf("too long entry!");
                    logf("length=%d", entry.tag_length[index_type]);
                    logf("pos=0x%02x", lseek(tmpfd, 0, SEEK_CUR));
                    error = true;
                    break ;
                }
                
                lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
                if (read(tmpfd, buf, entry.tag_length[index_type]) !=
                    entry.tag_length[index_type])
                {
                    logf("read fail #3");
                    logf("offset=0x%02x", entry.tag_offset[index_type]);
                    logf("length=0x%02x", entry.tag_length[index_type]);
                    error = true;
                    break ;
                }

                /* Write to index file. */
                idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
                fe.tag_length = entry.tag_length[index_type];
                fe.idx_id = tch.entry_count + i + j;
                write(fd, &fe, sizeof(struct tagfile_entry));
                write(fd, buf, fe.tag_length);
                tempbufidx++;

                /* Skip to next. */
                lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
                      entry.tag_length[index_type], SEEK_CUR);
            }
            else
            {
                /* Locate the correct entry from the sorted array. */
                idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
                if (idxbuf[j].tag_seek[index_type] < 0)
                {
                    logf("entry not found (%d)");
                    error = true;
                    break ;
                }
            }
        }

        /* Write index. */
        if (write(masterfd, idxbuf, sizeof(struct index_entry)*idxbuf_pos) !=
            (int)sizeof(struct index_entry)*idxbuf_pos)
        {
            logf("tagcache: write fail #4");
            error = true;
            break ;
        }
        
        yield();
    }
    logf("done");
    
    /* Finally write the header. */
    tch.magic = TAGCACHE_MAGIC;
    tch.entry_count = tempbufidx;
    tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
    lseek(fd, 0, SEEK_SET);
    write(fd, &tch, sizeof(struct tagcache_header));
    
    if (index_type != tag_filename)
        h->datasize += tch.datasize;
    logf("s:%d/%d/%d", index_type, tch.datasize, h->datasize);
    error_exit:
    
    close(fd);
    close(masterfd);

    if (error)
        return -2;
    
    return 1;
}

static bool commit(void)
{
    struct tagcache_header header, header_old;
    int i, len, rc;
    int tmpfd;
    int masterfd;
#ifdef HAVE_DIRCACHE
    bool dircache_buffer_stolen = false;
#endif
    bool local_allocation = false;
    
    logf("committing tagcache");
    
    tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
    if (tmpfd < 0)
    {
        logf("nothing to commit");
        return true;
    }
    
    /* Load the header. */
    len = sizeof(struct tagcache_header);
    rc = read(tmpfd, &header, len);
    
    if (header.magic != TAGCACHE_MAGIC || rc != len)
    {
        logf("incorrect header");
        close(tmpfd);
        remove_files();
        remove(TAGCACHE_FILE_TEMP);
        return false;
    }

    if (header.entry_count == 0)
    {
        logf("nothing to commit");
        close(tmpfd);
        remove(TAGCACHE_FILE_TEMP);
        return true;
    }

    /* Try to steal every buffer we can :) */
    if (tempbuf_size == 0)
        local_allocation = true;
    
#ifdef HAVE_DIRCACHE
    if (tempbuf_size == 0)
    {
        /* Try to steal the dircache buffer. */
        tempbuf = dircache_steal_buffer(&tempbuf_size);
        tempbuf_size &= ~0x03;
        
        if (tempbuf_size > 0)
        {
            dircache_buffer_stolen = true;
            stat.ramcache = false;
        }
    }
#endif    
    
#ifdef HAVE_TC_RAMCACHE
    if (tempbuf_size == 0 && stat.ramcache_allocated > 0)
    {
        stat.ramcache = false;
        tempbuf = (char *)(hdr + 1);
        tempbuf_size = stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
        tempbuf_size &= ~0x03;
    }
#endif
    
    /* And finally fail if there are no buffers available. */
    if (tempbuf_size == 0)
    {
        logf("delaying commit until next boot");
        stat.commit_delayed = true;
        close(tmpfd);
        return false;
    }
    
    logf("commit %d entries...", header.entry_count);
    
    /* Now create the index files. */
    stat.commit_step = 0;
    header.datasize = 0;
    stat.commit_delayed = false;
    
    for (i = 0; i < TAG_COUNT; i++)
    {
        int ret;
        
        stat.commit_step++;
        if (tagcache_is_numeric_tag(i))
        {
            build_numeric_index(i, &header, tmpfd);
            continue;
        }
        
        ret = build_index(i, &header, tmpfd);
        if (ret <= 0)
        {
            close(tmpfd);
            logf("tagcache failed init");
            if (ret < 0)
                remove_files();
            else
                stat.commit_delayed = true;
            stat.commit_step = 0;
            return false;
        }
    }
    
    close(tmpfd);
    stat.commit_step = 0;
    
    /* Update the master index headers. */
    masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
    if (masterfd < 0)
    {
        logf("failed to open master index");
        return false;
    }

    if (read(masterfd, &header_old, sizeof(struct tagcache_header))
        != sizeof(struct tagcache_header) ||
        header_old.magic != TAGCACHE_MAGIC)
    {
        logf("incorrect header");
        close(masterfd);
        remove_files();
        return false;
    }

    header.entry_count += header_old.entry_count;
    /* Datasize has been recalculated. */
    // header.datasize += header_old.datasize;

    lseek(masterfd, 0, SEEK_SET);
    write(masterfd, &header, sizeof(struct tagcache_header));
    close(masterfd);
    
    logf("tagcache committed");
    remove(TAGCACHE_FILE_TEMP);
    
    if (local_allocation)
    {
        tempbuf = NULL;
        tempbuf_size = 0;
    }
    
#ifdef HAVE_DIRCACHE
    /* Rebuild the dircache, if we stole the buffer. */
    if (dircache_buffer_stolen)
        dircache_build(0);
#endif

#ifdef HAVE_TC_RAMCACHE
    /* Reload tagcache. */
    if (stat.ramcache_allocated > 0 && !stat.ramcache)
        tagcache_start_scan();
#endif
    
    return true;
}

static void allocate_tempbuf(void)
{
    /* Yeah, malloc would be really nice now :) */
    tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
    tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
    audiobuf += tempbuf_size;
}

static void free_tempbuf(void)
{
    if (tempbuf_size == 0)
        return ;
    
    audiobuf -= tempbuf_size;
    tempbuf = NULL;
    tempbuf_size = 0;
}

static int open_master_fd(struct tagcache_header *hdr)
{
    int fd;
    
    fd = open(TAGCACHE_FILE_MASTER, O_RDWR);
    if (fd < 0)
    {
        logf("master file open failed for R/W");
        return fd;
    }
    
    /* Check the header. */
    read(fd, hdr, sizeof(struct tagcache_header));
    if (hdr->magic != TAGCACHE_MAGIC)
    {
        logf("header error");
        close(fd);
        return -2;
    }
    
    return fd;
}

static bool write_tag(int fd, const char *tagstr, const char *datastr)
{
    char buf[256];
    int i;
    
    snprintf(buf, sizeof buf, "%s=\"", tagstr);
    for (i = strlen(buf); i < (long)sizeof(buf)-2; i++)
    {
        if (*datastr == '\0')
            break;
        
        if (*datastr == '"')
        {
            buf[i] = '\\';
            datastr++;
            continue;
        }
        
        buf[i] = *(datastr++);
    }
    
    strcpy(&buf[i], "\" ");
    
    write(fd, buf, i + 2);
    
    return true;
}

bool tagcache_create_changelog(struct tagcache_search *tcs)
{
    static const char *tags_str[] = { "artist", "album", "genre", "title", 
        "filename", "playcount", "playtime", "lastplayed" };
    static const int  tags[] = { tag_artist, tag_album, tag_genre, tag_title, 
        tag_filename, tag_playcount, tag_playtime, tag_lastplayed };
    struct tagcache_header myhdr;
    struct index_entry idx;
    char buf[256];
    char temp[32];
    int clfd;
    int i, j;
    
    if (!tagcache_search(tcs, tag_filename))
        return false;
    
    /* Initialize the changelog */
    clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
    if (clfd < 0)
    {
        logf("failure to open changelog");
        return false;
    }
    
    if (tcs->masterfd < 0)
    {
        if ( (tcs->masterfd = open_master_fd(&myhdr)) < 0)
            return false;
    }
    else
    {
        lseek(tcs->masterfd, 0, SEEK_SET);
        read(tcs->masterfd, &myhdr, sizeof(struct tagcache_header));
    }
    
    write(clfd, "## Changelog version 1\n", 23);
    
    for (i = 0; i < myhdr.entry_count; i++)
    {
        if (read(tcs->masterfd, &idx, sizeof(struct index_entry)) 
            != sizeof(struct index_entry))
        {
            logf("read error");
            tagcache_search_finish(tcs);
            close(clfd);
            return false;
        }
        
        /* Skip until the entry found has been modified. */
        if (! (idx.flag & FLAG_DIRTYNUM) )
            continue;
        
        logf("Found!");
        
        /* Now retrieve all tags. */
        for (j = 0; j < (long)(sizeof(tags) / sizeof(tags[0])); j++)
        {
            if (tagcache_is_numeric_tag(tags[j]))
            {
                snprintf(temp, sizeof temp, "%d", idx.tag_seek[tags[j]]);
                write_tag(clfd, tags_str[j], temp);
                continue;
            }
            
            tcs->type = tags[j];
            tagcache_retrieve(tcs, i, buf, sizeof buf);
            logf("tag: %s", buf);
            write_tag(clfd, tags_str[j], buf);
        }
        
        write(clfd, "\n", 1);
    }
    
    close(clfd);
    
    tagcache_search_finish(tcs);
    
    return true;
}

bool tagcache_modify_numeric_entry(struct tagcache_search *tcs, 
                                   int tag, long data)
{
    struct index_entry idx;
    struct tagcache_header myhdr;
    
    if (!tagcache_is_numeric_tag(tag))
        return false;
    
#ifdef HAVE_TC_RAMCACHE
    /* Update ram entries first. */
    if (tcs->ramsearch)
    {
        hdr->indices[tcs->idx_id].tag_seek[tag] = data;
        hdr->indices[tcs->idx_id].flag |= FLAG_DIRTYNUM;
    }
#endif
    
    /* And now update the db on disk also. */
    if (tcs->masterfd < 0)
    {
        if ( (tcs->masterfd = open_master_fd(&myhdr)) < 0)
            return false;
    }
    
    lseek(tcs->masterfd, tcs->idx_id * sizeof(struct index_entry)
          + sizeof(struct tagcache_header), SEEK_SET);
    if (read(tcs->masterfd, &idx, sizeof(struct index_entry)) 
        != sizeof(struct index_entry))
    {
        logf("read error");
        return false;
    }
    
    idx.flag |= FLAG_DIRTYNUM;
    idx.tag_seek[tag] = data;
    
    lseek(tcs->masterfd, -sizeof(struct index_entry), SEEK_CUR);
    if (write(tcs->masterfd, &idx, sizeof(struct index_entry))
        != sizeof(struct index_entry))
    {
        logf("write error");
        return false;
    }
    
    return true;
}

static bool delete_entry(long idx_id)
{
    int fd;
    int tag, i;
    struct index_entry idx, myidx;
    struct tagcache_header myhdr;
    char buf[MAX_PATH];
    int in_use[TAG_COUNT];
    
#ifdef HAVE_TC_RAMCACHE
    /* At first mark the entry removed from ram cache. */
    if (hdr)
        hdr->indices[idx_id].flag |= FLAG_DELETED;
#endif
    
    if ( (fd = open_master_fd(&myhdr) < 0) )
        return false;
    
    lseek(fd, idx_id * sizeof(struct index_entry), SEEK_CUR);
    if (read(fd, &myidx, sizeof(struct index_entry)) 
        != sizeof(struct index_entry))
    {
        logf("read error");
        close(fd);
        return false;
    }
    
    myidx.flag |= FLAG_DELETED;
    lseek(fd, -sizeof(struct index_entry), SEEK_CUR);
    if (write(fd, &myidx, sizeof(struct index_entry))
        != sizeof(struct index_entry))
    {
        logf("write error");
        close(fd);
        return false;
    }
    
    /* Now check which tags are no longer in use (if any) */
    for (tag = 0; tag < TAG_COUNT; tag++)
        in_use[tag] = 0;
    
    lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
    for (i = 0; i < myhdr.entry_count; i++)
    {
        if (read(fd, &idx, sizeof(struct index_entry)) 
            != sizeof(struct index_entry))
        {
            logf("read error");
            close(fd);
            return false;
        }
        
        if (idx.flag & FLAG_DELETED)
            continue;
        
        for (tag = 0; tag < TAG_COUNT; tag++)
        {
            if (tagcache_is_numeric_tag(tag))
                continue;
            
            if (idx.tag_seek[tag] == myidx.tag_seek[tag])
                in_use[tag]++;
        }
    }
    
    close(fd);
    
    /* Now delete all tags no longer in use. */
    for (tag = 0; tag < TAG_COUNT; tag++)
    {
        if (tagcache_is_numeric_tag(tag))
            continue;
        
        if (in_use[tag])
        {
            logf("in use: %d/%d", tag, in_use[tag]);
            continue;
        }
        
        /* Open the index file, which contains the tag names. */
        snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
        fd = open(buf, O_RDWR);
        
        if (fd < 0)
        {
            logf("open failed");
            return false;
        }
        
        /* Skip the header block */
        lseek(fd, myidx.tag_seek[tag] + sizeof(struct tagfile_entry), SEEK_SET);
        
        read(fd, buf, 10);
        buf[10]='\0';
        logf("TAG:%s", buf);
        lseek(fd, -10, SEEK_CUR);
        
        /* Write first data byte in tag as \0 */
        write(fd, "", 1);
    
        /* Now tag data has been removed */
        close(fd);
    }
    
    return true;
}

#ifdef HAVE_TC_RAMCACHE
static bool allocate_tagcache(void)
{
    int rc, len;
    int fd;

    hdr = NULL;
    
    fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
    if (fd < 0)
    {
        logf("no tagcache file found.");
        return false;
    }

    /* Load the header. */
    hdr = (struct ramcache_header *)(((long)audiobuf & ~0x03) + 0x04);
    memset(hdr, 0, sizeof(struct ramcache_header));
    len = sizeof(struct tagcache_header);
    rc = read(fd, &hdr->h, len);
    close(fd);
    
    if (hdr->h.magic != TAGCACHE_MAGIC || rc != len)
    {
        logf("incorrect header");
        remove_files();
        hdr = NULL;
        return false;
    }

    hdr->indices = (struct index_entry *)(hdr + 1);
    
    /** 
     * Now calculate the required cache size plus 
     * some extra space for alignment fixes. 
     */
    stat.ramcache_allocated = hdr->h.datasize + 128 + TAGCACHE_RESERVE +
        sizeof(struct index_entry) * hdr->h.entry_count +
        sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
    logf("tagcache: %d bytes allocated.", stat.ramcache_allocated);
    logf("at: 0x%04x", audiobuf);
    audiobuf += (long)((stat.ramcache_allocated & ~0x03) + 128);

    return true;
}

static bool load_tagcache(void)
{
    struct tagcache_header *tch;
    long bytesleft = stat.ramcache_allocated;
    struct index_entry *idx;
    int rc, fd;
    char *p;
    int i, tag;

    logf("loading tagcache to ram...");
    
# ifdef HAVE_DIRCACHE
    while (dircache_is_initializing())
        sleep(1);
# endif
    
    fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
    if (fd < 0)
    {
        logf("tagcache open failed");
        return false;
    }
    
    if (read(fd, &hdr->h, sizeof(struct tagcache_header))
        != sizeof(struct tagcache_header)
        || hdr->h.magic != TAGCACHE_MAGIC)
    {
        logf("incorrect header");
        return false;
    }

    lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
    
    idx = hdr->indices;

    /* Load the master index table. */
    for (i = 0; i < hdr->h.entry_count; i++)
    {
        rc = read(fd, idx, sizeof(struct index_entry));
        if (rc != sizeof(struct index_entry))
        {
            logf("read error #1");
            close(fd);
            return false;
        }
    
        bytesleft -= sizeof(struct index_entry);
        if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= stat.ramcache_allocated)
        {
            logf("too big tagcache.");
            close(fd);
            return false;
        }

        idx++;
    }

    close(fd);

    /* Load the tags. */
    p = (char *)idx;
    for (tag = 0; tag < TAG_COUNT; tag++)
    {
        struct tagfile_entry *fe;
        char buf[MAX_PATH];

        if (tagcache_is_numeric_tag(tag))
            continue ;
        
        //p = ((void *)p+1);
        p = (char *)((long)p & ~0x03) + 0x04;
        hdr->tags[tag] = p;

        snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
        fd = open(buf, O_RDONLY);
        
        if (fd < 0)
        {
            logf("%s open fail", buf);
            return false;
        }

        /* Check the header. */
        tch = (struct tagcache_header *)p;
        rc = read(fd, tch, sizeof(struct tagcache_header));
        p += rc;
        if (rc != sizeof(struct tagcache_header) ||
            tch->magic != TAGCACHE_MAGIC)
        {
            logf("incorrect header");
            close(fd);
            return false;
        }
        
        for (hdr->entry_count[tag] = 0;
             hdr->entry_count[tag] < tch->entry_count;
             hdr->entry_count[tag]++)
        {
            long pos;
            
            yield();
            fe = (struct tagfile_entry *)p;
            pos = lseek(fd, 0, SEEK_CUR);
            rc = read(fd, fe, sizeof(struct tagfile_entry));
            if (rc != sizeof(struct tagfile_entry))
            {
                /* End of lookup table. */
                logf("read error");
                close(fd);
                return false;
            }

            /* We have a special handling for the filename tags. */
            if (tag == tag_filename)
            {
# ifdef HAVE_DIRCACHE
                const struct dircache_entry *dc;
# endif
                
                // FIXME: This is wrong!
                // idx = &hdr->indices[hdr->entry_count[i]];
                idx = &hdr->indices[fe->idx_id];
                
                if (fe->tag_length >= (long)sizeof(buf)-1)
                {
                    read(fd, buf, 10);
                    buf[10] = '\0';
                    logf("TAG:%s", buf);
                    logf("too long filename");
                    close(fd);
                    return false;
                }
                
                rc = read(fd, buf, fe->tag_length);
                if (rc != fe->tag_length)
                {
                    logf("read error #3");
                    close(fd);
                    return false;
                }
                
                /* Check if the entry has already been removed */
                if (idx->flag & FLAG_DELETED)
                    continue;
                    
                /* This flag must not be used yet. */
                if (idx->flag & FLAG_DIRCACHE)
                {
                    logf("internal error!");
                    close(fd);
                    return false;
                }
                
                if (idx->tag_seek[tag] != pos)
                {
                    logf("corrupt data structures!");
                    close(fd);
                    return false;
                }

# ifdef HAVE_DIRCACHE
                if (dircache_is_enabled())
                {
                    dc = dircache_get_entry_ptr(buf);
                    if (dc == NULL)
                    {
                        logf("Entry no longer valid.");
                        logf("-> %s", buf);
                        delete_entry(fe->idx_id);
                        continue ;
                    }

                    idx->flag |= FLAG_DIRCACHE;
                    idx->tag_seek[tag_filename] = (long)dc;
                }
                else
# endif
                {
                    /* Check if entry has been removed. */
                    if (global_settings.tagcache_autoupdate)
                    {
                        int testfd;
                        
                        testfd = open(buf, O_RDONLY);
                        if (testfd < 0)
                        {
                            logf("Entry no longer valid.");
                            logf("-> %s", buf);
                            delete_entry(fe->idx_id);
                            continue;
                        }
                        close(testfd);
                    }
                }
                
                continue ;
            }
            
            bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
            if (bytesleft < 0)
            {
                logf("too big tagcache #2");
                logf("tl: %d", fe->tag_length);
                logf("bl: %d", bytesleft);
                close(fd);
                return false;
            }

            p = fe->tag_data;
            rc = read(fd, fe->tag_data, fe->tag_length);
            p += rc;

            if (rc != fe->tag_length)
            {
                logf("read error #4");
                logf("rc=0x%04x", rc); // 0x431
                logf("len=0x%04x", fe->tag_length); // 0x4000
                logf("pos=0x%04x", lseek(fd, 0, SEEK_CUR)); // 0x433
                logf("tag=0x%02x", tag); // 0x00
                close(fd);
                return false;
            }
        }
        close(fd);
    }
    
    stat.ramcache_used = stat.ramcache_allocated - bytesleft;
    logf("tagcache loaded into ram!");

    return true;
}
#endif /* HAVE_TC_RAMCACHE */

static bool check_deleted_files(void)
{
    int fd, testfd;
    char buf[MAX_PATH];
    struct tagfile_entry tfe;
    
    logf("reverse scan...");
    snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
    fd = open(buf, O_RDONLY);
    
    if (fd < 0)
    {
        logf("%s open fail", buf);
        return false;
    }

    lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
    while (read(fd, &tfe, sizeof(struct tagfile_entry))
           == sizeof(struct tagfile_entry) && queue_empty(&tagcache_queue))
    {
        if (tfe.tag_length >= (long)sizeof(buf)-1)
        {
            logf("too long tag");
            close(fd);
            return false;
        }
        
        if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
        {
            logf("read error");
            close(fd);
            return false;
        }
        
        /* Now check if the file exists. */
        testfd = open(buf, O_RDONLY);
        if (testfd < 0)
        {
            logf("Entry no longer valid.");
            logf("-> %s", buf);
            delete_entry(tfe.idx_id);
        }
        close(testfd);
    }
    
    close(fd);
    
    logf("done");
    
    return true;
}

static bool check_dir(const char *dirname)
{
    DIRCACHED *dir;
    int len;
    int success = false;

    dir = opendir_cached(dirname);
    if (!dir)
    {
        logf("tagcache: opendir_cached() failed");
        return false;
    }
    
    /* Recursively scan the dir. */
    while (queue_empty(&tagcache_queue))
    {
        struct dircache_entry *entry;

        entry = readdir_cached(dir);
    
        if (entry == NULL)
        {
            success = true;
            break ;
        }
        
        if (!strcmp(entry->d_name, ".") ||
            !strcmp(entry->d_name, ".."))
            continue;

        yield();
        
        len = strlen(curpath);
        snprintf(&curpath[len], curpath_size - len, "/%s",
                 entry->d_name);
        
        processed_dir_count++;
        if (entry->attribute & ATTR_DIRECTORY)
            check_dir(curpath);
        else
#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
            add_tagcache(curpath, dir->internal_entry);
#else
            add_tagcache(curpath);
#endif

        curpath[len] = '\0';
    }
    
    closedir_cached(dir);

    return success;
}

static void build_tagcache(void)
{
    struct tagcache_header header;
    bool ret;
    char buf[MAX_PATH];

    curpath[0] = '\0';
    data_size = 0;
    total_entry_count = 0;
    processed_dir_count = 0;
    
    logf("updating tagcache");
    
    cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
    if (cachefd >= 0)
    {
        logf("skipping, cache already waiting for commit");
        close(cachefd);
        return ;
    }
    
    cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
    if (cachefd < 0)
    {
        logf("master file open failed");
        return ;
    }

    snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
    filenametag_fd = open(buf, O_RDONLY);

    if (filenametag_fd >= 0)
    {
        if (read(filenametag_fd, &header, sizeof(struct tagcache_header)) !=
            sizeof(struct tagcache_header) || header.magic != TAGCACHE_MAGIC)
        {
            logf("header error");
            close(filenametag_fd);
            filenametag_fd = -1;
        }
    }

            
    cpu_boost(true);

    /* Scan for new files. */
    memset(&header, 0, sizeof(struct tagcache_header));
    write(cachefd, &header, sizeof(struct tagcache_header));

    //strcpy(curpath, "/Best");
    ret = check_dir("/");
    
    /* Write the header. */
    header.magic = TAGCACHE_MAGIC;
    header.datasize = data_size;
    header.entry_count = total_entry_count;
    lseek(cachefd, 0, SEEK_SET);
    write(cachefd, &header, sizeof(struct tagcache_header));
    close(cachefd);

    if (filenametag_fd >= 0)
    {
        close(filenametag_fd);
        filenametag_fd = -1;
    }

    if (!ret)
    {
        logf("Aborted.");
        cpu_boost(false);
        return ;
    }

    /* Commit changes to the database. */
    if (commit())
    {
        remove(TAGCACHE_FILE_TEMP);
        logf("tagcache built!");
    }
    
    cpu_boost(false);
}

#ifdef HAVE_TC_RAMCACHE
static void load_ramcache(void)
{
    if (!hdr)
        return ;
        
    cpu_boost(true);
    
    /* At first we should load the cache (if exists). */
    stat.ramcache = load_tagcache();

    if (!stat.ramcache)
    {
        hdr = NULL;
        remove_files();
    }
    
    cpu_boost(false);
}
#endif

static void tagcache_thread(void)
{
    struct event ev;
    bool check_done = false;

    /* If the previous cache build/update was interrupted, commit
     * the changes first in foreground. */
    cpu_boost(true);
    allocate_tempbuf();
    commit();
    free_tempbuf();
    
#ifdef HAVE_TC_RAMCACHE
    /* Allocate space for the tagcache if found on disk. */
    if (global_settings.tagcache_ram)
        allocate_tagcache();
#endif
    
    cpu_boost(false);
    
    stat.initialized = true;
    
    while (1)
    {
        queue_wait_w_tmo(&tagcache_queue, &ev, HZ);

        switch (ev.id)
        {
            case Q_START_SCAN:
                check_done = false;
                break ;

            case Q_REBUILD:
                remove_files();
                build_tagcache();
                break;
            
            case Q_UPDATE:
                build_tagcache();
                check_deleted_files();
                break ;
                
            case SYS_TIMEOUT:
                if (check_done)
                    break ;
                
#ifdef HAVE_TC_RAMCACHE
                if (!stat.ramcache && global_settings.tagcache_ram)
                {
                    load_ramcache();
                    if (stat.ramcache)
                        build_tagcache();
                }
                else
#endif
                if (global_settings.tagcache_autoupdate)
                {
                    build_tagcache();
                    /* Don't do auto removal without dircache (very slow). */
#ifdef HAVE_DIRCACHE
                    if (dircache_is_enabled())
                        check_deleted_files();
#endif
                }
                
                check_done = true;
                break ;

            case Q_STOP_SCAN:
                break ;
                
            case SYS_POWEROFF:
                break ;
                
#ifndef SIMULATOR
            case SYS_USB_CONNECTED:
                logf("USB: TagCache");
                usb_acknowledge(SYS_USB_CONNECTED_ACK);
                usb_wait_for_disconnect(&tagcache_queue);
                break ;
#endif
        }
    }
}

static int get_progress(void)
{
    int total_count = -1;

#ifdef HAVE_DIRCACHE
    if (dircache_is_enabled())
    {
        total_count = dircache_get_entry_count();
    }
    else
#endif
#ifdef HAVE_TC_RAMCACHE
    {
        if (hdr && stat.ramcache)
            total_count = hdr->h.entry_count;
    }
#endif

    if (total_count < 0)
        return -1;
    
    return processed_dir_count * 100 / total_count;
}

struct tagcache_stat* tagcache_get_stat(void)
{
    stat.progress = get_progress();
    stat.processed_entries = processed_dir_count;
    
    return &stat;
}
    
void tagcache_start_scan(void)
{
    queue_post(&tagcache_queue, Q_START_SCAN, 0);
}

bool tagcache_update(void)
{
    queue_post(&tagcache_queue, Q_UPDATE, 0);
    gui_syncsplash(HZ*2, true, str(LANG_TAGCACHE_FORCE_UPDATE_SPLASH));
    
    return false;
}

bool tagcache_rebuild(void)
{
    queue_post(&tagcache_queue, Q_REBUILD, 0);
    gui_syncsplash(HZ*2, true, str(LANG_TAGCACHE_FORCE_UPDATE_SPLASH));
    
    return false;
}

void tagcache_stop_scan(void)
{
    queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
}

#ifdef HAVE_TC_RAMCACHE
bool tagcache_is_ramcache(void)
{
    return stat.ramcache;
}
#endif


void tagcache_init(void)
{
    stat.initialized = false;
    stat.commit_step = 0;
    filenametag_fd = -1;
    
    queue_init(&tagcache_queue);
    create_thread(tagcache_thread, tagcache_stack,
                  sizeof(tagcache_stack), tagcache_thread_name);
}

bool tagcache_is_initialized(void)
{
    return stat.initialized;
}
    
int tagcache_get_commit_step(void)
{
    return stat.commit_step;
}