Nyaaori/rockbox
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Much faster optimized version of tagcache commit. Added a few items to

Browse source 
the debug menu (still buggy) and fixed a problem with partial tagcache
updates.


git-svn-id: svn://svn.rockbox.org/rockbox/trunk@9775 a1c6a512-1295-4272-9138-f99709370657
This commit is contained in:
Miika Pekkarinen 2006-04-23 18:47:26 +00:00
parent 3f7292e13a
commit 4142710a47
3 changed files with 252 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
apps/debug_menu.c
View file

@ -1883,6 +1883,7 @@ static bool dbg_tagcache_info(void)
bool done = false; bool done = false;
int line; int line;
char buf[32]; char buf[32];
struct tagcache_stat *stat;
lcd_setmargins(0, 0); lcd_setmargins(0, 0);
lcd_setfont(FONT_SYSFIXED); lcd_setfont(FONT_SYSFIXED);
@ -1892,8 +1893,21 @@ static bool dbg_tagcache_info(void)
line = 0; line = 0;
lcd_clear_display(); lcd_clear_display();
snprintf(buf, sizeof(buf), "Current progress: %d%%", stat = tagcache_get_stat();
tagcache_get_progress()); snprintf(buf, sizeof(buf), "Busy: %s", stat->initialized ? "No" : "Yes");
lcd_puts(0, line++, buf);
snprintf(buf, sizeof(buf), "RAM Cache: %s", stat->ramcache ? "Yes" : "No");
lcd_puts(0, line++, buf);
snprintf(buf, sizeof(buf), "RAM: %d/%d B",
stat->ramcache_used, stat->ramcache_allocated);
lcd_puts(0, line++, buf);
snprintf(buf, sizeof(buf), "Progress: %d%% (%d entries)",
stat->progress, stat->processed_entries);
lcd_puts(0, line++, buf);
snprintf(buf, sizeof(buf), "Commit step: %d", stat->commit_step);
lcd_puts(0, line++, buf);
snprintf(buf, sizeof(buf), "Commit delayed: %s",
stat->commit_delayed ? "Yes" : "No");
lcd_puts(0, line++, buf); lcd_puts(0, line++, buf);
lcd_update(); lcd_update();

405
apps/tagcache.c
View file

@ -60,11 +60,8 @@ static const int unique_tags[] = { tag_artist, tag_album, tag_genre, tag_compose
/* Numeric tags (we can use these tags with conditional clauses). */ /* Numeric tags (we can use these tags with conditional clauses). */
static const int numeric_tags[] = { tag_year, tag_tracknumber, tag_length, tag_bitrate }; static const int numeric_tags[] = { tag_year, tag_tracknumber, tag_length, tag_bitrate };
/* When thread initialization and memory allocation has been made. */ /* Status information of the tagcache. */
static bool tagcache_init_done = false; static struct tagcache_stat stat;
/* Progress indicator while committing the cache. */
static int init_step;
/* Queue commands. */ /* Queue commands. */
enum tagcache_queue { enum tagcache_queue {
@ -106,8 +103,6 @@ struct ramcache_header {
}; };
static struct ramcache_header *hdr; static struct ramcache_header *hdr;
static bool ramcache = false;
static long tagcache_size = 0;
#endif #endif
/** /**
@ -120,18 +115,22 @@ struct temp_file_entry {
long data_length; long data_length;
}; };
struct tempbuf_id { struct tempbuf_id_list {
int id; long id;
struct tempbuf_id *next; struct tempbuf_id_list *next;
}; };
struct tempbuf_searchidx { struct tempbuf_searchidx {
struct tempbuf_id *id;
long idx_id; long idx_id;
char *str; char *str;
int seek; int seek;
struct tempbuf_id_list idlist;
}; };
#define LOOKUP_BUF_DEPTH (TAGFILE_MAX_ENTRIES*2 \
* (TAGFILE_ENTRY_AVG_LENGTH/TAGFILE_ENTRY_CHUNK_LENGTH))
struct tempbuf_searchidx **lookup;
/* Used when building the temporary file. */ /* Used when building the temporary file. */
static int cachefd = -1, filenametag_fd; static int cachefd = -1, filenametag_fd;
@ -187,7 +186,7 @@ static struct index_entry *find_entry_ram(const char *filename,
int i; int i;
/* Check if we tagcache is loaded into ram. */ /* Check if we tagcache is loaded into ram. */
if (!ramcache) if (!stat.ramcache)
return NULL; return NULL;
if (dc == NULL) if (dc == NULL)
@ -579,7 +578,7 @@ bool tagcache_search(struct tagcache_search *tcs, int tag)
tagcache_search_finish(tcs); tagcache_search_finish(tcs);
memset(tcs, 0, sizeof(struct tagcache_search)); memset(tcs, 0, sizeof(struct tagcache_search));
if (tagcache_get_commit_step() > 0) if (stat.commit_step > 0)
return false; return false;
tcs->position = sizeof(struct tagcache_header); tcs->position = sizeof(struct tagcache_header);
@ -597,7 +596,7 @@ bool tagcache_search(struct tagcache_search *tcs, int tag)
#ifndef HAVE_TC_RAMCACHE #ifndef HAVE_TC_RAMCACHE
tcs->ramsearch = false; tcs->ramsearch = false;
#else #else
tcs->ramsearch = ramcache; tcs->ramsearch = stat.ramcache;
if (tcs->ramsearch) if (tcs->ramsearch)
{ {
tcs->entry_count = hdr->entry_count[tcs->type]; tcs->entry_count = hdr->entry_count[tcs->type];
@ -951,7 +950,7 @@ bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
/* Find the corresponding entry in tagcache. */ /* Find the corresponding entry in tagcache. */
entry = find_entry_ram(filename, NULL); entry = find_entry_ram(filename, NULL);
if (entry == NULL || !ramcache) if (entry == NULL || !stat.ramcache)
return false; return false;
id3->title = get_tag(entry, tag_title)->tag_data; id3->title = get_tag(entry, tag_title)->tag_data;
@ -1008,7 +1007,7 @@ static void add_tagcache(const char *path)
/* Check if the file is already cached. */ /* Check if the file is already cached. */
#ifdef HAVE_TC_RAMCACHE #ifdef HAVE_TC_RAMCACHE
if (ramcache) if (stat.ramcache)
{ {
if (find_entry_ram(path, dc)) if (find_entry_ram(path, dc))
return ; return ;
@ -1115,33 +1114,58 @@ static void remove_files(void)
} }
} }
static bool tempbuf_insert(char *str, int id, int idx_id) static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
{ {
struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
int len = strlen(str)+1; int len = strlen(str)+1;
int i;
unsigned crc32;
unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
char buf[MAX_PATH];
for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
buf[i] = tolower(str[i]);
buf[i] = '\0';
crc32 = crc_32(buf, i, 0xffffffff);
if (unique)
{
/* Check if the crc does not exist -> entry does not exist for sure. */
for (i = 0; i < tempbufidx; i++)
{
if (crcbuf[-i] != crc32)
continue;
if (!strcasecmp(str, index[i].str))
{
if (id >= 0 && id < LOOKUP_BUF_DEPTH)
lookup[id] = &index[i];
return true;
}
}
}
/* Insert to CRC buffer. */
crcbuf[-tempbufidx] = crc32;
tempbuf_left -= 4;
/* Insert it to the buffer. */ /* Insert it to the buffer. */
tempbuf_left -= len + sizeof(struct tempbuf_id); tempbuf_left -= len;
if (tempbuf_left - 4 < 0 || tempbufidx >= TAGFILE_MAX_ENTRIES-1) if (tempbuf_left - 4 < 0 || tempbufidx >= TAGFILE_MAX_ENTRIES-1)
return false; return false;
index[tempbufidx].id = (struct tempbuf_id *)&tempbuf[tempbuf_pos]; if (id >= 0 && id < LOOKUP_BUF_DEPTH)
#ifdef TAGCACHE_STRICT_ALIGN
/* Make sure the entry is long aligned. */
if ((long)index[tempbufidx].id & 0x03)
{ {
int fix = 4 - ((long)index[tempbufidx].id & 0x03); lookup[id] = &index[tempbufidx];
tempbuf_left -= fix; index[tempbufidx].idlist.id = id;
tempbuf_pos += fix;
index[tempbufidx].id = (struct tempbuf_id *)((
(long)index[tempbufidx].id & ~0x03) + 0x04);
} }
#endif else
index[tempbufidx].id->id = id; index[tempbufidx].idlist.id = -1;
index[tempbufidx].id->next = NULL;
index[tempbufidx].idx_id = idx_id;
tempbuf_pos += sizeof(struct tempbuf_id);
index[tempbufidx].idlist.next = NULL;
index[tempbufidx].idx_id = idx_id;
index[tempbufidx].seek = -1; index[tempbufidx].seek = -1;
index[tempbufidx].str = &tempbuf[tempbuf_pos]; index[tempbufidx].str = &tempbuf[tempbuf_pos];
memcpy(index[tempbufidx].str, str, len); memcpy(index[tempbufidx].str, str, len);
@ -1151,59 +1175,6 @@ static bool tempbuf_insert(char *str, int id, int idx_id)
return true; return true;
} }
static bool tempbuf_unique_insert(char *str, int id)
{
struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
struct tempbuf_id *idp;
int i;
unsigned crc32;
unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
crc32 = crc_32(str, strlen(str), 0xffffffff);
/* Check if the crc does not exist -> entry does not exist for sure. */
for (i = 0; i < tempbufidx; i++)
{
if (*(crcbuf--) == crc32)
{
if (!strcasecmp(str, index[i].str))
{
tempbuf_left -= sizeof(struct tempbuf_id);
if (tempbuf_left - 4 < 0)
return false;
idp = index[i].id;
while (idp->next != NULL)
idp = idp->next;
idp->next = (struct tempbuf_id *)&tempbuf[tempbuf_pos];
#if TAGCACHE_STRICT_ALIGN
/* Make sure the entry is long aligned. */
if ((long)idp->next & 0x03)
{
int fix = 4 - ((long)idp->next & 0x03);
tempbuf_left -= fix;
tempbuf_pos += fix;
idp->next = (struct tempbuf_id *)
(((long)idp->next & ~0x03) + 0x04);
}
#endif
idp = idp->next;
idp->id = id;
idp->next = NULL;
tempbuf_pos += sizeof(struct tempbuf_id);
return true;
}
}
}
/* Insert and quit. */
*crcbuf = crc32;
tempbuf_left -= 4;
return tempbuf_insert(str, id, -1);
}
static int compare(const void *p1, const void *p2) static int compare(const void *p1, const void *p2)
{ {
struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1; struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
@ -1225,19 +1196,69 @@ static int tempbuf_sort(int fd)
struct tagfile_entry fe; struct tagfile_entry fe;
int i; int i;
int length; int length;
#ifdef TAGCACHE_STRICT_ALIGN
int fix; /* Generate reverse lookup entries. */
#endif for (i = 0; i < LOOKUP_BUF_DEPTH; i++)
{
struct tempbuf_id_list *idlist;
if (!lookup[i])
continue;
if (lookup[i]->idlist.id == i)
continue;
idlist = &lookup[i]->idlist;
while (idlist->next != NULL)
idlist = idlist->next;
tempbuf_left -= sizeof(struct tempbuf_id_list);
if (tempbuf_left - 4 < 0)
return -1;
idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
if (tempbuf_pos & 0x03)
{
tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
tempbuf_left -= 3;
idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
}
tempbuf_pos += sizeof(struct tempbuf_id_list);
idlist = idlist->next;
idlist->id = i;
idlist->next = NULL;
}
qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare); qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
memset(lookup, 0, LOOKUP_BUF_DEPTH * sizeof(struct tempbuf_searchidx **));
for (i = 0; i < tempbufidx; i++) for (i = 0; i < tempbufidx; i++)
{ {
struct tempbuf_id_list *idlist = &index[i].idlist;
/* Fix the lookup list. */
while (idlist != NULL)
{
if (idlist->id >= 0)
lookup[idlist->id] = &index[i];
idlist = idlist->next;
}
index[i].seek = lseek(fd, 0, SEEK_CUR); index[i].seek = lseek(fd, 0, SEEK_CUR);
length = strlen(index[i].str) + 1; length = strlen(index[i].str) + 1;
fe.tag_length = length; fe.tag_length = length;
fe.idx_id = index[i].idx_id; fe.idx_id = index[i].idx_id;
/* Check the chunk alignment. */
if ((fe.tag_length + sizeof(struct tagfile_entry))
% TAGFILE_ENTRY_CHUNK_LENGTH)
{
fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
((fe.tag_length + sizeof(struct tagfile_entry))
% TAGFILE_ENTRY_CHUNK_LENGTH);
}
#ifdef TAGCACHE_STRICT_ALIGN #ifdef TAGCACHE_STRICT_ALIGN
/* Make sure the entry is long aligned. */ /* Make sure the entry is long aligned. */
if (index[i].seek & 0x03) if (index[i].seek & 0x03)
@ -1245,12 +1266,6 @@ static int tempbuf_sort(int fd)
logf("tempbuf_sort: alignment error!"); logf("tempbuf_sort: alignment error!");
return -3; return -3;
} }
fix = (sizeof(struct tagfile_entry) + length) & 0x03;
if (fix)
fix = 4-fix;
fe.tag_length += fix;
#endif #endif
if (write(fd, &fe, sizeof(struct tagfile_entry)) != if (write(fd, &fe, sizeof(struct tagfile_entry)) !=
@ -1266,11 +1281,9 @@ static int tempbuf_sort(int fd)
return -2; return -2;
} }
#ifdef TAGCACHE_STRICT_ALIGN
/* Write some padding. */ /* Write some padding. */
if (fix) if (fe.tag_length - length > 0)
write(fd, "XXX", fix); write(fd, "XXXXXXXX", fe.tag_length - length);
#endif
} }
return i; return i;
@ -1278,33 +1291,10 @@ static int tempbuf_sort(int fd)
inline static struct tempbuf_searchidx* tempbuf_locate(int id) inline static struct tempbuf_searchidx* tempbuf_locate(int id)
{ {
struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; if (id < 0 || id >= LOOKUP_BUF_DEPTH)
struct tempbuf_id *idp; return NULL;
static int last_id = 0;
int i;
try_again: return lookup[id];
if (last_id >= tempbufidx)
last_id = 0;
/* Check if string already exists. */
/* FIXME: This check is extremely slow, O(n^2) */
for (i = last_id; i < tempbufidx; i++)
{
idp = index[i].id;
while (idp != NULL)
{
if (idp->id == id)
return &index[i];
idp = idp->next;
}
}
if (last_id)
goto try_again;
return NULL;
} }
@ -1415,7 +1405,13 @@ static bool build_numeric_index(int index_type, struct tagcache_header *h, int t
return true; return true;
} }
static bool build_index(int index_type, struct tagcache_header *h, int tmpfd) /**
* Return values:
* > 0 success
* == 0 temporary failure
* < 0 fatal error
*/
static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
{ {
int i; int i;
struct tagcache_header tch; struct tagcache_header tch;
@ -1431,13 +1427,18 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
tempbufidx = 0; tempbufidx = 0;
tempbuf_pos = TAGFILE_MAX_ENTRIES * sizeof(struct tempbuf_searchidx); tempbuf_pos = TAGFILE_MAX_ENTRIES * sizeof(struct tempbuf_searchidx);
tempbuf_left = tempbuf_size - tempbuf_pos; memset(tempbuf+tempbuf_pos, 0, LOOKUP_BUF_DEPTH * sizeof(void **));
if (tempbuf_left < 0) tempbuf_pos += LOOKUP_BUF_DEPTH * sizeof(void **);
tempbuf_left = tempbuf_size - tempbuf_pos - 8;
if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * TAGFILE_MAX_ENTRIES < 0)
{ {
logf("Buffer way too small!"); logf("Buffer way too small!");
return false; return 0;
} }
lookup = (struct tempbuf_searchidx **)
(tempbuf + sizeof(struct tempbuf_searchidx)*TAGFILE_MAX_ENTRIES);
/* Open the index file, which contains the tag names. */ /* Open the index file, which contains the tag names. */
snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type); snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
fd = open(buf, O_RDWR); fd = open(buf, O_RDWR);
@ -1450,7 +1451,7 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
{ {
logf("header error"); logf("header error");
close(fd); close(fd);
return false; return -2;
} }
/** /**
@ -1471,21 +1472,21 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
{ {
logf("read error"); logf("read error");
close(fd); close(fd);
return false; return -2;
} }
if (entry.tag_length >= (int)sizeof(buf)) if (entry.tag_length >= (int)sizeof(buf))
{ {
logf("too long tag"); logf("too long tag");
close(fd); close(fd);
return false; return -2;
} }
if (read(fd, buf, entry.tag_length) != entry.tag_length) if (read(fd, buf, entry.tag_length) != entry.tag_length)
{ {
logf("read error #2"); logf("read error #2");
close(fd); close(fd);
return false; return -2;
} }
/** /**
@ -1493,7 +1494,8 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
* is saved so we can later reindex the master lookup * is saved so we can later reindex the master lookup
* table when the index gets resorted. * table when the index gets resorted.
*/ */
tempbuf_insert(buf, loc + TAGFILE_MAX_ENTRIES, entry.idx_id); tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
+ TAGFILE_MAX_ENTRIES, entry.idx_id, false);
yield(); yield();
} }
logf("done"); logf("done");
@ -1511,7 +1513,7 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
if (fd < 0) if (fd < 0)
{ {
logf("%s open fail", buf); logf("%s open fail", buf);
return false; return -2;
} }
tch.magic = TAGCACHE_MAGIC; tch.magic = TAGCACHE_MAGIC;
@ -1523,7 +1525,7 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
{ {
logf("header write failed"); logf("header write failed");
close(fd); close(fd);
return false; return -2;
} }
} }
@ -1540,7 +1542,7 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
{ {
logf("Failure to create index file"); logf("Failure to create index file");
close(fd); close(fd);
return false; return -2;
} }
/* Write the header (write real values later). */ /* Write the header (write real values later). */
@ -1565,7 +1567,7 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
logf("header error"); logf("header error");
close(fd); close(fd);
close(masterfd); close(masterfd);
return false; return -2;
} }
/** /**
@ -1623,9 +1625,9 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
} }
if (tagcache_is_unique_tag(index_type)) if (tagcache_is_unique_tag(index_type))
error = !tempbuf_unique_insert(buf, i); error = !tempbuf_insert(buf, i, -1, true);
else else
error = !tempbuf_insert(buf, i, tch.entry_count + i); error = !tempbuf_insert(buf, i, tch.entry_count + i, false);
if (error) if (error)
{ {
@ -1671,7 +1673,8 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
for (j = 0; j < idxbuf_pos; j++) for (j = 0; j < idxbuf_pos; j++)
{ {
idxbuf[j].tag_seek[index_type] = tempbuf_find_location( idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
idxbuf[j].tag_seek[index_type]+TAGFILE_MAX_ENTRIES); idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
+ TAGFILE_MAX_ENTRIES);
if (idxbuf[j].tag_seek[index_type] < 0) if (idxbuf[j].tag_seek[index_type] < 0)
{ {
@ -1679,6 +1682,7 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
error = true; error = true;
goto error_exit; goto error_exit;
} }
yield(); yield();
} }
@ -1800,30 +1804,24 @@ static bool build_index(int index_type, struct tagcache_header *h, int tmpfd)
} }
logf("done"); logf("done");
/* Finally write the uniqued tag index file. */ /* Finally write the header. */
if (tagcache_is_sorted_tag(index_type)) tch.magic = TAGCACHE_MAGIC;
{ tch.entry_count = tempbufidx;
tch.magic = TAGCACHE_MAGIC; tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
tch.entry_count = tempbufidx; lseek(fd, 0, SEEK_SET);
tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header); write(fd, &tch, sizeof(struct tagcache_header));
lseek(fd, 0, SEEK_SET);
write(fd, &tch, sizeof(struct tagcache_header));
}
else
{
tch.magic = TAGCACHE_MAGIC;
tch.entry_count = tempbufidx;
tch.datasize = lseek(fd, 0, SEEK_CUR) - sizeof(struct tagcache_header);
lseek(fd, 0, SEEK_SET);
write(fd, &tch, sizeof(struct tagcache_header));
}
if (index_type != tag_filename)
h->datasize += tch.datasize;
error_exit: error_exit:
close(fd); close(fd);
close(masterfd); close(masterfd);
return !error; if (error)
return -2;
return 1;
} }
static bool commit(void) static bool commit(void)
@ -1865,11 +1863,11 @@ static bool commit(void)
/* Try to steal every buffer we can :) */ /* Try to steal every buffer we can :) */
#ifdef HAVE_TC_RAMCACHE #ifdef HAVE_TC_RAMCACHE
if (tempbuf_size == 0 && tagcache_size > 0) if (tempbuf_size == 0 && stat.ramcache_allocated > 0)
{ {
ramcache = false; stat.ramcache = false;
tempbuf = (char *)(hdr + 1); tempbuf = (char *)(hdr + 1);
tempbuf_size = tagcache_size - sizeof(struct ramcache_header); tempbuf_size = stat.ramcache_allocated - sizeof(struct ramcache_header);
} }
#endif #endif
@ -1885,6 +1883,7 @@ static bool commit(void)
if (tempbuf_size == 0) if (tempbuf_size == 0)
{ {
logf("delaying commit until next boot"); logf("delaying commit until next boot");
stat.commit_delayed = true;
close(tmpfd); close(tmpfd);
return false; return false;
} }
@ -1892,26 +1891,36 @@ static bool commit(void)
logf("commit %d entries...", header.entry_count); logf("commit %d entries...", header.entry_count);
/* Now create the index files. */ /* Now create the index files. */
init_step = 0; stat.commit_step = 0;
header.datasize = 0;
stat.commit_delayed = false;
for (i = 0; i < TAG_COUNT; i++) for (i = 0; i < TAG_COUNT; i++)
{ {
init_step++; int ret;
stat.commit_step++;
if (tagcache_is_numeric_tag(i)) if (tagcache_is_numeric_tag(i))
{ {
build_numeric_index(i, &header, tmpfd); build_numeric_index(i, &header, tmpfd);
continue;
} }
else if (!build_index(i, &header, tmpfd)) ret = build_index(i, &header, tmpfd);
if (ret <= 0)
{ {
logf("tagcache failed init"); logf("tagcache failed init");
remove_files(); if (ret < 0)
init_step = 0; remove_files();
else
stat.commit_delayed = true;
stat.commit_step = 0;
return false; return false;
} }
} }
close(tmpfd); close(tmpfd);
init_step = 0; stat.commit_step = 0;
/* Update the master index headers. */ /* Update the master index headers. */
masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR); masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
@ -1932,7 +1941,8 @@ static bool commit(void)
} }
header.entry_count += header_old.entry_count; header.entry_count += header_old.entry_count;
header.datasize += header_old.datasize; /* Datasize has been recalculated. */
// header.datasize += header_old.datasize;
lseek(masterfd, 0, SEEK_SET); lseek(masterfd, 0, SEEK_SET);
write(masterfd, &header, sizeof(struct tagcache_header)); write(masterfd, &header, sizeof(struct tagcache_header));
@ -1948,7 +1958,7 @@ static bool commit(void)
#ifdef HAVE_TC_RAMCACHE #ifdef HAVE_TC_RAMCACHE
/* Reload tagcache. */ /* Reload tagcache. */
if (tagcache_size > 0 && !ramcache) if (stat.ramcache_allocated > 0 && !stat.ramcache)
tagcache_start_scan(); tagcache_start_scan();
#endif #endif
@ -2009,12 +2019,12 @@ static bool allocate_tagcache(void)
* Now calculate the required cache size plus * Now calculate the required cache size plus
* some extra space for alignment fixes. * some extra space for alignment fixes.
*/ */
tagcache_size = hdr->h.datasize + 128 + TAGCACHE_RESERVE + stat.ramcache_allocated = hdr->h.datasize + 128 + TAGCACHE_RESERVE +
sizeof(struct index_entry) * hdr->h.entry_count + sizeof(struct index_entry) * hdr->h.entry_count +
sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *); sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
logf("tagcache: %d bytes allocated.", tagcache_size); logf("tagcache: %d bytes allocated.", stat.ramcache_allocated);
logf("at: 0x%04x", audiobuf); logf("at: 0x%04x", audiobuf);
audiobuf += (long)((tagcache_size & ~0x03) + 0x04); audiobuf += (long)((stat.ramcache_allocated & ~0x03) + 0x04);
return true; return true;
} }
@ -2022,7 +2032,7 @@ static bool allocate_tagcache(void)
static bool load_tagcache(void) static bool load_tagcache(void)
{ {
struct tagcache_header *tch; struct tagcache_header *tch;
long bytesleft = tagcache_size; long bytesleft = stat.ramcache_allocated;
struct index_entry *idx; struct index_entry *idx;
int rc, fd; int rc, fd;
char *p; char *p;
@ -2065,7 +2075,7 @@ static bool load_tagcache(void)
} }
bytesleft -= sizeof(struct index_entry); bytesleft -= sizeof(struct index_entry);
if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tagcache_size) if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= stat.ramcache_allocated)
{ {
logf("too big tagcache."); logf("too big tagcache.");
close(fd); close(fd);
@ -2189,6 +2199,7 @@ static bool load_tagcache(void)
close(fd); close(fd);
} }
stat.ramcache_used = stat.ramcache_allocated - bytesleft;
logf("tagcache loaded into ram!"); logf("tagcache loaded into ram!");
return true; return true;
@ -2341,9 +2352,9 @@ static void load_ramcache(void)
cpu_boost(true); cpu_boost(true);
/* At first we should load the cache (if exists). */ /* At first we should load the cache (if exists). */
ramcache = load_tagcache(); stat.ramcache = load_tagcache();
if (!ramcache) if (!stat.ramcache)
{ {
hdr = NULL; hdr = NULL;
remove_files(); remove_files();
@ -2364,9 +2375,16 @@ static void tagcache_thread(void)
allocate_tempbuf(); allocate_tempbuf();
commit(); commit();
free_tempbuf(); free_tempbuf();
#ifdef HAVE_TC_RAMCACHE
/* Allocate space for the tagcache if found on disk. */
if (global_settings.tagcache_ram)
allocate_tagcache();
#endif
cpu_boost(false); cpu_boost(false);
tagcache_init_done = true; stat.initialized = true;
while (1) while (1)
{ {
@ -2388,10 +2406,10 @@ static void tagcache_thread(void)
if (check_done || !dircache_is_enabled()) if (check_done || !dircache_is_enabled())
break ; break ;
if (!ramcache && global_settings.tagcache_ram) if (!stat.ramcache && global_settings.tagcache_ram)
load_ramcache(); load_ramcache();
if (ramcache) if (stat.ramcache)
build_tagcache(); build_tagcache();
check_done = true; check_done = true;
@ -2415,7 +2433,7 @@ static void tagcache_thread(void)
} }
} }
int tagcache_get_progress(void) static int get_progress(void)
{ {
int total_count = -1; int total_count = -1;
@ -2426,7 +2444,7 @@ int tagcache_get_progress(void)
} }
else else
{ {
if (hdr && ramcache) if (hdr && stat.ramcache)
total_count = hdr->h.entry_count; total_count = hdr->h.entry_count;
} }
#endif #endif
@ -2437,9 +2455,12 @@ int tagcache_get_progress(void)
return processed_dir_count * 100 / total_count; return processed_dir_count * 100 / total_count;
} }
int tagcache_get_processes_entrycount(void) struct tagcache_stat* tagcache_get_stat(void)
{ {
return processed_dir_count; stat.progress = get_progress();
stat.processed_entries = processed_dir_count;
return &stat;
} }
void tagcache_start_scan(void) void tagcache_start_scan(void)
@ -2463,19 +2484,15 @@ void tagcache_stop_scan(void)
#ifdef HAVE_TC_RAMCACHE #ifdef HAVE_TC_RAMCACHE
bool tagcache_is_ramcache(void) bool tagcache_is_ramcache(void)
{ {
return ramcache; return stat.ramcache;
} }
#endif #endif
void tagcache_init(void) void tagcache_init(void)
{ {
tagcache_init_done = false; stat.initialized = false;
init_step = 0; stat.commit_step = 0;
#ifdef HAVE_TC_RAMCACHE
/* Allocate space for the tagcache if found on disk. */
allocate_tagcache();
#endif
queue_init(&tagcache_queue); queue_init(&tagcache_queue);
create_thread(tagcache_thread, tagcache_stack, create_thread(tagcache_thread, tagcache_stack,
@ -2484,11 +2501,11 @@ void tagcache_init(void)
bool tagcache_is_initialized(void) bool tagcache_is_initialized(void)
{ {
return tagcache_init_done; return stat.initialized;
} }
int tagcache_get_commit_step(void) int tagcache_get_commit_step(void)
{ {
return init_step; return stat.commit_step;
} }

29
apps/tagcache.h
View file

@ -38,13 +38,22 @@ enum tag_type { tag_artist = 0, tag_album, tag_genre, tag_title,
#define IDX_BUF_DEPTH 64 #define IDX_BUF_DEPTH 64
/* Tag Cache Header version 'TCHxx'. Increment when changing internal structures. */ /* Tag Cache Header version 'TCHxx'. Increment when changing internal structures. */
#define TAGCACHE_MAGIC 0x54434803 #define TAGCACHE_MAGIC 0x54434804
/* How much to allocate extra space for ramcache. */ /* How much to allocate extra space for ramcache. */
#define TAGCACHE_RESERVE 32768 #define TAGCACHE_RESERVE 32768
/* How many entries we can create in one tag file (for sorting). */ /* How many entries we can create in one tag file (for sorting). */
#define TAGFILE_MAX_ENTRIES 20000 #define TAGFILE_MAX_ENTRIES 10000
/**
* Define how long one entry must be at least (longer -> less memory at commit).
* Must be at least 4 bytes in length for correct alignment.
*/
#define TAGFILE_ENTRY_CHUNK_LENGTH 8
/* Used to guess the necessary buffer size at commit. */
#define TAGFILE_ENTRY_AVG_LENGTH 16
/* How many entries to fetch to the seek table at once while searching. */ /* How many entries to fetch to the seek table at once while searching. */
#define SEEK_LIST_SIZE 50 #define SEEK_LIST_SIZE 50
@ -67,6 +76,17 @@ enum clause { clause_none, clause_is, clause_gt, clause_gteq, clause_lt,
clause_lteq, clause_contains, clause_begins_with, clause_ends_with }; clause_lteq, clause_contains, clause_begins_with, clause_ends_with };
enum modifiers { clause_mod_none, clause_mod_not }; enum modifiers { clause_mod_none, clause_mod_not };
struct tagcache_stat {
bool initialized; /* Is tagcache currently busy? */
bool ramcache; /* Is tagcache loaded in ram? */
bool commit_delayed; /* Has commit been delayed until next reboot? */
int commit_step; /* Commit progress */
int ramcache_allocated; /* Has ram been allocated for ramcache? */
int ramcache_used; /* How much ram has been really used */
int progress; /* Current progress of disk scan */
int processed_entries; /* Scanned disk entries so far */
};
struct tagcache_search_clause struct tagcache_search_clause
{ {
int tag; int tag;
@ -116,14 +136,15 @@ bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
void tagcache_search_finish(struct tagcache_search *tcs); void tagcache_search_finish(struct tagcache_search *tcs);
long tagcache_get_numeric(const struct tagcache_search *tcs, int tag); long tagcache_get_numeric(const struct tagcache_search *tcs, int tag);
int tagcache_get_progress(void); struct tagcache_stat* tagcache_get_stat(void);
int tagcache_get_commit_step(void);
#ifdef HAVE_TC_RAMCACHE #ifdef HAVE_TC_RAMCACHE
bool tagcache_is_ramcache(void); bool tagcache_is_ramcache(void);
bool tagcache_fill_tags(struct mp3entry *id3, const char *filename); bool tagcache_fill_tags(struct mp3entry *id3, const char *filename);
#endif #endif
void tagcache_init(void); void tagcache_init(void);
bool tagcache_is_initialized(void); bool tagcache_is_initialized(void);
int tagcache_get_commit_step(void);
void tagcache_start_scan(void); void tagcache_start_scan(void);
void tagcache_stop_scan(void); void tagcache_stop_scan(void);
bool tagcache_force_update(void); bool tagcache_force_update(void);