From 16a9f84571276a13f4cfd5c66db8cd63ce4e2e7f Mon Sep 17 00:00:00 2001 From: Michael Sevakis Date: Tue, 17 Jan 2017 14:45:07 -0500 Subject: [PATCH] Reenable database ramcache and playlist dircache Playlist dircache references should be back in working order. Reenabling dircache references in the database ramcache is not yet done as it requires quite a bit of rework. Otherwise, the database in RAM is functional again. Some buffer compatibility changes have been made for database commit because the dircache buffer can no longer be stolen, only freed by an API call. Change-Id: Ib57c3e98cb23e798d4439e9da7ebd73826e733a4 --- apps/playlist.c | 160 ++++++++++++++++++++++----------------- apps/playlist.h | 4 +- apps/tagcache.c | 152 +++++++++++++++++++------------------ firmware/export/config.h | 2 +- 4 files changed, 172 insertions(+), 146 deletions(-) diff --git a/apps/playlist.c b/apps/playlist.c index 015e41ae0e..176cfc3af7 100644 --- a/apps/playlist.c +++ b/apps/playlist.c @@ -106,8 +106,9 @@ #include "plugin.h" /* To borrow a temp buffer to rewrite a .m3u8 file */ #include "panic.h" #include "logdiskf.h" - -#undef HAVE_DIRCACHE +#ifdef HAVE_DIRCACHE +#include "dircache.h" +#endif #define PLAYLIST_CONTROL_FILE_VERSION 2 @@ -205,6 +206,25 @@ static const char playlist_thread_name[] = "playlist cachectrl"; static struct mutex current_playlist_mutex SHAREDBSS_ATTR; static struct mutex created_playlist_mutex SHAREDBSS_ATTR; +#ifdef HAVE_DIRCACHE +static void copy_filerefs(struct dircache_fileref *dcfto, + const struct dircache_fileref *dcffrom, + int count) +{ + if (!dcfto) + return; + + if (dcffrom) + memmove(dcfto, dcffrom, count * sizeof (*dcfto)); + else + { + /* just initialize the destination */ + for (int i = 0; i < count; i++, dcfto++) + dircache_fileref_init(dcfto); + } +} +#endif /* HAVE_DIRCACHE */ + /* Check if the filename suggests M3U or M3U8 format. */ static bool is_m3u8(const char* filename) { @@ -401,6 +421,8 @@ static int recreate_control(struct playlist_info* playlist) int i; int result = 0; + temp_file[0] = 0; + if(playlist->control_fd >= 0) { char* dir = playlist->filename; @@ -408,6 +430,7 @@ static int recreate_control(struct playlist_info* playlist) char c = playlist->filename[playlist->dirlen-1]; close(playlist->control_fd); + playlist->control_fd = 0; snprintf(temp_file, sizeof(temp_file), "%s_temp", playlist->control_filename); @@ -571,10 +594,9 @@ static int add_indices_to_playlist(struct playlist_info* playlist, /* Store a new entry */ playlist->indices[ playlist->amount ] = i+count; -#ifdef HAVE_DIRCACHE - if (playlist->filenames) - playlist->filenames[ playlist->amount ] = -1; -#endif + #ifdef HAVE_DIRCACHE + copy_filerefs(&playlist->dcfrefs[playlist->amount], NULL, 1); + #endif playlist->amount++; } } @@ -779,8 +801,8 @@ static int add_track_to_playlist(struct playlist_info* playlist, { playlist->indices[i] = playlist->indices[i-1]; #ifdef HAVE_DIRCACHE - if (playlist->filenames) - playlist->filenames[i] = playlist->filenames[i-1]; + if (playlist->dcfrefs) + playlist->dcfrefs[i] = playlist->dcfrefs[i-1]; #endif } @@ -814,8 +836,7 @@ static int add_track_to_playlist(struct playlist_info* playlist, playlist->indices[insert_position] = flags | seek_pos; #ifdef HAVE_DIRCACHE - if (playlist->filenames) - playlist->filenames[insert_position] = -1; + copy_filerefs(&playlist->dcfrefs[insert_position], NULL, 1); #endif playlist->amount++; @@ -886,8 +907,8 @@ static int remove_track_from_playlist(struct playlist_info* playlist, { playlist->indices[i] = playlist->indices[i+1]; #ifdef HAVE_DIRCACHE - if (playlist->filenames) - playlist->filenames[i] = playlist->filenames[i+1]; + if (playlist->dcfrefs) + playlist->dcfrefs[i] = playlist->dcfrefs[i+1]; #endif } @@ -934,7 +955,6 @@ static int randomise_playlist(struct playlist_info* playlist, { int count; int candidate; - long store; unsigned int current = playlist->indices[playlist->index]; /* seed 0 is used to identify sorted playlist for resume purposes */ @@ -951,15 +971,15 @@ static int randomise_playlist(struct playlist_info* playlist, candidate = rand() % (count + 1); /* now swap the values at the 'count' and 'candidate' positions */ - store = playlist->indices[candidate]; + int indextmp = playlist->indices[candidate]; playlist->indices[candidate] = playlist->indices[count]; - playlist->indices[count] = store; + playlist->indices[count] = indextmp; #ifdef HAVE_DIRCACHE - if (playlist->filenames) + if (playlist->dcfrefs) { - store = playlist->filenames[candidate]; - playlist->filenames[candidate] = playlist->filenames[count]; - playlist->filenames[count] = store; + struct dircache_fileref dcftmp = playlist->dcfrefs[candidate]; + playlist->dcfrefs[candidate] = playlist->dcfrefs[count]; + playlist->dcfrefs[count] = dcftmp; } #endif } @@ -1001,7 +1021,7 @@ static int sort_playlist(struct playlist_info* playlist, bool start_current, /** We need to re-check the song names from disk because qsort can't * sort two arrays at once :/ * FIXME: Please implement a better way to do this. */ - memset((void*)playlist->filenames, 0xff, playlist->max_playlist_size * sizeof(int)); + copy_filerefs(playlist->dcfrefs, NULL, playlist->max_playlist_size); queue_post(&playlist_queue, PLAYLIST_LOAD_POINTERS, 0); #endif @@ -1245,11 +1265,6 @@ static void playlist_flush_callback(void) } } -static bool is_dircache_pointers_intact(void) -{ - return dircache_get_appflag(DIRCACHE_APPFLAG_PLAYLIST) ? true : false; -} - static void playlist_thread(void) { struct queue_event ev; @@ -1290,24 +1305,26 @@ static void playlist_thread(void) register_storage_idle_func(playlist_flush_callback); } - if (!dircache_is_enabled() || !playlist->filenames - || playlist->amount <= 0) - { + if (!playlist->dcfrefs || playlist->amount <= 0) break ; - } /* Check if previously loaded pointers are intact. */ - if (is_dircache_pointers_intact() && !dirty_pointers) + if (!dirty_pointers) break ; -#ifdef HAVE_ADJUSTABLE_CPU_FREQ - cpu_boost(true); -#endif + struct dircache_info info; + dircache_get_info(&info); + + if (info.status != DIRCACHE_READY) + break ; + + trigger_cpu_boost(); + for (index = 0; index < playlist->amount && queue_empty(&playlist_queue); index++) { - /* Process only pointers that are not already loaded. */ - if (is_dircache_pointers_intact() && playlist->filenames[index] >= 0) + /* Process only pointers that are superficially stale. */ + if (dircache_search(DCS_FILEREF, &playlist->dcfrefs[index], NULL) == 0) continue ; control_file = playlist->indices[index] & PLAYLIST_INSERT_TYPE_MASK; @@ -1320,19 +1337,16 @@ static void playlist_thread(void) break ; } - /* Set the dircache entry pointer. */ - playlist->filenames[index] = dircache_get_entry_id(tmp); + /* Obtain the dircache file entry cookie. */ + dircache_search(DCS_CACHED_PATH | DCS_UPDATE_FILEREF, + &playlist->dcfrefs[index], tmp); /* And be on background so user doesn't notice any delays. */ yield(); } - - if (dircache_is_enabled()) - dircache_set_appflag(DIRCACHE_APPFLAG_PLAYLIST); - -#ifdef HAVE_ADJUSTABLE_CPU_FREQ - cpu_boost(false); -#endif + + cancel_cpu_boost(); + if (index == playlist->amount) dirty_pointers = false; @@ -1364,17 +1378,12 @@ static int get_filename(struct playlist_info* playlist, int index, int seek, buf_length = MAX_PATH+1; #ifdef HAVE_DIRCACHE - if (is_dircache_pointers_intact() && playlist->filenames) + if (playlist->dcfrefs) { - if (playlist->filenames[index] >= 0) - { - max = dircache_copy_path(playlist->filenames[index], - tmp_buf, sizeof(tmp_buf)-1); - } + max = dircache_get_fileref_path(&playlist->dcfrefs[index], + tmp_buf, sizeof(tmp_buf)); } -#else - (void)index; -#endif +#endif /* HAVE_DIRCACHE */ if (playlist->in_ram && !control_file && max < 0) { @@ -1406,12 +1415,16 @@ static int get_filename(struct playlist_info* playlist, int index, int seek, { max = read(fd, tmp_buf, MIN((size_t) buf_length, sizeof(tmp_buf))); - if ((max > 0) && !utf8) + if (max > 0) { + /* playlist file may end without a new line - terminate buffer */ + tmp_buf[MIN(max, (int)sizeof(tmp_buf) - 1)] = '\0'; + /* Use dir_buf as a temporary buffer. Note that dir_buf must * be as large as tmp_buf. */ - max = convert_m3u(tmp_buf, max, sizeof(tmp_buf), dir_buf); + if (!utf8) + max = convert_m3u(tmp_buf, max, sizeof(tmp_buf), dir_buf); } } } @@ -1432,6 +1445,8 @@ static int get_filename(struct playlist_info* playlist, int index, int seek, strlcpy(dir_buf, playlist->filename, playlist->dirlen); return format_track_path(buf, tmp_buf, buf_length, dir_buf); + + (void)index; } static int get_next_directory(char *dir){ @@ -1974,11 +1989,12 @@ static int move_callback(int handle, void* current, void* new) struct playlist_info* playlist = ¤t_playlist; if (current == playlist->indices) playlist->indices = new; - else if (current == playlist->filenames) - playlist->filenames = new; else if (current == playlist->buffer) playlist->buffer = new; - +#ifdef HAVE_DIRCACHE + else if (current == playlist->dcfrefs) + playlist->dcfrefs = new; +#endif /* HAVE_DIRCACHE */ return BUFLIB_CB_OK; } @@ -2019,15 +2035,15 @@ void playlist_init(void) #ifdef HAVE_DIRCACHE handle = core_alloc_ex("playlist dc", - playlist->max_playlist_size * sizeof(int), &ops); - playlist->filenames = core_get_data(handle); - memset((void*)playlist->filenames, 0xff, - playlist->max_playlist_size * sizeof(int)); + playlist->max_playlist_size * sizeof(*playlist->dcfrefs), &ops); + playlist->dcfrefs = core_get_data(handle); + copy_filerefs(playlist->dcfrefs, NULL, playlist->max_playlist_size); create_thread(playlist_thread, playlist_stack, sizeof(playlist_stack), 0, playlist_thread_name IF_PRIO(, PRIORITY_BACKGROUND) IF_COP(, CPU)); + queue_init(&playlist_queue, true); -#endif +#endif /* HAVE_DIRCACHE */ } /* @@ -2513,8 +2529,9 @@ int playlist_add(const char *filename) playlist->indices[playlist->amount] = playlist->buffer_end_pos; #ifdef HAVE_DIRCACHE - playlist->filenames[playlist->amount] = -1; + copy_filerefs(&playlist->dcfrefs[playlist->amount], NULL, 1); #endif + playlist->amount++; strcpy((char*)&playlist->buffer[playlist->buffer_end_pos], filename); @@ -2863,18 +2880,19 @@ int playlist_create_ex(struct playlist_info* playlist, if (index_buffer) { - int num_indices = index_buffer_size / sizeof(int); - + size_t unit_size = sizeof (*playlist->indices); #ifdef HAVE_DIRCACHE - num_indices /= 2; + unit_size += sizeof (*playlist->dcfrefs); #endif + int num_indices = index_buffer_size / unit_size; + if (num_indices > global_settings.max_files_in_playlist) num_indices = global_settings.max_files_in_playlist; playlist->max_playlist_size = num_indices; playlist->indices = index_buffer; #ifdef HAVE_DIRCACHE - playlist->filenames = (int*)&playlist->indices[num_indices]; + playlist->dcfrefs = (void *)&playlist->indices[num_indices]; #endif } else @@ -2882,7 +2900,7 @@ int playlist_create_ex(struct playlist_info* playlist, playlist->max_playlist_size = current_playlist.max_playlist_size; playlist->indices = current_playlist.indices; #ifdef HAVE_DIRCACHE - playlist->filenames = current_playlist.filenames; + playlist->dcfrefs = current_playlist.dcfrefs; #endif } @@ -2942,8 +2960,8 @@ int playlist_set_current(struct playlist_info* playlist) memcpy((void*)current_playlist.indices, (void*)playlist->indices, playlist->max_playlist_size*sizeof(int)); #ifdef HAVE_DIRCACHE - memcpy((void*)current_playlist.filenames, (void*)playlist->filenames, - playlist->max_playlist_size*sizeof(int)); + copy_filerefs(current_playlist.dcfrefs, playlist->dcfrefs, + playlist->max_playlist_size); #endif } diff --git a/apps/playlist.h b/apps/playlist.h index a331838fde..6048001ff7 100644 --- a/apps/playlist.h +++ b/apps/playlist.h @@ -82,7 +82,9 @@ struct playlist_info bool control_created; /* has control file been created? */ int dirlen; /* Length of the path to the playlist file */ volatile unsigned long *indices; /* array of indices */ - volatile int *filenames; /* Array of dircache indices */ +#ifdef HAVE_DIRCACHE + struct dircache_fileref *dcfrefs; /* Dircache entry shortcuts */ +#endif int max_playlist_size; /* Max number of files in playlist. Mirror of global_settings.max_files_in_playlist */ bool in_ram; /* playlist stored in ram (dirplay) */ diff --git a/apps/tagcache.c b/apps/tagcache.c index 2b6041227b..df252ca0bc 100644 --- a/apps/tagcache.c +++ b/apps/tagcache.c @@ -90,6 +90,10 @@ #undef HAVE_DIRCACHE +#ifdef HAVE_DIRCACHE +#include "dircache.h" +#endif + #ifdef __PCTOOL__ #define yield() do { } while(0) #define sim_sleep(timeout) do { } while(0) @@ -112,6 +116,9 @@ static long tempbufidx; /* Current location in buffer. */ static size_t tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */ static long tempbuf_left; /* Buffer space left. */ static long tempbuf_pos; +#ifndef __PCTOOL__ +static int tempbuf_handle; +#endif #define SORTED_TAGS_COUNT 8 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS) @@ -289,17 +296,6 @@ static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf) return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ); } -#ifdef HAVE_DIRCACHE -/** - * Returns true if specified flag is still present, i.e., dircache - * has not been reloaded. - */ -static bool is_dircache_intact(void) -{ - return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE); -} -#endif - static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write) { int fd; @@ -434,7 +430,7 @@ static long find_entry_ram(const char *filename, int dc) return -1; } -#endif +#endif /* defined (HAVE_TC_RAMCACHE) && defined (HAVE_DIRCACHE) */ static long find_entry_disk(const char *filename_raw, bool localfd) { @@ -590,7 +586,7 @@ static bool get_index(int masterfd, int idxid, if (ramcache_hdr->indices[idxid].flag & FLAG_DELETED) return false; -# ifdef HAVE_DIRCACHE +#ifdef HAVE_DIRCACHE if (!(ramcache_hdr->indices[idxid].flag & FLAG_DIRCACHE) || is_dircache_intact()) #endif @@ -599,8 +595,6 @@ static bool get_index(int masterfd, int idxid, return true; } } -#else - (void)use_ram; #endif if (masterfd < 0) @@ -632,6 +626,8 @@ static bool get_index(int masterfd, int idxid, return false; return true; + + (void)use_ram; } #ifndef __PCTOOL__ @@ -724,7 +720,7 @@ static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx, { struct tagfile_entry *ep; -# ifdef HAVE_DIRCACHE +#ifdef HAVE_DIRCACHE if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)) { /* for tag_filename, seek is a dircache index */ @@ -745,7 +741,7 @@ static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx, } } else -# endif +#endif /* HAVE_DIRCACHE */ if (tag != tag_filename) { ep = (struct tagfile_entry *)&ramcache_hdr->tags[tag][seek]; @@ -1511,8 +1507,7 @@ static bool get_next(struct tagcache_search *tcs) #ifdef HAVE_TC_RAMCACHE if (tcs->ramsearch) { - -#if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) +#ifdef HAVE_DIRCACHE if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)) { if (is_dircache_intact()) @@ -1537,7 +1532,7 @@ static bool get_next(struct tagcache_search *tcs) } } else -#endif +#endif /* HAVE_DIRCACHE */ if (tcs->type != tag_filename) { struct tagfile_entry *ep; @@ -1766,6 +1761,13 @@ bool tagcache_fill_tags(struct mp3entry *id3, const char *filename) return true; } +#elif defined (HAVE_TC_RAMCACHE) +/* temporary dummy function until integration is sorted out --jethead71 */ +bool tagcache_fill_tags(struct mp3entry *id3, const char *filename) +{ + return false; + (void)id3; (void)filename; +} #endif static inline void write_item(const char *item) @@ -2982,27 +2984,25 @@ static bool commit(void) #ifdef HAVE_TC_RAMCACHE tc_stat.ramcache = false; #endif - + + /* Beyond here, jump to commit_error to undo locks and restore dircache */ + rc = false; read_lock++; /* Try to steal every buffer we can :) */ if (tempbuf_size == 0) local_allocation = true; -#if 0 /* FIXME: How much big? dircache buffer can no longer be taken but - may be freed to make room and the cache resumed. --jethead71 */ #ifdef HAVE_DIRCACHE if (tempbuf_size == 0) { - /* Shut down dircache to free its allocation. */ + /* Suspend dircache to free its allocation. */ dircache_free_buffer(); - if (tempbuf_size > 0) - { - dircache_buffer_stolen = true; - } + dircache_buffer_stolen = true; + + allocate_tempbuf(); } -#endif -#endif +#endif /* HAVE_DIRCACHE */ #ifdef HAVE_TC_RAMCACHE if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0) @@ -3021,8 +3021,7 @@ static bool commit(void) logf("delaying commit until next boot"); tc_stat.commit_delayed = true; close(tmpfd); - read_lock--; - return false; + goto commit_error; } logf("commit %ld entries...", tch.entry_count); @@ -3053,8 +3052,7 @@ static bool commit(void) tc_stat.commit_delayed = true; tc_stat.commit_step = 0; - read_lock--; - return false; + goto commit_error; } } @@ -3063,8 +3061,7 @@ static bool commit(void) logf("Failure to commit numeric indices"); close(tmpfd); tc_stat.commit_step = 0; - read_lock--; - return false; + goto commit_error; } close(tmpfd); @@ -3073,10 +3070,7 @@ static bool commit(void) /* Update the master index headers. */ if ( (masterfd = open_master_fd(&tcmh, true)) < 0) - { - read_lock--; - return false; - } + goto commit_error; remove(TAGCACHE_FILE_TEMP); @@ -3101,39 +3095,53 @@ static bool commit(void) tempbuf_size = 0; } -#ifdef HAVE_DIRCACHE - /* Rebuild the dircache, if we stole the buffer. */ - if (dircache_buffer_stolen) - dircache_resume(); -#endif - #ifdef HAVE_TC_RAMCACHE if (ramcache_buffer_stolen) + { + ramcache_buffer_stolen = false; move_lock--; + } + /* Reload tagcache. */ if (tc_stat.ramcache_allocated > 0) tagcache_start_scan(); #endif - read_lock--; - - return true; -} + rc = true; -#ifndef __PCTOOL__ -static int tempbuf_handle; +commit_error: +#ifdef HAVE_TC_RAMCACHE + if (ramcache_buffer_stolen) + move_lock--; #endif + read_lock--; + +#ifdef HAVE_DIRCACHE + /* Resume the dircache, if we stole the buffer. */ + if (dircache_buffer_stolen) + dircache_resume(); +#endif + + return rc; +} + static void allocate_tempbuf(void) { /* Yeah, malloc would be really nice now :) */ + size_t size; + tempbuf_size = 0; + #ifdef __PCTOOL__ - tempbuf_size = 32*1024*1024; - tempbuf = malloc(tempbuf_size); + size = 32*1024*1024; + tempbuf = malloc(size); #else - tempbuf_handle = core_alloc_maximum("tc tempbuf", &tempbuf_size, NULL); + tempbuf_handle = core_alloc_maximum("tc tempbuf", &size, NULL); tempbuf = core_get_data(tempbuf_handle); #endif + + if (tempbuf) + tempbuf_size = size; } static void free_tempbuf(void) @@ -4133,9 +4141,14 @@ static bool load_tagcache(void) goto failure; } -# ifdef HAVE_DIRCACHE - if (dircache_is_enabled()) + if (global_settings.tagcache_autoupdate) { + /* Check if entry has been removed. */ + #ifdef HAVE_DIRCACHE + /* This will be very slow unless dircache is enabled + or target is flash based. */ +################## + if (dircache_search( /* will get this! -- jethead71 */ dc = dircache_get_entry_id(buf); if (dc < 0) { @@ -4148,24 +4161,17 @@ static bool load_tagcache(void) idx->flag |= FLAG_DIRCACHE; idx->tag_seek[tag_filename] = dc; - } - else -# endif - { - /* This will be very slow unless dircache is enabled - or target is flash based, but do it anyway for - consistency. */ - /* Check if entry has been removed. */ - if (global_settings.tagcache_autoupdate) + #else /* ndef HAVE_DIRCACHE */ + /* This will be very slow unless target is flash based; + do it anyway for consistency. */ + if (!file_exists(buf)) { - if (!file_exists(buf)) - { - logf("Entry no longer valid."); - logf("-> %s", buf); - delete_entry(fe->idx_id); - continue; - } + logf("Entry no longer valid."); + logf("-> %s", buf); + delete_entry(fe->idx_id); + continue; } + #endif /* HAVE_DIRCACHE */ } continue ; diff --git a/firmware/export/config.h b/firmware/export/config.h index b769b63c32..73464526a6 100644 --- a/firmware/export/config.h +++ b/firmware/export/config.h @@ -853,7 +853,7 @@ Lyre prototype 1 */ #define HAVE_DIRCACHE #endif #ifdef HAVE_TAGCACHE -//#define HAVE_TC_RAMCACHE +#define HAVE_TC_RAMCACHE #endif #endif