Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix constantly growing file pool. #56

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

fix constantly growing file pool. #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7bfb7af
fix constantly growing file pool. Resulted in poor performance when r…
mmoos-meteomatics Jun 10, 2021
3da8f7d
Merge branch 'develop' into fix_decreasing_loading_performance
mmoos-meteomatics Jun 10, 2021
3401ac7
Merge branch 'develop' into fix_decreasing_loading_performance
shahramn Jun 10, 2021
71d672b
Merge branch 'develop' into fix_decreasing_loading_performance
mmoos-meteomatics Jun 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/grib_api_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1190,6 +1190,8 @@ struct grib_file
char* buffer;
long refcount;
grib_file* next;
grib_file* pool_file;
long pool_file_refcount;
short id;
};

Expand Down
2 changes: 2 additions & 0 deletions src/grib_api_prototypes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1036,6 +1036,8 @@ int grib_file_pool_read(grib_context* c, FILE* fh);
int grib_file_pool_write(FILE* fh);
grib_file* grib_file_open(const char* filename, const char* mode, int* err);
void grib_file_pool_delete_file(grib_file* file);
grib_file* grib_file_pool_create_clone(grib_context* c, short clone_id, grib_file* file);
void grib_file_pool_delete_clone(grib_file* clone);
void grib_file_close(const char* filename, int force, int* err);
void grib_file_close_all(int* err);
grib_file* grib_get_file(const char* filename, int* err);
Expand Down
63 changes: 57 additions & 6 deletions src/grib_filepool.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,51 @@ grib_file* grib_file_open(const char* filename, const char* mode, int* err)
return file;
}

grib_file* grib_file_pool_create_clone(grib_context* c, short clone_id, grib_file* pool_file)
{
if(pool_file)
{
grib_file* newfile = (grib_file*)grib_context_malloc_clear(c, sizeof(grib_file));
newfile->id = clone_id;
newfile->name = strdup(pool_file->name);
newfile->handle = pool_file->handle;
newfile->pool_file = pool_file;
newfile->pool_file_refcount = 0;

GRIB_MUTEX_INIT_ONCE(&once, &init);
GRIB_MUTEX_LOCK(&mutex1);

++pool_file->pool_file_refcount;

GRIB_MUTEX_UNLOCK(&mutex1);

return newfile;
}
else
return 0;
}

void grib_file_pool_delete_clone(grib_file* cloned_file)
{
grib_file* pool_file = cloned_file->pool_file;
if(pool_file)
{
GRIB_MUTEX_INIT_ONCE(&once, &init);
GRIB_MUTEX_LOCK(&mutex1);
if(pool_file->pool_file_refcount > 0)
{
--pool_file->pool_file_refcount;

if (pool_file->pool_file_refcount == 0)
grib_file_pool_delete_file(pool_file);
}

GRIB_MUTEX_UNLOCK(&mutex1);
}

grib_file_delete(cloned_file);
}

void grib_file_pool_delete_file(grib_file* file)
{
grib_file* prev = NULL;
Expand All @@ -287,6 +332,7 @@ void grib_file_pool_delete_file(grib_file* file)
if (file == file_pool.first) {
file_pool.first = file->next;
file_pool.current = file->next;
file_pool.size--;
}
else {
prev = file_pool.first;
Expand All @@ -299,10 +345,13 @@ void grib_file_pool_delete_file(grib_file* file)
DebugAssert(prev);
if (prev) {
prev->next = file->next;
file_pool.size--;
}
}

if (file->handle) {
fclose(file->handle);
file->handle = NULL;
file_pool.number_of_opened_files--;
}
grib_file_delete(file);
Expand Down Expand Up @@ -425,12 +474,14 @@ grib_file* grib_file_new(grib_context* c, const char* name, int* err)
next_id++;
GRIB_MUTEX_UNLOCK(&mutex1);

file->mode = 0;
file->handle = 0;
file->refcount = 0;
file->context = c;
file->next = 0;
file->buffer = 0;
file->mode = 0;
file->handle = 0;
file->refcount = 0;
file->context = c;
file->next = 0;
file->pool_file = 0;
file->pool_file_refcount = 0;
file->buffer = 0;
return file;
}

Expand Down
15 changes: 3 additions & 12 deletions src/grib_index.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -767,7 +767,7 @@ void grib_index_delete(grib_index* index)
while (file) {
grib_file* f = file;
file = file->next;
grib_file_delete(f);
grib_file_pool_delete_clone(f);
}
grib_context_free(index->context, index);
}
Expand Down Expand Up @@ -1074,7 +1074,6 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t
char buf[1024] = {0,};
int err = 0;
grib_file* indfile;
grib_file* newfile;

grib_index_key* index_key = NULL;
grib_handle* h = NULL;
Expand All @@ -1094,11 +1093,7 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t

if (!index->files) {
grib_filesid++;
newfile = (grib_file*)grib_context_malloc_clear(c, sizeof(grib_file));
newfile->id = grib_filesid;
newfile->name = strdup(file->name);
newfile->handle = file->handle;
index->files = newfile;
index->files = grib_file_pool_create_clone(c, grib_filesid, file);
}
else {
indfile = index->files;
Expand All @@ -1111,11 +1106,7 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t
while (indfile->next)
indfile = indfile->next;
grib_filesid++;
newfile = (grib_file*)grib_context_malloc_clear(c, sizeof(grib_file));
newfile->id = grib_filesid;
newfile->name = strdup(file->name);
newfile->handle = file->handle;
indfile->next = newfile;
indfile->next = grib_file_pool_create_clone(c, grib_filesid, file);
}

fseeko(file->handle, 0, SEEK_SET);
Expand Down