Persist dcache across sessions

cache.c

@@ -15,6 +15,8 @@
 #include <glib.h>
 #include <pthread.h>
 
+#include "dcache.pb-c.h"
+
 #define DEFAULT_CACHE_TIMEOUT_SECS 20
 #define DEFAULT_MAX_CACHE_SIZE 10000
 #define DEFAULT_CACHE_CLEAN_INTERVAL_SECS 60
@@ -36,6 +38,7 @@ struct cache {
 };
 
 static struct cache cache;
+static const size_t STAT_SIZE = sizeof(struct stat);
 
 struct node {
  struct stat stat;
@@ -613,3 +616,196 @@ int cache_parse_options(struct fuse_args *args)
 
  return fuse_opt_parse(args, &cache, cache_opts, NULL);
 }
+
+int cache_load(const char *in_path) {
+ FILE* f = fopen(in_path, "r");
+ if (f == NULL) {
+ // if the dcache does not exist, that's ok
+ return errno == ENOENT ? 0 : errno;
+ }
+
+ fseek(f, 0, SEEK_END);
+ size_t sz = ftell(f);
+ rewind(f);
+
+ int ret;
+
+ void* buf = g_malloc(sz);
+ ret = fread(buf, sz, 1, f);
+ if (ret <= 0) {
+ free(buf);
+ ret = errno;
+ goto done;
+ }
+
+ Dcache* store_root = dcache__unpack(NULL, sz, buf);
+ free(buf);
+
+ if (store_root == NULL) {
+ fprintf(stderr, "Corrupted dcache [%s], ignored.\n", in_path);
+ ret = 0;
+ goto done;
+ }
+
+ size_t n_entries = store_root->n_entries;
+
+ int stat_sz_read = store_root->metadata->stat_size;
+ fprintf(stderr, "STAT SIZE = %d\n", stat_sz_read);
+ fprintf(stderr, "CREATED AT = %ld\n", store_root->metadata->creation_time);
+ fprintf(stderr, "LIST LEN = %zd\n", n_entries);
+
+ if (stat_sz_read != STAT_SIZE) {
+ fprintf(stderr, "Inconsistent stat size (%d read, %ld expected)!!\n",
+ stat_sz_read, STAT_SIZE);
+ goto done_free_unpacked;
+ }
+
+ pthread_mutex_lock(&cache.lock);
+
+ struct node* cvalues = calloc(n_entries, sizeof(struct node));
+
+ for(size_t i = 0; i < n_entries; i++) {
+ Dcache__EntriesEntry* entry = store_root->entries[i];
+
+ char* path = g_strdup(entry->key);
+ DcacheEntry* value = entry->value;
+ struct node* node = &cvalues[i];
+
+ fprintf(stderr, "Reading cache entry: %s\n", path);
+
+ node->valid = LONG_MAX;
+
+ if (value->opt_stat_case == DCACHE_ENTRY__OPT_STAT_STAT) {
+ ProtobufCBinaryData* stat_ptr = &value->stat;
+ memcpy(&node->stat, stat_ptr->data, MIN(STAT_SIZE, stat_ptr->len));
+ node->stat_valid = LONG_MAX;
+ }
+
+ if (value->opt_link_case == DCACHE_ENTRY__OPT_LINK_LINK) {
+ strcpy(node->link, value->link);
+ node->link_valid = LONG_MAX;
+ }
+
+ if (value->opt_dir_case == DCACHE_ENTRY__OPT_DIR_DIR) {
+ size_t n = value->dir->n_values;
+ char** dirents = g_malloc(sizeof(char*) * (n+1));
+ char** ds = value->dir->values;
+ for (size_t i = 0; i < n; i++) {
+ dirents[i] = g_strdup(ds[i]);
+ }
+ dirents[n] = NULL;
+ node->dir = dirents;
+ node->dir_valid = LONG_MAX;
+ }
+
+ g_hash_table_insert(cache.table, path, node);
+ }
+
+ pthread_mutex_unlock(&cache.lock);
+
+done_free_unpacked:
+ dcache__free_unpacked(store_root, NULL);
+done:
+ ret = fclose(f);
+ return ret;
+}
+
+int cache_dump(const char *out_path, size_t* entries_count) {
+ Dcache store_root = DCACHE__INIT;
+
+ DcacheMeta meta = DCACHE_META__INIT;
+ meta.version = 42;
+ meta.stat_size = STAT_SIZE;
+ store_root.metadata = &meta;
+
+ GPtrArray* _entries = g_ptr_array_new();
+
+ GHashTableIter iter;
+ gpointer path, _node;
+
+ pthread_mutex_lock(&cache.lock);
+
+ time_t now = time(NULL);
+ meta.creation_time = now;
+ GPtrArray* buffers = g_ptr_array_new();
+
+ g_hash_table_iter_init(&iter, cache.table);
+ while (g_hash_table_iter_next(&iter, &path, &_node)) {
+ struct node *node = _node;
+
+ Dcache__EntriesEntry* entry = g_malloc(sizeof(Dcache__EntriesEntry));
+ g_ptr_array_add(buffers, entry);
+ dcache__entries_entry__init(entry);
+
+ DcacheEntry* entry_value = g_malloc(sizeof(DcacheEntry));
+ g_ptr_array_add(buffers, entry_value);
+ dcache_entry__init(entry_value);
+
+ entry->key = (char*) path;
+ entry->value = entry_value;
+ g_ptr_array_add(_entries, entry);
+
+ if (node->stat_valid - now >= 0) {
+ entry_value->opt_stat_case = DCACHE_ENTRY__OPT_STAT_STAT;
+ entry_value->stat = (ProtobufCBinaryData) {
+ .len = STAT_SIZE,
+ .data = (uint8_t*) &node->stat,
+ };
+ }
+
+ if (node->link_valid - now >= 0) {
+ entry_value->opt_link_case = DCACHE_ENTRY__OPT_LINK_LINK;
+ }
+
+ if (node->dir_valid - now >= 0) {
+ ListString* dirs = g_malloc(sizeof(ListString));
+ g_ptr_array_add(buffers, dirs);
+ list_string__init(dirs);
+
+ char** dir_iter = node->dir;
+ dirs->values = dir_iter;
+ while (*dir_iter) {
+ dirs->n_values++;
+ dir_iter++;
+ }
+
+ entry_value->opt_dir_case = DCACHE_ENTRY__OPT_DIR_DIR;
+ entry_value->dir = dirs;
+ }
+ }
+
+ store_root.n_entries = _entries->len;
+ store_root.entries = (Dcache__EntriesEntry **) _entries->pdata;
+
+ if (entries_count) {
+ *entries_count = _entries->len;
+ }
+
+ // because of referring node's data,
+ // we need to output the content before releasing the lock
+ int ret;
+ FILE* f = fopen(out_path, "w");
+
+ if (f == NULL) {
+ ret = errno;
+ } else {
+ size_t len = dcache__get_packed_size(&store_root);
+ void* buf = g_malloc(len);
+ dcache__pack(&store_root, buf);
+ size_t written = fwrite(buf, len, 1, f);
+ if (written == 0) {
+ ret = errno;
+ }
+ free(buf);
+ }
+
+ g_ptr_array_free(_entries, TRUE);
+
+ g_ptr_array_add(buffers, NULL);
+ g_strfreev((char**) g_ptr_array_free(buffers, FALSE));
+
+ ret = fclose(f);
+ pthread_mutex_unlock(&cache.lock);
+
+ return ret;
+}

cache.h

@@ -14,3 +14,6 @@ int cache_parse_options(struct fuse_args *args);
 void cache_add_attr(const char *path, const struct stat *stbuf, uint64_t wrctr);
 void cache_invalidate(const char *path);
 uint64_t cache_get_write_ctr(void);
+
+int cache_load(const char *in_path);
+int cache_dump(const char *out_path, size_t* entries_count);

meson.build

@@ -44,9 +44,19 @@ endif
 configure_file(output: 'config.h',
  configuration : cfg)
 
+protoc = find_program('protoc')
+
+proto_gen = generator(protoc, \
+ output : ['@[email protected]', '@[email protected]'],
+ arguments : ['--proto_path=@CURRENT_SOURCE_DIR@/protos', '--c_out=@BUILD_DIR@', '@INPUT@'])
+
+dcache_proto = proto_gen.process('protos/dcache.proto')
+sshfs_sources += [ dcache_proto ]
+
 sshfs_deps = [ dependency('fuse3', version: '>= 3.1.0'),
  dependency('glib-2.0'),
- dependency('gthread-2.0') ]
+ dependency('gthread-2.0'),
+ dependency('libprotobuf-c', version: '>= 1.0.0') ]
 
 executable('sshfs', sshfs_sources,
  include_directories: include_dirs,

protos/dcache.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+message Dcache {
+ DcacheMeta metadata = 1;
+ map<string, DcacheEntry> entries = 2;
+}
+
+message ListString {
+ repeated string values = 1;
+}
+
+message DcacheMeta {
+ uint32 version = 1;
+ uint32 stat_size = 2;
+ uint64 creation_time = 3;
+ string note = 4;
+}
+
+message DcacheEntry {
+ oneof opt_stat {
+ bytes stat = 1;
+ }
+ oneof opt_link {
+ string link = 2;
+ }
+ oneof opt_dir {
+ ListString dir = 3;
+ }
+}

sshfs.c

@@ -4337,6 +4337,14 @@ int main(int argc, char *argv[])
  if (res == -1)
  exit(1);
 
+ char* cwd_orig = getcwd(NULL, 0);
+ if (!cwd_orig) {
+ perror("Error getting old working directory");
+ exit(1);
+ }
+ char* cache_persist_path = malloc(strlen(cwd_orig) + 32);
+ sprintf(cache_persist_path, "%s/%s", cwd_orig, "dcache.dump");
+
  sshfs.randseed = time(0);
 
  if (sshfs.max_read > 65536)
@@ -4350,10 +4358,19 @@ int main(int argc, char *argv[])
  g_free(tmp);
  g_free(fsname);
 
- if(sshfs.dir_cache)
+ if(sshfs.dir_cache) {
  sshfs.op = cache_wrap(&sshfs_oper);
- else
+
+ int ret = cache_load(cache_persist_path);
+ if (ret > 0) {
+ perror("Error loading dcache... Remove the file if that causes problems");
+ exit(1);
+ } else if (ret != 0) {
+ exit(-ret);
+ }
+ } else {
  sshfs.op = &sshfs_oper;
+ }
  fuse = fuse_new(&args, sshfs.op,
  sizeof(struct fuse_operations), NULL);
  if(fuse == NULL)
@@ -4405,9 +4422,9 @@ int main(int argc, char *argv[])
  else
  res = 0;
 
- fuse_remove_signal_handlers(se);
  fuse_unmount(fuse);
  fuse_destroy(fuse);
+ fuse_remove_signal_handlers(se);
 
  if (sshfs.debug) {
  unsigned int avg_rtt = 0;
@@ -4428,9 +4445,23 @@ int main(int argc, char *argv[])
  sshfs.num_connect);
  }
 
+ if (sshfs.dir_cache) {
+ DEBUG("Ready to persist dcache to \"%s\"\n", cache_persist_path);
+ size_t cnt;
+ int res = cache_dump(cache_persist_path, &cnt);
+ if (res) {
+ DEBUG("Failed to save dcache: %s\n", strerror(res));
+ } else {
+ DEBUG("Persisted dcache of %zd entries.\n", cnt);
+ }
+ }
+
  fuse_opt_free_args(&args);
  fuse_opt_free_args(&sshfs.ssh_args);
  free(sshfs.directport);
 
+ free(cwd_orig);
+ free(cache_persist_path);
+
  return res;
 }