save to disk

redis/redis-cache.conf

@@ -449,7 +449,8 @@ maxclients 100
 #
 # maxmemory <bytes>
 
-maxmemory 104857600
+#maxmemory 104857600
+maxmemory 10485760
 
 # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory
 # is reached. You can select among five behaviors:

server/flask_server.py

@@ -18,6 +18,7 @@
 import scheduler
 import sys
 import random
+import traceback
 
 app = Flask(__name__)
 app.debug = False
@@ -51,9 +52,7 @@ def save():
  auth_code = js['auth_code']
  token = auth.get_fresh_token(auth_code)
  user = token['user_id']
- print 'got program', json.dumps(program, indent=4)
- if 'pid' in program:
- print 'got pid', program['pid']
+ # print 'got program', json.dumps(program, indent=4)
  if is_valid_program(program):
  if 'pid' in program:
  pid = program['pid']
@@ -515,6 +514,7 @@ def force_error():
 @app.errorhandler(Exception)
 def handle_invalid_usage(error):
  print "error", error
+ traceback.print_stack()
  results = { 
  'status': 'internal_error',
  "message": str(error)

server/kvstore.py

@@ -0,0 +1,110 @@
+import os
+base = "db.store"
+
+def put(pid, contents):
+ path = get_path(pid)
+ dirname = os.path.dirname(path)
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+ f = open(path, "w")
+ f.write(contents)
+ f.close()
+
+ return True
+
+def delete(pid):
+ path = get_path(pid)
+ if os.path.exits(path):
+ os.remove(path)
+
+def get(pid):
+ path = get_path(pid)
+ if os.path.exists(path):
+ f = open(path, "r")
+ contents = f.read()
+ f.close()
+ return contents
+ else:
+ return None
+
+
+def get_path(pid):
+ if len(pid) < 4:
+ path = os.sep.join([base, pid])
+ else:
+ path = os.sep.join([base, pid[0:2], pid[2:4], pid])
+ return path
+
+if __name__ == '__main__':
+ import sys
+ import time
+ import json
+ import random
+
+ def batch_write(count):
+ start_time = time.time()
+ for i in xrange(count):
+ pid = "%d12345678%d" %(i, i)
+ payload = {
+ "contents": "contents for pid " + pid,
+ "extra": "time is %d" % (int(time.time()), )
+ }
+ put(pid, json.dumps(payload))
+
+ delta = time.time() - start_time
+ print "wrote %d records in %d ms. %.4f ms per record" % (count, int(delta * 1000), (delta * 1000 / count))
+
+ def batch_read(count):
+ pids = []
+ for i in xrange(count):
+ pid = "%d12345678%d" %(i, i)
+ pids.append(pid)
+
+ random.shuffle(pids)
+
+ start_time = time.time()
+ found = 0
+ for pid in pids:
+ js_contents = get(pid)
+ if js_contents:
+ contents = json.loads(js_contents)
+ if contents:
+ found += 1
+ if verbose:
+ print json.dumps(contents, indent=2)
+
+
+ delta = time.time() - start_time
+ print "found %d of %d records in %d ms. %.4f ms per record" % (found, count, int(delta * 1000), (delta * 1000 / count))
+
+
+ args = sys.argv[1:]
+ pid = "12345678"
+ verbose = False
+ write = False
+ count = 1
+
+ while args:
+ arg = args.pop(0)
+
+ if arg == '--pid':
+ pid = args.pop(0)
+ elif arg == '--count':
+ count = int(args.pop(0))
+ elif arg == '--verbose':
+ verbose = True
+ elif arg == '--write':
+ if args and not args[0].startswith("--"):
+ payload['extra'] = args.pop(0)
+ write = True
+ else:
+ print "unknown arg", arg
+ sys.exit(1)
+
+ if write:
+ batch_write(count)
+ else:
+ batch_read(count)
+
+
+

server/memory_stats.txt

@@ -0,0 +1,87 @@
+Before memory reduction
+
+ "used_memory_human": "585.71M",
+
+
+{
+ "aof_rewrite_in_progress": 0, 
+ "total_connections_received": 61, 
+ "used_cpu_sys_children": 439.3, 
+ "run_id": "397b9b9c9626c1017ab43d122a29a997f25e8faf", 
+ "rejected_connections": 0, 
+ "redis_build_id": "899788decbe81753", 
+ "used_memory_peak_human": "587.11M", 
+ "pubsub_patterns": 0, 
+ "redis_mode": "standalone", 
+ "connected_slaves": 0, 
+ "uptime_in_days": 3, 
+ "multiplexing_api": "epoll", 
+ "lru_clock": 13576134, 
+ "redis_version": "3.0.6", 
+ "redis_git_sha1": 0, 
+ "sync_partial_ok": 0, 
+ "migrate_cached_sockets": 0, 
+ "gcc_version": "4.8.4", 
+ "connected_clients": 12, 
+ "keyspace_misses": 31244, 
+ "used_memory": 614164000, 
+ "tcp_port": 6379, 
+ "master_repl_offset": 0, 
+ "used_cpu_user_children": 4083.65, 
+ "repl_backlog_first_byte_offset": 0, 
+ "rdb_current_bgsave_time_sec": -1, 
+ "pubsub_channels": 0, 
+ "used_cpu_user": 144.43, 
+ "used_memory_lua": 36864, 
+ "instantaneous_ops_per_sec": 0, 
+ "rdb_last_save_time": 1540302617, 
+ "total_commands_processed": 836496, 
+ "aof_last_write_status": "ok", 
+ "role": "master", 
+ "cluster_enabled": 0, 
+ "aof_rewrite_scheduled": 0, 
+ "sync_partial_err": 0, 
+ "used_memory_rss": 409247744, 
+ "hz": 10, 
+ "sync_full": 0, 
+ "aof_enabled": 0, 
+ "config_file": "/home/plamere/SmarterPlaylists/redis/./redis.conf", 
+ "used_cpu_sys": 163.97, 
+ "rdb_last_bgsave_status": "ok", 
+ "instantaneous_output_kbps": 0.0, 
+ "latest_fork_usec": 31750, 
+ "aof_last_bgrewrite_status": "ok", 
+ "aof_last_rewrite_time_sec": -1, 
+ "used_memory_human": "585.71M", 
+ "loading": 0, 
+ "blocked_clients": 1, 
+ "process_id": 1783, 
+ "db0": {
+ "keys": 210526, 
+ "expires": 0, 
+ "avg_ttl": 0
+ }, 
+ "repl_backlog_histlen": 0, 
+ "client_biggest_input_buf": 0, 
+ "aof_current_rewrite_time_sec": -1, 
+ "arch_bits": 64, 
+ "repl_backlog_active": 0, 
+ "mem_fragmentation_ratio": 0.67, 
+ "rdb_last_bgsave_time_sec": 15, 
+ "expired_keys": 0, 
+ "total_net_input_bytes": 90820189, 
+ "evicted_keys": 0, 
+ "rdb_bgsave_in_progress": 0, 
+ "repl_backlog_size": 1048576, 
+ "instantaneous_input_kbps": 0.0, 
+ "client_longest_output_list": 0, 
+ "mem_allocator": "jemalloc-3.6.0", 
+ "total_net_output_bytes": 999814640, 
+ "used_memory_peak": 615631112, 
+ "uptime_in_seconds": 268583, 
+ "rdb_changes_since_last_save": 285, 
+ "redis_git_dirty": 0, 
+ "os": "Linux 4.15.13-x86_64-linode106 x86_64", 
+ "keyspace_hits": 570278
+}
+dbsize 210526