forked from plamere/SmarterPlaylists
-
Notifications
You must be signed in to change notification settings - Fork 0
/
trim_db.py
151 lines (121 loc) · 3.9 KB
/
trim_db.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
import sys
import redis
import json
import datetime
max_days_to_keep = 30
r = redis.StrictRedis(host='localhost', port=6379, db=0)
unexpired_byte_count = 0
expired_byte_count = 0
total_byte_count = 0
total_jobs = 0
removed_jobs = 0
trimmed_jobs = 0
keep_all_jobs = 0
MAX_AGE_RESULTS = 30 * 24 * 60 * 60
def trim_results(name):
global total_jobs, removed_jobs, trimmed_jobs, keep_all_jobs
results = r.lrange(name, 0, 100)
last_result_to_keep = -1
total_jobs += 1
for i, js in enumerate(results):
results = json.loads(js)
# print name, i, fmt_date(results['runtime'])
age = get_age(results['runtime'])
if age < max_days_to_keep:
last_result_to_keep = i
else:
break
if last_result_to_keep == -1:
removed_jobs += 1
r.delete(name)
else:
r.ltrim(name, 0, last_result_to_keep)
trimmed_jobs += 1
print "%d %s %4d %8d %8d %8d %8d" % (i, fmt_date(results['runtime']), get_age(results['runtime']), total_jobs, removed_jobs, trimmed_jobs, keep_all_jobs)
def count_results(name):
global total_byte_count, expired_byte_count, unexpired_byte_count
results = r.lrange(name, 0, 100)
for i, js in enumerate(results):
total_byte_count += len(js)
results = json.loads(js)
# print name, i, fmt_date(results['runtime'])
age = get_age(results['runtime'])
if age < max_days_to_keep:
unexpired_byte_count += len(js)
else:
expired_byte_count += len(js)
print "%d %s %4d %8d %8d %8d" % (i, fmt_date(results['runtime']), get_age(results['runtime']), unexpired_byte_count, expired_byte_count, total_byte_count)
def show_results(name):
results = r.lrange(name, 0, 100)
ttl = r.ttl(name)
for i, js in enumerate(results):
results = json.loads(js)
# print json.dumps(results, indent=4)
print "TTL", ttl
show_result(results)
def expire_key(name):
r.expire(name, MAX_AGE_RESULTS)
def show_result(result):
print "%s %s %.2f" % (result['status'], fmt_date(result['runtime']), result['time'])
print result['oinfo']
print result['info']
if result['status'] == 'ok':
print result['name']
print result['uri']
else:
print result['message']
print
def fmt_date(ts):
date = datetime.date.fromtimestamp(ts)
return date.strftime("%Y-%m-%d")
def get_age(ts):
date = datetime.date.fromtimestamp(ts)
now = datetime.date.today()
delta = now - date
return delta.days
def trim():
cursor, knames = r.scan(0, match="sched-results-*")
for name in knames:
trim_results(name)
while cursor != 0:
cursor, knames = r.scan(cursor, match="sched-results-*")
for name in knames:
trim_results(name)
def count():
cursor, knames = r.scan(0, match="sched-results-*")
for name in knames:
count_results(name)
while cursor != 0:
cursor, knames = r.scan(cursor, match="sched-results-*")
for name in knames:
count_results(name)
def show():
cursor, knames = r.scan(0, match="sched-results-*")
for name in knames:
show_results(name)
while cursor != 0:
cursor, knames = r.scan(cursor, match="sched-results-*")
for name in knames:
show_results(name)
def expire():
cursor, knames = r.scan(0, match="sched-results-*")
for name in knames:
expire_key(name)
while cursor != 0:
cursor, knames = r.scan(cursor, match="sched-results-*")
for name in knames:
expire_key(name)
if __name__ == '__main__':
args = sys.argv[1:]
while args:
arg = args.pop(0)
if arg == '--trim':
trim()
elif arg == '--count':
count()
elif arg == '--show':
show()
elif arg == '--expire':
expire()
else:
print "unknown arg", arg