Skip to content

Commit

Permalink
Store statistics to persistent storage, read during startup
Browse files Browse the repository at this point in the history 
Correct CPR calculation and discard relative CPR (for now)
Version 2.3.6
  • Loading branch information
@Wolfrax
Wolfrax committed Jun 9, 2018
1 parent c541d6f commit 692120a
Show file tree
Hide file tree
Showing 6 changed files with 164 additions and 20 deletions.
12 changes: 12 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,18 @@ Configuration for spots is in `spots_config.json`. Follows json syntax with no e
* spots server address (localhost or ip-address): the address for the server
* spots server port (5051): the server port
* flight db name (string): name of database file to store flight counts, if the value is "" this function is skipped
* statistics file name (string): name of file to store statistics, which are read during start
* config file (string): name of file where personal email configuration data is stored, if the value is "" no emailing is done
* email recipient (string): name of email recipient, used when rare errors occur for notification. SMTP information is stored in the config file (above)

config file for SMTP info use the following syntax:
{
"//": "CAUTION: Stored in plain text!",
"SMTP_server": "smtp.gmail.com",
"SMTP_port": 587,
"GMAIL_username": "[email protected]",
"GMAIL_pw": "YourSercretPassword"
}

## Client/Server

Expand Down
80 changes: 74 additions & 6 deletions client/spots.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,41 @@ <h4 id="hdStatistics"></h4>
<div id="graphPreamble" style="width:150px"></div>
</div>
<div class="col_sm-12" id="summary"></div>
<div class="col_sm-12" id="footer">Mats Melander (c)</div>

<div class="col-sm-12">
<div id="header3">
<h3>Flights</h3>
<h4 id="hdFlightDB"></h4>
</div>
</div>

<div class="row" align="center">
<div class="col-sm-12">
<div class="col-sm-6">
<div id="FlightDB">
<table id="tbFlightDB">
<thead>
<tr>
<th>Top 10 Flights</th>
<th>Count</th>
<th>Bottom 10 Flights</th>
<th>Count</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
</div>
</div>
</div>
</div>

<br>
<div class="row">
<div class="col_sm-12" id="footer">Mats Melander (c)</div>
</div>



<script type="text/javascript">
function Spots() {
Expand Down Expand Up @@ -117,6 +151,30 @@ <h4 id="hdStatistics"></h4>
$(rows).appendTo("#tbCurrentData tbody");
}

function FlightDB(series) {
var rows = "";
var i;
var max_ind;

$("#hdFlightDB").html("Since " + series['start_date'] + " Total flight count: " + series['total_cnt']);
max_ind = Math.min(10, series['flights'].length);
min_ind = Math.max(0, series['flights'].length - 10);
for (i = 0; i < max_ind; i++) {
rows += "<tr><td>" +
"<a href='https://www.avdelphi.com/flight.html?callsign=" + series['flights'][i][0] + "'>" +
series['flights'][i][0] + "</a>" + "</td><td>" +
series['flights'][i][1] + "</td>";
rows += "<td>" +
"<a href='https://www.avdelphi.com/flight.html?callsign=" + series['flights'][min_ind + i][0] + "'>" +
series['flights'][min_ind + i][0] + "</a>" + "</td><td>" +
series['flights'][min_ind + i][1] + "</td></tr>";
}
for (i = max_ind; i < series['flights'].length; i++) {
}
$("#FlightDB").find("tbody").empty();
$(rows).appendTo("#FlightDB tbody");
}

function Statistics(series) {
var stats = [];
var preambles_per_secs = (series['valid_preambles'] - prev_nr_preambles) / 10;
Expand All @@ -126,7 +184,8 @@ <h4 id="hdStatistics"></h4>
prev_nr_preambles = series['valid_preambles'];

$("#hdStatistics").html("Version " + series['spots_version'] + " " +
"running since " + series['start_time_string']);
"running since " + series['start_time_string'] + " " +
"(latest restart " + series['latest_start_time_string'] + ")");

stats.push(series['df_total']);
stats.push(series['df_0']);
Expand Down Expand Up @@ -171,10 +230,6 @@ <h4 id="hdStatistics"></h4>
i.innerHTML = html;
}

function SignalCache(series) {
Plot.signal(series);
}

function updateTable() {
$.ajax({
url: "/spots/data",
Expand All @@ -187,6 +242,18 @@ <h4 id="hdStatistics"></h4>
});
}

function updateFlightDB() {
$.ajax({
url: "/spots/flight_db",
method: 'GET',
dataType: 'json',
cache: false
}).done(function (series) {
setTimeout(updateFlightDB, 1000);
FlightDB(series);
});
}

function updateStatistics() {
$.ajax({
url: "/spots/statistics",
Expand All @@ -200,6 +267,7 @@ <h4 id="hdStatistics"></h4>
}

updateTable();
updateFlightDB();
updateStatistics();
}
</script>
Expand Down
52 changes: 47 additions & 5 deletions radar/basic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import os
import simplejson
import logging
import shutil
import __init__ as init


Expand Down Expand Up @@ -200,6 +201,7 @@ class ADSB:
cfg_config_file = config["config file"]
cfg_use_email = True if cfg_config_file != "" else False
cfg_email_recipient = config["email recipient"]
cfg_stats_filename = config["statistics filename"]

def __init__(self):
pass
Expand Down Expand Up @@ -640,6 +642,8 @@ class Stats:
data = {'spots_version': "",
'start_time': 0,
'start_time_string': "",
'latest_start_time': 0,
'latest_start_time_string': "",
'valid_preambles': 0,
'valid_crc': 0,
'not_valid_crc': 0,
Expand Down Expand Up @@ -677,16 +681,44 @@ class Stats:
'df_31': 0,
'df_total': 0,
'no_unique_icao': 0,
'flights': 0
'flights': 0,
'max_lat': -90,
'min_lat': 90,
'max_lon': -180,
'min_lon': 180
}
icao_list = []
flight_list = {}

def __init__(self):
self['spots_version'] = ADSB.VERSION
self['start_time'] = time.time()
self['start_time_string'] = time.ctime(self['start_time'])
pass
self.logger = logging.getLogger('spots.Stats')

location = os.path.expanduser(ADSB.cfg_stats_filename)
self.loc = ADSB.cfg_stats_filename
self.loc_bck = self.loc + ".1"

if os.path.exists(location):
try:
self.data = simplejson.load(open(self.loc, 'rb'))

self['spots_version'] = ADSB.VERSION
self['latest_start_time'] = time.time()
self['latest_start_time_string'] = time.ctime(self['latest_start_time'])

if self['start_time'] == 0:
self['start_time'] = self['latest_start_time']
if self['start_time_string'] == "":
self['start_time_string'] = self['latest_start_time_string']
except simplejson.JSONDecodeError:
try:
self.logger.info("Init, stats file corrupt, using backup")
# Current file is corrupt, try to fallback to backup file
self.db = simplejson.load(open(self.loc_bck, 'rb'))
except simplejson.JSONDecodeError:
# No joy, give up and use default values
self.logger.info("Init, DB file and backup corrupt, using defaults")
self['start_time'] = self['latest_start_time']
self['start_time_string'] = self['latest_start_time_string']

def __setitem__(self, key, value):
self.data[key] = value
Expand All @@ -706,6 +738,13 @@ def add_flight(self, call_sign):
self.flight_list[call_sign] = 0
self['flights'] = len(self.flight_list)

def dump(self):
if os.path.isfile(self.loc):
shutil.copy2(self.loc, self.loc_bck) # Make a backup

with open(self.loc, 'wt') as stat_file:
simplejson.dump(self.data, stat_file, skipkeys=True, indent=4*' ')

def __str__(self):
st = "\n"
st += "Preambles:{}\n".format(self['valid_preambles'])
Expand Down Expand Up @@ -746,6 +785,9 @@ def __str__(self):
st += "DF30: {} ".format(self['df_30'])
st += "DF31: {} ".format(self['df_31'])
st += "\n"
st += "Max lat: {} Min lat: {}".format(self['max_lat'], self['min_lat'])
st += "Max lon: {} Min lat: {}".format(self['max_lon'], self['min_lon'])
st += "\n"
st += "DF Total: {} ".format(self['df_total'])
st += "\n"
st += "No of unique icao: {} ".format(self['no_unique_icao'])
Expand Down
14 changes: 11 additions & 3 deletions radar/radar.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,6 @@ def __init__(self):
def _dump_flight_db(self):
if basic.ADSB.cfg_use_flight_db:
# Save to persistent storage, flights and statistics
self.logger.info("Dumping DB to file")
self.flight_db.dump()

def _show_stats(self):
Expand Down Expand Up @@ -342,8 +341,16 @@ def _blip_add(self, msg):
else:
self.blips[icao] = {'msg': msg, 'timestamp': time.time(), 'count': 1}

if not self.blips[icao]['msg'].decodeCPR_relative():
self.blips[icao]['msg'].decodeCPR()
# if not self.blips[icao]['msg'].decodeCPR_relative():
# self.blips[icao]['msg'].decodeCPR()

self.blips[icao]['msg'].decodeCPR()

# Logic, decodeCPR returns True if
# 1. It successfully decodes one pair of odd+even position messages, not separated by more than 10 seconds
# 2. Give that condition 1) above is fulfilled
#if self.blips[icao]['msg'].decodeCPR():
# self.blips[icao]['msg'].decodeCPR_relative()

if basic.ADSB.cfg_use_flight_db and msg['call_sign'] != "":
self.flight_db.add(msg['call_sign'])
Expand Down Expand Up @@ -409,6 +416,7 @@ def _die(self):
self.stat_timer.cancel()
if self.cfg_use_text_display:
self.screen.close()
basic.statistics.dump()

def tuner_read(self, msgs, stop=False):
"""
Expand Down
1 change: 1 addition & 0 deletions radar/spots_config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"spots server address": "",
"spots server port": 5051,
"flight db name": "spots_flight_db.json",
"statistics filename": "spots_stats.json",
"config file": "/usr/etc/mm.json",
"email recipient": "[email protected]"
}
25 changes: 19 additions & 6 deletions radar/squitter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,8 +306,8 @@ def decodeCPR_relative(self):
lat_cpr = self.odd_raw_latitude / self.MAX_17_BITS
lon_cpr = self.odd_raw_longitude / self.MAX_17_BITS

j = math.floor(self.cfg_latitude / d_lat) + \
math.floor((self.cfg_latitude % d_lat) / d_lat - lat_cpr + 0.5) # latitude index
j = int(math.floor(self.cfg_latitude / d_lat)) + \
int(math.floor((self.cfg_latitude % d_lat) / d_lat - lat_cpr + 0.5)) # latitude index

latitude = d_lat * (j + lat_cpr)

Expand All @@ -316,13 +316,19 @@ def decodeCPR_relative(self):
else:
d_lon = 360.0 / CPR_NL(latitude)

m = math.floor(self.cfg_longitude / d_lon) + math.floor((self.cfg_longitude % d_lon) / d_lon - lon_cpr + 0.5)
m = int(math.floor(self.cfg_longitude / d_lon)) + \
int(math.floor((self.cfg_longitude % d_lon) / d_lon - lon_cpr + 0.5))

longitude = d_lon * (m + lon_cpr)

self.data['latitude'] = str(round(latitude, 3)) if latitude != 0.0 else ""
self.data['longitude'] = str(round(longitude, 3)) if longitude != 0.0 else ""

basic.statistics['max_lat'] = max(basic.statistics['max_lat'], self.data['latitude'])
basic.statistics['min_lat'] = min(basic.statistics['min_lat'], self.data['latitude'])
basic.statistics['max_lon'] = max(basic.statistics['max_lon'], self.data['longitude'])
basic.statistics['min_lon'] = min(basic.statistics['min_lon'], self.data['longitude'])

return True

def decodeCPR(self):
Expand All @@ -332,13 +338,15 @@ def decodeCPR(self):

if self.odd_time == 0 or self.even_time == 0:
return False # we need both even + odd messages
if abs(self.odd_time - self.even_time) > 10.0:
return False # Need to have odd and even messages within 10 seconds

cpr_lat_even = self.even_raw_latitude / self.MAX_17_BITS
cpr_lon_even = self.even_raw_longitude / self.MAX_17_BITS
cpr_lat_odd = self.odd_raw_latitude / self.MAX_17_BITS
cpr_lon_odd = self.odd_raw_longitude / self.MAX_17_BITS

j = math.floor(59 * cpr_lat_even - 60 * cpr_lat_odd + 0.5) # latitude index
j = int(math.floor(59 * cpr_lat_even - 60 * cpr_lat_odd + 0.5)) # latitude index

latitude_even = float((360.0 / 60.0) * (j % 60 + cpr_lat_even))
latitude_odd = float((360.0 / 59.0) * (j % 59 + cpr_lat_odd))
Expand All @@ -358,12 +366,12 @@ def decodeCPR(self):
if self.even_time >= self.odd_time:
ni = max(CPR_NL(latitude_even), 1)
dlon = 360.0 / ni
m = math.floor(cpr_lon_even * (CPR_NL(latitude_even) - 1) - cpr_lon_odd * CPR_NL(latitude_even) + 0.5)
m = int(math.floor(cpr_lon_even * (CPR_NL(latitude_even) - 1) - cpr_lon_odd * CPR_NL(latitude_even) + 0.5))
longitude = dlon * (m % ni + cpr_lon_even)
else:
ni = max(CPR_NL(latitude_odd) - 1, 1)
dlon = 360.0 / ni
m = math.floor(cpr_lon_even * (CPR_NL(latitude_odd) - 1) - cpr_lon_odd * CPR_NL(latitude_odd) + 0.5)
m = int(math.floor(cpr_lon_even * (CPR_NL(latitude_odd) - 1) - cpr_lon_odd * CPR_NL(latitude_odd) + 0.5))
longitude = dlon * (m % ni + cpr_lon_odd)

if longitude >= 180:
Expand All @@ -372,6 +380,11 @@ def decodeCPR(self):
self.data['latitude'] = str(round(latitude, 3)) if latitude != 0.0 else ""
self.data['longitude'] = str(round(longitude, 3)) if longitude != 0.0 else ""

basic.statistics['max_lat'] = max(basic.statistics['max_lat'], self.data['latitude'])
basic.statistics['min_lat'] = min(basic.statistics['min_lat'], self.data['latitude'])
basic.statistics['max_lon'] = max(basic.statistics['max_lon'], self.data['longitude'])
basic.statistics['min_lon'] = min(basic.statistics['min_lon'], self.data['longitude'])

# Reset all flags
self.odd_time = 0
self.even_time = 0
Expand Down

0 comments on commit 692120a

Please sign in to comment.