From 051042bc4c08968315f10936a1fd700d9b28f4d7 Mon Sep 17 00:00:00 2001 From: Tom Kooij Date: Mon, 25 Nov 2019 11:56:16 +0100 Subject: [PATCH 1/4] Add rain_rate plots to histogram jobs --- publicdb/histograms/esd.py | 10 ++++++++++ publicdb/histograms/jobs.py | 24 ++++++++++++++++++------ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/publicdb/histograms/esd.py b/publicdb/histograms/esd.py index 638958256..bd2cda05a 100644 --- a/publicdb/histograms/esd.py +++ b/publicdb/histograms/esd.py @@ -423,6 +423,16 @@ def get_barometer(summary): return get_time_series(summary, 'weather', 'barometer') +def get_rain_rate(summary): + """Get rain_rate data + + :param summary: summary of data source (station and date) + :type summary: histograms.models.Summary instance + + """ + return get_time_series(summary, 'weather', 'rain_rate') + + def get_event_data(summary, quantity): """Get event data of a specific quantity diff --git a/publicdb/histograms/jobs.py b/publicdb/histograms/jobs.py index c5915b46e..d4f4a8719 100644 --- a/publicdb/histograms/jobs.py +++ b/publicdb/histograms/jobs.py @@ -37,8 +37,7 @@ # Parameters for the datasets, intervals in seconds BIN_SINGLES_RATE = 180 -INTERVAL_TEMP = 150 -INTERVAL_BARO = 150 +INTERVAL_WEATHER = 150 # Maximum number of configs per station per day. If more configs are found # for a single day, all (new) configs will be treated as erroneous and skipped. @@ -283,6 +282,7 @@ def perform_weather_tasks(summary): logger.info("Updating weather datasets for %s", summary) update_temperature_dataset(summary) update_barometer_dataset(summary) + update_rain_rate_dataset(summary) return summary, [] @@ -431,7 +431,7 @@ def update_station_timing_offsets(network_summary): network_off = esd.DetermineStationTimingOffsetsESD(stations) for ref_sn, sn in network_off.get_station_pairs_within_max_distance(): - off = esd.DetermineStationTimingOffsetsESD([ref_sn, sn]) + off = esd.varm([ref_sn, sn]) cuts = off._get_cuts(sn, ref_sn) left, right = off.determine_first_and_last_date(summary_date, sn, ref_sn) # To update all affected offsets use: @@ -489,7 +489,7 @@ def update_temperature_dataset(summary): error_values = [-999, -2 ** 15] temperature = [(x, y) for x, y in temperature if y not in error_values] if temperature != []: - temperature = shrink_dataset(temperature, INTERVAL_TEMP) + temperature = shrink_dataset(temperature, INTERVAL_WEATHER) save_dataset(summary, 'temperature', *zip(*temperature)) @@ -501,10 +501,22 @@ def update_barometer_dataset(summary): error_values = [-999] barometer = [(x, y) for x, y in barometer if y not in error_values] if barometer != []: - barometer = shrink_dataset(barometer, INTERVAL_BARO) + barometer = shrink_dataset(barometer, INTERVAL_WEATHER) save_dataset(summary, 'barometer', *zip(*barometer)) +def update_rain_rate_dataset(summary): + """Create dataset of timestamped rain-rate data""" + + logger.debug("Updating rain-rate dataset for %s", summary) + rain_rate = esd.get_rain_rate(summary) + error_values = [-999] + rain_rate = [(x, y.astype(float)) for x, y in rain_rate if y not in error_values] + if rain_rate != []: + rain_rate = shrink_dataset(rain_rate, INTERVAL_WEATHER) + save_dataset(summary, 'rain_rate', *zip(*rain_rate)) + + def shrink_dataset(dataset, interval): """Shrink a dataset by skipping over data. @@ -606,7 +618,7 @@ def save_dataset(summary, slug, x, y): logger.debug("Saving dataset %s for %s", slug, summary) type = DatasetType.objects.get(slug=slug) dataset = {'x': x, 'y': y} - if slug in ['barometer', 'temperature']: + if slug in ['barometer', 'temperature', 'rain_rate']: DailyDataset.objects.update_or_create(summary=summary, type=type, defaults=dataset) else: MultiDailyDataset.objects.update_or_create(summary=summary, type=type, defaults=dataset) From 233dfe686a0f77f5afe8bd7ac04e2e1ecb8e3304 Mon Sep 17 00:00:00 2001 From: Tom Kooij Date: Mon, 25 Nov 2019 14:37:49 +0100 Subject: [PATCH 2/4] Add rain rate plot to status display --- .../static/scripts/flot_settings.js | 14 ++++++++++ .../static/styles/station_page.css | 1 + .../templates/source/rain_rate_dataset.tsv | 27 +++++++++++++++++++ .../status_display/station_data.html | 21 +++++++++++++++ publicdb/status_display/urls.py | 2 +- publicdb/status_display/views.py | 4 +-- 6 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 publicdb/status_display/templates/source/rain_rate_dataset.tsv diff --git a/publicdb/status_display/static/scripts/flot_settings.js b/publicdb/status_display/static/scripts/flot_settings.js index 0643dd12a..057375021 100644 --- a/publicdb/status_display/static/scripts/flot_settings.js +++ b/publicdb/status_display/static/scripts/flot_settings.js @@ -189,6 +189,19 @@ var _barometerdata = { max: 24} }; +var _rain_ratedata = { + lines: { + show: true}, + yaxis: { + axisLabel: 'Rain rate (mm/h)'}, + xaxis: { + axisLabel: 'Hour of day (GPS)', + ticks: [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24], + min: 0, + max: 24}, + colors: ["#0066FF"], +}; + var _singleslowhistogram = { yaxis: { axisLabel: 'Counts', @@ -286,6 +299,7 @@ var srl_options = $.extend(true, {}, _base, _singlesdata); var srh_options = $.extend(true, {}, _base, _singlesdata); var td_options = $.extend(true, {}, _base, _temperaturedata); var bd_options = $.extend(true, {}, _base, _barometerdata); +var rd_options = $.extend(true, {}, _base, _rain_ratedata); var vg_options = $.extend(true, {}, _base, _voltagegraph); var cg_options = $.extend(true, {}, _base, _currentgraph); var to_options = $.extend(true, {}, _base, _timingoffsetgraph); diff --git a/publicdb/status_display/static/styles/station_page.css b/publicdb/status_display/static/styles/station_page.css index ca65d0884..6398cf528 100644 --- a/publicdb/status_display/static/styles/station_page.css +++ b/publicdb/status_display/static/styles/station_page.css @@ -32,6 +32,7 @@ #eh_histogram, #bd_histogram, #td_histogram, +#rd_histogram, #vg_histogram, #cg_histogram, #to_histogram { diff --git a/publicdb/status_display/templates/source/rain_rate_dataset.tsv b/publicdb/status_display/templates/source/rain_rate_dataset.tsv new file mode 100644 index 000000000..6200f5674 --- /dev/null +++ b/publicdb/status_display/templates/source/rain_rate_dataset.tsv @@ -0,0 +1,27 @@ +# HiSPARC rain rate dataset source +# +# Station: {{ station_number }} +# Data from {{ date }} +# +# +# HiSPARC data and HiSPARC software is licensed for use within the HiSPARC collaboration +# and for educational purposes. Commercial use is prohibited. +# +# Academic publishing including, but not limited to preprints, journal articles, theses +# and books using HiSPARC data, is prohibited unless permission is explicitly granted by +# the HiSPARC management (beheer@hisparc.nl) +# +# For information about HiSPARC data usage and HiSPARC software application +# please contact: beheer@hisparc.nl or consult www.hisparc.nl +# +# +# +# This data contains the following columns: +# +# timestamp: time of measurement in seconds after 1970-1-1 [UNIX timestamp] +# value: rain rate [mm/h] +# +# +# timestamp value +{% for row in data %}{{ row.0 }} {{ row.1 }} +{% endfor %} diff --git a/publicdb/status_display/templates/status_display/station_data.html b/publicdb/status_display/templates/status_display/station_data.html index bc912979a..0203a63da 100644 --- a/publicdb/status_display/templates/status_display/station_data.html +++ b/publicdb/status_display/templates/status_display/station_data.html @@ -120,6 +120,27 @@

Singles (per second above low threshold)

{% endif %} + {% if plots.rain_rate %} +
+

Rain rate data

+
+ Image | + Source +
+
+ +
+ {% endif %} + {% if plots.singlesratehigh %}

Singles (per second above high threshold)

diff --git a/publicdb/status_display/urls.py b/publicdb/status_display/urls.py index d4443b5bf..df0875620 100644 --- a/publicdb/status_display/urls.py +++ b/publicdb/status_display/urls.py @@ -37,7 +37,7 @@ ] + [ # Datasets url(r'^{type}/'.format(type=type) + STATION_DATE_REGEX, views.get_specific_dataset_source, {'type': type}, name=type) - for type in ['barometer', 'temperature', 'singlesratelow', 'singlesratehigh'] + for type in ['barometer', 'temperature', 'rain_rate', 'singlesratelow', 'singlesratehigh'] ] + [ # Configurations url(r'^{type}/(?P\d+)/$'.format(type=type), views.get_specific_config_source, {'type': type}, name=type) diff --git a/publicdb/status_display/views.py b/publicdb/status_display/views.py index 2680b83f6..366f00cb8 100644 --- a/publicdb/status_display/views.py +++ b/publicdb/status_display/views.py @@ -938,7 +938,7 @@ def get_dataset_source(year, month, day, type, station_number): except ValueError: raise Http404 - if type in ['barometer', 'temperature']: + if type in ['barometer', 'temperature', 'rain_rate']: dataset_model = DailyDataset else: dataset_model = MultiDailyDataset @@ -948,7 +948,7 @@ def get_dataset_source(year, month, day, type, station_number): summary__date=date, type__slug=type) - if type in ['barometer', 'temperature']: + if type in ['barometer', 'temperature', 'rain_rate']: return zip(dataset.x, dataset.y) else: # Multiple value columns From 7834cb98e969d5b743726307af4acd7a3812a0f5 Mon Sep 17 00:00:00 2001 From: Tom Kooij Date: Mon, 25 Nov 2019 19:54:06 +0100 Subject: [PATCH 3/4] Fix typo / accidental commit --- publicdb/histograms/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publicdb/histograms/jobs.py b/publicdb/histograms/jobs.py index d4f4a8719..f06b0cb55 100644 --- a/publicdb/histograms/jobs.py +++ b/publicdb/histograms/jobs.py @@ -431,7 +431,7 @@ def update_station_timing_offsets(network_summary): network_off = esd.DetermineStationTimingOffsetsESD(stations) for ref_sn, sn in network_off.get_station_pairs_within_max_distance(): - off = esd.varm([ref_sn, sn]) + off = esd.DetermineStationTimingOffsetsESD([ref_sn, sn]) cuts = off._get_cuts(sn, ref_sn) left, right = off.determine_first_and_last_date(summary_date, sn, ref_sn) # To update all affected offsets use: From 456f6e30055ed8bd44f68f890d38f509282fbf8c Mon Sep 17 00:00:00 2001 From: Tom Kooij Date: Tue, 26 Nov 2019 20:53:58 +0100 Subject: [PATCH 4/4] Plot rain rate in singleratelow figure --- .../static/scripts/flot_settings.js | 2 +- .../status_display/station_data.html | 48 +++++++++++-------- .../status_display/templatetags/fix_data.py | 7 +++ 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/publicdb/status_display/static/scripts/flot_settings.js b/publicdb/status_display/static/scripts/flot_settings.js index 057375021..ca9bbc979 100644 --- a/publicdb/status_display/static/scripts/flot_settings.js +++ b/publicdb/status_display/static/scripts/flot_settings.js @@ -2,7 +2,7 @@ function HideTickLabels(v, axis) { return " ";} var _base = { - colors: ["#222", "#D22", "#1C2", "#1CC"], + colors: ["#222", "#D22", "#1C2", "#1CC", "#0066FF"], series: { lines: { lineWidth: 1.25, diff --git a/publicdb/status_display/templates/status_display/station_data.html b/publicdb/status_display/templates/status_display/station_data.html index 0203a63da..17f3882c9 100644 --- a/publicdb/status_display/templates/status_display/station_data.html +++ b/publicdb/status_display/templates/status_display/station_data.html @@ -102,6 +102,7 @@

Pulseintegral histogram

{% if plots.singlesratelow %}

Singles (per second above low threshold)

+ {% if plots.rain_rate %}rain rate data (arb unit) in blue.{% endif %}
Image | Source @@ -115,32 +116,16 @@

Singles (per second above low threshold)

{% for data in plots.singlesratelow.data %} {data: {{ data|fix_timestamps_in_data|slice_data:50|round_data:2 }}, yaxis: 1}, {% endfor %} + {% if plots.rain_rate %} + {% for data in plots.rain_rate.data %} + {data: {{ data|fix_timestamps_in_data|slice_data:50|scale_rain_rate|round_data:2 }}, yaxis: 1}, + {% endfor %} + {% endif %} {data: [0, 0], lines: {show: false}, xaxis: 2, yaxis: 2},], srl_options);
{% endif %} - {% if plots.rain_rate %} -
-

Rain rate data

-
- Image | - Source -
-
- -
- {% endif %} - {% if plots.singlesratehigh %}

Singles (per second above high threshold)

@@ -287,6 +272,27 @@

Temperature data

{% endif %} + + {% if plots.rain_rate %} +
+

Rain rate data

+
+ Image | + Source +
+
+ +
+ {% endif %}
{% endblock %} diff --git a/publicdb/status_display/templatetags/fix_data.py b/publicdb/status_display/templatetags/fix_data.py index b4de2f15c..7aaf0f2ea 100644 --- a/publicdb/status_display/templatetags/fix_data.py +++ b/publicdb/status_display/templatetags/fix_data.py @@ -113,3 +113,10 @@ def mv_to_adc(value): return value else: return int(round(value / -0.57 + 200)) + + +@register.filter +def scale_rain_rate(values, factor=10): + """Scale rain rate to singlerate (low threshold) plot""" + + return [[x, min(150, factor * y)] for x, y in values]