Last active
February 6, 2022 15:25
-
-
Save ludoo/cf863a4591fab76eb0f75358b9e864a7 to your computer and use it in GitHub Desktop.
Simple Stackdriver custom metrics
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| foo! spam! eggs! name this gist! |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2021 Google LLC. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| FROM alpine | |
| COPY requirements.txt / | |
| RUN \ | |
| apk update && \ | |
| apk add \ | |
| python3 python3-dev py3-wheel py3-pip py3-curl build-base libstdc++ linux-headers && \ | |
| /usr/bin/pip install --upgrade pip && \ | |
| /usr/bin/pip install -r requirements.txt && \ | |
| apk del build-base python3-dev | |
| COPY main.py run.sh / | |
| RUN chmod 755 /main.py /run.sh | |
| ENV MON_CRONSPEC "*/2 * * * *" | |
| ENV MON_HOSTSFILE "# no extra hosts to add from MON_HOSTSFILE variable" | |
| ENV CRONTAB /etc/crontabs/root | |
| ENV LOGFILE /var/log/latency.log | |
| CMD \ | |
| echo "$MON_HOSTSFILE" >> /etc/hosts && \ | |
| echo "$MON_CRONSPEC /run.sh >>$LOGFILE 2>&1" > $CRONTAB && \ | |
| crond -L /var/log/cron.log && \ | |
| touch $LOGFILE && \ | |
| tail -F $LOGFILE |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # Copyright 2021 Google LLC. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| # revised by @shanemhansen for Python3 | |
| import click | |
| import datetime | |
| import logging | |
| import pycurl | |
| import warnings | |
| from urllib.parse import urlparse | |
| from google.api_core.exceptions import GoogleAPIError | |
| from google.cloud import monitoring_v3 | |
| _BASE = 'custom.googleapis.com/mymetrics/latency' | |
| class LatencyError(Exception): | |
| pass | |
| def _logging_config(verbose=False): | |
| level = logging.INFO if verbose else logging.WARNING | |
| warnings.filterwarnings('ignore', r'.*end user credentials.*', UserWarning) | |
| logging.basicConfig(level=level) | |
| def _fetch_latency_data(url): | |
| c = pycurl.Curl() | |
| c.setopt(pycurl.URL, url) | |
| c.setopt(pycurl.FOLLOWLOCATION, 1) | |
| c.setopt(pycurl.WRITEFUNCTION, lambda _: None) | |
| try: | |
| c.perform() | |
| except pycurl.error as e: | |
| raise LatencyError(e.message, *e.args) | |
| data = { | |
| 'dns': c.getinfo(pycurl.NAMELOOKUP_TIME), | |
| 'tcp': c.getinfo(pycurl.CONNECT_TIME), | |
| 'ttfb': c.getinfo(pycurl.STARTTRANSFER_TIME), | |
| 'total': c.getinfo(pycurl.TOTAL_TIME), | |
| } | |
| c.close() | |
| return data | |
| def _get_series(metric_type, project_id, label, host, value, dt=None): | |
| series = monitoring_v3.types.TimeSeries() | |
| series.metric.type = '/'.join((_BASE, metric_type)) | |
| series.resource.type = 'global' | |
| series.metric.labels['label'] = label | |
| series.metric.labels['host'] = host | |
| point = monitoring_v3.types.Point() | |
| point.value.double_value = value | |
| point.interval.end_time = dt or datetime.datetime.utcnow() | |
| series.points.append(point) | |
| return series | |
| def _add_series(project_id, series, client=None): | |
| client = client or monitoring_v3.MetricServiceClient() | |
| seriesRequest = monitoring_v3.CreateTimeSeriesRequest() | |
| seriesRequest.name = client.common_project_path(project_id) | |
| if isinstance(series, monitoring_v3.types.TimeSeries): | |
| series = [series] | |
| seriesRequest.time_series = series | |
| try: | |
| client.create_time_series(seriesRequest) | |
| except GoogleAPIError as e: | |
| raise LatencyError('Error from monitoring API: %s' % e) | |
| @click.command() | |
| @click.option('--project-id', required=True, help='Stackdriver project id') | |
| @click.option('--label', required=True, help='Metric label') | |
| @click.option('--dry-run', default=False, is_flag=True, help='Skip Stackdriver') | |
| @click.option('--verbose', default=False, is_flag=True, help='Verbose logging') | |
| @click.argument('urls', required=True, nargs=-1) | |
| def main(project_id, urls, label, dry_run, verbose): | |
| _logging_config(verbose) | |
| logging.info('starting') | |
| for url in urls: | |
| logging.info('fetching URL %s', url) | |
| try: | |
| data = _fetch_latency_data(url) | |
| if not dry_run: | |
| series = [] | |
| for name, value in data.items(): | |
| series.append(_get_series(name, project_id, label, | |
| urlparse(url).netloc, value)) | |
| _add_series(project_id, series) | |
| except LatencyError as e: | |
| logging.exception(e) | |
| else: | |
| logging.info('data %s', data) | |
| if __name__ == '__main__': | |
| main(auto_envvar_prefix='MON') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| click | |
| google-cloud-monitoring |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh | |
| # Copyright 2018 Google LLC. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| /usr/bin/python /main.py $MON_URLS |
For me this didn't work on python3 so I made a few changes and put them here: https://gist.github.com/shanemhansen/eac6ffa84caeb7b378c2f1b5bfe89dce
Thanks a lot for the py3 fixes! Would you be ok with me incorporating them here (and giving attribution of course)?
No problem, feel free to use!
How do you pass --project-id and other flags into the main.py script?
Since it's using the click library it should be as simple as /usr/bin/python /main.py -project-id my-project $URLS
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
For me this didn't work on python3 so I made a few changes and put them here: https://gist.github.com/shanemhansen/eac6ffa84caeb7b378c2f1b5bfe89dce