Skip to content

Commit

Permalink
[ci/release] Do not store metrics in DB (#31755)
Browse files Browse the repository at this point in the history
We stop storing metric jsons in the release test database as they are now uploaded as separate buildkite artifacts and can be obtained using buildkite's API (metric collection is unaffected).

https://buildkite.com/docs/apis/rest-api/artifacts

Signed-off-by: Antoni Baum <[email protected]>
  • Loading branch information
Yard1 committed Jan 20, 2023
1 parent e0bd59a commit faa9a2c
Showing 1 changed file with 4 additions and 16 deletions.
20 changes: 4 additions & 16 deletions release/ray_release/reporter/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ def __init__(self):
def report_result(self, test: Test, result: Result):
logger.info("Persisting result to the databricks delta lake...")

# Prometheus metrics are saved as buildkite artifacts
# and can be obtained using buildkite API.

result_json = {
"_table": "release_test_result",
"report_timestamp_ms": int(time.time() * 1000),
Expand All @@ -34,7 +37,6 @@ def report_result(self, test: Test, result: Result):
"stable": result.stable,
"return_code": result.return_code,
"smoke_test": result.smoke_test,
"prometheus_metrics": result.prometheus_metrics or {},
"extra_tags": result.extra_tags or {},
}

Expand All @@ -46,20 +48,6 @@ def report_result(self, test: Test, result: Result):
Record={"Data": json.dumps(result_json)},
)
except Exception:
try:
# This may happen if metrics are too big.
# TODO persist big metrics in an alternative fashion
logger.warning(
"Couldn't persist with prometheus_metrics, trying without them"
)
result_json.pop("prometheus_metrics", None)
self.firehose.put_record(
DeliveryStreamName="ray-ci-results",
Record={"Data": json.dumps(result_json)},
)
except Exception:
logger.exception(
"Failed to persist result to the databricks delta lake"
)
logger.exception("Failed to persist result to the databricks delta lake")
else:
logger.info("Result has been persisted to the databricks delta lake")

0 comments on commit faa9a2c

Please sign in to comment.