Improved kubernetes error handling

gethomepage · Nov 6, 2022 · 4fc6db4 · 4fc6db4
1 parent 8887fcc
commit 4fc6db4
Show file tree

Hide file tree

Showing 8 changed files with 208 additions and 127 deletions.
diff --git a/src/components/widgets/longhorn/longhorn.jsx b/src/components/widgets/longhorn/longhorn.jsx
@@ -1,6 +1,6 @@
 import useSWR from "swr";
 import { BiError } from "react-icons/bi";
-import { i18n, useTranslation } from "next-i18next";
+import { useTranslation } from "next-i18next";
 
 import Node from "./node";
 

diff --git a/src/pages/api/kubernetes/stats/[...service].js b/src/pages/api/kubernetes/stats/[...service].js
@@ -2,15 +2,18 @@ import { CoreV1Api, Metrics } from "@kubernetes/client-node";
 
 import getKubeConfig from "../../../../utils/config/kubernetes";
 import { parseCpu, parseMemory } from "../../../../utils/kubernetes/kubernetes-utils";
+import createLogger from "../../../../utils/logger";
+
+const logger = createLogger("kubernetesStatsService");
 
 export default async function handler(req, res) {
- const APP_LABEL =  "app.kubernetes.io/name";
+ const APP_LABEL = "app.kubernetes.io/name";
  const { service } = req.query;
 
  const [namespace, appName] = service;
  if (!namespace && !appName) {
  res.status(400).send({
- error: "kubernetes query parameters are required",
+ error: "kubernetes query parameters are required"
  });
  return;
  }
@@ -20,12 +23,23 @@ export default async function handler(req, res) {
  const kc = getKubeConfig();
  const coreApi = kc.makeApiClient(CoreV1Api);
  const metricsApi = new Metrics(kc);
- const podsResponse = await coreApi.listNamespacedPod(namespace, null, null, null, null, labelSelector);
- const pods = podsResponse.body.items;
+ const podsResponse = await coreApi.listNamespacedPod(namespace, null, null, null, null, labelSelector)
+ .then((response) => response.body)
+ .catch((err) => {
+ logger.error("Error getting pods: %d %s %s", err.statusCode, err.body, err.response);
+ return null;
+ });
+ if (!podsResponse) {
+ res.status(500).send({
+ error: "Error communicating with kubernetes"
+ });
+ return;
+ }
+ const pods = podsResponse.items;
 
  if (pods.length === 0) {
- res.status(200).send({
- error: "not found",
+ res.status(404).send({
+ error: "not found"
  });
  return;
  }
@@ -46,34 +60,43 @@ export default async function handler(req, res) {
  const stats = await pods.map(async (pod) => {
  let depMem = 0;
  let depCpu = 0;
- const podMetrics = await metricsApi.getPodMetrics(namespace, pod.metadata.name);
- podMetrics.containers.forEach((container) => {
- depMem += parseMemory(container.usage.memory);
- depCpu += parseCpu(container.usage.cpu);
- });
+ const podMetrics = await metricsApi.getPodMetrics(namespace, pod.metadata.name)
+ .then((response) => response)
+ .catch((err) => {
+ // 404 generally means that the metrics have not been populated yet
+ if (err.statusCode !== 404) {
+ logger.error("Error getting pod metrics: %d %s %s", err.statusCode, err.body, err.response);
+ }
+ return null;
+ });
+ if (podMetrics) {
+ podMetrics.containers.forEach((container) => {
+ depMem += parseMemory(container.usage.memory);
+ depCpu += parseCpu(container.usage.cpu);
+ });
+ }
  return {
  mem: depMem,
  cpu: depCpu
- }
- }).reduce(async (finalStats, podStatPromise) => {
- const podStats = await podStatPromise;
- return {
- mem: finalStats.mem + podStats.mem,
- cpu: finalStats.cpu + podStats.cpu
  };
- });
+ }).reduce(async (finalStats, podStatPromise) => {
+ const podStats = await podStatPromise;
+ return {
+ mem: finalStats.mem + podStats.mem,
+ cpu: finalStats.cpu + podStats.cpu
+ };
+ });
  stats.cpuLimit = cpuLimit;
  stats.memLimit = memLimit;
- stats.cpuUsage = stats.cpu / cpuLimit;
- stats.memUsage = stats.mem / memLimit;
-
+ stats.cpuUsage = cpuLimit ? stats.cpu / cpuLimit : 0;
+ stats.memUsage = memLimit ? stats.mem / memLimit : 0;
  res.status(200).json({
- stats,
+ stats
  });
  } catch (e) {
- console.log("error", e);
+ logger.error(e);
  res.status(500).send({
- error: "unknown error",
+ error: "unknown error"
  });
  }
 }
diff --git a/src/pages/api/kubernetes/status/[...service].js b/src/pages/api/kubernetes/status/[...service].js
@@ -1,6 +1,9 @@
 import { CoreV1Api } from "@kubernetes/client-node";
 
 import getKubeConfig from "../../../../utils/config/kubernetes";
+import createLogger from "../../../../utils/logger";
+
+const logger = createLogger("kubernetesStatusService");
 
 export default async function handler(req, res) {
  const APP_LABEL = "app.kubernetes.io/name";
@@ -18,11 +21,22 @@ export default async function handler(req, res) {
  try {
  const kc = getKubeConfig();
  const coreApi = kc.makeApiClient(CoreV1Api);
- const podsResponse = await coreApi.listNamespacedPod(namespace, null, null, null, null, labelSelector);
- const pods = podsResponse.body.items;
+ const podsResponse = await coreApi.listNamespacedPod(namespace, null, null, null, null, labelSelector)
+ .then((response) => response.body)
+ .catch((err) => {
+ logger.error("Error getting pods: %d %s %s", err.statusCode, err.body, err.response);
+ return null;
+ });
+ if (!podsResponse) {
+ res.status(500).send({
+ error: "Error communicating with kubernetes"
+ });
+ return;
+ }
+ const pods = podsResponse.items;
 
  if (pods.length === 0) {
- res.status(200).send({
+ res.status(404).send({
  error: "not found",
  });
  return;
@@ -34,7 +48,8 @@ export default async function handler(req, res) {
  res.status(200).json({
  status
  });
- } catch {
+ } catch (e) {
+ logger.error(e);
  res.status(500).send({
  error: "unknown error",
  });

diff --git a/src/pages/api/widgets/kubernetes.js b/src/pages/api/widgets/kubernetes.js
@@ -2,59 +2,79 @@ import { CoreV1Api, Metrics } from "@kubernetes/client-node";
 
 import getKubeConfig from "../../../utils/config/kubernetes";
 import { parseCpu, parseMemory } from "../../../utils/kubernetes/kubernetes-utils";
+import createLogger from "../../../utils/logger";
+
+const logger = createLogger("kubernetes-widget");
 
 export default async function handler(req, res) {
  const { type } = req.query;
 
- const kc = getKubeConfig();
- const coreApi = kc.makeApiClient(CoreV1Api);
- const metricsApi = new Metrics(kc);
-
- const nodes = await coreApi.listNode();
- const nodeCapacity = new Map();
- let cpuTotal = 0;
- let cpuUsage = 0;
- let memTotal = 0;
- let memUsage = 0;
-
- nodes.body.items.forEach((node) => {
- nodeCapacity.set(node.metadata.name, node.status.capacity);
- cpuTotal += Number.parseInt(node.status.capacity.cpu, 10);
- memTotal += parseMemory(node.status.capacity.memory);
- });
-
- const nodeMetrics = await metricsApi.getNodeMetrics();
- const nodeUsage = new Map();
- nodeMetrics.items.forEach((metrics) => {
- nodeUsage.set(metrics.metadata.name, metrics.usage);
- cpuUsage += parseCpu(metrics.usage.cpu);
- memUsage += parseMemory(metrics.usage.memory);
- });
-
- if (type === "cpu") {
- return res.status(200).json({
- cpu: {
- usage: (cpuUsage / cpuTotal) * 100,
- load: cpuUsage
- }
+ try {
+ const kc = getKubeConfig();
+ const coreApi = kc.makeApiClient(CoreV1Api);
+ const metricsApi = new Metrics(kc);
+
+ const nodes = await coreApi.listNode()
+ .then((response) => response.body)
+ .catch((error) => {
+ logger.error("Error getting ingresses: %d %s %s", error.statusCode, error.body, error.response);
+ return null;
+ });
+ if (!nodes) {
+ return res.status(500).send({
+ error: "unknown error"
+ });
+ }
+ const nodeCapacity = new Map();
+ let cpuTotal = 0;
+ let cpuUsage = 0;
+ let memTotal = 0;
+ let memUsage = 0;
+
+ nodes.items.forEach((node) => {
+ nodeCapacity.set(node.metadata.name, node.status.capacity);
+ cpuTotal += Number.parseInt(node.status.capacity.cpu, 10);
+ memTotal += parseMemory(node.status.capacity.memory);
  });
- }
 
- if (type === "memory") {
- const SCALE_MB = 1024 * 1024;
- const usedMemMb = memUsage / SCALE_MB;
- const totalMemMb = memTotal / SCALE_MB;
- const freeMemMb = totalMemMb - usedMemMb;
- return res.status(200).json({
- memory: {
- usedMemMb,
- freeMemMb,
- totalMemMb
- }
+ const nodeMetrics = await metricsApi.getNodeMetrics();
+ const nodeUsage = new Map();
+ nodeMetrics.items.forEach((metrics) => {
+ nodeUsage.set(metrics.metadata.name, metrics.usage);
+ cpuUsage += parseCpu(metrics.usage.cpu);
+ memUsage += parseMemory(metrics.usage.memory);
  });
- }
 
- return res.status(400).json({
- error: "invalid type"
- });
+ if (type === "cpu") {
+ return res.status(200).json({
+ cpu: {
+ usage: (cpuUsage / cpuTotal) * 100,
+ load: cpuUsage
+ }
+ });
+ }
+
+ if (type === "memory") {
+ const SCALE_MB = 1024 * 1024;
+ const usedMemMb = memUsage / SCALE_MB;
+ const totalMemMb = memTotal / SCALE_MB;
+ const freeMemMb = totalMemMb - usedMemMb;
+ return res.status(200).json({
+ memory: {
+ usedMemMb,
+ freeMemMb,
+ totalMemMb
+ }
+ });
+ }
+
+ return res.status(400).json({
+ error: "invalid type"
+ });
+ } catch (e) {
+ logger.error("exception %s", e);
+ return res.status(500).send({
+ error: "unknown error"
+ });
+ }
 }
diff --git a/src/utils/config/api-response.js b/src/utils/config/api-response.js
@@ -64,7 +64,7 @@ export async function servicesResponse() {
  try {
  discoveredKubernetesServices = cleanServiceGroups(await servicesFromKubernetes());
  } catch (e) {
- console.error("Failed to discover services, please check docker.yaml for errors or remove example entries.");
+ console.error("Failed to discover services, please check kubernetes.yaml for errors or remove example entries.");
  if (e) console.error(e);
  discoveredKubernetesServices = [];
  }