Enhancement: support for Kubernetes gateway API (#4643)

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
Co-authored-by: lyons <gittea.sand@gmail.com>
Co-authored-by: Brett Dudo <brett@dudo.io>
This commit is contained in:
djeinstine 2025-02-12 03:57:22 +01:00 committed by GitHub
parent 2a95f88cdf
commit 91d5fc8e42
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 407 additions and 168 deletions

View file

@ -1,7 +1,7 @@
import { CoreV1Api, Metrics } from "@kubernetes/client-node";
import getKubeConfig from "../../../../utils/config/kubernetes";
import { parseCpu, parseMemory } from "../../../../utils/kubernetes/kubernetes-utils";
import { getKubeConfig } from "../../../../utils/config/kubernetes";
import { parseCpu, parseMemory } from "../../../../utils/kubernetes/utils";
import createLogger from "../../../../utils/logger";
const logger = createLogger("kubernetesStatsService");
@ -30,7 +30,10 @@ export default async function handler(req, res) {
const coreApi = kc.makeApiClient(CoreV1Api);
const metricsApi = new Metrics(kc);
const podsResponse = await coreApi
.listNamespacedPod(namespace, null, null, null, null, labelSelector)
.listNamespacedPod({
namespace,
labelSelector,
})
.then((response) => response.body)
.catch((err) => {
logger.error("Error getting pods: %d %s %s", err.statusCode, err.body, err.response);

View file

@ -1,6 +1,6 @@
import { CoreV1Api } from "@kubernetes/client-node";
import getKubeConfig from "../../../../utils/config/kubernetes";
import { getKubeConfig } from "../../../../utils/config/kubernetes";
import createLogger from "../../../../utils/logger";
const logger = createLogger("kubernetesStatusService");
@ -27,8 +27,11 @@ export default async function handler(req, res) {
}
const coreApi = kc.makeApiClient(CoreV1Api);
const podsResponse = await coreApi
.listNamespacedPod(namespace, null, null, null, null, labelSelector)
.then((response) => response.body)
.listNamespacedPod({
namespace,
labelSelector,
})
.then((response) => response)
.catch((err) => {
logger.error("Error getting pods: %d %s %s", err.statusCode, err.body, err.response);
return null;

View file

@ -1,10 +1,10 @@
import { CoreV1Api, Metrics } from "@kubernetes/client-node";
import getKubeConfig from "../../../utils/config/kubernetes";
import { parseCpu, parseMemory } from "../../../utils/kubernetes/kubernetes-utils";
import { getKubeConfig } from "../../../utils/config/kubernetes";
import { parseCpu, parseMemory } from "../../../utils/kubernetes/utils";
import createLogger from "../../../utils/logger";
const logger = createLogger("kubernetes-widget");
const logger = createLogger("widget");
export default async function handler(req, res) {
try {

View file

@ -2,18 +2,21 @@ import path from "path";
import { readFileSync } from "fs";
import yaml from "js-yaml";
import { KubeConfig } from "@kubernetes/client-node";
import { KubeConfig, ApiextensionsV1Api } from "@kubernetes/client-node";
import checkAndCopyConfig, { CONF_DIR, substituteEnvironmentVars } from "utils/config/config";
export default function getKubeConfig() {
export function getKubernetes() {
checkAndCopyConfig("kubernetes.yaml");
const configFile = path.join(CONF_DIR, "kubernetes.yaml");
const rawConfigData = readFileSync(configFile, "utf8");
const configData = substituteEnvironmentVars(rawConfigData);
const config = yaml.load(configData);
return yaml.load(configData);
}
export const getKubeConfig = () => {
const kc = new KubeConfig();
const config = getKubernetes();
switch (config?.mode) {
case "cluster":
@ -28,4 +31,31 @@ export default function getKubeConfig() {
}
return kc;
};
export async function checkCRD(name, kc, logger) {
const apiExtensions = kc.makeApiClient(ApiextensionsV1Api);
const exist = await apiExtensions
.readCustomResourceDefinitionStatus({
name,
})
.then(() => true)
.catch(async (error) => {
if (error.statusCode === 403) {
logger.error(
"Error checking if CRD %s exists. Make sure to add the following permission to your RBAC: %d %s %s",
name,
error.statusCode,
error.body.message,
);
}
return false;
});
return exist;
}
export const ANNOTATION_BASE = "gethomepage.dev";
export const ANNOTATION_WIDGET_BASE = `${ANNOTATION_BASE}/widget.`;
export const HTTPROUTE_API_GROUP = "gateway.networking.k8s.io";
export const HTTPROUTE_API_VERSION = "v1";

View file

@ -3,12 +3,12 @@ import path from "path";
import yaml from "js-yaml";
import Docker from "dockerode";
import { CustomObjectsApi, NetworkingV1Api, ApiextensionsV1Api } from "@kubernetes/client-node";
import createLogger from "utils/logger";
import checkAndCopyConfig, { CONF_DIR, getSettings, substituteEnvironmentVars } from "utils/config/config";
import getDockerArguments from "utils/config/docker";
import getKubeConfig from "utils/config/kubernetes";
import kubernetes from "utils/kubernetes/export";
import { getKubeConfig } from "utils/config/kubernetes";
import * as shvl from "utils/config/shvl";
const logger = createLogger("service-helpers");
@ -167,36 +167,7 @@ export async function servicesFromDocker() {
return mappedServiceGroups;
}
function getUrlFromIngress(ingress) {
const urlHost = ingress.spec.rules[0].host;
const urlPath = ingress.spec.rules[0].http.paths[0].path;
const urlSchema = ingress.spec.tls ? "https" : "http";
return `${urlSchema}://${urlHost}${urlPath}`;
}
export async function checkCRD(kc, name) {
const apiExtensions = kc.makeApiClient(ApiextensionsV1Api);
const exist = await apiExtensions
.readCustomResourceDefinitionStatus(name)
.then(() => true)
.catch(async (error) => {
if (error.statusCode === 403) {
logger.error(
"Error checking if CRD %s exists. Make sure to add the following permission to your RBAC: %d %s %s",
name,
error.statusCode,
error.body.message,
);
}
return false;
});
return exist;
}
export async function servicesFromKubernetes() {
const ANNOTATION_BASE = "gethomepage.dev";
const ANNOTATION_WIDGET_BASE = `${ANNOTATION_BASE}/widget.`;
const { instanceName } = getSettings();
checkAndCopyConfig("kubernetes.yaml");
@ -206,145 +177,46 @@ export async function servicesFromKubernetes() {
if (!kc) {
return [];
}
const networking = kc.makeApiClient(NetworkingV1Api);
const crd = kc.makeApiClient(CustomObjectsApi);
const ingressList = await networking
.listIngressForAllNamespaces(null, null, null, null)
.then((response) => response.body)
.catch((error) => {
logger.error("Error getting ingresses: %d %s %s", error.statusCode, error.body, error.response);
if (error) logger.debug(error);
return null;
});
// resource lists
const [ingressList, traefikIngressList, httpRouteList] = await Promise.all([
kubernetes.listIngress(),
kubernetes.listTraefikIngress(),
kubernetes.listHttpRoute(),
]);
const traefikContainoExists = await checkCRD(kc, "ingressroutes.traefik.containo.us");
const traefikExists = await checkCRD(kc, "ingressroutes.traefik.io");
const resources = [...ingressList, ...traefikIngressList, ...httpRouteList];
const traefikIngressListContaino = await crd
.listClusterCustomObject("traefik.containo.us", "v1alpha1", "ingressroutes")
.then((response) => response.body)
.catch(async (error) => {
if (traefikContainoExists) {
logger.error(
"Error getting traefik ingresses from traefik.containo.us: %d %s %s",
error.statusCode,
error.body,
error.response,
);
if (error) logger.debug(error);
}
return [];
});
const traefikIngressListIo = await crd
.listClusterCustomObject("traefik.io", "v1alpha1", "ingressroutes")
.then((response) => response.body)
.catch(async (error) => {
if (traefikExists) {
logger.error(
"Error getting traefik ingresses from traefik.io: %d %s %s",
error.statusCode,
error.body,
error.response,
);
if (error) logger.debug(error);
}
return [];
});
const traefikIngressList = [...(traefikIngressListContaino?.items ?? []), ...(traefikIngressListIo?.items ?? [])];
if (traefikIngressList.length > 0) {
const traefikServices = traefikIngressList.filter(
(ingress) => ingress.metadata.annotations && ingress.metadata.annotations[`${ANNOTATION_BASE}/href`],
);
ingressList.items.push(...traefikServices);
}
if (!ingressList) {
if (!resources) {
return [];
}
const services = ingressList.items
.filter(
(ingress) =>
ingress.metadata.annotations &&
ingress.metadata.annotations[`${ANNOTATION_BASE}/enabled`] === "true" &&
(!ingress.metadata.annotations[`${ANNOTATION_BASE}/instance`] ||
ingress.metadata.annotations[`${ANNOTATION_BASE}/instance`] === instanceName ||
`${ANNOTATION_BASE}/instance.${instanceName}` in ingress.metadata.annotations),
)
.map((ingress) => {
let constructedService = {
app: ingress.metadata.annotations[`${ANNOTATION_BASE}/app`] || ingress.metadata.name,
namespace: ingress.metadata.namespace,
href: ingress.metadata.annotations[`${ANNOTATION_BASE}/href`] || getUrlFromIngress(ingress),
name: ingress.metadata.annotations[`${ANNOTATION_BASE}/name`] || ingress.metadata.name,
group: ingress.metadata.annotations[`${ANNOTATION_BASE}/group`] || "Kubernetes",
weight: ingress.metadata.annotations[`${ANNOTATION_BASE}/weight`] || "0",
icon: ingress.metadata.annotations[`${ANNOTATION_BASE}/icon`] || "",
description: ingress.metadata.annotations[`${ANNOTATION_BASE}/description`] || "",
external: false,
type: "service",
};
if (ingress.metadata.annotations[`${ANNOTATION_BASE}/external`]) {
constructedService.external =
String(ingress.metadata.annotations[`${ANNOTATION_BASE}/external`]).toLowerCase() === "true";
}
if (ingress.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`] !== undefined) {
constructedService.podSelector = ingress.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`];
}
if (ingress.metadata.annotations[`${ANNOTATION_BASE}/ping`]) {
constructedService.ping = ingress.metadata.annotations[`${ANNOTATION_BASE}/ping`];
}
if (ingress.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`]) {
constructedService.siteMonitor = ingress.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`];
}
if (ingress.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`]) {
constructedService.statusStyle = ingress.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`];
}
Object.keys(ingress.metadata.annotations).forEach((annotation) => {
if (annotation.startsWith(ANNOTATION_WIDGET_BASE)) {
shvl.set(
constructedService,
annotation.replace(`${ANNOTATION_BASE}/`, ""),
ingress.metadata.annotations[annotation],
);
}
});
const services = await Promise.all(
resources
.filter((resource) => kubernetes.isDiscoverable(resource, instanceName))
.map(async (resource) => kubernetes.constructedServiceFromResource(resource)),
);
try {
constructedService = JSON.parse(substituteEnvironmentVars(JSON.stringify(constructedService)));
} catch (e) {
logger.error("Error attempting k8s environment variable substitution.");
if (e) logger.debug(e);
}
// map service groups
const mappedServiceGroups = services.reduce((groups, serverService) => {
let serverGroup = groups.find((group) => group.name === serverService.group);
return constructedService;
});
const mappedServiceGroups = [];
services.forEach((serverService) => {
let serverGroup = mappedServiceGroups.find((searchedGroup) => searchedGroup.name === serverService.group);
if (!serverGroup) {
mappedServiceGroups.push({
serverGroup = {
name: serverService.group,
services: [],
});
serverGroup = mappedServiceGroups[mappedServiceGroups.length - 1];
};
groups.push(serverGroup);
}
const { name: serviceName, group: serverServiceGroup, ...pushedService } = serverService;
const result = {
const { name: serviceName, group: _, ...pushedService } = serverService;
serverGroup.services.push({
name: serviceName,
...pushedService,
};
});
serverGroup.services.push(result);
});
return groups;
}, []);
return mappedServiceGroups;
} catch (e) {

View file

@ -0,0 +1,14 @@
import listIngress from "utils/kubernetes/ingress-list";
import listTraefikIngress from "utils/kubernetes/traefik-list";
import listHttpRoute from "utils/kubernetes/httproute-list";
import { isDiscoverable, constructedServiceFromResource } from "utils/kubernetes/resource-helpers";
const kubernetes = {
listIngress,
listTraefikIngress,
listHttpRoute,
isDiscoverable,
constructedServiceFromResource,
};
export default kubernetes;

View file

@ -0,0 +1,56 @@
import { CustomObjectsApi, CoreV1Api } from "@kubernetes/client-node";
import { getKubernetes, getKubeConfig, HTTPROUTE_API_GROUP, HTTPROUTE_API_VERSION } from "utils/config/kubernetes";
import createLogger from "utils/logger";
const logger = createLogger("httproute-list");
const kc = getKubeConfig();
export default async function listHttpRoute() {
const crd = kc.makeApiClient(CustomObjectsApi);
const core = kc.makeApiClient(CoreV1Api);
const { gateway } = getKubernetes();
let httpRouteList = [];
if (gateway) {
// httproutes
const getHttpRoute = async (namespace) =>
crd
.listNamespacedCustomObject({
group: HTTPROUTE_API_GROUP,
version: HTTPROUTE_API_VERSION,
namespace,
plural: "httproutes",
})
.then((response) => {
const [httpRoute] = response.items;
return httpRoute;
})
.catch((error) => {
logger.error("Error getting httproutes: %d %s %s", error.statusCode, error.body, error.response);
logger.debug(error);
return null;
});
// namespaces
const namespaces = await core
.listNamespace()
.then((response) => response.items.map((ns) => ns.metadata.name))
.catch((error) => {
logger.error("Error getting namespaces: %d %s %s", error.statusCode, error.body, error.response);
logger.debug(error);
return null;
});
if (namespaces) {
const httpRouteListUnfiltered = await Promise.all(
namespaces.map(async (namespace) => {
const httpRoute = await getHttpRoute(namespace);
return httpRoute;
}),
);
httpRouteList = httpRouteListUnfiltered.filter((httpRoute) => httpRoute !== undefined);
}
}
return httpRouteList;
}

View file

@ -0,0 +1,26 @@
import { NetworkingV1Api } from "@kubernetes/client-node";
import { getKubernetes, getKubeConfig } from "utils/config/kubernetes";
import createLogger from "utils/logger";
const logger = createLogger("ingress-list");
const kc = getKubeConfig();
export default async function listIngress() {
const networking = kc.makeApiClient(NetworkingV1Api);
const { ingress } = getKubernetes();
let ingressList = [];
if (ingress) {
const ingressData = await networking
.listIngressForAllNamespaces()
.then((response) => response)
.catch((error) => {
logger.error("Error getting ingresses: %d %s %s", error.statusCode, error.body, error.response);
logger.debug(error);
return null;
});
ingressList = ingressData.items;
}
return ingressList;
}

View file

@ -0,0 +1,130 @@
import { CustomObjectsApi } from "@kubernetes/client-node";
import {
getKubeConfig,
ANNOTATION_BASE,
ANNOTATION_WIDGET_BASE,
HTTPROUTE_API_GROUP,
HTTPROUTE_API_VERSION,
} from "utils/config/kubernetes";
import { substituteEnvironmentVars } from "utils/config/config";
import createLogger from "utils/logger";
import * as shvl from "utils/config/shvl";
const logger = createLogger("resource-helpers");
const kc = getKubeConfig();
const getSchemaFromGateway = async (gatewayRef) => {
const crd = kc.makeApiClient(CustomObjectsApi);
const schema = await crd
.getNamespacedCustomObject({
group: HTTPROUTE_API_GROUP,
version: HTTPROUTE_API_VERSION,
namespace: gatewayRef.namespace,
plural: "gateways",
name: gatewayRef.name,
})
.then((response) => {
const listner = response.spec.listeners.filter((listener) => listener.name === gatewayRef.sectionName)[0];
return listner.protocol.toLowerCase();
})
.catch((error) => {
logger.error("Error getting gateways: %d %s %s", error.statusCode, error.body, error.response);
logger.debug(error);
return "";
});
return schema;
};
async function getUrlFromHttpRoute(resource) {
let url = null;
const hasHostName = resource.spec?.hostnames;
if (hasHostName) {
if (resource.spec.rules[0].matches[0].path.type !== "RegularExpression") {
const urlHost = resource.spec.hostnames[0];
const urlPath = resource.spec.rules[0].matches[0].path.value;
const urlSchema = (await getSchemaFromGateway(resource.spec.parentRefs[0])) ? "https" : "http";
url = `${urlSchema}://${urlHost}${urlPath}`;
}
}
return url;
}
function getUrlFromIngress(resource) {
const urlHost = resource.spec.rules[0].host;
const urlPath = resource.spec.rules[0].http.paths[0].path;
const urlSchema = resource.spec.tls ? "https" : "http";
return `${urlSchema}://${urlHost}${urlPath}`;
}
async function getUrlSchema(resource) {
const isHttpRoute = resource.kind === "HTTPRoute";
let urlSchema;
if (isHttpRoute) {
urlSchema = getUrlFromHttpRoute(resource);
} else {
urlSchema = getUrlFromIngress(resource);
}
return urlSchema;
}
export function isDiscoverable(resource, instanceName) {
return (
resource.metadata.annotations &&
resource.metadata.annotations[`${ANNOTATION_BASE}/enabled`] === "true" &&
(!resource.metadata.annotations[`${ANNOTATION_BASE}/instance`] ||
resource.metadata.annotations[`${ANNOTATION_BASE}/instance`] === instanceName ||
`${ANNOTATION_BASE}/instance.${instanceName}` in resource.metadata.annotations)
);
}
export async function constructedServiceFromResource(resource) {
let constructedService = {
app: resource.metadata.annotations[`${ANNOTATION_BASE}/app`] || resource.metadata.name,
namespace: resource.metadata.namespace,
href: resource.metadata.annotations[`${ANNOTATION_BASE}/href`] || (await getUrlSchema(resource)),
name: resource.metadata.annotations[`${ANNOTATION_BASE}/name`] || resource.metadata.name,
group: resource.metadata.annotations[`${ANNOTATION_BASE}/group`] || "Kubernetes",
weight: resource.metadata.annotations[`${ANNOTATION_BASE}/weight`] || "0",
icon: resource.metadata.annotations[`${ANNOTATION_BASE}/icon`] || "",
description: resource.metadata.annotations[`${ANNOTATION_BASE}/description`] || "",
external: false,
type: "service",
};
if (resource.metadata.annotations[`${ANNOTATION_BASE}/external`]) {
constructedService.external =
String(resource.metadata.annotations[`${ANNOTATION_BASE}/external`]).toLowerCase() === "true";
}
if (resource.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`] !== undefined) {
constructedService.podSelector = resource.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`];
}
if (resource.metadata.annotations[`${ANNOTATION_BASE}/ping`]) {
constructedService.ping = resource.metadata.annotations[`${ANNOTATION_BASE}/ping`];
}
if (resource.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`]) {
constructedService.siteMonitor = resource.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`];
}
if (resource.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`]) {
constructedService.statusStyle = resource.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`];
}
Object.keys(resource.metadata.annotations).forEach((annotation) => {
if (annotation.startsWith(ANNOTATION_WIDGET_BASE)) {
shvl.set(
constructedService,
annotation.replace(`${ANNOTATION_BASE}/`, ""),
resource.metadata.annotations[annotation],
);
}
});
try {
constructedService = JSON.parse(substituteEnvironmentVars(JSON.stringify(constructedService)));
} catch (e) {
logger.error("Error attempting k8s environment variable substitution.");
logger.debug(e);
}
return constructedService;
}

View file

@ -0,0 +1,70 @@
import { CustomObjectsApi } from "@kubernetes/client-node";
import { getKubernetes, getKubeConfig, checkCRD, ANNOTATION_BASE } from "utils/config/kubernetes";
import createLogger from "utils/logger";
const logger = createLogger("traefik-list");
const kc = getKubeConfig();
export default async function listTraefikIngress() {
const { traefik } = getKubernetes();
const traefikList = [];
if (traefik) {
const crd = kc.makeApiClient(CustomObjectsApi);
const traefikContainoExists = await checkCRD("ingressroutes.traefik.containo.us", kc, logger);
const traefikExists = await checkCRD("ingressroutes.traefik.io", kc, logger);
const traefikIngressListContaino = await crd
.listClusterCustomObject({
group: "traefik.containo.us",
version: "v1alpha1",
plural: "ingressroutes",
})
.then((response) => response)
.catch(async (error) => {
if (traefikContainoExists) {
logger.error(
"Error getting traefik ingresses from traefik.containo.us: %d %s %s",
error.statusCode,
error.body,
error.response,
);
logger.debug(error);
}
return [];
});
const traefikIngressListIo = await crd
.listClusterCustomObject({
group: "traefik.io",
version: "v1alpha1",
plural: "ingressroutes",
})
.then((response) => response.body)
.catch(async (error) => {
if (traefikExists) {
logger.error(
"Error getting traefik ingresses from traefik.io: %d %s %s",
error.statusCode,
error.body,
error.response,
);
logger.debug(error);
}
return [];
});
const traefikIngressList = [...(traefikIngressListContaino?.items ?? []), ...(traefikIngressListIo?.items ?? [])];
if (traefikIngressList.length > 0) {
const traefikServices = traefikIngressList.filter(
(ingress) => ingress.metadata.annotations && ingress.metadata.annotations[`${ANNOTATION_BASE}/href`],
);
traefikList.push(...traefikServices);
}
}
return traefikList;
}