From 88cededc3c9308f54f59c572477238935874a21e Mon Sep 17 00:00:00 2001 From: mclarke47 Date: Sat, 2 Oct 2021 09:27:02 +0100 Subject: [PATCH 1/8] add top pods function --- src/top.ts | 78 ++++++++++++++- src/top_test.ts | 251 ++++++++++++++++++++++++++++++++++++++++++++++++ src/util.ts | 2 + 3 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 src/top_test.ts diff --git a/src/top.ts b/src/top.ts index 463b1d661c..51b0ba76be 100644 --- a/src/top.ts +++ b/src/top.ts @@ -1,4 +1,5 @@ -import { CoreV1Api, V1Node, V1Pod } from './gen/api'; +import { CoreV1Api, V1Node, V1Pod, V1PodList, V1Container } from './gen/api'; +import { Metrics, PodMetric } from './metrics'; import { add, podsForNode, quantityToScalar, totalCPU, totalMemory } from './util'; export class ResourceUsage { @@ -9,6 +10,14 @@ export class ResourceUsage { ) {} } +export class CurrentResourceUsage { + constructor( + public readonly CurrentUsage: number | BigInt, + public readonly RequestTotal: number | BigInt, + public readonly LimitTotal: number | BigInt, + ) {} +} + export class NodeStatus { constructor( public readonly Node: V1Node, @@ -17,6 +26,25 @@ export class NodeStatus { ) {} } +export class ContainerStatus { + constructor( + public readonly Container: string, + public readonly CPUUsage: number | BigInt, + public readonly Memory: number | BigInt, + ) {} +} + +export class PodStatus { + constructor( + public readonly Pod: V1Pod, + public readonly CPU: CurrentResourceUsage, + public readonly Memory: CurrentResourceUsage, + public readonly Containers: ContainerStatus[], + ) {} +} + + + export async function topNodes(api: CoreV1Api): Promise { // TODO: Support metrics APIs in the client and this library const nodes = await api.listNode(); @@ -46,3 +74,51 @@ export async function topNodes(api: CoreV1Api): Promise { } return result; } + +export async function topPods(api: CoreV1Api, metrics: Metrics, namespace?: string): Promise { + + const getPodList = async ():Promise => { + if (namespace) { + return (await api.listNamespacedPod(namespace)).body + } else { + return (await api.listPodForAllNamespaces()).body + } + } + + const [podMetrics, podList] = await Promise.all([metrics.getPodMetrics(namespace), getPodList()]) + + const podMetricsMap = podMetrics.items.reduce((accum, next) => { + accum.set(next.metadata.name, next) + return accum + }, (new Map())) + + const result: PodStatus[] = []; + for (const pod of podList.items) { + + const podMetric = podMetricsMap.get(pod.metadata!.name!) + + const cpuTotal = totalCPU(pod); + const memTotal = totalMemory(pod); + const containerStatuses: ContainerStatus[] = []; + let currentPodCPU: number | bigint = 0; + let currentPodMem: number | bigint = 0; + + if (podMetric !== undefined){ + podMetric.containers.forEach(container => { + const containerCPUUsage = quantityToScalar(container.usage.cpu); + const containerMemUsage = quantityToScalar(container.usage.memory); + currentPodCPU = add(currentPodCPU, containerCPUUsage) + currentPodMem = add(currentPodMem, containerMemUsage) + containerStatuses.push(new ContainerStatus(container.name, containerCPUUsage, containerMemUsage)) + }) + } + + const cpuUsage = new CurrentResourceUsage(currentPodCPU, cpuTotal.request, cpuTotal.limit); + const memUsage = new CurrentResourceUsage(currentPodMem, memTotal.request, memTotal.limit); + result.push(new PodStatus(pod, cpuUsage, memUsage, containerStatuses)); + + } + return result; + + +} diff --git a/src/top_test.ts b/src/top_test.ts new file mode 100644 index 0000000000..4a2cae6046 --- /dev/null +++ b/src/top_test.ts @@ -0,0 +1,251 @@ +import { fail } from 'assert'; +import { expect } from 'chai'; +import * as nock from 'nock'; +import { KubeConfig } from './config'; +import { V1Status, HttpError, V1Pod } from './gen/api'; +import { Metrics, NodeMetricsList, PodMetricsList } from './metrics'; +import { topPods } from './top'; +import { CoreV1Api, } from './gen/api'; + + +const emptyPodMetrics: PodMetricsList = { + kind: 'PodMetricsList', + apiVersion: 'metrics.k8s.io/v1beta1', + metadata: { + selfLink: '/apis/metrics.k8s.io/v1beta1/pods', + }, + items: [], +}; + +const mockedPodMetrics: PodMetricsList = { + kind: 'PodMetricsList', + apiVersion: 'metrics.k8s.io/v1beta1', + metadata: { selfLink: '/apis/metrics.k8s.io/v1beta1/pods/' }, + items: [ + { + metadata: { + name: 'dice-roller-7c76898b4d-shm9p', + namespace: 'default', + selfLink: '/apis/metrics.k8s.io/v1beta1/namespaces/default/pods/dice-roller-7c76898b4d-shm9p', + creationTimestamp: '2021-09-26T11:57:27Z', + }, + timestamp: '2021-09-26T11:57:21Z', + window: '30s', + containers: [{ name: 'nginx', usage: { cpu: '10', memory: '3912Ki' } }], + }, + { + metadata: { + name: 'other-pod-7c76898b4e-12kj', + namespace: 'default', + selfLink: '/apis/metrics.k8s.io/v1beta1/namespaces/default/pods/other-pod-7c76898b4e-12kj', + creationTimestamp: '2021-09-26T11:57:27Z', + }, + timestamp: '2021-09-26T11:57:21Z', + window: '30s', + containers: [ + { name: 'nginx', usage: { cpu: '15', memory: '4012Ki' } }, + { name: 'sidecar', usage: { cpu: '16', memory: '3012Ki' } }, + ], + }, + ], +}; + +const podList: V1Pod[] = [ + { + "metadata": { + "name": "dice-roller-7c76898b4d-shm9p" + }, + 'spec': { + containers: [ + { + name: "nginx", + resources: { + requests: { + memory: "100Mi", + cpu: "100m" + }, + limits: { + memory: "100Mi", + cpu: "100m" + }, + } + } + ] + } + }, + { + "metadata": { + "name": "other-pod-7c76898b4e-12kj" + }, + 'spec': { + containers: [ + { + name: "nginx", + resources: { + requests: { + memory: "100Mi", + cpu: "100m" + }, + limits: { + memory: "100Mi", + cpu: "100m" + }, + } + }, + { + name: "sidecar", + resources: { + requests: { + memory: "50Mi", + cpu: "1" + }, + limits: { + memory: "100Mi", + cpu: "1" + }, + } + } + ] + } + } +] + + +const TEST_NAMESPACE = 'test-namespace'; + +const testConfigOptions: any = { + clusters: [{ name: 'cluster', server: 'https://127.0.0.1:51010' }], + users: [{ name: 'user', password: 'password' }], + contexts: [{ name: 'currentContext', cluster: 'cluster', user: 'user' }], + currentContext: 'currentContext', +}; + +const systemUnderTest = (namespace?: string, options: any = testConfigOptions): [() => ReturnType, nock.Scope] => { + const kc = new KubeConfig(); + kc.loadFromOptions(options); + const metricsClient = new Metrics(kc); + const core = kc.makeApiClient(CoreV1Api); + const topPodsFunc = () => topPods(core, metricsClient, namespace); + + const scope = nock(testConfigOptions.clusters[0].server); + + return [topPodsFunc, scope]; +}; + +describe('Top', () => { + describe('topPods', () => { + it('should return empty when no pods', async () => { + const [topPodsFunc, scope] = systemUnderTest(); + const podMetrics = scope.get('/apis/metrics.k8s.io/v1beta1/pods').reply(200, emptyPodMetrics); + const pods = scope.get('/api/v1/pods').reply(200, { + items: [] + }); + const result = await topPodsFunc(); + expect(result).to.deep.equal([]); + podMetrics.done(); + pods.done(); + }); + it('should return cluster wide pod metrics', async () => { + const [topPodsFunc, scope] = systemUnderTest(); + const podMetrics = scope.get('/apis/metrics.k8s.io/v1beta1/pods').reply(200, mockedPodMetrics); + const pods = scope.get('/api/v1/pods').reply(200, { + items: podList + }); + const result = await topPodsFunc(); + expect(result.length).to.equal(2); + expect(result[0].CPU).to.deep.equal({ + // TODO fix this + "CurrentUsage": 10, + "LimitTotal": 0.1, + "RequestTotal": 0.1 + }); + expect(result[0].Memory).to.deep.equal({ + "CurrentUsage": BigInt("4005888"), + "RequestTotal": BigInt("104857600"), + "LimitTotal": BigInt("104857600"), + }); + expect(result[0].Containers).to.deep.equal([ + { + "CPUUsage": 10, + "Container": "nginx", + "Memory": BigInt("4005888") + } + ]); + expect(result[1].CPU).to.deep.equal({ + // TODO fix this + "CurrentUsage": 31, + "LimitTotal": 1.1, + "RequestTotal": 1.1 + }); + expect(result[1].Memory).to.deep.equal({ + "CurrentUsage": BigInt("7192576"), + "LimitTotal": BigInt("209715200"), + "RequestTotal": BigInt("157286400") + }); + expect(result[1].Containers).to.deep.equal( [ + { + "CPUUsage": 15, + "Container": "nginx", + "Memory": BigInt("4108288"), + }, + { + "CPUUsage": 16, + "Container": "sidecar", + "Memory": BigInt("3084288"), + } + ]); + podMetrics.done(); + pods.done(); + }); + it('should return namespace pod metrics', async () => { + const [topPodsFunc, scope] = systemUnderTest(TEST_NAMESPACE); + const podMetrics = scope.get(`/apis/metrics.k8s.io/v1beta1/namespaces/${TEST_NAMESPACE}/pods`).reply(200, mockedPodMetrics); + const pods = scope.get(`/api/v1/namespaces/${TEST_NAMESPACE}/pods`).reply(200, { + items: podList + }); + const result = await topPodsFunc(); + expect(result.length).to.equal(2); + expect(result[0].CPU).to.deep.equal({ + "CurrentUsage": 10, + "LimitTotal": 0.1, + "RequestTotal": 0.1 + }); + expect(result[0].Memory).to.deep.equal({ + "CurrentUsage": BigInt("4005888"), + "RequestTotal": BigInt("104857600"), + "LimitTotal": BigInt("104857600"), + }); + expect(result[0].Containers).to.deep.equal([ + { + "CPUUsage": 10, + "Container": "nginx", + "Memory": BigInt("4005888") + } + ]); + expect(result[1].CPU).to.deep.equal({ + "CurrentUsage": 31, + "LimitTotal": 1.1, + "RequestTotal": 1.1 + }); + expect(result[1].Memory).to.deep.equal({ + "CurrentUsage": BigInt("7192576"), + "LimitTotal": BigInt("209715200"), + "RequestTotal": BigInt("157286400") + }); + expect(result[1].Containers).to.deep.equal( [ + { + "CPUUsage": 15, + "Container": "nginx", + "Memory": BigInt("4108288"), + }, + { + "CPUUsage": 16, + "Container": "sidecar", + "Memory": BigInt("3084288"), + } + ]); + podMetrics.done(); + pods.done(); + }); + }); +}); \ No newline at end of file diff --git a/src/util.ts b/src/util.ts index 37efc3ef2b..103728eb20 100644 --- a/src/util.ts +++ b/src/util.ts @@ -27,6 +27,8 @@ export function quantityToScalar(quantity: string): number | bigint { return num; } switch (suffix) { + case 'n': + return Number(quantity.substr(0, quantity.length - 1)).valueOf() / 1_000_000.0; case 'm': return Number(quantity.substr(0, quantity.length - 1)).valueOf() / 1000.0; case 'Ki': From 4ac6d392941eb6d31c3ca0925865cfc0b895ce48 Mon Sep 17 00:00:00 2001 From: mclarke47 Date: Sat, 2 Oct 2021 10:31:25 +0100 Subject: [PATCH 2/8] fix container resource calculating --- src/top.ts | 102 ++++++++++++------ src/top_test.ts | 268 ++++++++++++++++++++++++++++-------------------- src/util.ts | 36 +++++-- 3 files changed, 256 insertions(+), 150 deletions(-) diff --git a/src/top.ts b/src/top.ts index 51b0ba76be..44e96be0d2 100644 --- a/src/top.ts +++ b/src/top.ts @@ -1,6 +1,15 @@ -import { CoreV1Api, V1Node, V1Pod, V1PodList, V1Container } from './gen/api'; -import { Metrics, PodMetric } from './metrics'; -import { add, podsForNode, quantityToScalar, totalCPU, totalMemory } from './util'; +import { CoreV1Api, V1Node, V1Pod, V1PodList } from './gen/api'; +import { ContainerMetric, Metrics, PodMetric } from './metrics'; +import { + add, + podsForNode, + quantityToScalar, + ResourceStatus, + totalCPU, + totalCPUForContainer, + totalMemory, + totalMemoryForContainer, +} from './util'; export class ResourceUsage { constructor( @@ -29,8 +38,8 @@ export class NodeStatus { export class ContainerStatus { constructor( public readonly Container: string, - public readonly CPUUsage: number | BigInt, - public readonly Memory: number | BigInt, + public readonly CPUUsage: CurrentResourceUsage, + public readonly MemoryUsage: CurrentResourceUsage, ) {} } @@ -43,8 +52,6 @@ export class PodStatus { ) {} } - - export async function topNodes(api: CoreV1Api): Promise { // TODO: Support metrics APIs in the client and this library const nodes = await api.listNode(); @@ -75,50 +82,81 @@ export async function topNodes(api: CoreV1Api): Promise { return result; } +// TODO describe what this method does export async function topPods(api: CoreV1Api, metrics: Metrics, namespace?: string): Promise { - - const getPodList = async ():Promise => { + const getPodList = async (): Promise => { if (namespace) { - return (await api.listNamespacedPod(namespace)).body + return (await api.listNamespacedPod(namespace)).body; } else { - return (await api.listPodForAllNamespaces()).body + return (await api.listPodForAllNamespaces()).body; } - } + }; - const [podMetrics, podList] = await Promise.all([metrics.getPodMetrics(namespace), getPodList()]) + const [podMetrics, podList] = await Promise.all([metrics.getPodMetrics(namespace), getPodList()]); + // Create a map of pod names to their metric usage + // to make it easier to look up when we need it later const podMetricsMap = podMetrics.items.reduce((accum, next) => { - accum.set(next.metadata.name, next) - return accum - }, (new Map())) + accum.set(next.metadata.name, next); + return accum; + }, new Map()); const result: PodStatus[] = []; for (const pod of podList.items) { + const podMetric = podMetricsMap.get(pod.metadata!.name!); - const podMetric = podMetricsMap.get(pod.metadata!.name!) - + // TODO we are calculating the contianer usages twice + // Once per pod, then below per container + // calculate the pod usage in this method instead to avoid + // duplicating work const cpuTotal = totalCPU(pod); const memTotal = totalMemory(pod); const containerStatuses: ContainerStatus[] = []; let currentPodCPU: number | bigint = 0; let currentPodMem: number | bigint = 0; - if (podMetric !== undefined){ - podMetric.containers.forEach(container => { - const containerCPUUsage = quantityToScalar(container.usage.cpu); - const containerMemUsage = quantityToScalar(container.usage.memory); - currentPodCPU = add(currentPodCPU, containerCPUUsage) - currentPodMem = add(currentPodMem, containerMemUsage) - containerStatuses.push(new ContainerStatus(container.name, containerCPUUsage, containerMemUsage)) - }) + if (podMetric !== undefined) { + // Create a map of container names to their resource spec + // to make it easier to look up when we need it later + const containerResourceStatuses = pod.spec!.containers.reduce((accum, next) => { + const containerCpuTotal = totalCPUForContainer(next); + const containerMemTotal = totalMemoryForContainer(next); + accum.set(next.name, [containerCpuTotal, containerMemTotal]); + return accum; + }, new Map()); + + podMetric.containers.forEach((containerMetrics: ContainerMetric) => { + const currentContainerCPUUsage = quantityToScalar(containerMetrics.usage.cpu); + const currentContainerMemUsage = quantityToScalar(containerMetrics.usage.memory); + currentPodCPU = add(currentPodCPU, currentContainerCPUUsage); + currentPodMem = add(currentPodMem, currentContainerMemUsage); + + const containerResourceStatus = containerResourceStatuses.get(containerMetrics.name); + + if (containerResourceStatus !== undefined) { + const [containerCpuTotal, containerMemTotal] = containerResourceStatus; + + const containerCpuUsage = new CurrentResourceUsage( + currentContainerCPUUsage, + containerCpuTotal.request, + containerCpuTotal.limit, + ); + const containerMemUsage = new CurrentResourceUsage( + currentContainerMemUsage, + containerMemTotal.request, + containerMemTotal.limit, + ); + + containerStatuses.push( + new ContainerStatus(containerMetrics.name, containerCpuUsage, containerMemUsage), + ); + } + }); } - const cpuUsage = new CurrentResourceUsage(currentPodCPU, cpuTotal.request, cpuTotal.limit); - const memUsage = new CurrentResourceUsage(currentPodMem, memTotal.request, memTotal.limit); - result.push(new PodStatus(pod, cpuUsage, memUsage, containerStatuses)); - + const podCpuUsage = new CurrentResourceUsage(currentPodCPU, cpuTotal.request, cpuTotal.limit); + const podMemUsage = new CurrentResourceUsage(currentPodMem, memTotal.request, memTotal.limit); + result.push(new PodStatus(pod, podCpuUsage, podMemUsage, containerStatuses)); } return result; - - } diff --git a/src/top_test.ts b/src/top_test.ts index 4a2cae6046..b42ab5b0c0 100644 --- a/src/top_test.ts +++ b/src/top_test.ts @@ -1,12 +1,10 @@ -import { fail } from 'assert'; import { expect } from 'chai'; import * as nock from 'nock'; import { KubeConfig } from './config'; -import { V1Status, HttpError, V1Pod } from './gen/api'; -import { Metrics, NodeMetricsList, PodMetricsList } from './metrics'; +import { V1Pod } from './gen/api'; +import { Metrics, PodMetricsList } from './metrics'; import { topPods } from './top'; -import { CoreV1Api, } from './gen/api'; - +import { CoreV1Api } from './gen/api'; const emptyPodMetrics: PodMetricsList = { kind: 'PodMetricsList', @@ -31,7 +29,7 @@ const mockedPodMetrics: PodMetricsList = { }, timestamp: '2021-09-26T11:57:21Z', window: '30s', - containers: [{ name: 'nginx', usage: { cpu: '10', memory: '3912Ki' } }], + containers: [{ name: 'nginx', usage: { cpu: '5000000n', memory: '3912Ki' } }], }, { metadata: { @@ -43,8 +41,8 @@ const mockedPodMetrics: PodMetricsList = { timestamp: '2021-09-26T11:57:21Z', window: '30s', containers: [ - { name: 'nginx', usage: { cpu: '15', memory: '4012Ki' } }, - { name: 'sidecar', usage: { cpu: '16', memory: '3012Ki' } }, + { name: 'nginx', usage: { cpu: '0', memory: '4012Ki' } }, + { name: 'sidecar', usage: { cpu: '140000000n', memory: '3012Ki' } }, ], }, ], @@ -52,64 +50,63 @@ const mockedPodMetrics: PodMetricsList = { const podList: V1Pod[] = [ { - "metadata": { - "name": "dice-roller-7c76898b4d-shm9p" + metadata: { + name: 'dice-roller-7c76898b4d-shm9p', }, - 'spec': { + spec: { containers: [ { - name: "nginx", + name: 'nginx', resources: { requests: { - memory: "100Mi", - cpu: "100m" + memory: '100Mi', + cpu: '100m', }, limits: { - memory: "100Mi", - cpu: "100m" + memory: '100Mi', + cpu: '100m', }, - } - } - ] - } + }, + }, + ], + }, }, { - "metadata": { - "name": "other-pod-7c76898b4e-12kj" + metadata: { + name: 'other-pod-7c76898b4e-12kj', }, - 'spec': { + spec: { containers: [ { - name: "nginx", + name: 'nginx', resources: { requests: { - memory: "100Mi", - cpu: "100m" + memory: '100Mi', + cpu: '100m', }, limits: { - memory: "100Mi", - cpu: "100m" + memory: '100Mi', + cpu: '100m', }, - } + }, }, { - name: "sidecar", + name: 'sidecar', resources: { requests: { - memory: "50Mi", - cpu: "1" + memory: '50Mi', + cpu: '2', }, limits: { - memory: "100Mi", - cpu: "1" + memory: '100Mi', + cpu: '2', }, - } - } - ] - } - } -] - + }, + }, + ], + }, + }, +]; const TEST_NAMESPACE = 'test-namespace'; @@ -120,7 +117,10 @@ const testConfigOptions: any = { currentContext: 'currentContext', }; -const systemUnderTest = (namespace?: string, options: any = testConfigOptions): [() => ReturnType, nock.Scope] => { +const systemUnderTest = ( + namespace?: string, + options: any = testConfigOptions, +): [() => ReturnType, nock.Scope] => { const kc = new KubeConfig(); kc.loadFromOptions(options); const metricsClient = new Metrics(kc); @@ -138,7 +138,7 @@ describe('Top', () => { const [topPodsFunc, scope] = systemUnderTest(); const podMetrics = scope.get('/apis/metrics.k8s.io/v1beta1/pods').reply(200, emptyPodMetrics); const pods = scope.get('/api/v1/pods').reply(200, { - items: [] + items: [], }); const result = await topPodsFunc(); expect(result).to.deep.equal([]); @@ -149,103 +149,153 @@ describe('Top', () => { const [topPodsFunc, scope] = systemUnderTest(); const podMetrics = scope.get('/apis/metrics.k8s.io/v1beta1/pods').reply(200, mockedPodMetrics); const pods = scope.get('/api/v1/pods').reply(200, { - items: podList + items: podList, }); const result = await topPodsFunc(); expect(result.length).to.equal(2); expect(result[0].CPU).to.deep.equal({ // TODO fix this - "CurrentUsage": 10, - "LimitTotal": 0.1, - "RequestTotal": 0.1 + CurrentUsage: 0.05, + LimitTotal: 0.1, + RequestTotal: 0.1, }); expect(result[0].Memory).to.deep.equal({ - "CurrentUsage": BigInt("4005888"), - "RequestTotal": BigInt("104857600"), - "LimitTotal": BigInt("104857600"), - }); + CurrentUsage: BigInt('4005888'), + RequestTotal: BigInt('104857600'), + LimitTotal: BigInt('104857600'), + }); expect(result[0].Containers).to.deep.equal([ { - "CPUUsage": 10, - "Container": "nginx", - "Memory": BigInt("4005888") - } + CPUUsage: { + CurrentUsage: 0.05, + LimitTotal: 0.1, + RequestTotal: 0.1, + }, + Container: 'nginx', + MemoryUsage: { + CurrentUsage: BigInt('4005888'), + LimitTotal: BigInt('104857600'), + RequestTotal: BigInt('104857600'), + }, + }, ]); expect(result[1].CPU).to.deep.equal({ // TODO fix this - "CurrentUsage": 31, - "LimitTotal": 1.1, - "RequestTotal": 1.1 - }); + CurrentUsage: 1.4, + LimitTotal: 2.1, + RequestTotal: 2.1, + }); expect(result[1].Memory).to.deep.equal({ - "CurrentUsage": BigInt("7192576"), - "LimitTotal": BigInt("209715200"), - "RequestTotal": BigInt("157286400") - }); - expect(result[1].Containers).to.deep.equal( [ - { - "CPUUsage": 15, - "Container": "nginx", - "Memory": BigInt("4108288"), - }, - { - "CPUUsage": 16, - "Container": "sidecar", - "Memory": BigInt("3084288"), - } - ]); + CurrentUsage: BigInt('7192576'), + LimitTotal: BigInt('209715200'), + RequestTotal: BigInt('157286400'), + }); + expect(result[1].Containers).to.deep.equal([ + { + CPUUsage: { + CurrentUsage: 0, + LimitTotal: 0.1, + RequestTotal: 0.1, + }, + Container: 'nginx', + MemoryUsage: { + CurrentUsage: BigInt('4108288'), + LimitTotal: BigInt('104857600'), + RequestTotal: BigInt('104857600'), + }, + }, + { + CPUUsage: { + CurrentUsage: 1.4, + LimitTotal: 2, + RequestTotal: 2, + }, + Container: 'sidecar', + MemoryUsage: { + CurrentUsage: BigInt('3084288'), + LimitTotal: BigInt('104857600'), + RequestTotal: BigInt('52428800'), + }, + }, + ]); podMetrics.done(); pods.done(); }); it('should return namespace pod metrics', async () => { const [topPodsFunc, scope] = systemUnderTest(TEST_NAMESPACE); - const podMetrics = scope.get(`/apis/metrics.k8s.io/v1beta1/namespaces/${TEST_NAMESPACE}/pods`).reply(200, mockedPodMetrics); + const podMetrics = scope + .get(`/apis/metrics.k8s.io/v1beta1/namespaces/${TEST_NAMESPACE}/pods`) + .reply(200, mockedPodMetrics); const pods = scope.get(`/api/v1/namespaces/${TEST_NAMESPACE}/pods`).reply(200, { - items: podList + items: podList, }); const result = await topPodsFunc(); expect(result.length).to.equal(2); expect(result[0].CPU).to.deep.equal({ - "CurrentUsage": 10, - "LimitTotal": 0.1, - "RequestTotal": 0.1 + CurrentUsage: 0.05, + LimitTotal: 0.1, + RequestTotal: 0.1, }); expect(result[0].Memory).to.deep.equal({ - "CurrentUsage": BigInt("4005888"), - "RequestTotal": BigInt("104857600"), - "LimitTotal": BigInt("104857600"), - }); + CurrentUsage: BigInt('4005888'), + RequestTotal: BigInt('104857600'), + LimitTotal: BigInt('104857600'), + }); expect(result[0].Containers).to.deep.equal([ { - "CPUUsage": 10, - "Container": "nginx", - "Memory": BigInt("4005888") - } + CPUUsage: { + CurrentUsage: 0.05, + LimitTotal: 0.1, + RequestTotal: 0.1, + }, + Container: 'nginx', + MemoryUsage: { + CurrentUsage: BigInt('4005888'), + LimitTotal: BigInt('104857600'), + RequestTotal: BigInt('104857600'), + }, + }, ]); expect(result[1].CPU).to.deep.equal({ - "CurrentUsage": 31, - "LimitTotal": 1.1, - "RequestTotal": 1.1 - }); + CurrentUsage: 1.4, + LimitTotal: 2.1, + RequestTotal: 2.1, + }); expect(result[1].Memory).to.deep.equal({ - "CurrentUsage": BigInt("7192576"), - "LimitTotal": BigInt("209715200"), - "RequestTotal": BigInt("157286400") - }); - expect(result[1].Containers).to.deep.equal( [ - { - "CPUUsage": 15, - "Container": "nginx", - "Memory": BigInt("4108288"), - }, - { - "CPUUsage": 16, - "Container": "sidecar", - "Memory": BigInt("3084288"), - } - ]); + CurrentUsage: BigInt('7192576'), + LimitTotal: BigInt('209715200'), + RequestTotal: BigInt('157286400'), + }); + expect(result[1].Containers).to.deep.equal([ + { + CPUUsage: { + CurrentUsage: 0, + LimitTotal: 0.1, + RequestTotal: 0.1, + }, + Container: 'nginx', + MemoryUsage: { + CurrentUsage: BigInt('4108288'), + LimitTotal: BigInt('104857600'), + RequestTotal: BigInt('104857600'), + }, + }, + { + CPUUsage: { + CurrentUsage: 1.4, + LimitTotal: 2, + RequestTotal: 2, + }, + Container: 'sidecar', + MemoryUsage: { + CurrentUsage: BigInt('3084288'), + LimitTotal: BigInt('104857600'), + RequestTotal: BigInt('52428800'), + }, + }, + ]); podMetrics.done(); pods.done(); }); }); -}); \ No newline at end of file +}); diff --git a/src/util.ts b/src/util.ts index 103728eb20..c0af5bd5e3 100644 --- a/src/util.ts +++ b/src/util.ts @@ -28,7 +28,7 @@ export function quantityToScalar(quantity: string): number | bigint { } switch (suffix) { case 'n': - return Number(quantity.substr(0, quantity.length - 1)).valueOf() / 1_000_000.0; + return Number(quantity.substr(0, quantity.length - 1)).valueOf() / 100_000_000.0; case 'm': return Number(quantity.substr(0, quantity.length - 1)).valueOf() / 1000.0; case 'Ki': @@ -66,6 +66,14 @@ export class ResourceStatus { ) {} } +export function totalCPUForContainer(container: V1Container): ResourceStatus { + return containerTotalForResource(container, 'cpu'); +} + +export function totalMemoryForContainer(container: V1Container): ResourceStatus { + return containerTotalForResource(container, 'memory'); +} + export function totalCPU(pod: V1Pod): ResourceStatus { return totalForResource(pod, 'cpu'); } @@ -86,18 +94,28 @@ export function add(n1: number | bigint, n2: number | bigint): number | bigint { return ((n1 as bigint) + n2) as bigint; } +export function containerTotalForResource(container: V1Container, resource: string): ResourceStatus { + let reqTotal: number | bigint = 0; + let limitTotal: number | bigint = 0; + if (container.resources) { + if (container.resources.requests) { + reqTotal = add(reqTotal, quantityToScalar(container.resources.requests[resource])); + } + if (container.resources.limits) { + limitTotal = add(limitTotal, quantityToScalar(container.resources.limits[resource])); + } + } + return new ResourceStatus(reqTotal, limitTotal, resource); +} + export function totalForResource(pod: V1Pod, resource: string): ResourceStatus { let reqTotal: number | bigint = 0; let limitTotal: number | bigint = 0; pod.spec!.containers.forEach((container: V1Container) => { - if (container.resources) { - if (container.resources.requests) { - reqTotal = add(reqTotal, quantityToScalar(container.resources.requests[resource])); - } - if (container.resources.limits) { - limitTotal = add(limitTotal, quantityToScalar(container.resources.limits[resource])); - } - } + const containerTotal = containerTotalForResource(container, resource); + + reqTotal = add(reqTotal, containerTotal.request); + limitTotal = add(limitTotal, containerTotal.limit); }); return new ResourceStatus(reqTotal, limitTotal, resource); } From 9b4ffe2ffc957a671bef58a9c692860d6f25a3cf Mon Sep 17 00:00:00 2001 From: mclarke47 Date: Sat, 2 Oct 2021 10:48:07 +0100 Subject: [PATCH 3/8] add example --- examples/top_pods.js | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 examples/top_pods.js diff --git a/examples/top_pods.js b/examples/top_pods.js new file mode 100644 index 0000000000..0b1ef457c2 --- /dev/null +++ b/examples/top_pods.js @@ -0,0 +1,42 @@ +const k8s = require('../dist/index'); + +const kc = new k8s.KubeConfig(); +kc.loadFromDefault(); + +const k8sApi = kc.makeApiClient(k8s.CoreV1Api); +const metricsClient = new k8s.Metrics(kc); + +k8s.topPods(k8sApi, metricsClient, "kube-system") +.then((pods) => { + + const podsColumns = pods.reduce((accum, next) => { + accum.push({ + "POD": next.Pod.metadata.name, + "CPU(cores)": next.CPU.CurrentUsage, + "MEMORY(bytes)": next.Memory.CurrentUsage, + }); + return accum; + }, []); + console.log("TOP PODS") + console.table(podsColumns) +}); + +k8s.topPods(k8sApi, metricsClient, "kube-system") +.then((pods) => { + + const podsColumns = pods.reduce((accum, next) => { + + next.Containers.forEach(containerUsage => { + accum.push({ + "POD": next.Pod.metadata.name, + "NAME": containerUsage.Container, + "CPU(cores)": next.CPU.CurrentUsage, + "MEMORY(bytes)": next.Memory.CurrentUsage, + }); + }) + return accum; + }, []); + + console.log("TOP CONTAINERS") + console.table(podsColumns) +}); \ No newline at end of file From 976e418758be4691180f1b2b1e3237417a74a149 Mon Sep 17 00:00:00 2001 From: mclarke47 Date: Sat, 2 Oct 2021 11:31:52 +0100 Subject: [PATCH 4/8] simplfy example --- examples/top_pods.js | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/examples/top_pods.js b/examples/top_pods.js index 0b1ef457c2..1f3bd5b120 100644 --- a/examples/top_pods.js +++ b/examples/top_pods.js @@ -9,14 +9,13 @@ const metricsClient = new k8s.Metrics(kc); k8s.topPods(k8sApi, metricsClient, "kube-system") .then((pods) => { - const podsColumns = pods.reduce((accum, next) => { - accum.push({ - "POD": next.Pod.metadata.name, - "CPU(cores)": next.CPU.CurrentUsage, - "MEMORY(bytes)": next.Memory.CurrentUsage, - }); - return accum; - }, []); + const podsColumns = pods.map((pod) => { + return { + "POD": pod.Pod.metadata.name, + "CPU(cores)": pod.CPU.CurrentUsage, + "MEMORY(bytes)": pod.Memory.CurrentUsage, + } + }); console.log("TOP PODS") console.table(podsColumns) }); @@ -24,19 +23,17 @@ k8s.topPods(k8sApi, metricsClient, "kube-system") k8s.topPods(k8sApi, metricsClient, "kube-system") .then((pods) => { - const podsColumns = pods.reduce((accum, next) => { - - next.Containers.forEach(containerUsage => { - accum.push({ - "POD": next.Pod.metadata.name, + const podsAndContainersColumns = pods.flatMap((pod) => { + return pod.Containers.map(containerUsage => { + return { + "POD": pod.Pod.metadata.name, "NAME": containerUsage.Container, - "CPU(cores)": next.CPU.CurrentUsage, - "MEMORY(bytes)": next.Memory.CurrentUsage, - }); + "CPU(cores)": containerUsage.CPUUsage.CurrentUsage, + "MEMORY(bytes)": containerUsage.MemoryUsage.CurrentUsage, + }; }) - return accum; - }, []); + }); console.log("TOP CONTAINERS") - console.table(podsColumns) + console.table(podsAndContainersColumns) }); \ No newline at end of file From bb133c0d3c00b215fcaab35f7721f7bcf7aaaf6d Mon Sep 17 00:00:00 2001 From: mclarke47 Date: Sun, 3 Oct 2021 16:22:54 +0100 Subject: [PATCH 5/8] refactor implementation --- src/top.ts | 92 ++++++++++++++++++++++------------------- src/top_test.ts | 108 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 155 insertions(+), 45 deletions(-) diff --git a/src/top.ts b/src/top.ts index 44e96be0d2..2cd1ad587d 100644 --- a/src/top.ts +++ b/src/top.ts @@ -82,8 +82,9 @@ export async function topNodes(api: CoreV1Api): Promise { return result; } -// TODO describe what this method does +// Returns the current pod CPU/Memory usage including the CPU/Memory usage of each container export async function topPods(api: CoreV1Api, metrics: Metrics, namespace?: string): Promise { + // Figure out which pod list endpoint to call const getPodList = async (): Promise => { if (namespace) { return (await api.listNamespacedPod(namespace)).body; @@ -105,57 +106,62 @@ export async function topPods(api: CoreV1Api, metrics: Metrics, namespace?: stri for (const pod of podList.items) { const podMetric = podMetricsMap.get(pod.metadata!.name!); - // TODO we are calculating the contianer usages twice - // Once per pod, then below per container - // calculate the pod usage in this method instead to avoid - // duplicating work - const cpuTotal = totalCPU(pod); - const memTotal = totalMemory(pod); const containerStatuses: ContainerStatus[] = []; let currentPodCPU: number | bigint = 0; let currentPodMem: number | bigint = 0; - - if (podMetric !== undefined) { - // Create a map of container names to their resource spec - // to make it easier to look up when we need it later - const containerResourceStatuses = pod.spec!.containers.reduce((accum, next) => { - const containerCpuTotal = totalCPUForContainer(next); - const containerMemTotal = totalMemoryForContainer(next); - accum.set(next.name, [containerCpuTotal, containerMemTotal]); - return accum; - }, new Map()); - - podMetric.containers.forEach((containerMetrics: ContainerMetric) => { + let podRequestsCPU: number | bigint = 0; + let podLimitsCPU: number | bigint = 0; + let podRequestsMem: number | bigint = 0; + let podLimitsMem: number | bigint = 0; + + pod.spec!.containers.forEach((container) => { + // get the the container CPU/Memory container.resources.requests + const containerCpuTotal = totalCPUForContainer(container); + const containerMemTotal = totalMemoryForContainer(container); + + // sum each container's CPU/Memory container.resources.requests + // to get the pod's overall request limit + podRequestsCPU = add(podRequestsCPU, containerCpuTotal.request); + podLimitsCPU = add(podLimitsCPU, containerCpuTotal.limit); + + podRequestsMem = add(podLimitsMem, containerMemTotal.request); + podLimitsMem = add(podLimitsMem, containerMemTotal.limit); + + // Find the container metrics by container.name + // if both the pod and container metrics exist + const containerMetrics = + podMetric !== undefined + ? podMetric.containers.find((c) => c.name === container.name) + : undefined; + + // Store the current usage of each container + // Sum each container to get the overall pod usage + if (containerMetrics !== undefined) { const currentContainerCPUUsage = quantityToScalar(containerMetrics.usage.cpu); const currentContainerMemUsage = quantityToScalar(containerMetrics.usage.memory); + currentPodCPU = add(currentPodCPU, currentContainerCPUUsage); currentPodMem = add(currentPodMem, currentContainerMemUsage); - const containerResourceStatus = containerResourceStatuses.get(containerMetrics.name); - - if (containerResourceStatus !== undefined) { - const [containerCpuTotal, containerMemTotal] = containerResourceStatus; - - const containerCpuUsage = new CurrentResourceUsage( - currentContainerCPUUsage, - containerCpuTotal.request, - containerCpuTotal.limit, - ); - const containerMemUsage = new CurrentResourceUsage( - currentContainerMemUsage, - containerMemTotal.request, - containerMemTotal.limit, - ); - - containerStatuses.push( - new ContainerStatus(containerMetrics.name, containerCpuUsage, containerMemUsage), - ); - } - }); - } + const containerCpuUsage = new CurrentResourceUsage( + currentContainerCPUUsage, + containerCpuTotal.request, + containerCpuTotal.limit, + ); + const containerMemUsage = new CurrentResourceUsage( + currentContainerMemUsage, + containerMemTotal.request, + containerMemTotal.limit, + ); + + containerStatuses.push( + new ContainerStatus(containerMetrics.name, containerCpuUsage, containerMemUsage), + ); + } + }); - const podCpuUsage = new CurrentResourceUsage(currentPodCPU, cpuTotal.request, cpuTotal.limit); - const podMemUsage = new CurrentResourceUsage(currentPodMem, memTotal.request, memTotal.limit); + const podCpuUsage = new CurrentResourceUsage(currentPodCPU, podRequestsCPU, podLimitsCPU); + const podMemUsage = new CurrentResourceUsage(currentPodMem, podRequestsMem, podLimitsMem); result.push(new PodStatus(pod, podCpuUsage, podMemUsage, containerStatuses)); } return result; diff --git a/src/top_test.ts b/src/top_test.ts index b42ab5b0c0..8d08b0620f 100644 --- a/src/top_test.ts +++ b/src/top_test.ts @@ -48,6 +48,21 @@ const mockedPodMetrics: PodMetricsList = { ], }; +const bestEffortPodList: V1Pod[] = [ + { + metadata: { + name: 'dice-roller-7c76898b4d-shm9p', + }, + spec: { + containers: [ + { + name: 'nginx', + }, + ], + }, + }, +]; + const podList: V1Pod[] = [ { metadata: { @@ -145,6 +160,17 @@ describe('Top', () => { podMetrics.done(); pods.done(); }); + it('should return use cluster scope when namespace empty string', async () => { + const [topPodsFunc, scope] = systemUnderTest(''); + const podMetrics = scope.get('/apis/metrics.k8s.io/v1beta1/pods').reply(200, emptyPodMetrics); + const pods = scope.get('/api/v1/pods').reply(200, { + items: [], + }); + const result = await topPodsFunc(); + expect(result).to.deep.equal([]); + podMetrics.done(); + pods.done(); + }); it('should return cluster wide pod metrics', async () => { const [topPodsFunc, scope] = systemUnderTest(); const podMetrics = scope.get('/apis/metrics.k8s.io/v1beta1/pods').reply(200, mockedPodMetrics); @@ -154,7 +180,6 @@ describe('Top', () => { const result = await topPodsFunc(); expect(result.length).to.equal(2); expect(result[0].CPU).to.deep.equal({ - // TODO fix this CurrentUsage: 0.05, LimitTotal: 0.1, RequestTotal: 0.1, @@ -180,7 +205,6 @@ describe('Top', () => { }, ]); expect(result[1].CPU).to.deep.equal({ - // TODO fix this CurrentUsage: 1.4, LimitTotal: 2.1, RequestTotal: 2.1, @@ -221,6 +245,86 @@ describe('Top', () => { podMetrics.done(); pods.done(); }); + it('should return best effort pod metrics', async () => { + const [topPodsFunc, scope] = systemUnderTest(); + const podMetrics = scope.get('/apis/metrics.k8s.io/v1beta1/pods').reply(200, mockedPodMetrics); + const pods = scope.get('/api/v1/pods').reply(200, { + items: bestEffortPodList, + }); + const result = await topPodsFunc(); + expect(result.length).to.equal(1); + expect(result[0].CPU).to.deep.equal({ + CurrentUsage: 0.05, + LimitTotal: 0, + RequestTotal: 0, + }); + expect(result[0].Memory).to.deep.equal({ + CurrentUsage: BigInt('4005888'), + RequestTotal: 0, + LimitTotal: 0, + }); + expect(result[0].Containers).to.deep.equal([ + { + CPUUsage: { + CurrentUsage: 0.05, + LimitTotal: 0, + RequestTotal: 0, + }, + Container: 'nginx', + MemoryUsage: { + CurrentUsage: BigInt('4005888'), + LimitTotal: 0, + RequestTotal: 0, + }, + }, + ]); + podMetrics.done(); + pods.done(); + }); + it('should return 0 when pod metrics missing', async () => { + const [topPodsFunc, scope] = systemUnderTest(); + const podMetrics = scope.get('/apis/metrics.k8s.io/v1beta1/pods').reply(200, emptyPodMetrics); + const pods = scope.get('/api/v1/pods').reply(200, { + items: podList, + }); + const result = await topPodsFunc(); + expect(result.length).to.equal(2); + expect(result[0].CPU).to.deep.equal({ + CurrentUsage: 0, + LimitTotal: 0.1, + RequestTotal: 0.1, + }); + expect(result[0].Memory).to.deep.equal({ + CurrentUsage: 0, + RequestTotal: BigInt('104857600'), + LimitTotal: BigInt('104857600'), + }); + expect(result[0].Containers).to.deep.equal([]); + expect(result[1].CPU).to.deep.equal({ + CurrentUsage: 0, + LimitTotal: 2.1, + RequestTotal: 2.1, + }); + expect(result[1].Memory).to.deep.equal({ + CurrentUsage: 0, + LimitTotal: BigInt('209715200'), + RequestTotal: BigInt('157286400'), + }); + expect(result[1].Containers).to.deep.equal([]); + podMetrics.done(); + pods.done(); + }); + it('should return empty array when pods missing', async () => { + const [topPodsFunc, scope] = systemUnderTest(); + const podMetrics = scope.get('/apis/metrics.k8s.io/v1beta1/pods').reply(200, mockedPodMetrics); + const pods = scope.get('/api/v1/pods').reply(200, { + items: [], + }); + const result = await topPodsFunc(); + expect(result.length).to.equal(0); + podMetrics.done(); + pods.done(); + }); it('should return namespace pod metrics', async () => { const [topPodsFunc, scope] = systemUnderTest(TEST_NAMESPACE); const podMetrics = scope From 8b88bd269a980251deb4e3c696fea9b251d4f63c Mon Sep 17 00:00:00 2001 From: mclarke47 Date: Sun, 3 Oct 2021 16:24:22 +0100 Subject: [PATCH 6/8] tweak comment --- src/top.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/top.ts b/src/top.ts index 2cd1ad587d..2c5ee30276 100644 --- a/src/top.ts +++ b/src/top.ts @@ -115,12 +115,12 @@ export async function topPods(api: CoreV1Api, metrics: Metrics, namespace?: stri let podLimitsMem: number | bigint = 0; pod.spec!.containers.forEach((container) => { - // get the the container CPU/Memory container.resources.requests + // get the the container CPU/Memory container.resources.requests/limits const containerCpuTotal = totalCPUForContainer(container); const containerMemTotal = totalMemoryForContainer(container); - // sum each container's CPU/Memory container.resources.requests - // to get the pod's overall request limit + // sum each container's CPU/Memory container.resources.requests/limits + // to get the pod's overall requests/limits podRequestsCPU = add(podRequestsCPU, containerCpuTotal.request); podLimitsCPU = add(podLimitsCPU, containerCpuTotal.limit); From a0e3f5f0496477ebc62e671c2bc07be58528ae0e Mon Sep 17 00:00:00 2001 From: mclarke47 Date: Mon, 4 Oct 2021 18:26:22 +0100 Subject: [PATCH 7/8] review feedback --- src/top.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/top.ts b/src/top.ts index 2c5ee30276..313872817e 100644 --- a/src/top.ts +++ b/src/top.ts @@ -88,9 +88,8 @@ export async function topPods(api: CoreV1Api, metrics: Metrics, namespace?: stri const getPodList = async (): Promise => { if (namespace) { return (await api.listNamespacedPod(namespace)).body; - } else { - return (await api.listPodForAllNamespaces()).body; } + return (await api.listPodForAllNamespaces()).body; }; const [podMetrics, podList] = await Promise.all([metrics.getPodMetrics(namespace), getPodList()]); From d9230a9ded153ab8ae613a16e5e1b97468c37d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Str=C3=BCbing?= Date: Wed, 6 Oct 2021 08:59:50 +0200 Subject: [PATCH 8/8] update kubernetes-client/gen --- settings | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings b/settings index 96244d9d3d..7ae65e14a7 100644 --- a/settings +++ b/settings @@ -15,7 +15,7 @@ # limitations under the License. # kubernetes-client/gen commit to use for code generation. -export GEN_COMMIT=d71ff1efd +export GEN_COMMIT=a3aef4d # GitHub username/organization to clone kubernetes repo from. export USERNAME=kubernetes