From 99ab6f511b192830ff8b66674f73f3d6cd5acdd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 28 Jan 2026 15:18:09 +0100 Subject: [PATCH 1/2] feat: naive performance test and handling results storing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stores performance tests as artifacts and compares the results with released version performance tests Signed-off-by: Attila Mészáros --- .github/workflows/integration-tests.yml | 7 + .github/workflows/pr.yml | 48 +++++ .../performance/PerformanceTestResult.java | 31 +++ .../performance/PerformanceTestSummary.java | 58 ++++++ .../performance/SimplePerformanceTestIT.java | 197 ++++++++++++++++++ .../SimplePerformanceTestReconciler.java | 85 ++++++++ .../SimplePerformanceTestResource.java | 29 +++ .../SimplePerformanceTestSpec.java | 29 +++ .../SimplePerformanceTestStatus.java | 34 +++ .../finalizer/SSAFinalizerIssueStatus.java | 2 +- 10 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestReconciler.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestResource.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestSpec.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestStatus.java diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 77c268d6bc..e795350c6c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -55,3 +55,10 @@ jobs: echo "Using profile: ${it_profile}" ./mvnw ${MAVEN_ARGS} -T1C -B install -DskipTests -Pno-apt --file pom.xml ./mvnw ${MAVEN_ARGS} -T1C -B package -P${it_profile} -Dfabric8-httpclient-impl.name=${{inputs.http-client}} --file pom.xml + + - name: Upload performance test results + uses: actions/upload-artifact@v4 + with: + name: performance-results-java${{ inputs.java-version }}-k8s${{ inputs.kube-version }}-${{ inputs.http-client }} + path: operator-framework/target/performance_test_result.json + if-no-files-found: ignore diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7a5964ba35..57e655ca4e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -32,3 +32,51 @@ jobs: build: uses: ./.github/workflows/build.yml + + performance_report: + name: Post Performance Results + runs-on: ubuntu-latest + needs: build + if: always() + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v6 + + - name: Download all performance artifacts + uses: actions/download-artifact@v4 + with: + pattern: performance-results-* + path: performance-results + merge-multiple: true + + - name: Check for performance results + id: check_results + run: | + if [ -d "performance-results" ] && [ "$(ls -A performance-results/*.json 2>/dev/null)" ]; then + echo "has_results=true" >> $GITHUB_OUTPUT + else + echo "has_results=false" >> $GITHUB_OUTPUT + fi + + - name: Convert performance results to markdown + if: steps.check_results.outputs.has_results == 'true' + id: convert + run: | + echo "# Performance Test Results" > comment.md + echo "" >> comment.md + for file in performance-results/*.json; do + if [ -f "$file" ]; then + echo "Processing $file" + python3 .github/scripts/performance-to-markdown.py "$file" >> comment.md + echo "" >> comment.md + fi + done + + - name: Post PR comment + if: steps.check_results.outputs.has_results == 'true' && github.event_name == 'pull_request' + uses: marocchino/sticky-pull-request-comment@v2 + with: + path: comment.md + recreate: true + diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java new file mode 100644 index 0000000000..61b0feacf2 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java @@ -0,0 +1,31 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +import java.util.List; + +public class PerformanceTestResult { + + private List summaries; + + public List getSummaries() { + return summaries; + } + + public void setSummaries(List summaries) { + this.summaries = summaries; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java new file mode 100644 index 0000000000..a8137dbad8 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java @@ -0,0 +1,58 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +public class PerformanceTestSummary { + + private String name; + + // data about the machine + private int numberOfProcessors; + private long maxMemory; + private long duration; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getNumberOfProcessors() { + return numberOfProcessors; + } + + public void setNumberOfProcessors(int numberOfProcessors) { + this.numberOfProcessors = numberOfProcessors; + } + + public long getMaxMemory() { + return maxMemory; + } + + public void setMaxMemory(long maxMemory) { + this.maxMemory = maxMemory; + } + + public long getDuration() { + return duration; + } + + public void setDuration(long duration) { + this.duration = duration; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java new file mode 100644 index 0000000000..bc4eab008a --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java @@ -0,0 +1,197 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.client.informers.ResourceEventHandler; +import io.fabric8.kubernetes.client.informers.SharedIndexInformer; +import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension; +import io.vertx.core.impl.ConcurrentHashSet; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class SimplePerformanceTestIT { + + private static final Logger log = LoggerFactory.getLogger(SimplePerformanceTestIT.class); + public static final String INITIAL_VALUE = "initialValue"; + public static final String RESOURCE_NAME_PREFIX = "resource"; + public static final String INDEX = "index"; + + @RegisterExtension + LocallyRunOperatorExtension extension = + LocallyRunOperatorExtension.builder() + .withReconciler(new SimplePerformanceTestReconciler()) + .build(); + + final int WARM_UP_RESOURCE_NUMBER = 10; + final int TEST_RESOURCE_NUMBER = 150; + + ExecutorService executor = Executors.newFixedThreadPool(TEST_RESOURCE_NUMBER); + + @Test + void simpleNaivePerformanceTest() { + var processors = Runtime.getRuntime().availableProcessors(); + long maxMemory = Runtime.getRuntime().maxMemory(); + log.info("Running performance test with memory: {} and processors: {}", maxMemory, processors); + + var primaryInformer = + extension + .getKubernetesClient() + .resources(SimplePerformanceTestResource.class) + .inNamespace(extension.getNamespace()) + .inform(); + + var statusChecker = + new StatusChecker(INITIAL_VALUE, 0, WARM_UP_RESOURCE_NUMBER, primaryInformer); + createResources(0, WARM_UP_RESOURCE_NUMBER, INITIAL_VALUE); + statusChecker.waitUntilAllInStatus(); + + long startTime = System.currentTimeMillis(); + statusChecker = + new StatusChecker( + INITIAL_VALUE, WARM_UP_RESOURCE_NUMBER, TEST_RESOURCE_NUMBER, primaryInformer); + createResources(WARM_UP_RESOURCE_NUMBER, TEST_RESOURCE_NUMBER, INITIAL_VALUE); + statusChecker.waitUntilAllInStatus(); + var duration = System.currentTimeMillis() - startTime; + + log.info("Create duration: {}", duration); + saveResults(duration); + } + + private void saveResults(long duration) { + try { + var result = new PerformanceTestResult(); + var summary = new PerformanceTestSummary(); + result.setSummaries(List.of(summary)); + summary.setName("Naive performance test"); + summary.setDuration(duration); + summary.setNumberOfProcessors(Runtime.getRuntime().availableProcessors()); + summary.setMaxMemory(Runtime.getRuntime().maxMemory()); + var objectMapper = new ObjectMapper(); + objectMapper.writeValue(new File("target/performance_test_result.json"), result); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void createResources(int startIndex, int number, String value) { + try { + List> callables = new ArrayList<>(number); + + for (int i = startIndex; i < startIndex + number; i++) { + var res = new SimplePerformanceTestResource(); + res.setMetadata( + new ObjectMetaBuilder() + .withAnnotations(Map.of(INDEX, "" + i)) + .withName(RESOURCE_NAME_PREFIX + i) + .build()); + res.setSpec(new SimplePerformanceTestSpec()); + res.getSpec().setValue(value); + callables.add( + () -> { + extension.create(res); + return null; + }); + } + var futures = executor.invokeAll(callables); + for (var future : futures) { + future.get(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + static class StatusChecker { + private final String expectedStatus; // null indicates deleted + private final Set remaining = new ConcurrentHashSet<>(); + + StatusChecker( + String expectedStatus, + int startIndex, + int number, + SharedIndexInformer primaryInformer) { + this.expectedStatus = expectedStatus; + for (int i = startIndex; i < startIndex + number; i++) { + remaining.add(i); + } + primaryInformer.addEventHandler( + new ResourceEventHandler<>() { + @Override + public void onAdd(SimplePerformanceTestResource obj) { + checkOnStatus(obj); + } + + @Override + public void onUpdate( + SimplePerformanceTestResource oldObj, SimplePerformanceTestResource newObj) { + checkOnStatus(newObj); + } + + @Override + public void onDelete( + SimplePerformanceTestResource obj, boolean deletedFinalStateUnknown) { + if (expectedStatus == null) { + synchronized (remaining) { + remaining.remove(Integer.parseInt(obj.getMetadata().getAnnotations().get(INDEX))); + remaining.notifyAll(); + } + } + } + }); + primaryInformer.getStore().list().forEach(this::checkOnStatus); + } + + private void checkOnStatus(SimplePerformanceTestResource res) { + if (expectedStatus != null + && res.getStatus() != null + && res.getStatus().getValue().equals(expectedStatus)) { + synchronized (remaining) { + remaining.remove(Integer.parseInt(res.getMetadata().getAnnotations().get(INDEX))); + remaining.notifyAll(); + } + } + } + + public void waitUntilAllInStatus() { + synchronized (remaining) { + while (!remaining.isEmpty()) { + try { + remaining.wait(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + } + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestReconciler.java new file mode 100644 index 0000000000..eff83d1514 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestReconciler.java @@ -0,0 +1,85 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +import java.util.List; +import java.util.Map; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.api.config.informer.InformerEventSourceConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Cleaner; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.DeleteControl; +import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import io.javaoperatorsdk.operator.processing.event.source.EventSource; +import io.javaoperatorsdk.operator.processing.event.source.informer.InformerEventSource; + +@ControllerConfiguration +public class SimplePerformanceTestReconciler + implements Reconciler, Cleaner { + + public static final String KEY = "key"; + + @Override + public UpdateControl reconcile( + SimplePerformanceTestResource resource, Context context) { + var cm = configMap(resource); + + context.getClient().resource(cm).serverSideApply(); + + resource.setStatus(new SimplePerformanceTestStatus()); + resource.getStatus().setValue(resource.getSpec().getValue()); + return UpdateControl.patchStatus(resource); + } + + private ConfigMap configMap(SimplePerformanceTestResource primary) { + var cm = + new ConfigMapBuilder() + .withMetadata( + new ObjectMetaBuilder() + .withName(primary.getMetadata().getName()) + .withNamespace(primary.getMetadata().getNamespace()) + .build()) + .withData(Map.of(KEY, primary.getSpec().getValue())) + .build(); + cm.addOwnerReference(primary); + return cm; + } + + @Override + public DeleteControl cleanup( + SimplePerformanceTestResource resource, Context context) { + return DeleteControl.defaultDelete(); + } + + @Override + public List> prepareEventSources( + EventSourceContext context) { + InformerEventSource es = + new InformerEventSource<>( + InformerEventSourceConfiguration.from( + ConfigMap.class, SimplePerformanceTestResource.class) + .withNamespacesInheritedFromController() + .build(), + context); + return List.of(es); + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestResource.java new file mode 100644 index 0000000000..cc1e5dc53d --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestResource.java @@ -0,0 +1,29 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("sample.javaoperatorsdk") +@Version("v1") +@ShortNames("spt") +public class SimplePerformanceTestResource + extends CustomResource + implements Namespaced {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestSpec.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestSpec.java new file mode 100644 index 0000000000..16c54e0f09 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestSpec.java @@ -0,0 +1,29 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +public class SimplePerformanceTestSpec { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestStatus.java new file mode 100644 index 0000000000..01df9a8ffe --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestStatus.java @@ -0,0 +1,34 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +public class SimplePerformanceTestStatus { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String configMapStatus) { + this.value = configMapStatus; + } + + @Override + public String toString() { + return "SimplePerformanceTestStatus{" + "configMapStatus='" + value + '\'' + '}'; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java index efd0377311..b59c393b04 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java @@ -29,6 +29,6 @@ public void setConfigMapStatus(String configMapStatus) { @Override public String toString() { - return "TestCustomResourceStatus{" + "configMapStatus='" + configMapStatus + '\'' + '}'; + return "SimplePerformanceTestStatus{" + "configMapStatus='" + configMapStatus + '\'' + '}'; } } From f5e5509a61c44953d67f5da8c1213f0f1c827fa5 Mon Sep 17 00:00:00 2001 From: Chris Laprun Date: Wed, 28 Jan 2026 18:49:38 +0100 Subject: [PATCH 2/2] fix: improper name Signed-off-by: Chris Laprun --- .../baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java index b59c393b04..c19eb3802d 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java @@ -29,6 +29,6 @@ public void setConfigMapStatus(String configMapStatus) { @Override public String toString() { - return "SimplePerformanceTestStatus{" + "configMapStatus='" + configMapStatus + '\'' + '}'; + return "SSAFinalizerIssueStatus{" + "configMapStatus='" + configMapStatus + '\'' + '}'; } }