Skip to content

feat: metrics contains the whole resource #1645

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 5, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.util.Map;
import java.util.Optional;

import io.fabric8.kubernetes.api.model.HasMetadata;
import io.javaoperatorsdk.operator.OperatorException;
import io.javaoperatorsdk.operator.api.monitoring.Metrics;
import io.javaoperatorsdk.operator.api.reconciler.Constants;
Expand Down Expand Up @@ -84,10 +85,10 @@ public void cleanupDoneFor(ResourceID resourceID, Map<String, Object> metadata)
}

@Override
public void reconcileCustomResource(ResourceID resourceID, RetryInfo retryInfoNullable,
public void reconcileCustomResource(HasMetadata resource, RetryInfo retryInfoNullable,
Map<String, Object> metadata) {
Optional<RetryInfo> retryInfo = Optional.ofNullable(retryInfoNullable);
incrementCounter(resourceID, RECONCILIATIONS + "started",
incrementCounter(ResourceID.fromResource(resource), RECONCILIATIONS + "started",
metadata,
RECONCILIATIONS + "retries.number",
"" + retryInfo.map(RetryInfo::getAttemptCount).orElse(0),
Expand All @@ -96,19 +97,20 @@ public void reconcileCustomResource(ResourceID resourceID, RetryInfo retryInfoNu
}

@Override
public void finishedReconciliation(ResourceID resourceID, Map<String, Object> metadata) {
incrementCounter(resourceID, RECONCILIATIONS + "success", metadata);
public void finishedReconciliation(HasMetadata resource, Map<String, Object> metadata) {
incrementCounter(ResourceID.fromResource(resource), RECONCILIATIONS + "success", metadata);
}

public void failedReconciliation(ResourceID resourceID, Exception exception,
public void failedReconciliation(HasMetadata resource, Exception exception,
Map<String, Object> metadata) {
var cause = exception.getCause();
if (cause == null) {
cause = exception;
} else if (cause instanceof RuntimeException) {
cause = cause.getCause() != null ? cause.getCause() : cause;
}
incrementCounter(resourceID, RECONCILIATIONS + "failed", metadata, "exception",
incrementCounter(ResourceID.fromResource(resource), RECONCILIATIONS + "failed", metadata,
"exception",
cause.getClass().getSimpleName());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,56 +29,39 @@ public interface Metrics {
*/
default void receivedEvent(Event event, Map<String, Object> metadata) {}

/**
*
* @deprecated Use (and implement) {@link #receivedEvent(Event, Map)} instead
*/
@Deprecated
default void receivedEvent(Event event) {
receivedEvent(event, Collections.emptyMap());
}

/**
*
* @deprecated Use (and implement) {@link #reconcileCustomResource(ResourceID, RetryInfo, Map)}
* instead
*/
@Deprecated
default void reconcileCustomResource(ResourceID resourceID, RetryInfo retryInfo) {
reconcileCustomResource(resourceID, retryInfo, Collections.emptyMap());
}
@Deprecated(forRemoval = true)
default void reconcileCustomResource(ResourceID resourceID, RetryInfo retryInfo,
Map<String, Object> metadata) {}

/**
* Called right before a resource is dispatched to the ExecutorService for reconciliation.
*
* @param resourceID the {@link ResourceID} associated with the resource
* @param resource the associated with the resource
* @param retryInfo the current retry state information for the reconciliation request
* @param metadata metadata associated with the resource being processed
*/
default void reconcileCustomResource(ResourceID resourceID, RetryInfo retryInfo,
Map<String, Object> metadata) {}

/**
*
* @deprecated Use (and implement) {@link #failedReconciliation(ResourceID, Exception, Map)}
* instead
*/
@Deprecated
default void failedReconciliation(ResourceID resourceID, Exception exception) {
failedReconciliation(resourceID, exception, Collections.emptyMap());
default void reconcileCustomResource(HasMetadata resource, RetryInfo retryInfo,
Map<String, Object> metadata) {
reconcileCustomResource(ResourceID.fromResource(resource), retryInfo, metadata);
}

@Deprecated(forRemoval = true)
default void failedReconciliation(ResourceID resourceID, Exception exception,
Map<String, Object> metadata) {}

/**
* Called when a precedent reconciliation for the resource associated with the specified
* {@link ResourceID} resulted in the provided exception, resulting in a retry of the
* reconciliation.
*
* @param resourceID the {@link ResourceID} associated with the resource being processed
* @param resource the {@link ResourceID} associated with the resource being processed
* @param exception the exception that caused the failed reconciliation resulting in a retry
* @param metadata metadata associated with the resource being processed
*/
default void failedReconciliation(ResourceID resourceID, Exception exception,
Map<String, Object> metadata) {}
default void failedReconciliation(HasMetadata resource, Exception exception,
Map<String, Object> metadata) {
failedReconciliation(ResourceID.fromResource(resource), exception, metadata);
}

/**
*
Expand Down Expand Up @@ -107,16 +90,21 @@ default void finishedReconciliation(ResourceID resourceID) {
finishedReconciliation(resourceID, Collections.emptyMap());
}

@Deprecated(forRemoval = true)
default void finishedReconciliation(ResourceID resourceID, Map<String, Object> metadata) {}

/**
* Called when the
* {@link io.javaoperatorsdk.operator.api.reconciler.Reconciler#reconcile(HasMetadata, Context)}
* method of the Reconciler associated with the resource associated with the specified
* {@link ResourceID} has sucessfully finished.
*
* @param resourceID the {@link ResourceID} associated with the resource being processed
* @param resource the {@link ResourceID} associated with the resource being processed
* @param metadata metadata associated with the resource being processed
*/
default void finishedReconciliation(ResourceID resourceID, Map<String, Object> metadata) {}
default void finishedReconciliation(HasMetadata resource, Map<String, Object> metadata) {
finishedReconciliation(ResourceID.fromResource(resource), metadata);
}

/**
* Encapsulates the information about a controller execution i.e. a call to either
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,25 +32,25 @@

import static io.javaoperatorsdk.operator.processing.KubernetesResourceUtils.getName;

public class EventProcessor<R extends HasMetadata> implements EventHandler, LifecycleAware {
public class EventProcessor<P extends HasMetadata> implements EventHandler, LifecycleAware {

private static final Logger log = LoggerFactory.getLogger(EventProcessor.class);
private static final long MINIMAL_RATE_LIMIT_RESCHEDULE_DURATION = 50;

private volatile boolean running;
private final ControllerConfiguration<?> controllerConfiguration;
private final ReconciliationDispatcher<R> reconciliationDispatcher;
private final ReconciliationDispatcher<P> reconciliationDispatcher;
private final Retry retry;
private final ExecutorService executor;
private final Metrics metrics;
private final Cache<R> cache;
private final EventSourceManager<R> eventSourceManager;
private final Cache<P> cache;
private final EventSourceManager<P> eventSourceManager;
private final RateLimiter<? extends RateLimitState> rateLimiter;
private final ResourceStateManager resourceStateManager = new ResourceStateManager();
private final Map<String, Object> metricsMetadata;


public EventProcessor(EventSourceManager<R> eventSourceManager) {
public EventProcessor(EventSourceManager<P> eventSourceManager) {
this(
eventSourceManager.getController().getConfiguration(),
eventSourceManager.getControllerResourceEventSource(),
Expand All @@ -63,8 +63,8 @@ public EventProcessor(EventSourceManager<R> eventSourceManager) {
@SuppressWarnings("rawtypes")
EventProcessor(
ControllerConfiguration controllerConfiguration,
ReconciliationDispatcher<R> reconciliationDispatcher,
EventSourceManager<R> eventSourceManager,
ReconciliationDispatcher<P> reconciliationDispatcher,
EventSourceManager<P> eventSourceManager,
Metrics metrics) {
this(
controllerConfiguration,
Expand All @@ -78,11 +78,11 @@ public EventProcessor(EventSourceManager<R> eventSourceManager) {
@SuppressWarnings({"rawtypes", "unchecked"})
private EventProcessor(
ControllerConfiguration controllerConfiguration,
Cache<R> cache,
Cache<P> cache,
ExecutorService executor,
ReconciliationDispatcher<R> reconciliationDispatcher,
ReconciliationDispatcher<P> reconciliationDispatcher,
Metrics metrics,
EventSourceManager<R> eventSourceManager) {
EventSourceManager<P> eventSourceManager) {
this.controllerConfiguration = controllerConfiguration;
this.running = false;
this.executor =
Expand Down Expand Up @@ -136,7 +136,7 @@ private void submitReconciliationExecution(ResourceState state) {
try {
boolean controllerUnderExecution = isControllerUnderExecution(state);
final var resourceID = state.getId();
Optional<R> maybeLatest = cache.get(resourceID);
Optional<P> maybeLatest = cache.get(resourceID);
maybeLatest.ifPresent(MDCUtils::addResourceInfo);
if (!controllerUnderExecution && maybeLatest.isPresent()) {
var rateLimit = state.getRateLimit();
Expand All @@ -151,9 +151,9 @@ private void submitReconciliationExecution(ResourceState state) {
}
state.setUnderProcessing(true);
final var latest = maybeLatest.get();
ExecutionScope<R> executionScope = new ExecutionScope<>(latest, state.getRetry());
ExecutionScope<P> executionScope = new ExecutionScope<>(latest, state.getRetry());
state.unMarkEventReceived();
metrics.reconcileCustomResource(resourceID, state.getRetry(), metricsMetadata);
metrics.reconcileCustomResource(latest, state.getRetry(), metricsMetadata);
log.debug("Executing events for custom resource. Scope: {}", executionScope);
executor.execute(new ReconcilerExecutor(executionScope));
} else {
Expand Down Expand Up @@ -221,7 +221,7 @@ private void handleRateLimitedSubmission(ResourceID resourceID, Duration minimal
}

synchronized void eventProcessingFinished(
ExecutionScope<R> executionScope, PostExecutionControl<R> postExecutionControl) {
ExecutionScope<P> executionScope, PostExecutionControl<P> postExecutionControl) {
if (!running) {
return;
}
Expand All @@ -244,7 +244,7 @@ synchronized void eventProcessingFinished(
return;
}
cleanupOnSuccessfulExecution(executionScope);
metrics.finishedReconciliation(resourceID, metricsMetadata);
metrics.finishedReconciliation(executionScope.getResource(), metricsMetadata);
if (state.deleteEventPresent()) {
cleanupForDeletedEvent(executionScope.getResourceID());
} else if (postExecutionControl.isFinalizerRemoved()) {
Expand All @@ -253,12 +253,12 @@ synchronized void eventProcessingFinished(
postExecutionControl
.getUpdatedCustomResource()
.ifPresent(
r -> {
p -> {
if (!postExecutionControl.updateIsStatusPatch()) {
eventSourceManager
.getControllerResourceEventSource()
.handleRecentResourceUpdate(
ResourceID.fromResource(r), r, executionScope.getResource());
ResourceID.fromResource(p), p, executionScope.getResource());
}
});
if (state.eventPresent()) {
Expand All @@ -270,7 +270,7 @@ synchronized void eventProcessingFinished(
}

private void reScheduleExecutionIfInstructed(
PostExecutionControl<R> postExecutionControl, R customResource) {
PostExecutionControl<P> postExecutionControl, P customResource) {

postExecutionControl
.getReScheduleDelay()
Expand All @@ -281,7 +281,7 @@ private void reScheduleExecutionIfInstructed(
}, () -> scheduleExecutionForMaxReconciliationInterval(customResource));
}

private void scheduleExecutionForMaxReconciliationInterval(R customResource) {
private void scheduleExecutionForMaxReconciliationInterval(P customResource) {
this.controllerConfiguration
.maxReconciliationInterval()
.ifPresent(m -> {
Expand All @@ -294,7 +294,7 @@ private void scheduleExecutionForMaxReconciliationInterval(R customResource) {
});
}

TimerEventSource<R> retryEventSource() {
TimerEventSource<P> retryEventSource() {
return eventSourceManager.retryEventSource();
}

Expand All @@ -304,7 +304,7 @@ TimerEventSource<R> retryEventSource() {
* according to the retry timing if there was an exception.
*/
private void handleRetryOnException(
ExecutionScope<R> executionScope, Exception exception) {
ExecutionScope<P> executionScope, Exception exception) {
final var state = getOrInitRetryExecution(executionScope);
var resourceID = state.getId();
boolean eventPresent = state.eventPresent();
Expand All @@ -323,7 +323,7 @@ private void handleRetryOnException(
"Scheduling timer event for retry with delay:{} for resource: {}",
delay,
resourceID);
metrics.failedReconciliation(resourceID, exception, metricsMetadata);
metrics.failedReconciliation(executionScope.getResource(), exception, metricsMetadata);
retryEventSource().scheduleOnce(resourceID, delay);
},
() -> {
Expand All @@ -332,7 +332,7 @@ private void handleRetryOnException(
});
}

private void cleanupOnSuccessfulExecution(ExecutionScope<R> executionScope) {
private void cleanupOnSuccessfulExecution(ExecutionScope<P> executionScope) {
log.debug(
"Cleanup for successful execution for resource: {}", getName(executionScope.getResource()));
if (isRetryConfigured()) {
Expand All @@ -341,7 +341,7 @@ private void cleanupOnSuccessfulExecution(ExecutionScope<R> executionScope) {
retryEventSource().cancelOnceSchedule(executionScope.getResourceID());
}

private ResourceState getOrInitRetryExecution(ExecutionScope<R> executionScope) {
private ResourceState getOrInitRetryExecution(ExecutionScope<P> executionScope) {
final var state = resourceStateManager.getOrCreate(executionScope.getResourceID());
RetryExecution retryExecution = state.getRetry();
if (retryExecution == null) {
Expand Down Expand Up @@ -387,9 +387,9 @@ private void handleAlreadyMarkedEvents() {
}

private class ReconcilerExecutor implements Runnable {
private final ExecutionScope<R> executionScope;
private final ExecutionScope<P> executionScope;

private ReconcilerExecutor(ExecutionScope<R> executionScope) {
private ReconcilerExecutor(ExecutionScope<P> executionScope) {
this.executionScope = executionScope;
}

Expand All @@ -401,7 +401,7 @@ public void run() {
try {
MDCUtils.addResourceInfo(executionScope.getResource());
thread.setName("ReconcilerExecutor-" + controllerName() + "-" + thread.getId());
PostExecutionControl<R> postExecutionControl =
PostExecutionControl<P> postExecutionControl =
reconciliationDispatcher.handleExecution(executionScope);
eventProcessingFinished(executionScope, postExecutionControl);
} finally {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ void startProcessedMarkedEventReceivedBefore() {
eventProcessor.start();

verify(reconciliationDispatcherMock, timeout(100).times(1)).handleExecution(any());
verify(metricsMock, times(1)).reconcileCustomResource(any(), isNull(), any());
verify(metricsMock, times(1)).reconcileCustomResource(any(HasMetadata.class), isNull(), any());
}

@Test
Expand Down