GH-3146: Propagate KT observation callback

Fixes: #3146

Propagate `KafkaTemplate` observation callback.

There is a requirement to see logs correlated in the `CompletableFuture` result 
after `KafkaTemplate.send()` operation.

* Add `observation.openScope()` into the `buildCallback()` logic

**Auto-cherry-pick to `3.1.x` & `3.0.x`**

spring-kafka/src/main/java/org/springframework/kafka/core/KafkaTemplate.java

@@ -829,6 +829,7 @@ private ProducerRecord<K, V> interceptorProducerRecord(ProducerRecord<K, V> prod
 		return producerRecord;
 	}
 
+	@SuppressWarnings("try")
 	private Callback buildCallback(final ProducerRecord<K, V> producerRecord, final Producer<K, V> producer,
 			final CompletableFuture<SendResult<K, V>> future, @Nullable Object sample, Observation observation) {
 
@@ -841,10 +842,9 @@ private Callback buildCallback(final ProducerRecord<K, V> producerRecord, final
 			catch (Exception e) {
 				this.logger.warn(e, () ->  "Error executing interceptor onAcknowledgement callback");
 			}
-			try {
+			try (Observation.Scope ignored = observation.openScope()) {
 				if (exception == null) {
 					successTimer(sample, producerRecord);
-					observation.stop();
 					future.complete(new SendResult<>(producerRecord, metadata));
 					if (this.producerListener != null) {
 						this.producerListener.onSuccess(producerRecord, metadata);
@@ -855,7 +855,6 @@ private Callback buildCallback(final ProducerRecord<K, V> producerRecord, final
 				else {
 					failureTimer(sample, exception, producerRecord);
 					observation.error(exception);
-					observation.stop();
 					future.completeExceptionally(
 							new KafkaProducerException(producerRecord, "Failed to send", exception));
 					if (this.producerListener != null) {
@@ -865,6 +864,7 @@ private Callback buildCallback(final ProducerRecord<K, V> producerRecord, final
 				}
 			}
 			finally {
+				observation.stop();
 				closeProducer(producer, this.transactional);
 			}
 		};

spring-kafka/src/test/java/org/springframework/kafka/support/micrometer/ObservationTests.java

@@ -29,6 +29,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -108,7 +109,14 @@ void endToEnd(@Autowired Listener listener, @Autowired KafkaTemplate<Integer, St
 			@Autowired Config config)
 					throws InterruptedException, ExecutionException, TimeoutException {
 
-		template.send("observation.testT1", "test").get(10, TimeUnit.SECONDS);
+		AtomicReference<SimpleSpan> spanFromCallback = new AtomicReference<>();
+
+		template.send("observation.testT1", "test")
+				.thenAccept((sendResult) -> spanFromCallback.set(tracer.currentSpan()))
+				.get(10, TimeUnit.SECONDS);
+
+		assertThat(spanFromCallback.get()).isNotNull();
+
 		assertThat(listener.latch1.await(10, TimeUnit.SECONDS)).isTrue();
 		assertThat(listener.record).isNotNull();
 		Headers headers = listener.record.headers();