DI
Idempotency
Distributed Systems core v1.0.0
Idempotency
Overview
Idempotency ensures that performing an operation multiple times has the same effect as performing it once. This is critical in distributed systems where retries, network duplicates, and at-least-once delivery are common.
Key Concepts
Idempotency Patterns
┌─────────────────────────────────────────────────────────────┐
│ Idempotency Spectrum │
├─────────────────────────────────────────────────────────────┤
│ │
│ Naturally Idempotent: │
│ ┌─────────────────────────────────────────────────────┐ │
│ │ GET /users/123 ✓ Always safe to retry │ │
│ │ PUT /users/123 ✓ Sets absolute state │ │
│ │ DELETE /users/123 ✓ Removes if exists │ │
│ └─────────────────────────────────────────────────────┘ │
│ │
│ NOT Naturally Idempotent: │
│ ┌─────────────────────────────────────────────────────┐ │
│ │ POST /users ✗ Creates new each time │ │
│ │ POST /transfer ✗ Moves money each time │ │
│ │ counter++ ✗ Increments each time │ │
│ └─────────────────────────────────────────────────────┘ │
│ │
│ Making Operations Idempotent: │
│ ┌─────────────────────────────────────────────────────┐ │
│ │ │ │
│ │ 1. Idempotency Key │ │
│ │ Client generates unique key per logical op │ │
│ │ Server deduplicates by key │ │
│ │ │ │
│ │ 2. Conditional Updates │ │
│ │ IF version = X THEN update SET version = X+1 │ │
│ │ Second attempt fails version check │ │
│ │ │ │
│ │ 3. Natural Keys │ │
│ │ Use business identifiers that are unique │ │
│ │ UPSERT by natural key │ │
│ │ │ │
│ │ 4. State Machine │ │
│ │ Operations only valid in certain states │ │
│ │ State transitions are idempotent │ │
│ │ │ │
│ └─────────────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────┘
Best Practices
1. Client-Generated Idempotency Keys
Let clients generate unique keys for retry safety.
2. Store Results, Not Just Keys
Return cached result for duplicate requests.
3. Set Appropriate TTL
Keep idempotency records long enough for retries.
4. Use Database Constraints
Leverage unique constraints for natural idempotency.
5. Make State Transitions Explicit
Model operations as state changes.
Code Examples
Example 1: Idempotency Key Implementation
@Service
public class IdempotentRequestService {
private final IdempotencyStore idempotencyStore;
private final ObjectMapper objectMapper;
/**
* Execute operation with idempotency guarantee
*/
public <T> T executeIdempotent(
String idempotencyKey,
Duration ttl,
Supplier<T> operation,
Class<T> resultType) {
// Check for existing result
Optional<IdempotencyRecord> existing = idempotencyStore.find(idempotencyKey);
if (existing.isPresent()) {
IdempotencyRecord record = existing.get();
switch (record.getStatus()) {
case COMPLETED:
log.info("Returning cached result for idempotency key: {}", idempotencyKey);
return deserialize(record.getResult(), resultType);
case IN_PROGRESS:
// Another request is processing
throw new ConflictException("Request in progress: " + idempotencyKey);
case FAILED:
// Previous attempt failed, allow retry
log.info("Previous attempt failed, retrying: {}", idempotencyKey);
break;
}
}
// Create in-progress record
IdempotencyRecord record = IdempotencyRecord.builder()
.key(idempotencyKey)
.status(IdempotencyStatus.IN_PROGRESS)
.createdAt(Instant.now())
.expiresAt(Instant.now().plus(ttl))
.build();
try {
idempotencyStore.create(record);
} catch (DuplicateKeyException e) {
// Race condition - another request started
throw new ConflictException("Request in progress: " + idempotencyKey);
}
try {
// Execute the operation
T result = operation.get();
// Store successful result
record = record.toBuilder()
.status(IdempotencyStatus.COMPLETED)
.result(serialize(result))
.completedAt(Instant.now())
.build();
idempotencyStore.update(record);
return result;
} catch (Exception e) {
// Store failure
record = record.toBuilder()
.status(IdempotencyStatus.FAILED)
.error(e.getMessage())
.build();
idempotencyStore.update(record);
throw e;
}
}
private String serialize(Object obj) {
try {
return objectMapper.writeValueAsString(obj);
} catch (JsonProcessingException e) {
throw new RuntimeException("Failed to serialize result", e);
}
}
private <T> T deserialize(String json, Class<T> type) {
try {
return objectMapper.readValue(json, type);
} catch (JsonProcessingException e) {
throw new RuntimeException("Failed to deserialize result", e);
}
}
}
@Repository
public class RedisIdempotencyStore implements IdempotencyStore {
private final RedisTemplate<String, IdempotencyRecord> redisTemplate;
@Override
public void create(IdempotencyRecord record) {
Boolean created = redisTemplate.opsForValue().setIfAbsent(
"idempotency:" + record.getKey(),
record,
Duration.between(Instant.now(), record.getExpiresAt())
);
if (!Boolean.TRUE.equals(created)) {
throw new DuplicateKeyException("Key already exists: " + record.getKey());
}
}
@Override
public Optional<IdempotencyRecord> find(String key) {
IdempotencyRecord record = redisTemplate.opsForValue().get("idempotency:" + key);
return Optional.ofNullable(record);
}
@Override
public void update(IdempotencyRecord record) {
redisTemplate.opsForValue().set(
"idempotency:" + record.getKey(),
record,
Duration.between(Instant.now(), record.getExpiresAt())
);
}
}
Example 2: REST API with Idempotency Header
@RestController
@RequestMapping("/api/payments")
public class PaymentController {
private final PaymentService paymentService;
private final IdempotentRequestService idempotentService;
@PostMapping
public ResponseEntity<PaymentResponse> createPayment(
@RequestHeader("Idempotency-Key") String idempotencyKey,
@Valid @RequestBody PaymentRequest request) {
// Validate idempotency key format
if (!isValidIdempotencyKey(idempotencyKey)) {
return ResponseEntity.badRequest()
.body(PaymentResponse.error("Invalid idempotency key format"));
}
// Create hash of request for validation
String requestHash = hashRequest(request);
try {
PaymentResponse response = idempotentService.executeIdempotent(
idempotencyKey,
Duration.ofHours(24),
() -> {
// Verify request matches original if replaying
verifyRequestConsistency(idempotencyKey, requestHash);
return paymentService.processPayment(request);
},
PaymentResponse.class
);
return ResponseEntity.ok()
.header("Idempotency-Key", idempotencyKey)
.body(response);
} catch (ConflictException e) {
return ResponseEntity.status(HttpStatus.CONFLICT)
.body(PaymentResponse.error("Request already in progress"));
}
}
private void verifyRequestConsistency(String key, String currentHash) {
// Optionally store and verify request hash to detect
// different requests with same idempotency key
}
private String hashRequest(PaymentRequest request) {
return Hashing.sha256()
.hashString(
request.getAmount() + request.getCurrency() + request.getRecipient(),
StandardCharsets.UTF_8
)
.toString();
}
}
// Client usage
public class PaymentClient {
public PaymentResponse createPayment(PaymentRequest request) {
// Generate idempotency key on client
String idempotencyKey = UUID.randomUUID().toString();
return retryWithBackoff(() ->
httpClient.post("/api/payments")
.header("Idempotency-Key", idempotencyKey)
.body(request)
.execute(PaymentResponse.class)
);
}
}
Example 3: Database-Level Idempotency
@Service
@Transactional
public class OrderService {
private final OrderRepository orderRepository;
private final ProcessedRequestRepository processedRequests;
/**
* Idempotent order creation using database constraints
*/
public Order createOrder(String clientOrderId, OrderRequest request) {
// Check if already processed
Optional<Order> existing = orderRepository.findByClientOrderId(clientOrderId);
if (existing.isPresent()) {
log.info("Order already exists for clientOrderId: {}", clientOrderId);
return existing.get();
}
// Create order with unique constraint on clientOrderId
Order order = Order.builder()
.clientOrderId(clientOrderId) // Unique constraint
.customerId(request.getCustomerId())
.items(request.getItems())
.total(calculateTotal(request))
.status(OrderStatus.CREATED)
.build();
try {
return orderRepository.save(order);
} catch (DataIntegrityViolationException e) {
// Race condition - another request created it
return orderRepository.findByClientOrderId(clientOrderId)
.orElseThrow(() -> new IllegalStateException("Order disappeared"));
}
}
/**
* Idempotent state transition using optimistic locking
*/
public Order confirmOrder(String orderId, int expectedVersion) {
Order order = orderRepository.findById(orderId)
.orElseThrow(() -> new NotFoundException("Order not found"));
// Version check for idempotency
if (order.getVersion() != expectedVersion) {
// State has changed, check if already confirmed
if (order.getStatus() == OrderStatus.CONFIRMED) {
log.info("Order already confirmed: {}", orderId);
return order;
}
throw new OptimisticLockException("Order was modified");
}
// Validate state transition
if (order.getStatus() != OrderStatus.CREATED) {
throw new InvalidStateException(
"Cannot confirm order in status: " + order.getStatus()
);
}
order.setStatus(OrderStatus.CONFIRMED);
order.setConfirmedAt(Instant.now());
return orderRepository.save(order);
}
/**
* Idempotent increment using conditional update
*/
public int incrementInventory(String productId, int amount, String operationId) {
// First check if operation was already applied
if (processedRequests.exists(operationId)) {
log.info("Operation already processed: {}", operationId);
return getInventory(productId);
}
// Use atomic conditional update
int updated = jdbcTemplate.update(
"""
UPDATE inventory
SET quantity = quantity + ?,
last_operation_id = ?
WHERE product_id = ?
AND (last_operation_id IS NULL OR last_operation_id != ?)
""",
amount, operationId, productId, operationId
);
if (updated == 0) {
// Either product doesn't exist or operation was duplicate
log.info("No update performed for operation: {}", operationId);
}
// Record operation
processedRequests.save(new ProcessedRequest(operationId, Instant.now()));
return getInventory(productId);
}
}
// Entity with optimistic locking
@Entity
public class Order {
@Id
private String id;
@Column(unique = true)
private String clientOrderId;
@Version
private int version;
@Enumerated(EnumType.STRING)
private OrderStatus status;
// ... other fields
}
Example 4: Event Processing Idempotency
@Service
public class IdempotentEventProcessor {
private final ProcessedEventRepository processedEvents;
private final EventHandler eventHandler;
@Transactional
public void processEvent(Event event) {
String eventId = event.getId();
// Check if already processed
if (processedEvents.existsById(eventId)) {
log.debug("Event already processed: {}", eventId);
return;
}
// Process the event
try {
eventHandler.handle(event);
// Record as processed
processedEvents.save(new ProcessedEvent(
eventId,
event.getType(),
Instant.now()
));
} catch (Exception e) {
// Don't mark as processed - allow retry
log.error("Failed to process event: {}", eventId, e);
throw e;
}
}
/**
* Idempotent event processing with deduplication window
*/
@KafkaListener(topics = "orders")
public void handleOrderEvent(
@Payload OrderEvent event,
@Header(KafkaHeaders.RECEIVED_KEY) String key,
@Header(KafkaHeaders.RECEIVED_PARTITION) int partition,
@Header(KafkaHeaders.OFFSET) long offset) {
// Create composite dedup key
String dedupKey = String.format("%s-%d-%d", key, partition, offset);
// Use Redis for distributed deduplication
Boolean isNew = redisTemplate.opsForValue().setIfAbsent(
"processed:" + dedupKey,
"1",
Duration.ofHours(24) // Dedup window
);
if (!Boolean.TRUE.equals(isNew)) {
log.debug("Duplicate event: {}", dedupKey);
return;
}
try {
processOrderEvent(event);
} catch (Exception e) {
// Remove dedup key to allow retry
redisTemplate.delete("processed:" + dedupKey);
throw e;
}
}
}
Example 5: Outbox Pattern for Idempotent Publishing
@Service
public class OutboxService {
private final OutboxRepository outboxRepository;
private final MessagePublisher publisher;
/**
* Transactional outbox pattern
* Ensures exactly-once publishing with at-least-once delivery
*/
@Transactional
public void saveWithOutbox(Order order, OrderEvent event) {
// Save business entity
orderRepository.save(order);
// Save outbox entry in same transaction
OutboxEntry entry = OutboxEntry.builder()
.id(UUID.randomUUID().toString())
.aggregateType("Order")
.aggregateId(order.getId())
.eventType(event.getClass().getSimpleName())
.payload(serialize(event))
.createdAt(Instant.now())
.status(OutboxStatus.PENDING)
.build();
outboxRepository.save(entry);
}
/**
* Outbox processor with idempotent publishing
*/
@Scheduled(fixedDelay = 1000)
public void processOutbox() {
List<OutboxEntry> pending = outboxRepository.findByStatus(
OutboxStatus.PENDING,
PageRequest.of(0, 100)
);
for (OutboxEntry entry : pending) {
try {
// Publish with deduplication key
publisher.publish(
entry.getEventType(),
entry.getPayload(),
Map.of(
"dedup-id", entry.getId(),
"aggregate-id", entry.getAggregateId()
)
);
// Mark as published
entry.setStatus(OutboxStatus.PUBLISHED);
entry.setPublishedAt(Instant.now());
outboxRepository.save(entry);
} catch (Exception e) {
log.error("Failed to publish outbox entry: {}", entry.getId(), e);
entry.setRetryCount(entry.getRetryCount() + 1);
if (entry.getRetryCount() >= 10) {
entry.setStatus(OutboxStatus.FAILED);
}
outboxRepository.save(entry);
}
}
}
/**
* Consumer-side idempotency
*/
@KafkaListener(topics = "order-events")
@Transactional
public void handleEvent(
@Payload String payload,
@Header("dedup-id") String dedupId) {
// Check if already processed using database
if (processedMessageRepository.existsById(dedupId)) {
log.debug("Message already processed: {}", dedupId);
return;
}
// Process the event
OrderEvent event = deserialize(payload);
eventHandler.handle(event);
// Mark as processed
processedMessageRepository.save(new ProcessedMessage(
dedupId,
Instant.now()
));
}
}
@Entity
@Table(name = "outbox")
class OutboxEntry {
@Id
private String id;
private String aggregateType;
private String aggregateId;
private String eventType;
@Lob
private String payload;
@Enumerated(EnumType.STRING)
private OutboxStatus status;
private Instant createdAt;
private Instant publishedAt;
private int retryCount;
}
Anti-Patterns
❌ Assuming Network Reliability
// WRONG - no idempotency protection
public void transfer(String from, String to, BigDecimal amount) {
debit(from, amount);
credit(to, amount); // What if this fails after debit?
}
// ✅ CORRECT - idempotent with compensation
public void transfer(String transferId, String from, String to, BigDecimal amount) {
// Check if already completed
if (isTransferComplete(transferId)) return;
// Idempotent debit with transfer reference
debit(transferId + "-debit", from, amount);
try {
credit(transferId + "-credit", to, amount);
markTransferComplete(transferId);
} catch (Exception e) {
// Compensate the debit
credit(transferId + "-debit-reversal", from, amount);
throw e;
}
}
❌ Short Idempotency TTL
Set TTL longer than maximum retry window.