From 0312cf699f2c31e35b4b4c844183ce56deb74839 Mon Sep 17 00:00:00 2001 From: Leo Galambos Date: Wed, 12 Aug 2026 01:48:11 +0200 Subject: [PATCH] feat(pki): reconcile recovered signing operations Add bounded durable reconciliation with retry metadata, fencing-safe status and cancellation handling, and server-managed background recovery. Include versioned persistence migration, bounded keyset paging, lifecycle-safe worker shutdown, redacted diagnostics, and restart/failure coverage. Closes #10 --- docs/pki-server-acme.md | 2 +- docs/pki-server-admin-https.md | 15 + docs/pki-server-architecture.md | 40 +- docs/pki-server-loopback-example.json | 3 +- docs/pki-server-production-example.json | 3 +- ...ki-server-trusted-proxy-nginx-example.json | 3 +- ...-server-trusted-proxy-rfc9440-example.json | 3 +- .../zeroecho/pki/server/PkiHttpsServer.java | 49 +- .../pki/server/PkiServerConfiguration.java | 50 +- .../server/PkiServerConfigurationCodec.java | 25 +- .../server/SigningReconciliationWorker.java | 130 +++++ .../zeroecho/pki/server/AcmeEndToEndTest.java | 5 +- .../pki/server/PkiHttpsServerTest.java | 20 +- .../SigningReconciliationWorkerTest.java | 127 +++++ .../pki/application/DefaultPkiSession.java | 8 + .../zeroecho/pki/application/PkiSession.java | 13 + .../SigningReconciliationRequest.java | 57 ++ .../SigningReconciliationResult.java | 31 + .../pki/impl/core/async/PkiSigningBus.java | 367 +++++++++++- .../ZeroEchoLibSignatureWorkflow.java | 160 +++++- .../bc/WorkflowProofOfPossessionVerifier.java | 7 +- .../pki/impl/fs/FilesystemPkiStore.java | 167 +++++- .../java/zeroecho/pki/impl/fs/FsCodec.java | 57 +- .../pki/impl/fs/MetadataStateIndex.java | 89 ++- .../impl/fs/PosixMetadataSnapshotSupport.java | 33 +- .../pki/impl/fs/PosixMetadataStoreEngine.java | 11 +- .../pki/spi/crypto/SignatureWorkflow.java | 71 ++- .../pki/spi/store/SignWorkflowStore.java | 72 ++- .../core/async/PkiSigningBusFailureTest.java | 538 ++++++++++++++++-- .../PkiSigningBusOperatorApprovalTest.java | 6 +- .../ZeroEchoLibKeyRefParsingTest.java | 15 +- ...hoLibSignatureWorkflowPersistenceTest.java | 156 +++-- ...gnatureWorkflowVerifyEncodedEcdsaTest.java | 6 +- ...LibSignatureWorkflowVerifyEncodedTest.java | 6 +- .../fs/FilesystemSignWorkflowStoreTest.java | 136 ++++- .../pki/impl/fs/MetadataStateIndexTest.java | 23 + .../pki/spi/bootstrap/PkiBootstrapTest.java | 9 +- .../DurableDelayedSignatureWorkflow.java | 17 +- ...ableOperatorApprovalSignatureWorkflow.java | 18 +- .../testkit/InMemorySignatureWorkflow.java | 17 +- .../OperatorApprovalSignatureWorkflow.java | 19 +- .../pki/testkit/TestSignIdentityRegistry.java | 9 + 42 files changed, 2318 insertions(+), 275 deletions(-) create mode 100644 pki-server/src/main/java/zeroecho/pki/server/SigningReconciliationWorker.java create mode 100644 pki-server/src/test/java/zeroecho/pki/server/SigningReconciliationWorkerTest.java create mode 100644 pki/src/main/java/zeroecho/pki/application/SigningReconciliationRequest.java create mode 100644 pki/src/main/java/zeroecho/pki/application/SigningReconciliationResult.java diff --git a/docs/pki-server-acme.md b/docs/pki-server-acme.md index 8b0f443..72046ab 100644 --- a/docs/pki-server-acme.md +++ b/docs/pki-server-acme.md @@ -8,7 +8,7 @@ An ACME directory is an immutable policy revision bound to one realm, logical au Direct deployments use server-authenticated TLS. A client TLS certificate is not ACME account authority; ACME identity is the account key authenticated by JWS. Trusted-reverse-proxy deployments require the established mutually authenticated proxy-to-ZeroEcho TLS hop and a dedicated enabled proxy principal with `FORWARD_AUTHENTICATED_CLIENT_IDENTITY`. Forwarded administrative identity is not used as an ACME account. Source addresses and `Forwarded` or `X-Forwarded-*` headers never authorize ACME. -The server configuration schema is version 5. ACME is disabled explicitly with: +The server configuration schema is version 6. ACME is disabled explicitly with: ```json "acmeListener": {"enabled": false} diff --git a/docs/pki-server-admin-https.md b/docs/pki-server-admin-https.md index 30baef0..bbd47b7 100644 --- a/docs/pki-server-admin-https.md +++ b/docs/pki-server-admin-https.md @@ -84,8 +84,23 @@ Responses are deterministic version-one JSON. They carry only safe typed results Transport workers, transport backlog, concurrent operation workers, waiting operation capacity, admitted requests, request body size, deadlines, and both shutdown periods are finite configuration values. Saturation never runs a request on the caller thread and a rejected request never invokes the gateway. +Configuration schema version 6 adds `signingReconciliation`. Its production +defaults are enabled, 256 examined records, 64 provider calls, a 30-second pass +timeout, a 10-second provider-call timeout, and a fixed 2-second idle interval. +`maxRecords` is 1 through 4096, `maxCalls` is 1 through the record +bound, all durations are positive, and the provider timeout cannot exceed the +pass timeout. Version 5 configurations migrate to these defaults. Set `enabled` +to `false` only when an external owner deliberately invokes the transport-neutral +session reconciliation API. + Shutdown first removes readiness and admission, stops listener acceptance, drains for the configured graceful period, cooperatively cancels remaining work, waits only for the forced period, then closes the one realm context. Already committed PKI changes remain committed. +Signing-provider deadlines and cancellation are cooperative for trusted +in-process implementations. A provider call that remains live after forced +shutdown causes `SIGNING_RECONCILIATION_SHUTDOWN_TIMEOUT`; the session, provider, +and store stay open and `close` may be retried after the call returns. Hard +isolation for non-cooperative providers requires an out-of-process boundary. + ## Request correlation Clients may supply `X-ZeroEcho-Request-Id` using 16–128 ASCII letters, digits, underscore, or hyphen. Otherwise the server creates a random opaque identifier. It is correlation metadata—not an idempotency key, object identity, approval, or capability token. diff --git a/docs/pki-server-architecture.md b/docs/pki-server-architecture.md index 4edff45..25f0ad1 100644 --- a/docs/pki-server-architecture.md +++ b/docs/pki-server-architecture.md @@ -243,9 +243,47 @@ retires advisory state before reporting that marker. The retained record does no regain a live content reference. Store restart recovery uses authoritative live references to reclaim the orphaned staged file safely. +Each fully opened realm owns exactly one signing-reconciliation worker. The +worker starts immediately before the server becomes ready, runs one synchronous +bounded pass at a time, and self-schedules the next pass only after the current +pass returns. It has no queue of missed ticks and creates no active-active lease; +the existing realm, provider, and filesystem ownership locks remain the +single-active authority. + +The reconciliation cursor is an advisory exclusive key into a metadata snapshot +ordered by canonical submission identifier. A pass examines no more than its +record bound, makes no more than its independent provider-call budget, and wraps +to the beginning after reaching the end so deferred records cannot starve other +records. Per-record failures are isolated and persisted separately from the +provider detail code with a store-time retry schedule of 2, 4, 8, 16, then 30 +seconds. Terminal and retired records carry no retry metadata. + +Shutdown first stops reconciliation admission and scheduling, then cooperatively +cancels and drains the active pass before closing the realm session, providers, +or store. If a provider ignores cancellation beyond the forced-shutdown period, +the server reports `SIGNING_RECONCILIATION_SHUTDOWN_TIMEOUT` and leaves those +dependencies open so close can be retried without use-after-close behavior. + +`SignatureWorkflow` implementations are trusted in-process components and every +submit, status, verify, and cancel call now requires a `CallControl` carrying an +absolute deadline and cancellation signal. This is an intentionally breaking +provider-SPI migration: implementations must check the control before and +between I/O, before an externally visible effect, and before returning. These +controls are cooperative bounds; the server does not kill provider threads or +close their dependencies while a violating call remains live. Providers that +need enforcement against untrusted or non-cooperative code require a future +out-of-process provider boundary with process or RPC isolation. + Payload staging remains streaming `O(n)` time and `O(1)` aggregate auxiliary heap excluding the signature. Synchronous waiting performs -`O(TTL / polling interval)` status observations. +`O(TTL / polling interval)` status observations. For `M` metadata keys and a +bound `B`, an ordered reconciliation page is `O(log M + B)` traversal and each +candidate point access is `O(log M)`, for `O(B log M)` candidate access in the +worst case. Normal pass auxiliary memory is `O(B)`. A concurrent metadata commit +may detach the currently pinned index generation once in `O(M)` time and memory; +later commits do not copy again merely because an older generation remains +pinned. A pass uses at most `maximumProviderCalls` external calls, and each realm +owns `O(1)` scheduler state. ## 7. Authorization architecture diff --git a/docs/pki-server-loopback-example.json b/docs/pki-server-loopback-example.json index 2f1cfe1..c100b6a 100644 --- a/docs/pki-server-loopback-example.json +++ b/docs/pki-server-loopback-example.json @@ -1,5 +1,5 @@ { - "version": 5, + "version": 6, "serverName": "zeroecho-admin", "realm": { "realmId": "production", @@ -74,6 +74,7 @@ "gracefulShutdownMillis": 30000, "forcedShutdownMillis": 10000 }, + "signingReconciliation": {"enabled": true, "maxRecords": 256, "maxCalls": 64, "passTimeoutMillis": 30000, "providerCallTimeoutMillis": 10000, "idleIntervalMillis": 2000}, "publicListener": { "enabled": true, "address": "127.0.0.1", diff --git a/docs/pki-server-production-example.json b/docs/pki-server-production-example.json index a7a5d0b..3dd2a60 100644 --- a/docs/pki-server-production-example.json +++ b/docs/pki-server-production-example.json @@ -1,5 +1,5 @@ { - "version": 5, + "version": 6, "serverName": "zeroecho-admin", "realm": { "realmId": "production", @@ -74,6 +74,7 @@ "gracefulShutdownMillis": 30000, "forcedShutdownMillis": 10000 }, + "signingReconciliation": {"enabled": true, "maxRecords": 256, "maxCalls": 64, "passTimeoutMillis": 30000, "providerCallTimeoutMillis": 10000, "idleIntervalMillis": 2000}, "publicListener": {"enabled": false}, "acmeListener": {"enabled": false}, "runtime": {} diff --git a/docs/pki-server-trusted-proxy-nginx-example.json b/docs/pki-server-trusted-proxy-nginx-example.json index 3aaecd6..a8a5061 100644 --- a/docs/pki-server-trusted-proxy-nginx-example.json +++ b/docs/pki-server-trusted-proxy-nginx-example.json @@ -1,5 +1,5 @@ { - "version": 5, + "version": 6, "serverName": "zeroecho-admin-nginx", "realm": { "realmId": "production", @@ -40,6 +40,7 @@ "maximumForwardedChainBytes": 524288 }, "execution": {"transportWorkers":8,"transportQueueCapacity":64,"operationWorkers":4,"operationQueueCapacity":32,"maximumAdmittedRequests":96,"defaultDeadlineMillis":30000,"maximumDeadlineMillis":120000,"gracefulShutdownMillis":30000,"forcedShutdownMillis":10000}, + "signingReconciliation": {"enabled":true,"maxRecords":256,"maxCalls":64,"passTimeoutMillis":30000,"providerCallTimeoutMillis":10000,"idleIntervalMillis":2000}, "publicListener": {"enabled":false}, "acmeListener": {"enabled":false}, "runtime": {} diff --git a/docs/pki-server-trusted-proxy-rfc9440-example.json b/docs/pki-server-trusted-proxy-rfc9440-example.json index 3e6efdf..5855c29 100644 --- a/docs/pki-server-trusted-proxy-rfc9440-example.json +++ b/docs/pki-server-trusted-proxy-rfc9440-example.json @@ -1,5 +1,5 @@ { - "version": 5, + "version": 6, "serverName": "zeroecho-admin-proxy", "realm": { "realmId": "production", @@ -41,6 +41,7 @@ "maximumForwardedChainBytes": 524288 }, "execution": {"transportWorkers":8,"transportQueueCapacity":64,"operationWorkers":4,"operationQueueCapacity":32,"maximumAdmittedRequests":96,"defaultDeadlineMillis":30000,"maximumDeadlineMillis":120000,"gracefulShutdownMillis":30000,"forcedShutdownMillis":10000}, + "signingReconciliation": {"enabled":true,"maxRecords":256,"maxCalls":64,"passTimeoutMillis":30000,"providerCallTimeoutMillis":10000,"idleIntervalMillis":2000}, "publicListener": { "enabled":true, "address":"127.0.0.1", diff --git a/pki-server/src/main/java/zeroecho/pki/server/PkiHttpsServer.java b/pki-server/src/main/java/zeroecho/pki/server/PkiHttpsServer.java index 66e1182..debbe9b 100644 --- a/pki-server/src/main/java/zeroecho/pki/server/PkiHttpsServer.java +++ b/pki-server/src/main/java/zeroecho/pki/server/PkiHttpsServer.java @@ -70,6 +70,7 @@ public final class PkiHttpsServer implements AutoCloseable { private final Optional publicAuthenticator; private final Optional publicTransport; private final Optional acmeTransport; + private final Optional signingReconciliation; private final AtomicReference state; private PkiHttpsServer(PkiServerConfiguration configuration, ServerRealmContext realm, @@ -77,6 +78,7 @@ public final class PkiHttpsServer implements AutoCloseable { Optional publicAuthenticator, Optional publicTransport, Optional acmeTransport, + Optional signingReconciliation, AtomicReference state) { this.configuration = configuration; this.realm = realm; @@ -85,6 +87,7 @@ public final class PkiHttpsServer implements AutoCloseable { this.publicAuthenticator = publicAuthenticator; this.publicTransport = publicTransport; this.acmeTransport = acmeTransport; + this.signingReconciliation = signingReconciliation; this.state = state; } @@ -116,6 +119,7 @@ public final class PkiHttpsServer implements AutoCloseable { PkiServerAuthenticator publicAuthenticator = null; PublicRepositoryTransport publicTransport = null; AcmeTransport acmeTransport = null; + SigningReconciliationWorker signingReconciliation = null; try { SSLContext tls = PkiHttpsTransport.resolveTls(exact, loader); realm = ServerRealmContext.open(exact.realm(), runtimeDependencies, clock, random); @@ -160,14 +164,22 @@ public final class PkiHttpsServer implements AutoCloseable { acmeTransport = AcmeTransport.start(acmeConfiguration, sharedRealm, clock, random, loader, () -> state.get() == State.READY && sharedRealm.state() == ServerRealmContext.State.OPEN); } - state.set(State.READY); + if (exact.signingReconciliation().enabled()) { + signingReconciliation = new SigningReconciliationWorker(realm, exact.signingReconciliation(), + exact.execution().forcedShutdown(), clock); + } realm.auditTransport("SERVER_READY", "system", Map.of("state", "READY")); - return new PkiHttpsServer(exact, realm, authenticator, transport, + PkiHttpsServer server = new PkiHttpsServer(exact, realm, authenticator, transport, Optional.ofNullable(publicAuthenticator), Optional.ofNullable(publicTransport), - Optional.ofNullable(acmeTransport), state); + Optional.ofNullable(acmeTransport), Optional.ofNullable(signingReconciliation), state); + if (signingReconciliation != null) { + signingReconciliation.start(); + } + state.set(State.READY); + return server; } catch (RuntimeException | Error primary) { - closePartial(acmeTransport, publicTransport, publicAuthenticator, transport, realm, authenticator, - state, primary); + closePartial(signingReconciliation, acmeTransport, publicTransport, publicAuthenticator, transport, realm, + authenticator, state, primary); throw primary; } } @@ -221,8 +233,10 @@ public final class PkiHttpsServer implements AutoCloseable { */ @Override public void close() throws Exception { - if (!state.compareAndSet(State.READY, State.QUIESCING) + State observed = state.get(); + if (observed != State.QUIESCING && !state.compareAndSet(State.READY, State.QUIESCING) && !state.compareAndSet(State.STARTING, State.QUIESCING)) return; + signingReconciliation.ifPresent(SigningReconciliationWorker::stopScheduling); Throwable primary = null; try { realm.auditTransport("SERVER_SHUTDOWN", "system", Map.of("state", "QUIESCING")); @@ -253,6 +267,15 @@ public final class PkiHttpsServer implements AutoCloseable { } catch (Throwable failure) { primary = suppress(primary, failure); } + try { + if (signingReconciliation.isPresent()) { + signingReconciliation.orElseThrow().awaitStopped(); + } + } catch (Throwable failure) { + if (primary != null && primary != failure) failure.addSuppressed(primary); + rethrow(failure); + return; + } if (publicAuthenticator.isPresent()) primary = close(publicAuthenticator.orElseThrow(), primary); primary = close(authenticator, primary); primary = close(realm, primary); @@ -260,9 +283,21 @@ public final class PkiHttpsServer implements AutoCloseable { rethrow(primary); } - private static void closePartial(AcmeTransport acmeTransport, PublicRepositoryTransport publicTransport, + private static void closePartial(SigningReconciliationWorker signingReconciliation, AcmeTransport acmeTransport, + PublicRepositoryTransport publicTransport, PkiServerAuthenticator publicAuthenticator, PkiHttpsTransport transport, ServerRealmContext realm, PkiServerAuthenticator authenticator, AtomicReference state, Throwable primary) { + if (signingReconciliation != null) { + try { + signingReconciliation.close(); + } catch (Throwable failure) { + if (primary != failure) primary.addSuppressed(failure); + close(acmeTransport, primary); + close(publicTransport, primary); + close(transport, primary); + return; + } + } primary = close(acmeTransport, primary); primary = close(publicTransport, primary); primary = close(publicAuthenticator, primary); diff --git a/pki-server/src/main/java/zeroecho/pki/server/PkiServerConfiguration.java b/pki-server/src/main/java/zeroecho/pki/server/PkiServerConfiguration.java index 8843730..3d6d527 100644 --- a/pki-server/src/main/java/zeroecho/pki/server/PkiServerConfiguration.java +++ b/pki-server/src/main/java/zeroecho/pki/server/PkiServerConfiguration.java @@ -61,14 +61,16 @@ import zeroecho.pki.spi.ProviderConfig; * @param runtime process-local capability references * @param publicListener optional separately bounded public repository listener * @param acmeListener optional separately bounded ACME protocol listener + * @param signingReconciliation bounded recovered-signing owner policy */ @SuppressWarnings("PMD") public record PkiServerConfiguration(int version, String serverName, ServerRealmConfiguration realm, Listener listener, Authentication authentication, Execution execution, RuntimeCapabilities runtime, - Optional publicListener, Optional acmeListener) { + Optional publicListener, Optional acmeListener, + SigningReconciliation signingReconciliation) { /** Current server configuration schema. */ - public static final int CURRENT_VERSION = 5; + public static final int CURRENT_VERSION = 6; /** Validates all security-sensitive fields before resource allocation. */ public PkiServerConfiguration { @@ -81,6 +83,7 @@ public record PkiServerConfiguration(int version, String serverName, ServerRealm Objects.requireNonNull(runtime, "runtime"); publicListener = Objects.requireNonNull(publicListener, "publicListener"); acmeListener = Objects.requireNonNull(acmeListener, "acmeListener"); + Objects.requireNonNull(signingReconciliation, "signingReconciliation"); if (!listener.clientCertificateRequired()) { throw new IllegalArgumentException("Administrative HTTPS requires client certificates"); } @@ -110,14 +113,53 @@ public record PkiServerConfiguration(int version, String serverName, ServerRealm Listener listener, Authentication authentication, Execution execution, RuntimeCapabilities runtime, Optional publicListener) { this(version, serverName, realm, listener, authentication, execution, runtime, publicListener, - Optional.empty()); + Optional.empty(), SigningReconciliation.defaults()); + } + + /** Creates a configuration with default signing reconciliation. */ + public PkiServerConfiguration(int version, String serverName, ServerRealmConfiguration realm, + Listener listener, Authentication authentication, Execution execution, RuntimeCapabilities runtime, + Optional publicListener, Optional acmeListener) { + this(version, serverName, realm, listener, authentication, execution, runtime, publicListener, acmeListener, + SigningReconciliation.defaults()); } /** Creates a configuration with the public repository listener disabled. */ public PkiServerConfiguration(int version, String serverName, ServerRealmConfiguration realm, Listener listener, Authentication authentication, Execution execution, RuntimeCapabilities runtime) { this(version, serverName, realm, listener, authentication, execution, runtime, Optional.empty(), - Optional.empty()); + Optional.empty(), SigningReconciliation.defaults()); + } + + /** + * Bounded recovered-signing reconciliation policy. + * + * @param enabled whether the realm owns a reconciliation scheduler + * @param maximumRecords maximum records examined per pass + * @param maximumProviderCalls maximum provider calls per pass + * @param passTimeout positive complete-pass timeout + * @param providerCallTimeout positive per-call timeout no greater than the pass timeout + * @param idleInterval positive delay after each completed pass + */ + public record SigningReconciliation(boolean enabled, int maximumRecords, int maximumProviderCalls, + Duration passTimeout, Duration providerCallTimeout, Duration idleInterval) { + /** Validates all reconciliation bounds. */ + public SigningReconciliation { + bounded(maximumRecords, 1, 4096, "signing reconciliation record count"); + bounded(maximumProviderCalls, 1, maximumRecords, "signing reconciliation provider call count"); + positive(passTimeout, Duration.ofMinutes(5), "signing reconciliation pass timeout"); + positive(providerCallTimeout, Duration.ofMinutes(5), "signing reconciliation provider call timeout"); + positive(idleInterval, Duration.ofMinutes(5), "signing reconciliation idle interval"); + if (providerCallTimeout.compareTo(passTimeout) > 0) { + throw new IllegalArgumentException("Signing reconciliation provider timeout exceeds pass timeout"); + } + } + + /** @return version-6 production defaults */ + public static SigningReconciliation defaults() { + return new SigningReconciliation(true, 256, 64, Duration.ofSeconds(30), Duration.ofSeconds(10), + Duration.ofSeconds(2)); + } } /** diff --git a/pki-server/src/main/java/zeroecho/pki/server/PkiServerConfigurationCodec.java b/pki-server/src/main/java/zeroecho/pki/server/PkiServerConfigurationCodec.java index a5577a2..98dc2af 100644 --- a/pki-server/src/main/java/zeroecho/pki/server/PkiServerConfigurationCodec.java +++ b/pki-server/src/main/java/zeroecho/pki/server/PkiServerConfigurationCodec.java @@ -86,12 +86,19 @@ public final class PkiServerConfigurationCodec { public static PkiServerConfiguration decode(byte[] document) { Fields root = Fields.of(StrictJson.parse(document, MAXIMUM_CONFIGURATION_BYTES)); root.allowed(Set.of("version", "serverName", "realm", "listener", "authentication", "execution", - "runtime", "publicListener", "acmeListener")); - PkiServerConfiguration configuration = new PkiServerConfiguration(root.integer("version"), + "runtime", "publicListener", "acmeListener", "signingReconciliation")); + int encodedVersion = root.integer("version"); + if (encodedVersion != 5 && encodedVersion != PkiServerConfiguration.CURRENT_VERSION) { + throw new IllegalArgumentException("Unsupported server schema version"); + } + PkiServerConfiguration.SigningReconciliation reconciliation = encodedVersion == 5 + ? PkiServerConfiguration.SigningReconciliation.defaults() + : signingReconciliation(root.object("signingReconciliation")); + PkiServerConfiguration configuration = new PkiServerConfiguration(PkiServerConfiguration.CURRENT_VERSION, root.text("serverName"), realm(root.object("realm")), listener(root.object("listener")), authentication(root.object("authentication")), execution(root.object("execution")), runtime(root.object("runtime")), publicListener(root.object("publicListener")), - acmeListener(root.object("acmeListener"))); + acmeListener(root.object("acmeListener")), reconciliation); root.complete(); return configuration; } @@ -273,6 +280,18 @@ public final class PkiServerConfigurationCodec { return result; } + private static PkiServerConfiguration.SigningReconciliation signingReconciliation(Fields value) { + value.exact("enabled", "maxRecords", "maxCalls", "passTimeoutMillis", "providerCallTimeoutMillis", + "idleIntervalMillis"); + PkiServerConfiguration.SigningReconciliation result = new PkiServerConfiguration.SigningReconciliation( + value.bool("enabled"), value.integer("maxRecords"), value.integer("maxCalls"), + Duration.ofMillis(value.longValue("passTimeoutMillis")), + Duration.ofMillis(value.longValue("providerCallTimeoutMillis")), + Duration.ofMillis(value.longValue("idleIntervalMillis"))); + value.complete(); + return result; + } + private static Optional publicListener(Fields value) { value.allowed(Set.of("enabled", "address", "port", "tlsProvider", "allowPlaintextLoopback", "authentication", "maximumHeaderBytes", "maximumBodyBytes", "execution", "maximumStreamDurationMillis", diff --git a/pki-server/src/main/java/zeroecho/pki/server/SigningReconciliationWorker.java b/pki-server/src/main/java/zeroecho/pki/server/SigningReconciliationWorker.java new file mode 100644 index 0000000..160bdf4 --- /dev/null +++ b/pki-server/src/main/java/zeroecho/pki/server/SigningReconciliationWorker.java @@ -0,0 +1,130 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + ******************************************************************************/ +package zeroecho.pki.server; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import zeroecho.pki.application.SigningReconciliationRequest; +import zeroecho.pki.application.SigningReconciliationResult; +import zeroecho.pki.application.PkiSession; + +/** Single-threaded, self-rescheduling signing recovery owner for one realm. */ +@SuppressWarnings({ "PMD.DoNotUseThreads", "PMD.AvoidUsingVolatile", "PMD.CommentDefaultAccessModifier" }) +final class SigningReconciliationWorker implements AutoCloseable { + private final PkiSession session; + private final AuditTransport audit; + private final PkiServerConfiguration.SigningReconciliation configuration; + private final Duration forcedShutdown; + private final Clock clock; + private final ScheduledExecutorService executor; + private final AtomicBoolean cancelled = new AtomicBoolean(); + private Optional cursor = Optional.empty(); + private volatile Future scheduled; + + /* default */ SigningReconciliationWorker(ServerRealmContext realm, + PkiServerConfiguration.SigningReconciliation configuration, Duration forcedShutdown, Clock clock) { + this(Objects.requireNonNull(realm, "realm").session(), realm::auditTransport, configuration, forcedShutdown, + clock); + } + + /* default */ SigningReconciliationWorker(PkiSession session, AuditTransport audit, + PkiServerConfiguration.SigningReconciliation configuration, Duration forcedShutdown, Clock clock) { + this.session = Objects.requireNonNull(session, "session"); + this.audit = Objects.requireNonNull(audit, "audit"); + this.configuration = Objects.requireNonNull(configuration, "configuration"); + this.forcedShutdown = Objects.requireNonNull(forcedShutdown, "forcedShutdown"); + this.clock = Objects.requireNonNull(clock, "clock"); + this.executor = Executors.newSingleThreadScheduledExecutor(runnable -> { + Thread thread = new Thread(runnable, "zeroecho-signing-reconciliation"); + thread.setDaemon(true); + return thread; + }); + } + + /* default */ void start() { + if (cancelled.get() || scheduled != null) { + throw new IllegalStateException("Signing reconciliation worker cannot start"); + } + scheduled = executor.submit(this::runPass); + } + + @SuppressWarnings("PMD.AvoidCatchingGenericException") + private void runPass() { + if (cancelled.get()) { + return; + } + Instant deadline = clock.instant().plus(configuration.passTimeout()); + Map auditDetails; + try { + SigningReconciliationResult result = session.reconcileSigning(new SigningReconciliationRequest( + cursor, configuration.maximumRecords(), configuration.maximumProviderCalls(), deadline, + configuration.providerCallTimeout(), cancelled::get)); + cursor = result.nextCursor(); + auditDetails = Map.of( + "code", result.retryable() == 0 ? "SIGNING_RECONCILIATION_OK" : "SIGNING_RECONCILIATION_RETRY", + "examined", Integer.toString(result.examined()), + "progressed", Integer.toString(result.progressed()), + "unresolved", Integer.toString(result.unresolved()), + "retryable", Integer.toString(result.retryable()), + "endReached", Boolean.toString(result.endReached())); + } catch (RuntimeException failure) { + auditDetails = Map.of("code", "SIGNING_RECONCILIATION_PASS_FAILED"); + } + try { + audit.record("SIGNING_RECONCILIATION_PASS", "system", auditDetails); + } catch (RuntimeException ignored) { + // The pass remains bounded and future passes remain authoritative. + } finally { + if (!cancelled.get()) { + scheduled = executor.schedule(this::runPass, configuration.idleInterval().toNanos(), + TimeUnit.NANOSECONDS); + } + } + } + + @Override + public void close() throws InterruptedException { + stopScheduling(); + awaitStopped(); + } + + /* default */ void stopScheduling() { + if (cancelled.compareAndSet(false, true)) { + Future current = scheduled; + if (current != null) { + current.cancel(true); + } + executor.shutdownNow(); + } + } + + /* default */ void awaitStopped() throws InterruptedException { + if (!cancelled.get()) { + throw new IllegalStateException("Signing reconciliation worker is still accepting work"); + } + if (!executor.awaitTermination(forcedShutdown.toNanos(), TimeUnit.NANOSECONDS)) { + throw new IllegalStateException( + "Signing reconciliation shutdown timed out: code=SIGNING_RECONCILIATION_SHUTDOWN_TIMEOUT"); + } + } + + /** Safe aggregate reconciliation audit boundary. */ + @FunctionalInterface + /* default */ + interface AuditTransport { + /** Records one aggregate event without record or provider details. */ + void record(String eventType, String principal, Map details); + } +} diff --git a/pki-server/src/test/java/zeroecho/pki/server/AcmeEndToEndTest.java b/pki-server/src/test/java/zeroecho/pki/server/AcmeEndToEndTest.java index 0a4ca52..46774fe 100644 --- a/pki-server/src/test/java/zeroecho/pki/server/AcmeEndToEndTest.java +++ b/pki-server/src/test/java/zeroecho/pki/server/AcmeEndToEndTest.java @@ -755,7 +755,10 @@ class AcmeEndToEndTest { + "\"port\":" + configuration.listener().port() + ",\"tlsProvider\":" + tlsJson + ",\"clientCertificateRequired\":true,\"maximumHeaderBytes\":16384," + "\"maximumBodyBytes\":1048576},\"authentication\":" + authenticationJson - + ",\"execution\":" + executionJson() + ",\"publicListener\":{\"enabled\":true," + + ",\"execution\":" + executionJson() + + ",\"signingReconciliation\":{\"enabled\":true,\"maxRecords\":256,\"maxCalls\":64," + + "\"passTimeoutMillis\":30000,\"providerCallTimeoutMillis\":10000," + + "\"idleIntervalMillis\":2000},\"publicListener\":{\"enabled\":true," + "\"address\":\"127.0.0.1\",\"port\":" + publicListener.port() + ",\"tlsProvider\":" + tlsJson + ",\"allowPlaintextLoopback\":false," + "\"authentication\":" + authenticationJson diff --git a/pki-server/src/test/java/zeroecho/pki/server/PkiHttpsServerTest.java b/pki-server/src/test/java/zeroecho/pki/server/PkiHttpsServerTest.java index e706820..578c2b2 100644 --- a/pki-server/src/test/java/zeroecho/pki/server/PkiHttpsServerTest.java +++ b/pki-server/src/test/java/zeroecho/pki/server/PkiHttpsServerTest.java @@ -121,8 +121,24 @@ class PkiHttpsServerTest { valid.realm(), valid.listener(), valid.authentication(), valid.execution(), valid.runtime())); String production = java.nio.file.Files.readString(java.nio.file.Path.of( "..", "docs", "pki-server-production-example.json")); - assertTrue(PkiServerConfigurationCodec.decode(production.getBytes( - java.nio.charset.StandardCharsets.UTF_8)).publicListener().isEmpty()); + PkiServerConfiguration decoded = PkiServerConfigurationCodec.decode(production.getBytes( + java.nio.charset.StandardCharsets.UTF_8)); + assertTrue(decoded.publicListener().isEmpty()); + assertEquals(PkiServerConfiguration.SigningReconciliation.defaults(), decoded.signingReconciliation()); + String versionFive = production.replace("\"version\": 6", "\"version\": 5") + .replace(" \"signingReconciliation\": {\"enabled\": true, \"maxRecords\": 256, " + + "\"maxCalls\": 64, \"passTimeoutMillis\": 30000, " + + "\"providerCallTimeoutMillis\": 10000, \"idleIntervalMillis\": 2000},\n", ""); + PkiServerConfiguration migrated = PkiServerConfigurationCodec.decode( + versionFive.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + assertEquals(PkiServerConfiguration.CURRENT_VERSION, migrated.version()); + assertEquals(PkiServerConfiguration.SigningReconciliation.defaults(), migrated.signingReconciliation()); + assertThrows(IllegalArgumentException.class, + () -> new PkiServerConfiguration.SigningReconciliation(true, 4, 5, Duration.ofSeconds(30), + Duration.ofSeconds(10), Duration.ofSeconds(2))); + assertThrows(IllegalArgumentException.class, + () -> new PkiServerConfiguration.SigningReconciliation(true, 4, 4, Duration.ofSeconds(5), + Duration.ofSeconds(10), Duration.ofSeconds(2))); assertThrows(IllegalArgumentException.class, () -> PkiServerConfigurationCodec.decode(production .replace("\"publicListener\": {\"enabled\": false},\n", "") .getBytes(java.nio.charset.StandardCharsets.UTF_8))); diff --git a/pki-server/src/test/java/zeroecho/pki/server/SigningReconciliationWorkerTest.java b/pki-server/src/test/java/zeroecho/pki/server/SigningReconciliationWorkerTest.java new file mode 100644 index 0000000..9c86df0 --- /dev/null +++ b/pki-server/src/test/java/zeroecho/pki/server/SigningReconciliationWorkerTest.java @@ -0,0 +1,127 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + ******************************************************************************/ +package zeroecho.pki.server; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Proxy; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import zeroecho.pki.application.PkiSession; +import zeroecho.pki.application.SigningReconciliationResult; + +final class SigningReconciliationWorkerTest { + + @Test + void timeoutStopsSchedulingAndNeverClosesSessionUnderViolatingCall() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + AtomicBoolean release = new AtomicBoolean(); + AtomicInteger calls = new AtomicInteger(); + AtomicInteger closes = new AtomicInteger(); + PkiSession session = session((method, arguments) -> { + if ("reconcileSigning".equals(method)) { + calls.incrementAndGet(); + entered.countDown(); + while (!release.get()) { + try { + Thread.sleep(1L); + } catch (InterruptedException ignored) { + // Deliberately violates cooperative cancellation for this lifecycle test. + } + } + return emptyResult(); + } + if ("close".equals(method)) { + closes.incrementAndGet(); + } + return null; + }); + SigningReconciliationWorker worker = new SigningReconciliationWorker(session, (type, principal, details) -> { + // Aggregate audit is irrelevant to the blocking lifecycle assertion. + }, configuration(Duration.ofSeconds(1)), Duration.ofMillis(20), fixedClock()); + worker.start(); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + + worker.stopScheduling(); + IllegalStateException timeout = assertThrows(IllegalStateException.class, worker::awaitStopped); + assertTrue(timeout.getMessage().contains("SIGNING_RECONCILIATION_SHUTDOWN_TIMEOUT")); + assertEquals(1, calls.get()); + assertEquals(0, closes.get()); + + release.set(true); + worker.awaitStopped(); + assertEquals(1, calls.get()); + assertEquals(0, closes.get()); + } + + @Test + void subMillisecondIdleIntervalSelfSchedulesWithoutOverlap() throws Exception { + AtomicInteger active = new AtomicInteger(); + AtomicInteger maximumActive = new AtomicInteger(); + CountDownLatch twoPasses = new CountDownLatch(2); + PkiSession session = session((method, arguments) -> { + if ("reconcileSigning".equals(method)) { + int current = active.incrementAndGet(); + maximumActive.accumulateAndGet(current, Math::max); + twoPasses.countDown(); + active.decrementAndGet(); + return emptyResult(); + } + return null; + }); + SigningReconciliationWorker worker = new SigningReconciliationWorker(session, (type, principal, details) -> { + // Safe aggregate event accepted. + }, configuration(Duration.ofNanos(1)), Duration.ofSeconds(1), fixedClock()); + worker.start(); + assertTrue(twoPasses.await(5, TimeUnit.SECONDS)); + worker.close(); + assertEquals(1, maximumActive.get()); + } + + private static PkiServerConfiguration.SigningReconciliation configuration(Duration idle) { + return new PkiServerConfiguration.SigningReconciliation(true, 1, 1, Duration.ofSeconds(1), + Duration.ofMillis(100), idle); + } + + private static SigningReconciliationResult emptyResult() { + return new SigningReconciliationResult(Optional.empty(), 0, 0, 0, 0, true); + } + + private static Clock fixedClock() { + return Clock.fixed(Instant.parse("2026-08-05T00:00:00Z"), ZoneOffset.UTC); + } + + private static PkiSession session(SessionInvocation invocation) { + return (PkiSession) Proxy.newProxyInstance(PkiSession.class.getClassLoader(), new Class[] { PkiSession.class }, + (proxy, method, arguments) -> { + if (method.getDeclaringClass() == Object.class) { + return switch (method.getName()) { + case "toString" -> "test-pki-session"; + case "hashCode" -> System.identityHashCode(proxy); + case "equals" -> proxy == arguments[0]; + default -> null; + }; + } + return invocation.invoke(method.getName(), arguments); + }); + } + + @FunctionalInterface + private interface SessionInvocation { + Object invoke(String method, Object[] arguments) throws Throwable; + } +} diff --git a/pki/src/main/java/zeroecho/pki/application/DefaultPkiSession.java b/pki/src/main/java/zeroecho/pki/application/DefaultPkiSession.java index 33e60f0..74049fc 100644 --- a/pki/src/main/java/zeroecho/pki/application/DefaultPkiSession.java +++ b/pki/src/main/java/zeroecho/pki/application/DefaultPkiSession.java @@ -273,6 +273,14 @@ final class DefaultPkiSession implements PkiSession { return operations; } + @Override + public SigningReconciliationResult reconcileSigning(SigningReconciliationRequest request) { + requireOpen(); + Objects.requireNonNull(request, "request"); + return signingBus.map(value -> value.reconcile(request)) + .orElseGet(() -> new SigningReconciliationResult(Optional.empty(), 0, 0, 0, 0, true)); + } + @Override public void close() throws Exception { if (!closed.compareAndSet(false, true)) { diff --git a/pki/src/main/java/zeroecho/pki/application/PkiSession.java b/pki/src/main/java/zeroecho/pki/application/PkiSession.java index 7a5b1ce..cfcd883 100644 --- a/pki/src/main/java/zeroecho/pki/application/PkiSession.java +++ b/pki/src/main/java/zeroecho/pki/application/PkiSession.java @@ -156,6 +156,19 @@ public interface PkiSession extends AutoCloseable { /** @return read-only authoritative repository facade owned by this session */ PkiRepository repository(); + /** + * Performs one synchronous, bounded signing-recovery pass. + * + *

The session creates no scheduler. Callers own invocation cadence and must + * pass the opaque exclusive cursor returned by the preceding pass. A session + * without signing returns an empty, end-reached result.

+ * + * @param request validated pass bounds and cancellation + * @return aggregate safe pass result + * @throws IllegalStateException if the session is closed + */ + SigningReconciliationResult reconcileSigning(SigningReconciliationRequest request); + /** * Closes services and backend resources in reverse construction order. * Repeated calls are harmless; primary and suppressed failures are preserved. diff --git a/pki/src/main/java/zeroecho/pki/application/SigningReconciliationRequest.java b/pki/src/main/java/zeroecho/pki/application/SigningReconciliationRequest.java new file mode 100644 index 0000000..87f3305 --- /dev/null +++ b/pki/src/main/java/zeroecho/pki/application/SigningReconciliationRequest.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + ******************************************************************************/ +package zeroecho.pki.application; + +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; + +import zeroecho.core.io.CancellationSignal; + +/** + * Immutable bounds for one transport-neutral signing reconciliation pass. + * + * @param cursor opaque exclusive cursor returned by an earlier pass + * @param maximumRecords maximum records examined, from 1 through 4096 + * @param maximumProviderCalls maximum provider calls, from 1 through + * {@code maximumRecords} + * @param deadline absolute pass deadline + * @param providerCallTimeout positive timeout for each provider call; the + * reconciler clamps each call to the absolute pass deadline + * @param cancellation cooperative pass cancellation + */ +public record SigningReconciliationRequest(Optional cursor, int maximumRecords, + int maximumProviderCalls, Instant deadline, Duration providerCallTimeout, + CancellationSignal cancellation) { + + /** Validates and snapshots one pass request. */ + public SigningReconciliationRequest { + cursor = Objects.requireNonNull(cursor, "cursor"); + if (maximumRecords < 1 || maximumRecords > 4096) { + throw new IllegalArgumentException("maximumRecords must be between 1 and 4096"); + } + if (maximumProviderCalls < 1 || maximumProviderCalls > maximumRecords) { + throw new IllegalArgumentException("maximumProviderCalls must be between 1 and maximumRecords"); + } + Objects.requireNonNull(deadline, "deadline"); + Objects.requireNonNull(providerCallTimeout, "providerCallTimeout"); + Objects.requireNonNull(cancellation, "cancellation"); + if (providerCallTimeout.isZero() || providerCallTimeout.isNegative()) { + throw new IllegalArgumentException("providerCallTimeout must be positive"); + } + cursor.ifPresent(value -> { + if (value.isBlank() || value.length() > 4096) { + throw new IllegalArgumentException("cursor is invalid"); + } + for (int index = 0; index < value.length(); index++) { + char current = value.charAt(index); + if (current < 0x21 || current > 0x7e || current == '/' || current == '\\') { + throw new IllegalArgumentException("cursor is invalid"); + } + } + }); + } +} diff --git a/pki/src/main/java/zeroecho/pki/application/SigningReconciliationResult.java b/pki/src/main/java/zeroecho/pki/application/SigningReconciliationResult.java new file mode 100644 index 0000000..6b060f2 --- /dev/null +++ b/pki/src/main/java/zeroecho/pki/application/SigningReconciliationResult.java @@ -0,0 +1,31 @@ +/******************************************************************************* + * Copyright (C) 2026, Leo Galambos + * All rights reserved. + ******************************************************************************/ +package zeroecho.pki.application; + +import java.util.Objects; +import java.util.Optional; + +/** + * Aggregate safe outcome of one bounded signing reconciliation pass. + * + * @param nextCursor opaque exclusive cursor for the next pass + * @param examined records examined + * @param progressed records whose durable state progressed + * @param unresolved records still requiring reconciliation + * @param retryable records deferred after an isolated retryable failure + * @param endReached whether the pass reached the end of the ordered record set + */ +public record SigningReconciliationResult(Optional nextCursor, int examined, + int progressed, int unresolved, int retryable, boolean endReached) { + + /** Validates aggregate counts. */ + public SigningReconciliationResult { + nextCursor = Objects.requireNonNull(nextCursor, "nextCursor"); + if (examined < 0 || progressed < 0 || unresolved < 0 || retryable < 0 + || progressed > examined || unresolved > examined || retryable > examined) { + throw new IllegalArgumentException("Signing reconciliation counts are inconsistent"); + } + } +} diff --git a/pki/src/main/java/zeroecho/pki/impl/core/async/PkiSigningBus.java b/pki/src/main/java/zeroecho/pki/impl/core/async/PkiSigningBus.java index 7394e2d..ea898cc 100644 --- a/pki/src/main/java/zeroecho/pki/impl/core/async/PkiSigningBus.java +++ b/pki/src/main/java/zeroecho/pki/impl/core/async/PkiSigningBus.java @@ -37,6 +37,9 @@ import java.nio.file.Path; import java.security.SecureRandom; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -62,6 +65,8 @@ import zeroecho.pki.api.audit.Principal; import zeroecho.pki.api.orch.OrchestrationDurabilityPolicy; import zeroecho.pki.api.orch.SigningSubmissionId; import zeroecho.pki.api.orch.WorkflowStateRecord; +import zeroecho.pki.application.SigningReconciliationRequest; +import zeroecho.pki.application.SigningReconciliationResult; import zeroecho.pki.impl.framework.x509.X509AlgorithmResolver; import zeroecho.pki.impl.framework.x509.X509AuthoritySnapshot; import zeroecho.pki.impl.framework.x509.X509ExecutionPlan; @@ -94,7 +99,9 @@ import zeroecho.pki.util.async.impl.DurableAsyncBus; */ // The collaborators counted here form one durable signing lifecycle; splitting // them would obscure the coordinator/reservation boundary that protects it. -@SuppressWarnings({ "PMD.CouplingBetweenObjects", "PMD.CyclomaticComplexity", "PMD.PreserveStackTrace" }) +@SuppressWarnings({ "PMD.CouplingBetweenObjects", "PMD.CyclomaticComplexity", "PMD.PreserveStackTrace", + "PMD.AvoidCatchingGenericException", "PMD.AvoidInstantiatingObjectsInLoops", + "PMD.AvoidLiteralsInIfCondition", "PMD.CollapsibleIfStatements" }) public final class PkiSigningBus implements AutoCloseable { private static final Logger LOG = Logger.getLogger(PkiSigningBus.class.getName()); @@ -109,6 +116,7 @@ public final class PkiSigningBus implements AutoCloseable { "Signing content cleanup failed: code=SIGNING_CONTENT_CLEANUP_FAILED"; private static final Duration CLAIM_LEASE = Duration.ofSeconds(30); private static final long INITIAL_FENCE = 0L; + private static final int MAXIMUM_SIGNATURE_BYTES = 1_048_576; /** * System property controlling the maximum number of characters appended after @@ -454,7 +462,8 @@ public final class PkiSigningBus implements AutoCloseable { EncodedObject persistedRequest = continuation.withSignerOpId(baseOpId).encode(); SignWorkflowStore.Record intent = new SignWorkflowStore.Record(baseOpId, namespace, fingerprint, owner, parsed.createdAt(), deadline, persistedRequest, SignWorkflowStore.State.INTENT, 0L, - 0L, Optional.empty(), Optional.of("INTENT"), Optional.empty(), Optional.empty()); + 0L, Optional.empty(), Optional.of("INTENT"), Optional.empty(), Optional.empty(), 0, + Optional.empty(), Optional.empty()); SignWorkflowStore.CreateResult created = store.createSignIntent(intent); if (created == SignWorkflowStore.CreateResult.CONFLICT) { releaseAttachedOrConflictingContent(content); @@ -545,6 +554,248 @@ public final class PkiSigningBus implements AutoCloseable { store.purgeExpiredSignRecords(); } + /** + * Performs one bounded synchronous recovery pass. The pass scans at most the + * requested record bound, retains only that page, and invokes the provider no + * more than the requested call budget. + * + * @param request immutable pass bounds + * @return aggregate safe pass outcome + */ + public SigningReconciliationResult reconcile(SigningReconciliationRequest request) { + Objects.requireNonNull(request, "request"); + ReconciliationPage page = reconciliationPage(request); + int progressed = 0; + int unresolved = page.failures(); + int retryable = page.failures(); + int providerCalls = 0; + for (SignWorkflowStore.Record candidate : page.records()) { + if (request.cancellation().isCancelled()) { + break; + } + ReconciliationStep step; + try { + if (!store.signingNow().isBefore(request.deadline())) { + break; + } + step = reconcileRecord(candidate, request, request.maximumProviderCalls() - providerCalls); + } catch (RuntimeException localFailure) { + step = isolateCandidateFailure(candidate, new ProviderCallCounter(), localFailure); + } + providerCalls += step.providerCalls(); + if (step.progressed()) { + progressed++; + } + if (step.unresolved()) { + unresolved++; + } + if (step.retryable()) { + retryable++; + } + } + return new SigningReconciliationResult(page.nextCursor(), page.examined(), progressed, unresolved, retryable, + page.endReached()); + } + + private ReconciliationPage reconciliationPage(SigningReconciliationRequest request) { + zeroecho.core.io.CancellationSignal scanControl = () -> request.cancellation().isCancelled() + || !store.signingNow().isBefore(request.deadline()); + SignWorkflowStore.Page first; + try { + first = store.pageSignRecords(request.cursor(), request.maximumRecords(), scanControl); + } catch (IllegalArgumentException invalidAdvisoryCursor) { + first = store.pageSignRecords(Optional.empty(), request.maximumRecords(), scanControl); + } + List records = new ArrayList<>(first.records()); + if (!first.endReached() || request.cursor().isEmpty() || first.examined() >= request.maximumRecords()) { + return new ReconciliationPage(records, first.nextCursor(), first.examined(), first.failures(), + first.endReached()); + } + int remaining = request.maximumRecords() - first.examined(); + SignWorkflowStore.Page wrapped = store.pageSignRecords(Optional.empty(), remaining, scanControl); + Set seen = new HashSet<>(); + for (SignWorkflowStore.Record record : records) { + seen.add(record.submissionId()); + } + for (SignWorkflowStore.Record record : wrapped.records()) { + if (seen.add(record.submissionId()) && records.size() < request.maximumRecords()) { + records.add(record); + } + } + return new ReconciliationPage(records, wrapped.nextCursor(), first.examined() + wrapped.examined(), + first.failures() + wrapped.failures(), true); + } + + private record ReconciliationPage(List records, Optional nextCursor, + int examined, int failures, boolean endReached) { + private ReconciliationPage { + records = List.copyOf(records); + } + } + + @SuppressWarnings("PMD.AvoidCatchingGenericException") + private ReconciliationStep reconcileRecord(SignWorkflowStore.Record candidate, + SigningReconciliationRequest request, int remainingCalls) { + ProviderCallCounter calls = new ProviderCallCounter(); + SignWorkflowStore.Record before = candidate; + try { + Optional currentOptional = store.getSignRecord(candidate.submissionId()); + if (currentOptional.isEmpty()) { + return ReconciliationStep.RESOLVED; + } + before = currentOptional.orElseThrow(); + if (isTerminalSignState(before.state())) { + return retireReconciled(before); + } + Instant now = store.signingNow(); + if (before.nextEligibleAt().filter(eligible -> eligible.isAfter(now)).isPresent()) { + return ReconciliationStep.UNRESOLVED; + } + if (remainingCalls <= 0) { + return ReconciliationStep.UNRESOLVED; + } + SignatureWorkflow.CallControl control = callControl(request, now); + if (before.state() == SignWorkflowStore.State.INTENT) { + if (before.fence() == 0L) { + endpoint.execute(before.submissionId(), control, false, calls); + } else { + endpoint.reconcileProviderStatus(before.submissionId(), control, calls, true); + } + } else { + endpoint.reconcileProviderStatus(before.submissionId(), control, calls, false); + } + SignWorkflowStore.Record afterStatus = store.getSignRecord(before.submissionId()).orElse(before); + if (!afterStatus.deadline().isAfter(store.signingNow()) + && afterStatus.state() == SignWorkflowStore.State.DISPATCHED) { + afterStatus = store.transitionSign(afterStatus.submissionId(), afterStatus.revision(), + afterStatus.fence(), SignWorkflowStore.State.CANCELLING, Optional.of("CANCEL_REQUESTED"), + Optional.empty(), Optional.empty()).orElse(afterStatus); + } + if (afterStatus.state() == SignWorkflowStore.State.CANCELLING) { + if (afterStatus.state() == SignWorkflowStore.State.CANCELLING + && afterStatus.detailCode().filter("CANCEL_REQUESTED"::equals).isPresent() + && remainingCalls - calls.attempts() > 0 && callActive(request)) { + submitReconciliationCancellation(afterStatus, callControl(request, store.signingNow()), calls); + } + } + SignWorkflowStore.Record after = store.getSignRecord(before.submissionId()).orElse(before); + if (isTerminalSignState(after.state())) { + ReconciliationStep retired = retireReconciled(after); + return new ReconciliationStep(calls.attempts(), true, retired.unresolved(), retired.retryable()); + } + clearRetry(after); + SignWorkflowStore.Record finalState = store.getSignRecord(before.submissionId()).orElse(after); + return new ReconciliationStep(calls.attempts(), finalState.revision() != before.revision(), true, false); + } catch (RuntimeException failure) { + return isolateCandidateFailure(before, calls, failure); + } + } + + @SuppressWarnings("PMD.AvoidCatchingGenericException") + private ReconciliationStep isolateCandidateFailure(SignWorkflowStore.Record fallback, + ProviderCallCounter calls, RuntimeException failure) { + rethrowStoreWideFailure(failure); + SignWorkflowStore.Record failed = fallback; + try { + failed = store.getSignRecord(fallback.submissionId()).orElse(fallback); + } catch (RuntimeException readFailure) { + rethrowStoreWideFailure(readFailure); + } + if (isTerminalSignState(failed.state())) { + return new ReconciliationStep(calls.attempts(), false, true, true); + } + SignWorkflowStore.ReconciliationFailureClass classification = switch (failed.state()) { + case INTENT -> SignWorkflowStore.ReconciliationFailureClass.SUBMISSION_UNCERTAIN; + case CANCELLING -> failed.detailCode().filter("CANCEL_REQUESTED"::equals).isPresent() + ? SignWorkflowStore.ReconciliationFailureClass.CANCELLATION_UNCERTAIN + : SignWorkflowStore.ReconciliationFailureClass.STATUS_UNAVAILABLE; + case DISPATCHED -> SignWorkflowStore.ReconciliationFailureClass.STATUS_UNAVAILABLE; + default -> SignWorkflowStore.ReconciliationFailureClass.LOCAL_FAILURE; + }; + Optional deferred = Optional.empty(); + try { + deferred = store.deferSignReconciliation(failed.submissionId(), failed.revision(), failed.fence(), + classification); + } catch (RuntimeException writeFailure) { + rethrowStoreWideFailure(writeFailure); + } + return new ReconciliationStep(calls.attempts(), deferred.isPresent(), true, true); + } + + private static void rethrowStoreWideFailure(RuntimeException failure) { + Throwable current = failure; + while (current != null) { + String message = current.getMessage(); + if (message != null && message.contains("code=STORE_DURABILITY_UNCONFIRMED")) { + throw failure; + } + current = current.getCause(); + } + } + + @SuppressWarnings("PMD.AvoidCatchingGenericException") + private ReconciliationStep retireReconciled(SignWorkflowStore.Record record) { + try { + SignWorkflowStore.Record retired = confirmRetirement(record.submissionId(), record); + store.deleteWorkflowState(record.submissionId()); + return new ReconciliationStep(0, retired.state() == SignWorkflowStore.State.RETIRED, false, false); + } catch (RuntimeException failure) { + rethrowStoreWideFailure(failure); + return new ReconciliationStep(0, false, true, true); + } + } + + private void clearRetry(SignWorkflowStore.Record record) { + if (record.failureCount() > 0) { + store.clearSignReconciliation(record.submissionId(), record.revision(), record.fence()); + } + } + + private void submitReconciliationCancellation(SignWorkflowStore.Record record, + SignatureWorkflow.CallControl control, ProviderCallCounter calls) { + Optional reservation = externalActions.tryReserve(record.submissionId(), + ExternalAction.CANCEL); + if (reservation.isEmpty()) { + return; + } + try (ExternalActionCoordinator.Reservation ignored = reservation.orElseThrow()) { + calls.beforeProviderCall(); + signer.cancel(record.submissionId(), record.fence(), "reconciliation", control); + control.requireActive(store.signingNow()); + } + markCancellationSubmitted(record.submissionId(), record); + } + + private SignatureWorkflow.CallControl callControl(SigningReconciliationRequest request, Instant now) { + Instant callDeadline = now.plus(request.providerCallTimeout()); + if (callDeadline.isAfter(request.deadline())) { + callDeadline = request.deadline(); + } + return new SignatureWorkflow.CallControl(callDeadline, request.cancellation()); + } + + private boolean callActive(SigningReconciliationRequest request) { + return !request.cancellation().isCancelled() && store.signingNow().isBefore(request.deadline()); + } + + private record ReconciliationStep(int providerCalls, boolean progressed, boolean unresolved, boolean retryable) { + private static final ReconciliationStep RESOLVED = new ReconciliationStep(0, false, false, false); + private static final ReconciliationStep UNRESOLVED = new ReconciliationStep(0, false, true, false); + } + + /** Exact attempts made during one candidate reconciliation. */ + private static final class ProviderCallCounter { + private int attempts; + + private void beforeProviderCall() { + attempts++; + } + + private int attempts() { + return attempts; + } + } + /** * Deletes workflow continuation state once finished. */ @@ -679,7 +930,9 @@ public final class PkiSigningBus implements AutoCloseable { return cancelling; } try (ExternalActionCoordinator.Reservation ignored = reserved.get()) { - signer.cancel(operationId, cancelling.fence(), reason); + signer.cancel(operationId, cancelling.fence(), reason, + new SignatureWorkflow.CallControl(store.signingNow().plus(CLAIM_LEASE), + zeroecho.core.io.CancellationSignal.NONE)); } catch (RuntimeException ex) { endpoint.reconcileProviderStatus(operationId); SignWorkflowStore.Record current = store.getSignRecord(operationId).orElse(cancelling); @@ -804,7 +1057,7 @@ public final class PkiSigningBus implements AutoCloseable { record.request().encoding(), Optional.of(record.request()))); Duration ttl = Duration.between(record.createdAt(), record.deadline()); bus.submit(record.submissionId(), TYPE_SIGN, record.owner(), ENDPOINT_SIGNER, record.createdAt(), ttl); - } catch (RuntimeException ex) { // NOPMD - projection is advisory + } catch (RuntimeException ex) { // projection is advisory if (LOG.isLoggable(Level.FINE)) { LOG.log(Level.FINE, "Advisory projection failed: code={0}, exception={1}", new Object[] { "PROJECTION_REFRESH_FAILED", ex.getClass().getName() }); @@ -1099,21 +1352,36 @@ public final class PkiSigningBus implements AutoCloseable { * missing */ public void execute(PkiId opId) { + execute(opId, new SignatureWorkflow.CallControl(store.signingNow().plus(CLAIM_LEASE), + zeroecho.core.io.CancellationSignal.NONE), true, null); + } + + private boolean execute(PkiId opId, SignatureWorkflow.CallControl control, boolean pollAfterAcceptance, + ProviderCallCounter calls) { Objects.requireNonNull(opId, "opId"); + Objects.requireNonNull(control, "control"); Optional prepared = prepareSubmission(opId); if (prepared.isEmpty()) { - return; + return false; } SubmissionCall call = prepared.get(); PkiId returned; try (ExternalActionCoordinator.Reservation ignored = call.reservation()) { authority.authorize(call.plan(), signer, AlgorithmExecutionCapability.Direction.SIGN); - returned = call.plan().executor().submitSign(call.request()); + control.requireActive(store.signingNow()); + if (calls != null) { + calls.beforeProviderCall(); + } + returned = call.plan().executor().submitSign(call.request(), control); + control.requireActive(store.signingNow()); } catch (RuntimeException ambiguousFailure) { // NOPMD - provider acceptance is unknown - return; + throw ambiguousFailure; } recordSubmissionAcceptance(call, returned); - reconcileProviderStatus(opId); + if (pollAfterAcceptance) { + reconcileProviderStatus(opId); + } + return true; } // On success the reservation ownership moves into SubmissionCall and spans @@ -1233,7 +1501,8 @@ public final class PkiSigningBus implements AutoCloseable { * *

* Once a downstream signer operation exists, the returned status is derived - * from {@link SignatureWorkflow#status(PkiId)} using this mapping: + * from {@link SignatureWorkflow#status(PkiId, SignatureWorkflow.CallControl)} + * using this mapping: *

*
    *
  • missing downstream status - {@link AsyncState#RUNNING} with detail code @@ -1325,29 +1594,46 @@ public final class PkiSigningBus implements AutoCloseable { // Provider implementations are an untrusted boundary and may throw any runtime // failure. - @SuppressWarnings("PMD.AvoidCatchingGenericException") private void reconcileProviderStatus(PkiId operationId) { + reconcileProviderStatus(operationId, new SignatureWorkflow.CallControl(store.signingNow().plus(CLAIM_LEASE), + zeroecho.core.io.CancellationSignal.NONE)); + } + + private boolean reconcileProviderStatus(PkiId operationId, SignatureWorkflow.CallControl control) { + return reconcileProviderStatus(operationId, control, null, false); + } + + private boolean reconcileProviderStatus(PkiId operationId, SignatureWorkflow.CallControl control, + ProviderCallCounter calls, boolean permitUncertainIntent) { removeAdvisory(operationId); - Optional prepared = prepareStatusCall(operationId); + Optional prepared = prepareStatusCall(operationId, permitUncertainIntent); if (prepared.isEmpty()) { - return; + return false; } StatusCall call = prepared.get(); try (ExternalActionCoordinator.Reservation ignored = call.reservation()) { try { - SignatureWorkflow.OperationStatus providerStatus = signer.status(operationId); + control.requireActive(store.signingNow()); + if (calls != null) { + calls.beforeProviderCall(); + } + SignatureWorkflow.OperationStatus providerStatus = signer.status(operationId, control); + control.requireActive(store.signingNow()); applyProviderStatus(call, providerStatus); } catch (RuntimeException providerFailure) { throw new PkiException("Provider status failed: code=PROVIDER_STATUS_FAILED"); } } + return true; } - private Optional prepareStatusCall(PkiId operationId) { + private Optional prepareStatusCall(PkiId operationId, boolean permitUncertainIntent) { try (OperationCoordinator.Lease ignored = coordinator.acquire(operationId)) { Optional beforeCall = store.getSignRecord(operationId); if (beforeCall.isEmpty() || beforeCall.get().state() != SignWorkflowStore.State.DISPATCHED - && beforeCall.get().state() != SignWorkflowStore.State.CANCELLING) { + && beforeCall.get().state() != SignWorkflowStore.State.CANCELLING + && (!permitUncertainIntent || beforeCall.get().state() != SignWorkflowStore.State.INTENT + || beforeCall.get().fence() <= 0L)) { return Optional.empty(); } Optional reserved = externalActions.tryReserve(operationId, @@ -1357,25 +1643,39 @@ public final class PkiSigningBus implements AutoCloseable { } private void applyProviderStatus(StatusCall call, SignatureWorkflow.OperationStatus providerStatus) { - if (!providerStatus.isTerminal()) { - return; - } try (OperationCoordinator.Lease ignored = coordinator.acquire(call.record().submissionId())) { Optional currentOptional = store.getSignRecord(call.record().submissionId()); if (currentOptional.isEmpty() || currentOptional.get().state() != SignWorkflowStore.State.DISPATCHED && currentOptional.get().state() != SignWorkflowStore.State.CANCELLING + && currentOptional.get().state() != SignWorkflowStore.State.INTENT || currentOptional.get().fence() != call.record().fence()) { return; } SignWorkflowStore.Record current = currentOptional.get(); + if (current.state() == SignWorkflowStore.State.INTENT) { + Optional attached = store.transitionSign(current.submissionId(), + current.revision(), current.fence(), SignWorkflowStore.State.DISPATCHED, + Optional.of("DISPATCHED"), Optional.empty(), Optional.empty()); + if (attached.isEmpty()) { + return; + } + current = attached.orElseThrow(); + } + if (!providerStatus.isTerminal()) { + return; + } Optional result = providerStatus.result() .flatMap(SignatureWorkflow.OperationResult::signature); SignWorkflowStore.State target = mapProviderState(providerStatus.state()); - Optional detail = sanitizeProviderDetail(providerStatus.detailCode()); - if (target == SignWorkflowStore.State.SUCCEEDED && result.isEmpty()) { + Optional detail = safeProviderOutcomeCode(providerStatus.state()); + boolean validSignature = providerStatus.result().isPresent() + && providerStatus.result().orElseThrow().verified().isEmpty() + && result.filter(SignatureWorkflowEndpoint::isValidSignatureResult).isPresent(); + if (target == SignWorkflowStore.State.SUCCEEDED && !validSignature) { target = SignWorkflowStore.State.FAILED; - detail = Optional.of("PROVIDER_RESULT_MISSING"); + detail = Optional.of("PROVIDER_RESULT_INVALID"); + result = Optional.empty(); } if (target == SignWorkflowStore.State.SUCCEEDED && !providerStatus.updatedAt().isBefore(current.deadline())) { @@ -1455,15 +1755,26 @@ public final class PkiSigningBus implements AutoCloseable { }; } - private static Optional sanitizeProviderDetail(Optional detail) { - if (detail.isEmpty()) { - return Optional.empty(); + private static boolean isValidSignatureResult(EncodedObject result) { + if (result.encoding() != Encoding.BINARY) { + return false; } - String value = detail.orElseThrow(); - if (value.length() > 64 || !value.matches("[A-Z0-9_]+")) { - return Optional.of("PROVIDER_DETAIL_INVALID"); + byte[] bytes = result.bytes(); + try { + return bytes.length > 0 && bytes.length <= MAXIMUM_SIGNATURE_BYTES; + } finally { + java.util.Arrays.fill(bytes, (byte) 0); } - return detail; + } + + private static Optional safeProviderOutcomeCode(SignatureWorkflow.State state) { + return Optional.of(switch (state) { + case SUCCEEDED -> "SIGNED"; + case CANCELLED -> "CANCELLED"; + case EXPIRED -> "EXPIRED"; + case FAILED -> "PROVIDER_FAILED"; + case PENDING, RUNNING, WAITING_APPROVAL -> "RUNNING"; + }); } private static AsyncStatus mapStoreStatus(SignWorkflowStore.Record record) { diff --git a/pki/src/main/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflow.java b/pki/src/main/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflow.java index 3548243..31d6c68 100644 --- a/pki/src/main/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflow.java +++ b/pki/src/main/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflow.java @@ -122,7 +122,7 @@ import zeroecho.sdk.ZeroEchoSession; * The implementation is intentionally synchronous from its internal execution * perspective, but it still conforms to the {@link SignatureWorkflow} contract * by returning an operation identifier and exposing the terminal outcome - * through {@link #status(PkiId)}. Each submitted operation is executed + * through {@link #status(PkiId, CallControl)}. Each submitted operation is executed * immediately in the caller thread. Signing requests and terminal outcomes are * retained durably in the configured operation root for the configured * operation horizon. @@ -130,15 +130,15 @@ import zeroecho.sdk.ZeroEchoSession; * *

    Supported operations

    *
      - *
    • {@link #submitSign(SignRequest)} resolves a private key from the configured + *
    • {@link #submitSign(SignRequest, CallControl)} resolves a private key from the configured * {@link KeyringStore}, validates the requested algorithm * compatibility, produces a signature over the supplied payload, and stores the * result as a terminal successful or failed operation status.
      • - *
    • {@link #submitVerify(VerifyRequest)} verifies a signature either against + *
    • {@link #submitVerify(VerifyRequest, CallControl)} verifies a signature either against * a key resolved from {@link VerifyRequest#publicKeyRef()} or against a caller- * supplied encoded public key from * {@link VerifyRequest#publicKeyEncoded()}.
      • - *
    • {@link #status(PkiId)} returns the retained operation status, or a stable + *
    • {@link #status(PkiId, CallControl)} returns the retained operation status, or a stable * failed status for unknown operation identifiers.
      • *
    • {@link #register(NotificationSink)} installs an in-memory notification * sink that is called whenever an operation status changes.
      • @@ -165,7 +165,7 @@ import zeroecho.sdk.ZeroEchoSession; * exceptions once an operation has been accepted for processing. Instead, the * provider always returns an operation identifier and records a terminal * {@link State#FAILED} status with a stable non-secret detail code retrievable - * through {@link #status(PkiId)}. + * through {@link #status(PkiId, CallControl)}. *

      * *

      @@ -240,7 +240,9 @@ import zeroecho.sdk.ZeroEchoSession; *

      */ // The provider deliberately centralizes operation lifecycle and cleanup in one implementation. -@SuppressWarnings({ "PMD.CouplingBetweenObjects", "PMD.CyclomaticComplexity", "PMD.TooManyMethods" }) +@SuppressWarnings({ "PMD.CouplingBetweenObjects", "PMD.CyclomaticComplexity", "PMD.TooManyMethods", + "PMD.NcssCount", "PMD.CloseResource", "PMD.ExceptionAsFlowControl", + "PMD.AvoidCatchingGenericException" }) public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, PublicKeyInfoSource { private static final Logger LOG = Logger.getLogger(ZeroEchoLibSignatureWorkflow.class.getName()); @@ -265,7 +267,8 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu private static final OperationStatus UNKNOWN_OPERATION_STATUS = new OperationStatus(State.FAILED, Instant.EPOCH, Optional.of(DC_UNKNOWN_OPERATION), Optional.empty()); - private static final int OPERATION_RECORD_VERSION = 4; + private static final int OPERATION_RECORD_VERSION = 5; + private static final int PREVIOUS_OPERATION_RECORD_VERSION = 4; private static final long MIN_FENCING_TOKEN = 1L; private final String id; @@ -281,6 +284,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu private final ConcurrentMap statuses; private final ConcurrentMap fingerprints; private final ConcurrentMap fences; + private final ConcurrentMap cancellationReasons; private final ConcurrentMap requests; private final ConcurrentMap sinks; private final ConcurrentMap operationLocks; @@ -370,12 +374,15 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu this.statuses = new ConcurrentHashMap<>(); this.fingerprints = new ConcurrentHashMap<>(); this.fences = new ConcurrentHashMap<>(); + this.cancellationReasons = new ConcurrentHashMap<>(); this.requests = new ConcurrentHashMap<>(); this.sinks = new ConcurrentHashMap<>(); this.operationLocks = new ConcurrentHashMap<>(); this.domainLock = new ReentrantLock(); this.keyringLifecycleLock = new ReentrantLock(); this.timeWatermarkLock = new ReentrantLock(); + FileChannel acquiredChannel = null; + FileLock acquiredLock = null; try { Files.createDirectories(this.operationRoot); restrictPermissions(this.operationRoot, true); @@ -388,11 +395,10 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu } else { Files.writeString(owner, id, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); } - this.ownershipChannel = FileChannel.open(this.operationRoot.resolve(".lock"), StandardOpenOption.CREATE, + acquiredChannel = FileChannel.open(this.operationRoot.resolve(".lock"), StandardOpenOption.CREATE, StandardOpenOption.WRITE); - this.ownershipLock = this.ownershipChannel.tryLock(); - if (this.ownershipLock == null) { - this.ownershipChannel.close(); + acquiredLock = acquiredChannel.tryLock(); + if (acquiredLock == null) { throw new IllegalStateException("Signing operation root is already in use"); } this.timeWatermark = new AtomicLong(loadTimeWatermark()); @@ -401,8 +407,31 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu Files.exists(domain) ? Files.readString(domain, StandardCharsets.US_ASCII).trim() : null); loadOperationRecords(); purgeExpiredOperations(); + this.ownershipChannel = acquiredChannel; + this.ownershipLock = acquiredLock; } catch (IOException ex) { + releaseFailedOwnership(acquiredLock, acquiredChannel, ex); throw new IllegalStateException("Cannot initialize signing operation root", ex); + } catch (RuntimeException | Error failure) { + releaseFailedOwnership(acquiredLock, acquiredChannel, failure); + throw failure; + } + } + + private static void releaseFailedOwnership(FileLock lock, FileChannel channel, Throwable failure) { + if (lock != null) { + try { + lock.release(); + } catch (IOException cleanup) { + failure.addSuppressed(cleanup); + } + } + if (channel != null) { + try { + channel.close(); + } catch (IOException cleanup) { + failure.addSuppressed(cleanup); + } } } @@ -485,10 +514,11 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu * @throws IllegalArgumentException if {@code request} is {@code null} */ @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { if (request == null) { throw new IllegalArgumentException("request must not be null"); } + Objects.requireNonNull(control, "control").requireActive(now()); PkiId opId = request.submissionId(); if (!request.namespace().endsWith("." + id)) { @@ -498,20 +528,43 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu SigningSubmissionId parsed = SigningSubmissionId.parse(opId); purgeExpiredOperations(); parsed.validate(request.namespace(), now(), operationHorizon, Duration.ZERO); - if (!beginSign(request)) { + if (!beginSign(request, control)) { return opId; } - SignExecutionResult execution = executeAcceptedSign(request); + SignExecutionResult execution = null; try { + execution = executeAcceptedSign(request, control); + Instant commitTime = now(); + control.requireActive(commitTime); completeSign(request, execution.status); return opId; + } catch (RuntimeException failure) { + Instant stoppedAt = now(); + if (callStopped(request, control, stoppedAt)) { + completeSign(request, stoppedSignStatus(request, control, stoppedAt)); + } + throw failure; } finally { - clearOwned("sign-result-copy", execution.signatureBytes); + if (execution != null) { + clearOwned("sign-result-copy", execution.signatureBytes); + } } } - private SignExecutionResult executeAcceptedSign(SignRequest request) { + private static boolean callStopped(SignRequest request, CallControl control, Instant observedAt) { + return request.cancellation().isCancelled() || control.cancellation().isCancelled() + || !observedAt.isBefore(control.deadline()) || deadlineReached(request.deadline(), observedAt); + } + + private static OperationStatus stoppedSignStatus(SignRequest request, CallControl control, Instant observedAt) { + boolean cancelled = request.cancellation().isCancelled() || control.cancellation().isCancelled(); + return cancelled + ? new OperationStatus(State.CANCELLED, observedAt, Optional.of(DC_CANCELLED), Optional.empty()) + : expiredStatus(observedAt); + } + + private SignExecutionResult executeAcceptedSign(SignRequest request, CallControl control) { byte[] signatureBytes = null; try { KeyRefParts parts = parseKeyRefOrThrow(request.keyRef(), true); @@ -525,13 +578,16 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu throw new InvalidRequestException(DC_INVALID_ALGORITHM_ID); } + control.requireActive(now()); if (deadlineReached(request.deadline(), now())) { return SignExecutionResult.terminal(expiredStatus()); } KeyringSignatureExecutor executor = requireSignatureExecutor(); signatureBytes = executor.sign(parts.privateAlias, resolvedAlgorithm.get(), request.content(), - request.cancellation()); + () -> request.cancellation().isCancelled() || control.cancellation().isCancelled() + || !now().isBefore(control.deadline())); + control.requireActive(now()); Encoding outEnc = request.preferredSignatureEncoding().orElse(Encoding.BINARY); EncodedObject signature = encodeSignatureOrThrow(outEnc, signatureBytes); @@ -555,7 +611,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu } return SignExecutionResult.withSignature(failedStatus(detailCode), signatureBytes); - } catch (InvalidRequestException inv) { // NOPMD + } catch (InvalidRequestException inv) { return SignExecutionResult.withSignature(failedStatus(inv.detailCode), signatureBytes); } catch (KeyringSignatureExecutor.Cancellation cancelled) { @@ -581,7 +637,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu return new KeyringSignatureExecutor(requireKeyringOrThrow(), session); } - private boolean beginSign(SignRequest request) { + private boolean beginSign(SignRequest request, CallControl control) { PkiId operationId = request.submissionId(); SignLockEntry entry = acquireOperationLock(operationId); OperationStatus event = null; @@ -604,6 +660,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu } return false; } + control.requireActive(now()); fingerprints.put(operationId, request.semanticFingerprint()); fences.put(operationId, request.fencingToken()); requests.put(operationId, request); @@ -684,10 +741,11 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu * @throws IllegalArgumentException if {@code request} is {@code null} */ @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { if (request == null) { throw new IllegalArgumentException("request must not be null"); } + Objects.requireNonNull(control, "control").requireActive(now()); PkiId opId = newOperationId(); putStatus(opId, new OperationStatus(State.RUNNING, now(), Optional.of(DC_SUBMITTED), Optional.empty())); @@ -705,11 +763,14 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu putStatus(opId, expiredStatus()); return opId; } + control.requireActive(now()); request.cancellation().throwIfCancelled(); boolean ok = verifyStreaming(request.algorithmId(), pub, request.content(), signatureBytes, - request.cancellation()); + () -> request.cancellation().isCancelled() || control.cancellation().isCancelled() + || !now().isBefore(control.deadline())); Instant completedAt = now(); + control.requireActive(completedAt); if (deadlineReached(request.deadline(), completedAt)) { putStatus(opId, expiredStatus(completedAt)); return opId; @@ -719,7 +780,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu putStatus(opId, new OperationStatus(State.SUCCEEDED, completedAt, Optional.of(dc), Optional.of(result))); return opId; - } catch (InvalidRequestException inv) { // NOPMD + } catch (InvalidRequestException inv) { putStatus(opId, new OperationStatus(State.FAILED, now(), Optional.of(inv.detailCode), Optional.empty())); return opId; @@ -760,10 +821,11 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu * @throws IllegalArgumentException if {@code operationId} is {@code null} */ @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { if (operationId == null) { throw new IllegalArgumentException("operationId must not be null"); } + Objects.requireNonNull(control, "control").requireActive(now()); purgeExpiredOperations(); OperationStatus st = this.statuses.get(operationId); if (st == null) { @@ -772,13 +834,18 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu String namespace = boundNamespace.get(); if (namespace != null && namespace.equals(parsed.namespace()) && !now().isBefore(parsed.createdAt().plus(operationHorizon))) { - return new OperationStatus(State.EXPIRED, now(), Optional.of("EXPIRED"), Optional.empty()); + OperationStatus expired = new OperationStatus(State.EXPIRED, now(), Optional.of("EXPIRED"), + Optional.empty()); + control.requireActive(now()); + return expired; } } catch (IllegalArgumentException ignored) { // unknown verify operation identifiers remain synthetic failed // handled by the stable unknown status below } + control.requireActive(now()); return UNKNOWN_OPERATION_STATUS; } + control.requireActive(now()); return st; } @@ -788,7 +855,8 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu *

      * Cancellation is serialized with completion for the same signing identifier, * validates the fencing token, and durably records {@link State#CANCELLED}. It - * succeeds only while the retained operation is non-terminal. + * A replay of an accepted cancellation with the same operation identifier and + * fence succeeds without another state change, regardless of its safe reason. *

      * * @param operationId workflow operation identifier; must not be {@code null} @@ -800,7 +868,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu * {@code reason} is {@code null} or blank */ @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { if (operationId == null) { throw new IllegalArgumentException("operationId must not be null"); } @@ -810,18 +878,27 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu if (fencingToken < MIN_FENCING_TOKEN) { throw new IllegalArgumentException("fencingToken must be positive"); } + Objects.requireNonNull(control, "control").requireActive(now()); SignLockEntry entry = acquireOperationLock(operationId); OperationStatus cancelled = null; try { OperationStatus st = this.statuses.get(operationId); - if (st == null || st.isTerminal()) { + if (st == null) { + control.requireActive(now()); return false; } long currentFence = this.fences.getOrDefault(operationId, 0L); + if (st.isTerminal()) { + control.requireActive(now()); + return st.state() == State.CANCELLED && fencingToken == currentFence; + } if (fencingToken < currentFence) { + control.requireActive(now()); return false; } + control.requireActive(now()); this.fences.put(operationId, fencingToken); + this.cancellationReasons.putIfAbsent(operationId, cancellationReasonFingerprint(reason)); cancelled = new OperationStatus(State.CANCELLED, now(), Optional.of(DC_CANCELLED), Optional.empty()); statuses.put(operationId, cancelled); persistOperationRecord(operationId, cancelled); @@ -831,6 +908,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu notifySinks(operationId, cancelled); } } + control.requireActive(now()); return true; } @@ -889,6 +967,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu this.statuses.clear(); this.fingerprints.clear(); this.fences.clear(); + this.cancellationReasons.clear(); this.requests.clear(); this.sinks.clear(); if (keyring != null) { @@ -1200,7 +1279,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu for (NotificationSink sink : this.sinks.values()) { try { sink.onStatusChanged(id, st); - } catch (Throwable ignore) { // NOPMD + } catch (Throwable ignore) { // sink must not break provider logSafeFailure("CALLBACK", "NOTIFICATION_FAILED", ignore); } @@ -1304,6 +1383,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu requests.remove(operationId); fingerprints.remove(operationId); fences.remove(operationId); + cancellationReasons.remove(operationId); statuses.remove(operationId); Files.deleteIfExists(operationRecordPath(operationId)); } @@ -1362,13 +1442,16 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu try (java.util.stream.Stream paths = Files.list(records)) { for (Path path : paths.filter(Files::isRegularFile).toList()) { try (DataInputStream input = new DataInputStream(Files.newInputStream(path))) { - if (input.readInt() != OPERATION_RECORD_VERSION) { + int version = input.readInt(); + if (version != OPERATION_RECORD_VERSION && version != PREVIOUS_OPERATION_RECORD_VERSION) { throw new IllegalStateException("Unsupported signing operation record version"); } PkiId operationId = new PkiId(input.readUTF()); SigningSubmissionId parsedId = SigningSubmissionId.parse(operationId); String fingerprint = input.readUTF(); long fence = input.readLong(); + Optional cancellationReason = version == OPERATION_RECORD_VERSION && input.readBoolean() + ? Optional.of(input.readUTF()) : Optional.empty(); SignRequest request = readSignRequest(input); if (!operationId.equals(request.submissionId()) || !constantTimeEquals(fingerprint, request.semanticFingerprint()) @@ -1400,6 +1483,7 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu } this.fingerprints.put(operationId, fingerprint); this.fences.put(operationId, fence); + cancellationReason.ifPresent(value -> this.cancellationReasons.put(operationId, value)); this.requests.put(operationId, request); this.statuses.put(operationId, loaded); if (repaired) { @@ -1419,6 +1503,10 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu output.writeUTF(operationId.value()); output.writeUTF(this.fingerprints.get(operationId)); output.writeLong(this.fences.getOrDefault(operationId, 0L)); + output.writeBoolean(this.cancellationReasons.containsKey(operationId)); + if (this.cancellationReasons.containsKey(operationId)) { + output.writeUTF(this.cancellationReasons.get(operationId)); + } writeSignRequest(output, requests.get(operationId)); output.writeInt(stateCode(status.state())); output.writeLong(status.updatedAt().getEpochSecond()); @@ -1470,6 +1558,20 @@ public final class ZeroEchoLibSignatureWorkflow implements SignatureWorkflow, Pu } } + private static String cancellationReasonFingerprint(String reason) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256") + .digest(reason.getBytes(StandardCharsets.UTF_8)); + try { + return HexFormat.of().formatHex(digest); + } finally { + Arrays.fill(digest, (byte) 0); + } + } catch (NoSuchAlgorithmException ex) { + throw new IllegalStateException("SHA-256 is unavailable", ex); + } + } + private void writeSignRequest(DataOutputStream output, SignRequest request) throws IOException { if (request == null) { throw new IllegalStateException("Missing signing request snapshot"); diff --git a/pki/src/main/java/zeroecho/pki/impl/framework/x509/bc/WorkflowProofOfPossessionVerifier.java b/pki/src/main/java/zeroecho/pki/impl/framework/x509/bc/WorkflowProofOfPossessionVerifier.java index 9316ec4..005c08f 100644 --- a/pki/src/main/java/zeroecho/pki/impl/framework/x509/bc/WorkflowProofOfPossessionVerifier.java +++ b/pki/src/main/java/zeroecho/pki/impl/framework/x509/bc/WorkflowProofOfPossessionVerifier.java @@ -356,8 +356,11 @@ public final class WorkflowProofOfPossessionVerifier implements ProofOfPossessio X509ExecutionPlan executionPlan) { authority.authorize(executionPlan, workflow, zeroecho.core.spi.AlgorithmExecutionCapability.Direction.VERIFY); - PkiId verifyOperationId = executionPlan.executor().submitVerify(verifyRequest); - SignatureWorkflow.OperationStatus status = executionPlan.executor().status(verifyOperationId); + Instant callDeadline = verifyRequest.deadline().orElseGet(() -> Instant.now().plusSeconds(30)); + SignatureWorkflow.CallControl control = new SignatureWorkflow.CallControl(callDeadline, + verifyRequest.cancellation()); + PkiId verifyOperationId = executionPlan.executor().submitVerify(verifyRequest, control); + SignatureWorkflow.OperationStatus status = executionPlan.executor().status(verifyOperationId, control); if (status == null) { return failed("Verifier returned no status"); } diff --git a/pki/src/main/java/zeroecho/pki/impl/fs/FilesystemPkiStore.java b/pki/src/main/java/zeroecho/pki/impl/fs/FilesystemPkiStore.java index 32863ab..bae6515 100644 --- a/pki/src/main/java/zeroecho/pki/impl/fs/FilesystemPkiStore.java +++ b/pki/src/main/java/zeroecho/pki/impl/fs/FilesystemPkiStore.java @@ -40,6 +40,7 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InterruptedIOException; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; @@ -82,6 +83,7 @@ import org.bouncycastle.operator.jcajce.JcaContentVerifierProviderBuilder; import zeroecho.pki.api.EncodedObject; +import zeroecho.pki.api.Encoding; import zeroecho.pki.api.PkiException; import zeroecho.pki.api.PkiId; import zeroecho.pki.api.ca.CaRecord; @@ -183,7 +185,7 @@ import zeroecho.pki.spi.store.RevocationHistory; "PMD.ExcessivePublicCount", "PMD.UseTryWithResources", "PMD.ExceptionAsFlowControl", "PMD.PreserveStackTrace", "PMD.NcssCount", "PMD.AvoidDuplicateLiterals", "PMD.ControlStatementBraces", "PMD.CollapsibleIfStatements", "PMD.AvoidDeeplyNestedIfStmts", - "PMD.AvoidLiteralsInIfCondition" }) + "PMD.AvoidLiteralsInIfCondition", "PMD.AvoidCatchingGenericException" }) public final class FilesystemPkiStore implements PkiStore, Closeable { private static final Logger LOG = Logger.getLogger(FilesystemPkiStore.class.getName()); @@ -199,6 +201,7 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { private static final int SIGN_OWNER_VALUE_VERSION = 1; private static final int STATUS_OWNER_VALUE_VERSION = 1; private static final int METADATA_TRANSFER_BUFFER_BYTES = 16 * 1024; + private static final int MAX_METADATA_VALUE_BYTES = FsCodec.MAX_COMPONENT_BYTES + 64 * 1024; private static final ThreadLocal STATUS_COMMIT_FAULT = new ThreadLocal<>(); private static final ThreadLocal PUBLICATION_COMMIT_FAULT = new ThreadLocal<>(); private static final String SIGN_FINGERPRINT_PREFIX = "signfp:v1:"; @@ -1636,6 +1639,137 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { return listStoredSigns().stream().map(StoredSign::record).toList(); } + @Override + public SignWorkflowStore.Page pageSignRecords(Optional exclusiveCursor, int maximumRecords, + CancellationSignal cancellation) { + requireStoreUsable(); + Objects.requireNonNull(exclusiveCursor, "exclusiveCursor"); + Objects.requireNonNull(cancellation, "cancellation"); + if (maximumRecords < 1 || maximumRecords > 4096) { + throw new IllegalArgumentException("maximumRecords must be between 1 and 4096"); + } + Optional lower = exclusiveCursor; + MetadataSnapshot.KeyRange range = new MetadataSnapshot.KeyRange(SIGN_RECORD_NAMESPACE, lower, + Optional.empty()); + List records = new ArrayList<>(maximumRecords); + int examined = 0; + int failures = 0; + Optional nextCursor = exclusiveCursor; + if (cancellation.isCancelled()) { + return new SignWorkflowStore.Page(records, nextCursor, examined, failures, false); + } + try (MetadataSnapshot snapshot = metadataStore.snapshot(); + MetadataCursor cursor = snapshot.scan(range, cancellation)) { + while (examined < maximumRecords) { + Optional next; + try { + next = cursor.next(cancellation); + } catch (InterruptedIOException cancelled) { + if (cancellation.isCancelled()) { + return new SignWorkflowStore.Page(records, nextCursor, examined, failures, false); + } + throw cancelled; + } + if (next.isEmpty()) { + return new SignWorkflowStore.Page(records, nextCursor, examined, failures, true); + } + try (MetadataSnapshot.Record candidate = next.orElseThrow()) { + String candidateCursor = candidate.key().key(); + if (exclusiveCursor.filter(candidateCursor::equals).isPresent()) { + continue; + } + if (cancellation.isCancelled()) { + return new SignWorkflowStore.Page(records, nextCursor, examined, failures, false); + } + examined++; + nextCursor = Optional.of(candidateCursor); + try { + records.add(decodeStoredSign(snapshot, candidate).record()); + } catch (RuntimeException | IOException malformedCandidate) { + failures++; + } + } + } + boolean endReached; + try { + endReached = cursor.next(cancellation).isEmpty(); + } catch (InterruptedIOException cancelled) { + if (!cancellation.isCancelled()) { + throw cancelled; + } + endReached = false; + } + return new SignWorkflowStore.Page(records, nextCursor, examined, failures, endReached); + } catch (InterruptedIOException exception) { + if (cancellation.isCancelled()) { + return new SignWorkflowStore.Page(records, nextCursor, examined, failures, false); + } + throw new IllegalStateException("Failed to page authoritative signing records", exception); + } catch (IOException exception) { + throw new IllegalStateException("Failed to page authoritative signing records", exception); + } + } + + @Override + public Optional deferSignReconciliation(PkiId submissionId, long expectedRevision, + long fence, SignWorkflowStore.ReconciliationFailureClass failureClass) { + requireStoreUsable(); + Objects.requireNonNull(failureClass, "failureClass"); + SignLockEntry lock = acquireSignLock(submissionId); + try { + Optional optional = readStoredSign(submissionId); + if (optional.isEmpty()) { + return Optional.empty(); + } + StoredSign stored = optional.orElseThrow(); + SignWorkflowStore.Record current = stored.record(); + if (current.revision() != expectedRevision || current.fence() != fence + || isRetirableSignState(current.state())) { + return Optional.empty(); + } + int failureCount = current.failureCount() == Integer.MAX_VALUE + ? Integer.MAX_VALUE : current.failureCount() + 1; + long delaySeconds = failureCount >= 5 ? 30L : 2L << failureCount - 1; + Instant now = signingNow(); + Instant proposedEligibility = now.plusSeconds(Math.min(delaySeconds, 30L)); + Instant horizonEnd = current.createdAt().plus(options.signingOperationHorizon()); + Instant nextEligible = proposedEligibility.isAfter(horizonEnd) ? horizonEnd : proposedEligibility; + SignWorkflowStore.Record deferred = copySignRecord(current, current.state(), current.revision() + 1L, + current.fence(), current.leaseUntil(), current.detailCode(), current.result(), + current.providerUpdatedAt(), failureCount, Optional.of(nextEligible), Optional.of(failureClass)); + return replaceSignMetadata(stored, deferred, false) ? Optional.of(deferred) : Optional.empty(); + } finally { + releaseSignLock(submissionId, lock); + } + } + + @Override + public Optional clearSignReconciliation(PkiId submissionId, long expectedRevision, + long fence) { + requireStoreUsable(); + SignLockEntry lock = acquireSignLock(submissionId); + try { + Optional optional = readStoredSign(submissionId); + if (optional.isEmpty()) { + return Optional.empty(); + } + StoredSign stored = optional.orElseThrow(); + SignWorkflowStore.Record current = stored.record(); + if (current.revision() != expectedRevision || current.fence() != fence) { + return Optional.empty(); + } + if (current.failureCount() == 0) { + return Optional.of(current); + } + SignWorkflowStore.Record cleared = copySignRecord(current, current.state(), current.revision() + 1L, + current.fence(), current.leaseUntil(), current.detailCode(), current.result(), + current.providerUpdatedAt(), 0, Optional.empty(), Optional.empty()); + return replaceSignMetadata(stored, cleared, false) ? Optional.of(cleared) : Optional.empty(); + } finally { + releaseSignLock(submissionId, lock); + } + } + private List listStoredSigns() { List storedSigns = new ArrayList<>(); Set recordIdentities = new HashSet<>(); @@ -1806,7 +1940,7 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { } SignWorkflowStore.Record claimed = copySignRecord(current, current.state(), current.revision() + 1L, current.fence() + 1L, Optional.of(now.plus(lease)), current.detailCode(), current.result(), - current.providerUpdatedAt()); + current.providerUpdatedAt(), 0, Optional.empty(), Optional.empty()); return replaceSignMetadata(stored, claimed, false) ? Optional.of(claimed) : Optional.empty(); } finally { releaseSignLock(submissionId, lock); @@ -1831,7 +1965,8 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { } SignWorkflowStore.Record renewed = copySignRecord(current, current.state(), current.revision() + 1L, fence, Optional.of(signingNow().plus(lease)), current.detailCode(), current.result(), - current.providerUpdatedAt()); + current.providerUpdatedAt(), current.failureCount(), current.nextEligibleAt(), + current.reconciliationFailureClass()); return replaceSignMetadata(stored, renewed, false) ? Optional.of(renewed) : Optional.empty(); } finally { releaseSignLock(submissionId, lock); @@ -1868,7 +2003,7 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { throw new IllegalArgumentException("Successful signing completion timestamp is not trustworthy"); } SignWorkflowStore.Record transitioned = copySignRecord(current, target, current.revision() + 1L, fence, - Optional.empty(), detailCode, result, providerUpdatedAt); + Optional.empty(), detailCode, result, providerUpdatedAt, 0, Optional.empty(), Optional.empty()); return replaceSignMetadata(stored, transitioned, false) ? Optional.of(transitioned) : Optional.empty(); } finally { releaseSignLock(submissionId, lock); @@ -1896,7 +2031,8 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { SignWorkflowStore.Record retired = new SignWorkflowStore.Record(current.submissionId(), current.namespace(), current.fingerprint(), current.owner(), current.createdAt(), current.deadline(), retiredRequest, SignWorkflowStore.State.RETIRED, current.revision() + 1L, - fence, Optional.empty(), Optional.of("RETIRED"), current.result(), current.providerUpdatedAt()); + fence, Optional.empty(), Optional.of("RETIRED"), current.result(), current.providerUpdatedAt(), 0, + Optional.empty(), Optional.empty()); if (!replaceSignMetadata(stored, retired, true)) { return Optional.empty(); } @@ -2308,8 +2444,16 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { requireValidSignRecord(leaseUntil.isAfter(record.createdAt()) && !leaseUntil.isAfter(horizonEnd), record, "LEASE_TIME_INVALID"); } + if (record.nextEligibleAt().isPresent()) { + Instant nextEligible = record.nextEligibleAt().orElseThrow(); + requireValidSignRecord(!nextEligible.isAfter(horizonEnd) + && !nextEligible.isAfter(signingNow().plusSeconds(30L)), record, + "RECONCILIATION_ELIGIBILITY_INVALID"); + } if (record.result().isPresent()) { - requireValidSignRecord(record.result().get().bytes().length <= FsCodec.MAX_COMPONENT_BYTES, record, + requireValidSignRecord(record.result().get().encoding() == Encoding.BINARY + && record.result().get().bytes().length > 0 + && record.result().get().bytes().length <= 1_048_576, record, "RESULT_SIZE_INVALID"); } @@ -2363,12 +2507,15 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { return result; } + @SuppressWarnings("PMD.ExcessiveParameterList") private static SignWorkflowStore.Record copySignRecord(SignWorkflowStore.Record current, SignWorkflowStore.State state, long revision, long fence, Optional leaseUntil, - Optional detailCode, Optional result, Optional providerUpdatedAt) { + Optional detailCode, Optional result, Optional providerUpdatedAt, + int failureCount, Optional nextEligibleAt, + Optional failureClass) { return new SignWorkflowStore.Record(current.submissionId(), current.namespace(), current.fingerprint(), current.owner(), current.createdAt(), current.deadline(), current.request(), state, revision, fence, - leaseUntil, detailCode, result, providerUpdatedAt); + leaseUntil, detailCode, result, providerUpdatedAt, failureCount, nextEligibleAt, failureClass); } private static MetadataKey signingRecordKey(PkiId submissionId) { @@ -2564,7 +2711,7 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { private static byte[] readMetadataValue(MetadataSnapshot.Record record) throws IOException { long length = record.length().orElseThrow(); - if (length < 0L || length > FsCodec.MAX_COMPONENT_BYTES) { + if (length < 0L || length > MAX_METADATA_VALUE_BYTES) { throw new IOException("Metadata value length is invalid"); } byte[] result = new byte[Math.toIntExact(length)]; @@ -2643,7 +2790,7 @@ public final class FilesystemPkiStore implements PkiStore, Closeable { String ownerId = readOwnerField(input); String storeId = readOwnerField(input); String contentId = readOwnerField(input); - zeroecho.pki.api.Encoding encoding = zeroecho.pki.api.Encoding.valueOf(readOwnerField(input)); + Encoding encoding = Encoding.valueOf(readOwnerField(input)); String digest = readOwnerField(input); DurableContentReference.Lifecycle lifecycle = DurableContentReference.Lifecycle.valueOf( readOwnerField(input)); diff --git a/pki/src/main/java/zeroecho/pki/impl/fs/FsCodec.java b/pki/src/main/java/zeroecho/pki/impl/fs/FsCodec.java index a41f2e8..d24f8cb 100644 --- a/pki/src/main/java/zeroecho/pki/impl/fs/FsCodec.java +++ b/pki/src/main/java/zeroecho/pki/impl/fs/FsCodec.java @@ -114,8 +114,9 @@ import zeroecho.pki.spi.store.SignWorkflowStore; @SuppressWarnings("PMD.CouplingBetweenObjects") final class FsCodec { - /* package */ static final int MAX_COMPONENT_BYTES = 256 * 1024; - /* package */ static final int CURRENT_CODEC_VERSION = 4; + /* package */ static final int MAX_COMPONENT_BYTES = 1024 * 1024; + /* package */ static final int CURRENT_CODEC_VERSION = 5; + private static final int PREVIOUS_CODEC_VERSION = 4; private static final int CODEC_MAGIC = 0x5A454346; private static final int MAX_COLLECTION_ELEMENTS = MAX_COMPONENT_BYTES; @@ -168,6 +169,7 @@ final class FsCodec { private static final int TYPE_PROFILE_BINDING = 73; private static final int TYPE_DURABLE_CONTENT = 74; private static final int TYPE_ISSUER_GENERATION_STATE_ENUM = 75; + private static final int TYPE_RETRY_FAILURE_ENUM = 76; private static final int ATTRIBUTE_STRING = 1; private static final int ATTRIBUTE_BOOLEAN = 2; @@ -281,6 +283,19 @@ final class FsCodec { throw new IOException("unknown SignWorkflowStore.State code " + code, ex); } }); + private static final ValueSchema RECONCILIATION_FAILURE_CLASS = + enumSchema(TYPE_RETRY_FAILURE_ENUM, value -> switch (value) { + case SUBMISSION_UNCERTAIN -> 1; + case STATUS_UNAVAILABLE -> 2; + case CANCELLATION_UNCERTAIN -> 3; + case LOCAL_FAILURE -> 4; + }, code -> switch (code) { + case 1 -> SignWorkflowStore.ReconciliationFailureClass.SUBMISSION_UNCERTAIN; + case 2 -> SignWorkflowStore.ReconciliationFailureClass.STATUS_UNAVAILABLE; + case 3 -> SignWorkflowStore.ReconciliationFailureClass.CANCELLATION_UNCERTAIN; + case 4 -> SignWorkflowStore.ReconciliationFailureClass.LOCAL_FAILURE; + default -> throw unknownEnum("ReconciliationFailureClass", code); + }); private static final ValueSchema SUBJECT_RDN_TYPE = enumSchema(TYPE_SUBJECT_RDN_TYPE_ENUM, value -> switch (value) { case COMMON_NAME -> 1; @@ -376,6 +391,8 @@ final class FsCodec { private static final ValueSchema> OPTIONAL_STRING = optionalOf(STRING); private static final ValueSchema> OPTIONAL_INSTANT = optionalOf(INSTANT); private static final ValueSchema> OPTIONAL_ENCODED_OBJECT = optionalOf(ENCODED_OBJECT); + private static final ValueSchema> + OPTIONAL_RETRY_FAILURE = optionalOf(RECONCILIATION_FAILURE_CLASS); private static final ValueSchema CREDENTIAL_VALUE = valueSchema(TYPE_CREDENTIAL_RECORD, FsCodec::writeCredential, FsCodec::readCredential); @@ -459,9 +476,10 @@ final class FsCodec { throw new IOException("codec magic mismatch"); } int version = reader.readUnsignedByte(); - if (version != CURRENT_CODEC_VERSION) { + if (version != CURRENT_CODEC_VERSION && version != PREVIOUS_CODEC_VERSION) { throw new IOException("unsupported codec version"); } + reader.codecVersion = version; int typeId = reader.readUnsignedByte(); Schema encodedSchema = TOP_LEVEL_SCHEMAS.get(typeId); if (encodedSchema == null) { @@ -818,15 +836,35 @@ final class FsCodec { writer.writeValue(OPTIONAL_STRING, value.detailCode()); writer.writeValue(OPTIONAL_ENCODED_OBJECT, value.result()); writer.writeValue(OPTIONAL_INSTANT, value.providerUpdatedAt()); + writer.writeValue(LONG, (long) value.failureCount()); + writer.writeValue(OPTIONAL_INSTANT, value.nextEligibleAt()); + writer.writeValue(OPTIONAL_RETRY_FAILURE, value.reconciliationFailureClass()); } private static SignWorkflowStore.Record readSignWorkflowRecord(Reader reader) throws IOException { - return new SignWorkflowStore.Record(reader.readValue(PKI_ID), reader.readValue(STRING), - reader.readValue(STRING), reader.readValue(PRINCIPAL), reader.readValue(INSTANT), - reader.readValue(INSTANT), reader.readValue(ENCODED_OBJECT), reader.readValue(SIGN_STATE), - reader.readValue(LONG), reader.readValue(LONG), reader.readValue(OPTIONAL_INSTANT), - reader.readValue(OPTIONAL_STRING), reader.readValue(OPTIONAL_ENCODED_OBJECT), - reader.readValue(OPTIONAL_INSTANT)); + PkiId submissionId = reader.readValue(PKI_ID); + String namespace = reader.readValue(STRING); + String fingerprint = reader.readValue(STRING); + Principal owner = reader.readValue(PRINCIPAL); + Instant createdAt = reader.readValue(INSTANT); + Instant deadline = reader.readValue(INSTANT); + EncodedObject request = reader.readValue(ENCODED_OBJECT); + SignWorkflowStore.State state = reader.readValue(SIGN_STATE); + long revision = reader.readValue(LONG); + long fence = reader.readValue(LONG); + Optional leaseUntil = reader.readValue(OPTIONAL_INSTANT); + Optional detailCode = reader.readValue(OPTIONAL_STRING); + Optional result = reader.readValue(OPTIONAL_ENCODED_OBJECT); + Optional providerUpdatedAt = reader.readValue(OPTIONAL_INSTANT); + if (reader.codecVersion < CURRENT_CODEC_VERSION) { + return new SignWorkflowStore.Record(submissionId, namespace, fingerprint, owner, createdAt, deadline, + request, state, revision, fence, leaseUntil, detailCode, result, providerUpdatedAt, 0, + Optional.empty(), Optional.empty()); + } + int failureCount = Math.toIntExact(reader.readValue(LONG)); + return new SignWorkflowStore.Record(submissionId, namespace, fingerprint, owner, createdAt, deadline, + request, state, revision, fence, leaseUntil, detailCode, result, providerUpdatedAt, failureCount, + reader.readValue(OPTIONAL_INSTANT), reader.readValue(OPTIONAL_RETRY_FAILURE)); } private static Schema topLevel(int typeId, String name, ValueSchema valueSchema) { @@ -1046,6 +1084,7 @@ final class FsCodec { private final InputStream input; private final StagedContentStore stagedContent; + private int codecVersion; private Reader(InputStream input, StagedContentStore stagedContent) { this.input = input; diff --git a/pki/src/main/java/zeroecho/pki/impl/fs/MetadataStateIndex.java b/pki/src/main/java/zeroecho/pki/impl/fs/MetadataStateIndex.java index e4aa134..cef3666 100644 --- a/pki/src/main/java/zeroecho/pki/impl/fs/MetadataStateIndex.java +++ b/pki/src/main/java/zeroecho/pki/impl/fs/MetadataStateIndex.java @@ -34,9 +34,9 @@ package zeroecho.pki.impl.fs; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; @@ -52,6 +52,7 @@ import zeroecho.pki.spi.store.MetadataKey; import zeroecho.pki.spi.store.MetadataStoreException; /** Atomic current-record index reconstructed from committed mutation descriptors. */ +@SuppressWarnings({ "PMD.CyclomaticComplexity", "PMD.AvoidLiteralsInIfCondition", "PMD.ConfusingTernary" }) final class MetadataStateIndex { private static final long INITIAL_STORE_REVISION = 0L; private static final long MINIMUM_VALUE_POSITION = 0L; @@ -62,6 +63,8 @@ final class MetadataStateIndex { private NavigableMap current = new TreeMap<>(); private Object stateToken = new StateToken(); private long storeRevision = INITIAL_STORE_REVISION; + private final Map, Integer> snapshotPins = new IdentityHashMap<>(); + private long snapshotCopyCount; /* default */ static RecoveryBuilder recoveryBuilder() { return new RecoveryBuilder(); @@ -86,6 +89,18 @@ final class MetadataStateIndex { } } + /* default */ SnapshotView snapshot() { + writeLock.lock(); + try { + NavigableMap generation = current; + snapshotPins.merge(generation, 1, Math::addExact); + return new SnapshotView(java.util.Collections.unmodifiableNavigableMap(generation), + () -> releaseSnapshot(generation)); + } finally { + writeLock.unlock(); + } + } + /* default */ List records() { readLock.lock(); try { @@ -95,8 +110,35 @@ final class MetadataStateIndex { } } + private void releaseSnapshot(NavigableMap generation) { + writeLock.lock(); + try { + Integer pins = snapshotPins.get(generation); + if (pins == null || pins <= 0) { + throw new IllegalStateException("Metadata snapshot pin accounting is inconsistent"); + } + if (pins == 1) { + snapshotPins.remove(generation); + } else { + snapshotPins.put(generation, pins - 1); + } + } finally { + writeLock.unlock(); + } + } + + /* default */ long snapshotCopyCount() { + readLock.lock(); + try { + return snapshotCopyCount; + } finally { + readLock.unlock(); + } + } + /* - * Copy-then-publish keeps every conflict and validation failure invisible. + * Validate-then-publish keeps every conflict and validation failure invisible. + * A full map copy occurs only while a stable snapshot pins the prior identity. * CREATE and REPLACE record revisions are the authoritative committed store * revision, never an independently incremented per-record counter. */ @@ -114,15 +156,14 @@ final class MetadataStateIndex { try { requireNextRevision(committedStoreRevision); rejectDuplicateKeys(validated); - NavigableMap candidate = new TreeMap<>(current); for (ValidatedMutation mutation : validated) { - apply(candidate, committedStoreRevision, mutation); + validateAgainstCurrent(current, mutation); } return new PreparedUpdate( stateToken, storeRevision, committedStoreRevision, - Collections.unmodifiableNavigableMap(candidate)); + validated); } catch (ArithmeticException | IllegalArgumentException | NullPointerException | IndexOutOfBoundsException failure) { throw integrity("Committed metadata transaction contains a malformed descriptor", failure); @@ -138,7 +179,16 @@ final class MetadataStateIndex { if (!stateToken.equals(update.baseToken()) || storeRevision != update.baseRevision()) { throw integrity("Prepared metadata state no longer has its exact base revision"); } - current = update.candidate(); + boolean currentGenerationPinned = snapshotPins.containsKey(current); + NavigableMap target = !currentGenerationPinned + ? current : new TreeMap<>(current); + if (currentGenerationPinned) { + snapshotCopyCount++; + } + for (ValidatedMutation mutation : update.mutations()) { + apply(target, update.targetRevision(), mutation); + } + current = target; storeRevision = update.targetRevision(); stateToken = new StateToken(); } finally { @@ -285,6 +335,19 @@ final class MetadataStateIndex { } } + private static void validateAgainstCurrent(Map records, + ValidatedMutation mutation) throws MetadataStoreException { + CurrentRecord existing = records.get(mutation.key()); + switch (mutation.kind()) { + case CREATE -> { + if (existing != null) { + throw conflict("Metadata create precondition failed"); + } + } + case REPLACE, DELETE -> requireExpected(existing, mutation); + } + } + private static void create( Map candidate, long committedStoreRevision, @@ -449,10 +512,18 @@ final class MetadataStateIndex { Object baseToken, long baseRevision, long targetRevision, - NavigableMap candidate) { + List mutations) { PreparedUpdate { Objects.requireNonNull(baseToken, "baseToken"); - Objects.requireNonNull(candidate, "candidate"); + mutations = List.copyOf(mutations); + } + } + + /** Pinned immutable map identity released when its snapshot closes. */ + /* default */ record SnapshotView(NavigableMap records, Runnable release) { + SnapshotView { + Objects.requireNonNull(records, "records"); + Objects.requireNonNull(release, "release"); } } diff --git a/pki/src/main/java/zeroecho/pki/impl/fs/PosixMetadataSnapshotSupport.java b/pki/src/main/java/zeroecho/pki/impl/fs/PosixMetadataSnapshotSupport.java index c435180..085282f 100644 --- a/pki/src/main/java/zeroecho/pki/impl/fs/PosixMetadataSnapshotSupport.java +++ b/pki/src/main/java/zeroecho/pki/impl/fs/PosixMetadataSnapshotSupport.java @@ -45,7 +45,6 @@ import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Logger; import zeroecho.core.io.CancellationSignal; @@ -55,6 +54,7 @@ import zeroecho.pki.spi.store.MetadataSnapshot; import zeroecho.pki.spi.store.MetadataStoreId; /** Stable snapshot, lazy cursor, and bounded log-slice content lifecycles. */ +@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") final class PosixMetadataSnapshotSupport { private static final Logger LOGGER = Logger.getLogger(PosixMetadataSnapshotSupport.class.getName()); private static final String CLEANUP_WARNING = @@ -91,7 +91,8 @@ final class PosixMetadataSnapshotSupport { private final class SnapshotImpl implements MetadataSnapshot, PosixMetadataAdapterLifecycle.ManagedResource { private final long revision; - private final NavigableMap records; + private final NavigableMap records; + private final Runnable release; private final ReentrantLock lock = new ReentrantLock(); private final Set recordChildren = Collections.newSetFromMap(new IdentityHashMap<>()); @@ -102,10 +103,8 @@ final class PosixMetadataSnapshotSupport { private SnapshotImpl(PosixMetadataStoreEngine.SnapshotState captured) { super(); revision = captured.revision(); - NavigableMap detached = new TreeMap<>(); - captured.records().forEach(record -> - detached.put(record.key(), new RecordMetadata(record))); - records = Collections.unmodifiableNavigableMap(detached); + records = captured.records(); + release = captured.release(); } @Override @@ -126,10 +125,10 @@ final class PosixMetadataSnapshotSupport { lock.lock(); try { requireOpenLocked(); - RecordMetadata metadata = records.get(key); + MetadataStateIndex.CurrentRecord metadata = records.get(key); return metadata == null ? Optional.empty() - : Optional.of(createRecordLocked(metadata)); + : Optional.of(createRecordLocked(new RecordMetadata(metadata))); } finally { lock.unlock(); } @@ -143,7 +142,9 @@ final class PosixMetadataSnapshotSupport { lock.lock(); try { requireOpenLocked(); - CursorImpl cursor = new CursorImpl(range, records.entrySet().iterator()); + String lower = range.lowerInclusive().orElse("!"); + MetadataKey first = new MetadataKey(range.namespace(), lower); + CursorImpl cursor = new CursorImpl(range, records.tailMap(first, true).entrySet().iterator()); cursorChildren.add(cursor); return cursor; } finally { @@ -170,6 +171,7 @@ final class PosixMetadataSnapshotSupport { return null; } closed = true; + release.run(); cursors = List.copyOf(cursorChildren); children = List.copyOf(recordChildren); cursorChildren.clear(); @@ -244,13 +246,13 @@ final class PosixMetadataSnapshotSupport { /** Cursor retains only one map iterator and one current record. */ private final class CursorImpl implements MetadataCursor { private final KeyRange range; - private final Iterator> iterator; + private final Iterator> iterator; private RecordImpl current; private boolean cursorClosed; private CursorImpl( KeyRange range, - Iterator> iterator) { + Iterator> iterator) { super(); this.range = range; this.iterator = iterator; @@ -278,11 +280,16 @@ final class PosixMetadataSnapshotSupport { if (!iterator.hasNext()) { return Optional.empty(); } - Map.Entry candidate = iterator.next(); + Map.Entry candidate = iterator.next(); if (range.contains(candidate.getKey())) { - current = createRecordLocked(candidate.getValue()); + current = createRecordLocked(new RecordMetadata(candidate.getValue())); return Optional.of(current); } + if (candidate.getKey().namespace().compareTo(range.namespace()) > 0 + || candidate.getKey().namespace().equals(range.namespace()) + && range.upperExclusive().isPresent()) { + return Optional.empty(); + } } finally { lock.unlock(); } diff --git a/pki/src/main/java/zeroecho/pki/impl/fs/PosixMetadataStoreEngine.java b/pki/src/main/java/zeroecho/pki/impl/fs/PosixMetadataStoreEngine.java index 71cbab5..228cf26 100644 --- a/pki/src/main/java/zeroecho/pki/impl/fs/PosixMetadataStoreEngine.java +++ b/pki/src/main/java/zeroecho/pki/impl/fs/PosixMetadataStoreEngine.java @@ -46,6 +46,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; @@ -253,7 +254,9 @@ final class PosixMetadataStoreEngine implements AutoCloseable { transactionLock.lock(); try { requireOperational(); - return new SnapshotState(stateIndex.storeRevision(), stateIndex.records()); + long revision = stateIndex.storeRevision(); + MetadataStateIndex.SnapshotView view = stateIndex.snapshot(); + return new SnapshotState(revision, view.records(), view.release()); } finally { transactionLock.unlock(); } @@ -458,12 +461,14 @@ final class PosixMetadataStoreEngine implements AutoCloseable { /** Immutable finite current-state metadata captured under engine serialization. */ /* default */ record SnapshotState( - long revision, List records) { + long revision, NavigableMap records, + Runnable release) { SnapshotState { if (revision < MINIMUM_VALUE_BOUNDARY) { throw new IllegalArgumentException("Snapshot revision must not be negative"); } - records = List.copyOf(records); + Objects.requireNonNull(records, "records"); + Objects.requireNonNull(release, "release"); } } diff --git a/pki/src/main/java/zeroecho/pki/spi/crypto/SignatureWorkflow.java b/pki/src/main/java/zeroecho/pki/spi/crypto/SignatureWorkflow.java index e2e8e50..4263c8b 100644 --- a/pki/src/main/java/zeroecho/pki/spi/crypto/SignatureWorkflow.java +++ b/pki/src/main/java/zeroecho/pki/spi/crypto/SignatureWorkflow.java @@ -83,7 +83,7 @@ import zeroecho.pki.api.audit.AccessContext; *
        *
      • Validation and policy failures must not be surfaced as uncaught * exceptions. Instead, the provider must return an operation id and expose the - * failure via {@link #status(PkiId)} using {@link OperationStatus#state()} == + * failure via {@link #status(PkiId, CallControl)} using {@link OperationStatus#state()} == * {@link State#FAILED} and a stable {@link OperationStatus#detailCode()}.
        • *
      • {@link IllegalArgumentException} may be thrown only for programmer errors * such as {@code request == null} or {@code operationId == null}. These are not @@ -153,7 +153,7 @@ public interface SignatureWorkflow extends Closeable { * committing success. A completion at the deadline is late; it must become * {@link State#EXPIRED} without exposing a result. Terminal states are * immutable. At and after the configured horizon, submission is rejected and - * {@link #status(PkiId)} reports {@link State#EXPIRED}, including after + * {@link #status(PkiId, CallControl)} reports {@link State#EXPIRED}, including after * payload/result purge and restart. Provider callbacks must run after operation * state locks are released. *

        @@ -161,7 +161,7 @@ public interface SignatureWorkflow extends Closeable { *

        Failure model (normative)

        *
          *
        • For validation and policy failures: do not throw; return operation id and - * expose failure via {@link #status(PkiId)} with {@code FAILED} and stable + * expose failure via {@link #status(PkiId, CallControl)} with {@code FAILED} and stable * {@code detailCode}.
          • *
        • Throws {@link IllegalStateException} for identifier/fingerprint conflicts * or stale fencing tokens.
          • @@ -170,9 +170,10 @@ public interface SignatureWorkflow extends Closeable { *
        * * @param request request (never {@code null}) + * @param control cooperative call deadline and cancellation * @return operation id (never {@code null}) */ - PkiId submitSign(SignRequest request); + PkiId submitSign(SignRequest request, CallControl control); /** * Submits a verification request. @@ -190,24 +191,26 @@ public interface SignatureWorkflow extends Closeable { *

        Failure model (normative)

        *
          *
        • For validation and policy failures: do not throw; return operation id and - * expose failure via {@link #status(PkiId)} with {@code FAILED} and stable + * expose failure via {@link #status(PkiId, CallControl)} with {@code FAILED} and stable * {@code detailCode}.
          • *
        • May throw {@link IllegalArgumentException} only for programmer errors * (e.g., {@code request == null}).
          • *
        * * @param request request (never {@code null}) + * @param control cooperative call deadline and cancellation * @return operation id (never {@code null}) */ - PkiId submitVerify(VerifyRequest request); + PkiId submitVerify(VerifyRequest request, CallControl control); /** * Reads current status of an operation. * * @param operationId operation id (never {@code null}) + * @param control cooperative call deadline and cancellation * @return status (never {@code null}) */ - OperationStatus status(PkiId operationId); + OperationStatus status(PkiId operationId, CallControl control); /** * Best-effort cancellation. @@ -215,15 +218,26 @@ public interface SignatureWorkflow extends Closeable { *

        * A {@code true} return value means only that the provider accepted the * cancellation request. It does not prove that the operation is terminal. - * Callers must re-read {@link #status(PkiId)} and may retire state only after + * Callers must re-read {@link #status(PkiId, CallControl)} and may retire state only after * an immutable terminal status is observed. *

        * + *

        The operation identifier and fencing-token pair is idempotent across + * ambiguous acceptance and restart, regardless of the safe reason supplied on + * a replay. A provider may retain the first accepted reason for audit, but a + * replay with another reason must not cause another external effect. A lower + * fencing token must not mutate provider state, and every terminal state is + * immutable.

        + * * @param operationId operation id (never {@code null}) + * @param fencingToken monotonic fencing token * @param reason non-sensitive reason (never blank) - * @return true if cancellation was accepted; false if already terminal/unknown + * @param control cooperative call deadline and cancellation + * @return true if cancellation was accepted or the exact operation/fence pair + * replays an accepted cancellation; false if unknown, stale, superseded, + * or terminal for another outcome */ - boolean cancel(PkiId operationId, long fencingToken, String reason); + boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control); /** * Registers a notification sink for status changes. @@ -248,6 +262,34 @@ public interface SignatureWorkflow extends Closeable { @Override void close(); + /** + * Immutable cooperative deadline and cancellation control for one provider + * invocation. + * + * @param deadline absolute exclusive call deadline + * @param cancellation cooperative cancellation signal + */ + record CallControl(Instant deadline, CancellationSignal cancellation) { + /** Validates one call control. */ + public CallControl { + Objects.requireNonNull(deadline, "deadline"); + Objects.requireNonNull(cancellation, "cancellation"); + } + + /** + * Rejects work at or after the deadline or after cancellation. + * + * @param now authoritative current time + * @throws IllegalStateException when the call may no longer continue + */ + public void requireActive(Instant now) { + Objects.requireNonNull(now, "now"); + if (cancellation.isCancelled() || !now.isBefore(deadline)) { + throw new IllegalStateException("Signature workflow call is no longer active"); + } + } + } + /** * Signing request. * @@ -256,7 +298,7 @@ public interface SignatureWorkflow extends Closeable { * {@link IllegalArgumentException} at construction time. Callers building * requests for external transports are expected to catch such exceptions and * convert them to an appropriate error state before invoking - * {@link #submitSign(SignRequest)}. + * {@link #submitSign(SignRequest, CallControl)}. *

        * * @param submissionId stable caller-assigned submission @@ -401,7 +443,7 @@ public interface SignatureWorkflow extends Closeable { * {@link IllegalArgumentException} at construction time. Callers building * requests for external transports are expected to catch such exceptions and * convert them to an appropriate error state before invoking - * {@link #submitVerify(VerifyRequest)}. + * {@link #submitVerify(VerifyRequest, CallControl)}. *

        * * @param accessContext audit/governance context (never {@code null}) @@ -443,7 +485,7 @@ public interface SignatureWorkflow extends Closeable { * of an operation previously submitted via {@code submitSign} or * {@code submitVerify}. Providers must ensure that status transitions are * monotonic and observable through repeated calls to - * {@link SignatureWorkflow#status(PkiId)}. + * {@link SignatureWorkflow#status(PkiId, CallControl)}. *

        * *

        Failure and audit model

        @@ -586,7 +628,8 @@ public interface SignatureWorkflow extends Closeable { *

        * The callback must be treated as a best-effort notification mechanism and must * not be relied upon as the sole source of truth; callers should always be able - * to query the authoritative state via {@link SignatureWorkflow#status(PkiId)}. + * to query the authoritative state via + * {@link SignatureWorkflow#status(PkiId, CallControl)}. * Delivery may be coalesced or dropped under load. Providers must not require * callback processing to finish an operation, and sink implementations should * return promptly without waiting for operation-level coordination. diff --git a/pki/src/main/java/zeroecho/pki/spi/store/SignWorkflowStore.java b/pki/src/main/java/zeroecho/pki/spi/store/SignWorkflowStore.java index ab1acf2..0a7cb6b 100644 --- a/pki/src/main/java/zeroecho/pki/spi/store/SignWorkflowStore.java +++ b/pki/src/main/java/zeroecho/pki/spi/store/SignWorkflowStore.java @@ -61,6 +61,18 @@ import zeroecho.pki.api.audit.Principal; */ public interface SignWorkflowStore { + /** Durable classification for one deferred reconciliation retry. */ + enum ReconciliationFailureClass { + /** Submission may have been accepted. */ + SUBMISSION_UNCERTAIN, + /** Provider status could not be obtained. */ + STATUS_UNAVAILABLE, + /** Provider cancellation outcome is uncertain. */ + CANCELLATION_UNCERTAIN, + /** Local durable reconciliation failed. */ + LOCAL_FAILURE + } + /** * Signing orchestration states. * @@ -135,7 +147,8 @@ public interface SignWorkflowStore { record Record(PkiId submissionId, String namespace, String fingerprint, Principal owner, Instant createdAt, Instant deadline, EncodedObject request, State state, long revision, long fence, Optional leaseUntil, Optional detailCode, Optional result, - Optional providerUpdatedAt) { + Optional providerUpdatedAt, int failureCount, Optional nextEligibleAt, + Optional reconciliationFailureClass) { public Record { Objects.requireNonNull(submissionId, "submissionId"); Objects.requireNonNull(namespace, "namespace"); @@ -149,6 +162,8 @@ public interface SignWorkflowStore { Objects.requireNonNull(detailCode, "detailCode"); Objects.requireNonNull(result, "result"); Objects.requireNonNull(providerUpdatedAt, "providerUpdatedAt"); + Objects.requireNonNull(nextEligibleAt, "nextEligibleAt"); + Objects.requireNonNull(reconciliationFailureClass, "reconciliationFailureClass"); if (namespace.isBlank() || fingerprint.isBlank()) { throw new IllegalArgumentException("namespace and fingerprint must not be blank"); } @@ -158,6 +173,34 @@ public interface SignWorkflowStore { if (revision < 0L || fence < 0L) { throw new IllegalArgumentException("revision and fence must not be negative"); } + boolean retryMetadataPresent = nextEligibleAt.isPresent() && reconciliationFailureClass.isPresent(); + if (failureCount < 0 || failureCount == 0 != !retryMetadataPresent) { + throw new IllegalArgumentException("Reconciliation retry metadata is inconsistent"); + } + if ((state == State.SUCCEEDED || state == State.FAILED || state == State.CANCELLED + || state == State.EXPIRED || state == State.RETIRED) && failureCount != 0) { + throw new IllegalArgumentException("Terminal signing records cannot retain retry metadata"); + } + } + } + + /** + * One bounded snapshot page ordered by canonical submission identifier. + * + * @param records immutable page records + * @param nextCursor exclusive cursor after the last returned record + * @param endReached whether the snapshot namespace was exhausted + */ + record Page(List records, Optional nextCursor, int examined, int failures, + boolean endReached) { + /** Validates one immutable page. */ + public Page { + records = List.copyOf(Objects.requireNonNull(records, "records")); + nextCursor = Objects.requireNonNull(nextCursor, "nextCursor"); + if (examined < records.size() || failures < 0 || failures > examined + || records.size() + failures != examined) { + throw new IllegalArgumentException("Signing page counts are inconsistent"); + } } } @@ -217,6 +260,33 @@ public interface SignWorkflowStore { */ List listSignRecords(); + /** + * Returns at most {@code maximumRecords} records after an advisory exclusive + * cursor, ordered by canonical submission identifier. + * + * @param exclusiveCursor opaque cursor from an earlier page + * @param maximumRecords page bound from 1 through 4096 + * @param cancellation cooperative scan cancellation + * @return bounded snapshot page + */ + Page pageSignRecords(Optional exclusiveCursor, int maximumRecords, + zeroecho.core.io.CancellationSignal cancellation); + + /** + * Defers reconciliation with store-authoritative exponential backoff. + * + * @return updated record, or empty when the revision/fence CAS loses + */ + Optional deferSignReconciliation(PkiId submissionId, long expectedRevision, long fence, + ReconciliationFailureClass failureClass); + + /** + * Clears durable retry metadata against the current revision and fence. + * + * @return updated record, or empty when the revision/fence CAS loses + */ + Optional clearSignReconciliation(PkiId submissionId, long expectedRevision, long fence); + /** * Atomically claims an intent and increments its revision and fencing token. * diff --git a/pki/src/test/java/zeroecho/pki/impl/core/async/PkiSigningBusFailureTest.java b/pki/src/test/java/zeroecho/pki/impl/core/async/PkiSigningBusFailureTest.java index 5db11ad..0362415 100644 --- a/pki/src/test/java/zeroecho/pki/impl/core/async/PkiSigningBusFailureTest.java +++ b/pki/src/test/java/zeroecho/pki/impl/core/async/PkiSigningBusFailureTest.java @@ -36,6 +36,7 @@ package zeroecho.pki.impl.core.async; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -71,6 +72,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import zeroecho.core.alg.BootstrapAlgorithmIdentities; +import zeroecho.core.io.CancellationSignal; import zeroecho.core.spec.AlgorithmIdentity; import zeroecho.core.spec.AlgorithmSuite; import zeroecho.core.spi.AlgorithmExecutionCapability; @@ -85,6 +87,8 @@ import zeroecho.pki.api.audit.Principal; import zeroecho.pki.api.audit.Purpose; import zeroecho.pki.api.orch.OrchestrationDurabilityPolicy; import zeroecho.pki.api.orch.WorkflowStateRecord; +import zeroecho.pki.application.SigningReconciliationRequest; +import zeroecho.pki.application.SigningReconciliationResult; import zeroecho.pki.impl.fs.FilesystemPkiStore; import zeroecho.pki.impl.fs.FsPkiStoreOptions; import zeroecho.pki.impl.framework.x509.X509AlgorithmResolver; @@ -97,6 +101,264 @@ import zeroecho.pki.util.async.AsyncState; final class PkiSigningBusFailureTest { + @Test + void reconciliationEnforcesIndependentScanAndProviderBudgetsAndWrapsFairly(@TempDir Path tempDir) + throws Exception { + MutableClock clock = new MutableClock(Instant.parse("2026-08-01T00:00:00Z")); + ControlledWorkflow signer = new ControlledWorkflow(clock); + try (FilesystemPkiStore store = new FilesystemPkiStore(tempDir.resolve("store"), + FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus(store, signer, tempDir.resolve("bus.log"), + signingAuthority(signer))) { + submit(bus); + clock.set(clock.instant().plusMillis(1)); + submit(bus); + clock.set(clock.instant().plusMillis(1)); + submit(bus); + + SigningReconciliationResult first = bus.reconcile(reconciliation(Optional.empty(), 2, 1, clock)); + assertEquals(2, first.examined()); + assertEquals(1, signer.submissions.get()); + assertEquals(1, first.progressed()); + assertEquals(2, first.unresolved()); + + SigningReconciliationResult second = bus.reconcile(reconciliation(first.nextCursor(), 2, 1, clock)); + assertEquals(2, second.examined()); + assertEquals(2, signer.submissions.get()); + assertTrue(second.endReached()); + + SigningReconciliationResult third = bus.reconcile(reconciliation(second.nextCursor(), 2, 1, clock)); + assertEquals(2, third.examined()); + assertEquals(3, signer.submissions.get()); + } + } + + @Test + void reconciliationDeadlineStopsPagingBeforeCandidateDecode(@TempDir Path tempDir) throws Exception { + MutableClock clock = new MutableClock(Instant.parse("2026-08-01T01:00:00Z")); + assertThrows(IllegalArgumentException.class, () -> new SigningReconciliationRequest(Optional.of("bad/cursor"), + 1, 1, clock.instant().plusSeconds(1), Duration.ofSeconds(1), CancellationSignal.NONE)); + assertThrows(IllegalArgumentException.class, () -> new SigningReconciliationRequest(Optional.of("bad-\u2603"), + 1, 1, clock.instant().plusSeconds(1), Duration.ofSeconds(1), CancellationSignal.NONE)); + ControlledWorkflow signer = new ControlledWorkflow(clock); + try (FilesystemPkiStore store = new FilesystemPkiStore(tempDir.resolve("store"), + FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus(store, signer, tempDir.resolve("bus.log"), + signingAuthority(signer))) { + submit(bus); + SigningReconciliationRequest expired = new SigningReconciliationRequest(Optional.empty(), 256, 64, + clock.instant(), Duration.ofSeconds(10), CancellationSignal.NONE); + + SigningReconciliationResult result = bus.reconcile(expired); + + assertEquals(0, result.examined()); + assertEquals(0, signer.submissions.get()); + assertFalse(result.endReached()); + } + } + + @Test + void reconciliationUsesStatusBeforeCancellationAndRetiresTerminalSamePass(@TempDir Path tempDir) + throws Exception { + MutableClock clock = new MutableClock(Instant.parse("2026-08-02T00:00:00Z")); + ControlledWorkflow signer = new ControlledWorkflow(clock); + try (FilesystemPkiStore store = new FilesystemPkiStore(tempDir.resolve("store"), + FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus(store, signer, tempDir.resolve("bus.log"), + signingAuthority(signer))) { + PkiId id = submit(bus); + assertEquals(AsyncState.RUNNING, bus.status(id).orElseThrow().state()); + SignWorkflowStore.Record dispatched = store.getSignRecord(id).orElseThrow(); + store.transitionSign(id, dispatched.revision(), dispatched.fence(), SignWorkflowStore.State.CANCELLING, + Optional.of("CANCEL_REQUESTED"), Optional.empty(), Optional.empty()).orElseThrow(); + + bus.reconcile(reconciliation(Optional.empty(), 1, 1, clock)); + assertEquals(0, signer.cancellations.get()); + assertEquals("CANCEL_REQUESTED", store.getSignRecord(id).orElseThrow().detailCode().orElseThrow()); + + bus.reconcile(reconciliation(Optional.empty(), 2, 2, clock)); + assertEquals(1, signer.cancellations.get()); + assertEquals(SignWorkflowStore.State.CANCELLING, store.getSignRecord(id).orElseThrow().state()); + + SigningReconciliationResult terminal = bus.reconcile(reconciliation(Optional.empty(), 1, 1, clock)); + assertEquals(SignWorkflowStore.State.RETIRED, store.getSignRecord(id).orElseThrow().state()); + assertEquals(1, terminal.progressed()); + assertEquals(0, terminal.unresolved()); + } + } + + @Test + void throwingCancellationConsumesExactTwoCallBudgetAndDoesNotTouchLaterRecord(@TempDir Path tempDir) + throws Exception { + MutableClock clock = new MutableClock(Instant.parse("2026-08-03T00:00:00Z")); + ThrowingCancellationWorkflow signer = new ThrowingCancellationWorkflow(clock); + try (FilesystemPkiStore store = new FilesystemPkiStore(tempDir.resolve("store"), + FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus(store, signer, tempDir.resolve("bus.log"), + signingAuthority(signer))) { + PkiId first = submit(bus); + clock.set(clock.instant().plusMillis(1)); + PkiId second = submit(bus); + assertEquals(AsyncState.RUNNING, bus.status(first).orElseThrow().state()); + assertEquals(AsyncState.RUNNING, bus.status(second).orElseThrow().state()); + requestCancellation(store, first); + requestCancellation(store, second); + signer.resetObservations(); + + SigningReconciliationResult result = bus.reconcile(reconciliation(Optional.empty(), 2, 2, clock)); + + assertEquals(2, result.examined()); + assertEquals(1, signer.statusReads.get()); + assertEquals(1, signer.cancellations.get()); + assertEquals(1, store.getSignRecord(first).orElseThrow().failureCount() + + store.getSignRecord(second).orElseThrow().failureCount()); + } + } + + @Test + void pointReadFailureIsolatedAndLaterCandidateUsesRemainingProviderBudget(@TempDir Path tempDir) + throws Exception { + MutableClock clock = new MutableClock(Instant.parse("2026-08-03T01:00:00Z")); + ControlledWorkflow signer = new ControlledWorkflow(clock); + AtomicReference faultTarget = new AtomicReference<>(); + AtomicBoolean faultArmed = new AtomicBoolean(); + try (FilesystemPkiStore delegate = new FilesystemPkiStore(tempDir.resolve("store"), + FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus( + faultingStore(delegate, "getSignRecord", faultTarget, faultArmed), signer, + tempDir.resolve("bus.log"), signingAuthority(signer))) { + PkiId first = submit(bus); + clock.set(clock.instant().plusMillis(1)); + submit(bus); + faultTarget.set(first); + faultArmed.set(true); + + SigningReconciliationResult result = bus.reconcile(reconciliation(Optional.empty(), 2, 1, clock)); + + assertEquals(2, result.examined()); + assertEquals(1, signer.submissions.get()); + assertEquals(2, result.unresolved()); + assertFalse(faultArmed.get()); + } + } + + @Test + void retirementFailureIsolatedAndLaterCandidateProgresses(@TempDir Path tempDir) throws Exception { + MutableClock clock = new MutableClock(Instant.parse("2026-08-03T02:00:00Z")); + ControlledWorkflow signer = new ControlledWorkflow(clock); + AtomicReference faultTarget = new AtomicReference<>(); + AtomicBoolean faultArmed = new AtomicBoolean(); + try (FilesystemPkiStore delegate = new FilesystemPkiStore(tempDir.resolve("store"), + FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus( + faultingStore(delegate, "retireSign", faultTarget, faultArmed), signer, + tempDir.resolve("bus.log"), signingAuthority(signer))) { + PkiId terminal = submit(bus); + assertEquals(AsyncState.RUNNING, bus.status(terminal).orElseThrow().state()); + signer.succeedAt(terminal, clock.instant(), (byte) 7); + assertEquals(AsyncState.SUCCEEDED, bus.status(terminal).orElseThrow().state()); + clock.set(clock.instant().plusMillis(1)); + submit(bus); + faultTarget.set(terminal); + faultArmed.set(true); + + SigningReconciliationResult result = bus.reconcile(reconciliation(Optional.empty(), 2, 1, clock)); + + assertEquals(2, result.examined()); + assertEquals(2, signer.submissions.get()); + assertTrue(result.progressed() >= 1); + assertTrue(result.retryable() >= 1); + assertFalse(faultArmed.get()); + } + } + + @Test + void deferWriteFailureIsolatedWithExactProviderAttemptsAndLaterProgress(@TempDir Path tempDir) + throws Exception { + MutableClock clock = new MutableClock(Instant.parse("2026-08-03T03:00:00Z")); + AcceptedThenThrowsWorkflow signer = new AcceptedThenThrowsWorkflow(clock); + AtomicReference faultTarget = new AtomicReference<>(); + AtomicBoolean faultArmed = new AtomicBoolean(); + try (FilesystemPkiStore delegate = new FilesystemPkiStore(tempDir.resolve("store"), + FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus( + faultingStore(delegate, "deferSignReconciliation", faultTarget, faultArmed), signer, + tempDir.resolve("bus.log"), signingAuthority(signer))) { + PkiId first = submit(bus); + clock.set(clock.instant().plusMillis(1)); + submit(bus); + faultTarget.set(first); + faultArmed.set(true); + + SigningReconciliationResult result = bus.reconcile(reconciliation(Optional.empty(), 2, 2, clock)); + + assertEquals(2, result.examined()); + assertEquals(2, signer.submissions.get()); + assertEquals(1, result.progressed()); + assertEquals(2, result.retryable()); + assertFalse(faultArmed.get()); + } + } + + @Test + void durabilityUncertainStoreFailureFailsPassClosed(@TempDir Path tempDir) throws Exception { + MutableClock clock = new MutableClock(Instant.parse("2026-08-03T04:00:00Z")); + ControlledWorkflow signer = new ControlledWorkflow(clock); + AtomicReference faultTarget = new AtomicReference<>(); + AtomicBoolean faultArmed = new AtomicBoolean(); + try (FilesystemPkiStore delegate = new FilesystemPkiStore(tempDir.resolve("store"), + FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus( + faultingStore(delegate, "getSignRecord", faultTarget, faultArmed, + new PkiException("Store durability unconfirmed: code=STORE_DURABILITY_UNCONFIRMED")), + signer, tempDir.resolve("bus.log"), signingAuthority(signer))) { + PkiId first = submit(bus); + faultTarget.set(first); + faultArmed.set(true); + + PkiException failure = assertThrows(PkiException.class, + () -> bus.reconcile(reconciliation(Optional.empty(), 1, 1, clock))); + + assertTrue(failure.getMessage().contains("code=STORE_DURABILITY_UNCONFIRMED")); + assertEquals(0, signer.submissions.get()); + } + } + + @Test + void acceptedUncertainIntentIsStatusReconciledAcrossRestartAndCancelledAfterDeadline(@TempDir Path tempDir) + throws Exception { + Instant createdAt = Instant.parse("2026-08-04T00:00:00Z"); + MutableClock clock = new MutableClock(createdAt); + AcceptedThenThrowsWorkflow signer = new AcceptedThenThrowsWorkflow(clock); + Path storeRoot = tempDir.resolve("store"); + Path busLog = tempDir.resolve("bus.log"); + PkiId id; + try (FilesystemPkiStore store = new FilesystemPkiStore(storeRoot, FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus(store, signer, busLog, signingAuthority(signer))) { + id = submit(bus, Duration.ofSeconds(10)); + assertThrows(IllegalStateException.class, () -> bus.status(id)); + SignWorkflowStore.Record uncertain = store.getSignRecord(id).orElseThrow(); + assertEquals(SignWorkflowStore.State.INTENT, uncertain.state()); + assertTrue(uncertain.fence() > 0L); + assertEquals(1, signer.submissions.get()); + } + + clock.set(createdAt.plusSeconds(10)); + try (FilesystemPkiStore reopened = new FilesystemPkiStore(storeRoot, FsPkiStoreOptions.defaults(), clock); + PkiSigningBus replayed = new PkiSigningBus(reopened, signer, busLog, signingAuthority(signer))) { + SigningReconciliationResult cancellation = replayed.reconcile( + reconciliation(Optional.empty(), 2, 2, clock)); + assertEquals(1, cancellation.progressed()); + assertEquals(1, signer.submissions.get()); + assertEquals(1, signer.statusReads.get()); + assertEquals(1, signer.cancellations.get()); + + replayed.reconcile(reconciliation(Optional.empty(), 1, 1, clock)); + assertEquals(SignWorkflowStore.State.RETIRED, reopened.getSignRecord(id).orElseThrow().state()); + assertEquals(1, signer.submissions.get()); + } + } + @Test void constructorsRequireExplicitOwningSignAuthority(@TempDir Path tempDir) throws Exception { System.out.println("constructorsRequireExplicitOwningSignAuthority"); @@ -276,6 +538,55 @@ final class PkiSigningBusFailureTest { } } + @Test + void signResultAdmissionAcceptsExactLimitAndRejectsOversizeWrongMissingAndMixed(@TempDir Path tempDir) + throws Exception { + Instant createdAt = Instant.parse("2026-06-07T08:09:10Z"); + MutableClock clock = new MutableClock(createdAt); + ControlledWorkflow signer = new ControlledWorkflow(clock); + try (FilesystemPkiStore store = new FilesystemPkiStore(tempDir.resolve("store"), + FsPkiStoreOptions.defaults(), clock); + PkiSigningBus bus = new PkiSigningBus(store, signer, tempDir.resolve("bus.log"), + signingAuthority(signer))) { + PkiId exact = submit(bus); + assertEquals(AsyncState.RUNNING, bus.status(exact).orElseThrow().state()); + signer.succeedWith(exact, createdAt.plusSeconds(1), new SignatureWorkflow.OperationResult( + Optional.of(new EncodedObject(Encoding.BINARY, new byte[1_048_576])), Optional.empty())); + assertEquals(AsyncState.SUCCEEDED, bus.status(exact).orElseThrow().state()); + + PkiId oversize = submit(bus); + assertEquals(AsyncState.RUNNING, bus.status(oversize).orElseThrow().state()); + signer.succeedWith(oversize, createdAt.plusSeconds(1), new SignatureWorkflow.OperationResult( + Optional.of(new EncodedObject(Encoding.BINARY, new byte[1_048_577])), Optional.empty())); + assertInvalidProviderResult(bus, store, oversize); + + PkiId wrongEncoding = submit(bus); + assertEquals(AsyncState.RUNNING, bus.status(wrongEncoding).orElseThrow().state()); + signer.succeedWith(wrongEncoding, createdAt.plusSeconds(1), new SignatureWorkflow.OperationResult( + Optional.of(new EncodedObject(Encoding.DER, new byte[] { 1 })), Optional.empty())); + assertInvalidProviderResult(bus, store, wrongEncoding); + + PkiId missing = submit(bus); + assertEquals(AsyncState.RUNNING, bus.status(missing).orElseThrow().state()); + signer.succeedWith(missing, createdAt.plusSeconds(1), + new SignatureWorkflow.OperationResult(Optional.empty(), Optional.empty())); + assertInvalidProviderResult(bus, store, missing); + + PkiId mixed = submit(bus); + assertEquals(AsyncState.RUNNING, bus.status(mixed).orElseThrow().state()); + signer.succeedWith(mixed, createdAt.plusSeconds(1), new SignatureWorkflow.OperationResult( + Optional.of(new EncodedObject(Encoding.BINARY, new byte[] { 1 })), Optional.of(true))); + assertInvalidProviderResult(bus, store, mixed); + } + } + + private static void assertInvalidProviderResult(PkiSigningBus bus, FilesystemPkiStore store, PkiId operationId) { + assertEquals(AsyncState.FAILED, bus.status(operationId).orElseThrow().state()); + SignWorkflowStore.Record failed = store.getSignRecord(operationId).orElseThrow(); + assertEquals(Optional.of("PROVIDER_RESULT_INVALID"), failed.detailCode()); + assertEquals(Optional.empty(), failed.result()); + } + @Test void rejectedCancellationReconcilesProviderSuccessBeforeRetirement(@TempDir Path tempDir) throws Exception { CancelRejectedAfterCompletionWorkflow signer = new CancelRejectedAfterCompletionWorkflow(); @@ -515,7 +826,10 @@ final class PkiSigningBusFailureTest { assertEquals(1, signer.submissions.get()); second.get(5, TimeUnit.SECONDS); signer.release.countDown(); - first.get(5, TimeUnit.SECONDS); + java.util.concurrent.ExecutionException late = assertThrows(java.util.concurrent.ExecutionException.class, + () -> first.get(5, TimeUnit.SECONDS)); + assertInstanceOf(IllegalStateException.class, late.getCause()); + assertEquals(SignWorkflowStore.State.INTENT, store.getSignRecord(id).orElseThrow().state()); assertEquals(1, signer.submissions.get()); } } @@ -651,6 +965,41 @@ final class PkiSigningBusFailureTest { return submit(bus, Duration.ofMinutes(5)); } + private static SigningReconciliationRequest reconciliation(Optional cursor, int maximumRecords, + int maximumCalls, Clock clock) { + return new SigningReconciliationRequest(cursor, maximumRecords, maximumCalls, + clock.instant().plusSeconds(30), Duration.ofSeconds(10), () -> false); + } + + private static PkiStore faultingStore(PkiStore delegate, String methodName, + AtomicReference target, AtomicBoolean armed) { + return faultingStore(delegate, methodName, target, armed, + new IllegalStateException("injected candidate-local store failure")); + } + + private static PkiStore faultingStore(PkiStore delegate, String methodName, + AtomicReference target, AtomicBoolean armed, RuntimeException injectedFailure) { + return (PkiStore) Proxy.newProxyInstance(PkiStore.class.getClassLoader(), new Class[] { PkiStore.class }, + (proxy, method, arguments) -> { + if (methodName.equals(method.getName()) && arguments != null && arguments.length > 0 + && target.get() != null && target.get().equals(arguments[0]) + && armed.compareAndSet(true, false)) { + throw injectedFailure; + } + try { + return method.invoke(delegate, arguments); + } catch (InvocationTargetException failure) { + throw failure.getCause(); + } + }); + } + + private static void requestCancellation(SignWorkflowStore store, PkiId id) { + SignWorkflowStore.Record dispatched = store.getSignRecord(id).orElseThrow(); + store.transitionSign(id, dispatched.revision(), dispatched.fence(), SignWorkflowStore.State.CANCELLING, + Optional.of("CANCEL_REQUESTED"), Optional.empty(), Optional.empty()).orElseThrow(); + } + private static PkiId submit(PkiSigningBus bus, Duration ttl) { Principal owner = new Principal("TEST", "owner"); PkiId id = bus.newSubmissionId(); @@ -705,6 +1054,125 @@ final class PkiSigningBusFailureTest { }; } + private static final class ThrowingCancellationWorkflow implements SignatureWorkflow { + private final Clock clock; + private final Map statuses = new java.util.concurrent.ConcurrentHashMap<>(); + private final AtomicInteger statusReads = new AtomicInteger(); + private final AtomicInteger cancellations = new AtomicInteger(); + + private ThrowingCancellationWorkflow(Clock clock) { + this.clock = clock; + } + + @Override + public String id() { + return "throwing-cancellation"; + } + + @Override + public PkiId submitSign(SignRequest request, CallControl control) { + statuses.put(request.submissionId(), + new OperationStatus(State.RUNNING, clock.instant(), Optional.of("RUNNING"), Optional.empty())); + return request.submissionId(); + } + + @Override + public PkiId submitVerify(VerifyRequest request, CallControl control) { + throw new UnsupportedOperationException(); + } + + @Override + public OperationStatus status(PkiId operationId, CallControl control) { + statusReads.incrementAndGet(); + return statuses.get(operationId); + } + + @Override + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { + cancellations.incrementAndGet(); + throw new IllegalStateException("accepted outcome is deliberately uncertain"); + } + + @Override + public Registration register(NotificationSink sink) { + return () -> { }; + } + + @Override + public Set supportedAlgorithms() { + return Set.of("SHA256withRSA"); + } + + @Override + public void close() { + // No owned resources. + } + + private void resetObservations() { + statusReads.set(0); + cancellations.set(0); + } + } + + private static final class AcceptedThenThrowsWorkflow implements SignatureWorkflow { + private final Clock clock; + private final Map statuses = new java.util.concurrent.ConcurrentHashMap<>(); + private final AtomicInteger submissions = new AtomicInteger(); + private final AtomicInteger statusReads = new AtomicInteger(); + private final AtomicInteger cancellations = new AtomicInteger(); + + private AcceptedThenThrowsWorkflow(Clock clock) { + this.clock = clock; + } + + @Override + public String id() { + return "accepted-then-throws"; + } + + @Override + public PkiId submitSign(SignRequest request, CallControl control) { + submissions.incrementAndGet(); + statuses.putIfAbsent(request.submissionId(), + new OperationStatus(State.RUNNING, clock.instant(), Optional.of("RUNNING"), Optional.empty())); + throw new IllegalStateException("accepted before transport failure"); + } + + @Override + public PkiId submitVerify(VerifyRequest request, CallControl control) { + throw new UnsupportedOperationException(); + } + + @Override + public OperationStatus status(PkiId operationId, CallControl control) { + statusReads.incrementAndGet(); + return statuses.get(operationId); + } + + @Override + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { + cancellations.incrementAndGet(); + statuses.put(operationId, + new OperationStatus(State.CANCELLED, clock.instant(), Optional.of("CANCELLED"), Optional.empty())); + return true; + } + + @Override + public Registration register(NotificationSink sink) { + return () -> { }; + } + + @Override + public Set supportedAlgorithms() { + return Set.of("SHA256withRSA"); + } + + @Override + public void close() { + // Shared durable-provider simulation remains available across bus restart. + } + } + private static final class AcceptedDelayedCancellationWorkflow implements SignatureWorkflow { private final Clock clock; private final Map statuses = new java.util.concurrent.ConcurrentHashMap<>(); @@ -725,7 +1193,7 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { submissions.incrementAndGet(); statuses.put(request.submissionId(), new OperationStatus(State.RUNNING, clock.instant(), Optional.of("RUNNING"), Optional.empty())); @@ -733,17 +1201,17 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { throw new UnsupportedOperationException(); } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { return statuses.get(operationId); } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { cancellations.incrementAndGet(); return true; } @@ -794,7 +1262,7 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { submissions.incrementAndGet(); statuses.put(request.submissionId(), new OperationStatus(State.RUNNING, clock.instant(), Optional.of("RUNNING"), Optional.empty())); @@ -802,17 +1270,17 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { throw new UnsupportedOperationException(); } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { return statuses.get(operationId); } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { cancellations.incrementAndGet(); statuses.put(operationId, new OperationStatus(State.CANCELLED, clock.instant(), Optional.of("CANCELLED"), Optional.empty())); @@ -839,6 +1307,10 @@ final class PkiSigningBusFailureTest { private void succeedAt(PkiId operationId, Instant completedAt, byte value) { OperationResult result = new OperationResult( Optional.of(new EncodedObject(Encoding.BINARY, new byte[] { value })), Optional.empty()); + succeedWith(operationId, completedAt, result); + } + + private void succeedWith(PkiId operationId, Instant completedAt, OperationResult result) { statuses.put(operationId, new OperationStatus(State.SUCCEEDED, completedAt, Optional.of("SIGNED"), Optional.of(result))); } @@ -861,7 +1333,7 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { submissions.incrementAndGet(); operationId.set(request.submissionId()); status.set(new OperationStatus(State.RUNNING, Instant.now(), Optional.of("RUNNING"), Optional.empty())); @@ -869,12 +1341,12 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { throw new UnsupportedOperationException(); } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { if (blockStatus.compareAndSet(true, false)) { statusEntered.countDown(); try { @@ -888,7 +1360,7 @@ final class PkiSigningBusFailureTest { } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { OperationStatus current = status.get(); if (current.isTerminal()) { return false; @@ -946,23 +1418,23 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { status.set(new OperationStatus(State.RUNNING, Instant.now(), Optional.of("RUNNING"), Optional.empty())); return request.submissionId(); } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { throw new UnsupportedOperationException(); } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { return status.get(); } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { cancellations.incrementAndGet(); OperationResult result = new OperationResult( Optional.of(new EncodedObject(Encoding.BINARY, new byte[] { 11, 12 })), Optional.empty()); @@ -1001,7 +1473,7 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { submissions.incrementAndGet(); statuses.put(request.submissionId(), new OperationStatus(State.RUNNING, Instant.now(), Optional.of("RUNNING"), Optional.empty())); @@ -1016,17 +1488,17 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { throw new UnsupportedOperationException(); } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { return statuses.get(operationId); } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { return false; } @@ -1077,7 +1549,7 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { submissions.incrementAndGet(); status.set(new OperationStatus(State.RUNNING, Instant.now(), Optional.of("RUNNING"), Optional.empty())); reenterStatus(request.submissionId()); @@ -1085,19 +1557,19 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { throw new UnsupportedOperationException(); } @Override - public OperationStatus status(PkiId requested) { + public OperationStatus status(PkiId requested, CallControl control) { statusReads.incrementAndGet(); reenterStatus(requested); return status.get(); } @Override - public boolean cancel(PkiId requested, long fencingToken, String reason) { + public boolean cancel(PkiId requested, long fencingToken, String reason, CallControl control) { cancellations.incrementAndGet(); reenterStatus(requested); status.set(new OperationStatus(State.CANCELLED, Instant.now(), Optional.of("CANCELLED"), Optional.empty())); @@ -1143,7 +1615,7 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { submissions.incrementAndGet(); statuses.put(request.submissionId(), new OperationStatus(State.RUNNING, Instant.now(), Optional.of("RUNNING"), Optional.empty())); @@ -1161,17 +1633,17 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { throw new UnsupportedOperationException(); } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { return statuses.get(operationId); } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { statuses.put(operationId, new OperationStatus(State.CANCELLED, Instant.now(), Optional.of("CANCELLED"), Optional.empty())); return true; @@ -1205,18 +1677,18 @@ final class PkiSigningBusFailureTest { } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { status.set(new OperationStatus(State.RUNNING, Instant.now(), Optional.of("RUNNING"), Optional.empty())); return request.submissionId(); } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { throw new UnsupportedOperationException(); } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { if (statusReads.incrementAndGet() == 1) { throw new IllegalStateException("injected status failure"); } @@ -1224,7 +1696,7 @@ final class PkiSigningBusFailureTest { } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { status.set(new OperationStatus(State.CANCELLED, Instant.now(), Optional.of("CANCELLED"), Optional.empty())); return true; } diff --git a/pki/src/test/java/zeroecho/pki/impl/core/async/PkiSigningBusOperatorApprovalTest.java b/pki/src/test/java/zeroecho/pki/impl/core/async/PkiSigningBusOperatorApprovalTest.java index 201dad2..b4003fe 100644 --- a/pki/src/test/java/zeroecho/pki/impl/core/async/PkiSigningBusOperatorApprovalTest.java +++ b/pki/src/test/java/zeroecho/pki/impl/core/async/PkiSigningBusOperatorApprovalTest.java @@ -85,10 +85,12 @@ public final class PkiSigningBusOperatorApprovalTest { "test", 1L, access, keyRef, "SHA256withRSA", new ImmutableByteContent(new byte[] { 1 }), Optional.of(Encoding.BINARY), Optional.of(Instant.EPOCH)); - PkiId operationId = signer.submitSign(request); + SignatureWorkflow.CallControl control = new SignatureWorkflow.CallControl(Instant.MAX, + zeroecho.core.io.CancellationSignal.NONE); + PkiId operationId = signer.submitSign(request, control); signer.approve(operationId); - SignatureWorkflow.OperationStatus status = signer.status(operationId); + SignatureWorkflow.OperationStatus status = signer.status(operationId, control); assertEquals(SignatureWorkflow.State.EXPIRED, status.state()); assertTrue(status.result().isEmpty()); } diff --git a/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibKeyRefParsingTest.java b/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibKeyRefParsingTest.java index 1504d9c..c01170a 100644 --- a/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibKeyRefParsingTest.java +++ b/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibKeyRefParsingTest.java @@ -80,8 +80,10 @@ public final class ZeroEchoLibKeyRefParsingTest { new KeyRef("zeroecho-lib:abc"), "ECDSA", new ImmutableByteContent(new byte[] { 0x01 }), Optional.of(Encoding.BINARY), Optional.of(Instant.now())); - PkiId opId = wf.submitSign(req); - SignatureWorkflow.OperationStatus st = wf.status(opId); + SignatureWorkflow.CallControl control = new SignatureWorkflow.CallControl(Instant.MAX, + CancellationSignal.NONE); + PkiId opId = wf.submitSign(req, control); + SignatureWorkflow.OperationStatus st = wf.status(opId, control); System.out.println("...state=" + st.state()); System.out.println("...detailCode=" + st.detailCode().orElse("")); @@ -109,8 +111,10 @@ public final class ZeroEchoLibKeyRefParsingTest { Optional.of(new EncodedObject(Encoding.BINARY, new byte[] { 0x03 })), // unsupported form Optional.of(Instant.now()), CancellationSignal.NONE); - PkiId opId = wf.submitVerify(req); - SignatureWorkflow.OperationStatus st = wf.status(opId); + SignatureWorkflow.CallControl control = new SignatureWorkflow.CallControl(Instant.MAX, + CancellationSignal.NONE); + PkiId opId = wf.submitVerify(req, control); + SignatureWorkflow.OperationStatus st = wf.status(opId, control); System.out.println("...state=" + st.state()); System.out.println("...detailCode=" + st.detailCode().orElse("")); @@ -130,7 +134,8 @@ public final class ZeroEchoLibKeyRefParsingTest { TestKeyringUnlocks.provider())) { PkiId unknown = new PkiId("00000000-0000-0000-0000-000000000000"); - SignatureWorkflow.OperationStatus st = wf.status(unknown); + SignatureWorkflow.OperationStatus st = wf.status(unknown, + new SignatureWorkflow.CallControl(Instant.MAX, CancellationSignal.NONE)); System.out.println("...state=" + st.state()); System.out.println("...detailCode=" + st.detailCode().orElse("")); diff --git a/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowPersistenceTest.java b/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowPersistenceTest.java index 046bdcd..31b81b3 100644 --- a/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowPersistenceTest.java +++ b/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowPersistenceTest.java @@ -143,16 +143,16 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { PkiId signId = SigningSubmissionId.create(NAMESPACE, now, new SecureRandom()).id(); SignatureWorkflow.SignRequest signRequest = request(signId, 1L, message, new KeyRef("zeroecho-lib:test.prv"), Optional.empty()); - workflow.submitSign(signRequest); - EncodedObject signature = workflow.status(signId).result().orElseThrow().signature().orElseThrow(); + workflow.submitSign(signRequest, control()); + EncodedObject signature = workflow.status(signId, control()).result().orElseThrow().signature().orElseThrow(); assertTrue(signature.bytes().length > 0); AccessContext access = signRequest.accessContext(); SignatureWorkflow.VerifyRequest verifyRequest = new SignatureWorkflow.VerifyRequest(access, "SHA256withRSA", new ImmutableByteContent(message), signature, Optional.of(new KeyRef("zeroecho-lib:test.pub")), Optional.empty(), Optional.empty(), CancellationSignal.NONE); - PkiId verifyId = workflow.submitVerify(verifyRequest); - assertEquals(Optional.of(true), workflow.status(verifyId).result().orElseThrow().verified()); + PkiId verifyId = workflow.submitVerify(verifyRequest, control()); + assertEquals(Optional.of(true), workflow.status(verifyId, control()).result().orElseThrow().verified()); byte[] invalidBytes = signature.bytes(); invalidBytes[0] ^= 0x01; @@ -160,8 +160,8 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { "SHA256withRSA", new ImmutableByteContent(message), new EncodedObject(Encoding.BINARY, invalidBytes), Optional.of(new KeyRef("zeroecho-lib:test.pub")), Optional.empty(), Optional.empty(), CancellationSignal.NONE); - PkiId invalidId = workflow.submitVerify(invalidRequest); - assertEquals(Optional.of(false), workflow.status(invalidId).result().orElseThrow().verified()); + PkiId invalidId = workflow.submitVerify(invalidRequest, control()); + assertEquals(Optional.of(false), workflow.status(invalidId, control()).result().orElseThrow().verified()); assertTrue(cleared.stream().anyMatch(value -> "sign-result-copy".equals(value.category()))); assertTrue(cleared.stream().anyMatch(value -> "verify-signature".equals(value.category()))); @@ -216,8 +216,8 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { SignatureWorkflow.SignRequest canonical = request(canonicalId, 1L, new byte[] { 1, 2, 3 }, new KeyRef("zeroecho-lib:test.prv"), Optional.empty(), BootstrapAlgorithmIdentities.RSA_PKCS1_SHA256.canonicalForm()); - workflow.submitSign(canonical); - assertEquals(SignatureWorkflow.State.SUCCEEDED, workflow.status(canonicalId).state()); + workflow.submitSign(canonical, control()); + assertEquals(SignatureWorkflow.State.SUCCEEDED, workflow.status(canonicalId, control()).state()); assertSigningFailure(workflow, now, "SHA1withRSA", new KeyRef("zeroecho-lib:test.prv"), ZeroEchoLibSignatureWorkflow.DC_INVALID_ALGORITHM_ID); @@ -262,11 +262,11 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { throw new IllegalStateException("CANCELLATION_RECHECK_SENTINEL"); } return true; - })); + }), control()); assertEquals(1, cancellationChecks.get()); - assertEquals(SignatureWorkflow.State.CANCELLED, workflow.status(beforeId).state()); + assertEquals(SignatureWorkflow.State.CANCELLED, workflow.status(beforeId, control()).state()); assertEquals(Optional.of(ZeroEchoLibSignatureWorkflow.DC_CANCELLED), - workflow.status(beforeId).detailCode()); + workflow.status(beforeId, control()).detailCode()); AtomicBoolean armed = new AtomicBoolean(); AtomicBoolean cancelled = new AtomicBoolean(); @@ -280,10 +280,10 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { SignatureWorkflow.SignRequest streamingRequest = request(streamingId, 1L, streaming, new KeyRef("zeroecho-lib:test.prv"), "SHA256withRSA", cancelled::get); armed.set(true); - workflow.submitSign(streamingRequest); - assertEquals(SignatureWorkflow.State.CANCELLED, workflow.status(streamingId).state()); + workflow.submitSign(streamingRequest, control()); + assertEquals(SignatureWorkflow.State.CANCELLED, workflow.status(streamingId, control()).state()); assertEquals(Optional.of(ZeroEchoLibSignatureWorkflow.DC_CANCELLED), - workflow.status(streamingId).detailCode()); + workflow.status(streamingId, control()).detailCode()); AtomicBoolean failReads = new AtomicBoolean(); RepeatableContent interruptedIo = new RepeatableContent() { @@ -319,10 +319,10 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { SignatureWorkflow.SignRequest ioRequest = request(ioId, 1L, interruptedIo, new KeyRef("zeroecho-lib:test.prv"), "SHA256withRSA", CancellationSignal.NONE); failReads.set(true); - workflow.submitSign(ioRequest); - assertEquals(SignatureWorkflow.State.FAILED, workflow.status(ioId).state()); + workflow.submitSign(ioRequest, control()); + assertEquals(SignatureWorkflow.State.FAILED, workflow.status(ioId, control()).state()); assertEquals(Optional.of(ZeroEchoLibSignatureWorkflow.DC_KEYRING_IO_ERROR), - workflow.status(ioId).detailCode()); + workflow.status(ioId, control()).detailCode()); assertEquals(3, terminalPublications.get()); } System.out.println("signingCancellationIsTerminalOnlyForRequestedCancellation...ok"); @@ -359,10 +359,10 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { })) { PkiId id = SigningSubmissionId.create(NAMESPACE, now, new SecureRandom()).id(); workflow.submitSign(request(id, 1L, new byte[] { 6, 7, 8 }, new KeyRef("zeroecho-lib:test.prv"), - Optional.empty())); - assertEquals(SignatureWorkflow.State.FAILED, workflow.status(id).state()); + Optional.empty()), control()); + assertEquals(SignatureWorkflow.State.FAILED, workflow.status(id, control()).state()); assertEquals(Optional.of(ZeroEchoLibSignatureWorkflow.DC_CRYPTO_FAILURE), - workflow.status(id).detailCode()); + workflow.status(id, control()).detailCode()); assertEquals(1, terminalPublications.get()); } System.out.println("providerFailureTerminalizesExactlyOnce...ok"); @@ -376,11 +376,12 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { PkiId id = SigningSubmissionId.create(NAMESPACE, now, new SecureRandom()).id(); byte[] payload = new byte[] { 91, 92, 93 }; try (ZeroEchoLibSignatureWorkflow workflow = workflow(root, operations, clock)) { - workflow.submitSign(request(id, 1L, payload)); + workflow.submitSign(request(id, 1L, payload), control()); } Path record = onlyOperationRecord(operations); byte[] encoded = Files.readAllBytes(record); + byte[] valid = encoded.clone(); ByteBuffer.wrap(encoded).putInt(99); Files.write(record, encoded); @@ -388,6 +389,11 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { () -> workflow(root, operations, clock)); assertFalse(failure.toString().contains(id.value())); assertFalse(failure.toString().contains(java.util.Base64.getEncoder().encodeToString(payload))); + + Files.write(record, valid); + try (ZeroEchoLibSignatureWorkflow reopened = workflow(root, operations, clock)) { + assertEquals(SignatureWorkflow.State.FAILED, reopened.status(id, control()).state()); + } } @Test @@ -406,8 +412,8 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { try (ZeroEchoLibSignatureWorkflow workflow = workflow(root, root.resolve("on-time"), keyring, onTimeClock)) { SignatureWorkflow.SignRequest onTime = request(onTimeId, 1L, new byte[] { 1 }, new KeyRef("zeroecho-lib:test.prv"), Optional.of(base.plusNanos(1))); - workflow.submitSign(onTime); - assertEquals(SignatureWorkflow.State.SUCCEEDED, workflow.status(onTimeId).state()); + workflow.submitSign(onTime, control()); + assertEquals(SignatureWorkflow.State.SUCCEEDED, workflow.status(onTimeId, control()).state()); } PkiId exactId = SigningSubmissionId.create(NAMESPACE, base, new SecureRandom()).id(); @@ -415,9 +421,9 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { try (ZeroEchoLibSignatureWorkflow workflow = workflow(root, root.resolve("exact"), keyring, exactClock)) { SignatureWorkflow.SignRequest exact = request(exactId, 1L, new byte[] { 2 }, new KeyRef("zeroecho-lib:test.prv"), Optional.of(base)); - workflow.submitSign(exact); - assertEquals(SignatureWorkflow.State.EXPIRED, workflow.status(exactId).state()); - assertEquals(Optional.empty(), workflow.status(exactId).result()); + workflow.submitSign(exact, control()); + assertEquals(SignatureWorkflow.State.EXPIRED, workflow.status(exactId, control()).state()); + assertEquals(Optional.empty(), workflow.status(exactId, control()).result()); } PkiId crossingId = SigningSubmissionId.create(NAMESPACE, base, new SecureRandom()).id(); @@ -427,14 +433,44 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { try (ZeroEchoLibSignatureWorkflow workflow = workflow(root, crossingOperations, keyring, crossingClock)) { SignatureWorkflow.SignRequest crossing = request(crossingId, 1L, new byte[] { 3 }, new KeyRef("zeroecho-lib:test.prv"), Optional.of(deadline)); - workflow.submitSign(crossing); - assertEquals(SignatureWorkflow.State.EXPIRED, workflow.status(crossingId).state()); - assertEquals(Optional.empty(), workflow.status(crossingId).result()); + workflow.submitSign(crossing, control()); + assertEquals(SignatureWorkflow.State.EXPIRED, workflow.status(crossingId, control()).state()); + assertEquals(Optional.empty(), workflow.status(crossingId, control()).result()); } try (ZeroEchoLibSignatureWorkflow restarted = workflow(root, crossingOperations, keyring, Clock.fixed(deadline, ZoneOffset.UTC))) { - assertEquals(SignatureWorkflow.State.EXPIRED, restarted.status(crossingId).state()); - assertEquals(Optional.empty(), restarted.status(crossingId).result()); + assertEquals(SignatureWorkflow.State.EXPIRED, restarted.status(crossingId, control()).state()); + assertEquals(Optional.empty(), restarted.status(crossingId, control()).result()); + } + } + + @Test + void callDeadlineAfterDurableAcceptanceTerminalizesBeforePropagation(@TempDir Path root) throws Exception { + Instant acceptedAt = Instant.parse("2026-02-03T04:05:06Z"); + Instant callDeadline = acceptedAt.plusSeconds(1); + LatchClock clock = new LatchClock(acceptedAt); + Path operations = root.resolve("post-accept-call-deadline"); + PkiId operationId = SigningSubmissionId.create(NAMESPACE, acceptedAt, new SecureRandom()).id(); + SignatureWorkflow.CallControl expiring = new SignatureWorkflow.CallControl(callDeadline, + CancellationSignal.NONE); + + try (ZeroEchoLibSignatureWorkflow workflow = workflow(root, operations, clock); + SignatureWorkflow.Registration ignored = workflow.register((id, status) -> { + if (operationId.equals(id) && status.state() == SignatureWorkflow.State.RUNNING) { + clock.set(callDeadline); + } + })) { + assertThrows(IllegalStateException.class, + () -> workflow.submitSign(request(operationId, 1L, new byte[] { 4 }), expiring)); + SignatureWorkflow.OperationStatus terminal = workflow.status(operationId, control()); + assertEquals(SignatureWorkflow.State.EXPIRED, terminal.state()); + assertEquals(Optional.empty(), terminal.result()); + } + + try (ZeroEchoLibSignatureWorkflow restarted = workflow(root, operations, + Clock.fixed(callDeadline, ZoneOffset.UTC))) { + assertEquals(SignatureWorkflow.State.EXPIRED, restarted.status(operationId, control()).state()); + assertEquals(Optional.empty(), restarted.status(operationId, control()).result()); } } @@ -469,28 +505,30 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { SignatureWorkflow.SignRequest request = request(id, 2L, new byte[] { 1 }); try (ZeroEchoLibSignatureWorkflow workflow = workflow(root, operations, clock)) { - assertEquals(id, workflow.submitSign(request)); - assertEquals(SignatureWorkflow.State.FAILED, workflow.status(id).state()); + assertEquals(id, workflow.submitSign(request, control())); + assertEquals(SignatureWorkflow.State.FAILED, workflow.status(id, control()).state()); } try (ZeroEchoLibSignatureWorkflow restarted = workflow(root, operations, clock)) { - assertEquals(SignatureWorkflow.State.FAILED, restarted.status(id).state()); - assertEquals(id, restarted.submitSign(request)); - assertThrows(IllegalStateException.class, () -> restarted.submitSign(request(id, 1L, new byte[] { 1 }))); - assertThrows(IllegalStateException.class, () -> restarted.submitSign(request(id, 3L, new byte[] { 2 }))); + assertEquals(SignatureWorkflow.State.FAILED, restarted.status(id, control()).state()); + assertEquals(id, restarted.submitSign(request, control())); + assertThrows(IllegalStateException.class, + () -> restarted.submitSign(request(id, 1L, new byte[] { 1 }), control())); + assertThrows(IllegalStateException.class, + () -> restarted.submitSign(request(id, 3L, new byte[] { 2 }), control())); } forcePersistedStateCode(operations, 50, 30); try (ZeroEchoLibSignatureWorkflow recovered = workflow(root, operations, clock)) { - assertEquals(SignatureWorkflow.State.FAILED, recovered.status(id).state()); - assertEquals("RECOVERY_INCOMPLETE", recovered.status(id).detailCode().orElseThrow()); + assertEquals(SignatureWorkflow.State.FAILED, recovered.status(id, control()).state()); + assertEquals("RECOVERY_INCOMPLETE", recovered.status(id, control()).detailCode().orElseThrow()); } Clock expired = Clock.fixed(now.plus(Duration.ofDays(90)), ZoneOffset.UTC); try (ZeroEchoLibSignatureWorkflow restarted = workflow(root, operations, expired)) { - assertEquals(SignatureWorkflow.State.EXPIRED, restarted.status(id).state()); + assertEquals(SignatureWorkflow.State.EXPIRED, restarted.status(id, control()).state()); PkiId expiredId = SigningSubmissionId.create(NAMESPACE, now, new SecureRandom()).id(); assertThrows(IllegalArgumentException.class, - () -> restarted.submitSign(request(expiredId, 1L, new byte[] { 3 }))); + () -> restarted.submitSign(request(expiredId, 1L, new byte[] { 3 }), control())); } } @@ -508,7 +546,7 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { ExecutorService executor = Executors.newFixedThreadPool(3); SignatureWorkflow.Registration registration = workflow.register((operationId, status) -> { if (firstId.equals(operationId) && status.state() == SignatureWorkflow.State.RUNNING) { - assertEquals(SignatureWorkflow.State.RUNNING, workflow.status(operationId).state()); + assertEquals(SignatureWorkflow.State.RUNNING, workflow.status(operationId, control()).state()); callbackEntered.countDown(); try { callbackRelease.await(); @@ -518,15 +556,21 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { } } })) { - Future firstFuture = executor.submit(() -> workflow.submitSign(first)); + Future firstFuture = executor.submit(() -> workflow.submitSign(first, control())); assertEquals(true, callbackEntered.await(5, TimeUnit.SECONDS)); - assertEquals(firstId, executor.submit(() -> workflow.submitSign(first)).get(5, TimeUnit.SECONDS)); - assertEquals(secondId, executor.submit(() -> workflow.submitSign(second)).get(5, TimeUnit.SECONDS)); - assertEquals(true, workflow.cancel(firstId, 2L, "test cancellation")); + assertEquals(firstId, executor.submit(() -> workflow.submitSign(first, control())).get(5, TimeUnit.SECONDS)); + assertEquals(secondId, executor.submit(() -> workflow.submitSign(second, control())).get(5, TimeUnit.SECONDS)); + assertEquals(true, workflow.cancel(firstId, 2L, "test cancellation", control())); callbackRelease.countDown(); assertEquals(firstId, firstFuture.get(5, TimeUnit.SECONDS)); - assertEquals(SignatureWorkflow.State.CANCELLED, workflow.status(firstId).state()); - assertEquals(false, workflow.cancel(firstId, 1L, "stale")); + assertEquals(SignatureWorkflow.State.CANCELLED, workflow.status(firstId, control()).state()); + assertTrue(workflow.cancel(firstId, 2L, "test cancellation", control())); + assertTrue(workflow.cancel(firstId, 2L, "different cancellation", control())); + assertEquals(false, workflow.cancel(firstId, 1L, "stale", control())); + } + try (ZeroEchoLibSignatureWorkflow restarted = workflow(root, root.resolve("operations"), clock)) { + assertTrue(restarted.cancel(firstId, 2L, "different cancellation after restart", control())); + assertFalse(restarted.cancel(firstId, 1L, "test cancellation", control())); } } @@ -539,10 +583,10 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { try (ZeroEchoLibSignatureWorkflow workflow = workflow(root, operations, clock); ExecutorService executor = Executors.newFixedThreadPool(2)) { clock.arm(high); - Future highRead = executor.submit(() -> workflow.status(new PkiId("unknown-high"))); + Future highRead = executor.submit(() -> workflow.status(new PkiId("unknown-high"), control())); assertEquals(true, clock.observed.await(5, TimeUnit.SECONDS)); clock.set(base.minusSeconds(60)); - Future rollbackRead = executor.submit(() -> workflow.status(new PkiId("unknown-low"))); + Future rollbackRead = executor.submit(() -> workflow.status(new PkiId("unknown-low"), control())); clock.release.countDown(); highRead.get(5, TimeUnit.SECONDS); rollbackRead.get(5, TimeUnit.SECONDS); @@ -551,7 +595,7 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { } clock.set(base.minusSeconds(120)); try (ZeroEchoLibSignatureWorkflow restarted = workflow(root, operations, clock)) { - restarted.status(new PkiId("unknown-restart")); + restarted.status(new PkiId("unknown-restart"), control()); assertEquals(high.toEpochMilli(), Long.parseLong(Files.readString(operations.resolve("TIME_WATERMARK")).trim())); } @@ -644,9 +688,13 @@ final class ZeroEchoLibSignatureWorkflowPersistenceTest { private static void assertSigningFailure(ZeroEchoLibSignatureWorkflow workflow, Instant now, String algorithmId, KeyRef keyRef, String expectedDetailCode) { PkiId id = SigningSubmissionId.create(NAMESPACE, now, new SecureRandom()).id(); - workflow.submitSign(request(id, 1L, new byte[] { 9 }, keyRef, Optional.empty(), algorithmId)); - assertEquals(SignatureWorkflow.State.FAILED, workflow.status(id).state()); - assertEquals(Optional.of(expectedDetailCode), workflow.status(id).detailCode()); + workflow.submitSign(request(id, 1L, new byte[] { 9 }, keyRef, Optional.empty(), algorithmId), control()); + assertEquals(SignatureWorkflow.State.FAILED, workflow.status(id, control()).state()); + assertEquals(Optional.of(expectedDetailCode), workflow.status(id, control()).detailCode()); + } + + private static SignatureWorkflow.CallControl control() { + return new SignatureWorkflow.CallControl(Instant.MAX, CancellationSignal.NONE); } private static boolean contains(byte[] haystack, byte[] needle) { diff --git a/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowVerifyEncodedEcdsaTest.java b/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowVerifyEncodedEcdsaTest.java index 42c5f95..2fa87d8 100644 --- a/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowVerifyEncodedEcdsaTest.java +++ b/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowVerifyEncodedEcdsaTest.java @@ -102,8 +102,10 @@ public final class ZeroEchoLibSignatureWorkflowVerifyEncodedEcdsaTest { new ImmutableByteContent(payload), sigObj, Optional.empty(), Optional.of(spkiObj), Optional.of(Instant.now().plusSeconds(5)), CancellationSignal.NONE); - PkiId opId = wf.submitVerify(vr); - SignatureWorkflow.OperationStatus st = wf.status(opId); + SignatureWorkflow.CallControl control = new SignatureWorkflow.CallControl(Instant.MAX, + CancellationSignal.NONE); + PkiId opId = wf.submitVerify(vr, control); + SignatureWorkflow.OperationStatus st = wf.status(opId, control); assertTrue(st.state() == SignatureWorkflow.State.SUCCEEDED); assertTrue(st.result().isPresent()); assertTrue(st.result().get().verified().isPresent()); diff --git a/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowVerifyEncodedTest.java b/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowVerifyEncodedTest.java index 1c51608..75649d9 100644 --- a/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowVerifyEncodedTest.java +++ b/pki/src/test/java/zeroecho/pki/impl/crypto/zeroecholib/ZeroEchoLibSignatureWorkflowVerifyEncodedTest.java @@ -96,8 +96,10 @@ public final class ZeroEchoLibSignatureWorkflowVerifyEncodedTest { new ImmutableByteContent(payload), sigObj, Optional.empty(), Optional.of(spkiObj), Optional.of(Instant.now().plusSeconds(5)), CancellationSignal.NONE); - PkiId opId = wf.submitVerify(vr); - SignatureWorkflow.OperationStatus st = wf.status(opId); + SignatureWorkflow.CallControl control = new SignatureWorkflow.CallControl(Instant.MAX, + CancellationSignal.NONE); + PkiId opId = wf.submitVerify(vr, control); + SignatureWorkflow.OperationStatus st = wf.status(opId, control); assertTrue(st.state() == SignatureWorkflow.State.SUCCEEDED); assertTrue(st.result().isPresent()); assertTrue(st.result().get().verified().isPresent()); diff --git a/pki/src/test/java/zeroecho/pki/impl/fs/FilesystemSignWorkflowStoreTest.java b/pki/src/test/java/zeroecho/pki/impl/fs/FilesystemSignWorkflowStoreTest.java index 1622b61..d05c507 100644 --- a/pki/src/test/java/zeroecho/pki/impl/fs/FilesystemSignWorkflowStoreTest.java +++ b/pki/src/test/java/zeroecho/pki/impl/fs/FilesystemSignWorkflowStoreTest.java @@ -76,8 +76,10 @@ import zeroecho.pki.api.audit.Principal; import zeroecho.pki.api.audit.Purpose; import zeroecho.pki.api.orch.SigningSubmissionId; import zeroecho.core.io.CancellationSignal; +import zeroecho.core.io.ImmutableByteContent; import zeroecho.core.io.RepeatableContent; import zeroecho.pki.impl.core.async.PkiSigningBus; +import zeroecho.pki.spi.store.MetadataCommitResult; import zeroecho.pki.spi.store.SignWorkflowStore; import zeroecho.pki.spi.store.MetadataKey; import zeroecho.pki.spi.store.MetadataSnapshot; @@ -573,6 +575,129 @@ final class FilesystemSignWorkflowStoreTest { System.out.println("...ok"); } + @Test + void signingPagesAndRetryBackoffAreBoundedAndRestartSafe(@TempDir Path root) throws Exception { + Instant createdAt = Instant.parse("2026-01-02T03:04:05.123Z"); + MutableClock clock = new MutableClock(createdAt); + try (FilesystemPkiStore store = new FilesystemPkiStore(root, FsPkiStoreOptions.defaults(), clock)) { + PkiId first = persistIntent(store, createdAt, 1); + PkiId second = persistIntent(store, createdAt, 2); + PkiId third = persistIntent(store, createdAt, 3); + List expected = java.util.stream.Stream.of(first, second, third).map(PkiId::value).sorted().toList(); + + SignWorkflowStore.Page pageOne = store.pageSignRecords(Optional.empty(), 2, CancellationSignal.NONE); + assertEquals(expected.subList(0, 2), pageOne.records().stream() + .map(record -> record.submissionId().value()).toList()); + assertFalse(pageOne.endReached()); + SignWorkflowStore.Page pageTwo = store.pageSignRecords(pageOne.nextCursor(), 2, CancellationSignal.NONE); + assertEquals(List.of(expected.get(2)), pageTwo.records().stream() + .map(record -> record.submissionId().value()).toList()); + assertTrue(pageTwo.endReached()); + SignWorkflowStore.Page maximumCursor = store.pageSignRecords(Optional.of("~".repeat(4096)), 1, + CancellationSignal.NONE); + assertEquals(0, maximumCursor.examined()); + assertTrue(maximumCursor.endReached()); + + SignWorkflowStore.Record retry = store.tryClaimSign(first, 0L, Duration.ofSeconds(30)).orElseThrow(); + long[] delays = { 2L, 4L, 8L, 16L, 30L, 30L }; + for (int index = 0; index < delays.length; index++) { + Instant deferredAt = clock.instant(); + retry = store.deferSignReconciliation(first, retry.revision(), retry.fence(), + SignWorkflowStore.ReconciliationFailureClass.SUBMISSION_UNCERTAIN).orElseThrow(); + assertEquals(index + 1, retry.failureCount()); + assertEquals(deferredAt.plusSeconds(delays[index]), retry.nextEligibleAt().orElseThrow()); + clock.set(deferredAt.plusSeconds(1)); + } + retry = store.clearSignReconciliation(first, retry.revision(), retry.fence()).orElseThrow(); + assertEquals(0, retry.failureCount()); + assertEquals(Optional.empty(), retry.nextEligibleAt()); + } + } + + @Test + void malformedCandidateAdvancesCursorWithoutStarvingLaterRecords(@TempDir Path root) throws Exception { + Instant createdAt = Instant.parse("2026-01-02T03:04:05.123Z"); + List ordered; + try (FilesystemPkiStore store = new FilesystemPkiStore(root, FsPkiStoreOptions.defaults(), + Clock.fixed(createdAt, java.time.ZoneOffset.UTC))) { + ordered = java.util.stream.Stream.of( + persistIntent(store, createdAt, 1), + persistIntent(store, createdAt, 2), + persistIntent(store, createdAt, 3)) + .map(PkiId::value).sorted().toList(); + } + + Path metadataLog = root.resolve("metadata/transactions.log"); + MetadataKey malformed = new MetadataKey("io.zeroecho.pki.signing-record", ordered.get(0)); + try (PosixTransactionalMetadataStore metadata = PosixTransactionalMetadataStore.open(metadataLog); + MetadataSnapshot snapshot = metadata.snapshot()) { + long revision; + try (MetadataSnapshot.Record record = snapshot.get(malformed).orElseThrow()) { + revision = record.recordRevision(); + } + try (MetadataTransaction transaction = metadata.beginTransaction()) { + transaction.replace(malformed, revision, new ImmutableByteContent(new byte[] { 1, 2, 3 }), + CancellationSignal.NONE); + assertEquals(MetadataCommitResult.Outcome.COMMITTED, transaction.commit().outcome()); + } + } + + try (FilesystemPkiStore reopened = new FilesystemPkiStore(root, FsPkiStoreOptions.defaults(), + Clock.fixed(createdAt, java.time.ZoneOffset.UTC))) { + SignWorkflowStore.Page page = reopened.pageSignRecords(Optional.empty(), 3, CancellationSignal.NONE); + assertEquals(3, page.examined()); + assertEquals(1, page.failures()); + assertEquals(ordered.subList(1, 3), page.records().stream() + .map(record -> record.submissionId().value()).toList()); + assertEquals(Optional.of(ordered.get(2)), page.nextCursor()); + assertTrue(page.endReached()); + } + } + + @Test + void farFutureRetryEligibilityIsRejectedAsCorrupt(@TempDir Path root) throws Exception { + Instant createdAt = Instant.parse("2026-01-02T03:04:05.123Z"); + MutableClock clock = new MutableClock(createdAt); + PkiId id; + SignWorkflowStore.Record corrupted; + try (FilesystemPkiStore store = new FilesystemPkiStore(root, FsPkiStoreOptions.defaults(), clock)) { + id = persistClaimed(store, createdAt, 4); + SignWorkflowStore.Record claimed = store.getSignRecord(id).orElseThrow(); + corrupted = new SignWorkflowStore.Record(claimed.submissionId(), claimed.namespace(), + claimed.fingerprint(), claimed.owner(), claimed.createdAt(), claimed.deadline(), claimed.request(), + claimed.state(), claimed.revision(), claimed.fence(), claimed.leaseUntil(), claimed.detailCode(), + claimed.result(), claimed.providerUpdatedAt(), 1, Optional.of(createdAt.plusSeconds(31)), + Optional.of(SignWorkflowStore.ReconciliationFailureClass.STATUS_UNAVAILABLE)); + } + writeRawCurrentRecord(root, id, corrupted); + + try (FilesystemPkiStore reopened = new FilesystemPkiStore(root, FsPkiStoreOptions.defaults(), clock)) { + IllegalStateException failure = assertThrows(IllegalStateException.class, + () -> reopened.getSignRecord(id)); + assertTrue(failure.getMessage().contains("code=RECONCILIATION_ELIGIBILITY_INVALID")); + assertFalse(failure.toString().contains(id.value())); + } + } + + @Test + void previousSigningRecordCodecDecodesWithEmptyRetryMetadata(@TempDir Path root) throws Exception { + Instant createdAt = Instant.parse("2026-01-02T03:04:05.123Z"); + try (FilesystemPkiStore store = new FilesystemPkiStore(root, FsPkiStoreOptions.defaults())) { + PkiId id = SigningSubmissionId + .create(store.signingNamespace() + ".test-signer", createdAt, new SecureRandom()).id(); + SignWorkflowStore.Record record = intent(store, id, createdAt, + new EncodedObject(Encoding.BINARY, new byte[] { 7 }), TEST_ALGORITHM); + byte[] current = FsCodec.encode(FsCodec.SIGN_WORKFLOW_RECORD, record); + byte[] previous = Arrays.copyOf(current, current.length - 15); + previous[Integer.BYTES] = 4; + SignWorkflowStore.Record decoded = FsCodec.decode(FsCodec.SIGN_WORKFLOW_RECORD, previous, + store.stagedContent()); + assertEquals(0, decoded.failureCount()); + assertEquals(Optional.empty(), decoded.nextEligibleAt()); + assertEquals(Optional.empty(), decoded.reconciliationFailureClass()); + } + } + private static SignWorkflowStore.Record intent(FilesystemPkiStore store, PkiId id, Instant createdAt, EncodedObject request, String algorithmId) { @@ -582,7 +707,7 @@ final class FilesystemSignWorkflowStoreTest { String fingerprint = continuation.semanticFingerprint(namespace, deadline); return new SignWorkflowStore.Record(id, namespace, fingerprint, TEST_OWNER, createdAt, deadline, continuation.encode(), SignWorkflowStore.State.INTENT, 0L, 0L, Optional.empty(), Optional.of("INTENT"), - Optional.empty(), Optional.empty()); + Optional.empty(), Optional.empty(), 0, Optional.empty(), Optional.empty()); } private static PkiId persistIntent(FilesystemPkiStore store, Instant createdAt, int marker) { @@ -637,7 +762,14 @@ final class FilesystemSignWorkflowStoreTest { Optional detailCode, Optional result, Optional providerUpdatedAt) { return new SignWorkflowStore.Record(source.submissionId(), source.namespace(), fingerprint, source.owner(), source.createdAt(), source.deadline(), request, state, revision, fence, leaseUntil, detailCode, result, - providerUpdatedAt); + providerUpdatedAt, state == SignWorkflowStore.State.INTENT + || state == SignWorkflowStore.State.DISPATCHED + || state == SignWorkflowStore.State.CANCELLING ? source.failureCount() : 0, + state == SignWorkflowStore.State.INTENT || state == SignWorkflowStore.State.DISPATCHED + || state == SignWorkflowStore.State.CANCELLING ? source.nextEligibleAt() : Optional.empty(), + state == SignWorkflowStore.State.INTENT || state == SignWorkflowStore.State.DISPATCHED + || state == SignWorkflowStore.State.CANCELLING + ? source.reconciliationFailureClass() : Optional.empty()); } private static String flipFingerprint(String fingerprint) { diff --git a/pki/src/test/java/zeroecho/pki/impl/fs/MetadataStateIndexTest.java b/pki/src/test/java/zeroecho/pki/impl/fs/MetadataStateIndexTest.java index b5d954b..b8c96bd 100644 --- a/pki/src/test/java/zeroecho/pki/impl/fs/MetadataStateIndexTest.java +++ b/pki/src/test/java/zeroecho/pki/impl/fs/MetadataStateIndexTest.java @@ -170,6 +170,29 @@ final class MetadataStateIndexTest { System.out.println("...ok"); } + @Test + void updatesAvoidFullIndexCopiesWithoutPinnedSnapshotAndPreservePinnedView() throws Exception { + MetadataStateIndex index = new MetadataStateIndex(); + for (int revision = 1; revision <= 100; revision++) { + index.applyCommitted(revision, List.of(create(key("key-" + revision), revision, 1L))); + } + assertEquals(0L, index.snapshotCopyCount()); + + MetadataStateIndex.SnapshotView pinned = index.snapshot(); + assertEquals(100, pinned.records().size()); + index.applyCommitted(101L, List.of(create(key("key-101"), 101L, 1L))); + assertEquals(1L, index.snapshotCopyCount()); + assertEquals(100, pinned.records().size()); + index.applyCommitted(102L, List.of(create(key("key-102"), 102L, 1L))); + assertEquals(1L, index.snapshotCopyCount()); + assertEquals(100, pinned.records().size()); + pinned.release().run(); + + index.applyCommitted(103L, List.of(create(key("key-103"), 103L, 1L))); + assertEquals(1L, index.snapshotCopyCount()); + assertEquals(103, index.records().size()); + } + private static MetadataMutationPayloadCodec.Descriptor create( MetadataKey key, long offset, long length) { return new MetadataMutationPayloadCodec.Descriptor( diff --git a/pki/src/test/java/zeroecho/pki/spi/bootstrap/PkiBootstrapTest.java b/pki/src/test/java/zeroecho/pki/spi/bootstrap/PkiBootstrapTest.java index d634075..e62d207 100644 --- a/pki/src/test/java/zeroecho/pki/spi/bootstrap/PkiBootstrapTest.java +++ b/pki/src/test/java/zeroecho/pki/spi/bootstrap/PkiBootstrapTest.java @@ -332,10 +332,13 @@ public final class PkiBootstrapTest { SignatureWorkflow.SignRequest request = SignatureWorkflow.SignRequest.create(submissionId, SIGNING_NAMESPACE, 1L, access, new KeyRef("test-prefix:bootstrap"), "SHA256withRSA", new ImmutableByteContent(payload), Optional.of(Encoding.BINARY), Optional.empty()); - workflow.submitSign(request); - assertEquals(SignatureWorkflow.State.SUCCEEDED, workflow.status(submissionId).state()); + SignatureWorkflow.CallControl control = new SignatureWorkflow.CallControl(Instant.MAX, + zeroecho.core.io.CancellationSignal.NONE); + workflow.submitSign(request, control); + assertEquals(SignatureWorkflow.State.SUCCEEDED, workflow.status(submissionId, control).state()); assertTrue( - workflow.status(submissionId).result().orElseThrow().signature().orElseThrow().bytes().length > 0); + workflow.status(submissionId, control).result().orElseThrow().signature().orElseThrow() + .bytes().length > 0); } try (KeyringPassword password = password(); KeyringStore reopened = KeyringStore.open(keyringPath, password)) { diff --git a/pki/src/test/java/zeroecho/pki/testkit/DurableDelayedSignatureWorkflow.java b/pki/src/test/java/zeroecho/pki/testkit/DurableDelayedSignatureWorkflow.java index 358937d..46aca97 100644 --- a/pki/src/test/java/zeroecho/pki/testkit/DurableDelayedSignatureWorkflow.java +++ b/pki/src/test/java/zeroecho/pki/testkit/DurableDelayedSignatureWorkflow.java @@ -101,7 +101,7 @@ public final class DurableDelayedSignatureWorkflow implements SignatureWorkflow, } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { Objects.requireNonNull(request, "request"); PkiId opId = request.submissionId(); @@ -124,13 +124,13 @@ public final class DurableDelayedSignatureWorkflow implements SignatureWorkflow, } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { Objects.requireNonNull(request, "request"); return new PkiId("verify-" + UUID.randomUUID()); } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { Objects.requireNonNull(operationId, "operationId"); OperationStatus st = loadStatus(operationId); if (st.isTerminal()) { @@ -172,7 +172,7 @@ public final class DurableDelayedSignatureWorkflow implements SignatureWorkflow, } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(reason, "reason"); if (fencingToken <= 0L) { @@ -181,11 +181,12 @@ public final class DurableDelayedSignatureWorkflow implements SignatureWorkflow, if (reason.isBlank()) { throw new IllegalArgumentException("reason must not be blank"); } - if (!identities.acceptFence(operationId, fencingToken)) { - return false; - } - OperationStatus st = status(operationId); + OperationStatus st = status(operationId, control); if (st.isTerminal()) { + return st.state() == State.CANCELLED + && identities.fence(operationId).orElse(-1L) == fencingToken; + } + if (!identities.acceptFence(operationId, fencingToken)) { return false; } OperationStatus cancelled = new OperationStatus(State.CANCELLED, Instant.now(), Optional.of("CANCELLED"), diff --git a/pki/src/test/java/zeroecho/pki/testkit/DurableOperatorApprovalSignatureWorkflow.java b/pki/src/test/java/zeroecho/pki/testkit/DurableOperatorApprovalSignatureWorkflow.java index aa57929..7eb6034 100644 --- a/pki/src/test/java/zeroecho/pki/testkit/DurableOperatorApprovalSignatureWorkflow.java +++ b/pki/src/test/java/zeroecho/pki/testkit/DurableOperatorApprovalSignatureWorkflow.java @@ -71,7 +71,7 @@ import zeroecho.pki.spi.crypto.SignatureWorkflow; *

          *
        1. {@code submitSign()} creates operation in {@code WAITING_APPROVAL}.
          2. *
        2. Operator calls {@link #approve(PkiId)} or {@link #deny(PkiId)}.
          3. - *
        3. On next {@link #status(PkiId)} poll, the operation either expires, fails, + *
        4. On next {@link #status(PkiId, CallControl)} poll, the operation either expires, fails, * or signs and succeeds.
          5. *
        */ @@ -136,7 +136,7 @@ public final class DurableOperatorApprovalSignatureWorkflow implements Signature } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { Objects.requireNonNull(request, "request"); PkiId opId = request.submissionId(); @@ -160,7 +160,7 @@ public final class DurableOperatorApprovalSignatureWorkflow implements Signature } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { Objects.requireNonNull(request, "request"); return new PkiId("verify-" + UUID.randomUUID()); } @@ -194,7 +194,7 @@ public final class DurableOperatorApprovalSignatureWorkflow implements Signature } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { Objects.requireNonNull(operationId, "operationId"); OperationStatus st = loadStatus(operationId); @@ -256,7 +256,7 @@ public final class DurableOperatorApprovalSignatureWorkflow implements Signature } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(reason, "reason"); if (fencingToken <= 0L) { @@ -265,12 +265,12 @@ public final class DurableOperatorApprovalSignatureWorkflow implements Signature if (reason.isBlank()) { throw new IllegalArgumentException("reason must not be blank"); } - if (!identities.acceptFence(operationId, fencingToken)) { - return false; - } - OperationStatus st = loadStatus(operationId); if (st.isTerminal()) { + return st.state() == State.CANCELLED + && identities.fence(operationId).orElse(-1L) == fencingToken; + } + if (!identities.acceptFence(operationId, fencingToken)) { return false; } diff --git a/pki/src/test/java/zeroecho/pki/testkit/InMemorySignatureWorkflow.java b/pki/src/test/java/zeroecho/pki/testkit/InMemorySignatureWorkflow.java index 0c022df..0aff47b 100644 --- a/pki/src/test/java/zeroecho/pki/testkit/InMemorySignatureWorkflow.java +++ b/pki/src/test/java/zeroecho/pki/testkit/InMemorySignatureWorkflow.java @@ -86,7 +86,7 @@ public final class InMemorySignatureWorkflow implements SignatureWorkflow, Publi } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { Objects.requireNonNull(request, "request"); PkiId opId = request.submissionId(); Object lock = operationLocks.computeIfAbsent(opId, ignored -> new Object()); @@ -194,7 +194,7 @@ public final class InMemorySignatureWorkflow implements SignatureWorkflow, Publi } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { Objects.requireNonNull(request, "request"); PkiId opId = new PkiId("verify:" + (counter++)); OperationStatus st = new OperationStatus(State.FAILED, Instant.now(), Optional.of("UNSUPPORTED"), @@ -204,7 +204,7 @@ public final class InMemorySignatureWorkflow implements SignatureWorkflow, Publi } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { Objects.requireNonNull(operationId, "operationId"); OperationStatus st = status.get(operationId); if (st == null) { @@ -214,7 +214,7 @@ public final class InMemorySignatureWorkflow implements SignatureWorkflow, Publi } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(reason, "reason"); if (fencingToken <= 0L) { @@ -223,7 +223,14 @@ public final class InMemorySignatureWorkflow implements SignatureWorkflow, Publi Object lock = operationLocks.computeIfAbsent(operationId, ignored -> new Object()); synchronized (lock) { OperationStatus existing = status.get(operationId); - if (existing == null || existing.isTerminal() || fencingToken < fences.get(operationId)) { + if (existing == null) { + return false; + } + long currentFence = fences.get(operationId); + if (existing.isTerminal()) { + return existing.state() == State.CANCELLED && fencingToken == currentFence; + } + if (fencingToken < currentFence) { return false; } fences.put(operationId, fencingToken); diff --git a/pki/src/test/java/zeroecho/pki/testkit/OperatorApprovalSignatureWorkflow.java b/pki/src/test/java/zeroecho/pki/testkit/OperatorApprovalSignatureWorkflow.java index 12f4db7..9bb347e 100644 --- a/pki/src/test/java/zeroecho/pki/testkit/OperatorApprovalSignatureWorkflow.java +++ b/pki/src/test/java/zeroecho/pki/testkit/OperatorApprovalSignatureWorkflow.java @@ -68,7 +68,7 @@ import zeroecho.pki.spi.crypto.SignatureWorkflow; * The approval flow is intentionally explicit: *

        *
          - *
        • After {@link #submitSign(SignRequest)} the operation enters + *
        • After {@link #submitSign(SignRequest, CallControl)} the operation enters * {@link State#WAITING_APPROVAL}.
          • *
        • Tests can call {@link #approve(PkiId)} or {@link #deny(PkiId)}.
          • *
        • If the operator does not act within {@link #approvalWindow}, the @@ -139,7 +139,7 @@ public final class OperatorApprovalSignatureWorkflow implements SignatureWorkflo } @Override - public PkiId submitSign(SignRequest request) { + public PkiId submitSign(SignRequest request, CallControl control) { Objects.requireNonNull(request, "request"); long n = seq.incrementAndGet(); @@ -182,7 +182,7 @@ public final class OperatorApprovalSignatureWorkflow implements SignatureWorkflo } @Override - public PkiId submitVerify(VerifyRequest request) { + public PkiId submitVerify(VerifyRequest request, CallControl control) { Objects.requireNonNull(request, "request"); // Not needed for these tests. long n = seq.incrementAndGet(); @@ -250,7 +250,7 @@ public final class OperatorApprovalSignatureWorkflow implements SignatureWorkflo } @Override - public OperationStatus status(PkiId operationId) { + public OperationStatus status(PkiId operationId, CallControl control) { Objects.requireNonNull(operationId, "operationId"); Path dir = opDir(operationId); java.util.Properties p = readPropsSafe(dir.resolve(FILE_META)); @@ -331,7 +331,7 @@ public final class OperatorApprovalSignatureWorkflow implements SignatureWorkflo } @Override - public boolean cancel(PkiId operationId, long fencingToken, String reason) { + public boolean cancel(PkiId operationId, long fencingToken, String reason, CallControl control) { Objects.requireNonNull(operationId, "operationId"); Objects.requireNonNull(reason, "reason"); if (fencingToken <= 0L) { @@ -340,9 +340,6 @@ public final class OperatorApprovalSignatureWorkflow implements SignatureWorkflo if (reason.isBlank()) { throw new IllegalArgumentException("reason must not be blank"); } - if (!identities.acceptFence(operationId, fencingToken)) { - return false; - } Path dir = opDir(operationId); java.util.Properties p = readPropsSafe(dir.resolve(FILE_META)); if (p.isEmpty()) { @@ -350,6 +347,9 @@ public final class OperatorApprovalSignatureWorkflow implements SignatureWorkflo } State s = parseState(p.getProperty(K_STATE)); if (isTerminalState(s)) { + return s == State.CANCELLED && identities.fence(operationId).orElse(-1L) == fencingToken; + } + if (!identities.acceptFence(operationId, fencingToken)) { return false; } Instant now = Instant.now(); @@ -427,7 +427,8 @@ public final class OperatorApprovalSignatureWorkflow implements SignatureWorkflo private void notifySink(PkiId opId) { for (NotificationSink sink : sinks.values()) { try { - sink.onStatusChanged(opId, status(opId)); + sink.onStatusChanged(opId, status(opId, + new CallControl(Instant.MAX, zeroecho.core.io.CancellationSignal.NONE))); } catch (RuntimeException ex) { // ignore } diff --git a/pki/src/test/java/zeroecho/pki/testkit/TestSignIdentityRegistry.java b/pki/src/test/java/zeroecho/pki/testkit/TestSignIdentityRegistry.java index e44e3c7..222cd6d 100644 --- a/pki/src/test/java/zeroecho/pki/testkit/TestSignIdentityRegistry.java +++ b/pki/src/test/java/zeroecho/pki/testkit/TestSignIdentityRegistry.java @@ -41,6 +41,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HexFormat; import java.util.Map; +import java.util.OptionalLong; import java.util.concurrent.ConcurrentHashMap; import zeroecho.pki.api.PkiId; @@ -95,6 +96,14 @@ final class TestSignIdentityRegistry { } } + OptionalLong fence(PkiId operationId) { + Object lock = locks.computeIfAbsent(operationId, ignored -> new Object()); + synchronized (lock) { + Path path = path(operationId); + return Files.exists(path) ? OptionalLong.of(read(path).fence) : OptionalLong.empty(); + } + } + private Path path(PkiId id) { try { byte[] hash = MessageDigest.getInstance("SHA-256").digest(id.value().getBytes(StandardCharsets.UTF_8));