fix(volume): return conflict and explicit proof for read-only upload refusals (#12236)

* fix(volume): report read-only HTTP refusals in Rust

* fix(volume): mirror HTTP write refusal proof in Go

* test(volume): cover HTTP read-only refusal parity

* test(volume): cover local refusal after remote append

* fix(volume): honor decoded replica controls before retry proof

* fix(operation): reassign proven HTTP conflict refusals

* fix(operation): stop same-target retries on HTTP conflict

* fix(volume): replace replica marker when forwarding requests

* fix(java): gate conflict reassignment on refusal proof

* fix(volume): preserve authentication query encoding in replication

* fix(operation): stop unproven conflicts in forever retries

other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java

@@ -2,6 +2,7 @@ package seaweedfs.client;
 
 import com.google.common.base.Strings;
 import com.google.protobuf.ByteString;
+import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
@@ -35,6 +36,7 @@ public class SeaweedWrite {
             final String path) throws IOException {
 
         IOException lastException = null;
+        boolean reassigned = false;
         for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) {
             try {
                 FilerProto.FileChunk.Builder chunkBuilder = writeChunk(
@@ -47,6 +49,16 @@ public class SeaweedWrite {
             } catch (IOException ioe) {
                 LOG.debug("writeData:{}", ioe);
                 lastException = ioe;
+                if (ioe instanceof UploadException) {
+                    UploadException error = (UploadException) ioe;
+                    if (error.status == 409) {
+                        if (!error.refusedBeforeAppend || reassigned) {
+                            throw error;
+                        }
+                        reassigned = true;
+                        continue;
+                    }
+                }
             }
             try {
                 Thread.sleep(waitTime);
@@ -195,17 +207,24 @@ public class SeaweedWrite {
                 .addBinaryBody("upload", inputStream)
                 .build());
 
-        CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);
-
-        try {
-            if (response.getStatusLine().getStatusCode() / 100 != 2) {
+        try (CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post)) {
+            int status = response.getStatusLine().getStatusCode();
+            if (status / 100 != 2) {
+                Header outcome = response.getFirstHeader("X-Seaweed-Write-Outcome");
+                boolean refusedBeforeAppend = status == 409 && outcome != null
+                        && "refused-before-append".equals(outcome.getValue());
+                String message = response.getStatusLine().getReasonPhrase();
+                IOException cause = null;
                 HttpEntity entity = response.getEntity();
-                if (entity != null && entity.getContentType() != null
-                        && entity.getContentType().getValue().equals("application/json")) {
-                    throw new IOException(EntityUtils.toString(entity, "UTF-8"));
-                } else {
-                    throw new IOException(response.getStatusLine().getReasonPhrase());
+                try {
+                    if (entity != null && entity.getContentType() != null
+                            && entity.getContentType().getValue().equals("application/json")) {
+                        message = EntityUtils.toString(entity, "UTF-8");
+                    }
+                } catch (IOException e) {
+                    cause = e;
                 }
+                throw new UploadException(status, refusedBeforeAppend, message, cause);
             }
 
             String etag = response.getLastHeader("ETag").getValue();
@@ -218,12 +237,22 @@ public class SeaweedWrite {
 
             return etag;
         } finally {
-            response.close();
             post.releaseConnection();
         }
 
     }
 
+    private static final class UploadException extends IOException {
+        private final int status;
+        private final boolean refusedBeforeAppend;
+
+        private UploadException(int status, boolean refusedBeforeAppend, String message, IOException cause) {
+            super(message, cause);
+            this.status = status;
+            this.refusedBeforeAppend = refusedBeforeAppend;
+        }
+    }
+
     private static byte[] genCipherKey() {
         byte[] b = new byte[32];
         random.nextBytes(b);

other/java/client/src/test/java/seaweedfs/client/SeaweedWriteIntegrationTest.java

@@ -0,0 +1,207 @@
+package seaweedfs.client;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import io.grpc.Server;
+import io.grpc.ServerBuilder;
+import io.grpc.stub.StreamObserver;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.Assert.*;
+
+public class SeaweedWriteIntegrationTest {
+    private static final byte[] BYTES = "prefix-upload\u0000payload-suffix".getBytes(StandardCharsets.UTF_8);
+    private static final byte[] PAYLOAD = Arrays.copyOfRange(BYTES, 7, BYTES.length - 7);
+    private static final String PROOF = "refused-before-append";
+
+    @Test(timeout = 30000)
+    public void testProvenRefusalReassignsOnce() throws Exception {
+        for (boolean cipher : new boolean[]{false, true}) {
+            try (UploadServer server = new UploadServer(cipher, 1, 409, PROOF, false)) {
+                FilerProto.Entry.Builder entry = server.write();
+                server.assertAttempts(2);
+                assertEquals(1, entry.getChunksCount());
+                FilerProto.FileChunk chunk = entry.getChunks(0);
+                assertEquals("7,200000000", chunk.getFileId());
+                assertEquals(23, chunk.getOffset());
+                assertEquals(PAYLOAD.length, chunk.getSize());
+                assertEquals("uploaded", chunk.getETag());
+                byte[] uploaded = server.payloads.get(1);
+                if (cipher) {
+                    uploaded = SeaweedCipher.decrypt(uploaded, chunk.getCipherKey().toByteArray());
+                } else {
+                    assertArrayEquals(PAYLOAD, server.payloads.get(0));
+                }
+                assertArrayEquals(PAYLOAD, uploaded);
+            }
+        }
+    }
+
+    @Test(timeout = 30000)
+    public void testUnprovenConflictDoesNotRetry() throws Exception {
+        for (String outcome : new String[]{null, "unknown", "REFUSED-BEFORE-APPEND"}) {
+            try (UploadServer server = new UploadServer(false, 1, 409, outcome, false)) {
+                server.assertWriteFails(1);
+            }
+        }
+    }
+
+    @Test(timeout = 30000)
+    public void testRepeatedProvenRefusalDoesNotRetryAgain() throws Exception {
+        try (UploadServer server = new UploadServer(false, 2, 409, PROOF, false)) {
+            server.assertWriteFails(2);
+        }
+    }
+
+    @Test(timeout = 30000)
+    public void testTruncatedConflictRetainsRetryProof() throws Exception {
+        try (UploadServer server = new UploadServer(false, 1, 409, null, true)) {
+            assertNotNull(server.assertWriteFails(1).getCause());
+        }
+        try (UploadServer server = new UploadServer(false, 1, 409, PROOF, true)) {
+            assertEquals(1, server.write().getChunksCount());
+            server.assertAttempts(2);
+        }
+    }
+
+    @Test(timeout = 30000)
+    public void testOtherUploadErrorsKeepExistingRetry() throws Exception {
+        try (UploadServer server = new UploadServer(false, 1, 500, PROOF, false)) {
+            assertEquals(1, server.write().getChunksCount());
+            server.assertAttempts(2);
+        }
+    }
+
+    private static final class UploadServer implements AutoCloseable {
+        private final AtomicInteger assignments = new AtomicInteger();
+        private final AtomicInteger uploads = new AtomicInteger();
+        private final List<byte[]> payloads = Collections.synchronizedList(new ArrayList<>());
+        private final List<String> paths = Collections.synchronizedList(new ArrayList<>());
+        private final List<String> auth = Collections.synchronizedList(new ArrayList<>());
+        private final List<String> digests = Collections.synchronizedList(new ArrayList<>());
+        private final HttpServer volume;
+        private final Server filer;
+        private final FilerClient client;
+
+        private UploadServer(boolean cipher, int refusals, int status, String outcome, boolean truncated)
+                throws IOException {
+            volume = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+            volume.createContext("/", exchange -> upload(exchange, refusals, status, outcome, truncated));
+            volume.start();
+            filer = ServerBuilder.forPort(0).addService(new SeaweedFilerGrpc.SeaweedFilerImplBase() {
+                @Override
+                public void getFilerConfiguration(FilerProto.GetFilerConfigurationRequest request,
+                        StreamObserver<FilerProto.GetFilerConfigurationResponse> observer) {
+                    observer.onNext(FilerProto.GetFilerConfigurationResponse.newBuilder().setCipher(cipher).build());
+                    observer.onCompleted();
+                }
+
+                @Override
+                public void assignVolume(FilerProto.AssignVolumeRequest request,
+                        StreamObserver<FilerProto.AssignVolumeResponse> observer) {
+                    int attempt = assignments.incrementAndGet();
+                    observer.onNext(FilerProto.AssignVolumeResponse.newBuilder()
+                            .setFileId("7," + attempt + "00000000")
+                            .setAuth("token-" + attempt)
+                            .setLocation(FilerProto.Location.newBuilder()
+                                    .setUrl("127.0.0.1:" + volume.getAddress().getPort()))
+                            .build());
+                    observer.onCompleted();
+                }
+            }).build().start();
+            client = new FilerClient("127.0.0.1", filer.getPort());
+        }
+
+        private void upload(HttpExchange exchange, int refusals, int status, String outcome, boolean truncated)
+                throws IOException {
+            try {
+                ByteArrayOutputStream body = new ByteArrayOutputStream();
+                byte[] buffer = new byte[1024];
+                int count;
+                while ((count = exchange.getRequestBody().read(buffer)) != -1) {
+                    body.write(buffer, 0, count);
+                }
+                String multipart = new String(body.toByteArray(), StandardCharsets.ISO_8859_1);
+                String boundary = exchange.getRequestHeaders().getFirst("Content-Type").split("boundary=")[1];
+                boundary = boundary.replace("\"", "");
+                int start = multipart.indexOf("\r\n\r\n") + 4;
+                int end = multipart.lastIndexOf("\r\n--" + boundary);
+                payloads.add(Arrays.copyOfRange(body.toByteArray(), start, end));
+                paths.add(exchange.getRequestURI().getPath());
+                auth.add(exchange.getRequestHeaders().getFirst("Authorization"));
+                digests.add(exchange.getRequestHeaders().getFirst("Content-MD5"));
+                int attempt = uploads.incrementAndGet();
+                byte[] response = "{\"error\":\"read only; refused-before-append\"}"
+                        .getBytes(StandardCharsets.UTF_8);
+                if (attempt <= refusals) {
+                    exchange.getResponseHeaders().add("Content-Type", "application/json");
+                    if (outcome != null) {
+                        exchange.getResponseHeaders().add("X-Seaweed-Write-Outcome", outcome);
+                    }
+                    exchange.sendResponseHeaders(status, response.length + (truncated ? 20 : 0));
+                } else {
+                    exchange.getResponseHeaders().add("ETag", "\"uploaded\"");
+                    exchange.sendResponseHeaders(201, response.length);
+                }
+                exchange.getResponseBody().write(response);
+            } finally {
+                exchange.close();
+            }
+        }
+
+        private FilerProto.Entry.Builder write() throws IOException {
+            FilerProto.Entry.Builder entry = FilerProto.Entry.newBuilder();
+            SeaweedWrite.writeData(entry, "", "", client, 23, BYTES, 7, PAYLOAD.length, "/upload");
+            return entry;
+        }
+
+        private IOException assertWriteFails(int attempts) throws Exception {
+            FilerProto.Entry.Builder entry = FilerProto.Entry.newBuilder();
+            IOException error = null;
+            try {
+                SeaweedWrite.writeData(entry, "", "", client, 23, BYTES, 7, PAYLOAD.length, "/upload");
+                fail("expected upload conflict");
+            } catch (IOException expected) {
+                assertNotNull(expected.getMessage());
+                error = expected;
+            }
+            assertEquals(0, entry.getChunksCount());
+            assertAttempts(attempts);
+            return error;
+        }
+
+        private void assertAttempts(int attempts) throws Exception {
+            assertEquals(attempts, assignments.get());
+            assertEquals(attempts, uploads.get());
+            for (int i = 0; i < attempts; i++) {
+                assertEquals("/7," + (i + 1) + "00000000", paths.get(i));
+                assertEquals("BEARER token-" + (i + 1), auth.get(i));
+                assertEquals(Base64.getEncoder().encodeToString(MessageDigest.getInstance("MD5")
+                        .digest(payloads.get(i))), digests.get(i));
+            }
+        }
+
+        @Override
+        public void close() throws InterruptedException {
+            try {
+                client.shutdown();
+            } finally {
+                filer.shutdownNow().awaitTermination(5, TimeUnit.SECONDS);
+                volume.stop(0);
+            }
+        }
+    }
+}