From ff53da88b44b958a3de54a86d92be4791f0aeff3 Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 12 Oct 2026 19:55:01 +0800 Subject: [PATCH 01/14] fix(loader): guard startup mode cleanup - require confirmed loading mode before cleanup - keep cleanup state local and reset it on clone - cover failed startup and cleanup error paths - document network uncertainty and shared mode limits --- hugegraph-loader/README.md | 11 + .../loader/executor/LoadContext.java | 15 +- .../loader/test/unit/LoadContextTest.java | 300 +++++++++++++++++- 3 files changed, 317 insertions(+), 9 deletions(-) diff --git a/hugegraph-loader/README.md b/hugegraph-loader/README.md index decc2b3a9..33ad5dd3e 100644 --- a/hugegraph-loader/README.md +++ b/hugegraph-loader/README.md @@ -133,6 +133,17 @@ Paths in the argument file resolve from the working directory. Protect files con the credentials can then stay out of the process arguments. The mapping directory must be writable for progress output, so copy a mapping mounted from a read-only Kubernetes ConfigMap to a writable volume before running Loader. +#### Graph mode during startup and cleanup + +Loader resets `LOADING` mode only after that Loader context successfully sets it. A startup failure before the mode +change, a rejected mode request, or a restore run does not clear another run's `LOADING` mode. Cleanup preserves a +different current graph mode; successful cleanup is not repeated. Concurrent successful Loader runs on the same +graph still require external coordination; this startup safeguard does not provide shared ownership of graph mode. + +A network failure can leave a mode change applied without a successful response. Loader reports that failure and +leaves the unconfirmed mode unchanged during cleanup. Inspect the graph mode and confirm there are no active imports +before explicitly restoring `NONE` mode; an unconfirmed request cannot be rolled back automatically. + ### 3.2 Load data with docker > If the `loader` and `server` is in the same docker network (for example, you deploy `loader` and `server` with `docker-compose`), we can set `-h {server_container_name}`. In our example, the container name of `server` is `graph` diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java index 4b5863596..94a878ef9 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java @@ -53,6 +53,8 @@ private enum StopReason { private final String timestamp; private volatile boolean closed; + // Only a successful LOADING request permits this context to reset it. + private boolean loadingModeSet; private volatile StopReason stopReason; private volatile boolean noError; private volatile Throwable failure; @@ -206,10 +208,11 @@ public void updateSchemaCache() { this.schemaCache.updateAll(); } - public void setLoadingMode() { + public synchronized void setLoadingMode() { String graph = this.client.graph().graph(); try { this.client.graphs().mode(graph, GraphMode.LOADING); + this.loadingModeSet = true; } catch (ServerException e) { if (e.getMessage().contains("Can not deserialize value of type")) { LOG.warn("HugeGraphServer doesn't support loading mode"); @@ -232,13 +235,17 @@ public void setRestoreMode() { } } - public void unsetLoadingMode() { + public synchronized void unsetLoadingMode() { + if (!this.loadingModeSet) { + return; + } try { String graph = this.client.graph().graph(); GraphMode mode = this.client.graphs().mode(graph); if (mode.loading()) { this.client.graphs().mode(graph, GraphMode.NONE); } + this.loadingModeSet = false; } catch (Exception e) { throw new LoadException("Failed to unset mode %s for server", e, GraphMode.LOADING); @@ -291,6 +298,8 @@ public void close() { @Override public Object clone() throws CloneNotSupportedException { - return super.clone(); + LoadContext context = (LoadContext) super.clone(); + context.loadingModeSet = false; + return context; } } diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java index ea3f2dd45..33a414543 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java @@ -17,39 +17,50 @@ package org.apache.hugegraph.loader.test.unit; -import java.io.File; -import java.io.IOException; import java.io.BufferedWriter; -import java.io.StringWriter; +import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.io.StringWriter; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.util.LinkedHashMap; -import java.lang.reflect.Field; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import org.apache.hugegraph.driver.HugeClient; import org.apache.hugegraph.driver.HugeClientBuilder; +import org.apache.hugegraph.exception.ServerException; +import org.apache.hugegraph.loader.HugeGraphLoader; import org.apache.hugegraph.loader.builder.SchemaCache; import org.apache.hugegraph.loader.constant.ElemType; -import org.apache.hugegraph.loader.exception.LoadException; import org.apache.hugegraph.loader.exception.InsertException; +import org.apache.hugegraph.loader.exception.LoadException; import org.apache.hugegraph.loader.exception.ParseException; import org.apache.hugegraph.loader.executor.LoadContext; import org.apache.hugegraph.loader.executor.LoadOptions; import org.apache.hugegraph.loader.failure.FailLogger; import org.apache.hugegraph.loader.failure.FailWriter; import org.apache.hugegraph.loader.mapping.InputStruct; +import org.apache.hugegraph.loader.mapping.LoadMapping; import org.apache.hugegraph.loader.metrics.LoadSummary; import org.apache.hugegraph.loader.progress.LoadProgress; import org.apache.hugegraph.loader.source.file.FileSource; +import org.apache.hugegraph.loader.source.graph.GraphSource; +import org.apache.hugegraph.structure.constant.GraphMode; +import org.apache.hugegraph.util.JsonUtil; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import com.sun.net.httpserver.HttpServer; + public class LoadContextTest { @Rule @@ -418,6 +429,283 @@ public void testUnrecoverableWorkerErrorPreventsProgressAndClosesClients() throw Assert.assertFalse(new File(LoadProgress.format(context.options(), "test")).exists()); } + @Test + public void testMixedSourceStartupDoesNotClearAnotherLoadingMode() throws Exception { + try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { + LoadContext context = this.context(client, client); + InputStruct graph = new InputStruct(null, null); + graph.input(new GraphSource()); + InputStruct file = new InputStruct(null, null); + file.input(new FileSource()); + HugeGraphLoader loader = loader(context, graph, file); + try { + loader.load(); + Assert.fail("Expected mixed Graph and File startup failure"); + } catch (LoadException expected) { + Assert.assertEquals("All inputs must be of Graph Type", expected.getMessage()); + } + try { + loader.shutdown(); + Assert.fail("Expected the original startup failure during close"); + } catch (LoadException expected) { + Assert.assertEquals("All inputs must be of Graph Type", expected.getMessage()); + } + loader.shutdown(); + Assert.assertTrue(context.closed()); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(0, server.modeReads); + Assert.assertEquals(0, server.modeWrites); + } + } + + @Test + public void testFailedLoadingModeSetDoesNotClearAnotherLoadingMode() throws Exception { + try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { + LoadContext context = this.context(client, client); + server.failure = "Mode change denied"; + try { + context.setLoadingMode(); + Assert.fail("Expected mode set failure"); + } catch (ServerException expected) { + Assert.assertEquals("Mode change denied", expected.getMessage()); + } + server.failure = null; + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(0, server.modeReads); + Assert.assertEquals(1, server.modeWrites); + } + } + + @Test + public void testUnsupportedLoadingModeDoesNotAcquireCleanup() throws Exception { + try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { + LoadContext context = this.context(client, client); + server.failure = "Can not deserialize value of type GraphMode"; + context.setLoadingMode(); + server.failure = null; + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(0, server.modeReads); + Assert.assertEquals(1, server.modeWrites); + } + } + + @Test + public void testSuccessfulLoadingModeCleanupRunsOnce() throws Exception { + try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { + LoadContext context = this.context(client, client); + server.mode = GraphMode.NONE; + context.setLoadingMode(); + Assert.assertEquals(GraphMode.LOADING, server.mode); + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.NONE, server.mode); + // Another Loader starts after this context has finished cleanup. + server.mode = GraphMode.LOADING; + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(1, server.modeReads); + Assert.assertEquals(2, server.modeWrites); + } + } + + @Test + public void testLoadingCleanupPreservesOtherMode() throws Exception { + try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { + LoadContext context = this.context(client, client); + context.setLoadingMode(); + server.mode = GraphMode.RESTORING; + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.RESTORING, server.mode); + server.mode = GraphMode.LOADING; + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(1, server.modeReads); + Assert.assertEquals(1, server.modeWrites); + } + } + + @Test + public void testRestoreModeDoesNotAcquireLoadingCleanup() throws Exception { + try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { + LoadContext context = this.context(client, client); + context.setRestoreMode(); + Assert.assertEquals(GraphMode.RESTORING, server.mode); + server.mode = GraphMode.LOADING; + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(0, server.modeReads); + Assert.assertEquals(1, server.modeWrites); + } + } + + @Test + public void testClonedContextDoesNotInheritLoadingCleanup() throws Exception { + try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { + LoadContext context = this.context(client, client); + context.setLoadingMode(); + LoadContext cloned = (LoadContext) context.clone(); + cloned.unsetLoadingMode(); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(0, server.modeReads); + Assert.assertEquals(1, server.modeWrites); + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.NONE, server.mode); + } + } + + @Test + public void testCleanupReadFailureCanBeRetried() throws Exception { + this.assertCleanupFailureCanBeRetried(true); + } + + @Test + public void testCleanupWriteFailureCanBeRetried() throws Exception { + this.assertCleanupFailureCanBeRetried(false); + } + + private void assertCleanupFailureCanBeRetried(boolean readFailure) throws Exception { + try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { + LoadContext context = this.context(client, client); + context.setLoadingMode(); + if (readFailure) { + server.readFailure = "Cleanup read denied"; + } else { + server.failure = "Cleanup write denied"; + } + try { + context.unsetLoadingMode(); + Assert.fail("Expected cleanup failure"); + } catch (LoadException expected) { + Assert.assertTrue(expected.getCause() instanceof ServerException); + } + Assert.assertEquals(GraphMode.LOADING, server.mode); + server.readFailure = null; + server.failure = null; + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.NONE, server.mode); + Assert.assertEquals(2, server.modeReads); + Assert.assertEquals(readFailure ? 2 : 3, server.modeWrites); + // A completed retry must not acquire cleanup of a subsequent import. + server.mode = GraphMode.LOADING; + context.unsetLoadingMode(); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(2, server.modeReads); + Assert.assertEquals(readFailure ? 2 : 3, server.modeWrites); + } + } + + @Test + public void testUnconfirmedModeSetPreservesModeAndStartupFailure() throws Exception { + try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { + LoadContext context = this.context(client, client); + InputStruct file = new InputStruct(null, null); + file.input(new FileSource()); + HugeGraphLoader loader = loader(context, file); + server.mode = GraphMode.NONE; + server.dropModeResponse = true; + IOException failure = null; + try { + loader.load(); + Assert.fail("Expected failure after the mode change was applied"); + } catch (RuntimeException expected) { + Assert.assertTrue(expected.getCause() instanceof IOException); + failure = (IOException) expected.getCause(); + } + Assert.assertNotNull(failure); + Assert.assertEquals(GraphMode.LOADING, server.mode); + int writes = server.modeWrites; + Assert.assertTrue(writes > 0); + server.dropModeResponse = false; + try { + loader.shutdown(); + Assert.fail("Expected the original unconfirmed mode change failure"); + } catch (LoadException expected) { + Assert.assertSame(failure, expected.getCause()); + } + loader.shutdown(); + Assert.assertTrue(context.closed()); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(0, server.modeReads); + Assert.assertEquals(writes, server.modeWrites); + } + } + + private static HugeGraphLoader loader(LoadContext context, InputStruct... inputs) throws Exception { + HugeGraphLoader loader = allocate(HugeGraphLoader.class); + set(loader, "context", context); + set(loader, "options", context.options()); + set(loader, "mapping", new LoadMapping(Arrays.asList(inputs))); + return loader; + } + + private static class ModeServer implements AutoCloseable { + + private final HttpServer server; + private volatile GraphMode mode = GraphMode.LOADING; + private volatile String failure; + private volatile String readFailure; + private volatile boolean dropModeResponse; + private volatile int modeReads; + private volatile int modeWrites; + + private ModeServer() throws IOException { + this.server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + this.server.createContext("/", exchange -> { + String response; + int status = 200; + String path = exchange.getRequestURI().getPath(); + if (path.equals("/versions")) { + response = "{\"versions\":{\"core\":\"1.8.0\",\"api\":\"0.74\"}}"; + } else if (path.equals("/graphspaces/DEFAULT/graphs/hugegraph/mode")) { + if (exchange.getRequestMethod().equals("PUT")) { + this.modeWrites++; + if (this.failure != null) { + status = 400; + response = JsonUtil.toJson(Collections.singletonMap("message", this.failure)); + } else { + String body = new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + this.mode = GraphMode.valueOf(JsonUtil.fromJson(body, String.class)); + if (this.dropModeResponse) { + exchange.close(); + return; + } + response = "{}"; + } + } else { + this.modeReads++; + if (this.readFailure != null) { + status = 400; + response = JsonUtil.toJson(Collections.singletonMap("message", this.readFailure)); + } else { + response = JsonUtil.toJson(Collections.singletonMap("mode", this.mode.name())); + } + } + } else { + status = 404; + response = "{}"; + } + byte[] body = response.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, body.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(body); + } + }); + this.server.start(); + } + + private HugeClient client() { + return HugeClient.builder("http://127.0.0.1:" + this.server.getAddress().getPort(), "hugegraph") + .configTimeout(1).build(); + } + + @Override + public void close() { + this.server.stop(0); + } + } + private static void assertClosedWithoutRetrying(LoadContext context, RecordingClient client, RecordingClient indirect) { Assert.assertTrue(context.closed()); From f5a07f0d08f7f8c402ece1faddab4a9e7ea50cc2 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 02:12:17 +0800 Subject: [PATCH 02/14] fix(loader): report uncertain loading mode writes - log the graph when a mode write loses its response - preserve the original checked transport exception - retain cleanup ownership and avoid blind mode resets - verify the diagnostic and explain manual recovery --- hugegraph-loader/README.md | 7 ++-- .../loader/executor/LoadContext.java | 6 +++ .../loader/test/unit/LoadContextTest.java | 38 +++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/hugegraph-loader/README.md b/hugegraph-loader/README.md index 33ad5dd3e..c3e69388d 100644 --- a/hugegraph-loader/README.md +++ b/hugegraph-loader/README.md @@ -140,9 +140,10 @@ change, a rejected mode request, or a restore run does not clear another run's ` different current graph mode; successful cleanup is not repeated. Concurrent successful Loader runs on the same graph still require external coordination; this startup safeguard does not provide shared ownership of graph mode. -A network failure can leave a mode change applied without a successful response. Loader reports that failure and -leaves the unconfirmed mode unchanged during cleanup. Inspect the graph mode and confirm there are no active imports -before explicitly restoring `NONE` mode; an unconfirmed request cannot be rolled back automatically. +A network failure can leave a mode change applied without a successful response. Loader preserves the original +failure and logs an error naming the graph and warning that it may still be in `LOADING` mode. Cleanup leaves the +unconfirmed mode unchanged. Inspect the graph mode and confirm there are no active imports before explicitly +restoring `NONE` mode; an unconfirmed request cannot be rolled back automatically. ### 3.2 Load data with docker diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java index 94a878ef9..bbd935143 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java @@ -208,6 +208,7 @@ public void updateSchemaCache() { this.schemaCache.updateAll(); } + @SneakyThrows public synchronized void setLoadingMode() { String graph = this.client.graph().graph(); try { @@ -219,6 +220,11 @@ public synchronized void setLoadingMode() { } else { throw e; } + } catch (Exception e) { + LOG.error("Failed to confirm LOADING mode for graph '{}'; it may still be in LOADING mode. " + + "Confirm no imports are active, inspect the graph mode, and manually reset it to NONE if needed.", + graph); + throw e; } } diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java index 33a414543..1684d411a 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java @@ -54,6 +54,10 @@ import org.apache.hugegraph.loader.source.graph.GraphSource; import org.apache.hugegraph.structure.constant.GraphMode; import org.apache.hugegraph.util.JsonUtil; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -597,6 +601,10 @@ private void assertCleanupFailureCanBeRetried(boolean readFailure) throws Except @Test public void testUnconfirmedModeSetPreservesModeAndStartupFailure() throws Exception { + Logger logger = Logger.getLogger(LoadContext.class.getName()); + CapturingAppender appender = new CapturingAppender(); + appender.setThreshold(Level.ERROR); + logger.addAppender(appender); try (ModeServer server = new ModeServer(); HugeClient client = server.client()) { LoadContext context = this.context(client, client); InputStruct file = new InputStruct(null, null); @@ -613,6 +621,10 @@ public void testUnconfirmedModeSetPreservesModeAndStartupFailure() throws Except failure = (IOException) expected.getCause(); } Assert.assertNotNull(failure); + Assert.assertTrue(appender.contains("graph 'hugegraph'")); + Assert.assertTrue(appender.contains("may still be in LOADING mode")); + Assert.assertTrue(appender.contains("no imports are active")); + Assert.assertTrue(appender.contains("manually reset it to NONE")); Assert.assertEquals(GraphMode.LOADING, server.mode); int writes = server.modeWrites; Assert.assertTrue(writes > 0); @@ -628,6 +640,32 @@ public void testUnconfirmedModeSetPreservesModeAndStartupFailure() throws Except Assert.assertEquals(GraphMode.LOADING, server.mode); Assert.assertEquals(0, server.modeReads); Assert.assertEquals(writes, server.modeWrites); + } finally { + logger.removeAppender(appender); + } + } + + private static final class CapturingAppender extends AppenderSkeleton { + + private final StringBuilder messages = new StringBuilder(); + + @Override + protected void append(LoggingEvent event) { + this.messages.append(event.getRenderedMessage()).append('\n'); + } + + private boolean contains(String text) { + return this.messages.toString().contains(text); + } + + @Override + public void close() { + // No resources owned by this appender. + } + + @Override + public boolean requiresLayout() { + return false; } } From ade05333165c7291b6ccd87bd394617724ec699a Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 10:05:45 +0800 Subject: [PATCH 03/14] fix(hubble): close owned loaders before deletion - retain loader ownership through cleanup and recovery - preserve typed file progress during cancellation - serialize job admission with metadata deletion - verify transaction rollback and stale callbacks --- hugegraph-hubble/README.md | 44 + .../controller/load/LoadTaskController.java | 87 +- .../hugegraph/entity/load/LoadTask.java | 58 +- .../hugegraph/handler/LoadTaskExecutor.java | 5 +- .../service/load/JobManagerService.java | 60 +- .../service/load/LoadTaskService.java | 484 +++- .../IngestTransactionIntegrationTest.java | 12 + .../hugegraph/unit/LoadTaskCleanupTest.java | 2581 +++++++++++++++++ .../hugegraph/unit/LoadTaskServiceTest.java | 18 +- .../apache/hugegraph/unit/UnitTestSuite.java | 1 + hugegraph-loader/README.md | 19 + .../hugegraph/loader/HugeGraphLoader.java | 91 +- .../loader/executor/LoadContext.java | 10 +- .../loader/progress/InputItemProgress.java | 10 +- .../loader/progress/InputProgress.java | 10 + .../loader/progress/LoadProgress.java | 6 +- .../loader/reader/AbstractReader.java | 3 +- .../loader/reader/file/FileReader.java | 4 +- .../loader/test/unit/LoadContextTest.java | 206 ++ .../loader/test/unit/TaskManagerTest.java | 15 + 20 files changed, 3519 insertions(+), 205 deletions(-) create mode 100644 hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index 9338594eb..8b4b0917c 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -49,6 +49,50 @@ The compatibility boundary is deliberately small: Use matching Server, PD, and Store minor versions in a distributed cluster. **Server/PD 1.8 or later is strongly recommended for the best Hubble experience.** Support for 1.5 and 1.7 is intentionally limited to minimum usability and does not imply feature parity with the current release. Version checks stay in the client adapter/resolver rather than being scattered through controllers or pages. See [`AGENTS.md`](AGENTS.md) for verification rules. +## Import task lifecycle + +Import tasks finish their Loader shutdown before reporting completion. Shutdown +also persists progress and closes the task's clients and worker pools; a shutdown +failure marks the task as failed, including after a pause or stop request. The +synchronous Loader example also shuts down before returning. A task that cannot +be saved, whose transaction rolls back, or whose async dispatch is rejected closes +its Loader without starting a run. Pause and stop wait for the running Loader and +its final database callback. Within one Hubble service, pause, stop, resume, retry +and initial task publication are serialized for a task, including the database +read and final stop persistence; other requests for that task wait for admission. +The bounded weak lock stripes can also serialize unrelated task IDs that collide. +Retry and resume reject an owner whose workers have not closed. An accepted +pause or stop also prevents delayed initial dispatch after commit; the unused +Loader closes without overwriting the accepted status. A delayed callback also +closes only its unused Loader if the task was deleted, finished, or replaced; it +does not change the newer task or job state. Task deletion rechecks state and +closed resources under admission. The Job task create/start endpoints and Job +management deletion share Job admission, so deletion enumerates tasks started before it and later +starts recheck that the Job still exists. Job deletion commits physical stops before +atomically deleting task and job metadata; a metadata rollback leaves those +tasks stopped. If delayed initial publication cannot read task metadata, it closes +its unused Loader and retains that closed owner for the existing progress poll. +The poll marks only a still-RUNNING row as failed and retires the owner after commit; +a missing or advanced row is preserved. +Finished tasks release their native JVM shutdown hook once their context closes. +If shutdown cannot terminate the task's workers, Hubble retains the failed owner +and rejects retry/resume until its context is actually closed; restart Hubble to +finish recovery when cleanup remains incomplete. This cleanup applies when load +exits and reaches shutdown; an ongoing backend call can still block existing +Loader waits and controls queued on the same stripe. The existing progress poll retries final metadata writes for +finished, closed owners and releases an owner only after its final metadata +transaction commits. Closing an owner before load starts preserves its old +checkpoint. A pause/stop using separately restored task state, or a resume/retry, +rejects a conflicting persisted status change; read the current state before +retrying the request. Incremental pause/resume also keeps offsets for inputs not yet read +and already completed counts. This cleanup keeps the existing graph-mode +behavior and does not coordinate concurrent imports from separate processes or +Server instances. + +The Loader [startup mode ownership guard](https://github.com/apache/hugegraph-toolchain/pull/812) +is a prerequisite: without it, cleanup after startup fails can reset another +importer's `LOADING` mode. + ## Local development feedback loop Run the frontend with third-party source-map noise disabled: diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/LoadTaskController.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/LoadTaskController.java index 5b45e7800..1602339a9 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/LoadTaskController.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/LoadTaskController.java @@ -50,6 +50,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.locks.Lock; @Log4j2 @RestController @@ -114,19 +115,25 @@ public LoadTask create(@PathVariable("graphspace") String graphSpace, @PathVariable("jobId") int jobId, @RequestBody LoadTask entity) { this.requireGraphSpaceWrite(graphSpace); - JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); - Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); - Ex.check(jobEntity.getJobStatus() == JobStatus.SETTING, - "load.task.create.no-permission"); - synchronized (this.service) { - Ex.check(this.service.count() < LIMIT, - "load.task.reached-limit", LIMIT); - entity.setGraphSpace(graphSpace); - entity.setGraph(graph); - entity.setJobId(jobId); - this.service.save(entity); + Lock control = this.jobService.jobControl(jobId); + control.lock(); + try { + JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); + Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); + Ex.check(jobEntity.getJobStatus() == JobStatus.SETTING, + "load.task.create.no-permission"); + synchronized (this.service) { + Ex.check(this.service.count() < LIMIT, + "load.task.reached-limit", LIMIT); + entity.setGraphSpace(graphSpace); + entity.setGraph(graph); + entity.setJobId(jobId); + this.service.save(entity); + } + return entity; + } finally { + control.unlock(); } - return entity; } @DeleteMapping("{id}") @@ -171,35 +178,41 @@ public List start(@PathVariable("graphspace") String graphSpace, connection.setPort(host.getPort()); } - JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); - Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); - Ex.check(jobEntity.getJobStatus() == JobStatus.SETTING, - "load.task.start.no-permission"); - boolean existError = false; + Lock control = this.jobService.jobControl(jobId); + control.lock(); try { - List tasks = new ArrayList<>(); - HugeClient client = this.authClient(graphSpace, graph); - for (Integer fileId : fileIds) { - FileMapping fileMapping = this.fmService.get(graphSpace, graph, - jobId, fileId); - if (fileMapping == null) { - throw new ExternalException("file-mapping.not-exist.id", - fileId); + JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); + Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); + Ex.check(jobEntity.getJobStatus() == JobStatus.SETTING, + "load.task.start.no-permission"); + boolean existError = false; + try { + List tasks = new ArrayList<>(); + HugeClient client = this.authClient(graphSpace, graph); + for (Integer fileId : fileIds) { + FileMapping fileMapping = this.fmService.get(graphSpace, graph, + jobId, fileId); + if (fileMapping == null) { + throw new ExternalException("file-mapping.not-exist.id", + fileId); + } + tasks.add(this.service.start(connection, fileMapping, client)); + } + return tasks; + } catch (Exception e) { + existError = true; + throw e; + } finally { + if (existError) { + jobEntity.setJobStatus(JobStatus.FAILED); + } else { + jobEntity.setJobStatus(JobStatus.LOADING); } - tasks.add(this.service.start(connection, fileMapping, client)); + jobEntity.setUpdateTime(HubbleUtil.nowDate()); + this.jobService.update(jobEntity); } - return tasks; - } catch (Exception e) { - existError = true; - throw e; } finally { - if (existError) { - jobEntity.setJobStatus(JobStatus.FAILED); - } else { - jobEntity.setJobStatus(JobStatus.LOADING); - } - jobEntity.setUpdateTime(HubbleUtil.nowDate()); - this.jobService.update(jobEntity); + control.unlock(); } } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java index ce6fd04af..9a68e0206 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java @@ -68,6 +68,10 @@ public class LoadTask implements Runnable { @JsonIgnore private transient volatile boolean finished; + @TableField(exist = false) + @JsonIgnore + private transient volatile boolean initialStateReadFailed; + @TableId(type = IdType.AUTO) @MergeProperty(useNew = false) @JsonProperty("id") @@ -174,21 +178,57 @@ public LoadTask(LoadOptions options, GraphConnection connection, @Override public void run() { + this.run(() -> { }); + } + + public void run(Runnable callback) { + try { + this.runLoader(); + } finally { + this.lock.lock(); + try { + callback.run(); + } finally { + // Stop/retry must not race a callback from the previous run. + this.finished = true; + this.lock.unlock(); + } + } + } + + private void runLoader() { Ex.check(this.options != null, "The load options shouldn't be null"); log.info("LoadTask is start running : {}", this.id); - boolean noError; + boolean noError = false; + boolean cleanupFailed = false; + Throwable failure = null; try { noError = this.loader.load(); } catch (Throwable e) { - noError = false; - log.error("Run task {} failed", this.id, e); + failure = e; + } finally { + // Only the running thread closes this loader, after load has stopped. + try { + this.loader.shutdown(); + } catch (Throwable e) { + noError = false; + cleanupFailed = true; + if (failure == null) { + failure = e; + } else if (failure != e) { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + log.error("Run task {} failed", this.id, failure); } this.lock.lock(); try { // Pay attention to whether the user stops actively or // the program stops by itself - if (this.status.inRunning()) { + if (cleanupFailed || this.status.inRunning()) { if (noError) { this.status = LoadStatus.SUCCEED; } else { @@ -204,7 +244,6 @@ public void run() { this.lastDuration += this.context().summary().totalTime(); this.currDuration = 0L; } finally { - this.finished = true; this.lock.unlock(); } } @@ -218,7 +257,10 @@ public void unlock() { } public void stop() { - this.context().stopLoading(); + HugeGraphLoader owner = this.loader; + if (owner != null) { + owner.context().stopLoading(); + } while (!this.finished) { try { Thread.sleep(1000); @@ -226,7 +268,9 @@ public void stop() { // pass } } - this.loader = null; + if (owner == null || owner.context().closed()) { + this.loader = null; + } log.info("LoadTask {} stopped", this.id); } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/LoadTaskExecutor.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/LoadTaskExecutor.java index 07e7ca164..402b14bd9 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/LoadTaskExecutor.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/LoadTaskExecutor.java @@ -32,12 +32,9 @@ public class LoadTaskExecutor { public void execute(LoadTask task, Runnable callback) { log.info("Executing task: {}", task.getId()); try { - task.run(); + task.run(callback); } catch (Throwable t) { log.warn("Executing task: {} error", task.getId(), t); - } finally { - log.info("Executed task: {}, update status to db", task.getId()); - callback.run(); } } } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java index deb05a8f7..60b049710 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java @@ -23,12 +23,14 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.concurrent.locks.Lock; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.Striped; import lombok.extern.log4j.Log4j2; import org.apache.hugegraph.driver.HugeClient; @@ -46,10 +48,14 @@ import org.apache.hugegraph.util.PageUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.annotation.Isolation; +import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionSynchronization; import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.transaction.support.TransactionTemplate; @Log4j2 @Service @@ -61,6 +67,14 @@ public class JobManagerService { private LoadTaskService taskService; @Autowired private FileMappingService fileMappingService; + @Autowired + private PlatformTransactionManager transactionManager; + + private final Striped jobControls = Striped.lazyWeakLock(1024); + + public Lock jobControl(int id) { + return this.jobControls.get(id); + } public int count() { return Math.toIntExact(this.mapper.selectCount(null)); @@ -264,28 +278,42 @@ public void removeByGraph(String graphSpace, String graph) { "graph", graph)); } - @Transactional(isolation = Isolation.READ_COMMITTED) + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void deleteJob(int id) { - JobManager job = this.get(id); - if (job == null) { - throw new ExternalException("job.manager.not-exist.id", id); - } + Lock control = this.jobControl(id); + control.lock(); + try { + JobManager job = this.get(id); + if (job == null) { + throw new ExternalException("job.manager.not-exist.id", id); + } - List loadTasks = this.taskService.taskListByJob(id); - for (LoadTask loadTask : loadTasks) { - if (loadTask.getStatus().inRunning() || - loadTask.getStatus() == LoadStatus.PAUSED) { - this.taskService.stop(loadTask.getId()); + List loadTasks = this.taskService.taskListByJob(id); + for (LoadTask loadTask : loadTasks) { + if (loadTask.getStatus().inRunning() || + loadTask.getStatus() == LoadStatus.PAUSED) { + this.taskService.stop(loadTask.getId()); + } } - this.taskService.remove(loadTask.getId()); - } - List mappings = this.fileMappingService.listByJob(id); - this.remove(id); - this.deleteDiskFilesAfterCommit(mappings); + // Physical stops cannot be rolled back. Commit them before the metadata transaction. + TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); + transaction.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED); + transaction.execute(status -> { + for (LoadTask loadTask : loadTasks) { + this.taskService.remove(loadTask.getId()); + } + List mappings = this.fileMappingService.listByJob(id); + this.remove(id); + this.deleteDiskFilesAfterCommit(mappings); + return null; + }); + } finally { + control.unlock(); + } } - @Transactional(isolation = Isolation.READ_COMMITTED) + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void deleteJob(String graphSpace, String graph, int id) { JobManager job = this.get(graphSpace, graph, id); if (job == null) { diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java index 23b3656c4..3f4c3bf27 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java @@ -23,6 +23,7 @@ import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Striped; import lombok.extern.log4j.Log4j2; import org.apache.commons.io.FileUtils; import org.apache.hugegraph.common.Constant; @@ -84,6 +85,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; import org.apache.commons.lang3.StringUtils; @@ -112,6 +114,8 @@ public class LoadTaskService { private Map runningTaskContainer; + // Request admission precedes task locks. Workers and progress polling never take this lock. + private final Striped taskControls = Striped.lazyWeakLock(1024); public LoadTaskService() { this.runningTaskContainer = new ConcurrentHashMap<>(); @@ -187,11 +191,22 @@ public void save(LoadTask entity) { @Transactional(isolation = Isolation.READ_COMMITTED) public void update(LoadTask entity) { + this.update(entity, null); + } + + private void update(LoadTask entity, LoadStatus previousStatus) { LoadOptions runtimeOptions = entity.getOptions(); entity.setOptions(withoutCredentials(runtimeOptions)); try { - if (this.mapper.updateById(entity) != 1) { - throw new InternalException("entity.update.failed"); + if (previousStatus == null) { + if (this.mapper.updateById(entity) != 1) { + throw new InternalException("entity.update.failed"); + } + } else { + int updated = this.mapper.update(entity, Wrappers.update() + .eq("id", entity.getId()) + .eq("load_status", previousStatus.getValue())); + Ex.check(updated == 1, "The load task state has changed; retry the request"); } } finally { entity.setOptions(runtimeOptions); @@ -217,9 +232,33 @@ private static LoadOptions withoutCredentials(LoadOptions options) { @Transactional(isolation = Isolation.READ_COMMITTED) public void remove(int id) { - this.runningTaskContainer.remove(id); - if (this.mapper.deleteById(id) != 1) { - throw new InternalException("entity.delete.failed", id); + Lock control = this.taskControls.get(id); + control.lock(); + try { + LoadTask persisted = this.get(id); + Ex.check(persisted != null, "load.task.not-exist.id", id); + Ex.check(!persisted.getStatus().inRunning(), "load.task.delete.task-in-running", id); + LoadTask owner = this.runningTaskContainer.get(id); + if (owner != null) { + owner.lock(); + } + try { + Ex.check(owner == null || (owner.isFinished() && + (owner.getLoader() == null || owner.context().closed())), + "load.task.delete.task-in-running", id); + int deleted = this.mapper.delete(Wrappers.query().eq("id", id) + .eq("load_status", persisted.getStatus().getValue())); + Ex.check(deleted == 1, "The load task state has changed; retry the request"); + if (owner != null) { + this.removeOwnerAfterCommit(owner); + } + } finally { + if (owner != null) { + owner.unlock(); + } + } + } finally { + control.unlock(); } } @@ -232,9 +271,15 @@ public List batchTasks(int jobId) { public LoadTask start(GraphConnection connection, FileMapping fileMapping, HugeClient client) { LoadTask task = this.buildLoadTask(connection, fileMapping, client); - this.save(task); - this.executeAfterCommit(task); - return task; + try { + this.save(task); + this.executeAfterCommit(task); + return task; + } catch (RuntimeException | Error failure) { + closeUnscheduled(task, failure); + this.removeClosedOwner(task); + throw failure; + } } protected void executeAfterCommit(LoadTask task) { @@ -248,164 +293,369 @@ protected void executeAfterCommit(LoadTask task) { public void afterCommit() { executeSafely(task); } + + @Override + public void afterCompletion(int status) { + if (status != TransactionSynchronization.STATUS_COMMITTED) { + closeUnscheduled(task, null); + } + } }); } private void executeSafely(LoadTask task) { + Lock control = this.taskControls.get(task.getId()); + control.lock(); try { - this.execute(task); - } catch (RuntimeException e) { - this.markDispatchFailed(task, e); + this.executeControlled(task); + } finally { + control.unlock(); } } - private void markDispatchFailed(LoadTask task, RuntimeException cause) { - log.error("Failed to dispatch load task {}, marking task and job " + - "as FAILED", task.getId(), cause); - task.setStatus(LoadStatus.FAILED); - this.runningTaskContainer.remove(task.getId()); + private void executeControlled(LoadTask task) { + LoadTask current = this.runningTaskContainer.get(task.getId()); + if (current == task) { + // The same owner was already scheduled; do not run or close it twice. + return; + } + if (current != null) { + this.cancelInitial(task, LoadStatus.FAILED, null); + return; + } + LoadTask persisted; + try { + persisted = this.get(task.getId()); + } catch (RuntimeException failure) { + log.error("Failed to read metadata for initial load task {}", task.getId(), failure); + task.setInitialStateReadFailed(true); + this.cancelInitial(task, LoadStatus.FAILED, failure); + // Keep even a closed owner until the existing poll can conditionally persist this failure. + this.runningTaskContainer.putIfAbsent(task.getId(), task); + return; + } + HugeGraphLoader loader = task.getLoader(); + if (persisted == null || persisted.getStatus() != LoadStatus.RUNNING || current != null || + task.isFinished() || loader == null || loader.context().closed() || loader.context().stopped()) { + LoadStatus status = persisted == null || persisted.getStatus() == LoadStatus.RUNNING ? + LoadStatus.FAILED : persisted.getStatus(); + this.cancelInitial(task, status, null); + return; + } + if (this.runningTaskContainer.putIfAbsent(task.getId(), task) != null) { + this.cancelInitial(task, LoadStatus.FAILED, null); + return; + } + try { + this.dispatch(task); + } catch (RuntimeException failure) { + this.markDispatchFailed(task, failure); + } + } + private void cancelInitial(LoadTask task, LoadStatus status, Throwable failure) { + // A delayed callback owns only its unused Loader, never a newer task's metadata. + closeUnscheduled(task, failure, status); + HugeGraphLoader owner = task.getLoader(); + if (owner != null && !owner.context().closed()) { + task.setStatus(LoadStatus.FAILED); + this.runningTaskContainer.putIfAbsent(task.getId(), task); + } + } + + private void markDispatchFailed(LoadTask task, RuntimeException cause) { + if (this.runningTaskContainer.get(task.getId()) != task) { + this.cancelInitial(task, LoadStatus.FAILED, cause); + return; + } + log.error("Failed to dispatch load task {}, marking task and job as FAILED", task.getId(), cause); + closeUnscheduled(task, cause); try { - TransactionTemplate transaction = - new TransactionTemplate(this.transactionManager); - transaction.setPropagationBehavior( - TransactionDefinition.PROPAGATION_REQUIRES_NEW); + TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); + transaction.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); transaction.execute(status -> { - int updated = this.mapper.update( - null, Wrappers.update() - .eq("id", task.getId()) - .set("load_status", - LoadStatus.FAILED.getValue())); - if (updated != 1) { - throw new InternalException("entity.update.failed", task); - } - if (task.getJobId() == null) { + if (this.runningTaskContainer.get(task.getId()) != task) { return null; } - JobManager job = this.jobManagerMapper.selectById( - task.getJobId()); - if (job == null) { + int updated = this.mapper.update(null, Wrappers.update() + .eq("id", task.getId()) + .eq("load_status", LoadStatus.RUNNING.getValue()) + .set("load_status", LoadStatus.FAILED.getValue())); + if (updated == 0) { return null; } - job.setJobStatus(JobStatus.FAILED); - job.setUpdateTime(HubbleUtil.nowDate()); - if (this.jobManagerMapper.updateById(job) != 1) { - throw new InternalException("entity.update.failed", job); + if (updated != 1) { + throw new InternalException("entity.update.failed", task); + } + if (task.getJobId() != null) { + JobManager job = this.jobManagerMapper.selectById(task.getJobId()); + if (job != null) { + job.setJobStatus(JobStatus.FAILED); + job.setUpdateTime(HubbleUtil.nowDate()); + if (this.jobManagerMapper.updateById(job) != 1) { + throw new InternalException("entity.update.failed", job); + } + } } return null; }); - } catch (RuntimeException e) { - // The ingest transaction is already committed. Never turn a - // dispatch failure into a retryable request failure. - log.error("Failed to persist dispatch failure for load task {}", - task.getId(), e); + } catch (RuntimeException failure) { + // The ingest transaction is committed. Do not report a retryable dispatch failure. + log.error("Failed to persist dispatch failure for load task {}", task.getId(), failure); + } finally { + this.removeClosedOwner(task); + } + } + + private static void closeUnscheduled(LoadTask task, Throwable failure) { + closeUnscheduled(task, failure, LoadStatus.FAILED); + } + + private static void closeUnscheduled(LoadTask task, Throwable failure, LoadStatus status) { + task.lock(); + try { + task.setStatus(status); + if (task.getLoader() != null) { + try { + task.getLoader().shutdown(); + } catch (Throwable cleanup) { + task.setStatus(LoadStatus.FAILED); + if (failure != null && failure != cleanup) { + failure.addSuppressed(cleanup); + } + log.error("Failed to close unscheduled load task {}", task.getId(), cleanup); + } + } + } finally { + // No runner was scheduled to publish completion for this owner. + task.setFinished(true); + task.unlock(); } } - private void execute(LoadTask task) { - // Executed in other threads - this.taskExecutor.execute(task, () -> this.update(task)); - // Save current load task - this.runningTaskContainer.put(task.getId(), task); + private void dispatch(LoadTask task) { + this.taskExecutor.execute(task, () -> { + this.update(task); + // Pause/stop retain ownership until their caller has also persisted + // its terminal request. A failed callback can be retried there. + if (task.getStatus() != LoadStatus.PAUSED && task.getStatus() != LoadStatus.STOPPED) { + this.removeClosedOwner(task); + } + }); + } + + private void persistStopped(LoadTask task, boolean loading, LoadStatus previousStatus) { + task.lock(); + try { + // The runner already persisted its final state before finishing. + // Never let an old pause/stop callback overwrite a new owner. + if (!loading || this.runningTaskContainer.get(task.getId()) == task) { + this.update(task, loading ? null : previousStatus); + this.removeClosedOwner(task); + } + } finally { + task.unlock(); + } + } + + private void removeClosedOwner(LoadTask task) { + HugeGraphLoader owner = task.getLoader(); + if (task.getId() != null && (owner == null || owner.context().closed())) { + this.removeOwner(task); + } + } + + private void removeOwner(LoadTask expected) { + this.runningTaskContainer.computeIfPresent(expected.getId(), + (id, current) -> current == expected ? null : current); + } + + private void removeOwnerAfterCommit(LoadTask task) { + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + this.removeOwner(task); + return; + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + if (status == TransactionSynchronization.STATUS_COMMITTED) { + removeOwner(task); + } + } + }); + } + + private void releasePreviousOwner(int taskId) { + LoadTask previous = this.runningTaskContainer.get(taskId); + if (previous == null) { + return; + } + Ex.check(previous.isFinished(), "The previous load task is still finishing"); + previous.lock(); + try { + HugeGraphLoader owner = previous.getLoader(); + Ex.check(owner == null || owner.context().closed(), + "The previous Loader has not closed; finish its cleanup or restart Hubble before retrying"); + this.removeOwner(previous); + } finally { + previous.unlock(); + } } public LoadTask pause(int taskId) { + Lock control = this.taskControls.get(taskId); + control.lock(); + try { + return this.pauseControlled(taskId); + } finally { + control.unlock(); + } + } + + private LoadTask pauseControlled(int taskId) { /* * The in-memory container is not repopulated on startup, so a task * that was RUNNING before a restart is only visible in the database */ LoadTask task = this.runningTaskContainer.get(taskId); boolean loading = task != null; + LoadStatus readStatus = null; if (!loading) { task = this.get(taskId); + readStatus = task == null ? null : task.getStatus(); + // A restart may have published an owner during the database read. + LoadTask owner = this.runningTaskContainer.get(taskId); + if (owner != null) { + task = owner; + loading = true; + } } Ex.check(task != null, "load.task.not-exist.id", taskId); - Ex.check(task.getStatus() == LoadStatus.RUNNING, - "Can only pause the RUNNING task"); - // Mark status as paused, should set before context.stopLoading() - task.setStatus(LoadStatus.PAUSED); - if (loading) { - // Let HugeGraphLoader stop - task.stop(); - } - task.lock(); try { - this.update(task); - this.runningTaskContainer.remove(taskId); + Ex.check(!loading || readStatus == null || task.getStatus() == readStatus, + "The load task state has changed; retry the request"); + Ex.check(task.getStatus() == LoadStatus.RUNNING, + "Can only pause the RUNNING task"); + if (loading) { + LoadTask owner = this.runningTaskContainer.putIfAbsent(taskId, task); + Ex.check(owner == null || owner == task, "The load task has already restarted"); + } + task.setStatus(LoadStatus.PAUSED); } finally { task.unlock(); } + if (loading) { + // Let HugeGraphLoader stop + task.stop(); + } + + this.persistStopped(task, loading, LoadStatus.RUNNING); return task; } public LoadTask resume(int taskId, String token) { - LoadTask task = this.get(taskId); - Ex.check(task.getStatus() == LoadStatus.PAUSED || - task.getStatus() == LoadStatus.FAILED, - "Can only resume the PAUSED or FAILED task"); - task.lock(); + return this.restart(taskId, token, true); + } + + public LoadTask stop(int taskId) { + Lock control = this.taskControls.get(taskId); + control.lock(); try { - // Set work mode in incrental mode, load from last breakpoint - task.getOptions().incrementalMode = true; - task.reconnect(token); - task.setStatus(LoadStatus.RUNNING); - this.update(task); - this.taskExecutor.execute(task, () -> this.update(task)); - this.runningTaskContainer.put(taskId, task); + return this.stopControlled(taskId); } finally { - task.unlock(); + control.unlock(); } - return task; } - public LoadTask stop(int taskId) { + private LoadTask stopControlled(int taskId) { LoadTask task = this.runningTaskContainer.get(taskId); boolean loading = task != null; + LoadStatus readStatus = null; if (!loading) { task = this.get(taskId); + readStatus = task == null ? null : task.getStatus(); + // A restart may have published an owner during the database read. + LoadTask owner = this.runningTaskContainer.get(taskId); + if (owner != null) { + task = owner; + loading = true; + } } Ex.check(task != null, "load.task.not-exist.id", taskId); - LoadStatus status = task.getStatus(); - Ex.check(status == LoadStatus.RUNNING || status == LoadStatus.PAUSED, - "Can only stop the RUNNING or PAUSED task"); - // Mark status as stopped - task.setStatus(LoadStatus.STOPPED); + LoadStatus status; + task.lock(); + try { + status = task.getStatus(); + Ex.check(!loading || readStatus == null || status == readStatus, + "The load task state has changed; retry the request"); + Ex.check(status == LoadStatus.RUNNING || status == LoadStatus.PAUSED, + "Can only stop the RUNNING or PAUSED task"); + if (loading) { + LoadTask owner = this.runningTaskContainer.putIfAbsent(taskId, task); + Ex.check(owner == null || owner == task, "The load task has already restarted"); + } + task.setStatus(LoadStatus.STOPPED); + } finally { + task.unlock(); + } /* * Only a task taken from the in-memory container owns a loader, one * restored from the database has nothing running to interrupt, so * just persist the terminal status for it */ - if (loading && status == LoadStatus.RUNNING) { + if (loading) { task.stop(); } - task.lock(); + this.persistStopped(task, loading, status); + return task; + } + + public LoadTask retry(int taskId, String token) { + return this.restart(taskId, token, false); + } + + private LoadTask restart(int taskId, String token, boolean incremental) { + Lock control = this.taskControls.get(taskId); + control.lock(); try { - this.update(task); - this.runningTaskContainer.remove(taskId); + return this.restartControlled(taskId, token, incremental); } finally { - task.unlock(); + control.unlock(); } - return task; } - public LoadTask retry(int taskId, String token) { + private LoadTask restartControlled(int taskId, String token, boolean incremental) { LoadTask task = this.get(taskId); - Ex.check(task.getStatus() == LoadStatus.FAILED || - task.getStatus() == LoadStatus.STOPPED, - "Can only retry the FAILED or STOPPED task"); + Ex.check(task != null, "load.task.not-exist.id", taskId); + LoadStatus previousStatus = task.getStatus(); + Ex.check(previousStatus == LoadStatus.FAILED || + previousStatus == (incremental ? LoadStatus.PAUSED : LoadStatus.STOPPED), + incremental ? "Can only resume the PAUSED or FAILED task" : + "Can only retry the FAILED or STOPPED task"); + this.releasePreviousOwner(taskId); task.lock(); try { - // Set work mode in normal mode, load from begin - task.getOptions().incrementalMode = false; + Ex.check(this.runningTaskContainer.putIfAbsent(taskId, task) == null, + "The previous load task is still finishing"); + task.getOptions().incrementalMode = incremental; task.reconnect(token); task.setStatus(LoadStatus.RUNNING); - task.setLastDuration(0L); - task.setCurrDuration(0L); - this.update(task); - this.taskExecutor.execute(task, () -> this.update(task)); - this.runningTaskContainer.put(taskId, task); + if (!incremental) { + task.setLastDuration(0L); + task.setCurrDuration(0L); + } + this.update(task, previousStatus); + try { + this.dispatch(task); + } catch (RuntimeException failure) { + this.markDispatchFailed(task, failure); + } + } catch (RuntimeException | Error failure) { + closeUnscheduled(task, failure); + this.removeClosedOwner(task); + throw failure; } finally { task.unlock(); } @@ -465,12 +715,29 @@ public void pauseAllTasks() { @Transactional(isolation = Isolation.READ_COMMITTED) public void updateLoadTaskProgress() { for (LoadTask task : this.runningTaskContainer.values()) { - if (!task.getStatus().inRunning()) { - continue; - } task.lock(); try { - if (task.getStatus().inRunning()) { + if (this.runningTaskContainer.get(task.getId()) != task) { + continue; + } + HugeGraphLoader owner = task.getLoader(); + if (task.isFinished() && (owner == null || owner.context().closed())) { + if (task.isInitialStateReadFailed()) { + // Unknown initial metadata must not overwrite a later terminal state + // or recreate a missing row. + int updated = this.mapper.update(null, + Wrappers.update().eq("id", task.getId()) + .eq("load_status", LoadStatus.RUNNING.getValue()) + .set("load_status", LoadStatus.FAILED.getValue())); + if (updated != 0 && updated != 1) { + throw new InternalException("entity.update.failed", task); + } + } else { + // Retry a final write that failed in the runner or pause/stop. + this.update(task); + } + this.removeOwnerAfterCommit(task); + } else if (task.getStatus().inRunning()) { LoadContext context = task.context(); long readLines = context.newProgress().totalInputRead(); if (readLines == 0L) { @@ -771,13 +1038,28 @@ public void startCovid19(GraphConnection connection, } public void loader(LoadOptions options) { - HugeGraphLoader loader; + HugeGraphLoader loader = null; + Throwable failure = null; try { loader = new HugeGraphLoader(options); loader.load(); } catch (Throwable e) { - Printer.printError("Failed to start loading", e); - return; + failure = e; + } finally { + if (loader != null) { + try { + loader.shutdown(); + } catch (Throwable e) { + if (failure == null) { + failure = e; + } else if (failure != e) { + failure.addSuppressed(e); + } + } + } + } + if (failure != null) { + Printer.printError("Failed to start loading", failure); } } } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java index 38f997593..c55a1690b 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java @@ -59,6 +59,8 @@ import org.apache.hugegraph.entity.load.LoadParameter; import org.apache.hugegraph.entity.load.LoadTask; import org.apache.hugegraph.handler.LoadTaskExecutor; +import org.apache.hugegraph.loader.HugeGraphLoader; +import org.apache.hugegraph.loader.executor.LoadContext; import org.apache.hugegraph.loader.executor.LoadOptions; import org.apache.hugegraph.options.HubbleOptions; import org.apache.hugegraph.service.schema.EdgeLabelService; @@ -465,7 +467,17 @@ public LoadTask start(GraphConnection connection, FileMapping mapping, String graph = this.failTaskInsert ? null : connection.getGraph(); LoadOptions options = new LoadOptions(); options.token = connection.getToken(); + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class); + java.util.concurrent.atomic.AtomicBoolean closed = new java.util.concurrent.atomic.AtomicBoolean(); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(context.closed()).thenAnswer(invocation -> closed.get()); + Mockito.doAnswer(invocation -> { + closed.set(true); + return null; + }).when(loader).shutdown(); LoadTask task = LoadTask.builder() + .loader(loader) .graphSpace(connection.getGraphSpace()) .graph(graph) .jobId(mapping.getJobId()) diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java new file mode 100644 index 000000000..5678d868a --- /dev/null +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java @@ -0,0 +1,2581 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.unit; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.entity.GraphConnection; +import org.apache.hugegraph.entity.enums.LoadStatus; +import org.apache.hugegraph.entity.load.FileMapping; +import org.apache.hugegraph.entity.load.FileSetting; +import org.apache.hugegraph.entity.load.LoadParameter; +import org.apache.hugegraph.handler.LoadTaskExecutor; +import org.apache.hugegraph.mapper.load.LoadTaskMapper; +import org.apache.hugegraph.entity.load.LoadTask; +import org.apache.hugegraph.loader.HugeGraphLoader; +import org.apache.hugegraph.loader.executor.LoadContext; +import org.apache.hugegraph.loader.executor.LoadOptions; +import org.apache.hugegraph.service.load.LoadTaskService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.mockito.MockedConstruction; +import org.mockito.Mockito; + +public class LoadTaskCleanupTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testSuccessIsPublishedAfterShutdown() { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + Mockito.when(loader.load()).thenReturn(true); + Mockito.doAnswer(invocation -> { + Assert.assertEquals(LoadStatus.RUNNING, task.getStatus()); + Assert.assertFalse(task.isFinished()); + return null; + }).when(loader).shutdown(); + + task.run(); + + Mockito.verify(loader).shutdown(); + Assert.assertEquals(LoadStatus.SUCCEED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(Long.valueOf(7L), task.getFileReadLines()); + Assert.assertEquals(Long.valueOf(12L), task.getLastDuration()); + } + + @Test + public void testLoadFailureKeepsCleanupFailureSuppressed() { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + RuntimeException loadFailure = new RuntimeException("load failed"); + RuntimeException cleanupFailure = new RuntimeException("cleanup failed"); + Mockito.when(loader.load()).thenThrow(loadFailure); + Mockito.doThrow(cleanupFailure).when(loader).shutdown(); + + task.run(); + + Mockito.verify(loader).shutdown(); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + Assert.assertArrayEquals(new Throwable[]{cleanupFailure}, loadFailure.getSuppressed()); + } + + @Test + public void testFalseLoadResultStillShutsDown() { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + Mockito.when(loader.load()).thenReturn(false); + + task.run(); + + Mockito.verify(loader).shutdown(); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + } + + @Test + public void testCleanupFailureMarksPausedAndStoppedTasksFailed() { + for (LoadStatus status : new LoadStatus[]{LoadStatus.RUNNING, LoadStatus.PAUSED, LoadStatus.STOPPED}) { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setStatus(status); + Mockito.when(loader.load()).thenReturn(true); + Mockito.doThrow(new RuntimeException("cleanup failed")).when(loader).shutdown(); + + task.run(); + + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + } + } + + @Test + public void testSameFailureIsNotSuppressedOntoItself() { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + RuntimeException failure = new RuntimeException("same failure"); + Mockito.when(loader.load()).thenThrow(failure); + Mockito.doThrow(failure).when(loader).shutdown(); + + task.run(); + + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(0, failure.getSuppressed().length); + } + + @Test + public void testStopLeavesShutdownToRunAndWaitsForCleanup() throws Exception { + HugeGraphLoader loader = this.loader(); + LoadContext context = loader.context(); + LoadTask task = this.task(loader); + CountDownLatch loading = new CountDownLatch(1); + CountDownLatch releaseLoad = new CountDownLatch(1); + CountDownLatch stopped = new CountDownLatch(1); + CountDownLatch cleaning = new CountDownLatch(1); + CountDownLatch releaseCleanup = new CountDownLatch(1); + Mockito.when(loader.load()).thenAnswer(invocation -> { + loading.countDown(); + Assert.assertTrue(releaseLoad.await(5, TimeUnit.SECONDS)); + return true; + }); + Mockito.doAnswer(invocation -> { + stopped.countDown(); + return null; + }).when(context).stopLoading(); + Mockito.doAnswer(invocation -> { + cleaning.countDown(); + Assert.assertTrue(releaseCleanup.await(5, TimeUnit.SECONDS)); + Mockito.when(context.closed()).thenReturn(true); + return null; + }).when(loader).shutdown(); + Thread running = new Thread(task); + Thread stopping = new Thread(task::stop); + try { + running.start(); + Assert.assertTrue(loading.await(5, TimeUnit.SECONDS)); + task.setStatus(LoadStatus.PAUSED); + stopping.start(); + Assert.assertTrue(stopped.await(5, TimeUnit.SECONDS)); + Mockito.verify(loader, Mockito.never()).shutdown(); + releaseLoad.countDown(); + Assert.assertTrue(cleaning.await(5, TimeUnit.SECONDS)); + Assert.assertFalse(task.isFinished()); + Assert.assertTrue(stopping.isAlive()); + Assert.assertSame(loader, task.getLoader()); + releaseCleanup.countDown(); + running.join(5000); + stopping.join(5000); + Assert.assertFalse(running.isAlive()); + Assert.assertFalse(stopping.isAlive()); + Mockito.verify(loader).shutdown(); + Assert.assertNull(task.getLoader()); + Assert.assertEquals(LoadStatus.PAUSED, task.getStatus()); + } finally { + releaseLoad.countDown(); + releaseCleanup.countDown(); + running.join(5000); + stopping.join(5000); + } + } + + @Test + public void testPreviewShutsDownAfterSuccess() { + try (MockedConstruction construction = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, context) -> Mockito.when(loader.load()).thenReturn(true))) { + new LoadTaskService().loader(new LoadOptions()); + + Assert.assertEquals(1, construction.constructed().size()); + Mockito.verify(construction.constructed().get(0)).shutdown(); + } + } + + @Test + public void testPreviewKeepsPrimaryFailureWhenShutdownFails() { + RuntimeException loadFailure = new RuntimeException("preview load failed"); + RuntimeException cleanupFailure = new RuntimeException("preview cleanup failed"); + try (MockedConstruction construction = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, context) -> { + Mockito.when(loader.load()).thenThrow(loadFailure); + Mockito.doThrow(cleanupFailure).when(loader).shutdown(); + })) { + new LoadTaskService().loader(new LoadOptions()); + + Mockito.verify(construction.constructed().get(0)).shutdown(); + Assert.assertArrayEquals(new Throwable[]{cleanupFailure}, loadFailure.getSuppressed()); + } + } + + @Test + public void testSaveFailureClosesEagerLoaderAndKeepsPrimaryFailure() throws Exception { + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + set(service, "mapper", mapper); + RuntimeException failure = new RuntimeException("database unavailable"); + RuntimeException cleanup = new RuntimeException("cleanup failed"); + Mockito.when(mapper.insert(Mockito.any(LoadTask.class))).thenThrow(failure); + FileMapping mapping = new FileMapping(); + mapping.setPath(this.folder.newFile("input.csv").getAbsolutePath()); + mapping.setVertexMappings(Collections.emptySet()); + mapping.setEdgeMappings(Collections.emptySet()); + mapping.setLoadParameter(new LoadParameter()); + FileSetting setting = new FileSetting(); + setting.setColumnNames(Collections.singletonList("id")); + mapping.setFileSetting(setting); + try (MockedConstruction owners = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, context) -> { + LoadContext resources = Mockito.mock(LoadContext.class); + Mockito.when(loader.context()).thenReturn(resources); + Mockito.when(resources.closed()).thenReturn(true); + Mockito.doThrow(cleanup).when(loader).shutdown(); + })) { + try { + service.start(new GraphConnection(), mapping, null); + Assert.fail("save must fail"); + } catch (RuntimeException actual) { + Assert.assertSame(failure, actual); + } + Assert.assertEquals(1, owners.constructed().size()); + Mockito.verify(owners.constructed().get(0)).shutdown(); + Assert.assertArrayEquals(new Throwable[]{cleanup}, failure.getSuppressed()); + } + } + + @Test + public void testOuterRollbackClosesUnscheduledLoader() throws Exception { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + LoadTaskService service = new LoadTaskService(); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + set(service, "taskExecutor", executor); + TransactionSynchronizationManager.initSynchronization(); + try { + dispatch(service, task); + TransactionSynchronizationManager.getSynchronizations().get(0) + .afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK); + Mockito.verify(loader).shutdown(); + Mockito.verifyNoInteractions(executor); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + } + + @Test + public void testDispatchRejectionClosesUnscheduledLoader() throws Exception { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + LoadTaskService service = new LoadTaskService(); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + RuntimeException rejection = new java.util.concurrent.RejectedExecutionException("executor closed"); + RuntimeException cleanup = new RuntimeException("cleanup failed"); + Mockito.doThrow(rejection).when(executor).execute(Mockito.eq(task), Mockito.any()); + Mockito.doThrow(cleanup).when(loader).shutdown(); + set(service, "taskExecutor", executor); + dispatch(service, task); + Mockito.verify(loader).shutdown(); + Assert.assertArrayEquals(new Throwable[]{cleanup}, rejection.getSuppressed()); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + } + + @Test + public void testFinishedIncludesPersistenceCallback() { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + Mockito.when(loader.load()).thenReturn(true); + new LoadTaskExecutor().execute(task, () -> { + Mockito.verify(loader).shutdown(); + Assert.assertFalse("runner is not finished until its callback returns", task.isFinished()); + }); + Assert.assertTrue(task.isFinished()); + } + + @Test + public void testStopOfPausedOwnerWaitsForRunner() throws Exception { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + task.setStatus(LoadStatus.PAUSED); + CountDownLatch loading = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch stopReturned = new CountDownLatch(1); + Mockito.when(loader.load()).thenAnswer(invocation -> { + loading.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + return true; + }); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.updateById(task)).thenReturn(1); + set(service, "mapper", mapper); + Map owners = new ConcurrentHashMap<>(); + owners.put(9, task); + set(service, "runningTaskContainer", owners); + Thread runner = new Thread(() -> new LoadTaskExecutor().execute(task, () -> service.update(task))); + Thread stopper = new Thread(() -> { + service.stop(9); + stopReturned.countDown(); + }); + try { + runner.start(); + Assert.assertTrue(loading.await(5, TimeUnit.SECONDS)); + stopper.start(); + Assert.assertFalse("PAUSED owner is still loading", stopReturned.await(300, TimeUnit.MILLISECONDS)); + Assert.assertSame(task, owners.get(9)); + } finally { + release.countDown(); + runner.join(5000); + stopper.join(5000); + } + Assert.assertFalse(runner.isAlive()); + Assert.assertFalse(stopper.isAlive()); + Assert.assertEquals(LoadStatus.STOPPED, task.getStatus()); + Assert.assertTrue(owners.isEmpty()); + } + + @Test + public void testStopAndRetryCannotPassOldPersistenceCallback() throws Exception { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + task.setStatus(LoadStatus.PAUSED); + Mockito.when(loader.load()).thenReturn(true); + LoadTask restored = Mockito.mock(LoadTask.class); + Mockito.when(restored.getStatus()).thenReturn(LoadStatus.STOPPED); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.selectById(9)).thenReturn( + LoadTask.builder().id(9).status(LoadStatus.RUNNING).build(), restored); + CountDownLatch callback = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch stopReturned = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference persisted = + new java.util.concurrent.atomic.AtomicReference<>(); + Mockito.when(mapper.updateById(task)).thenAnswer(invocation -> { + if (callback.getCount() > 0) { + callback.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + } + persisted.set(task.getStatus()); + return 1; + }); + set(service, "mapper", mapper); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + Thread[] runner = new Thread[1]; + Mockito.doAnswer(invocation -> { + Runnable completion = invocation.getArgument(1); + runner[0] = new Thread(() -> new LoadTaskExecutor().execute(task, completion)); + runner[0].start(); + return null; + }).when(executor).execute(Mockito.eq(task), Mockito.any()); + set(service, "taskExecutor", executor); + dispatch(service, task); + Thread stopper = new Thread(() -> { + service.stop(9); + stopReturned.countDown(); + }); + CountDownLatch retryReturned = new CountDownLatch(1); + Thread retry = new Thread(() -> { + try { + service.retry(9, "token"); + } catch (RuntimeException expected) { + // The mock restored task cannot reconnect, after the old callback has finished. + } finally { + retryReturned.countDown(); + } + }); + try { + Assert.assertTrue(callback.await(5, TimeUnit.SECONDS)); + Assert.assertFalse(task.isFinished()); + stopper.start(); + Assert.assertFalse(stopReturned.await(300, TimeUnit.MILLISECONDS)); + retry.start(); + Assert.assertFalse(retryReturned.await(300, TimeUnit.MILLISECONDS)); + Mockito.verify(restored, Mockito.never()).reconnect(Mockito.anyString()); + } finally { + release.countDown(); + runner[0].join(5000); + stopper.join(5000); + retry.join(5000); + } + Assert.assertFalse(runner[0].isAlive()); + Assert.assertFalse(stopper.isAlive()); + Assert.assertFalse(retry.isAlive()); + Assert.assertEquals(LoadStatus.STOPPED, persisted.get()); + } + + @Test + public void testFastRunnerDoesNotLeaveFinishedOwnerInContainer() throws Exception { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + Mockito.when(loader.load()).thenReturn(true); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.updateById(task)).thenReturn(1); + set(service, "mapper", mapper); + set(service, "taskExecutor", new LoadTaskExecutor()); + Map owners = new ConcurrentHashMap<>(); + set(service, "runningTaskContainer", owners); + dispatch(service, task); + Assert.assertTrue(task.isFinished()); + Assert.assertTrue(owners.isEmpty()); + } + + @Test + public void testStopRetriesFailedCallbackBeforeRemovingOwner() throws Exception { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + task.setStatus(LoadStatus.PAUSED); + Mockito.when(loader.load()).thenReturn(true); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.updateById(task)).thenThrow(new IllegalStateException("database offline")) + .thenReturn(1); + set(service, "mapper", mapper); + set(service, "taskExecutor", new LoadTaskExecutor()); + Map owners = new ConcurrentHashMap<>(); + set(service, "runningTaskContainer", owners); + dispatch(service, task); + Assert.assertTrue(task.isFinished()); + Assert.assertSame(task, owners.get(9)); + service.stop(9); + Mockito.verify(mapper, Mockito.times(2)).updateById(task); + Mockito.verify(loader).shutdown(); + Assert.assertEquals(LoadStatus.STOPPED, task.getStatus()); + Assert.assertTrue(owners.isEmpty()); + } + + @Test + public void testResumeSaveFailureClosesNewOwnerAndKeepsPrimaryFailure() throws Exception { + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + LoadTask task = LoadTask.builder().id(9).options(new LoadOptions()).status(LoadStatus.PAUSED).build(); + Mockito.when(mapper.selectById(9)).thenReturn(task); + RuntimeException failure = new RuntimeException("database unavailable"); + RuntimeException cleanup = new RuntimeException("cleanup failed"); + Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenThrow(failure); + set(service, "mapper", mapper); + try (MockedConstruction owners = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, context) -> { + LoadContext resources = Mockito.mock(LoadContext.class); + Mockito.when(loader.context()).thenReturn(resources); + Mockito.when(resources.closed()).thenReturn(true); + Mockito.doThrow(cleanup).when(loader).shutdown(); + })) { + try { + service.resume(9, "session-token"); + Assert.fail("save must fail"); + } catch (RuntimeException actual) { + Assert.assertSame(failure, actual); + } + Mockito.verify(owners.constructed().get(0)).shutdown(); + Assert.assertArrayEquals(new Throwable[]{cleanup}, failure.getSuppressed()); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + } + } + + @Test(timeout = 15000) + public void testRealWorkerTimeoutRetainsOwnerUntilContextCloses() throws Exception { + com.sun.net.httpserver.HttpServer server = com.sun.net.httpserver.HttpServer.create( + new java.net.InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/versions", exchange -> { + byte[] body = "{\"versions\":{\"core\":\"1.8.0\",\"api\":\"0.74\"}}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch running = new CountDownLatch(1); + CountDownLatch stopped = new CountDownLatch(1); + HugeGraphLoader loader = null; + Thread hook = null; + try { + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.getAddress().getPort(); + options.file = this.folder.newFile("timeout.json").getAbsolutePath(); + options.batchInsertThreads = 1; + options.singleInsertThreads = 1; + options.shutdownTimeout = 1; + loader = Mockito.spy(new HugeGraphLoader(options, + new org.apache.hugegraph.loader.mapping.LoadMapping(Collections.emptyList()))); + Mockito.doReturn(true).when(loader).load(); + Field hookField = HugeGraphLoader.class.getDeclaredField("shutdownHook"); + hookField.setAccessible(true); + hook = (Thread) hookField.get(loader); + Field managerField = HugeGraphLoader.class.getDeclaredField("manager"); + managerField.setAccessible(true); + Object manager = managerField.get(loader); + Field poolField = manager.getClass().getDeclaredField("batchService"); + poolField.setAccessible(true); + java.util.concurrent.ExecutorService pool = + (java.util.concurrent.ExecutorService) poolField.get(manager); + pool.submit(() -> { + running.countDown(); + try { + while (release.getCount() > 0) { + try { + release.await(100, TimeUnit.MILLISECONDS); + } catch (InterruptedException ignored) { + // Deliberately model a backend call that ignores cancellation. + } + } + } finally { + stopped.countDown(); + } + }); + Assert.assertTrue(running.await(5, TimeUnit.SECONDS)); + LoadTask task = this.task(loader); + task.setId(9); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.updateById(Mockito.any(LoadTask.class))).thenReturn(1); + set(service, "mapper", mapper); + set(service, "taskExecutor", new LoadTaskExecutor()); + Map owners = new ConcurrentHashMap<>(); + set(service, "runningTaskContainer", owners); + dispatch(service, task); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertFalse(loader.context().closed()); + Assert.assertSame(task, owners.get(9)); + Assert.assertFalse(pool.isTerminated()); + task.stop(); + Assert.assertSame(loader, task.getLoader()); + RuntimeException primary; + try { + loader.context().throwIfFailed(); + throw new AssertionError("timeout diagnosis is missing"); + } catch (org.apache.hugegraph.loader.exception.LoadException failure) { + primary = failure; + } + Assert.assertTrue(Runtime.getRuntime().removeShutdownHook(hook)); + Runtime.getRuntime().addShutdownHook(hook); + LoadTask restored = LoadTask.builder().id(9).options(options).status(LoadStatus.FAILED) + .lastDuration(0L).build(); + Mockito.when(mapper.selectById(9)).thenReturn(restored); + try (MockedConstruction fresh = Mockito.mockConstruction(HugeGraphLoader.class)) { + org.apache.hugegraph.testutil.Assert.assertThrows( + RuntimeException.class, () -> service.retry(9, "session-token")); + org.apache.hugegraph.testutil.Assert.assertThrows( + RuntimeException.class, () -> service.resume(9, "session-token")); + Assert.assertTrue(fresh.constructed().isEmpty()); + Assert.assertSame(task, owners.get(9)); + release.countDown(); + Assert.assertTrue(stopped.await(5, TimeUnit.SECONDS)); + try { + loader.shutdown(); + Assert.fail("original timeout must remain visible"); + } catch (org.apache.hugegraph.loader.exception.LoadException failure) { + Assert.assertSame(primary, failure); + } + Assert.assertTrue(pool.isTerminated()); + Assert.assertTrue(loader.context().closed()); + Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); + set(service, "taskExecutor", Mockito.mock(LoadTaskExecutor.class)); + service.retry(9, "session-token"); + Assert.assertEquals(1, fresh.constructed().size()); + Assert.assertSame(restored, owners.get(9)); + Assert.assertEquals(LoadStatus.RUNNING, restored.getStatus()); + fresh.constructed().get(0).shutdown(); + } + } finally { + release.countDown(); + if (loader != null) { + try { + loader.shutdown(); + } catch (org.apache.hugegraph.loader.exception.LoadException ignored) { + // Preserve the original timeout diagnosis after closing resources. + } + } + if (hook != null) { + Runtime.getRuntime().removeShutdownHook(hook); + } + server.stop(0); + } + } + + @Test + public void testUnscheduledResumeKeepsCheckpointAndFileReaderOffset() throws Exception { + com.sun.net.httpserver.HttpServer server = com.sun.net.httpserver.HttpServer.create( + new java.net.InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/versions", exchange -> { + byte[] body = "{\"versions\":{\"core\":\"1.8.0\",\"api\":\"0.74\"}}" + .getBytes(java.nio.charset.StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + }); + server.start(); + try { + java.io.File input = this.folder.newFile("checkpoint.csv"); + java.nio.file.Files.writeString(input.toPath(), "prefix0\nprefix1\nsuffix2\n"); + org.apache.hugegraph.loader.source.file.FileSource source = + new org.apache.hugegraph.loader.source.file.FileSource(); + source.path(input.getAbsolutePath()); + source.header(new String[]{"id"}); + source.hasHeader(false); + org.apache.hugegraph.loader.mapping.VertexMapping vertex = + new org.apache.hugegraph.loader.mapping.VertexMapping("id", true); + vertex.label("record"); + org.apache.hugegraph.loader.mapping.InputStruct struct = + new org.apache.hugegraph.loader.mapping.InputStruct( + Collections.singletonList(vertex), Collections.emptyList()); + struct.id("1"); + struct.input(source); + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.getAddress().getPort(); + options.file = this.folder.newFile("checkpoint.json").getAbsolutePath(); + options.incrementalMode = true; + org.apache.hugegraph.loader.util.MappingUtil.write( + new org.apache.hugegraph.loader.mapping.LoadMapping(Collections.singletonList(struct)), + options.file); + org.apache.hugegraph.loader.progress.LoadProgress checkpoint = + new org.apache.hugegraph.loader.progress.LoadProgress(); + org.apache.hugegraph.loader.progress.InputItemProgress item = + new org.apache.hugegraph.loader.progress.FileItemProgress(input.getName(), input.lastModified(), + String.valueOf(org.apache.commons.io.FileUtils.checksumCRC32(input)), 2L); + item.offset(2L); + item.confirmOffset(); + checkpoint.addStruct(struct).addLoadingItem(input.getName(), item); + checkpoint.plusVertexLoaded(2L); + java.nio.file.Path saved = java.nio.file.Paths.get( + org.apache.hugegraph.loader.progress.LoadProgress.format(options, "20000101-000000")); + java.nio.file.Files.createDirectories(saved.getParent()); + String original = org.apache.hugegraph.loader.util.JsonUtil.toJson(checkpoint); + java.nio.file.Files.writeString(saved, original); + LoadTask restored = LoadTask.builder().id(9).options(options).status(LoadStatus.PAUSED).build(); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.selectById(9)).thenReturn(restored); + RuntimeException failure = new RuntimeException("database update rejected"); + Mockito.when(mapper.update(Mockito.eq(restored), Mockito.any())).thenThrow(failure); + set(service, "mapper", mapper); + try { + service.resume(9, "session-token"); + Assert.fail("database update must fail before dispatch"); + } catch (RuntimeException actual) { + Assert.assertSame(failure, actual); + } + Assert.assertTrue(restored.context().closed()); + LoadContext next = new LoadContext(options); + try { + org.apache.hugegraph.loader.reader.file.LocalFileReader scanner = + new org.apache.hugegraph.loader.reader.file.LocalFileReader(source); + try (org.apache.hugegraph.loader.reader.InputReader reader = scanner.split().get(0)) { + reader.init(next, struct); + String first = reader.next().rawLine(); + Assert.assertEquals("suffix2", first); + Assert.assertEquals(2L, next.oldProgress().totalInputRead()); + Assert.assertEquals(2L, next.oldProgress().vertexLoaded()); + Assert.assertEquals(original, java.nio.file.Files.readString(saved)); + try (java.util.stream.Stream paths = + java.nio.file.Files.list(saved.getParent())) { + Assert.assertEquals(1L, paths.count()); + } + } + } finally { + next.close(); + } + } finally { + server.stop(0); + } + } + + @Test + public void testProgressPollRetriesTerminalCallbackFailure() throws Exception { + this.assertTerminalPersistenceRecovery(false); + } + + @Test + public void testProgressPollRetriesStopPersistenceFailure() throws Exception { + this.assertTerminalPersistenceRecovery(true); + } + + private void assertTerminalPersistenceRecovery(boolean stop) throws Exception { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + if (stop) { + task.setStatus(LoadStatus.PAUSED); + } + Mockito.when(loader.load()).thenReturn(true); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + RuntimeException callbackFailure = new RuntimeException("callback database unavailable"); + RuntimeException stopFailure = new RuntimeException("stop database unavailable"); + if (stop) { + Mockito.when(mapper.updateById(task)).thenThrow(callbackFailure).thenThrow(stopFailure).thenReturn(1); + } else { + Mockito.when(mapper.updateById(task)).thenThrow(callbackFailure).thenReturn(1); + } + set(service, "mapper", mapper); + Map owners = new ConcurrentHashMap<>(); + set(service, "runningTaskContainer", owners); + CountDownLatch release = new CountDownLatch(1); + Thread[] runner = new Thread[1]; + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + Mockito.doAnswer(invocation -> { + Runnable callback = invocation.getArgument(1); + runner[0] = new Thread(() -> { + try { + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + new LoadTaskExecutor().execute(task, callback); + } catch (Throwable failure) { + Assert.assertSame(callbackFailure, failure); + } + }); + runner[0].start(); + return null; + }).when(executor).execute(Mockito.eq(task), Mockito.any()); + set(service, "taskExecutor", executor); + dispatch(service, task); + release.countDown(); + runner[0].join(5000); + Assert.assertFalse(runner[0].isAlive()); + Assert.assertTrue(task.isFinished()); + Assert.assertSame(task, owners.get(9)); + if (stop) { + try { + service.stop(9); + Assert.fail("stop must preserve its database failure"); + } catch (RuntimeException failure) { + Assert.assertSame(stopFailure, failure); + } + } + service.updateLoadTaskProgress(); + Mockito.verify(mapper, Mockito.times(stop ? 3 : 2)).updateById(task); + Assert.assertTrue(owners.isEmpty()); + Assert.assertEquals(stop ? LoadStatus.STOPPED : LoadStatus.SUCCEED, task.getStatus()); + } + + @Test + public void testDatabaseOnlyPauseDoesNotRemoveResumedOwner() throws Exception { + this.assertDatabaseOnlyTransitionKeepsNewOwner(true); + } + + @Test + public void testDatabaseOnlyStopDoesNotRemoveRetriedOwner() throws Exception { + this.assertDatabaseOnlyTransitionKeepsNewOwner(false); + } + + private void assertDatabaseOnlyTransitionKeepsNewOwner(boolean pause) throws Exception { + LoadTask old = LoadTask.builder().id(9).options(new LoadOptions()) + .status(pause ? LoadStatus.RUNNING : LoadStatus.PAUSED).build(); + LoadTask fresh = LoadTask.builder().id(9).options(new LoadOptions()).lastDuration(0L) + .status(pause ? LoadStatus.PAUSED : LoadStatus.STOPPED).build(); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.selectById(9)).thenReturn(old, fresh); + CountDownLatch saved = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference row = + new java.util.concurrent.atomic.AtomicReference<>(); + org.mockito.stubbing.Answer saveOld = invocation -> { + row.set(old.getStatus()); + saved.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + return 1; + }; + Mockito.when(mapper.updateById(old)).thenAnswer(saveOld); + Mockito.when(mapper.update(Mockito.eq(old), Mockito.any())).thenAnswer(saveOld); + org.mockito.stubbing.Answer saveFresh = invocation -> { + row.set(fresh.getStatus()); + return 1; + }; + Mockito.when(mapper.updateById(fresh)).thenAnswer(saveFresh); + Mockito.when(mapper.update(Mockito.eq(fresh), Mockito.any())).thenAnswer(saveFresh); + set(service, "mapper", mapper); + set(service, "taskExecutor", Mockito.mock(LoadTaskExecutor.class)); + Map owners = new ConcurrentHashMap<>(); + set(service, "runningTaskContainer", owners); + java.util.concurrent.atomic.AtomicReference failure = + new java.util.concurrent.atomic.AtomicReference<>(); + Thread stopping = new Thread(() -> { + try { + if (pause) { + service.pause(9); + } else { + service.stop(9); + } + } catch (Throwable error) { + failure.set(error); + } + }); + try (MockedConstruction construction = Mockito.mockConstruction(HugeGraphLoader.class)) { + try { + stopping.start(); + Assert.assertTrue(saved.await(5, TimeUnit.SECONDS)); + if (pause) { + peer(service).resume(9, "session-token"); + } else { + peer(service).retry(9, "session-token"); + } + Assert.assertSame(fresh, owners.get(9)); + release.countDown(); + stopping.join(5000); + Assert.assertFalse(stopping.isAlive()); + Assert.assertNull(failure.get()); + Assert.assertSame(fresh, owners.get(9)); + Assert.assertEquals(LoadStatus.RUNNING, row.get()); + } finally { + release.countDown(); + stopping.join(5000); + if (!construction.constructed().isEmpty()) { + construction.constructed().get(0).shutdown(); + } + } + } + } + + @Test + public void testFinalProgressPollDoesNotSaveReplacedOwner() throws Exception { + HugeGraphLoader loader = this.loader(); + LoadTask old = Mockito.spy(this.task(loader)); + old.setId(9); + Mockito.when(loader.load()).thenReturn(false); + old.run(); + LoadTask fresh = LoadTask.builder().id(9).options(new LoadOptions()).lastDuration(0L) + .status(LoadStatus.FAILED).build(); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.selectById(9)).thenReturn(fresh); + Mockito.when(mapper.updateById(fresh)).thenReturn(1); + set(service, "mapper", mapper); + set(service, "taskExecutor", Mockito.mock(LoadTaskExecutor.class)); + Map owners = new ConcurrentHashMap<>(); + owners.put(9, old); + set(service, "runningTaskContainer", owners); + CountDownLatch selected = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + if (Thread.currentThread().getName().equals("old-owner-poll")) { + selected.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + } + return invocation.callRealMethod(); + }).when(old).lock(); + java.util.concurrent.atomic.AtomicReference failure = + new java.util.concurrent.atomic.AtomicReference<>(); + Thread poll = new Thread(() -> { + try { + service.updateLoadTaskProgress(); + } catch (Throwable error) { + failure.set(error); + } + }, "old-owner-poll"); + try (MockedConstruction construction = Mockito.mockConstruction(HugeGraphLoader.class)) { + try { + poll.start(); + Assert.assertTrue(selected.await(5, TimeUnit.SECONDS)); + service.retry(9, "session-token"); + release.countDown(); + poll.join(5000); + Assert.assertFalse(poll.isAlive()); + Assert.assertNull(failure.get()); + Mockito.verify(mapper, Mockito.never()).updateById(old); + Mockito.verify(mapper).update(Mockito.eq(fresh), Mockito.any()); + Assert.assertSame(fresh, owners.get(9)); + } finally { + release.countDown(); + poll.join(5000); + if (!construction.constructed().isEmpty()) { + construction.constructed().get(0).shutdown(); + } + } + } + } + + @Test(timeout = 15000) + public void testStopRoutesToOwnerPublishedDuringDatabaseRead() throws Exception { + this.assertOwnerPublishedDuringDatabaseRead(false); + } + + @Test(timeout = 15000) + public void testPauseRoutesToOwnerPublishedDuringDatabaseRead() throws Exception { + this.assertOwnerPublishedDuringDatabaseRead(true); + } + + private void assertOwnerPublishedDuringDatabaseRead(boolean pause) throws Exception { + org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = + new org.springframework.jdbc.datasource.DriverManagerDataSource(); + dataSource.setDriverClassName("org.h2.Driver"); + dataSource.setUrl("jdbc:h2:mem:owner-routing-" + java.util.UUID.randomUUID()); + org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(dataSource); + CountDownLatch selecting = new CountDownLatch(1); + CountDownLatch releaseRead = new CountDownLatch(1); + CountDownLatch releaseRun = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicBoolean stopped = new java.util.concurrent.atomic.AtomicBoolean(); + java.util.concurrent.atomic.AtomicBoolean closed = + new java.util.concurrent.atomic.AtomicBoolean(); + java.util.concurrent.atomic.AtomicInteger reads = new java.util.concurrent.atomic.AtomicInteger(); + java.util.concurrent.atomic.AtomicReference failure = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference returned = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference snapshot = + new java.util.concurrent.atomic.AtomicReference<>(); + Map owners = new ConcurrentHashMap<>(); + Thread[] runner = new Thread[1]; + try (java.sql.Connection keeper = dataSource.getConnection()) { + jdbc.execute("CREATE TABLE routing (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); + jdbc.update("INSERT INTO routing VALUES (9, ?)", LoadStatus.PAUSED.getValue()); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.selectById(9)).thenAnswer(invocation -> { + boolean operation = Thread.currentThread().getName().equals("routing-operation"); + if (operation) { + selecting.countDown(); + Assert.assertTrue(releaseRead.await(5, TimeUnit.SECONDS)); + } + byte code = jdbc.queryForObject("SELECT load_status FROM routing WHERE id=9", Byte.class); + LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) + .filter(value -> value.getValue() == code).findFirst().get(); + LoadTask task = LoadTask.builder().id(9).options(new LoadOptions()) + .lastDuration(0L).status(status).build(); + if (operation) { + snapshot.set(task); + } + return task; + }); + Mockito.when(mapper.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + return jdbc.update("UPDATE routing SET load_status=? WHERE id=?", + task.getStatus().getValue(), task.getId()); + }); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = + invocation.getArgument(1); + condition.getSqlSegment(); + Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Byte).findFirst().get(); + return jdbc.update("UPDATE routing SET load_status=? WHERE id=? AND load_status=?", + task.getStatus().getValue(), task.getId(), expected); + }); + LoadTaskService service = new LoadTaskService(); + set(service, "mapper", mapper); + set(service, "runningTaskContainer", owners); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + Mockito.doAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + Runnable callback = invocation.getArgument(1); + runner[0] = new Thread(() -> { + try { + Assert.assertTrue(releaseRun.await(5, TimeUnit.SECONDS)); + new LoadTaskExecutor().execute(task, callback); + } catch (Throwable error) { + failure.compareAndSet(null, error); + } + }); + runner[0].start(); + return null; + }).when(executor).execute(Mockito.any(), Mockito.any()); + set(service, "taskExecutor", executor); + Thread operation = new Thread(() -> { + try { + returned.set(pause ? service.pause(9) : service.stop(9)); + } catch (Throwable error) { + failure.compareAndSet(null, error); + } + }, "routing-operation"); + try (MockedConstruction construction = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, ignored) -> { + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(context.closed()).thenAnswer(invocation -> closed.get()); + Mockito.when(context.stopped()).thenAnswer(invocation -> stopped.get()); + Mockito.doAnswer(invocation -> { + stopped.set(true); + return null; + }).when(context).stopLoading(); + Mockito.doAnswer(invocation -> { + closed.set(true); + return null; + }).when(loader).shutdown(); + Mockito.when(loader.load()).thenAnswer(invocation -> { + if (!stopped.get()) { + reads.incrementAndGet(); + } + return true; + }); + })) { + try { + operation.start(); + Assert.assertTrue(selecting.await(5, TimeUnit.SECONDS)); + LoadTask owner = peer(service).resume(9, "session-token"); + releaseRead.countDown(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (!stopped.get() && System.nanoTime() < deadline) { + Thread.sleep(5); + } + releaseRun.countDown(); + runner[0].join(5000); + operation.join(5000); + Assert.assertFalse(runner[0].isAlive()); + Assert.assertFalse(operation.isAlive()); + Assert.assertNull(failure.get()); + Assert.assertNotSame(snapshot.get(), owner); + Assert.assertSame(owner, returned.get()); + Assert.assertTrue(stopped.get()); + Assert.assertTrue(closed.get()); + Assert.assertEquals(0, reads.get()); + LoadStatus expected = pause ? LoadStatus.PAUSED : LoadStatus.STOPPED; + Assert.assertEquals(expected, owner.getStatus()); + Assert.assertEquals(expected.getValue(), jdbc.queryForObject( + "SELECT load_status FROM routing WHERE id=9", Byte.class)); + Assert.assertTrue(owners.isEmpty()); + } finally { + releaseRead.countDown(); + releaseRun.countDown(); + if (runner[0] != null) { + runner[0].join(5000); + } + operation.join(5000); + for (HugeGraphLoader loader : construction.constructed()) { + loader.shutdown(); + } + } + } + } + } + + @Test(timeout = 15000) + public void testConditionalResumeRejectsCommittedDatabaseOnlyStop() throws Exception { + this.assertConditionalTransitionWinner(true); + } + + @Test(timeout = 15000) + public void testConditionalDatabaseOnlyStopRejectsCommittedResume() throws Exception { + this.assertConditionalTransitionWinner(false); + } + + @Test(timeout = 15000) + public void testConditionalDatabaseOnlyPauseRejectsCommittedStop() throws Exception { + this.assertConditionalTransitionWinner(true, true); + } + + private void assertConditionalTransitionWinner(boolean stopWins) throws Exception { + this.assertConditionalTransitionWinner(stopWins, false); + } + + private void assertConditionalTransitionWinner(boolean stopWins, boolean pauseLoser) throws Exception { + org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = + new org.springframework.jdbc.datasource.DriverManagerDataSource(); + dataSource.setDriverClassName("org.h2.Driver"); + dataSource.setUrl("jdbc:h2:mem:state-winner-" + java.util.UUID.randomUUID()); + org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(dataSource); + CountDownLatch selected = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference failure = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference snapshot = + new java.util.concurrent.atomic.AtomicReference<>(); + Map owners = new ConcurrentHashMap<>(); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + try (java.sql.Connection keeper = dataSource.getConnection()) { + jdbc.execute("CREATE TABLE state_transition (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); + jdbc.update("INSERT INTO state_transition VALUES (9, ?)", + (pauseLoser ? LoadStatus.RUNNING : LoadStatus.PAUSED).getValue()); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.selectById(9)).thenAnswer(invocation -> { + byte state = jdbc.queryForObject("SELECT load_status FROM state_transition WHERE id=9", Byte.class); + LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) + .filter(value -> value.getValue() == state).findFirst().get(); + LoadTask task = LoadTask.builder().id(9).options(new LoadOptions()) + .status(status).lastDuration(0L).build(); + if (Thread.currentThread().getName().equals("delayed-transition")) { + snapshot.set(task); + selected.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + } + return task; + }); + Mockito.when(mapper.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + return jdbc.update("UPDATE state_transition SET load_status=? WHERE id=?", + task.getStatus().getValue(), task.getId()); + }); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = + invocation.getArgument(1); + Assert.assertTrue(condition.getSqlSegment().contains("id =")); + Assert.assertTrue(condition.getSqlSegment().contains("load_status =")); + Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Byte).findFirst().get(); + Assert.assertTrue(condition.getParamNameValuePairs().containsValue(9)); + return jdbc.update("UPDATE state_transition SET load_status=? WHERE id=? AND load_status=?", + task.getStatus().getValue(), task.getId(), expected); + }); + LoadTaskService service = new LoadTaskService(); + set(service, "mapper", mapper); + set(service, "taskExecutor", executor); + set(service, "runningTaskContainer", owners); + Thread delayed = new Thread(() -> { + try (MockedConstruction construction = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, ignored) -> { + LoadContext context = Mockito.mock(LoadContext.class); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(context.closed()).thenReturn(true); + })) { + if (pauseLoser) { + service.pause(9); + } else if (stopWins) { + service.resume(9, "session-token"); + } else { + service.stop(9); + } + } catch (Throwable error) { + failure.set(error); + } + }, "delayed-transition"); + try (MockedConstruction construction = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, ignored) -> { + LoadContext context = Mockito.mock(LoadContext.class); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(context.closed()).thenReturn(true); + })) { + try { + delayed.start(); + Assert.assertTrue(selected.await(5, TimeUnit.SECONDS)); + LoadTaskService competing = peer(service); + LoadTask winner = stopWins ? competing.stop(9) : competing.resume(9, "session-token"); + LoadStatus expected = stopWins ? LoadStatus.STOPPED : LoadStatus.RUNNING; + Assert.assertEquals(expected, winner.getStatus()); + release.countDown(); + delayed.join(5000); + Assert.assertFalse(delayed.isAlive()); + Assert.assertTrue(failure.get() instanceof org.apache.hugegraph.exception.ExternalException); + Assert.assertTrue(failure.get().getMessage().contains("state has changed")); + Assert.assertEquals(expected.getValue(), jdbc.queryForObject( + "SELECT load_status FROM state_transition WHERE id=9", Byte.class)); + Assert.assertNotSame(winner, snapshot.get()); + if (stopWins) { + if (!pauseLoser) { + Assert.assertEquals(LoadStatus.FAILED, snapshot.get().getStatus()); + Assert.assertTrue(snapshot.get().isFinished()); + Assert.assertTrue(snapshot.get().context().closed()); + } + Assert.assertTrue(owners.isEmpty()); + Mockito.verifyNoInteractions(executor); + } else { + Assert.assertSame(winner, owners.get(9)); + Mockito.verify(executor).execute(Mockito.eq(winner), Mockito.any()); + } + } finally { + release.countDown(); + delayed.join(5000); + for (HugeGraphLoader loader : construction.constructed()) { + loader.shutdown(); + } + } + } + } + } + + @Test + public void testUnscheduledCleanupFailureRetainsOwnerAndRejectsResume() throws Exception { + LoadTask task = LoadTask.builder().id(9).options(new LoadOptions()).status(LoadStatus.PAUSED).build(); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.selectById(9)).thenReturn(task); + RuntimeException primary = new RuntimeException("save rejected"); + RuntimeException cleanup = new RuntimeException("resources still open"); + Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenThrow(primary); + set(service, "mapper", mapper); + Map owners = new ConcurrentHashMap<>(); + set(service, "runningTaskContainer", owners); + try (MockedConstruction construction = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, ignored) -> { + LoadContext context = Mockito.mock(LoadContext.class); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(context.closed()).thenReturn(false); + Mockito.doThrow(cleanup).when(loader).shutdown(); + })) { + try { + service.resume(9, "session-token"); + Assert.fail("save must fail"); + } catch (RuntimeException actual) { + Assert.assertSame(primary, actual); + } + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + Assert.assertFalse(task.context().closed()); + Assert.assertSame(task, owners.get(9)); + Assert.assertArrayEquals(new Throwable[]{cleanup}, primary.getSuppressed()); + try { + service.resume(9, "session-token"); + Assert.fail("the old Loader must close before a new one is constructed"); + } catch (RuntimeException expected) { + Assert.assertTrue(expected.getMessage().contains("previous Loader has not closed")); + } + Assert.assertEquals(1, construction.constructed().size()); + Assert.assertSame(task, owners.get(9)); + } + } + + @Test(timeout = 15000) + public void testCapturedStopFinishesWhenResumeSaveFailsBeforeDispatch() throws Exception { + LoadTask restored = LoadTask.builder().id(9).options(new LoadOptions()) + .status(LoadStatus.PAUSED).lastDuration(0L).build(); + CountDownLatch updating = new CountDownLatch(1); + CountDownLatch selected = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + Map owners = new ConcurrentHashMap() { + @Override + public LoadTask get(Object id) { + LoadTask owner = super.get(id); + if (owner != null && Thread.currentThread().getName().equals("unscheduled-stop")) { + selected.countDown(); + } + return owner; + } + }; + RuntimeException primary = new RuntimeException("resume update rejected"); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.selectById(9)).thenReturn(restored); + Mockito.when(mapper.update(Mockito.eq(restored), Mockito.any())).thenAnswer(invocation -> { + updating.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + throw primary; + }); + LoadTaskService service = new LoadTaskService(); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + set(service, "mapper", mapper); + set(service, "taskExecutor", executor); + set(service, "runningTaskContainer", owners); + java.util.concurrent.atomic.AtomicReference resumeFailure = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference stopFailure = + new java.util.concurrent.atomic.AtomicReference<>(); + Thread resume = new Thread(() -> { + try (MockedConstruction construction = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, ignored) -> { + LoadContext context = Mockito.mock(LoadContext.class); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(context.closed()).thenReturn(true); + })) { + service.resume(9, "session-token"); + } catch (Throwable failure) { + resumeFailure.set(failure); + } + }, "unscheduled-resume"); + Thread stop = new Thread(() -> { + try { + service.stop(9); + } catch (Throwable failure) { + stopFailure.set(failure); + } + }, "unscheduled-stop"); + stop.setDaemon(true); + try { + resume.start(); + Assert.assertTrue(updating.await(5, TimeUnit.SECONDS)); + stop.start(); + Assert.assertFalse("Stop waits for admission before selecting the unscheduled owner", + selected.await(300, TimeUnit.MILLISECONDS)); + Assert.assertSame(restored, owners.get(9)); + release.countDown(); + resume.join(5000); + stop.join(1500); + Assert.assertFalse(resume.isAlive()); + Assert.assertFalse("No unscheduled runner remains to finish this stop waiter", stop.isAlive()); + Assert.assertSame(primary, resumeFailure.get()); + Assert.assertNotNull("FAILED owners cannot accept a stop request", stopFailure.get()); + Assert.assertEquals(LoadStatus.FAILED, restored.getStatus()); + Assert.assertTrue(restored.isFinished()); + Assert.assertTrue(restored.context().closed()); + Assert.assertFalse(owners.containsKey(9)); + Mockito.verifyNoInteractions(executor); + } finally { + release.countDown(); + restored.setFinished(true); + resume.join(5000); + stop.join(5000); + } + } + + @Test + public void testPublishedRestartIsLockedUntilInitialized() throws Exception { + LoadTask restored = LoadTask.builder().id(9).options(new LoadOptions()) + .status(LoadStatus.PAUSED).lastDuration(0L).build(); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + Mockito.when(mapper.selectById(9)).thenReturn(restored); + Mockito.when(mapper.updateById(restored)).thenReturn(1); + set(service, "mapper", mapper); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + set(service, "taskExecutor", executor); + java.util.concurrent.atomic.AtomicBoolean locked = new java.util.concurrent.atomic.AtomicBoolean(); + Map owners = new ConcurrentHashMap() { + @Override + public LoadTask putIfAbsent(Integer id, LoadTask task) { + LoadTask previous = super.putIfAbsent(id, task); + if (previous == null) { + Thread observer = new Thread(() -> { + try { + Field field = LoadTask.class.getDeclaredField("lock"); + field.setAccessible(true); + java.util.concurrent.locks.ReentrantLock lock = + (java.util.concurrent.locks.ReentrantLock) field.get(task); + boolean acquired = lock.tryLock(); + if (acquired) { + lock.unlock(); + } + locked.set(!acquired); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + }); + observer.start(); + try { + observer.join(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + Assert.assertFalse(observer.isAlive()); + } + return previous; + } + }; + set(service, "runningTaskContainer", owners); + try (MockedConstruction construction = Mockito.mockConstruction(HugeGraphLoader.class)) { + service.resume(9, "session-token"); + Assert.assertTrue("The published owner must already hold its task lock", locked.get()); + Assert.assertSame(restored, owners.get(9)); + Assert.assertSame(construction.constructed().get(0), restored.getLoader()); + Assert.assertEquals(LoadStatus.RUNNING, restored.getStatus()); + Mockito.verify(executor).execute(Mockito.eq(restored), Mockito.any()); + } + } + + @Test + public void testClosedCallbackKeepsEqualButDistinctOwner() throws Exception { + LoadTask old = this.task(this.loader()); + old.setId(9); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.updateById(old)).thenReturn(1); + set(service, "mapper", mapper); + Map owners = new ConcurrentHashMap<>(); + set(service, "runningTaskContainer", owners); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + java.util.concurrent.atomic.AtomicReference callback = + new java.util.concurrent.atomic.AtomicReference<>(); + Mockito.doAnswer(invocation -> { + callback.set(invocation.getArgument(1)); + return null; + }).when(executor).execute(Mockito.eq(old), Mockito.any()); + set(service, "taskExecutor", executor); + dispatch(service, old); + old.run(); + LoadTask replacement = this.equalRuntimeReplacement(old); + owners.put(9, replacement); + callback.get().run(); + Assert.assertEquals(old, replacement); + Assert.assertSame(replacement, owners.get(9)); + Mockito.verify(mapper).updateById(old); + } + + @Test + public void testNoTransactionRemovalKeepsEqualButDistinctOwner() throws Exception { + LoadTask old = this.task(this.loader()); + old.setId(9); + old.run(); + LoadTask replacement = this.equalRuntimeReplacement(old); + LoadTaskService service = new LoadTaskService(); + Map owners = new ConcurrentHashMap<>(); + owners.put(9, replacement); + set(service, "runningTaskContainer", owners); + Assert.assertFalse(TransactionSynchronizationManager.isSynchronizationActive()); + Method remove = LoadTaskService.class.getDeclaredMethod("removeOwnerAfterCommit", LoadTask.class); + remove.setAccessible(true); + remove.invoke(service, old); + Assert.assertSame(replacement, owners.get(9)); + } + + @Test + public void testCommittedPollKeepsEqualButDistinctOwner() throws Exception { + this.assertPollCommitBoundary(false, true); + } + + private LoadTask equalRuntimeReplacement(LoadTask previous) { + HugeGraphLoader loader = this.loader(); + Mockito.when(loader.context().closed()).thenReturn(false); + LoadTask replacement = LoadTask.builder().id(previous.getId()).options(previous.getOptions()) + .loader(loader).finished(false).status(previous.getStatus()).fileReadLines(previous.getFileReadLines()) + .lastDuration(previous.getLastDuration()).currDuration(previous.getCurrDuration()).build(); + Assert.assertEquals(previous, replacement); + Assert.assertNotSame(previous, replacement); + Assert.assertNotSame(previous.getLoader(), replacement.getLoader()); + Assert.assertTrue(previous.isFinished()); + Assert.assertFalse(replacement.isFinished()); + return replacement; + } + + @Test + public void testPollTransactionRollbackRetainsBothFinishedOwners() throws Exception { + this.assertPollCommitBoundary(false); + } + + @Test + public void testPollCommitFailureRetainsBothFinishedOwners() throws Exception { + this.assertPollCommitBoundary(true); + } + + private void assertPollCommitBoundary(boolean commitFailure) throws Exception { + this.assertPollCommitBoundary(commitFailure, false); + } + + private void assertPollCommitBoundary(boolean commitFailure, boolean replaceBeforeCommit) throws Exception { + org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = + new org.springframework.jdbc.datasource.DriverManagerDataSource(); + dataSource.setDriverClassName("org.h2.Driver"); + dataSource.setUrl("jdbc:h2:mem:cleanup-poll-" + java.util.UUID.randomUUID()); + org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(dataSource); + try (java.sql.Connection keeper = dataSource.getConnection()) { + jdbc.execute("CREATE TABLE task_progress (id INT PRIMARY KEY, load_status VARCHAR(16))"); + jdbc.update("INSERT INTO task_progress VALUES (1, 'RUNNING'), (2, 'RUNNING')"); + LoadTask first = this.task(this.loader()); + LoadTask second = this.task(this.loader()); + first.setId(1); + second.setId(2); + first.run(); + second.run(); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); + java.util.concurrent.atomic.AtomicBoolean fail = new java.util.concurrent.atomic.AtomicBoolean(true); + RuntimeException failure = new RuntimeException("later owner SQL failed"); + Mockito.when(mapper.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + if (!commitFailure && task == second && fail.get()) { + throw failure; + } + return jdbc.update("UPDATE task_progress SET load_status=? WHERE id=?", + task.getStatus().name(), task.getId()); + }); + Map owners = new java.util.concurrent.ConcurrentSkipListMap<>(); + owners.put(1, first); + owners.put(2, second); + set(service, "mapper", mapper); + set(service, "runningTaskContainer", owners); + org.springframework.jdbc.datasource.DataSourceTransactionManager manager = + new org.springframework.jdbc.datasource.DataSourceTransactionManager(dataSource) { + @Override + protected void doCommit( + org.springframework.transaction.support.DefaultTransactionStatus status) { + if (commitFailure && fail.get()) { + throw failure; + } + super.doCommit(status); + } + }; + manager.setRollbackOnCommitFailure(true); + org.springframework.transaction.support.TransactionTemplate transaction = + new org.springframework.transaction.support.TransactionTemplate(manager); + try { + transaction.execute(status -> { + service.updateLoadTaskProgress(); + return null; + }); + Assert.fail("transaction must fail"); + } catch (RuntimeException actual) { + Assert.assertSame(failure, actual); + } + Assert.assertEquals("RUNNING", jdbc.queryForObject( + "SELECT load_status FROM task_progress WHERE id=1", String.class)); + Assert.assertEquals("RUNNING", jdbc.queryForObject( + "SELECT load_status FROM task_progress WHERE id=2", String.class)); + Assert.assertSame(first, owners.get(1)); + Assert.assertSame(second, owners.get(2)); + fail.set(false); + LoadTask replacement = replaceBeforeCommit ? this.equalRuntimeReplacement(first) : null; + transaction.execute(status -> { + service.updateLoadTaskProgress(); + Assert.assertSame(first, owners.get(1)); + Assert.assertSame(second, owners.get(2)); + if (replaceBeforeCommit) { + owners.put(1, replacement); + } + return null; + }); + if (replaceBeforeCommit) { + Assert.assertSame(replacement, owners.get(1)); + Assert.assertNull(owners.get(2)); + } else { + Assert.assertTrue(owners.isEmpty()); + } + Assert.assertEquals("FAILED", jdbc.queryForObject( + "SELECT load_status FROM task_progress WHERE id=1", String.class)); + Assert.assertEquals("FAILED", jdbc.queryForObject( + "SELECT load_status FROM task_progress WHERE id=2", String.class)); + } + } + + @Test(timeout = 15000) + public void testTaskAdmissionPreventsPauseResumeStatusAba() throws Exception { + org.springframework.jdbc.datasource.DriverManagerDataSource ds = + new org.springframework.jdbc.datasource.DriverManagerDataSource(); + ds.setDriverClassName("org.h2.Driver"); + ds.setUrl("jdbc:h2:mem:admission-" + java.util.UUID.randomUUID()); + org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(ds); + CountDownLatch saving = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch competitors = new CountDownLatch(2); + java.util.concurrent.atomic.AtomicInteger reads = new java.util.concurrent.atomic.AtomicInteger(); + java.util.concurrent.atomic.AtomicReference stopFailure = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference pauseFailure = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference resumeFailure = + new java.util.concurrent.atomic.AtomicReference<>(); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + Map owners = new ConcurrentHashMap<>(); + set(service, "mapper", mapper); + set(service, "taskExecutor", executor); + set(service, "runningTaskContainer", owners); + try (java.sql.Connection keeper = ds.getConnection()) { + jdbc.execute("CREATE TABLE admission (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); + jdbc.update("INSERT INTO admission VALUES (9, ?)", LoadStatus.RUNNING.getValue()); + Mockito.when(mapper.selectById(9)).thenAnswer(invocation -> { + reads.incrementAndGet(); + byte code = jdbc.queryForObject("SELECT load_status FROM admission WHERE id=9", Byte.class); + LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) + .filter(value -> value.getValue() == code).findFirst().get(); + return LoadTask.builder().id(9).options(new LoadOptions()).status(status).build(); + }); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = + invocation.getArgument(1); + condition.getSqlSegment(); + Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Byte).findFirst().get(); + saving.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + return jdbc.update("UPDATE admission SET load_status=? WHERE id=? AND load_status=?", + task.getStatus().getValue(), task.getId(), expected); + }); + Thread stop = new Thread(() -> { + try { + service.stop(9); + } catch (Throwable error) { + stopFailure.set(error); + } + }); + Thread pause = new Thread(() -> { + competitors.countDown(); + try { + service.pause(9); + } catch (Throwable error) { + pauseFailure.set(error); + } + }); + Thread resume = new Thread(() -> { + competitors.countDown(); + try { + service.resume(9, "session-token"); + } catch (Throwable error) { + resumeFailure.set(error); + } + }); + try { + stop.start(); + Assert.assertTrue(saving.await(5, TimeUnit.SECONDS)); + pause.start(); + resume.start(); + Assert.assertTrue(competitors.await(5, TimeUnit.SECONDS)); + Thread.sleep(100); + Assert.assertTrue(pause.isAlive()); + Assert.assertTrue(resume.isAlive()); + Assert.assertEquals("Later controls cannot read an ABA snapshot", 1, reads.get()); + Assert.assertEquals(LoadStatus.RUNNING.getValue(), jdbc.queryForObject( + "SELECT load_status FROM admission WHERE id=9", Byte.class)); + Mockito.verifyNoInteractions(executor); + } finally { + release.countDown(); + stop.join(5000); + pause.join(5000); + resume.join(5000); + } + Assert.assertFalse(stop.isAlive()); + Assert.assertFalse(pause.isAlive()); + Assert.assertFalse(resume.isAlive()); + Assert.assertNull(stopFailure.get()); + Assert.assertTrue(pauseFailure.get() instanceof org.apache.hugegraph.exception.ExternalException); + Assert.assertTrue(resumeFailure.get() instanceof org.apache.hugegraph.exception.ExternalException); + Assert.assertEquals(LoadStatus.STOPPED.getValue(), jdbc.queryForObject( + "SELECT load_status FROM admission WHERE id=9", Byte.class)); + Assert.assertTrue(owners.isEmpty()); + Mockito.verifyNoInteractions(executor); + } + } + + @Test(timeout = 15000) + public void testInitialPublicationWaitsForTaskAdmissionAfterCommit() throws Exception { + LoadTaskService service = new LoadTaskService(); + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + set(service, "taskExecutor", executor); + Map owners = new ConcurrentHashMap<>(); + set(service, "runningTaskContainer", owners); + Field field = LoadTaskService.class.getDeclaredField("taskControls"); + field.setAccessible(true); + com.google.common.util.concurrent.Striped controls = + (com.google.common.util.concurrent.Striped) field.get(service); + java.util.concurrent.locks.Lock control = controls.get(9); + CountDownLatch committed = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference failure = + new java.util.concurrent.atomic.AtomicReference<>(); + Thread publication = new Thread(() -> { + TransactionSynchronizationManager.initSynchronization(); + try { + dispatch(service, task); + TransactionSynchronization synchronization = + TransactionSynchronizationManager.getSynchronizations().get(0); + committed.countDown(); + synchronization.afterCommit(); + synchronization.afterCompletion(TransactionSynchronization.STATUS_COMMITTED); + } catch (Throwable error) { + failure.set(error); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + }); + control.lock(); + try { + publication.start(); + Assert.assertTrue(committed.await(5, TimeUnit.SECONDS)); + Thread.sleep(100); + Assert.assertTrue(publication.isAlive()); + Assert.assertTrue(owners.isEmpty()); + Mockito.verifyNoInteractions(executor); + int independentId = 10; + while (controls.get(independentId) == control) { + independentId++; + } + LoadTask independent = this.task(this.loader()); + independent.setId(independentId); + dispatch(service, independent); + Assert.assertSame(independent, owners.get(independentId)); + Mockito.verify(executor).execute(Mockito.eq(independent), Mockito.any()); + } finally { + control.unlock(); + publication.join(5000); + loader.shutdown(); + } + Assert.assertFalse(publication.isAlive()); + Assert.assertNull(failure.get()); + Assert.assertSame(task, owners.get(9)); + Mockito.verify(executor).execute(Mockito.eq(task), Mockito.any()); + } + + @Test(timeout = 15000) + public void testInitialPublicationHonorsCommittedStop() throws Exception { + this.assertInitialPublicationHonorsTerminalRequest(false, false); + } + + @Test(timeout = 15000) + public void testInitialPublicationHonorsCommittedPause() throws Exception { + this.assertInitialPublicationHonorsTerminalRequest(true, false); + } + + @Test(timeout = 15000) + public void testCancelledInitialPublicationRetainsUnclosedOwner() throws Exception { + this.assertInitialPublicationHonorsTerminalRequest(false, true); + } + + @Test(timeout = 15000) + public void testCancelledInitialPublicationRetainsStillOpenContext() throws Exception { + this.assertInitialPublicationHonorsTerminalRequest(false, true, false); + } + + private void assertInitialPublicationHonorsTerminalRequest(boolean pause, boolean cleanupFailure) throws Exception { + this.assertInitialPublicationHonorsTerminalRequest(pause, cleanupFailure, true); + } + + private void assertInitialPublicationHonorsTerminalRequest(boolean pause, boolean cleanupFailure, + boolean throwsOnCleanup) throws Exception { + org.springframework.jdbc.datasource.DriverManagerDataSource ds = + new org.springframework.jdbc.datasource.DriverManagerDataSource(); + ds.setDriverClassName("org.h2.Driver"); + ds.setUrl("jdbc:h2:mem:late-publication-" + java.util.UUID.randomUUID()); + org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(ds); + HugeGraphLoader loader = this.loader(); + Mockito.when(loader.context().closed()).thenReturn(!cleanupFailure); + if (cleanupFailure && throwsOnCleanup) { + Mockito.doThrow(new RuntimeException("worker cleanup failed")).when(loader).shutdown(); + } + LoadTask initial = this.task(loader); + initial.setId(9); + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + Map owners = new ConcurrentHashMap<>(); + set(service, "mapper", mapper); + set(service, "taskExecutor", executor); + set(service, "runningTaskContainer", owners); + CountDownLatch registered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference failure = + new java.util.concurrent.atomic.AtomicReference<>(); + try (java.sql.Connection keeper = ds.getConnection()) { + jdbc.execute("CREATE TABLE initial_task (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); + jdbc.update("INSERT INTO initial_task VALUES (9, ?)", LoadStatus.RUNNING.getValue()); + Mockito.when(mapper.selectById(9)).thenAnswer(invocation -> { + byte code = jdbc.queryForObject("SELECT load_status FROM initial_task WHERE id=9", Byte.class); + LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) + .filter(value -> value.getValue() == code).findFirst().get(); + return LoadTask.builder().id(9).options(new LoadOptions()).status(status).build(); + }); + Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = + invocation.getArgument(1); + condition.getSqlSegment(); + Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Byte).findFirst().get(); + return jdbc.update("UPDATE initial_task SET load_status=? WHERE id=? AND load_status=?", + task.getStatus().getValue(), task.getId(), expected); + }); + Thread publication = new Thread(() -> { + TransactionSynchronizationManager.initSynchronization(); + try { + dispatch(service, initial); + TransactionSynchronization synchronization = + TransactionSynchronizationManager.getSynchronizations().get(0); + registered.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + synchronization.afterCommit(); + synchronization.afterCompletion(TransactionSynchronization.STATUS_COMMITTED); + } catch (Throwable error) { + failure.set(error); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + }); + try { + publication.start(); + Assert.assertTrue(registered.await(5, TimeUnit.SECONDS)); + LoadTask terminal = pause ? service.pause(9) : service.stop(9); + Assert.assertEquals(pause ? LoadStatus.PAUSED : LoadStatus.STOPPED, terminal.getStatus()); + Assert.assertNotSame(initial, terminal); + } finally { + release.countDown(); + publication.join(5000); + } + Assert.assertFalse(publication.isAlive()); + Assert.assertNull(failure.get()); + LoadStatus expected = pause ? LoadStatus.PAUSED : LoadStatus.STOPPED; + Assert.assertEquals(expected.getValue(), jdbc.queryForObject( + "SELECT load_status FROM initial_task WHERE id=9", Byte.class)); + Assert.assertEquals(cleanupFailure ? LoadStatus.FAILED : expected, initial.getStatus()); + Assert.assertTrue(initial.isFinished()); + Mockito.verify(loader).shutdown(); + Mockito.verify(loader, Mockito.never()).load(); + Mockito.verifyNoInteractions(executor); + if (cleanupFailure) { + Assert.assertSame(initial, owners.get(9)); + Assert.assertFalse(initial.context().closed()); + } else { + Assert.assertTrue(initial.context().closed()); + Assert.assertTrue(owners.isEmpty()); + } + } + } + + @Test + public void testStaleInitialPublicationDoesNotDispatchMissingOrTerminalRow() throws Exception { + for (LoadStatus status : new LoadStatus[]{null, LoadStatus.FAILED, LoadStatus.SUCCEED, LoadStatus.INIT}) { + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.selectById(9)).thenReturn(status == null ? null : + LoadTask.builder().id(9).status(status).build()); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + HugeGraphLoader loader = this.loader(); + LoadTask old = this.task(loader); + old.setId(9); + set(service, "mapper", mapper); + set(service, "taskExecutor", executor); + dispatch(service, old); + Assert.assertTrue(old.isFinished()); + Assert.assertTrue(loader.context().closed()); + Mockito.verify(loader, Mockito.never()).load(); + Mockito.verify(mapper, Mockito.never()).updateById(Mockito.any(LoadTask.class)); + Mockito.verify(mapper, Mockito.never()).update(Mockito.any(), Mockito.any()); + Mockito.verifyNoInteractions(executor); + } + } + + @Test + public void testStaleInitialPublicationClosesOnlyItsOwnLoader() throws Exception { + LoadTaskService service = new LoadTaskService(); + HugeGraphLoader oldLoader = this.loader(); + HugeGraphLoader newLoader = this.loader(); + LoadTask old = this.task(oldLoader); + old.setId(9); + LoadTask fresh = LoadTask.builder().id(9).options(old.getOptions()).lastDuration(0L) + .status(LoadStatus.RUNNING).loader(newLoader).build(); + Assert.assertEquals(old, fresh); + Assert.assertNotSame(old, fresh); + Map owners = new ConcurrentHashMap<>(); + owners.put(9, fresh); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.selectById(9)).thenReturn(fresh); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + set(service, "mapper", mapper); + set(service, "taskExecutor", executor); + set(service, "runningTaskContainer", owners); + dispatch(service, old); + Assert.assertSame(fresh, owners.get(9)); + Assert.assertEquals(LoadStatus.RUNNING, fresh.getStatus()); + Assert.assertFalse(fresh.isFinished()); + Assert.assertFalse(newLoader.context().closed()); + Assert.assertTrue(old.isFinished()); + Assert.assertTrue(oldLoader.context().closed()); + Mockito.verify(newLoader, Mockito.never()).shutdown(); + Mockito.verifyNoInteractions(executor); + Mockito.verify(mapper, Mockito.never()).update(Mockito.any(), Mockito.any()); + Mockito.verify(mapper, Mockito.never()).updateById(Mockito.any(LoadTask.class)); + } + + @Test + public void testInitialPublicationRejectsUnusableOwnLoader() throws Exception { + for (int state = 0; state < 4; state++) { + LoadTaskService service = new LoadTaskService(); + HugeGraphLoader loader = this.loader(); + LoadTask old = this.task(loader); + old.setId(9); + if (state == 0) { + old.setFinished(true); + } else if (state == 1) { + old.setLoader(null); + } else if (state == 2) { + Mockito.when(loader.context().closed()).thenReturn(true); + } else { + Mockito.when(loader.context().stopped()).thenReturn(true); + } + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.selectById(9)).thenReturn(LoadTask.builder().id(9).status(LoadStatus.RUNNING).build()); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + set(service, "mapper", mapper); + set(service, "taskExecutor", executor); + dispatch(service, old); + Assert.assertTrue(old.isFinished()); + Mockito.verifyNoInteractions(executor); + Mockito.verify(mapper, Mockito.never()).update(Mockito.any(), Mockito.any()); + } + } + + @Test + public void testDeleteRejectsTerminalSnapshotWithUnclosedOwner() throws Exception { + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.selectById(9)).thenReturn(LoadTask.builder().id(9).status(LoadStatus.PAUSED).build()); + LoadTask owner = this.task(this.loader()); + owner.setId(9); + Map owners = new ConcurrentHashMap<>(); + owners.put(9, owner); + set(service, "mapper", mapper); + set(service, "runningTaskContainer", owners); + org.apache.hugegraph.testutil.Assert.assertThrows(RuntimeException.class, () -> service.remove(9)); + Assert.assertSame(owner, owners.get(9)); + Mockito.verify(mapper, Mockito.never()).delete(Mockito.any()); + } + + @Test + public void testJobDeletionCommitsStopsBeforeCallerTransactionCompletes() throws Exception { + for (boolean scoped : new boolean[]{false, true}) { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + new org.springframework.transaction.support.TransactionTemplate(fixture.manager).execute(status -> { + fixture.delete(scoped); + Assert.assertTrue(TransactionSynchronizationManager.isActualTransactionActive()); + // Independent connections must see committed deletion before the caller commits. + try (java.sql.Connection other = fixture.dataSource.getConnection(); + java.sql.Statement statement = other.createStatement(); + java.sql.ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM delete_tasks")) { + Assert.assertTrue(rows.next()); + Assert.assertEquals(0, rows.getInt(1)); + } catch (java.sql.SQLException failure) { + throw new RuntimeException(failure); + } + status.setRollbackOnly(); + return null; + }); + Assert.assertEquals(0, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_tasks", Integer.class) + .intValue()); + Assert.assertEquals(0, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_jobs", Integer.class) + .intValue()); + } + } + } + + @Test + public void testMetadataRollbackKeepsCommittedStopsAndAllTaskRows() throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + RuntimeException primary = new RuntimeException("job metadata delete rejected"); + Mockito.doThrow(primary).when(fixture.jobs).deleteById(7); + try { + fixture.delete(true); + Assert.fail("metadata must roll back"); + } catch (RuntimeException actual) { + Assert.assertSame(primary, actual); + } + Assert.assertEquals(2, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_tasks", Integer.class) + .intValue()); + Assert.assertEquals(2, fixture.jdbc.queryForObject( + "SELECT COUNT(*) FROM delete_tasks WHERE load_status=?", Integer.class, + LoadStatus.STOPPED.getValue()).intValue()); + Assert.assertEquals(1, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_jobs", Integer.class) + .intValue()); + } + } + + @Test + public void testRetryWinningBeforeMetadataDeleteRollsBackEarlierTaskDeletion() throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + LoadTask fresh = this.task(this.loader()); + fresh.setId(9); + Mockito.doAnswer(invocation -> { + com.baomidou.mybatisplus.core.conditions.query.QueryWrapper condition = + invocation.getArgument(0); + condition.getSqlSegment(); + Integer id = (Integer) condition.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Integer).findFirst().get(); + Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Byte).findFirst().get(); + int deleted = fixture.jdbc.update("DELETE FROM delete_tasks WHERE id=? AND load_status=?", + id, expected); + if (id == 10) { + // A separately committed restart wins before the next task is admitted for deletion. + try (java.sql.Connection other = fixture.dataSource.getConnection(); + java.sql.PreparedStatement update = other.prepareStatement( + "UPDATE delete_tasks SET load_status=? WHERE id=9 AND load_status=?")) { + update.setByte(1, LoadStatus.RUNNING.getValue()); + update.setByte(2, LoadStatus.STOPPED.getValue()); + Assert.assertEquals(1, update.executeUpdate()); + } + fixture.owners.put(9, fresh); + } + return deleted; + }).when(fixture.tasks).delete(Mockito.any()); + org.apache.hugegraph.testutil.Assert.assertThrows(RuntimeException.class, () -> fixture.delete(false)); + Assert.assertSame(fresh, fixture.owners.get(9)); + Assert.assertFalse(fresh.context().closed()); + Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); + Assert.assertEquals(LoadStatus.STOPPED.getValue(), fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=10", Byte.class)); + Assert.assertEquals(1, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_jobs", Integer.class) + .intValue()); + } + } + + @Test + public void testOwnDispatchRejectionConditionallyMarksTaskAndJobFailed() throws Exception { + this.assertOwnDispatchFailureState(false); + } + + @Test + public void testOwnDispatchRejectionDoesNotOverwriteAdvancedTaskState() throws Exception { + this.assertOwnDispatchFailureState(true); + } + + private void assertOwnDispatchFailureState(boolean advanced) throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + task.setJobId(7); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + Mockito.doAnswer(invocation -> { + if (advanced) { + fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=9", + LoadStatus.SUCCEED.getValue()); + } + throw new java.util.concurrent.RejectedExecutionException("own dispatch rejected"); + }).when(executor).execute(Mockito.eq(task), Mockito.any()); + set(fixture.load, "taskExecutor", executor); + set(fixture.load, "transactionManager", fixture.manager); + set(fixture.load, "jobManagerMapper", fixture.jobs); + Method method = LoadTaskService.class.getDeclaredMethod("executeAfterCommit", LoadTask.class); + method.setAccessible(true); + method.invoke(fixture.load, task); + Assert.assertEquals(advanced ? LoadStatus.SUCCEED.getValue() : LoadStatus.FAILED.getValue(), + fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); + Assert.assertEquals(advanced ? org.apache.hugegraph.entity.enums.JobStatus.LOADING.getValue() : + org.apache.hugegraph.entity.enums.JobStatus.FAILED.getValue(), + fixture.jdbc.queryForObject("SELECT status FROM delete_jobs WHERE id=7", Byte.class)); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + Assert.assertTrue(loader.context().closed()); + Assert.assertFalse(fixture.owners.containsKey(9)); + Mockito.verify(loader, Mockito.never()).load(); + } + } + + @Test + public void testInitialReadFailureReasonIsRuntimeOnly() throws Exception { + LoadTask task = LoadTask.builder().status(LoadStatus.FAILED).lastDuration(0L).currDuration(0L) + .fileReadLines(0L).fileTotalLines(0L).build(); + task.setInitialStateReadFailed(true); + com.fasterxml.jackson.databind.ObjectMapper json = new com.fasterxml.jackson.databind.ObjectMapper(); + com.fasterxml.jackson.databind.JsonNode value = json.valueToTree(task); + Assert.assertFalse(value.has("initialStateReadFailed")); + Assert.assertFalse(value.has("initial_state_read_failed")); + LoadTask restored = json.readValue("{\"initialStateReadFailed\":true}", LoadTask.class); + Assert.assertFalse(restored.isInitialStateReadFailed()); + Field field = LoadTask.class.getDeclaredField("initialStateReadFailed"); + Assert.assertFalse(field.getAnnotation(com.baomidou.mybatisplus.annotation.TableField.class).exist()); + com.baomidou.mybatisplus.core.metadata.TableInfo table = + com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo( + new org.apache.ibatis.builder.MapperBuilderAssistant( + new com.baomidou.mybatisplus.core.MybatisConfiguration(), "cleanup-test"), + LoadTask.class); + Assert.assertFalse(table.getFieldList().stream() + .anyMatch(item -> item.getProperty().equals("initialStateReadFailed"))); + } + + @Test + public void testInitialReadFailurePollPersistsFailedAndRetiresOwner() throws Exception { + this.assertInitialReadRecovery(LoadStatus.RUNNING, false, false); + } + + @Test + public void testInitialReadFailurePollPreservesSucceededTask() throws Exception { + this.assertInitialReadRecovery(LoadStatus.SUCCEED, false, false); + } + + @Test + public void testInitialReadFailurePollDoesNotRecreateDeletedRow() throws Exception { + this.assertInitialReadRecovery(null, false, false); + } + + @Test + public void testInitialReadFailurePollRetainsOwnerAfterSqlFailure() throws Exception { + this.assertInitialReadRecovery(LoadStatus.RUNNING, true, false); + } + + @Test + public void testInitialReadFailurePollRetainsOwnerAfterCommitFailure() throws Exception { + this.assertInitialReadRecovery(LoadStatus.RUNNING, false, true); + } + + @Test + public void testDifferentInitialOwnerCancelsBeforeDatabaseRead() throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + HugeGraphLoader oldLoader = this.loader(); + LoadTask old = this.task(oldLoader); + old.setId(9); + HugeGraphLoader newLoader = this.loader(); + LoadTask replacement = this.task(newLoader); + replacement.setId(9); + fixture.owners.put(9, replacement); + Mockito.doThrow(new RuntimeException("must not read metadata for a replaced initial owner")) + .when(fixture.tasks).selectById(9); + + dispatch(fixture.load, old); + + Assert.assertSame(replacement, fixture.owners.get(9)); + Assert.assertTrue(oldLoader.context().closed()); + Assert.assertFalse(newLoader.context().closed()); + Mockito.verify(fixture.tasks, Mockito.never()).selectById(9); + Mockito.verify(fixture.tasks, Mockito.never()).update(Mockito.any(), Mockito.any()); + Mockito.verify(newLoader, Mockito.never()).shutdown(); + newLoader.shutdown(); + } + } + + private void assertInitialReadRecovery(LoadStatus afterRead, boolean sqlFailure, + boolean commitFailure) throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + HugeGraphLoader loader = this.loader(); + LoadTask task = this.task(loader); + task.setId(9); + task.setJobId(7); + java.util.concurrent.atomic.AtomicBoolean first = new java.util.concurrent.atomic.AtomicBoolean(true); + RuntimeException readFailure = new RuntimeException("initial metadata read temporarily unavailable"); + Mockito.doAnswer(invocation -> { + if (first.getAndSet(false)) { + throw readFailure; + } + throw new AssertionError("initial failure recovery must use conditional SQL, not another snapshot"); + }).when(fixture.tasks).selectById(9); + + dispatch(fixture.load, task); + + Assert.assertSame("the closed initial owner must remain recoverable", task, fixture.owners.get(9)); + Assert.assertTrue(loader.context().closed()); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + if (afterRead == null) { + fixture.jdbc.update("DELETE FROM delete_tasks WHERE id=9"); + } else { + fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=9", afterRead.getValue()); + } + RuntimeException writeFailure = new RuntimeException("conditional failure write unavailable"); + java.util.concurrent.atomic.AtomicBoolean rejectSql = + new java.util.concurrent.atomic.AtomicBoolean(sqlFailure); + Mockito.doAnswer(invocation -> { + Assert.assertNull(invocation.getArgument(0)); + com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = + invocation.getArgument(1); + String sql = condition.getSqlSegment(); + Assert.assertTrue(sql.contains("id =")); + Assert.assertTrue(sql.contains("load_status =")); + Assert.assertTrue(condition.getParamNameValuePairs().containsValue(LoadStatus.RUNNING.getValue())); + Assert.assertTrue(condition.getSqlSet().contains("load_status=")); + if (rejectSql.getAndSet(false)) { + throw writeFailure; + } + return fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=9 AND load_status=?", + LoadStatus.FAILED.getValue(), LoadStatus.RUNNING.getValue()); + }).when(fixture.tasks).update(Mockito.nullable(LoadTask.class), Mockito.any()); + org.springframework.jdbc.datasource.DataSourceTransactionManager tx = fixture.manager; + if (commitFailure) { + java.util.concurrent.atomic.AtomicBoolean rejectCommit = + new java.util.concurrent.atomic.AtomicBoolean(true); + tx = new org.springframework.jdbc.datasource.DataSourceTransactionManager(fixture.dataSource) { + @Override + protected void doCommit(org.springframework.transaction.support.DefaultTransactionStatus status) { + if (rejectCommit.getAndSet(false)) { + throw new org.springframework.transaction.TransactionSystemException("commit unavailable"); + } + super.doCommit(status); + } + }; + tx.setRollbackOnCommitFailure(true); + } + org.springframework.transaction.support.TransactionTemplate transaction = + new org.springframework.transaction.support.TransactionTemplate(tx); + if (sqlFailure || commitFailure) { + try { + transaction.execute(status -> { + fixture.load.updateLoadTaskProgress(); + return null; + }); + Assert.fail("the original SQL/commit failure must be propagated"); + } catch (RuntimeException failure) { + if (sqlFailure) { + Assert.assertSame(writeFailure, failure); + } else { + Assert.assertTrue( + failure instanceof org.springframework.transaction.TransactionSystemException); + } + } + Assert.assertSame(task, fixture.owners.get(9)); + Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); + } + transaction.execute(status -> { + fixture.load.updateLoadTaskProgress(); + Assert.assertSame("owner retirement waits for commit", task, fixture.owners.get(9)); + return null; + }); + Assert.assertFalse(fixture.owners.containsKey(9)); + java.util.List rows = fixture.jdbc.queryForList( + "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class); + Assert.assertEquals(afterRead == null ? Collections.emptyList() : + Collections.singletonList(afterRead == LoadStatus.RUNNING ? + LoadStatus.FAILED.getValue() : afterRead.getValue()), rows); + Mockito.verify(fixture.tasks, Mockito.never()).updateById(Mockito.any(LoadTask.class)); + Mockito.verify(loader, Mockito.never()).load(); + } + } + + @Test + public void testDeleteAdmissionRejectsLaterControllerStart() throws Exception { + this.assertJobAdmission(true, false); + } + + @Test + public void testControllerStartAdmissionMakesDeleteEnumerateNewTask() throws Exception { + this.assertJobAdmission(false, false); + } + + @Test + public void testDeleteAdmissionRejectsLaterControllerCreate() throws Exception { + this.assertJobAdmission(true, true); + } + + private void assertJobAdmission(boolean deleteFirst, boolean createOnly) throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + fixture.jdbc.update("UPDATE delete_jobs SET status=? WHERE id=7", + org.apache.hugegraph.entity.enums.JobStatus.SETTING.getValue()); + Mockito.doAnswer(invocation -> { + java.util.List rows = fixture.jdbc.queryForList( + "SELECT status FROM delete_jobs WHERE id=7", Byte.class); + if (rows.isEmpty()) { + return null; + } + return org.apache.hugegraph.entity.load.JobManager.builder().id(7) + .jobStatus(java.util.Arrays.stream(org.apache.hugegraph.entity.enums.JobStatus.values()) + .filter(status -> status.getValue().equals(rows.get(0))).findFirst().get()).build(); + }).when(fixture.jobs).selectById(7); + Mockito.doAnswer(invocation -> fixture.jobs.selectById(7)) + .when(fixture.jobs).selectOne(Mockito.any()); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference startFailure = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference deleteFailure = + new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicInteger constructions = new java.util.concurrent.atomic.AtomicInteger(); + Mockito.doAnswer(invocation -> { + java.util.List snapshot = new java.util.ArrayList<>(); + for (Integer id : fixture.jdbc.queryForList("SELECT id FROM delete_tasks ORDER BY id", Integer.class)) { + snapshot.add(fixture.tasks.selectById(id)); + } + if (deleteFirst) { + entered.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + } + return snapshot; + }).when(fixture.tasks).selectList(Mockito.any()); + Mockito.when(fixture.tasks.insert(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + task.setId(11); + if (!deleteFirst) { + entered.countDown(); + Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); + } + return fixture.jdbc.update("INSERT INTO delete_tasks VALUES (11, ?)", task.getStatus().getValue()); + }); + Mockito.when(fixture.tasks.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + return fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=?", + task.getStatus().getValue(), task.getId()); + }); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + Mockito.doAnswer(invocation -> { + new LoadTaskExecutor().execute(invocation.getArgument(0), invocation.getArgument(1)); + return null; + }).when(executor).execute(Mockito.any(), Mockito.any()); + set(fixture.load, "taskExecutor", executor); + FileMapping mapping = new FileMapping(); + mapping.setId(8); + mapping.setJobId(7); + mapping.setPath(this.folder.newFile("job-admission.csv").getAbsolutePath()); + mapping.setVertexMappings(Collections.emptySet()); + mapping.setEdgeMappings(Collections.emptySet()); + mapping.setLoadParameter(new LoadParameter()); + FileSetting setting = new FileSetting(); + setting.setColumnNames(Collections.singletonList("id")); + mapping.setFileSetting(setting); + org.apache.hugegraph.service.load.FileMappingService files = + Mockito.mock(org.apache.hugegraph.service.load.FileMappingService.class); + Mockito.when(files.get("space", "graph", 7, 8)).thenReturn(mapping); + org.apache.hugegraph.config.HugeConfig config = Mockito.mock(org.apache.hugegraph.config.HugeConfig.class); + Mockito.when(config.get(org.apache.hugegraph.options.HubbleOptions.PD_ENABLED)).thenReturn(false); + Mockito.when(config.get(org.apache.hugegraph.options.HubbleOptions.SERVER_URL)) + .thenReturn("http://127.0.0.1:19880"); + org.apache.hugegraph.controller.load.LoadTaskController controller = + new org.apache.hugegraph.controller.load.LoadTaskController(fixture.load) { + @Override + protected org.apache.hugegraph.driver.HugeClient requireGraphSpaceWrite(String graphSpace) { + return null; + } + + @Override + protected org.apache.hugegraph.driver.HugeClient authClient(String graphSpace, String graph) { + return null; + } + + @Override + protected String getToken() { + return "token"; + } + + @Override + protected String getUser() { + return null; + } + }; + Field jobField = org.apache.hugegraph.controller.load.LoadTaskController.class + .getDeclaredField("jobService"); + jobField.setAccessible(true); + jobField.set(controller, fixture.service); + Field filesField = org.apache.hugegraph.controller.load.LoadTaskController.class + .getDeclaredField("fmService"); + filesField.setAccessible(true); + filesField.set(controller, files); + Field configField = org.apache.hugegraph.controller.load.LoadTaskController.class + .getDeclaredField("config"); + configField.setAccessible(true); + configField.set(controller, config); + Thread start = new Thread(() -> { + try (MockedConstruction loaders = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, context) -> { + constructions.incrementAndGet(); + LoadContext resources = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + java.util.concurrent.atomic.AtomicBoolean closed = + new java.util.concurrent.atomic.AtomicBoolean(); + Mockito.when(loader.context()).thenReturn(resources); + Mockito.when(resources.closed()).thenAnswer(invocation -> closed.get()); + Mockito.when(resources.newProgress().totalInputRead()).thenReturn(0L); + Mockito.when(resources.summary().totalTime()).thenReturn(1L); + Mockito.when(loader.load()).thenReturn(true); + Mockito.doAnswer(invocation -> { + closed.set(true); + return null; + }).when(loader).shutdown(); + })) { + if (createOnly) { + controller.create("space", "graph", 7, LoadTask.builder().status(LoadStatus.INIT).build()); + } else { + controller.start("space", "graph", 7, Collections.singletonList(8)); + } + } catch (Throwable failure) { + startFailure.set(failure); + failure.printStackTrace(); + } + }); + Thread delete = new Thread(() -> { + try { + fixture.delete(true); + } catch (Throwable failure) { + deleteFailure.set(failure); + } + }); + Thread first = deleteFirst ? delete : start; + Thread second = deleteFirst ? start : delete; + first.start(); + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + second.start(); + try { + second.join(250); + System.out.println("JOB_ADMISSION deleteFirst=" + deleteFirst + " createOnly=" + createOnly + + " competitorWaiting=" + second.isAlive()); + Assert.assertTrue("same-job competitor must await admission", second.isAlive()); + } finally { + release.countDown(); + first.join(5000); + second.join(5000); + } + Assert.assertFalse(first.isAlive()); + Assert.assertFalse(second.isAlive()); + Assert.assertNull(deleteFailure.get()); + if (deleteFirst) { + Assert.assertTrue(startFailure.get() instanceof org.apache.hugegraph.exception.ExternalException); + Assert.assertEquals(0, constructions.get()); + Mockito.verifyNoInteractions(executor); + } else { + Assert.assertNull(startFailure.get()); + Assert.assertEquals(1, constructions.get()); + Mockito.verify(executor).execute(Mockito.any(), Mockito.any()); + } + Assert.assertEquals(Integer.valueOf(0), fixture.jdbc.queryForObject( + "SELECT COUNT(*) FROM delete_tasks", Integer.class)); + Assert.assertEquals(Integer.valueOf(0), fixture.jdbc.queryForObject( + "SELECT COUNT(*) FROM delete_jobs", Integer.class)); + Assert.assertTrue(fixture.owners.isEmpty()); + } + } + + private static class JobDeleteFixture implements AutoCloseable { + private final org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = + new org.springframework.jdbc.datasource.DriverManagerDataSource(); + private final org.springframework.jdbc.core.JdbcTemplate jdbc; + private final org.springframework.jdbc.datasource.DataSourceTransactionManager manager; + private final java.sql.Connection keeper; + private final LoadTaskMapper tasks = Mockito.mock(LoadTaskMapper.class); + private final org.apache.hugegraph.mapper.load.JobManagerMapper jobs = + Mockito.mock(org.apache.hugegraph.mapper.load.JobManagerMapper.class); + private final Map owners = new ConcurrentHashMap<>(); + private final org.apache.hugegraph.service.load.JobManagerService service; + private final LoadTaskService load = new LoadTaskService(); + + private JobDeleteFixture() throws Exception { + this.dataSource.setDriverClassName("org.h2.Driver"); + this.dataSource.setUrl("jdbc:h2:mem:job-delete-" + java.util.UUID.randomUUID()); + this.keeper = this.dataSource.getConnection(); + this.jdbc = new org.springframework.jdbc.core.JdbcTemplate(this.dataSource); + this.manager = new org.springframework.jdbc.datasource.DataSourceTransactionManager(this.dataSource); + this.jdbc.execute("CREATE TABLE delete_tasks (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); + this.jdbc.execute("CREATE TABLE delete_jobs (id INT PRIMARY KEY, status TINYINT NOT NULL)"); + this.jdbc.update("INSERT INTO delete_tasks VALUES (9, ?), (10, ?)", + LoadStatus.RUNNING.getValue(), LoadStatus.RUNNING.getValue()); + this.jdbc.update("INSERT INTO delete_jobs VALUES (7, ?)", + org.apache.hugegraph.entity.enums.JobStatus.LOADING.getValue()); + Mockito.when(this.tasks.selectById(Mockito.anyInt())).thenAnswer(invocation -> { + int id = invocation.getArgument(0); + java.util.List rows = this.jdbc.queryForList( + "SELECT load_status FROM delete_tasks WHERE id=?", Byte.class, id); + if (rows.isEmpty()) { + return null; + } + LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) + .filter(value -> value.getValue().equals(rows.get(0))).findFirst().get(); + return LoadTask.builder().id(id).jobId(7).options(new LoadOptions()).status(status).build(); + }); + Mockito.when(this.tasks.selectList(Mockito.any())).thenAnswer(invocation -> + java.util.Arrays.asList(this.tasks.selectById(10), this.tasks.selectById(9))); + Mockito.when(this.tasks.update(Mockito.nullable(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = + invocation.getArgument(1); + String where = condition.getSqlSegment(); + Byte expected = (Byte) condition.getParamNameValuePairs().entrySet().stream() + .filter(entry -> entry.getValue() instanceof Byte && where.contains("." + entry.getKey() + "}")) + .map(Map.Entry::getValue).findFirst().get(); + Integer id = (Integer) condition.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Integer).findFirst().get(); + Byte target = task == null ? LoadStatus.FAILED.getValue() : task.getStatus().getValue(); + return this.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=? AND load_status=?", + target, id, expected); + }); + Mockito.when(this.tasks.delete(Mockito.any())).thenAnswer(invocation -> { + com.baomidou.mybatisplus.core.conditions.query.QueryWrapper condition = + invocation.getArgument(0); + condition.getSqlSegment(); + Integer id = (Integer) condition.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Integer).findFirst().get(); + Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Byte).findFirst().get(); + return this.jdbc.update("DELETE FROM delete_tasks WHERE id=? AND load_status=?", id, expected); + }); + org.apache.hugegraph.entity.load.JobManager job = + org.apache.hugegraph.entity.load.JobManager.builder().id(7) + .jobStatus(org.apache.hugegraph.entity.enums.JobStatus.LOADING).build(); + Mockito.when(this.jobs.selectById(7)).thenReturn(job); + Mockito.when(this.jobs.selectOne(Mockito.any())).thenReturn(job); + Mockito.when(this.jobs.deleteById(7)).thenAnswer(invocation -> + this.jdbc.update("DELETE FROM delete_jobs WHERE id=7")); + Mockito.when(this.jobs.updateById(Mockito.any(org.apache.hugegraph.entity.load.JobManager.class))) + .thenAnswer(invocation -> { + org.apache.hugegraph.entity.load.JobManager updated = invocation.getArgument(0); + return this.jdbc.update("UPDATE delete_jobs SET status=? WHERE id=?", + updated.getJobStatus().getValue(), updated.getId()); + }); + set(this.load, "mapper", this.tasks); + set(this.load, "runningTaskContainer", this.owners); + org.apache.hugegraph.service.load.JobManagerService target = + new org.apache.hugegraph.service.load.JobManagerService(); + set(target, "mapper", this.jobs); + set(target, "taskService", this.load); + set(target, "transactionManager", this.manager); + org.apache.hugegraph.service.load.FileMappingService files = + Mockito.mock(org.apache.hugegraph.service.load.FileMappingService.class); + Mockito.when(files.listByJob(7)).thenReturn(Collections.emptyList()); + set(target, "fileMappingService", files); + org.springframework.aop.framework.ProxyFactory factory = + new org.springframework.aop.framework.ProxyFactory(target); + factory.setProxyTargetClass(true); + factory.addAdvice(new org.springframework.transaction.interceptor.TransactionInterceptor( + this.manager, + new org.springframework.transaction.annotation.AnnotationTransactionAttributeSource())); + this.service = (org.apache.hugegraph.service.load.JobManagerService) factory.getProxy(); + } + + private void delete(boolean scoped) { + if (scoped) { + this.service.deleteJob("space", "graph", 7); + } else { + this.service.deleteJob(7); + } + } + + @Override + public void close() throws Exception { + this.keeper.close(); + } + } + + // Separate admission instances intentionally keep the defensive old-tail SQL/identity + // controls concurrent. This is not a cross-Hubble lifecycle coordination guarantee. + private static LoadTaskService peer(LoadTaskService source) throws Exception { + LoadTaskService peer = new LoadTaskService(); + for (String name : new String[]{"mapper", "taskExecutor", "runningTaskContainer"}) { + Field field = LoadTaskService.class.getDeclaredField(name); + field.setAccessible(true); + set(peer, name, field.get(source)); + } + return peer; + } + + private static void set(Object object, String name, Object value) throws Exception { + Field field = object.getClass().getDeclaredField(name); + field.setAccessible(true); + field.set(object, value); + } + + private static void dispatch(LoadTaskService service, LoadTask task) throws Exception { + Field mapperField = LoadTaskService.class.getDeclaredField("mapper"); + mapperField.setAccessible(true); + LoadTaskMapper mapper = (LoadTaskMapper) mapperField.get(service); + if (mapper == null) { + mapper = Mockito.mock(LoadTaskMapper.class); + set(service, "mapper", mapper); + } + boolean configured = Mockito.mockingDetails(mapper).getStubbings().stream() + .anyMatch(stub -> stub.getInvocation().getMethod().getName().equals("selectById") && + java.util.Objects.equals(stub.getInvocation().getArguments()[0], task.getId())); + if (!configured) { + Mockito.when(mapper.selectById(task.getId())).thenReturn( + LoadTask.builder().id(task.getId()).status(LoadStatus.RUNNING).build()); + } + Method method = LoadTaskService.class.getDeclaredMethod("executeAfterCommit", LoadTask.class); + method.setAccessible(true); + method.invoke(service, task); + } + + private HugeGraphLoader loader() { + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + java.util.concurrent.atomic.AtomicBoolean closed = + new java.util.concurrent.atomic.AtomicBoolean(); + Mockito.when(context.closed()).thenAnswer(invocation -> closed.get()); + Mockito.doAnswer(invocation -> { + closed.set(true); + return null; + }).when(loader).shutdown(); + Mockito.when(context.newProgress().totalInputRead()).thenReturn(7L); + Mockito.when(context.summary().totalTime()).thenReturn(12L); + return loader; + } + + private LoadTask task(HugeGraphLoader loader) { + return LoadTask.builder().loader(loader).options(new LoadOptions()).status(LoadStatus.RUNNING) + .lastDuration(0L).build(); + } +} diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java index 68c0ab58e..11fd1bb0c 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java @@ -84,7 +84,13 @@ public void testLoadExecutionWaitsForOuterTransactionCommit() LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); this.setField(service, "taskExecutor", executor); this.setField(service, "runningTaskContainer", new ConcurrentHashMap<>()); - LoadTask task = LoadTask.builder().id(9).build(); + org.apache.hugegraph.loader.HugeGraphLoader loader = + Mockito.mock(org.apache.hugegraph.loader.HugeGraphLoader.class); + Mockito.when(loader.context()).thenReturn(Mockito.mock(org.apache.hugegraph.loader.executor.LoadContext.class)); + LoadTask task = LoadTask.builder().id(9).status(LoadStatus.RUNNING).loader(loader).build(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.selectById(9)).thenReturn(task); + this.setField(service, "mapper", mapper); Method method = LoadTaskService.class.getDeclaredMethod( "executeAfterCommit", LoadTask.class); method.setAccessible(true); @@ -114,7 +120,8 @@ public void testResumeRehydratesLoaderWithCurrentSessionToken() LoadTask task = Mockito.mock(LoadTask.class); LoadOptions options = new LoadOptions(); Mockito.when(mapper.selectById(9)).thenReturn(task); - Mockito.when(mapper.updateById(task)).thenReturn(1); + Mockito.when(task.getId()).thenReturn(9); + Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenReturn(1); Mockito.when(task.getStatus()).thenReturn(LoadStatus.PAUSED); Mockito.when(task.getOptions()).thenReturn(options); this.setField(service, "mapper", mapper); @@ -137,7 +144,8 @@ public void testResumeDoesNotPersistRunningStateWhenReconnectFails() LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); LoadTask task = Mockito.mock(LoadTask.class); Mockito.when(mapper.selectById(9)).thenReturn(task); - Mockito.when(mapper.updateById(task)).thenReturn(1); + Mockito.when(task.getId()).thenReturn(9); + Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenReturn(1); Mockito.when(task.getStatus()).thenReturn(LoadStatus.PAUSED); Mockito.when(task.getOptions()).thenReturn(new LoadOptions()); Mockito.doThrow(new IllegalArgumentException("missing token")) @@ -152,6 +160,7 @@ public void testResumeDoesNotPersistRunningStateWhenReconnectFails() }); Mockito.verify(mapper, Mockito.never()).updateById(task); + Mockito.verify(mapper, Mockito.never()).update(Mockito.eq(task), Mockito.any()); Mockito.verify(executor, Mockito.never()) .execute(Mockito.any(), Mockito.any()); } @@ -164,7 +173,8 @@ public void testRetryRehydratesLoaderWithCurrentSessionToken() LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); LoadTask task = Mockito.mock(LoadTask.class); Mockito.when(mapper.selectById(9)).thenReturn(task); - Mockito.when(mapper.updateById(task)).thenReturn(1); + Mockito.when(task.getId()).thenReturn(9); + Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenReturn(1); Mockito.when(task.getStatus()).thenReturn(LoadStatus.STOPPED); Mockito.when(task.getOptions()).thenReturn(new LoadOptions()); this.setField(service, "mapper", mapper); diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java index 07800995c..d7607498c 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -86,6 +86,7 @@ GroovySchemaCompatibilityTest.class, JobManagerServiceTest.class, K8sTokenEndpointSecurityTest.class, + LoadTaskCleanupTest.class, LoadTaskServiceTest.class, LoaderScopeControllerTest.class, LoginAttemptGuardTest.class, diff --git a/hugegraph-loader/README.md b/hugegraph-loader/README.md index c3e69388d..2cf290e88 100644 --- a/hugegraph-loader/README.md +++ b/hugegraph-loader/README.md @@ -171,3 +171,22 @@ Server 1.7 compatibility CI uses the pinned official ASF 1.7.0 binary archive. T hugegraph-loader is licensed under Apache 2.0 License. See the [Flink CDC Java 17 validation](docs/flink-cdc-java17.md) for its current limits and launcher syntax. + +### Embedded Loader shutdown + +Call `HugeGraphLoader.shutdown()` when an embedded load finishes or its dispatch +fails. Shutdown waits for that Loader's workers, closes its context, and removes +its own JVM shutdown hook once the context is closed. If workers cannot terminate, +the context and hook remain available for a later shutdown attempt. A hook already +running during JVM shutdown still closes its Loader. This does not shut down the +process-wide executor manager or coordinate graph mode across separate imports. + +Closing a Loader before `load()` starts does not write a new checkpoint. An actual +load attempt and direct `LoadContext.close()` keep their existing progress-save +behavior. Incremental loading keeps independent copies of the previous loaded +and loading offsets, including inputs paused before their first read, and adds +newly loaded counts to the completed counts. It also preserves mapped inputs +outside the selected file range; removed mappings do not retain source markers +in the next checkpoint. Non-incremental replay starts fresh. +An ongoing insert can still block the existing wait before shutdown +reaches cancellation; this change adds no load deadline. diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java index 42cccff7e..45b023c3f 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java @@ -90,6 +90,8 @@ public final class HugeGraphLoader { private final LoadMapping mapping; private final TaskManager manager; private final LoadOptions options; + private final Thread shutdownHook; + private volatile boolean loadStarted; public static class InputTaskItem { @@ -148,14 +150,11 @@ public HugeGraphLoader(LoadOptions options, LoadMapping mapping) { this.options = options; this.mapping = mapping; this.manager = new TaskManager(this.context); - this.addShutdownHook(); - } - - private void addShutdownHook() { - Runtime.getRuntime().addShutdownHook(new Thread(() -> { + this.shutdownHook = new Thread(() -> { LOG.info("Shutdown hook was triggered"); this.stopThenShutdown(); - })); + }); + Runtime.getRuntime().addShutdownHook(this.shutdownHook); } public LoadContext context() { @@ -203,6 +202,7 @@ private void setGraphMode() { } public boolean load() { + this.loadStarted = true; this.options.dumpParams(); try { @@ -607,6 +607,15 @@ private void loadInputs(List structs) { private List prepareTaskItems(List structs, boolean scatter) { + if (this.context.options().incrementalMode) { + // Range filtering must not discard progress of other mapped inputs. + for (InputStruct struct : structs) { + if (this.context.newProgress().get(struct.id()) == null && + this.context.oldProgress().get(struct.id()) != null) { + this.context.newProgress().addStruct(struct, this.context.oldProgress().get(struct.id())); + } + } + } ArrayList tasks = new ArrayList<>(); ArrayList readers = new ArrayList<>(); int curFile = 0; @@ -903,37 +912,58 @@ private boolean reachedMaxReadLines() { * TODO: How to distinguish load task finished normally or abnormally */ private synchronized void stopThenShutdown() { - if (this.context.closed()) { - return; - } - LOG.info("Stop loading then shutdown HugeGraphLoader"); - boolean tasksStopped = this.manager == null; - boolean interrupted = Thread.currentThread().isInterrupted(); + Throwable failure = null; try { - this.context.stopLoading(); - if (this.manager != null) { - // Wait all insert tasks stopped before exit - try { - this.manager.waitFinished(); - } finally { - this.manager.shutdown(); - tasksStopped = true; - } + if (this.context.closed()) { + return; } - } finally { - interrupted |= Thread.currentThread().isInterrupted(); + LOG.info("Stop loading then shutdown HugeGraphLoader"); + boolean tasksStopped = this.manager == null; + boolean interrupted = Thread.currentThread().isInterrupted(); try { - // Keep dependencies alive if cancellation could not terminate the workers. - if (tasksStopped) { + this.context.stopLoading(); + if (this.manager != null) { + // Wait all insert tasks stopped before exit try { - this.context.unsetLoadingMode(); + this.manager.waitFinished(); } finally { - this.context.close(); + this.manager.shutdown(); + tasksStopped = true; } } } finally { - if (interrupted) { - Thread.currentThread().interrupt(); + interrupted |= Thread.currentThread().isInterrupted(); + try { + // Keep dependencies alive if cancellation could not terminate the workers. + if (tasksStopped) { + try { + this.context.unsetLoadingMode(); + } finally { + this.context.close(this.loadStarted); + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + } catch (RuntimeException | Error e) { + failure = e; + throw e; + } finally { + if (this.context.closed() && this.shutdownHook != null) { + try { + Runtime.getRuntime().removeShutdownHook(this.shutdownHook); + } catch (IllegalStateException ignored) { + // JVM shutdown already started; this hook can still finish closing. + } catch (RuntimeException | Error e) { + if (failure == null) { + throw e; + } + if (failure != e) { + failure.addSuppressed(e); + } } } } @@ -941,7 +971,8 @@ private synchronized void stopThenShutdown() { private void cleanupEmptyProgress() { Map inputProgressMap = this.context.newProgress().inputProgress(); - inputProgressMap.entrySet().removeIf(entry -> entry.getValue().loadedItems().isEmpty()); + inputProgressMap.entrySet().removeIf(entry -> entry.getValue().loadedItems().isEmpty() && + entry.getValue().loadingItems().isEmpty()); } private static class SplitInputStructs { diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java index bbd935143..cc0d82714 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java @@ -96,6 +96,10 @@ private LoadContext(LoadOptions options, SchemaCache schemaCache) { this.summary = new LoadSummary(); this.oldProgress = LoadProgress.parse(options); this.newProgress = new LoadProgress(); + if (options.incrementalMode) { + this.newProgress.plusVertexLoaded(this.oldProgress.vertexLoaded()); + this.newProgress.plusEdgeLoaded(this.oldProgress.edgeLoaded()); + } this.loggers = new ConcurrentHashMap<>(); this.client = schemaCache == null ? HugeClientHolder.create(options) : null; if (this.client != null && this.options.direct) { @@ -259,6 +263,10 @@ public synchronized void unsetLoadingMode() { } public void close() { + this.close(true); + } + + public void close(boolean saveProgress) { if (this.closed) { return; } @@ -284,7 +292,7 @@ public void close() { this.throwIfFailed(); this.newProgress.plusVertexLoaded(this.summary.vertexLoaded()); this.newProgress.plusEdgeLoaded(this.summary.edgeLoaded()); - if (this.client != null) { + if (this.client != null && saveProgress) { try { this.newProgress.write(this); } catch (IOException e) { diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputItemProgress.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputItemProgress.java index a1c78b71a..88dd50a2d 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputItemProgress.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputItemProgress.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -public abstract class InputItemProgress { +public abstract class InputItemProgress implements Cloneable { /** * NOTE: this offset refers to the parsed part in input instead of loaded. @@ -50,4 +50,12 @@ public void offset(long offset) { public void confirmOffset() { this.confirmOffset = this.offset; } + + public InputItemProgress copy() { + try { + return (InputItemProgress) super.clone(); + } catch (CloneNotSupportedException e) { + throw new AssertionError("Input progress must be cloneable", e); + } + } } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputProgress.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputProgress.java index 66e2898ab..66c340384 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputProgress.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputProgress.java @@ -50,10 +50,20 @@ public InputProgress(@JsonProperty("type") SourceType type, } public InputProgress(InputStruct struct) { + this(struct, null); + } + + public InputProgress(InputStruct struct, InputProgress previous) { this.type = struct.input().type(); this.loadedItems = Collections.synchronizedMap( InsertionOrderUtil.newMap()); this.loadingItems = new ConcurrentHashMap<>(); + if (previous != null && previous.type == this.type) { + synchronized (previous) { + previous.loadedItems.forEach((name, item) -> this.loadedItems.put(name, item.copy())); + previous.loadingItems.forEach((name, item) -> this.loadingItems.put(name, item.copy())); + } + } } public synchronized Map loadedItems() { diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/LoadProgress.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/LoadProgress.java index 9fbeb4745..e78da8334 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/LoadProgress.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/LoadProgress.java @@ -101,8 +101,12 @@ public long totalInputRead() { } public InputProgress addStruct(InputStruct struct) { + return this.addStruct(struct, null); + } + + public InputProgress addStruct(InputStruct struct, InputProgress previous) { E.checkNotNull(struct, "mapping mapping"); - this.inputProgress.put(struct.id(), new InputProgress(struct)); + this.inputProgress.put(struct.id(), new InputProgress(struct, previous)); return this.inputProgress.get(struct.id()); } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/AbstractReader.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/AbstractReader.java index 6c0ecae2e..4221cfe55 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/AbstractReader.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/AbstractReader.java @@ -34,7 +34,8 @@ public void progress(LoadContext context, InputStruct struct) { // Update loading vertex/edge mapping this.newProgress = context.newProgress().get(struct.id()); if (this.newProgress == null) { - this.newProgress = context.newProgress().addStruct(struct); + this.newProgress = context.newProgress().addStruct( + struct, context.options().incrementalMode ? this.oldProgress : null); } } } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/file/FileReader.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/file/FileReader.java index e6e32af01..d961dc1ae 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/file/FileReader.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/file/FileReader.java @@ -238,14 +238,14 @@ private LoadStatus checkLastLoadStatus(Readable readable) { InputItemProgress loaded = this.oldProgress.matchLoadedItem(input); // The file has been loaded before, and it is not changed if (loaded != null) { - this.newProgress.addLoadedItem(readable.name(), loaded); + this.newProgress.addLoadedItem(readable.name(), loaded.copy()); return LoadStatus.LOADED; } InputItemProgress loading = this.oldProgress.matchLoadingItem(input); if (loading != null) { // The file has been loaded half before and it is not changed - this.newProgress.addLoadingItem(readable.name(), loading); + this.newProgress.addLoadingItem(readable.name(), loading.copy()); return LoadStatus.LOADED_HALF; } else { this.newProgress.addLoadingItem(readable.name(), input); diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java index 1684d411a..edfa54deb 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java @@ -25,10 +25,12 @@ import java.io.OutputStreamWriter; import java.io.StringWriter; import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.Arrays; +import java.util.List; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -669,6 +671,210 @@ public boolean requiresLayout() { } } + @Test + public void testIncrementalReaderInitializationCopiesCheckpointWithoutAliases() throws Exception { + this.assertReaderCheckpointInitialization(true); + } + + @Test + public void testReplayReaderInitializationDoesNotInheritCheckpoint() throws Exception { + this.assertReaderCheckpointInitialization(false); + } + + @Test + public void testIncrementalRangeFilteringPreservesOnlyCurrentMappingCheckpoints() throws Exception { + this.assertReaderCheckpointInitialization(true, true); + } + + private void assertReaderCheckpointInitialization(boolean incremental) throws Exception { + this.assertReaderCheckpointInitialization(incremental, false); + } + + private void assertReaderCheckpointInitialization(boolean incremental, boolean filtered) throws Exception { + try (ModeServer server = new ModeServer()) { + File firstFile = this.folder.newFile("copied-loaded.csv"); + File secondFile = this.folder.newFile("copied-loading.csv"); + java.nio.file.Files.writeString(firstFile.toPath(), "a0\na1\n"); + java.nio.file.Files.writeString(secondFile.toPath(), "b0\nb1\n"); + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = this.folder.newFile(incremental ? "copy-inc.json" : "copy-replay.json").getAbsolutePath(); + LoadProgress checkpoint = new LoadProgress(); + InputStruct[] structs = new InputStruct[2]; + File[] files = {firstFile, secondFile}; + for (int i = 0; i < files.length; i++) { + FileSource source = new FileSource(); + source.path(files[i].getAbsolutePath()); + source.header(new String[]{"id"}); + source.hasHeader(false); + InputStruct struct = new InputStruct(null, null); + struct.id(Integer.toString(i)); + struct.input(source); + structs[i] = struct; + org.apache.hugegraph.loader.progress.FileItemProgress item = + new org.apache.hugegraph.loader.progress.FileItemProgress( + files[i].getName(), files[i].lastModified(), + String.valueOf(org.apache.commons.io.FileUtils.checksumCRC32(files[i])), + i == 0 ? 2L : 1L); + org.apache.hugegraph.loader.progress.InputProgress progress = checkpoint.addStruct(struct); + if (i == 0) { + progress.addLoadedItem(files[i].getName(), item); + } else { + progress.addLoadingItem(files[i].getName(), item); + } + } + if (filtered) { + InputStruct removed = new InputStruct(null, null); + removed.id("removed"); + removed.input(structs[0].input()); + checkpoint.addStruct(removed).addLoadedItem(firstFile.getName(), + new org.apache.hugegraph.loader.progress.FileItemProgress( + firstFile.getName(), firstFile.lastModified(), "removed", 1L)); + options.startFile = 1; + options.endFile = 2; + } + checkpoint.plusVertexLoaded(5L); + checkpoint.plusEdgeLoaded(2L); + java.nio.file.Path saved = java.nio.file.Paths.get(LoadProgress.format(options, "20000101-000000")); + java.nio.file.Files.createDirectories(saved.getParent()); + java.nio.file.Files.writeString(saved, JsonUtil.toJson(checkpoint)); + options.incrementalMode = incremental; + LoadContext context = new LoadContext(options); + try { + if (filtered) { + HugeGraphLoader loader = loader(context, structs); + Method prepare = HugeGraphLoader.class.getDeclaredMethod( + "prepareTaskItems", List.class, boolean.class); + prepare.setAccessible(true); + @SuppressWarnings("unchecked") + List tasks = + (List) prepare.invoke(loader, Arrays.asList(structs), false); + Assert.assertEquals(1, tasks.size()); + for (HugeGraphLoader.InputTaskItem item : tasks) { + item.reader.close(); + } + Assert.assertNull(context.newProgress().get("removed")); + } else { + for (InputStruct struct : structs) { + org.apache.hugegraph.loader.reader.file.LocalFileReader scanner = + new org.apache.hugegraph.loader.reader.file.LocalFileReader( + (FileSource) struct.input()); + try (org.apache.hugegraph.loader.reader.InputReader reader = scanner.split().get(0)) { + reader.init(context, struct); + } + } + } + if (incremental) { + org.apache.hugegraph.loader.progress.InputItemProgress oldLoaded = + context.oldProgress().get("0").loadedItem(firstFile.getName()); + org.apache.hugegraph.loader.progress.InputItemProgress newLoaded = + context.newProgress().get("0").loadedItem(firstFile.getName()); + org.apache.hugegraph.loader.progress.InputItemProgress oldLoading = + context.oldProgress().get("1").loadingItem(secondFile.getName()); + org.apache.hugegraph.loader.progress.InputItemProgress newLoading = + context.newProgress().get("1").loadingItem(secondFile.getName()); + Assert.assertNotSame(oldLoaded, newLoaded); + Assert.assertNotSame(oldLoading, newLoading); + Assert.assertTrue(newLoaded instanceof org.apache.hugegraph.loader.progress.FileItemProgress); + Assert.assertEquals(oldLoaded, newLoaded); + newLoading.offset(2L); + newLoading.confirmOffset(); + Assert.assertEquals(1L, oldLoading.offset()); + newLoading.offset(1L); + newLoading.confirmOffset(); + HugeGraphLoader loader = loader(context, structs); + Method cleanup = HugeGraphLoader.class.getDeclaredMethod("cleanupEmptyProgress"); + cleanup.setAccessible(true); + cleanup.invoke(loader); + Assert.assertEquals(2, context.newProgress().inputProgress().size()); + Assert.assertEquals(3L, context.newProgress().totalInputRead()); + context.stopLoading(); + context.close(); + LoadProgress next = LoadProgress.parse(options); + Assert.assertEquals(3L, next.totalInputRead()); + Assert.assertEquals(5L, next.vertexLoaded()); + Assert.assertEquals(2L, next.edgeLoaded()); + Assert.assertNull(next.get("removed")); + } else { + Assert.assertEquals(0L, context.newProgress().totalInputRead()); + Assert.assertEquals(0L, context.newProgress().vertexLoaded()); + Assert.assertEquals(0L, context.newProgress().edgeLoaded()); + Assert.assertTrue(context.newProgress().get("0").loadedItems().isEmpty()); + Assert.assertTrue(context.newProgress().get("1").loadingItems().isEmpty()); + } + } finally { + context.close(false); + } + } + } + + @Test + public void testDirectContextCloseStillSavesEmptyProgress() throws Exception { + try (ModeServer server = new ModeServer()) { + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = this.folder.newFile("direct-empty.json").getAbsolutePath(); + LoadContext context = new LoadContext(options); + context.close(); + Assert.assertTrue(context.closed()); + Assert.assertTrue(new File(LoadProgress.format(options, context.timestamp())).exists()); + Assert.assertTrue(context.newProgress().inputProgress().isEmpty()); + } + } + + @Test + public void testShutdownUnregistersOwnNativeHook() throws Exception { + try (ModeServer server = new ModeServer()) { + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = this.folder.newFile("hook-success.json").getAbsolutePath(); + HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList())); + Field field = HugeGraphLoader.class.getDeclaredField("shutdownHook"); + field.setAccessible(true); + Thread hook = (Thread) field.get(loader); + try { + loader.shutdown(); + Assert.assertTrue(loader.context().closed()); + Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); + loader.shutdown(); + } finally { + Runtime.getRuntime().removeShutdownHook(hook); + } + } + } + + @Test + public void testLoadFailureStillUnregistersClosedLoaderHook() throws Exception { + try (ModeServer server = new ModeServer()) { + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = this.folder.newFile("hook-failure.json").getAbsolutePath(); + HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList())); + Field field = HugeGraphLoader.class.getDeclaredField("shutdownHook"); + field.setAccessible(true); + Thread hook = (Thread) field.get(loader); + IllegalStateException failure = new IllegalStateException("original loading failure"); + loader.context().failLoading(failure); + try { + try { + loader.shutdown(); + Assert.fail("shutdown must retain the loading failure"); + } catch (IllegalStateException actual) { + Assert.assertSame(failure, actual); + } + Assert.assertTrue(loader.context().closed()); + Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); + loader.shutdown(); + } finally { + Runtime.getRuntime().removeShutdownHook(hook); + } + } + } + private static HugeGraphLoader loader(LoadContext context, InputStruct... inputs) throws Exception { HugeGraphLoader loader = allocate(HugeGraphLoader.class); set(loader, "context", context); diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/TaskManagerTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/TaskManagerTest.java index 46eeeb91e..deb89442f 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/TaskManagerTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/TaskManagerTest.java @@ -593,6 +593,9 @@ private void assertInterruptedShutdown(boolean cancel, boolean timeout, boolean setField(loader, "context", context); setField(loader, "manager", manager); setField(loader, "options", options); + Thread hook = new Thread(loader::shutdown); + setField(loader, "shutdownHook", hook); + Runtime.getRuntime().addShutdownHook(hook); ExecutorService caller = Executors.newSingleThreadExecutor(); try { manager.submitBatch(struct, mapping, Arrays.asList(new Record("in-flight", new Edge("knows")))); @@ -620,6 +623,11 @@ private void assertInterruptedShutdown(boolean cancel, boolean timeout, boolean } Assert.assertTrue(shutdown.get(5, TimeUnit.SECONDS)); Assert.assertEquals(!timeout, context.closed()); + Assert.assertEquals(timeout, Runtime.getRuntime().removeShutdownHook(hook)); + if (timeout) { + // An unfinished owner keeps its native hook and can close on retry. + Runtime.getRuntime().addShutdownHook(hook); + } Assert.assertFalse(client.closedWhileRunning); Assert.assertFalse(context.noError()); if (!timeout) { @@ -634,7 +642,14 @@ private void assertInterruptedShutdown(boolean cancel, boolean timeout, boolean Assert.assertEquals(1L, client.closed.getCount()); } Assert.assertFalse(new File(LoadProgress.format(options, "test")).exists()); + if (timeout) { + graph.release.countDown(); + Assert.assertThrows(LoadException.class, loader::shutdown); + Assert.assertTrue(context.closed()); + Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); + } } finally { + Runtime.getRuntime().removeShutdownHook(hook); graph.release.countDown(); caller.shutdownNow(); caller.awaitTermination(5, TimeUnit.SECONDS); From ad0de8f56936a603aff04723fc221f2e2d0823c5 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 12:19:02 +0800 Subject: [PATCH 04/14] fix(hubble): complete deletion and owner recovery - route Ingest and graph cleanup through scoped job deletion - preserve permission and remote-drop failure boundaries - retry rejected dispatch after SQL and commit failures - isolate polling writes and retire owners after commit - document and verify deletion and recovery contracts --- hugegraph-hubble/README.md | 29 +- .../controller/ingest/IngestController.java | 2 +- .../hugegraph/entity/load/LoadTask.java | 4 + .../service/load/JobManagerService.java | 11 +- .../service/load/LoadTaskService.java | 125 ++++---- .../ingest/IngestControllerTest.java | 54 ++++ .../IngestTransactionIntegrationTest.java | 41 +++ .../unit/GraphsControllerCanonicalTest.java | 63 ++++ .../hugegraph/unit/LoadTaskCleanupTest.java | 268 +++++++++++++++++- 9 files changed, 534 insertions(+), 63 deletions(-) diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index 8b4b0917c..b98ce1b48 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -70,10 +70,27 @@ closed resources under admission. The Job task create/start endpoints and Job management deletion share Job admission, so deletion enumerates tasks started before it and later starts recheck that the Job still exists. Job deletion commits physical stops before atomically deleting task and job metadata; a metadata rollback leaves those -tasks stopped. If delayed initial publication cannot read task metadata, it closes +tasks stopped. The Ingest task deletion route uses this same scoped Job deletion. +Graph deletion keeps its remote-first order: if the Server refuses the drop, local +Job/task metadata and owners are preserved. After a successful drop, Hubble +enumerates that graph's existing Jobs and applies the same deletion to each one, +waiting for its Loaders to close before deleting its metadata. A cleanup failure +leaves that Job and its task rows recoverable; earlier Jobs have already committed +their cleanup, and the remote graph drop cannot be rolled back. This enumeration +does not prevent new Jobs from being created, so Jobs outside the snapshot are +preserved. It provides no graph-wide admission barrier. + +If delayed initial publication cannot read task metadata, it closes its unused Loader and retains that closed owner for the existing progress poll. The poll marks only a still-RUNNING row as failed and retires the owner after commit; -a missing or advanced row is preserved. +a missing or advanced row is preserved. A rejected dispatch whose failure cannot +be committed also retains its closed owner. The poll retries the conditional +Task-to-FAILED and Job-to-FAILED changes in one transaction, preserving a task row +that has since advanced or disappeared. Each owner's polling write uses an H2 +savepoint: a write failure rolls back that owner's changes and leaves it for a +later poll while other owners can finish persistence. Owners are retired only +after the polling transaction commits; a deleted final task row retires its closed +owner without recreating metadata. Finished tasks release their native JVM shutdown hook once their context closes. If shutdown cannot terminate the task's workers, Hubble retains the failed owner and rejects retry/resume until its context is actually closed; restart Hubble to @@ -91,7 +108,13 @@ Server instances. The Loader [startup mode ownership guard](https://github.com/apache/hugegraph-toolchain/pull/812) is a prerequisite: without it, cleanup after startup fails can reset another -importer's `LOADING` mode. +importer's `LOADING` mode. Multi-file jobs start a Loader for each file, so shared +graph-mode ownership also requires the matching Server, Client and Loader ownership +protocol 1 support. The paired [Server change](https://github.com/apache/hugegraph/pull/3331) +and [website documentation](https://github.com/apache/hugegraph-doc/pull/525) describe +that protocol; the Client and Loader must use it too. The startup guard alone does +not keep `LOADING` until every sibling Loader finishes. This lifecycle cleanup does +not implement that protocol, and multi-file validation requires their combined build. ## Local development feedback loop diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java index 3fadd9af0..37735dc11 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java @@ -378,7 +378,7 @@ public Response deleteTask(@PathVariable("id") int id) { .message("Task not found: " + id).build(); } this.requireJobWrite(job); - jobManagerService.remove(id); + jobManagerService.deleteJob(job.getGraphSpace(), job.getGraph(), id); return Response.builder().status(Constant.STATUS_OK).build(); } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java index 9a68e0206..37bd2f773 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java @@ -72,6 +72,10 @@ public class LoadTask implements Runnable { @JsonIgnore private transient volatile boolean initialStateReadFailed; + @TableField(exist = false) + @JsonIgnore + private transient volatile boolean dispatchFailurePending; + @TableId(type = IdType.AUTO) @MergeProperty(useNew = false) @JsonProperty("id") diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java index 60b049710..8ea6f27f2 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java @@ -29,7 +29,6 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Striped; import lombok.extern.log4j.Log4j2; @@ -272,10 +271,14 @@ public void remove(int id) { } } - @Transactional(isolation = Isolation.READ_COMMITTED) + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void removeByGraph(String graphSpace, String graph) { - this.mapper.deleteByMap(ImmutableMap.of("graphspace", graphSpace, - "graph", graph)); + QueryWrapper query = Wrappers.query(); + query.eq("graphspace", graphSpace).eq("graph", graph); + // Clean only the enlisted Jobs; newly created Jobs must not lose their task owners. + for (JobManager job : this.mapper.selectList(query)) { + this.deleteJob(graphSpace, graph, job.getId()); + } } @Transactional(propagation = Propagation.NOT_SUPPORTED) diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java index 3f4c3bf27..5b0a0e6c1 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java @@ -369,6 +369,7 @@ private void markDispatchFailed(LoadTask task, RuntimeException cause) { return; } log.error("Failed to dispatch load task {}, marking task and job as FAILED", task.getId(), cause); + task.setDispatchFailurePending(true); closeUnscheduled(task, cause); try { TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); @@ -377,33 +378,36 @@ private void markDispatchFailed(LoadTask task, RuntimeException cause) { if (this.runningTaskContainer.get(task.getId()) != task) { return null; } - int updated = this.mapper.update(null, Wrappers.update() - .eq("id", task.getId()) - .eq("load_status", LoadStatus.RUNNING.getValue()) - .set("load_status", LoadStatus.FAILED.getValue())); - if (updated == 0) { - return null; - } - if (updated != 1) { - throw new InternalException("entity.update.failed", task); - } - if (task.getJobId() != null) { - JobManager job = this.jobManagerMapper.selectById(task.getJobId()); - if (job != null) { - job.setJobStatus(JobStatus.FAILED); - job.setUpdateTime(HubbleUtil.nowDate()); - if (this.jobManagerMapper.updateById(job) != 1) { - throw new InternalException("entity.update.failed", job); - } - } - } + this.persistDispatchFailure(task); return null; }); + this.removeClosedOwner(task); } catch (RuntimeException failure) { // The ingest transaction is committed. Do not report a retryable dispatch failure. log.error("Failed to persist dispatch failure for load task {}", task.getId(), failure); - } finally { - this.removeClosedOwner(task); + } + } + + private void persistDispatchFailure(LoadTask task) { + int updated = this.mapper.update(null, Wrappers.update() + .eq("id", task.getId()) + .eq("load_status", LoadStatus.RUNNING.getValue()) + .set("load_status", LoadStatus.FAILED.getValue())); + if (updated == 0) { + return; + } + if (updated != 1) { + throw new InternalException("entity.update.failed", task); + } + if (task.getJobId() != null) { + JobManager job = this.jobManagerMapper.selectById(task.getJobId()); + if (job != null) { + job.setJobStatus(JobStatus.FAILED); + job.setUpdateTime(HubbleUtil.nowDate()); + if (this.jobManagerMapper.updateById(job) != 1) { + throw new InternalException("entity.update.failed", job); + } + } } } @@ -714,42 +718,61 @@ public void pauseAllTasks() { @Scheduled(fixedDelay = 1 * 1000) @Transactional(isolation = Isolation.READ_COMMITTED) public void updateLoadTaskProgress() { + TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); + transaction.setPropagationBehavior(TransactionDefinition.PROPAGATION_NESTED); for (LoadTask task : this.runningTaskContainer.values()) { task.lock(); try { - if (this.runningTaskContainer.get(task.getId()) != task) { - continue; + transaction.execute(status -> { + this.updateLoadTaskProgress(task); + return null; + }); + } catch (RuntimeException failure) { + log.warn("Failed to persist progress for load task {}; retaining its owner", task.getId(), failure); + } finally { + task.unlock(); + } + } + } + + private void updateLoadTaskProgress(LoadTask task) { + if (this.runningTaskContainer.get(task.getId()) != task) { + return; + } + HugeGraphLoader owner = task.getLoader(); + if (task.isFinished() && (owner == null || owner.context().closed())) { + if (task.isDispatchFailurePending()) { + this.persistDispatchFailure(task); + } else if (task.isInitialStateReadFailed()) { + // Unknown initial metadata must not overwrite a later terminal state or recreate a missing row. + int updated = this.mapper.update(null, + Wrappers.update().eq("id", task.getId()) + .eq("load_status", LoadStatus.RUNNING.getValue()) + .set("load_status", LoadStatus.FAILED.getValue())); + if (updated != 0 && updated != 1) { + throw new InternalException("entity.update.failed", task); } - HugeGraphLoader owner = task.getLoader(); - if (task.isFinished() && (owner == null || owner.context().closed())) { - if (task.isInitialStateReadFailed()) { - // Unknown initial metadata must not overwrite a later terminal state - // or recreate a missing row. - int updated = this.mapper.update(null, - Wrappers.update().eq("id", task.getId()) - .eq("load_status", LoadStatus.RUNNING.getValue()) - .set("load_status", LoadStatus.FAILED.getValue())); - if (updated != 0 && updated != 1) { - throw new InternalException("entity.update.failed", task); - } - } else { - // Retry a final write that failed in the runner or pause/stop. - this.update(task); - } - this.removeOwnerAfterCommit(task); - } else if (task.getStatus().inRunning()) { - LoadContext context = task.context(); - long readLines = context.newProgress().totalInputRead(); - if (readLines == 0L) { - readLines = context.oldProgress().totalInputRead(); - } - task.setFileReadLines(readLines); - task.setCurrDuration(context.summary().totalTime()); + } else { + try { + // Retry a final write that failed in the runner or pause/stop. this.update(task); + } catch (RuntimeException failure) { + if (this.get(task.getId()) != null) { + throw failure; + } + // A deleted row has no final state to persist; retire only this closed owner. } - } finally { - task.unlock(); } + this.removeOwnerAfterCommit(task); + } else if (task.getStatus().inRunning()) { + LoadContext context = task.context(); + long readLines = context.newProgress().totalInputRead(); + if (readLines == 0L) { + readLines = context.oldProgress().totalInputRead(); + } + task.setFileReadLines(readLines); + task.setCurrDuration(context.summary().totalTime()); + this.update(task); } } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/controller/ingest/IngestControllerTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/controller/ingest/IngestControllerTest.java index d3f765bfd..a668186eb 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/controller/ingest/IngestControllerTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/controller/ingest/IngestControllerTest.java @@ -61,6 +61,56 @@ public class IngestControllerTest { + @Test + public void testDeleteTaskUsesScopedJobLifecycleAfterPermissionCheck() + throws Exception { + TestIngestController controller = new TestIngestController(); + JobManagerService jobs = Mockito.mock(JobManagerService.class); + this.setField(controller, "jobManagerService", jobs); + Mockito.when(jobs.get(7)).thenReturn(JobManager.builder().id(7) + .graphSpace("space-a").graph("graph-a").build()); + Mockito.doAnswer(invocation -> { + Assert.assertEquals("space-a", controller.writeGraphSpace); + return null; + }).when(jobs).deleteJob("space-a", "graph-a", 7); + + Assert.assertEquals(Constant.STATUS_OK, controller.deleteTask(7).getStatus()); + + Mockito.verify(jobs).deleteJob("space-a", "graph-a", 7); + Mockito.verify(jobs, Mockito.never()).remove(Mockito.anyInt()); + } + + @Test + public void testDeleteTaskKeepsMissingAndDeniedResponses() throws Exception { + TestIngestController controller = new TestIngestController(); + JobManagerService jobs = Mockito.mock(JobManagerService.class); + this.setField(controller, "jobManagerService", jobs); + Assert.assertEquals(Constant.STATUS_NOT_FOUND, controller.deleteTask(7).getStatus()); + Assert.assertNull(controller.writeGraphSpace); + + Mockito.when(jobs.get(7)).thenReturn(JobManager.builder().id(7) + .graphSpace("space-a").graph("graph-a").build()); + controller.denyWrite = true; + Assert.assertThrows(org.apache.hugegraph.exception.ForbiddenException.class, + () -> controller.deleteTask(7)); + Mockito.verify(jobs, Mockito.never()).deleteJob(Mockito.anyString(), Mockito.anyString(), Mockito.anyInt()); + Mockito.verify(jobs, Mockito.never()).remove(Mockito.anyInt()); + } + + @Test + public void testDeleteTaskPropagatesLifecycleRefusal() throws Exception { + TestIngestController controller = new TestIngestController(); + JobManagerService jobs = Mockito.mock(JobManagerService.class); + this.setField(controller, "jobManagerService", jobs); + Mockito.when(jobs.get(7)).thenReturn(JobManager.builder().id(7) + .graphSpace("space-a").graph("graph-a").build()); + RuntimeException refusal = new IllegalStateException("owner is not closed"); + Mockito.doThrow(refusal).when(jobs).deleteJob("space-a", "graph-a", 7); + + Assert.assertThrows(IllegalStateException.class, () -> controller.deleteTask(7)); + Mockito.verify(jobs, Mockito.never()).remove(Mockito.anyInt()); + } + @After public void tearDown() { RequestContextHolder.resetRequestAttributes(); @@ -579,6 +629,7 @@ private static class TestIngestController extends IngestController { private String checkedGraphSpace; private String writeGraphSpace; + private boolean denyWrite; @Override protected HugeClient authClient(String graphSpace, String graph) { @@ -594,6 +645,9 @@ protected void requireGraphSpaceAccess(HugeClient client, @Override protected HugeClient requireGraphSpaceWrite(String graphSpace) { this.writeGraphSpace = graphSpace; + if (this.denyWrite) { + throw new org.apache.hugegraph.exception.ForbiddenException("denied graph write"); + } return Mockito.mock(HugeClient.class); } } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java index c55a1690b..7aeecabe8 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java @@ -294,6 +294,34 @@ public void testFailedDiskCleanupLeavesRetryableOrphanRecord() { Assert.assertEquals(0, this.count("file_mapping")); } + @Test + public void testRejectedDispatchRetriesTaskAndJobAfterDatabaseRecovery() throws Exception { + this.executor.reject(true); + this.jdbc.execute("ALTER TABLE job_manager ADD CONSTRAINT reject_failed_job CHECK (job_status <> " + + JobStatus.FAILED.getValue() + ")"); + LoadTask task; + try { + task = this.jobService.createIngestTask( + this.job(), this.mapping(), this.connection(), this.client()); + } finally { + this.jdbc.execute("ALTER TABLE job_manager DROP CONSTRAINT reject_failed_job"); + } + Assert.assertEquals(LoadStatus.RUNNING, this.taskService.get(task.getId()).getStatus()); + Assert.assertEquals(JobStatus.LOADING, this.jobService.get(task.getJobId()).getJobStatus()); + java.lang.reflect.Field ownersField = LoadTaskService.class.getDeclaredField("runningTaskContainer"); + ownersField.setAccessible(true); + java.util.Map owners = (java.util.Map) ownersField.get( + org.springframework.test.util.AopTestUtils.getUltimateTargetObject(this.taskService)); + Assert.assertSame(task, owners.get(task.getId())); + Assert.assertTrue(task.getLoader().context().closed()); + + this.taskService.updateLoadTaskProgress(); + + Assert.assertEquals(LoadStatus.FAILED, this.taskService.get(task.getId()).getStatus()); + Assert.assertEquals(JobStatus.FAILED, this.jobService.get(task.getJobId()).getJobStatus()); + Assert.assertFalse(owners.containsKey(task.getId())); + } + @Test public void testRejectedDispatchCommitsFailedTaskAndJobWithoutThrowing() { this.executor.reject(true); @@ -402,6 +430,7 @@ private HugeClient client() { } private void cleanup() { + this.taskService.clearOwners(); JdbcTemplate template = this.jdbc == null ? new JdbcTemplate(this.dataSource) : this.jdbc; template.update("DELETE FROM load_task"); @@ -457,6 +486,18 @@ public static class TestLoadTaskService extends LoadTaskService { private boolean failTaskInsert; + @SuppressWarnings("unchecked") + public void clearOwners() { + java.util.Map owners = (java.util.Map) + org.springframework.test.util.ReflectionTestUtils.getField(this, "runningTaskContainer"); + for (LoadTask task : owners.values()) { + if (task.getLoader() != null) { + task.getLoader().shutdown(); + } + } + owners.clear(); + } + public void failTaskInsert(boolean fail) { this.failTaskInsert = fail; } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphsControllerCanonicalTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphsControllerCanonicalTest.java index c13130e9b..4000d7f10 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphsControllerCanonicalTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphsControllerCanonicalTest.java @@ -38,6 +38,10 @@ import org.apache.hugegraph.exception.ExternalException; import org.apache.hugegraph.options.HubbleOptions; import org.apache.hugegraph.service.graphs.GraphsService; +import org.apache.hugegraph.service.load.JobManagerService; +import org.apache.hugegraph.service.query.ExecuteHistoryService; +import org.apache.hugegraph.service.query.GremlinCollectionService; +import org.apache.hugegraph.service.space.VermeerService; import org.apache.hugegraph.testutil.Assert; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; @@ -51,6 +55,9 @@ public class GraphsControllerCanonicalTest { private MockMvc mvc; private HugeClient client; private GraphsService graphsService; + private JobManagerService jobs; + private ExecuteHistoryService history; + private GremlinCollectionService collections; private RequestPostProcessor withClient; @Before @@ -58,6 +65,13 @@ public void setup() throws Exception { GraphsController controller = new GraphsController(); this.graphsService = Mockito.mock(GraphsService.class); this.setField(controller, "graphsService", this.graphsService); + this.jobs = Mockito.mock(JobManagerService.class); + this.history = Mockito.mock(ExecuteHistoryService.class); + this.collections = Mockito.mock(GremlinCollectionService.class); + this.setField(controller, "jobManagerService", this.jobs); + this.setField(controller, "executeHistoryService", this.history); + this.setField(controller, "gremlinCollectionService", this.collections); + this.setField(controller, "vermeerService", Mockito.mock(VermeerService.class)); HugeConfig config = Mockito.mock(HugeConfig.class); Mockito.when(config.get(HubbleOptions.PD_ENABLED)).thenReturn(false); this.setField(controller, "config", config); @@ -71,6 +85,55 @@ public void setup() throws Exception { this.mvc = MockMvcBuilders.standaloneSetup(controller).build(); } + @Test + public void testGraphDeletionCleansScopedJobsAfterRemoteSuccess() throws Exception { + this.mvc.perform(delete("/api/v1.3/graphspaces/DEFAULT/graphs/graph_a") + .with(this.withClient)) + .andExpect(status().isOk()); + + org.mockito.InOrder order = Mockito.inOrder(this.graphsService, this.history, this.collections, this.jobs); + order.verify(this.graphsService).delete(this.client, "graph_a", "I'm sure to drop the graph"); + order.verify(this.history).deleteByGraph("DEFAULT", "graph_a"); + order.verify(this.collections).deleteByGraph("DEFAULT", "graph_a"); + order.verify(this.jobs).removeByGraph("DEFAULT", "graph_a"); + } + + @Test + public void testGraphDeletionKeepsLocalStateWhenRemoteDropFails() throws Exception { + Mockito.doThrow(new ExternalException("remote drop rejected")).when(this.graphsService) + .delete(this.client, "graph_a", "I'm sure to drop the graph"); + try { + this.mvc.perform(delete("/api/v1.3/graphspaces/DEFAULT/graphs/graph_a").with(this.withClient)); + org.junit.Assert.fail("Expected remote deletion rejection"); + } catch (ServletException e) { + Assert.assertInstanceOf(ExternalException.class, e.getCause()); + } + Mockito.verifyNoInteractions(this.jobs, this.history, this.collections); + } + + @Test + public void testGraphDeletionKeepsLocalStateWhenAuthenticationFails() throws Exception { + GraphsController controller = new GraphsController() { + @Override + public boolean isVermeerEnabled() { + return false; + } + + @Override + protected HugeClient authClient(String graphspace, String graph) { + throw new org.apache.hugegraph.exception.ForbiddenException("graph deletion denied"); + } + }; + this.setField(controller, "graphsService", this.graphsService); + this.setField(controller, "jobManagerService", this.jobs); + this.setField(controller, "executeHistoryService", this.history); + this.setField(controller, "gremlinCollectionService", this.collections); + + Assert.assertThrows(org.apache.hugegraph.exception.ForbiddenException.class, + () -> controller.delete("DEFAULT", "graph_a", "I'm sure to drop the graph")); + Mockito.verifyNoInteractions(this.graphsService, this.jobs, this.history, this.collections); + } + @Test public void testCanonicalCreateGraphUsesPathNameAndJsonBody() throws Exception { diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java index 5678d868a..1fb07496e 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java @@ -730,6 +730,7 @@ private void assertTerminalPersistenceRecovery(boolean stop) throws Exception { LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); RuntimeException callbackFailure = new RuntimeException("callback database unavailable"); + enablePollTransactions(service); RuntimeException stopFailure = new RuntimeException("stop database unavailable"); if (stop) { Mockito.when(mapper.updateById(task)).thenThrow(callbackFailure).thenThrow(stopFailure).thenReturn(1); @@ -866,6 +867,7 @@ public void testFinalProgressPollDoesNotSaveReplacedOwner() throws Exception { LoadTask fresh = LoadTask.builder().id(9).options(new LoadOptions()).lastDuration(0L) .status(LoadStatus.FAILED).build(); LoadTaskService service = new LoadTaskService(); + enablePollTransactions(service); LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); Mockito.when(mapper.selectById(9)).thenReturn(fresh); @@ -1505,14 +1507,20 @@ protected void doCommit( } }; manager.setRollbackOnCommitFailure(true); + set(service, "transactionManager", manager); org.springframework.transaction.support.TransactionTemplate transaction = new org.springframework.transaction.support.TransactionTemplate(manager); try { transaction.execute(status -> { service.updateLoadTaskProgress(); + if (!commitFailure) { + status.setRollbackOnly(); + } return null; }); - Assert.fail("transaction must fail"); + if (commitFailure) { + Assert.fail("commit must fail"); + } } catch (RuntimeException actual) { Assert.assertSame(failure, actual); } @@ -1921,6 +1929,77 @@ public void testDeleteRejectsTerminalSnapshotWithUnclosedOwner() throws Exceptio Mockito.verify(mapper, Mockito.never()).delete(Mockito.any()); } + @Test + public void testGraphDeletionCommitsTaskCleanupOutsideCallerTransaction() throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + fixture.enableGraphDeletion(); + new org.springframework.transaction.support.TransactionTemplate(fixture.manager).execute(status -> { + fixture.service.removeByGraph("space", "graph"); + Assert.assertTrue(TransactionSynchronizationManager.isActualTransactionActive()); + try (java.sql.Connection other = fixture.dataSource.getConnection(); + java.sql.Statement statement = other.createStatement(); + java.sql.ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM delete_tasks")) { + Assert.assertTrue(rows.next()); + Assert.assertEquals(0, rows.getInt(1)); + } catch (java.sql.SQLException failure) { + throw new RuntimeException(failure); + } + status.setRollbackOnly(); + return null; + }); + Assert.assertEquals(Integer.valueOf(0), fixture.jdbc.queryForObject( + "SELECT COUNT(*) FROM delete_jobs", Integer.class)); + Mockito.verify(fixture.jobs, Mockito.never()).deleteByMap(Mockito.anyMap()); + } + } + + @Test + public void testGraphDeletionKeepsNewJobsAndOtherGraphScopes() throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + fixture.enableGraphDeletion(); + fixture.jdbc.update("INSERT INTO delete_jobs VALUES (8, ?, 'other-space', 'graph'), " + + "(9, ?, 'space', 'other-graph')", 0, 0); + Mockito.doAnswer(invocation -> { + java.util.List snapshot = + Collections.singletonList(fixture.jobs.selectById(7)); + fixture.jdbc.update("INSERT INTO delete_jobs VALUES (11, ?, 'space', 'graph')", 0); + fixture.jdbc.update("INSERT INTO delete_tasks VALUES (12, ?, 11)", LoadStatus.RUNNING.getValue()); + return snapshot; + }).when(fixture.jobs).selectList(Mockito.any()); + + fixture.service.removeByGraph("space", "graph"); + + Assert.assertEquals(java.util.Arrays.asList(8, 9, 11), fixture.jdbc.queryForList( + "SELECT id FROM delete_jobs ORDER BY id", Integer.class)); + Assert.assertEquals(Collections.singletonList(12), fixture.jdbc.queryForList( + "SELECT id FROM delete_tasks", Integer.class)); + Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=12", Byte.class)); + Mockito.verify(fixture.jobs, Mockito.never()).deleteByMap(Mockito.anyMap()); + } + } + + @Test + public void testGraphDeletionFailureRetainsFailedJobAndCommittedStops() throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + fixture.enableGraphDeletion(); + fixture.jdbc.update("INSERT INTO delete_jobs VALUES (8, ?, 'space', 'graph')", 0); + fixture.jdbc.update("INSERT INTO delete_tasks VALUES (11, ?, 8)", LoadStatus.RUNNING.getValue()); + RuntimeException refusal = new IllegalStateException("metadata deletion rejected"); + Mockito.doThrow(refusal).when(fixture.jobs).deleteById(8); + + org.apache.hugegraph.testutil.Assert.assertThrows(IllegalStateException.class, + () -> fixture.service.removeByGraph("space", "graph")); + + Assert.assertEquals(Collections.singletonList(8), fixture.jdbc.queryForList( + "SELECT id FROM delete_jobs", Integer.class)); + Assert.assertEquals(Collections.singletonList(11), fixture.jdbc.queryForList( + "SELECT id FROM delete_tasks", Integer.class)); + Assert.assertEquals(LoadStatus.STOPPED.getValue(), fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=11", Byte.class)); + } + } + @Test public void testJobDeletionCommitsStopsBeforeCallerTransactionCompletes() throws Exception { for (boolean scoped : new boolean[]{false, true}) { @@ -2009,6 +2088,112 @@ public void testRetryWinningBeforeMetadataDeleteRollsBackEarlierTaskDeletion() t } } + @Test + public void testDispatchCommitFailureKeepsClosedOwnerForConditionalRecovery() throws Exception { + for (LoadStatus advanced : new LoadStatus[]{LoadStatus.RUNNING, LoadStatus.SUCCEED, null}) { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + LoadTask task = this.task(this.loader()); + task.setId(9); + task.setJobId(7); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + Mockito.doThrow(new java.util.concurrent.RejectedExecutionException("queue full")) + .when(executor).execute(Mockito.any(), Mockito.any()); + org.springframework.jdbc.datasource.DataSourceTransactionManager rejecting = + new org.springframework.jdbc.datasource.DataSourceTransactionManager(fixture.dataSource) { + @Override + protected void doCommit( + org.springframework.transaction.support.DefaultTransactionStatus status) { + throw new org.springframework.transaction.TransactionSystemException("commit failed"); + } + }; + rejecting.setRollbackOnCommitFailure(true); + set(fixture.load, "taskExecutor", executor); + set(fixture.load, "transactionManager", rejecting); + set(fixture.load, "jobManagerMapper", fixture.jobs); + + dispatch(fixture.load, task); + + Assert.assertSame(task, fixture.owners.get(9)); + Assert.assertTrue(task.getLoader().context().closed()); + Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); + if (advanced == null) { + fixture.jdbc.update("DELETE FROM delete_tasks WHERE id=9"); + } else { + fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=9", advanced.getValue()); + } + set(fixture.load, "transactionManager", fixture.manager); + new org.springframework.transaction.support.TransactionTemplate(fixture.manager).execute(status -> { + fixture.load.updateLoadTaskProgress(); + Assert.assertSame("retire only after recovery commits", task, fixture.owners.get(9)); + return null; + }); + Assert.assertFalse(fixture.owners.containsKey(9)); + Assert.assertEquals(advanced == null ? Collections.emptyList() : Collections.singletonList( + advanced == LoadStatus.RUNNING ? LoadStatus.FAILED.getValue() : advanced.getValue()), + fixture.jdbc.queryForList("SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); + Assert.assertEquals(advanced == LoadStatus.RUNNING ? + org.apache.hugegraph.entity.enums.JobStatus.FAILED.getValue() : + org.apache.hugegraph.entity.enums.JobStatus.LOADING.getValue(), + fixture.jdbc.queryForObject("SELECT status FROM delete_jobs WHERE id=7", + Byte.class)); + } + } + } + + @Test + public void testPollMissingFinalRowDoesNotBlockOtherOwners() throws Exception { + this.assertPollFailureIsolation(true); + } + + @Test + public void testPollRollsBackFailedOwnerWriteAndContinuesOtherOwners() throws Exception { + this.assertPollFailureIsolation(false); + } + + private void assertPollFailureIsolation(boolean missing) throws Exception { + try (JobDeleteFixture fixture = new JobDeleteFixture()) { + LoadTask first = this.task(this.loader()); + first.setId(9); + first.run(); + LoadTask second = this.task(this.loader()); + second.setId(10); + second.run(); + Map owners = new java.util.concurrent.ConcurrentSkipListMap<>(); + owners.put(9, first); + owners.put(10, second); + set(fixture.load, "runningTaskContainer", owners); + set(fixture.load, "transactionManager", fixture.manager); + if (missing) { + fixture.jdbc.update("DELETE FROM delete_tasks WHERE id=9"); + } + Mockito.when(fixture.tasks.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { + LoadTask task = invocation.getArgument(0); + int count = fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=?", + task.getStatus().getValue(), task.getId()); + if (!missing && task == first) { + throw new IllegalStateException("failed after partial SQL"); + } + return count; + }); + + new org.springframework.transaction.support.TransactionTemplate(fixture.manager).execute(status -> { + fixture.load.updateLoadTaskProgress(); + Assert.assertSame(second, owners.get(10)); + return null; + }); + + Assert.assertFalse(owners.containsKey(10)); + Assert.assertEquals(missing ? null : first, owners.get(9)); + Assert.assertEquals(LoadStatus.FAILED.getValue(), fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=10", Byte.class)); + if (!missing) { + Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); + } + } + } + @Test public void testOwnDispatchRejectionConditionallyMarksTaskAndJobFailed() throws Exception { this.assertOwnDispatchFailureState(false); @@ -2054,16 +2239,21 @@ private void assertOwnDispatchFailureState(boolean advanced) throws Exception { } @Test - public void testInitialReadFailureReasonIsRuntimeOnly() throws Exception { + public void testRecoveryReasonsAreRuntimeOnly() throws Exception { LoadTask task = LoadTask.builder().status(LoadStatus.FAILED).lastDuration(0L).currDuration(0L) .fileReadLines(0L).fileTotalLines(0L).build(); task.setInitialStateReadFailed(true); + task.setDispatchFailurePending(true); com.fasterxml.jackson.databind.ObjectMapper json = new com.fasterxml.jackson.databind.ObjectMapper(); com.fasterxml.jackson.databind.JsonNode value = json.valueToTree(task); Assert.assertFalse(value.has("initialStateReadFailed")); Assert.assertFalse(value.has("initial_state_read_failed")); - LoadTask restored = json.readValue("{\"initialStateReadFailed\":true}", LoadTask.class); + Assert.assertFalse(value.has("dispatchFailurePending")); + Assert.assertFalse(value.has("dispatch_failure_pending")); + LoadTask restored = json.readValue( + "{\"initialStateReadFailed\":true,\"dispatchFailurePending\":true}", LoadTask.class); Assert.assertFalse(restored.isInitialStateReadFailed()); + Assert.assertFalse(restored.isDispatchFailurePending()); Field field = LoadTask.class.getDeclaredField("initialStateReadFailed"); Assert.assertFalse(field.getAnnotation(com.baomidou.mybatisplus.annotation.TableField.class).exist()); com.baomidou.mybatisplus.core.metadata.TableInfo table = @@ -2073,6 +2263,8 @@ public void testInitialReadFailureReasonIsRuntimeOnly() throws Exception { LoadTask.class); Assert.assertFalse(table.getFieldList().stream() .anyMatch(item -> item.getProperty().equals("initialStateReadFailed"))); + Assert.assertFalse(table.getFieldList().stream() + .anyMatch(item -> item.getProperty().equals("dispatchFailurePending"))); } @Test @@ -2187,7 +2379,15 @@ protected void doCommit(org.springframework.transaction.support.DefaultTransacti } org.springframework.transaction.support.TransactionTemplate transaction = new org.springframework.transaction.support.TransactionTemplate(tx); - if (sqlFailure || commitFailure) { + if (sqlFailure) { + transaction.execute(status -> { + fixture.load.updateLoadTaskProgress(); + return null; + }); + Assert.assertSame(task, fixture.owners.get(9)); + Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( + "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); + } else if (commitFailure) { try { transaction.execute(status -> { fixture.load.updateLoadTaskProgress(); @@ -2488,6 +2688,7 @@ private JobDeleteFixture() throws Exception { }); set(this.load, "mapper", this.tasks); set(this.load, "runningTaskContainer", this.owners); + set(this.load, "transactionManager", this.manager); org.apache.hugegraph.service.load.JobManagerService target = new org.apache.hugegraph.service.load.JobManagerService(); set(target, "mapper", this.jobs); @@ -2514,6 +2715,57 @@ private void delete(boolean scoped) { } } + private void enableGraphDeletion() { + this.jdbc.execute("ALTER TABLE delete_jobs ADD graphspace VARCHAR(48) DEFAULT 'space'"); + this.jdbc.execute("ALTER TABLE delete_jobs ADD graph_name VARCHAR(48) DEFAULT 'graph'"); + this.jdbc.execute("ALTER TABLE delete_tasks ADD job_id INT DEFAULT 7"); + Mockito.when(this.jobs.selectList(Mockito.any())).thenAnswer(invocation -> { + Assert.assertFalse("the graph cleanup must suspend the caller transaction", + TransactionSynchronizationManager.isActualTransactionActive()); + com.baomidou.mybatisplus.core.conditions.query.QueryWrapper query = invocation.getArgument(0); + query.getSqlSegment(); + Assert.assertTrue(query.getSqlSegment().contains("graphspace")); + Assert.assertTrue(query.getSqlSegment().contains("graph")); + Assert.assertTrue(query.getParamNameValuePairs().containsValue("space")); + Assert.assertTrue(query.getParamNameValuePairs().containsValue("graph")); + java.util.List result = new java.util.ArrayList<>(); + for (Integer id : this.jdbc.queryForList( + "SELECT id FROM delete_jobs WHERE graphspace='space' AND graph_name='graph' ORDER BY id", + Integer.class)) { + result.add(this.jobs.selectById(id)); + } + return result; + }); + Mockito.when(this.jobs.selectById(Mockito.anyInt())).thenAnswer(invocation -> { + int id = invocation.getArgument(0); + return this.jdbc.queryForList("SELECT id FROM delete_jobs WHERE id=?", Integer.class, id).isEmpty() ? + null : org.apache.hugegraph.entity.load.JobManager.builder().id(id) + .graphSpace("space").graph("graph").build(); + }); + Mockito.when(this.jobs.selectOne(Mockito.any())).thenAnswer(invocation -> { + com.baomidou.mybatisplus.core.conditions.query.QueryWrapper query = invocation.getArgument(0); + query.getSqlSegment(); + int id = (Integer) query.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Integer).findFirst().get(); + return this.jobs.selectById(id); + }); + Mockito.when(this.jobs.deleteById(Mockito.anyInt())).thenAnswer(invocation -> + this.jdbc.update("DELETE FROM delete_jobs WHERE id=?", (Integer) invocation.getArgument(0))); + Mockito.when(this.jobs.deleteByMap(Mockito.anyMap())).thenAnswer(invocation -> + this.jdbc.update("DELETE FROM delete_jobs WHERE graphspace='space' AND graph_name='graph'")); + Mockito.doAnswer(invocation -> { + com.baomidou.mybatisplus.core.conditions.query.QueryWrapper query = invocation.getArgument(0); + query.getSqlSegment(); + int jobId = (Integer) query.getParamNameValuePairs().values().stream().findFirst().get(); + java.util.List result = new java.util.ArrayList<>(); + for (Integer id : this.jdbc.queryForList( + "SELECT id FROM delete_tasks WHERE job_id=? ORDER BY id", Integer.class, jobId)) { + result.add(this.tasks.selectById(id)); + } + return result; + }).when(this.tasks).selectList(Mockito.any()); + } + @Override public void close() throws Exception { this.keeper.close(); @@ -2538,6 +2790,14 @@ private static void set(Object object, String name, Object value) throws Excepti field.set(object, value); } + private static void enablePollTransactions(LoadTaskService service) throws Exception { + org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = + new org.springframework.jdbc.datasource.DriverManagerDataSource(); + dataSource.setUrl("jdbc:h2:mem:poll-" + java.util.UUID.randomUUID()); + set(service, "transactionManager", + new org.springframework.jdbc.datasource.DataSourceTransactionManager(dataSource)); + } + private static void dispatch(LoadTaskService service, LoadTask task) throws Exception { Field mapperField = LoadTaskService.class.getDeclaredField("mapper"); mapperField.setAccessible(true); From 96946b70b74e02f323e81c95fe03d16984735763 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 12:38:46 +0800 Subject: [PATCH 05/14] fix(hubble): abort polls on transaction failures - propagate Spring transaction and savepoint failures - retain earlier owners when the polling transaction aborts - verify real deadlocks and ordinary SQL failure isolation - document the polling rollback boundary --- hugegraph-hubble/README.md | 6 +- .../service/load/LoadTaskService.java | 4 + .../hugegraph/unit/LoadTaskCleanupTest.java | 106 +++++++++++++++++- 3 files changed, 113 insertions(+), 3 deletions(-) diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index b98ce1b48..d0acca059 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -87,8 +87,10 @@ a missing or advanced row is preserved. A rejected dispatch whose failure cannot be committed also retains its closed owner. The poll retries the conditional Task-to-FAILED and Job-to-FAILED changes in one transaction, preserving a task row that has since advanced or disappeared. Each owner's polling write uses an H2 -savepoint: a write failure rolls back that owner's changes and leaves it for a -later poll while other owners can finish persistence. Owners are retired only +savepoint: an ordinary SQL failure whose savepoint rollback succeeds leaves that +owner for a later poll while other owners can finish persistence. A transaction or +savepoint operation failure aborts the whole polling transaction and retains its +owners, including those processed earlier. Owners are retired only after the polling transaction commits; a deleted final task row retires its closed owner without recreating metadata. Finished tasks release their native JVM shutdown hook once their context closes. diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java index 5b0a0e6c1..91fc1ecc2 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java @@ -67,6 +67,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.TransactionException; import org.springframework.transaction.annotation.Isolation; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionSynchronization; @@ -727,6 +728,9 @@ public void updateLoadTaskProgress() { this.updateLoadTaskProgress(task); return null; }); + } catch (TransactionException failure) { + // A failed savepoint/transaction operation cannot safely commit earlier owner writes. + throw failure; } catch (RuntimeException failure) { log.warn("Failed to persist progress for load task {}; retaining its owner", task.getId(), failure); } finally { diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java index 1fb07496e..631310760 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java @@ -2141,6 +2141,97 @@ protected void doCommit( } } + @Test + public void testPollSavepointRollbackFailureAbortsOuterTransaction() throws Exception { + org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = + new org.springframework.jdbc.datasource.DriverManagerDataSource( + "jdbc:h2:mem:poll-deadlock-" + java.util.UUID.randomUUID() + ";LOCK_TIMEOUT=3000", "sa", ""); + org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(dataSource); + org.springframework.jdbc.datasource.DataSourceTransactionManager manager = + new org.springframework.jdbc.datasource.DataSourceTransactionManager(dataSource); + try (java.sql.Connection keeper = dataSource.getConnection()) { + jdbc.execute("CREATE TABLE tasks (id INT PRIMARY KEY, load_status TINYINT, n INT)"); + jdbc.update("INSERT INTO tasks VALUES (9, ?, 0), (10, ?, 0), (11, ?, 0), (12, ?, 0)", + LoadStatus.RUNNING.getValue(), LoadStatus.RUNNING.getValue(), + LoadStatus.RUNNING.getValue(), LoadStatus.RUNNING.getValue()); + CountDownLatch otherLocked = new CountDownLatch(1); + CountDownLatch pollAtSecond = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference otherFailure = + new java.util.concurrent.atomic.AtomicReference<>(); + Thread other = new Thread(() -> { + try { + new org.springframework.transaction.support.TransactionTemplate(manager).execute(status -> { + jdbc.update("UPDATE tasks SET n=1 WHERE id=10"); + jdbc.update("UPDATE tasks SET n=1 WHERE id=11"); + otherLocked.countDown(); + try { + Assert.assertTrue(pollAtSecond.await(10, TimeUnit.SECONDS)); + Thread.sleep(150); + } catch (InterruptedException failure) { + throw new RuntimeException(failure); + } + jdbc.update("UPDATE tasks SET n=1 WHERE id=9"); + return null; + }); + } catch (Throwable failure) { + otherFailure.set(failure); + } + }, "poll-deadlock-competitor"); + Map owners = new java.util.concurrent.ConcurrentSkipListMap<>(); + for (int id : new int[]{9, 10, 12}) { + owners.put(id, LoadTask.builder().id(id).finished(true).initialStateReadFailed(true) + .status(LoadStatus.FAILED).build()); + } + LoadTask first = owners.get(9); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + Mockito.when(mapper.update(Mockito.nullable(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { + com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper query = invocation.getArgument(1); + query.getSqlSegment(); + int id = (Integer) query.getParamNameValuePairs().values().stream() + .filter(value -> value instanceof Integer).findFirst().get(); + if (id == 10) { + pollAtSecond.countDown(); + } + return jdbc.update("UPDATE tasks SET load_status=? WHERE id=? AND load_status=?", + LoadStatus.FAILED.getValue(), id, LoadStatus.RUNNING.getValue()); + }); + LoadTaskService service = new LoadTaskService(); + set(service, "mapper", mapper); + set(service, "transactionManager", manager); + set(service, "runningTaskContainer", owners); + org.springframework.aop.framework.ProxyFactory factory = + new org.springframework.aop.framework.ProxyFactory(service); + factory.setProxyTargetClass(true); + factory.addAdvice(new org.springframework.transaction.interceptor.TransactionInterceptor( + manager, new org.springframework.transaction.annotation.AnnotationTransactionAttributeSource())); + Throwable pollFailure = null; + other.start(); + try { + Assert.assertTrue(otherLocked.await(5, TimeUnit.SECONDS)); + try { + ((LoadTaskService) factory.getProxy()).updateLoadTaskProgress(); + } catch (Throwable failure) { + pollFailure = failure; + } + } finally { + pollAtSecond.countDown(); + other.join(5000); + } + Assert.assertFalse(other.isAlive()); + Assert.assertNull(otherFailure.get()); + Assert.assertTrue(pollFailure instanceof org.springframework.transaction.TransactionSystemException); + Assert.assertTrue(pollFailure.getCause() instanceof java.sql.SQLException); + Assert.assertEquals(90063, ((java.sql.SQLException) pollFailure.getCause()).getErrorCode()); + Assert.assertEquals(LoadStatus.RUNNING.getValue(), jdbc.queryForObject( + "SELECT load_status FROM tasks WHERE id=9", Byte.class)); + Assert.assertEquals(LoadStatus.RUNNING.getValue(), jdbc.queryForObject( + "SELECT load_status FROM tasks WHERE id=12", Byte.class)); + Assert.assertSame(first, owners.get(9)); + Assert.assertEquals(3, owners.size()); + Mockito.verify(mapper, Mockito.times(2)).update(Mockito.nullable(LoadTask.class), Mockito.any()); + } + } + @Test public void testPollMissingFinalRowDoesNotBlockOtherOwners() throws Exception { this.assertPollFailureIsolation(true); @@ -2151,7 +2242,16 @@ public void testPollRollsBackFailedOwnerWriteAndContinuesOtherOwners() throws Ex this.assertPollFailureIsolation(false); } + @Test + public void testPollConstraintFailureContinuesOtherOwners() throws Exception { + this.assertPollFailureIsolation(false, true); + } + private void assertPollFailureIsolation(boolean missing) throws Exception { + this.assertPollFailureIsolation(missing, false); + } + + private void assertPollFailureIsolation(boolean missing, boolean constraint) throws Exception { try (JobDeleteFixture fixture = new JobDeleteFixture()) { LoadTask first = this.task(this.loader()); first.setId(9); @@ -2167,11 +2267,15 @@ private void assertPollFailureIsolation(boolean missing) throws Exception { if (missing) { fixture.jdbc.update("DELETE FROM delete_tasks WHERE id=9"); } + if (constraint) { + fixture.jdbc.execute("ALTER TABLE delete_tasks ADD CONSTRAINT reject_first_failed CHECK " + + "(id <> 9 OR load_status <> " + LoadStatus.FAILED.getValue() + ")"); + } Mockito.when(fixture.tasks.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { LoadTask task = invocation.getArgument(0); int count = fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=?", task.getStatus().getValue(), task.getId()); - if (!missing && task == first) { + if (!missing && !constraint && task == first) { throw new IllegalStateException("failed after partial SQL"); } return count; From e87d0433701368da1bc2b7aa26ddcfe464808c80 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 14:14:55 +0800 Subject: [PATCH 06/14] fix(hubble): handle completion during deletion - use deletion-specific stopping while keeping stop requests strict - preserve completed states and wait for existing owner cleanup - cover H2 completion races, closure refusal and completion waits - document deletion of tasks that finish after enumeration --- hugegraph-hubble/README.md | 4 +- .../service/load/JobManagerService.java | 2 +- .../service/load/LoadTaskService.java | 32 +++-- .../IngestTransactionIntegrationTest.java | 116 ++++++++++++++++++ 4 files changed, 144 insertions(+), 10 deletions(-) diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index d0acca059..9ad2949dd 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -70,7 +70,9 @@ closed resources under admission. The Job task create/start endpoints and Job management deletion share Job admission, so deletion enumerates tasks started before it and later starts recheck that the Job still exists. Job deletion commits physical stops before atomically deleting task and job metadata; a metadata rollback leaves those -tasks stopped. The Ingest task deletion route uses this same scoped Job deletion. +tasks stopped. Tasks that finish after enumeration keep their terminal state; +any remaining owner must finish and close before metadata can be removed. The Ingest +task deletion route uses this same scoped Job deletion. Graph deletion keeps its remote-first order: if the Server refuses the drop, local Job/task metadata and owners are preserved. After a successful drop, Hubble enumerates that graph's existing Jobs and applies the same deletion to each one, diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java index 8ea6f27f2..1dba8586e 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java @@ -295,7 +295,7 @@ public void deleteJob(int id) { for (LoadTask loadTask : loadTasks) { if (loadTask.getStatus().inRunning() || loadTask.getStatus() == LoadStatus.PAUSED) { - this.taskService.stop(loadTask.getId()); + this.taskService.stopForDeletion(loadTask.getId()); } } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java index 91fc1ecc2..b952d5804 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java @@ -564,16 +564,24 @@ public LoadTask resume(int taskId, String token) { } public LoadTask stop(int taskId) { + return this.stop(taskId, false); + } + + LoadTask stopForDeletion(int taskId) { + return this.stop(taskId, true); + } + + private LoadTask stop(int taskId, boolean deleting) { Lock control = this.taskControls.get(taskId); control.lock(); try { - return this.stopControlled(taskId); + return this.stopControlled(taskId, deleting); } finally { control.unlock(); } } - private LoadTask stopControlled(int taskId) { + private LoadTask stopControlled(int taskId, boolean deleting) { LoadTask task = this.runningTaskContainer.get(taskId); boolean loading = task != null; LoadStatus readStatus = null; @@ -594,13 +602,21 @@ private LoadTask stopControlled(int taskId) { status = task.getStatus(); Ex.check(!loading || readStatus == null || status == readStatus, "The load task state has changed; retry the request"); - Ex.check(status == LoadStatus.RUNNING || status == LoadStatus.PAUSED, - "Can only stop the RUNNING or PAUSED task"); - if (loading) { - LoadTask owner = this.runningTaskContainer.putIfAbsent(taskId, task); - Ex.check(owner == null || owner == task, "The load task has already restarted"); + if (deleting && (status == LoadStatus.SUCCEED || status == LoadStatus.FAILED || + status == LoadStatus.STOPPED)) { + // Completion may race the deletion snapshot. Keep its terminal state and drain any owner below. + if (!loading) { + return task; + } + } else { + Ex.check(status == LoadStatus.RUNNING || status == LoadStatus.PAUSED, + "Can only stop the RUNNING or PAUSED task"); + if (loading) { + LoadTask owner = this.runningTaskContainer.putIfAbsent(taskId, task); + Ex.check(owner == null || owner == task, "The load task has already restarted"); + } + task.setStatus(LoadStatus.STOPPED); } - task.setStatus(LoadStatus.STOPPED); } finally { task.unlock(); } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java index 7aeecabe8..11ae8eadf 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.Date; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -58,6 +59,7 @@ import org.apache.hugegraph.entity.load.ListFormat; import org.apache.hugegraph.entity.load.LoadParameter; import org.apache.hugegraph.entity.load.LoadTask; +import org.apache.hugegraph.exception.ExternalException; import org.apache.hugegraph.handler.LoadTaskExecutor; import org.apache.hugegraph.loader.HugeGraphLoader; import org.apache.hugegraph.loader.executor.LoadContext; @@ -119,6 +121,7 @@ public void setup() { this.cleanup(); this.executor.reset(); this.taskService.failTaskInsert(false); + this.taskService.afterTaskSnapshot(null); Mockito.when(this.config.get(HubbleOptions.UPLOAD_FILE_LOCATION)) .thenReturn(UPLOAD_ROOT); } @@ -337,6 +340,103 @@ public void testRejectedDispatchCommitsFailedTaskAndJobWithoutThrowing() { this.jobService.get(task.getJobId()).getJobStatus()); } + @Test + public void testDeleteJobAcceptsSuccessfulTaskAfterSnapshot() { + this.deleteJobAfterTaskCompletion(LoadStatus.SUCCEED); + } + + @Test + public void testDeleteJobAcceptsFailedTaskAfterSnapshot() { + this.deleteJobAfterTaskCompletion(LoadStatus.FAILED); + } + + @Test + public void testStopStillRejectsCompletedTasks() { + LoadTask task = this.jobService.createIngestTask( + this.job(), this.mapping(), this.connection(), this.client()); + this.taskService.clearOwners(); + task.setFinished(true); + for (LoadStatus status : new LoadStatus[]{LoadStatus.SUCCEED, LoadStatus.FAILED, LoadStatus.STOPPED}) { + task.setStatus(status); + this.taskService.update(task); + Assert.assertThrows(ExternalException.class, + () -> this.taskService.stop(task.getId())); + Assert.assertEquals(status, this.taskService.get(task.getId()).getStatus()); + } + } + + @Test + public void testDeleteJobRejectsCompletedOwnerThatHasNotClosed() { + LoadTask task = this.jobService.createIngestTask( + this.job(), this.mapping(), this.connection(), this.client()); + this.taskService.afterTaskSnapshot(() -> { + task.setStatus(LoadStatus.SUCCEED); + task.setFinished(true); + this.taskService.update(task); + }); + + Assert.assertThrows(ExternalException.class, + () -> this.jobService.deleteJob("DEFAULT", "hugegraph", task.getJobId())); + + this.assertTableCounts(1, 1, 1); + Assert.assertEquals(LoadStatus.SUCCEED, this.taskService.get(task.getId()).getStatus()); + Assert.assertFalse(task.context().closed()); + } + + @Test + public void testDeleteJobWaitsForCompletedOwnerToFinish() throws Exception { + FileMapping mapping = this.mapping(); + mapping.setPath(UPLOAD_ROOT + "/snapshot-completion/" + mapping.getName()); + LoadTask task = this.jobService.createIngestTask( + this.job(), mapping, this.connection(), this.client()); + CountDownLatch stopping = new CountDownLatch(1); + LoadContext context = task.context(); + Mockito.doAnswer(invocation -> { + stopping.countDown(); + return null; + }).when(context).stopLoading(); + this.taskService.afterTaskSnapshot(() -> { + task.setStatus(LoadStatus.SUCCEED); + this.taskService.update(task); + task.getLoader().shutdown(); + }); + ExecutorService pool = Executors.newSingleThreadExecutor(); + try { + Future deleting = pool.submit(() -> + this.jobService.deleteJob("DEFAULT", "hugegraph", task.getJobId())); + Assert.assertTrue(stopping.await(5L, TimeUnit.SECONDS)); + Assert.assertFalse(deleting.isDone()); + this.assertTableCounts(1, 1, 1); + Assert.assertEquals(LoadStatus.SUCCEED, this.taskService.get(task.getId()).getStatus()); + + task.setFinished(true); + deleting.get(5L, TimeUnit.SECONDS); + + this.assertTableCounts(0, 0, 0); + } finally { + task.setFinished(true); + pool.shutdown(); + Assert.assertTrue(pool.awaitTermination(5L, TimeUnit.SECONDS)); + } + } + + private void deleteJobAfterTaskCompletion(LoadStatus status) { + FileMapping mapping = this.mapping(); + mapping.setPath(UPLOAD_ROOT + "/snapshot-completion/" + mapping.getName()); + LoadTask task = this.jobService.createIngestTask( + this.job(), mapping, this.connection(), this.client()); + this.taskService.afterTaskSnapshot(() -> { + task.setStatus(status); + task.setFinished(true); + this.taskService.update(task); + this.taskService.clearOwners(); + }); + + this.jobService.deleteJob("DEFAULT", "hugegraph", task.getJobId()); + + this.assertTableCounts(0, 0, 0); + } + private JobManager job() { Date now = new Date(); return JobManager.builder() @@ -485,6 +585,22 @@ public HugeConfig hugeConfig() { public static class TestLoadTaskService extends LoadTaskService { private boolean failTaskInsert; + private Runnable afterTaskSnapshot; + + public void afterTaskSnapshot(Runnable action) { + this.afterTaskSnapshot = action; + } + + @Override + public List taskListByJob(int jobId) { + List tasks = super.taskListByJob(jobId); + Runnable action = this.afterTaskSnapshot; + this.afterTaskSnapshot = null; + if (action != null) { + action.run(); + } + return tasks; + } @SuppressWarnings("unchecked") public void clearOwners() { From a5243081eb02ac3b59578df2d668a2400a571e7d Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 17:48:59 +0800 Subject: [PATCH 07/14] fix(hubble): close loaders on task exit - close task and example loaders after loading - close unused loaders on save, dispatch or transaction failure - remove owned shutdown hooks and preserve unused checkpoints - defer metadata and concurrency recovery to separate work --- hugegraph-hubble/README.md | 75 +- .../controller/ingest/IngestController.java | 2 +- .../controller/load/LoadTaskController.java | 87 +- .../hugegraph/entity/load/LoadTask.java | 36 +- .../hugegraph/handler/LoadTaskExecutor.java | 5 +- .../service/load/JobManagerService.java | 71 +- .../service/load/LoadTaskService.java | 514 +-- .../ingest/IngestControllerTest.java | 54 - .../IngestTransactionIntegrationTest.java | 216 +- .../unit/GraphsControllerCanonicalTest.java | 63 - .../hugegraph/unit/LoadTaskCleanupTest.java | 2945 ----------------- .../hugegraph/unit/LoadTaskServiceTest.java | 61 +- .../apache/hugegraph/unit/UnitTestSuite.java | 1 - hugegraph-loader/README.md | 23 +- .../hugegraph/loader/HugeGraphLoader.java | 93 +- .../loader/executor/LoadContext.java | 4 - .../loader/progress/InputItemProgress.java | 10 +- .../loader/progress/InputProgress.java | 10 - .../loader/progress/LoadProgress.java | 6 +- .../loader/reader/AbstractReader.java | 3 +- .../loader/reader/file/FileReader.java | 4 +- .../loader/test/unit/LoadContextTest.java | 240 +- .../loader/test/unit/TaskManagerTest.java | 15 - 23 files changed, 381 insertions(+), 4157 deletions(-) delete mode 100644 hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index 9ad2949dd..727cbb129 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -49,76 +49,11 @@ The compatibility boundary is deliberately small: Use matching Server, PD, and Store minor versions in a distributed cluster. **Server/PD 1.8 or later is strongly recommended for the best Hubble experience.** Support for 1.5 and 1.7 is intentionally limited to minimum usability and does not imply feature parity with the current release. Version checks stay in the client adapter/resolver rather than being scattered through controllers or pages. See [`AGENTS.md`](AGENTS.md) for verification rules. -## Import task lifecycle - -Import tasks finish their Loader shutdown before reporting completion. Shutdown -also persists progress and closes the task's clients and worker pools; a shutdown -failure marks the task as failed, including after a pause or stop request. The -synchronous Loader example also shuts down before returning. A task that cannot -be saved, whose transaction rolls back, or whose async dispatch is rejected closes -its Loader without starting a run. Pause and stop wait for the running Loader and -its final database callback. Within one Hubble service, pause, stop, resume, retry -and initial task publication are serialized for a task, including the database -read and final stop persistence; other requests for that task wait for admission. -The bounded weak lock stripes can also serialize unrelated task IDs that collide. -Retry and resume reject an owner whose workers have not closed. An accepted -pause or stop also prevents delayed initial dispatch after commit; the unused -Loader closes without overwriting the accepted status. A delayed callback also -closes only its unused Loader if the task was deleted, finished, or replaced; it -does not change the newer task or job state. Task deletion rechecks state and -closed resources under admission. The Job task create/start endpoints and Job -management deletion share Job admission, so deletion enumerates tasks started before it and later -starts recheck that the Job still exists. Job deletion commits physical stops before -atomically deleting task and job metadata; a metadata rollback leaves those -tasks stopped. Tasks that finish after enumeration keep their terminal state; -any remaining owner must finish and close before metadata can be removed. The Ingest -task deletion route uses this same scoped Job deletion. -Graph deletion keeps its remote-first order: if the Server refuses the drop, local -Job/task metadata and owners are preserved. After a successful drop, Hubble -enumerates that graph's existing Jobs and applies the same deletion to each one, -waiting for its Loaders to close before deleting its metadata. A cleanup failure -leaves that Job and its task rows recoverable; earlier Jobs have already committed -their cleanup, and the remote graph drop cannot be rolled back. This enumeration -does not prevent new Jobs from being created, so Jobs outside the snapshot are -preserved. It provides no graph-wide admission barrier. - -If delayed initial publication cannot read task metadata, it closes -its unused Loader and retains that closed owner for the existing progress poll. -The poll marks only a still-RUNNING row as failed and retires the owner after commit; -a missing or advanced row is preserved. A rejected dispatch whose failure cannot -be committed also retains its closed owner. The poll retries the conditional -Task-to-FAILED and Job-to-FAILED changes in one transaction, preserving a task row -that has since advanced or disappeared. Each owner's polling write uses an H2 -savepoint: an ordinary SQL failure whose savepoint rollback succeeds leaves that -owner for a later poll while other owners can finish persistence. A transaction or -savepoint operation failure aborts the whole polling transaction and retains its -owners, including those processed earlier. Owners are retired only -after the polling transaction commits; a deleted final task row retires its closed -owner without recreating metadata. -Finished tasks release their native JVM shutdown hook once their context closes. -If shutdown cannot terminate the task's workers, Hubble retains the failed owner -and rejects retry/resume until its context is actually closed; restart Hubble to -finish recovery when cleanup remains incomplete. This cleanup applies when load -exits and reaches shutdown; an ongoing backend call can still block existing -Loader waits and controls queued on the same stripe. The existing progress poll retries final metadata writes for -finished, closed owners and releases an owner only after its final metadata -transaction commits. Closing an owner before load starts preserves its old -checkpoint. A pause/stop using separately restored task state, or a resume/retry, -rejects a conflicting persisted status change; read the current state before -retrying the request. Incremental pause/resume also keeps offsets for inputs not yet read -and already completed counts. This cleanup keeps the existing graph-mode -behavior and does not coordinate concurrent imports from separate processes or -Server instances. - -The Loader [startup mode ownership guard](https://github.com/apache/hugegraph-toolchain/pull/812) -is a prerequisite: without it, cleanup after startup fails can reset another -importer's `LOADING` mode. Multi-file jobs start a Loader for each file, so shared -graph-mode ownership also requires the matching Server, Client and Loader ownership -protocol 1 support. The paired [Server change](https://github.com/apache/hugegraph/pull/3331) -and [website documentation](https://github.com/apache/hugegraph-doc/pull/525) describe -that protocol; the Client and Loader must use it too. The startup guard alone does -not keep `LOADING` until every sibling Loader finishes. This lifecycle cleanup does -not implement that protocol, and multi-file validation requires their combined build. +## Import resource cleanup + +Import tasks close their Loader before publishing completion. The built-in example import also closes its Loader; +tasks that cannot be saved or dispatched, or whose transaction rolls back, close their unused Loader. +Shutdown failures are reported as task failures. ## Local development feedback loop diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java index 37735dc11..3fadd9af0 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java @@ -378,7 +378,7 @@ public Response deleteTask(@PathVariable("id") int id) { .message("Task not found: " + id).build(); } this.requireJobWrite(job); - jobManagerService.deleteJob(job.getGraphSpace(), job.getGraph(), id); + jobManagerService.remove(id); return Response.builder().status(Constant.STATUS_OK).build(); } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/LoadTaskController.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/LoadTaskController.java index 1602339a9..5b45e7800 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/LoadTaskController.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/LoadTaskController.java @@ -50,7 +50,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.concurrent.locks.Lock; @Log4j2 @RestController @@ -115,25 +114,19 @@ public LoadTask create(@PathVariable("graphspace") String graphSpace, @PathVariable("jobId") int jobId, @RequestBody LoadTask entity) { this.requireGraphSpaceWrite(graphSpace); - Lock control = this.jobService.jobControl(jobId); - control.lock(); - try { - JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); - Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); - Ex.check(jobEntity.getJobStatus() == JobStatus.SETTING, - "load.task.create.no-permission"); - synchronized (this.service) { - Ex.check(this.service.count() < LIMIT, - "load.task.reached-limit", LIMIT); - entity.setGraphSpace(graphSpace); - entity.setGraph(graph); - entity.setJobId(jobId); - this.service.save(entity); - } - return entity; - } finally { - control.unlock(); + JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); + Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); + Ex.check(jobEntity.getJobStatus() == JobStatus.SETTING, + "load.task.create.no-permission"); + synchronized (this.service) { + Ex.check(this.service.count() < LIMIT, + "load.task.reached-limit", LIMIT); + entity.setGraphSpace(graphSpace); + entity.setGraph(graph); + entity.setJobId(jobId); + this.service.save(entity); } + return entity; } @DeleteMapping("{id}") @@ -178,41 +171,35 @@ public List start(@PathVariable("graphspace") String graphSpace, connection.setPort(host.getPort()); } - Lock control = this.jobService.jobControl(jobId); - control.lock(); + JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); + Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); + Ex.check(jobEntity.getJobStatus() == JobStatus.SETTING, + "load.task.start.no-permission"); + boolean existError = false; try { - JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); - Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); - Ex.check(jobEntity.getJobStatus() == JobStatus.SETTING, - "load.task.start.no-permission"); - boolean existError = false; - try { - List tasks = new ArrayList<>(); - HugeClient client = this.authClient(graphSpace, graph); - for (Integer fileId : fileIds) { - FileMapping fileMapping = this.fmService.get(graphSpace, graph, - jobId, fileId); - if (fileMapping == null) { - throw new ExternalException("file-mapping.not-exist.id", - fileId); - } - tasks.add(this.service.start(connection, fileMapping, client)); - } - return tasks; - } catch (Exception e) { - existError = true; - throw e; - } finally { - if (existError) { - jobEntity.setJobStatus(JobStatus.FAILED); - } else { - jobEntity.setJobStatus(JobStatus.LOADING); + List tasks = new ArrayList<>(); + HugeClient client = this.authClient(graphSpace, graph); + for (Integer fileId : fileIds) { + FileMapping fileMapping = this.fmService.get(graphSpace, graph, + jobId, fileId); + if (fileMapping == null) { + throw new ExternalException("file-mapping.not-exist.id", + fileId); } - jobEntity.setUpdateTime(HubbleUtil.nowDate()); - this.jobService.update(jobEntity); + tasks.add(this.service.start(connection, fileMapping, client)); } + return tasks; + } catch (Exception e) { + existError = true; + throw e; } finally { - control.unlock(); + if (existError) { + jobEntity.setJobStatus(JobStatus.FAILED); + } else { + jobEntity.setJobStatus(JobStatus.LOADING); + } + jobEntity.setUpdateTime(HubbleUtil.nowDate()); + this.jobService.update(jobEntity); } } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java index 37bd2f773..c33ce30b7 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java @@ -68,14 +68,6 @@ public class LoadTask implements Runnable { @JsonIgnore private transient volatile boolean finished; - @TableField(exist = false) - @JsonIgnore - private transient volatile boolean initialStateReadFailed; - - @TableField(exist = false) - @JsonIgnore - private transient volatile boolean dispatchFailurePending; - @TableId(type = IdType.AUTO) @MergeProperty(useNew = false) @JsonProperty("id") @@ -182,25 +174,6 @@ public LoadTask(LoadOptions options, GraphConnection connection, @Override public void run() { - this.run(() -> { }); - } - - public void run(Runnable callback) { - try { - this.runLoader(); - } finally { - this.lock.lock(); - try { - callback.run(); - } finally { - // Stop/retry must not race a callback from the previous run. - this.finished = true; - this.lock.unlock(); - } - } - } - - private void runLoader() { Ex.check(this.options != null, "The load options shouldn't be null"); log.info("LoadTask is start running : {}", this.id); @@ -212,7 +185,6 @@ private void runLoader() { } catch (Throwable e) { failure = e; } finally { - // Only the running thread closes this loader, after load has stopped. try { this.loader.shutdown(); } catch (Throwable e) { @@ -248,6 +220,7 @@ private void runLoader() { this.lastDuration += this.context().summary().totalTime(); this.currDuration = 0L; } finally { + this.finished = true; this.lock.unlock(); } } @@ -261,10 +234,7 @@ public void unlock() { } public void stop() { - HugeGraphLoader owner = this.loader; - if (owner != null) { - owner.context().stopLoading(); - } + this.context().stopLoading(); while (!this.finished) { try { Thread.sleep(1000); @@ -272,7 +242,7 @@ public void stop() { // pass } } - if (owner == null || owner.context().closed()) { + if (this.context().closed()) { this.loader = null; } log.info("LoadTask {} stopped", this.id); diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/LoadTaskExecutor.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/LoadTaskExecutor.java index 402b14bd9..07e7ca164 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/LoadTaskExecutor.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/handler/LoadTaskExecutor.java @@ -32,9 +32,12 @@ public class LoadTaskExecutor { public void execute(LoadTask task, Runnable callback) { log.info("Executing task: {}", task.getId()); try { - task.run(callback); + task.run(); } catch (Throwable t) { log.warn("Executing task: {} error", task.getId(), t); + } finally { + log.info("Executed task: {}, update status to db", task.getId()); + callback.run(); } } } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java index 1dba8586e..deb05a8f7 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java @@ -23,13 +23,12 @@ import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.concurrent.locks.Lock; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.google.common.util.concurrent.Striped; +import com.google.common.collect.ImmutableMap; import lombok.extern.log4j.Log4j2; import org.apache.hugegraph.driver.HugeClient; @@ -47,14 +46,10 @@ import org.apache.hugegraph.util.PageUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.annotation.Isolation; -import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionSynchronization; import org.springframework.transaction.support.TransactionSynchronizationManager; -import org.springframework.transaction.support.TransactionTemplate; @Log4j2 @Service @@ -66,14 +61,6 @@ public class JobManagerService { private LoadTaskService taskService; @Autowired private FileMappingService fileMappingService; - @Autowired - private PlatformTransactionManager transactionManager; - - private final Striped jobControls = Striped.lazyWeakLock(1024); - - public Lock jobControl(int id) { - return this.jobControls.get(id); - } public int count() { return Math.toIntExact(this.mapper.selectCount(null)); @@ -271,52 +258,34 @@ public void remove(int id) { } } - @Transactional(propagation = Propagation.NOT_SUPPORTED) + @Transactional(isolation = Isolation.READ_COMMITTED) public void removeByGraph(String graphSpace, String graph) { - QueryWrapper query = Wrappers.query(); - query.eq("graphspace", graphSpace).eq("graph", graph); - // Clean only the enlisted Jobs; newly created Jobs must not lose their task owners. - for (JobManager job : this.mapper.selectList(query)) { - this.deleteJob(graphSpace, graph, job.getId()); - } + this.mapper.deleteByMap(ImmutableMap.of("graphspace", graphSpace, + "graph", graph)); } - @Transactional(propagation = Propagation.NOT_SUPPORTED) + @Transactional(isolation = Isolation.READ_COMMITTED) public void deleteJob(int id) { - Lock control = this.jobControl(id); - control.lock(); - try { - JobManager job = this.get(id); - if (job == null) { - throw new ExternalException("job.manager.not-exist.id", id); - } + JobManager job = this.get(id); + if (job == null) { + throw new ExternalException("job.manager.not-exist.id", id); + } - List loadTasks = this.taskService.taskListByJob(id); - for (LoadTask loadTask : loadTasks) { - if (loadTask.getStatus().inRunning() || - loadTask.getStatus() == LoadStatus.PAUSED) { - this.taskService.stopForDeletion(loadTask.getId()); - } + List loadTasks = this.taskService.taskListByJob(id); + for (LoadTask loadTask : loadTasks) { + if (loadTask.getStatus().inRunning() || + loadTask.getStatus() == LoadStatus.PAUSED) { + this.taskService.stop(loadTask.getId()); } - - // Physical stops cannot be rolled back. Commit them before the metadata transaction. - TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); - transaction.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED); - transaction.execute(status -> { - for (LoadTask loadTask : loadTasks) { - this.taskService.remove(loadTask.getId()); - } - List mappings = this.fileMappingService.listByJob(id); - this.remove(id); - this.deleteDiskFilesAfterCommit(mappings); - return null; - }); - } finally { - control.unlock(); + this.taskService.remove(loadTask.getId()); } + + List mappings = this.fileMappingService.listByJob(id); + this.remove(id); + this.deleteDiskFilesAfterCommit(mappings); } - @Transactional(propagation = Propagation.NOT_SUPPORTED) + @Transactional(isolation = Isolation.READ_COMMITTED) public void deleteJob(String graphSpace, String graph, int id) { JobManager job = this.get(graphSpace, graph, id); if (job == null) { diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java index b952d5804..366c5074a 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java @@ -23,7 +23,6 @@ import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.Striped; import lombok.extern.log4j.Log4j2; import org.apache.commons.io.FileUtils; import org.apache.hugegraph.common.Constant; @@ -67,7 +66,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionDefinition; -import org.springframework.transaction.TransactionException; import org.springframework.transaction.annotation.Isolation; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionSynchronization; @@ -86,7 +84,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.Lock; import org.apache.commons.lang3.StringUtils; @@ -115,8 +112,6 @@ public class LoadTaskService { private Map runningTaskContainer; - // Request admission precedes task locks. Workers and progress polling never take this lock. - private final Striped taskControls = Striped.lazyWeakLock(1024); public LoadTaskService() { this.runningTaskContainer = new ConcurrentHashMap<>(); @@ -192,22 +187,11 @@ public void save(LoadTask entity) { @Transactional(isolation = Isolation.READ_COMMITTED) public void update(LoadTask entity) { - this.update(entity, null); - } - - private void update(LoadTask entity, LoadStatus previousStatus) { LoadOptions runtimeOptions = entity.getOptions(); entity.setOptions(withoutCredentials(runtimeOptions)); try { - if (previousStatus == null) { - if (this.mapper.updateById(entity) != 1) { - throw new InternalException("entity.update.failed"); - } - } else { - int updated = this.mapper.update(entity, Wrappers.update() - .eq("id", entity.getId()) - .eq("load_status", previousStatus.getValue())); - Ex.check(updated == 1, "The load task state has changed; retry the request"); + if (this.mapper.updateById(entity) != 1) { + throw new InternalException("entity.update.failed"); } } finally { entity.setOptions(runtimeOptions); @@ -233,33 +217,9 @@ private static LoadOptions withoutCredentials(LoadOptions options) { @Transactional(isolation = Isolation.READ_COMMITTED) public void remove(int id) { - Lock control = this.taskControls.get(id); - control.lock(); - try { - LoadTask persisted = this.get(id); - Ex.check(persisted != null, "load.task.not-exist.id", id); - Ex.check(!persisted.getStatus().inRunning(), "load.task.delete.task-in-running", id); - LoadTask owner = this.runningTaskContainer.get(id); - if (owner != null) { - owner.lock(); - } - try { - Ex.check(owner == null || (owner.isFinished() && - (owner.getLoader() == null || owner.context().closed())), - "load.task.delete.task-in-running", id); - int deleted = this.mapper.delete(Wrappers.query().eq("id", id) - .eq("load_status", persisted.getStatus().getValue())); - Ex.check(deleted == 1, "The load task state has changed; retry the request"); - if (owner != null) { - this.removeOwnerAfterCommit(owner); - } - } finally { - if (owner != null) { - owner.unlock(); - } - } - } finally { - control.unlock(); + this.runningTaskContainer.remove(id); + if (this.mapper.deleteById(id) != 1) { + throw new InternalException("entity.delete.failed", id); } } @@ -278,7 +238,6 @@ public LoadTask start(GraphConnection connection, FileMapping fileMapping, return task; } catch (RuntimeException | Error failure) { closeUnscheduled(task, failure); - this.removeClosedOwner(task); throw failure; } } @@ -305,378 +264,176 @@ public void afterCompletion(int status) { } private void executeSafely(LoadTask task) { - Lock control = this.taskControls.get(task.getId()); - control.lock(); try { - this.executeControlled(task); - } finally { - control.unlock(); + this.execute(task); + } catch (RuntimeException e) { + closeUnscheduled(task, e); + this.markDispatchFailed(task, e); } } - private void executeControlled(LoadTask task) { - LoadTask current = this.runningTaskContainer.get(task.getId()); - if (current == task) { - // The same owner was already scheduled; do not run or close it twice. - return; - } - if (current != null) { - this.cancelInitial(task, LoadStatus.FAILED, null); - return; - } - LoadTask persisted; - try { - persisted = this.get(task.getId()); - } catch (RuntimeException failure) { - log.error("Failed to read metadata for initial load task {}", task.getId(), failure); - task.setInitialStateReadFailed(true); - this.cancelInitial(task, LoadStatus.FAILED, failure); - // Keep even a closed owner until the existing poll can conditionally persist this failure. - this.runningTaskContainer.putIfAbsent(task.getId(), task); - return; - } - HugeGraphLoader loader = task.getLoader(); - if (persisted == null || persisted.getStatus() != LoadStatus.RUNNING || current != null || - task.isFinished() || loader == null || loader.context().closed() || loader.context().stopped()) { - LoadStatus status = persisted == null || persisted.getStatus() == LoadStatus.RUNNING ? - LoadStatus.FAILED : persisted.getStatus(); - this.cancelInitial(task, status, null); - return; - } - if (this.runningTaskContainer.putIfAbsent(task.getId(), task) != null) { - this.cancelInitial(task, LoadStatus.FAILED, null); - return; - } + private static void closeUnscheduled(LoadTask task, Throwable failure) { try { - this.dispatch(task); - } catch (RuntimeException failure) { - this.markDispatchFailed(task, failure); - } - } - - private void cancelInitial(LoadTask task, LoadStatus status, Throwable failure) { - // A delayed callback owns only its unused Loader, never a newer task's metadata. - closeUnscheduled(task, failure, status); - HugeGraphLoader owner = task.getLoader(); - if (owner != null && !owner.context().closed()) { - task.setStatus(LoadStatus.FAILED); - this.runningTaskContainer.putIfAbsent(task.getId(), task); + if (task.getLoader() != null) { + task.getLoader().shutdown(); + } + } catch (Throwable cleanup) { + if (failure != null && failure != cleanup) { + failure.addSuppressed(cleanup); + } + log.error("Failed to close unscheduled load task {}", task.getId(), cleanup); + } finally { + task.setFinished(true); } } private void markDispatchFailed(LoadTask task, RuntimeException cause) { - if (this.runningTaskContainer.get(task.getId()) != task) { - this.cancelInitial(task, LoadStatus.FAILED, cause); - return; - } - log.error("Failed to dispatch load task {}, marking task and job as FAILED", task.getId(), cause); - task.setDispatchFailurePending(true); - closeUnscheduled(task, cause); + log.error("Failed to dispatch load task {}, marking task and job " + + "as FAILED", task.getId(), cause); + task.setStatus(LoadStatus.FAILED); + this.runningTaskContainer.remove(task.getId()); + try { - TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); - transaction.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + TransactionTemplate transaction = + new TransactionTemplate(this.transactionManager); + transaction.setPropagationBehavior( + TransactionDefinition.PROPAGATION_REQUIRES_NEW); transaction.execute(status -> { - if (this.runningTaskContainer.get(task.getId()) != task) { + int updated = this.mapper.update( + null, Wrappers.update() + .eq("id", task.getId()) + .set("load_status", + LoadStatus.FAILED.getValue())); + if (updated != 1) { + throw new InternalException("entity.update.failed", task); + } + if (task.getJobId() == null) { + return null; + } + JobManager job = this.jobManagerMapper.selectById( + task.getJobId()); + if (job == null) { return null; } - this.persistDispatchFailure(task); - return null; - }); - this.removeClosedOwner(task); - } catch (RuntimeException failure) { - // The ingest transaction is committed. Do not report a retryable dispatch failure. - log.error("Failed to persist dispatch failure for load task {}", task.getId(), failure); - } - } - - private void persistDispatchFailure(LoadTask task) { - int updated = this.mapper.update(null, Wrappers.update() - .eq("id", task.getId()) - .eq("load_status", LoadStatus.RUNNING.getValue()) - .set("load_status", LoadStatus.FAILED.getValue())); - if (updated == 0) { - return; - } - if (updated != 1) { - throw new InternalException("entity.update.failed", task); - } - if (task.getJobId() != null) { - JobManager job = this.jobManagerMapper.selectById(task.getJobId()); - if (job != null) { job.setJobStatus(JobStatus.FAILED); job.setUpdateTime(HubbleUtil.nowDate()); if (this.jobManagerMapper.updateById(job) != 1) { throw new InternalException("entity.update.failed", job); } - } - } - } - - private static void closeUnscheduled(LoadTask task, Throwable failure) { - closeUnscheduled(task, failure, LoadStatus.FAILED); - } - - private static void closeUnscheduled(LoadTask task, Throwable failure, LoadStatus status) { - task.lock(); - try { - task.setStatus(status); - if (task.getLoader() != null) { - try { - task.getLoader().shutdown(); - } catch (Throwable cleanup) { - task.setStatus(LoadStatus.FAILED); - if (failure != null && failure != cleanup) { - failure.addSuppressed(cleanup); - } - log.error("Failed to close unscheduled load task {}", task.getId(), cleanup); - } - } - } finally { - // No runner was scheduled to publish completion for this owner. - task.setFinished(true); - task.unlock(); - } - } - - private void dispatch(LoadTask task) { - this.taskExecutor.execute(task, () -> { - this.update(task); - // Pause/stop retain ownership until their caller has also persisted - // its terminal request. A failed callback can be retried there. - if (task.getStatus() != LoadStatus.PAUSED && task.getStatus() != LoadStatus.STOPPED) { - this.removeClosedOwner(task); - } - }); - } - - private void persistStopped(LoadTask task, boolean loading, LoadStatus previousStatus) { - task.lock(); - try { - // The runner already persisted its final state before finishing. - // Never let an old pause/stop callback overwrite a new owner. - if (!loading || this.runningTaskContainer.get(task.getId()) == task) { - this.update(task, loading ? null : previousStatus); - this.removeClosedOwner(task); - } - } finally { - task.unlock(); - } - } - - private void removeClosedOwner(LoadTask task) { - HugeGraphLoader owner = task.getLoader(); - if (task.getId() != null && (owner == null || owner.context().closed())) { - this.removeOwner(task); - } - } - - private void removeOwner(LoadTask expected) { - this.runningTaskContainer.computeIfPresent(expected.getId(), - (id, current) -> current == expected ? null : current); - } - - private void removeOwnerAfterCommit(LoadTask task) { - if (!TransactionSynchronizationManager.isSynchronizationActive()) { - this.removeOwner(task); - return; + return null; + }); + } catch (RuntimeException e) { + // The ingest transaction is already committed. Never turn a + // dispatch failure into a retryable request failure. + log.error("Failed to persist dispatch failure for load task {}", + task.getId(), e); } - TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { - @Override - public void afterCompletion(int status) { - if (status == TransactionSynchronization.STATUS_COMMITTED) { - removeOwner(task); - } - } - }); } - private void releasePreviousOwner(int taskId) { - LoadTask previous = this.runningTaskContainer.get(taskId); - if (previous == null) { - return; - } - Ex.check(previous.isFinished(), "The previous load task is still finishing"); - previous.lock(); - try { - HugeGraphLoader owner = previous.getLoader(); - Ex.check(owner == null || owner.context().closed(), - "The previous Loader has not closed; finish its cleanup or restart Hubble before retrying"); - this.removeOwner(previous); - } finally { - previous.unlock(); - } + private void execute(LoadTask task) { + // Executed in other threads + this.taskExecutor.execute(task, () -> this.update(task)); + // Save current load task + this.runningTaskContainer.put(task.getId(), task); } public LoadTask pause(int taskId) { - Lock control = this.taskControls.get(taskId); - control.lock(); - try { - return this.pauseControlled(taskId); - } finally { - control.unlock(); - } - } - - private LoadTask pauseControlled(int taskId) { /* * The in-memory container is not repopulated on startup, so a task * that was RUNNING before a restart is only visible in the database */ LoadTask task = this.runningTaskContainer.get(taskId); boolean loading = task != null; - LoadStatus readStatus = null; if (!loading) { task = this.get(taskId); - readStatus = task == null ? null : task.getStatus(); - // A restart may have published an owner during the database read. - LoadTask owner = this.runningTaskContainer.get(taskId); - if (owner != null) { - task = owner; - loading = true; - } } Ex.check(task != null, "load.task.not-exist.id", taskId); - task.lock(); - try { - Ex.check(!loading || readStatus == null || task.getStatus() == readStatus, - "The load task state has changed; retry the request"); - Ex.check(task.getStatus() == LoadStatus.RUNNING, - "Can only pause the RUNNING task"); - if (loading) { - LoadTask owner = this.runningTaskContainer.putIfAbsent(taskId, task); - Ex.check(owner == null || owner == task, "The load task has already restarted"); - } - task.setStatus(LoadStatus.PAUSED); - } finally { - task.unlock(); - } + Ex.check(task.getStatus() == LoadStatus.RUNNING, + "Can only pause the RUNNING task"); + // Mark status as paused, should set before context.stopLoading() + task.setStatus(LoadStatus.PAUSED); if (loading) { // Let HugeGraphLoader stop task.stop(); } - this.persistStopped(task, loading, LoadStatus.RUNNING); + task.lock(); + try { + this.update(task); + this.runningTaskContainer.remove(taskId); + } finally { + task.unlock(); + } return task; } public LoadTask resume(int taskId, String token) { - return this.restart(taskId, token, true); - } - - public LoadTask stop(int taskId) { - return this.stop(taskId, false); - } - - LoadTask stopForDeletion(int taskId) { - return this.stop(taskId, true); - } - - private LoadTask stop(int taskId, boolean deleting) { - Lock control = this.taskControls.get(taskId); - control.lock(); + LoadTask task = this.get(taskId); + Ex.check(task.getStatus() == LoadStatus.PAUSED || + task.getStatus() == LoadStatus.FAILED, + "Can only resume the PAUSED or FAILED task"); + task.lock(); try { - return this.stopControlled(taskId, deleting); + // Set work mode in incrental mode, load from last breakpoint + task.getOptions().incrementalMode = true; + task.reconnect(token); + task.setStatus(LoadStatus.RUNNING); + this.update(task); + this.taskExecutor.execute(task, () -> this.update(task)); + this.runningTaskContainer.put(taskId, task); } finally { - control.unlock(); + task.unlock(); } + return task; } - private LoadTask stopControlled(int taskId, boolean deleting) { + public LoadTask stop(int taskId) { LoadTask task = this.runningTaskContainer.get(taskId); boolean loading = task != null; - LoadStatus readStatus = null; if (!loading) { task = this.get(taskId); - readStatus = task == null ? null : task.getStatus(); - // A restart may have published an owner during the database read. - LoadTask owner = this.runningTaskContainer.get(taskId); - if (owner != null) { - task = owner; - loading = true; - } } Ex.check(task != null, "load.task.not-exist.id", taskId); - LoadStatus status; - task.lock(); - try { - status = task.getStatus(); - Ex.check(!loading || readStatus == null || status == readStatus, - "The load task state has changed; retry the request"); - if (deleting && (status == LoadStatus.SUCCEED || status == LoadStatus.FAILED || - status == LoadStatus.STOPPED)) { - // Completion may race the deletion snapshot. Keep its terminal state and drain any owner below. - if (!loading) { - return task; - } - } else { - Ex.check(status == LoadStatus.RUNNING || status == LoadStatus.PAUSED, - "Can only stop the RUNNING or PAUSED task"); - if (loading) { - LoadTask owner = this.runningTaskContainer.putIfAbsent(taskId, task); - Ex.check(owner == null || owner == task, "The load task has already restarted"); - } - task.setStatus(LoadStatus.STOPPED); - } - } finally { - task.unlock(); - } + LoadStatus status = task.getStatus(); + Ex.check(status == LoadStatus.RUNNING || status == LoadStatus.PAUSED, + "Can only stop the RUNNING or PAUSED task"); + // Mark status as stopped + task.setStatus(LoadStatus.STOPPED); /* * Only a task taken from the in-memory container owns a loader, one * restored from the database has nothing running to interrupt, so * just persist the terminal status for it */ - if (loading) { + if (loading && status == LoadStatus.RUNNING) { task.stop(); } - this.persistStopped(task, loading, status); - return task; - } - - public LoadTask retry(int taskId, String token) { - return this.restart(taskId, token, false); - } - - private LoadTask restart(int taskId, String token, boolean incremental) { - Lock control = this.taskControls.get(taskId); - control.lock(); + task.lock(); try { - return this.restartControlled(taskId, token, incremental); + this.update(task); + this.runningTaskContainer.remove(taskId); } finally { - control.unlock(); + task.unlock(); } + return task; } - private LoadTask restartControlled(int taskId, String token, boolean incremental) { + public LoadTask retry(int taskId, String token) { LoadTask task = this.get(taskId); - Ex.check(task != null, "load.task.not-exist.id", taskId); - LoadStatus previousStatus = task.getStatus(); - Ex.check(previousStatus == LoadStatus.FAILED || - previousStatus == (incremental ? LoadStatus.PAUSED : LoadStatus.STOPPED), - incremental ? "Can only resume the PAUSED or FAILED task" : - "Can only retry the FAILED or STOPPED task"); - this.releasePreviousOwner(taskId); + Ex.check(task.getStatus() == LoadStatus.FAILED || + task.getStatus() == LoadStatus.STOPPED, + "Can only retry the FAILED or STOPPED task"); task.lock(); try { - Ex.check(this.runningTaskContainer.putIfAbsent(taskId, task) == null, - "The previous load task is still finishing"); - task.getOptions().incrementalMode = incremental; + // Set work mode in normal mode, load from begin + task.getOptions().incrementalMode = false; task.reconnect(token); task.setStatus(LoadStatus.RUNNING); - if (!incremental) { - task.setLastDuration(0L); - task.setCurrDuration(0L); - } - this.update(task, previousStatus); - try { - this.dispatch(task); - } catch (RuntimeException failure) { - this.markDispatchFailed(task, failure); - } - } catch (RuntimeException | Error failure) { - closeUnscheduled(task, failure); - this.removeClosedOwner(task); - throw failure; + task.setLastDuration(0L); + task.setCurrDuration(0L); + this.update(task); + this.taskExecutor.execute(task, () -> this.update(task)); + this.runningTaskContainer.put(taskId, task); } finally { task.unlock(); } @@ -735,64 +492,25 @@ public void pauseAllTasks() { @Scheduled(fixedDelay = 1 * 1000) @Transactional(isolation = Isolation.READ_COMMITTED) public void updateLoadTaskProgress() { - TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); - transaction.setPropagationBehavior(TransactionDefinition.PROPAGATION_NESTED); for (LoadTask task : this.runningTaskContainer.values()) { + if (!task.getStatus().inRunning()) { + continue; + } task.lock(); try { - transaction.execute(status -> { - this.updateLoadTaskProgress(task); - return null; - }); - } catch (TransactionException failure) { - // A failed savepoint/transaction operation cannot safely commit earlier owner writes. - throw failure; - } catch (RuntimeException failure) { - log.warn("Failed to persist progress for load task {}; retaining its owner", task.getId(), failure); - } finally { - task.unlock(); - } - } - } - - private void updateLoadTaskProgress(LoadTask task) { - if (this.runningTaskContainer.get(task.getId()) != task) { - return; - } - HugeGraphLoader owner = task.getLoader(); - if (task.isFinished() && (owner == null || owner.context().closed())) { - if (task.isDispatchFailurePending()) { - this.persistDispatchFailure(task); - } else if (task.isInitialStateReadFailed()) { - // Unknown initial metadata must not overwrite a later terminal state or recreate a missing row. - int updated = this.mapper.update(null, - Wrappers.update().eq("id", task.getId()) - .eq("load_status", LoadStatus.RUNNING.getValue()) - .set("load_status", LoadStatus.FAILED.getValue())); - if (updated != 0 && updated != 1) { - throw new InternalException("entity.update.failed", task); - } - } else { - try { - // Retry a final write that failed in the runner or pause/stop. - this.update(task); - } catch (RuntimeException failure) { - if (this.get(task.getId()) != null) { - throw failure; + if (task.getStatus().inRunning()) { + LoadContext context = task.context(); + long readLines = context.newProgress().totalInputRead(); + if (readLines == 0L) { + readLines = context.oldProgress().totalInputRead(); } - // A deleted row has no final state to persist; retire only this closed owner. + task.setFileReadLines(readLines); + task.setCurrDuration(context.summary().totalTime()); + this.update(task); } + } finally { + task.unlock(); } - this.removeOwnerAfterCommit(task); - } else if (task.getStatus().inRunning()) { - LoadContext context = task.context(); - long readLines = context.newProgress().totalInputRead(); - if (readLines == 0L) { - readLines = context.oldProgress().totalInputRead(); - } - task.setFileReadLines(readLines); - task.setCurrDuration(context.summary().totalTime()); - this.update(task); } } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/controller/ingest/IngestControllerTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/controller/ingest/IngestControllerTest.java index a668186eb..d3f765bfd 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/controller/ingest/IngestControllerTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/controller/ingest/IngestControllerTest.java @@ -61,56 +61,6 @@ public class IngestControllerTest { - @Test - public void testDeleteTaskUsesScopedJobLifecycleAfterPermissionCheck() - throws Exception { - TestIngestController controller = new TestIngestController(); - JobManagerService jobs = Mockito.mock(JobManagerService.class); - this.setField(controller, "jobManagerService", jobs); - Mockito.when(jobs.get(7)).thenReturn(JobManager.builder().id(7) - .graphSpace("space-a").graph("graph-a").build()); - Mockito.doAnswer(invocation -> { - Assert.assertEquals("space-a", controller.writeGraphSpace); - return null; - }).when(jobs).deleteJob("space-a", "graph-a", 7); - - Assert.assertEquals(Constant.STATUS_OK, controller.deleteTask(7).getStatus()); - - Mockito.verify(jobs).deleteJob("space-a", "graph-a", 7); - Mockito.verify(jobs, Mockito.never()).remove(Mockito.anyInt()); - } - - @Test - public void testDeleteTaskKeepsMissingAndDeniedResponses() throws Exception { - TestIngestController controller = new TestIngestController(); - JobManagerService jobs = Mockito.mock(JobManagerService.class); - this.setField(controller, "jobManagerService", jobs); - Assert.assertEquals(Constant.STATUS_NOT_FOUND, controller.deleteTask(7).getStatus()); - Assert.assertNull(controller.writeGraphSpace); - - Mockito.when(jobs.get(7)).thenReturn(JobManager.builder().id(7) - .graphSpace("space-a").graph("graph-a").build()); - controller.denyWrite = true; - Assert.assertThrows(org.apache.hugegraph.exception.ForbiddenException.class, - () -> controller.deleteTask(7)); - Mockito.verify(jobs, Mockito.never()).deleteJob(Mockito.anyString(), Mockito.anyString(), Mockito.anyInt()); - Mockito.verify(jobs, Mockito.never()).remove(Mockito.anyInt()); - } - - @Test - public void testDeleteTaskPropagatesLifecycleRefusal() throws Exception { - TestIngestController controller = new TestIngestController(); - JobManagerService jobs = Mockito.mock(JobManagerService.class); - this.setField(controller, "jobManagerService", jobs); - Mockito.when(jobs.get(7)).thenReturn(JobManager.builder().id(7) - .graphSpace("space-a").graph("graph-a").build()); - RuntimeException refusal = new IllegalStateException("owner is not closed"); - Mockito.doThrow(refusal).when(jobs).deleteJob("space-a", "graph-a", 7); - - Assert.assertThrows(IllegalStateException.class, () -> controller.deleteTask(7)); - Mockito.verify(jobs, Mockito.never()).remove(Mockito.anyInt()); - } - @After public void tearDown() { RequestContextHolder.resetRequestAttributes(); @@ -629,7 +579,6 @@ private static class TestIngestController extends IngestController { private String checkedGraphSpace; private String writeGraphSpace; - private boolean denyWrite; @Override protected HugeClient authClient(String graphSpace, String graph) { @@ -645,9 +594,6 @@ protected void requireGraphSpaceAccess(HugeClient client, @Override protected HugeClient requireGraphSpaceWrite(String graphSpace) { this.writeGraphSpace = graphSpace; - if (this.denyWrite) { - throw new org.apache.hugegraph.exception.ForbiddenException("denied graph write"); - } return Mockito.mock(HugeClient.class); } } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java index 11ae8eadf..19f0e7afd 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java @@ -20,7 +20,6 @@ import java.util.Collections; import java.util.Date; -import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -35,6 +34,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; +import org.mockito.MockedConstruction; import org.mybatis.spring.annotation.MapperScan; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringBootConfiguration; @@ -46,6 +46,7 @@ import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.transaction.support.TransactionTemplate; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.driver.HugeClient; @@ -59,11 +60,9 @@ import org.apache.hugegraph.entity.load.ListFormat; import org.apache.hugegraph.entity.load.LoadParameter; import org.apache.hugegraph.entity.load.LoadTask; -import org.apache.hugegraph.exception.ExternalException; import org.apache.hugegraph.handler.LoadTaskExecutor; -import org.apache.hugegraph.loader.HugeGraphLoader; -import org.apache.hugegraph.loader.executor.LoadContext; import org.apache.hugegraph.loader.executor.LoadOptions; +import org.apache.hugegraph.loader.HugeGraphLoader; import org.apache.hugegraph.options.HubbleOptions; import org.apache.hugegraph.service.schema.EdgeLabelService; import org.apache.hugegraph.service.schema.VertexLabelService; @@ -108,6 +107,8 @@ public class IngestTransactionIntegrationTest { private DataSource dataSource; @Autowired private HugeConfig config; + @Autowired + private PlatformTransactionManager transactionManager; @MockBean private VertexLabelService vertexLabelService; @MockBean @@ -121,7 +122,6 @@ public void setup() { this.cleanup(); this.executor.reset(); this.taskService.failTaskInsert(false); - this.taskService.afterTaskSnapshot(null); Mockito.when(this.config.get(HubbleOptions.UPLOAD_FILE_LOCATION)) .thenReturn(UPLOAD_ROOT); } @@ -297,34 +297,6 @@ public void testFailedDiskCleanupLeavesRetryableOrphanRecord() { Assert.assertEquals(0, this.count("file_mapping")); } - @Test - public void testRejectedDispatchRetriesTaskAndJobAfterDatabaseRecovery() throws Exception { - this.executor.reject(true); - this.jdbc.execute("ALTER TABLE job_manager ADD CONSTRAINT reject_failed_job CHECK (job_status <> " + - JobStatus.FAILED.getValue() + ")"); - LoadTask task; - try { - task = this.jobService.createIngestTask( - this.job(), this.mapping(), this.connection(), this.client()); - } finally { - this.jdbc.execute("ALTER TABLE job_manager DROP CONSTRAINT reject_failed_job"); - } - Assert.assertEquals(LoadStatus.RUNNING, this.taskService.get(task.getId()).getStatus()); - Assert.assertEquals(JobStatus.LOADING, this.jobService.get(task.getJobId()).getJobStatus()); - java.lang.reflect.Field ownersField = LoadTaskService.class.getDeclaredField("runningTaskContainer"); - ownersField.setAccessible(true); - java.util.Map owners = (java.util.Map) ownersField.get( - org.springframework.test.util.AopTestUtils.getUltimateTargetObject(this.taskService)); - Assert.assertSame(task, owners.get(task.getId())); - Assert.assertTrue(task.getLoader().context().closed()); - - this.taskService.updateLoadTaskProgress(); - - Assert.assertEquals(LoadStatus.FAILED, this.taskService.get(task.getId()).getStatus()); - Assert.assertEquals(JobStatus.FAILED, this.jobService.get(task.getJobId()).getJobStatus()); - Assert.assertFalse(owners.containsKey(task.getId())); - } - @Test public void testRejectedDispatchCommitsFailedTaskAndJobWithoutThrowing() { this.executor.reject(true); @@ -341,102 +313,48 @@ public void testRejectedDispatchCommitsFailedTaskAndJobWithoutThrowing() { } @Test - public void testDeleteJobAcceptsSuccessfulTaskAfterSnapshot() { - this.deleteJobAfterTaskCompletion(LoadStatus.SUCCEED); - } - - @Test - public void testDeleteJobAcceptsFailedTaskAfterSnapshot() { - this.deleteJobAfterTaskCompletion(LoadStatus.FAILED); - } - - @Test - public void testStopStillRejectsCompletedTasks() { - LoadTask task = this.jobService.createIngestTask( - this.job(), this.mapping(), this.connection(), this.client()); - this.taskService.clearOwners(); - task.setFinished(true); - for (LoadStatus status : new LoadStatus[]{LoadStatus.SUCCEED, LoadStatus.FAILED, LoadStatus.STOPPED}) { - task.setStatus(status); - this.taskService.update(task); - Assert.assertThrows(ExternalException.class, - () -> this.taskService.stop(task.getId())); - Assert.assertEquals(status, this.taskService.get(task.getId()).getStatus()); - } - } - - @Test - public void testDeleteJobRejectsCompletedOwnerThatHasNotClosed() { - LoadTask task = this.jobService.createIngestTask( - this.job(), this.mapping(), this.connection(), this.client()); - this.taskService.afterTaskSnapshot(() -> { - task.setStatus(LoadStatus.SUCCEED); - task.setFinished(true); - this.taskService.update(task); - }); - - Assert.assertThrows(ExternalException.class, - () -> this.jobService.deleteJob("DEFAULT", "hugegraph", task.getJobId())); - - this.assertTableCounts(1, 1, 1); - Assert.assertEquals(LoadStatus.SUCCEED, this.taskService.get(task.getId()).getStatus()); - Assert.assertFalse(task.context().closed()); - } - - @Test - public void testDeleteJobWaitsForCompletedOwnerToFinish() throws Exception { - FileMapping mapping = this.mapping(); - mapping.setPath(UPLOAD_ROOT + "/snapshot-completion/" + mapping.getName()); - LoadTask task = this.jobService.createIngestTask( - this.job(), mapping, this.connection(), this.client()); - CountDownLatch stopping = new CountDownLatch(1); - LoadContext context = task.context(); - Mockito.doAnswer(invocation -> { - stopping.countDown(); - return null; - }).when(context).stopLoading(); - this.taskService.afterTaskSnapshot(() -> { - task.setStatus(LoadStatus.SUCCEED); - this.taskService.update(task); - task.getLoader().shutdown(); - }); - ExecutorService pool = Executors.newSingleThreadExecutor(); - try { - Future deleting = pool.submit(() -> - this.jobService.deleteJob("DEFAULT", "hugegraph", task.getJobId())); - Assert.assertTrue(stopping.await(5L, TimeUnit.SECONDS)); - Assert.assertFalse(deleting.isDone()); - this.assertTableCounts(1, 1, 1); - Assert.assertEquals(LoadStatus.SUCCEED, this.taskService.get(task.getId()).getStatus()); - - task.setFinished(true); - deleting.get(5L, TimeUnit.SECONDS); - - this.assertTableCounts(0, 0, 0); - } finally { - task.setFinished(true); - pool.shutdown(); - Assert.assertTrue(pool.awaitTermination(5L, TimeUnit.SECONDS)); + public void testUnscheduledLoadersCloseAfterSaveFailureRollbackAndRejection() { + for (int outcome = 0; outcome < 3; outcome++) { + this.cleanup(); + this.executor.reset(); + this.executor.reject(outcome == 2); + JobManager job = this.job(); + this.jobService.save(job); + FileMapping mapping = this.mapping(); + mapping.setJobId(job.getId()); + mapping.setPath(UPLOAD_ROOT + "/cleanup/people.csv"); + this.mappingService.save(mapping); + TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); + if (outcome == 0) { + this.jdbc.execute("ALTER TABLE load_task ADD CONSTRAINT reject_task CHECK (id < 0)"); + } + final int result = outcome; + try (MockedConstruction construction = Mockito.mockConstruction(HugeGraphLoader.class)) { + if (outcome == 0) { + Assert.assertThrows(RuntimeException.class, () -> transaction.execute(status -> + this.taskService.startLoader(this.connection(), mapping, this.client()))); + } else { + transaction.execute(status -> { + this.taskService.startLoader(this.connection(), mapping, this.client()); + if (result == 1) { + status.setRollbackOnly(); + } + return null; + }); + } + Assert.assertEquals(1, construction.constructed().size()); + Mockito.verify(construction.constructed().get(0)).shutdown(); + Assert.assertTrue(this.taskService.createdTask().isFinished()); + Assert.assertEquals(outcome == 2 ? 1 : 0, this.count("load_task")); + Assert.assertEquals(outcome == 2 ? 1 : 0, this.executor.executions()); + } finally { + if (outcome == 0) { + this.jdbc.execute("ALTER TABLE load_task DROP CONSTRAINT reject_task"); + } + } } } - private void deleteJobAfterTaskCompletion(LoadStatus status) { - FileMapping mapping = this.mapping(); - mapping.setPath(UPLOAD_ROOT + "/snapshot-completion/" + mapping.getName()); - LoadTask task = this.jobService.createIngestTask( - this.job(), mapping, this.connection(), this.client()); - this.taskService.afterTaskSnapshot(() -> { - task.setStatus(status); - task.setFinished(true); - this.taskService.update(task); - this.taskService.clearOwners(); - }); - - this.jobService.deleteJob("DEFAULT", "hugegraph", task.getJobId()); - - this.assertTableCounts(0, 0, 0); - } - private JobManager job() { Date now = new Date(); return JobManager.builder() @@ -530,7 +448,6 @@ private HugeClient client() { } private void cleanup() { - this.taskService.clearOwners(); JdbcTemplate template = this.jdbc == null ? new JdbcTemplate(this.dataSource) : this.jdbc; template.update("DELETE FROM load_task"); @@ -585,37 +502,24 @@ public HugeConfig hugeConfig() { public static class TestLoadTaskService extends LoadTaskService { private boolean failTaskInsert; - private Runnable afterTaskSnapshot; + private LoadTask createdTask; - public void afterTaskSnapshot(Runnable action) { - this.afterTaskSnapshot = action; + public void failTaskInsert(boolean fail) { + this.failTaskInsert = fail; } - @Override - public List taskListByJob(int jobId) { - List tasks = super.taskListByJob(jobId); - Runnable action = this.afterTaskSnapshot; - this.afterTaskSnapshot = null; - if (action != null) { - action.run(); - } - return tasks; + public LoadTask startLoader(GraphConnection connection, FileMapping mapping, HugeClient client) { + return super.start(connection, mapping, client); } - @SuppressWarnings("unchecked") - public void clearOwners() { - java.util.Map owners = (java.util.Map) - org.springframework.test.util.ReflectionTestUtils.getField(this, "runningTaskContainer"); - for (LoadTask task : owners.values()) { - if (task.getLoader() != null) { - task.getLoader().shutdown(); - } - } - owners.clear(); + @Override + public void save(LoadTask task) { + this.createdTask = task; + super.save(task); } - public void failTaskInsert(boolean fail) { - this.failTaskInsert = fail; + public LoadTask createdTask() { + return this.createdTask; } @Override @@ -624,17 +528,7 @@ public LoadTask start(GraphConnection connection, FileMapping mapping, String graph = this.failTaskInsert ? null : connection.getGraph(); LoadOptions options = new LoadOptions(); options.token = connection.getToken(); - HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); - LoadContext context = Mockito.mock(LoadContext.class); - java.util.concurrent.atomic.AtomicBoolean closed = new java.util.concurrent.atomic.AtomicBoolean(); - Mockito.when(loader.context()).thenReturn(context); - Mockito.when(context.closed()).thenAnswer(invocation -> closed.get()); - Mockito.doAnswer(invocation -> { - closed.set(true); - return null; - }).when(loader).shutdown(); LoadTask task = LoadTask.builder() - .loader(loader) .graphSpace(connection.getGraphSpace()) .graph(graph) .jobId(mapping.getJobId()) diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphsControllerCanonicalTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphsControllerCanonicalTest.java index 4000d7f10..c13130e9b 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphsControllerCanonicalTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/GraphsControllerCanonicalTest.java @@ -38,10 +38,6 @@ import org.apache.hugegraph.exception.ExternalException; import org.apache.hugegraph.options.HubbleOptions; import org.apache.hugegraph.service.graphs.GraphsService; -import org.apache.hugegraph.service.load.JobManagerService; -import org.apache.hugegraph.service.query.ExecuteHistoryService; -import org.apache.hugegraph.service.query.GremlinCollectionService; -import org.apache.hugegraph.service.space.VermeerService; import org.apache.hugegraph.testutil.Assert; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; @@ -55,9 +51,6 @@ public class GraphsControllerCanonicalTest { private MockMvc mvc; private HugeClient client; private GraphsService graphsService; - private JobManagerService jobs; - private ExecuteHistoryService history; - private GremlinCollectionService collections; private RequestPostProcessor withClient; @Before @@ -65,13 +58,6 @@ public void setup() throws Exception { GraphsController controller = new GraphsController(); this.graphsService = Mockito.mock(GraphsService.class); this.setField(controller, "graphsService", this.graphsService); - this.jobs = Mockito.mock(JobManagerService.class); - this.history = Mockito.mock(ExecuteHistoryService.class); - this.collections = Mockito.mock(GremlinCollectionService.class); - this.setField(controller, "jobManagerService", this.jobs); - this.setField(controller, "executeHistoryService", this.history); - this.setField(controller, "gremlinCollectionService", this.collections); - this.setField(controller, "vermeerService", Mockito.mock(VermeerService.class)); HugeConfig config = Mockito.mock(HugeConfig.class); Mockito.when(config.get(HubbleOptions.PD_ENABLED)).thenReturn(false); this.setField(controller, "config", config); @@ -85,55 +71,6 @@ public void setup() throws Exception { this.mvc = MockMvcBuilders.standaloneSetup(controller).build(); } - @Test - public void testGraphDeletionCleansScopedJobsAfterRemoteSuccess() throws Exception { - this.mvc.perform(delete("/api/v1.3/graphspaces/DEFAULT/graphs/graph_a") - .with(this.withClient)) - .andExpect(status().isOk()); - - org.mockito.InOrder order = Mockito.inOrder(this.graphsService, this.history, this.collections, this.jobs); - order.verify(this.graphsService).delete(this.client, "graph_a", "I'm sure to drop the graph"); - order.verify(this.history).deleteByGraph("DEFAULT", "graph_a"); - order.verify(this.collections).deleteByGraph("DEFAULT", "graph_a"); - order.verify(this.jobs).removeByGraph("DEFAULT", "graph_a"); - } - - @Test - public void testGraphDeletionKeepsLocalStateWhenRemoteDropFails() throws Exception { - Mockito.doThrow(new ExternalException("remote drop rejected")).when(this.graphsService) - .delete(this.client, "graph_a", "I'm sure to drop the graph"); - try { - this.mvc.perform(delete("/api/v1.3/graphspaces/DEFAULT/graphs/graph_a").with(this.withClient)); - org.junit.Assert.fail("Expected remote deletion rejection"); - } catch (ServletException e) { - Assert.assertInstanceOf(ExternalException.class, e.getCause()); - } - Mockito.verifyNoInteractions(this.jobs, this.history, this.collections); - } - - @Test - public void testGraphDeletionKeepsLocalStateWhenAuthenticationFails() throws Exception { - GraphsController controller = new GraphsController() { - @Override - public boolean isVermeerEnabled() { - return false; - } - - @Override - protected HugeClient authClient(String graphspace, String graph) { - throw new org.apache.hugegraph.exception.ForbiddenException("graph deletion denied"); - } - }; - this.setField(controller, "graphsService", this.graphsService); - this.setField(controller, "jobManagerService", this.jobs); - this.setField(controller, "executeHistoryService", this.history); - this.setField(controller, "gremlinCollectionService", this.collections); - - Assert.assertThrows(org.apache.hugegraph.exception.ForbiddenException.class, - () -> controller.delete("DEFAULT", "graph_a", "I'm sure to drop the graph")); - Mockito.verifyNoInteractions(this.graphsService, this.jobs, this.history, this.collections); - } - @Test public void testCanonicalCreateGraphUsesPathNameAndJsonBody() throws Exception { diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java deleted file mode 100644 index 631310760..000000000 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCleanupTest.java +++ /dev/null @@ -1,2945 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.unit; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.hugegraph.entity.GraphConnection; -import org.apache.hugegraph.entity.enums.LoadStatus; -import org.apache.hugegraph.entity.load.FileMapping; -import org.apache.hugegraph.entity.load.FileSetting; -import org.apache.hugegraph.entity.load.LoadParameter; -import org.apache.hugegraph.handler.LoadTaskExecutor; -import org.apache.hugegraph.mapper.load.LoadTaskMapper; -import org.apache.hugegraph.entity.load.LoadTask; -import org.apache.hugegraph.loader.HugeGraphLoader; -import org.apache.hugegraph.loader.executor.LoadContext; -import org.apache.hugegraph.loader.executor.LoadOptions; -import org.apache.hugegraph.service.load.LoadTaskService; -import org.junit.Assert; -import org.junit.Test; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; -import org.springframework.transaction.support.TransactionSynchronization; -import org.springframework.transaction.support.TransactionSynchronizationManager; -import org.mockito.MockedConstruction; -import org.mockito.Mockito; - -public class LoadTaskCleanupTest { - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Test - public void testSuccessIsPublishedAfterShutdown() { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - Mockito.when(loader.load()).thenReturn(true); - Mockito.doAnswer(invocation -> { - Assert.assertEquals(LoadStatus.RUNNING, task.getStatus()); - Assert.assertFalse(task.isFinished()); - return null; - }).when(loader).shutdown(); - - task.run(); - - Mockito.verify(loader).shutdown(); - Assert.assertEquals(LoadStatus.SUCCEED, task.getStatus()); - Assert.assertTrue(task.isFinished()); - Assert.assertEquals(Long.valueOf(7L), task.getFileReadLines()); - Assert.assertEquals(Long.valueOf(12L), task.getLastDuration()); - } - - @Test - public void testLoadFailureKeepsCleanupFailureSuppressed() { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - RuntimeException loadFailure = new RuntimeException("load failed"); - RuntimeException cleanupFailure = new RuntimeException("cleanup failed"); - Mockito.when(loader.load()).thenThrow(loadFailure); - Mockito.doThrow(cleanupFailure).when(loader).shutdown(); - - task.run(); - - Mockito.verify(loader).shutdown(); - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - Assert.assertTrue(task.isFinished()); - Assert.assertArrayEquals(new Throwable[]{cleanupFailure}, loadFailure.getSuppressed()); - } - - @Test - public void testFalseLoadResultStillShutsDown() { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - Mockito.when(loader.load()).thenReturn(false); - - task.run(); - - Mockito.verify(loader).shutdown(); - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - Assert.assertTrue(task.isFinished()); - } - - @Test - public void testCleanupFailureMarksPausedAndStoppedTasksFailed() { - for (LoadStatus status : new LoadStatus[]{LoadStatus.RUNNING, LoadStatus.PAUSED, LoadStatus.STOPPED}) { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setStatus(status); - Mockito.when(loader.load()).thenReturn(true); - Mockito.doThrow(new RuntimeException("cleanup failed")).when(loader).shutdown(); - - task.run(); - - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - Assert.assertTrue(task.isFinished()); - } - } - - @Test - public void testSameFailureIsNotSuppressedOntoItself() { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - RuntimeException failure = new RuntimeException("same failure"); - Mockito.when(loader.load()).thenThrow(failure); - Mockito.doThrow(failure).when(loader).shutdown(); - - task.run(); - - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - Assert.assertTrue(task.isFinished()); - Assert.assertEquals(0, failure.getSuppressed().length); - } - - @Test - public void testStopLeavesShutdownToRunAndWaitsForCleanup() throws Exception { - HugeGraphLoader loader = this.loader(); - LoadContext context = loader.context(); - LoadTask task = this.task(loader); - CountDownLatch loading = new CountDownLatch(1); - CountDownLatch releaseLoad = new CountDownLatch(1); - CountDownLatch stopped = new CountDownLatch(1); - CountDownLatch cleaning = new CountDownLatch(1); - CountDownLatch releaseCleanup = new CountDownLatch(1); - Mockito.when(loader.load()).thenAnswer(invocation -> { - loading.countDown(); - Assert.assertTrue(releaseLoad.await(5, TimeUnit.SECONDS)); - return true; - }); - Mockito.doAnswer(invocation -> { - stopped.countDown(); - return null; - }).when(context).stopLoading(); - Mockito.doAnswer(invocation -> { - cleaning.countDown(); - Assert.assertTrue(releaseCleanup.await(5, TimeUnit.SECONDS)); - Mockito.when(context.closed()).thenReturn(true); - return null; - }).when(loader).shutdown(); - Thread running = new Thread(task); - Thread stopping = new Thread(task::stop); - try { - running.start(); - Assert.assertTrue(loading.await(5, TimeUnit.SECONDS)); - task.setStatus(LoadStatus.PAUSED); - stopping.start(); - Assert.assertTrue(stopped.await(5, TimeUnit.SECONDS)); - Mockito.verify(loader, Mockito.never()).shutdown(); - releaseLoad.countDown(); - Assert.assertTrue(cleaning.await(5, TimeUnit.SECONDS)); - Assert.assertFalse(task.isFinished()); - Assert.assertTrue(stopping.isAlive()); - Assert.assertSame(loader, task.getLoader()); - releaseCleanup.countDown(); - running.join(5000); - stopping.join(5000); - Assert.assertFalse(running.isAlive()); - Assert.assertFalse(stopping.isAlive()); - Mockito.verify(loader).shutdown(); - Assert.assertNull(task.getLoader()); - Assert.assertEquals(LoadStatus.PAUSED, task.getStatus()); - } finally { - releaseLoad.countDown(); - releaseCleanup.countDown(); - running.join(5000); - stopping.join(5000); - } - } - - @Test - public void testPreviewShutsDownAfterSuccess() { - try (MockedConstruction construction = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, context) -> Mockito.when(loader.load()).thenReturn(true))) { - new LoadTaskService().loader(new LoadOptions()); - - Assert.assertEquals(1, construction.constructed().size()); - Mockito.verify(construction.constructed().get(0)).shutdown(); - } - } - - @Test - public void testPreviewKeepsPrimaryFailureWhenShutdownFails() { - RuntimeException loadFailure = new RuntimeException("preview load failed"); - RuntimeException cleanupFailure = new RuntimeException("preview cleanup failed"); - try (MockedConstruction construction = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, context) -> { - Mockito.when(loader.load()).thenThrow(loadFailure); - Mockito.doThrow(cleanupFailure).when(loader).shutdown(); - })) { - new LoadTaskService().loader(new LoadOptions()); - - Mockito.verify(construction.constructed().get(0)).shutdown(); - Assert.assertArrayEquals(new Throwable[]{cleanupFailure}, loadFailure.getSuppressed()); - } - } - - @Test - public void testSaveFailureClosesEagerLoaderAndKeepsPrimaryFailure() throws Exception { - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - set(service, "mapper", mapper); - RuntimeException failure = new RuntimeException("database unavailable"); - RuntimeException cleanup = new RuntimeException("cleanup failed"); - Mockito.when(mapper.insert(Mockito.any(LoadTask.class))).thenThrow(failure); - FileMapping mapping = new FileMapping(); - mapping.setPath(this.folder.newFile("input.csv").getAbsolutePath()); - mapping.setVertexMappings(Collections.emptySet()); - mapping.setEdgeMappings(Collections.emptySet()); - mapping.setLoadParameter(new LoadParameter()); - FileSetting setting = new FileSetting(); - setting.setColumnNames(Collections.singletonList("id")); - mapping.setFileSetting(setting); - try (MockedConstruction owners = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, context) -> { - LoadContext resources = Mockito.mock(LoadContext.class); - Mockito.when(loader.context()).thenReturn(resources); - Mockito.when(resources.closed()).thenReturn(true); - Mockito.doThrow(cleanup).when(loader).shutdown(); - })) { - try { - service.start(new GraphConnection(), mapping, null); - Assert.fail("save must fail"); - } catch (RuntimeException actual) { - Assert.assertSame(failure, actual); - } - Assert.assertEquals(1, owners.constructed().size()); - Mockito.verify(owners.constructed().get(0)).shutdown(); - Assert.assertArrayEquals(new Throwable[]{cleanup}, failure.getSuppressed()); - } - } - - @Test - public void testOuterRollbackClosesUnscheduledLoader() throws Exception { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - LoadTaskService service = new LoadTaskService(); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - set(service, "taskExecutor", executor); - TransactionSynchronizationManager.initSynchronization(); - try { - dispatch(service, task); - TransactionSynchronizationManager.getSynchronizations().get(0) - .afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK); - Mockito.verify(loader).shutdown(); - Mockito.verifyNoInteractions(executor); - } finally { - TransactionSynchronizationManager.clearSynchronization(); - } - } - - @Test - public void testDispatchRejectionClosesUnscheduledLoader() throws Exception { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - LoadTaskService service = new LoadTaskService(); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - RuntimeException rejection = new java.util.concurrent.RejectedExecutionException("executor closed"); - RuntimeException cleanup = new RuntimeException("cleanup failed"); - Mockito.doThrow(rejection).when(executor).execute(Mockito.eq(task), Mockito.any()); - Mockito.doThrow(cleanup).when(loader).shutdown(); - set(service, "taskExecutor", executor); - dispatch(service, task); - Mockito.verify(loader).shutdown(); - Assert.assertArrayEquals(new Throwable[]{cleanup}, rejection.getSuppressed()); - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - } - - @Test - public void testFinishedIncludesPersistenceCallback() { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - Mockito.when(loader.load()).thenReturn(true); - new LoadTaskExecutor().execute(task, () -> { - Mockito.verify(loader).shutdown(); - Assert.assertFalse("runner is not finished until its callback returns", task.isFinished()); - }); - Assert.assertTrue(task.isFinished()); - } - - @Test - public void testStopOfPausedOwnerWaitsForRunner() throws Exception { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - task.setStatus(LoadStatus.PAUSED); - CountDownLatch loading = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - CountDownLatch stopReturned = new CountDownLatch(1); - Mockito.when(loader.load()).thenAnswer(invocation -> { - loading.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - return true; - }); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.updateById(task)).thenReturn(1); - set(service, "mapper", mapper); - Map owners = new ConcurrentHashMap<>(); - owners.put(9, task); - set(service, "runningTaskContainer", owners); - Thread runner = new Thread(() -> new LoadTaskExecutor().execute(task, () -> service.update(task))); - Thread stopper = new Thread(() -> { - service.stop(9); - stopReturned.countDown(); - }); - try { - runner.start(); - Assert.assertTrue(loading.await(5, TimeUnit.SECONDS)); - stopper.start(); - Assert.assertFalse("PAUSED owner is still loading", stopReturned.await(300, TimeUnit.MILLISECONDS)); - Assert.assertSame(task, owners.get(9)); - } finally { - release.countDown(); - runner.join(5000); - stopper.join(5000); - } - Assert.assertFalse(runner.isAlive()); - Assert.assertFalse(stopper.isAlive()); - Assert.assertEquals(LoadStatus.STOPPED, task.getStatus()); - Assert.assertTrue(owners.isEmpty()); - } - - @Test - public void testStopAndRetryCannotPassOldPersistenceCallback() throws Exception { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - task.setStatus(LoadStatus.PAUSED); - Mockito.when(loader.load()).thenReturn(true); - LoadTask restored = Mockito.mock(LoadTask.class); - Mockito.when(restored.getStatus()).thenReturn(LoadStatus.STOPPED); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.selectById(9)).thenReturn( - LoadTask.builder().id(9).status(LoadStatus.RUNNING).build(), restored); - CountDownLatch callback = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - CountDownLatch stopReturned = new CountDownLatch(1); - java.util.concurrent.atomic.AtomicReference persisted = - new java.util.concurrent.atomic.AtomicReference<>(); - Mockito.when(mapper.updateById(task)).thenAnswer(invocation -> { - if (callback.getCount() > 0) { - callback.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - } - persisted.set(task.getStatus()); - return 1; - }); - set(service, "mapper", mapper); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - Thread[] runner = new Thread[1]; - Mockito.doAnswer(invocation -> { - Runnable completion = invocation.getArgument(1); - runner[0] = new Thread(() -> new LoadTaskExecutor().execute(task, completion)); - runner[0].start(); - return null; - }).when(executor).execute(Mockito.eq(task), Mockito.any()); - set(service, "taskExecutor", executor); - dispatch(service, task); - Thread stopper = new Thread(() -> { - service.stop(9); - stopReturned.countDown(); - }); - CountDownLatch retryReturned = new CountDownLatch(1); - Thread retry = new Thread(() -> { - try { - service.retry(9, "token"); - } catch (RuntimeException expected) { - // The mock restored task cannot reconnect, after the old callback has finished. - } finally { - retryReturned.countDown(); - } - }); - try { - Assert.assertTrue(callback.await(5, TimeUnit.SECONDS)); - Assert.assertFalse(task.isFinished()); - stopper.start(); - Assert.assertFalse(stopReturned.await(300, TimeUnit.MILLISECONDS)); - retry.start(); - Assert.assertFalse(retryReturned.await(300, TimeUnit.MILLISECONDS)); - Mockito.verify(restored, Mockito.never()).reconnect(Mockito.anyString()); - } finally { - release.countDown(); - runner[0].join(5000); - stopper.join(5000); - retry.join(5000); - } - Assert.assertFalse(runner[0].isAlive()); - Assert.assertFalse(stopper.isAlive()); - Assert.assertFalse(retry.isAlive()); - Assert.assertEquals(LoadStatus.STOPPED, persisted.get()); - } - - @Test - public void testFastRunnerDoesNotLeaveFinishedOwnerInContainer() throws Exception { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - Mockito.when(loader.load()).thenReturn(true); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.updateById(task)).thenReturn(1); - set(service, "mapper", mapper); - set(service, "taskExecutor", new LoadTaskExecutor()); - Map owners = new ConcurrentHashMap<>(); - set(service, "runningTaskContainer", owners); - dispatch(service, task); - Assert.assertTrue(task.isFinished()); - Assert.assertTrue(owners.isEmpty()); - } - - @Test - public void testStopRetriesFailedCallbackBeforeRemovingOwner() throws Exception { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - task.setStatus(LoadStatus.PAUSED); - Mockito.when(loader.load()).thenReturn(true); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.updateById(task)).thenThrow(new IllegalStateException("database offline")) - .thenReturn(1); - set(service, "mapper", mapper); - set(service, "taskExecutor", new LoadTaskExecutor()); - Map owners = new ConcurrentHashMap<>(); - set(service, "runningTaskContainer", owners); - dispatch(service, task); - Assert.assertTrue(task.isFinished()); - Assert.assertSame(task, owners.get(9)); - service.stop(9); - Mockito.verify(mapper, Mockito.times(2)).updateById(task); - Mockito.verify(loader).shutdown(); - Assert.assertEquals(LoadStatus.STOPPED, task.getStatus()); - Assert.assertTrue(owners.isEmpty()); - } - - @Test - public void testResumeSaveFailureClosesNewOwnerAndKeepsPrimaryFailure() throws Exception { - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - LoadTask task = LoadTask.builder().id(9).options(new LoadOptions()).status(LoadStatus.PAUSED).build(); - Mockito.when(mapper.selectById(9)).thenReturn(task); - RuntimeException failure = new RuntimeException("database unavailable"); - RuntimeException cleanup = new RuntimeException("cleanup failed"); - Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenThrow(failure); - set(service, "mapper", mapper); - try (MockedConstruction owners = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, context) -> { - LoadContext resources = Mockito.mock(LoadContext.class); - Mockito.when(loader.context()).thenReturn(resources); - Mockito.when(resources.closed()).thenReturn(true); - Mockito.doThrow(cleanup).when(loader).shutdown(); - })) { - try { - service.resume(9, "session-token"); - Assert.fail("save must fail"); - } catch (RuntimeException actual) { - Assert.assertSame(failure, actual); - } - Mockito.verify(owners.constructed().get(0)).shutdown(); - Assert.assertArrayEquals(new Throwable[]{cleanup}, failure.getSuppressed()); - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - Assert.assertTrue(task.isFinished()); - } - } - - @Test(timeout = 15000) - public void testRealWorkerTimeoutRetainsOwnerUntilContextCloses() throws Exception { - com.sun.net.httpserver.HttpServer server = com.sun.net.httpserver.HttpServer.create( - new java.net.InetSocketAddress("127.0.0.1", 0), 0); - server.createContext("/versions", exchange -> { - byte[] body = "{\"versions\":{\"core\":\"1.8.0\",\"api\":\"0.74\"}}" - .getBytes(java.nio.charset.StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(200, body.length); - exchange.getResponseBody().write(body); - exchange.close(); - }); - server.start(); - CountDownLatch release = new CountDownLatch(1); - CountDownLatch running = new CountDownLatch(1); - CountDownLatch stopped = new CountDownLatch(1); - HugeGraphLoader loader = null; - Thread hook = null; - try { - LoadOptions options = new LoadOptions(); - options.host = "127.0.0.1"; - options.port = server.getAddress().getPort(); - options.file = this.folder.newFile("timeout.json").getAbsolutePath(); - options.batchInsertThreads = 1; - options.singleInsertThreads = 1; - options.shutdownTimeout = 1; - loader = Mockito.spy(new HugeGraphLoader(options, - new org.apache.hugegraph.loader.mapping.LoadMapping(Collections.emptyList()))); - Mockito.doReturn(true).when(loader).load(); - Field hookField = HugeGraphLoader.class.getDeclaredField("shutdownHook"); - hookField.setAccessible(true); - hook = (Thread) hookField.get(loader); - Field managerField = HugeGraphLoader.class.getDeclaredField("manager"); - managerField.setAccessible(true); - Object manager = managerField.get(loader); - Field poolField = manager.getClass().getDeclaredField("batchService"); - poolField.setAccessible(true); - java.util.concurrent.ExecutorService pool = - (java.util.concurrent.ExecutorService) poolField.get(manager); - pool.submit(() -> { - running.countDown(); - try { - while (release.getCount() > 0) { - try { - release.await(100, TimeUnit.MILLISECONDS); - } catch (InterruptedException ignored) { - // Deliberately model a backend call that ignores cancellation. - } - } - } finally { - stopped.countDown(); - } - }); - Assert.assertTrue(running.await(5, TimeUnit.SECONDS)); - LoadTask task = this.task(loader); - task.setId(9); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.updateById(Mockito.any(LoadTask.class))).thenReturn(1); - set(service, "mapper", mapper); - set(service, "taskExecutor", new LoadTaskExecutor()); - Map owners = new ConcurrentHashMap<>(); - set(service, "runningTaskContainer", owners); - dispatch(service, task); - Assert.assertTrue(task.isFinished()); - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - Assert.assertFalse(loader.context().closed()); - Assert.assertSame(task, owners.get(9)); - Assert.assertFalse(pool.isTerminated()); - task.stop(); - Assert.assertSame(loader, task.getLoader()); - RuntimeException primary; - try { - loader.context().throwIfFailed(); - throw new AssertionError("timeout diagnosis is missing"); - } catch (org.apache.hugegraph.loader.exception.LoadException failure) { - primary = failure; - } - Assert.assertTrue(Runtime.getRuntime().removeShutdownHook(hook)); - Runtime.getRuntime().addShutdownHook(hook); - LoadTask restored = LoadTask.builder().id(9).options(options).status(LoadStatus.FAILED) - .lastDuration(0L).build(); - Mockito.when(mapper.selectById(9)).thenReturn(restored); - try (MockedConstruction fresh = Mockito.mockConstruction(HugeGraphLoader.class)) { - org.apache.hugegraph.testutil.Assert.assertThrows( - RuntimeException.class, () -> service.retry(9, "session-token")); - org.apache.hugegraph.testutil.Assert.assertThrows( - RuntimeException.class, () -> service.resume(9, "session-token")); - Assert.assertTrue(fresh.constructed().isEmpty()); - Assert.assertSame(task, owners.get(9)); - release.countDown(); - Assert.assertTrue(stopped.await(5, TimeUnit.SECONDS)); - try { - loader.shutdown(); - Assert.fail("original timeout must remain visible"); - } catch (org.apache.hugegraph.loader.exception.LoadException failure) { - Assert.assertSame(primary, failure); - } - Assert.assertTrue(pool.isTerminated()); - Assert.assertTrue(loader.context().closed()); - Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); - set(service, "taskExecutor", Mockito.mock(LoadTaskExecutor.class)); - service.retry(9, "session-token"); - Assert.assertEquals(1, fresh.constructed().size()); - Assert.assertSame(restored, owners.get(9)); - Assert.assertEquals(LoadStatus.RUNNING, restored.getStatus()); - fresh.constructed().get(0).shutdown(); - } - } finally { - release.countDown(); - if (loader != null) { - try { - loader.shutdown(); - } catch (org.apache.hugegraph.loader.exception.LoadException ignored) { - // Preserve the original timeout diagnosis after closing resources. - } - } - if (hook != null) { - Runtime.getRuntime().removeShutdownHook(hook); - } - server.stop(0); - } - } - - @Test - public void testUnscheduledResumeKeepsCheckpointAndFileReaderOffset() throws Exception { - com.sun.net.httpserver.HttpServer server = com.sun.net.httpserver.HttpServer.create( - new java.net.InetSocketAddress("127.0.0.1", 0), 0); - server.createContext("/versions", exchange -> { - byte[] body = "{\"versions\":{\"core\":\"1.8.0\",\"api\":\"0.74\"}}" - .getBytes(java.nio.charset.StandardCharsets.UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(200, body.length); - exchange.getResponseBody().write(body); - exchange.close(); - }); - server.start(); - try { - java.io.File input = this.folder.newFile("checkpoint.csv"); - java.nio.file.Files.writeString(input.toPath(), "prefix0\nprefix1\nsuffix2\n"); - org.apache.hugegraph.loader.source.file.FileSource source = - new org.apache.hugegraph.loader.source.file.FileSource(); - source.path(input.getAbsolutePath()); - source.header(new String[]{"id"}); - source.hasHeader(false); - org.apache.hugegraph.loader.mapping.VertexMapping vertex = - new org.apache.hugegraph.loader.mapping.VertexMapping("id", true); - vertex.label("record"); - org.apache.hugegraph.loader.mapping.InputStruct struct = - new org.apache.hugegraph.loader.mapping.InputStruct( - Collections.singletonList(vertex), Collections.emptyList()); - struct.id("1"); - struct.input(source); - LoadOptions options = new LoadOptions(); - options.host = "127.0.0.1"; - options.port = server.getAddress().getPort(); - options.file = this.folder.newFile("checkpoint.json").getAbsolutePath(); - options.incrementalMode = true; - org.apache.hugegraph.loader.util.MappingUtil.write( - new org.apache.hugegraph.loader.mapping.LoadMapping(Collections.singletonList(struct)), - options.file); - org.apache.hugegraph.loader.progress.LoadProgress checkpoint = - new org.apache.hugegraph.loader.progress.LoadProgress(); - org.apache.hugegraph.loader.progress.InputItemProgress item = - new org.apache.hugegraph.loader.progress.FileItemProgress(input.getName(), input.lastModified(), - String.valueOf(org.apache.commons.io.FileUtils.checksumCRC32(input)), 2L); - item.offset(2L); - item.confirmOffset(); - checkpoint.addStruct(struct).addLoadingItem(input.getName(), item); - checkpoint.plusVertexLoaded(2L); - java.nio.file.Path saved = java.nio.file.Paths.get( - org.apache.hugegraph.loader.progress.LoadProgress.format(options, "20000101-000000")); - java.nio.file.Files.createDirectories(saved.getParent()); - String original = org.apache.hugegraph.loader.util.JsonUtil.toJson(checkpoint); - java.nio.file.Files.writeString(saved, original); - LoadTask restored = LoadTask.builder().id(9).options(options).status(LoadStatus.PAUSED).build(); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.selectById(9)).thenReturn(restored); - RuntimeException failure = new RuntimeException("database update rejected"); - Mockito.when(mapper.update(Mockito.eq(restored), Mockito.any())).thenThrow(failure); - set(service, "mapper", mapper); - try { - service.resume(9, "session-token"); - Assert.fail("database update must fail before dispatch"); - } catch (RuntimeException actual) { - Assert.assertSame(failure, actual); - } - Assert.assertTrue(restored.context().closed()); - LoadContext next = new LoadContext(options); - try { - org.apache.hugegraph.loader.reader.file.LocalFileReader scanner = - new org.apache.hugegraph.loader.reader.file.LocalFileReader(source); - try (org.apache.hugegraph.loader.reader.InputReader reader = scanner.split().get(0)) { - reader.init(next, struct); - String first = reader.next().rawLine(); - Assert.assertEquals("suffix2", first); - Assert.assertEquals(2L, next.oldProgress().totalInputRead()); - Assert.assertEquals(2L, next.oldProgress().vertexLoaded()); - Assert.assertEquals(original, java.nio.file.Files.readString(saved)); - try (java.util.stream.Stream paths = - java.nio.file.Files.list(saved.getParent())) { - Assert.assertEquals(1L, paths.count()); - } - } - } finally { - next.close(); - } - } finally { - server.stop(0); - } - } - - @Test - public void testProgressPollRetriesTerminalCallbackFailure() throws Exception { - this.assertTerminalPersistenceRecovery(false); - } - - @Test - public void testProgressPollRetriesStopPersistenceFailure() throws Exception { - this.assertTerminalPersistenceRecovery(true); - } - - private void assertTerminalPersistenceRecovery(boolean stop) throws Exception { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - if (stop) { - task.setStatus(LoadStatus.PAUSED); - } - Mockito.when(loader.load()).thenReturn(true); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - RuntimeException callbackFailure = new RuntimeException("callback database unavailable"); - enablePollTransactions(service); - RuntimeException stopFailure = new RuntimeException("stop database unavailable"); - if (stop) { - Mockito.when(mapper.updateById(task)).thenThrow(callbackFailure).thenThrow(stopFailure).thenReturn(1); - } else { - Mockito.when(mapper.updateById(task)).thenThrow(callbackFailure).thenReturn(1); - } - set(service, "mapper", mapper); - Map owners = new ConcurrentHashMap<>(); - set(service, "runningTaskContainer", owners); - CountDownLatch release = new CountDownLatch(1); - Thread[] runner = new Thread[1]; - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - Mockito.doAnswer(invocation -> { - Runnable callback = invocation.getArgument(1); - runner[0] = new Thread(() -> { - try { - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - new LoadTaskExecutor().execute(task, callback); - } catch (Throwable failure) { - Assert.assertSame(callbackFailure, failure); - } - }); - runner[0].start(); - return null; - }).when(executor).execute(Mockito.eq(task), Mockito.any()); - set(service, "taskExecutor", executor); - dispatch(service, task); - release.countDown(); - runner[0].join(5000); - Assert.assertFalse(runner[0].isAlive()); - Assert.assertTrue(task.isFinished()); - Assert.assertSame(task, owners.get(9)); - if (stop) { - try { - service.stop(9); - Assert.fail("stop must preserve its database failure"); - } catch (RuntimeException failure) { - Assert.assertSame(stopFailure, failure); - } - } - service.updateLoadTaskProgress(); - Mockito.verify(mapper, Mockito.times(stop ? 3 : 2)).updateById(task); - Assert.assertTrue(owners.isEmpty()); - Assert.assertEquals(stop ? LoadStatus.STOPPED : LoadStatus.SUCCEED, task.getStatus()); - } - - @Test - public void testDatabaseOnlyPauseDoesNotRemoveResumedOwner() throws Exception { - this.assertDatabaseOnlyTransitionKeepsNewOwner(true); - } - - @Test - public void testDatabaseOnlyStopDoesNotRemoveRetriedOwner() throws Exception { - this.assertDatabaseOnlyTransitionKeepsNewOwner(false); - } - - private void assertDatabaseOnlyTransitionKeepsNewOwner(boolean pause) throws Exception { - LoadTask old = LoadTask.builder().id(9).options(new LoadOptions()) - .status(pause ? LoadStatus.RUNNING : LoadStatus.PAUSED).build(); - LoadTask fresh = LoadTask.builder().id(9).options(new LoadOptions()).lastDuration(0L) - .status(pause ? LoadStatus.PAUSED : LoadStatus.STOPPED).build(); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.selectById(9)).thenReturn(old, fresh); - CountDownLatch saved = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - java.util.concurrent.atomic.AtomicReference row = - new java.util.concurrent.atomic.AtomicReference<>(); - org.mockito.stubbing.Answer saveOld = invocation -> { - row.set(old.getStatus()); - saved.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - return 1; - }; - Mockito.when(mapper.updateById(old)).thenAnswer(saveOld); - Mockito.when(mapper.update(Mockito.eq(old), Mockito.any())).thenAnswer(saveOld); - org.mockito.stubbing.Answer saveFresh = invocation -> { - row.set(fresh.getStatus()); - return 1; - }; - Mockito.when(mapper.updateById(fresh)).thenAnswer(saveFresh); - Mockito.when(mapper.update(Mockito.eq(fresh), Mockito.any())).thenAnswer(saveFresh); - set(service, "mapper", mapper); - set(service, "taskExecutor", Mockito.mock(LoadTaskExecutor.class)); - Map owners = new ConcurrentHashMap<>(); - set(service, "runningTaskContainer", owners); - java.util.concurrent.atomic.AtomicReference failure = - new java.util.concurrent.atomic.AtomicReference<>(); - Thread stopping = new Thread(() -> { - try { - if (pause) { - service.pause(9); - } else { - service.stop(9); - } - } catch (Throwable error) { - failure.set(error); - } - }); - try (MockedConstruction construction = Mockito.mockConstruction(HugeGraphLoader.class)) { - try { - stopping.start(); - Assert.assertTrue(saved.await(5, TimeUnit.SECONDS)); - if (pause) { - peer(service).resume(9, "session-token"); - } else { - peer(service).retry(9, "session-token"); - } - Assert.assertSame(fresh, owners.get(9)); - release.countDown(); - stopping.join(5000); - Assert.assertFalse(stopping.isAlive()); - Assert.assertNull(failure.get()); - Assert.assertSame(fresh, owners.get(9)); - Assert.assertEquals(LoadStatus.RUNNING, row.get()); - } finally { - release.countDown(); - stopping.join(5000); - if (!construction.constructed().isEmpty()) { - construction.constructed().get(0).shutdown(); - } - } - } - } - - @Test - public void testFinalProgressPollDoesNotSaveReplacedOwner() throws Exception { - HugeGraphLoader loader = this.loader(); - LoadTask old = Mockito.spy(this.task(loader)); - old.setId(9); - Mockito.when(loader.load()).thenReturn(false); - old.run(); - LoadTask fresh = LoadTask.builder().id(9).options(new LoadOptions()).lastDuration(0L) - .status(LoadStatus.FAILED).build(); - LoadTaskService service = new LoadTaskService(); - enablePollTransactions(service); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.selectById(9)).thenReturn(fresh); - Mockito.when(mapper.updateById(fresh)).thenReturn(1); - set(service, "mapper", mapper); - set(service, "taskExecutor", Mockito.mock(LoadTaskExecutor.class)); - Map owners = new ConcurrentHashMap<>(); - owners.put(9, old); - set(service, "runningTaskContainer", owners); - CountDownLatch selected = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - Mockito.doAnswer(invocation -> { - if (Thread.currentThread().getName().equals("old-owner-poll")) { - selected.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - } - return invocation.callRealMethod(); - }).when(old).lock(); - java.util.concurrent.atomic.AtomicReference failure = - new java.util.concurrent.atomic.AtomicReference<>(); - Thread poll = new Thread(() -> { - try { - service.updateLoadTaskProgress(); - } catch (Throwable error) { - failure.set(error); - } - }, "old-owner-poll"); - try (MockedConstruction construction = Mockito.mockConstruction(HugeGraphLoader.class)) { - try { - poll.start(); - Assert.assertTrue(selected.await(5, TimeUnit.SECONDS)); - service.retry(9, "session-token"); - release.countDown(); - poll.join(5000); - Assert.assertFalse(poll.isAlive()); - Assert.assertNull(failure.get()); - Mockito.verify(mapper, Mockito.never()).updateById(old); - Mockito.verify(mapper).update(Mockito.eq(fresh), Mockito.any()); - Assert.assertSame(fresh, owners.get(9)); - } finally { - release.countDown(); - poll.join(5000); - if (!construction.constructed().isEmpty()) { - construction.constructed().get(0).shutdown(); - } - } - } - } - - @Test(timeout = 15000) - public void testStopRoutesToOwnerPublishedDuringDatabaseRead() throws Exception { - this.assertOwnerPublishedDuringDatabaseRead(false); - } - - @Test(timeout = 15000) - public void testPauseRoutesToOwnerPublishedDuringDatabaseRead() throws Exception { - this.assertOwnerPublishedDuringDatabaseRead(true); - } - - private void assertOwnerPublishedDuringDatabaseRead(boolean pause) throws Exception { - org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = - new org.springframework.jdbc.datasource.DriverManagerDataSource(); - dataSource.setDriverClassName("org.h2.Driver"); - dataSource.setUrl("jdbc:h2:mem:owner-routing-" + java.util.UUID.randomUUID()); - org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(dataSource); - CountDownLatch selecting = new CountDownLatch(1); - CountDownLatch releaseRead = new CountDownLatch(1); - CountDownLatch releaseRun = new CountDownLatch(1); - java.util.concurrent.atomic.AtomicBoolean stopped = new java.util.concurrent.atomic.AtomicBoolean(); - java.util.concurrent.atomic.AtomicBoolean closed = - new java.util.concurrent.atomic.AtomicBoolean(); - java.util.concurrent.atomic.AtomicInteger reads = new java.util.concurrent.atomic.AtomicInteger(); - java.util.concurrent.atomic.AtomicReference failure = - new java.util.concurrent.atomic.AtomicReference<>(); - java.util.concurrent.atomic.AtomicReference returned = - new java.util.concurrent.atomic.AtomicReference<>(); - java.util.concurrent.atomic.AtomicReference snapshot = - new java.util.concurrent.atomic.AtomicReference<>(); - Map owners = new ConcurrentHashMap<>(); - Thread[] runner = new Thread[1]; - try (java.sql.Connection keeper = dataSource.getConnection()) { - jdbc.execute("CREATE TABLE routing (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); - jdbc.update("INSERT INTO routing VALUES (9, ?)", LoadStatus.PAUSED.getValue()); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.selectById(9)).thenAnswer(invocation -> { - boolean operation = Thread.currentThread().getName().equals("routing-operation"); - if (operation) { - selecting.countDown(); - Assert.assertTrue(releaseRead.await(5, TimeUnit.SECONDS)); - } - byte code = jdbc.queryForObject("SELECT load_status FROM routing WHERE id=9", Byte.class); - LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) - .filter(value -> value.getValue() == code).findFirst().get(); - LoadTask task = LoadTask.builder().id(9).options(new LoadOptions()) - .lastDuration(0L).status(status).build(); - if (operation) { - snapshot.set(task); - } - return task; - }); - Mockito.when(mapper.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - return jdbc.update("UPDATE routing SET load_status=? WHERE id=?", - task.getStatus().getValue(), task.getId()); - }); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = - invocation.getArgument(1); - condition.getSqlSegment(); - Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Byte).findFirst().get(); - return jdbc.update("UPDATE routing SET load_status=? WHERE id=? AND load_status=?", - task.getStatus().getValue(), task.getId(), expected); - }); - LoadTaskService service = new LoadTaskService(); - set(service, "mapper", mapper); - set(service, "runningTaskContainer", owners); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - Mockito.doAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - Runnable callback = invocation.getArgument(1); - runner[0] = new Thread(() -> { - try { - Assert.assertTrue(releaseRun.await(5, TimeUnit.SECONDS)); - new LoadTaskExecutor().execute(task, callback); - } catch (Throwable error) { - failure.compareAndSet(null, error); - } - }); - runner[0].start(); - return null; - }).when(executor).execute(Mockito.any(), Mockito.any()); - set(service, "taskExecutor", executor); - Thread operation = new Thread(() -> { - try { - returned.set(pause ? service.pause(9) : service.stop(9)); - } catch (Throwable error) { - failure.compareAndSet(null, error); - } - }, "routing-operation"); - try (MockedConstruction construction = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, ignored) -> { - LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); - Mockito.when(loader.context()).thenReturn(context); - Mockito.when(context.closed()).thenAnswer(invocation -> closed.get()); - Mockito.when(context.stopped()).thenAnswer(invocation -> stopped.get()); - Mockito.doAnswer(invocation -> { - stopped.set(true); - return null; - }).when(context).stopLoading(); - Mockito.doAnswer(invocation -> { - closed.set(true); - return null; - }).when(loader).shutdown(); - Mockito.when(loader.load()).thenAnswer(invocation -> { - if (!stopped.get()) { - reads.incrementAndGet(); - } - return true; - }); - })) { - try { - operation.start(); - Assert.assertTrue(selecting.await(5, TimeUnit.SECONDS)); - LoadTask owner = peer(service).resume(9, "session-token"); - releaseRead.countDown(); - long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); - while (!stopped.get() && System.nanoTime() < deadline) { - Thread.sleep(5); - } - releaseRun.countDown(); - runner[0].join(5000); - operation.join(5000); - Assert.assertFalse(runner[0].isAlive()); - Assert.assertFalse(operation.isAlive()); - Assert.assertNull(failure.get()); - Assert.assertNotSame(snapshot.get(), owner); - Assert.assertSame(owner, returned.get()); - Assert.assertTrue(stopped.get()); - Assert.assertTrue(closed.get()); - Assert.assertEquals(0, reads.get()); - LoadStatus expected = pause ? LoadStatus.PAUSED : LoadStatus.STOPPED; - Assert.assertEquals(expected, owner.getStatus()); - Assert.assertEquals(expected.getValue(), jdbc.queryForObject( - "SELECT load_status FROM routing WHERE id=9", Byte.class)); - Assert.assertTrue(owners.isEmpty()); - } finally { - releaseRead.countDown(); - releaseRun.countDown(); - if (runner[0] != null) { - runner[0].join(5000); - } - operation.join(5000); - for (HugeGraphLoader loader : construction.constructed()) { - loader.shutdown(); - } - } - } - } - } - - @Test(timeout = 15000) - public void testConditionalResumeRejectsCommittedDatabaseOnlyStop() throws Exception { - this.assertConditionalTransitionWinner(true); - } - - @Test(timeout = 15000) - public void testConditionalDatabaseOnlyStopRejectsCommittedResume() throws Exception { - this.assertConditionalTransitionWinner(false); - } - - @Test(timeout = 15000) - public void testConditionalDatabaseOnlyPauseRejectsCommittedStop() throws Exception { - this.assertConditionalTransitionWinner(true, true); - } - - private void assertConditionalTransitionWinner(boolean stopWins) throws Exception { - this.assertConditionalTransitionWinner(stopWins, false); - } - - private void assertConditionalTransitionWinner(boolean stopWins, boolean pauseLoser) throws Exception { - org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = - new org.springframework.jdbc.datasource.DriverManagerDataSource(); - dataSource.setDriverClassName("org.h2.Driver"); - dataSource.setUrl("jdbc:h2:mem:state-winner-" + java.util.UUID.randomUUID()); - org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(dataSource); - CountDownLatch selected = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - java.util.concurrent.atomic.AtomicReference failure = - new java.util.concurrent.atomic.AtomicReference<>(); - java.util.concurrent.atomic.AtomicReference snapshot = - new java.util.concurrent.atomic.AtomicReference<>(); - Map owners = new ConcurrentHashMap<>(); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - try (java.sql.Connection keeper = dataSource.getConnection()) { - jdbc.execute("CREATE TABLE state_transition (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); - jdbc.update("INSERT INTO state_transition VALUES (9, ?)", - (pauseLoser ? LoadStatus.RUNNING : LoadStatus.PAUSED).getValue()); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.selectById(9)).thenAnswer(invocation -> { - byte state = jdbc.queryForObject("SELECT load_status FROM state_transition WHERE id=9", Byte.class); - LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) - .filter(value -> value.getValue() == state).findFirst().get(); - LoadTask task = LoadTask.builder().id(9).options(new LoadOptions()) - .status(status).lastDuration(0L).build(); - if (Thread.currentThread().getName().equals("delayed-transition")) { - snapshot.set(task); - selected.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - } - return task; - }); - Mockito.when(mapper.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - return jdbc.update("UPDATE state_transition SET load_status=? WHERE id=?", - task.getStatus().getValue(), task.getId()); - }); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = - invocation.getArgument(1); - Assert.assertTrue(condition.getSqlSegment().contains("id =")); - Assert.assertTrue(condition.getSqlSegment().contains("load_status =")); - Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Byte).findFirst().get(); - Assert.assertTrue(condition.getParamNameValuePairs().containsValue(9)); - return jdbc.update("UPDATE state_transition SET load_status=? WHERE id=? AND load_status=?", - task.getStatus().getValue(), task.getId(), expected); - }); - LoadTaskService service = new LoadTaskService(); - set(service, "mapper", mapper); - set(service, "taskExecutor", executor); - set(service, "runningTaskContainer", owners); - Thread delayed = new Thread(() -> { - try (MockedConstruction construction = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, ignored) -> { - LoadContext context = Mockito.mock(LoadContext.class); - Mockito.when(loader.context()).thenReturn(context); - Mockito.when(context.closed()).thenReturn(true); - })) { - if (pauseLoser) { - service.pause(9); - } else if (stopWins) { - service.resume(9, "session-token"); - } else { - service.stop(9); - } - } catch (Throwable error) { - failure.set(error); - } - }, "delayed-transition"); - try (MockedConstruction construction = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, ignored) -> { - LoadContext context = Mockito.mock(LoadContext.class); - Mockito.when(loader.context()).thenReturn(context); - Mockito.when(context.closed()).thenReturn(true); - })) { - try { - delayed.start(); - Assert.assertTrue(selected.await(5, TimeUnit.SECONDS)); - LoadTaskService competing = peer(service); - LoadTask winner = stopWins ? competing.stop(9) : competing.resume(9, "session-token"); - LoadStatus expected = stopWins ? LoadStatus.STOPPED : LoadStatus.RUNNING; - Assert.assertEquals(expected, winner.getStatus()); - release.countDown(); - delayed.join(5000); - Assert.assertFalse(delayed.isAlive()); - Assert.assertTrue(failure.get() instanceof org.apache.hugegraph.exception.ExternalException); - Assert.assertTrue(failure.get().getMessage().contains("state has changed")); - Assert.assertEquals(expected.getValue(), jdbc.queryForObject( - "SELECT load_status FROM state_transition WHERE id=9", Byte.class)); - Assert.assertNotSame(winner, snapshot.get()); - if (stopWins) { - if (!pauseLoser) { - Assert.assertEquals(LoadStatus.FAILED, snapshot.get().getStatus()); - Assert.assertTrue(snapshot.get().isFinished()); - Assert.assertTrue(snapshot.get().context().closed()); - } - Assert.assertTrue(owners.isEmpty()); - Mockito.verifyNoInteractions(executor); - } else { - Assert.assertSame(winner, owners.get(9)); - Mockito.verify(executor).execute(Mockito.eq(winner), Mockito.any()); - } - } finally { - release.countDown(); - delayed.join(5000); - for (HugeGraphLoader loader : construction.constructed()) { - loader.shutdown(); - } - } - } - } - } - - @Test - public void testUnscheduledCleanupFailureRetainsOwnerAndRejectsResume() throws Exception { - LoadTask task = LoadTask.builder().id(9).options(new LoadOptions()).status(LoadStatus.PAUSED).build(); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.selectById(9)).thenReturn(task); - RuntimeException primary = new RuntimeException("save rejected"); - RuntimeException cleanup = new RuntimeException("resources still open"); - Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenThrow(primary); - set(service, "mapper", mapper); - Map owners = new ConcurrentHashMap<>(); - set(service, "runningTaskContainer", owners); - try (MockedConstruction construction = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, ignored) -> { - LoadContext context = Mockito.mock(LoadContext.class); - Mockito.when(loader.context()).thenReturn(context); - Mockito.when(context.closed()).thenReturn(false); - Mockito.doThrow(cleanup).when(loader).shutdown(); - })) { - try { - service.resume(9, "session-token"); - Assert.fail("save must fail"); - } catch (RuntimeException actual) { - Assert.assertSame(primary, actual); - } - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - Assert.assertTrue(task.isFinished()); - Assert.assertFalse(task.context().closed()); - Assert.assertSame(task, owners.get(9)); - Assert.assertArrayEquals(new Throwable[]{cleanup}, primary.getSuppressed()); - try { - service.resume(9, "session-token"); - Assert.fail("the old Loader must close before a new one is constructed"); - } catch (RuntimeException expected) { - Assert.assertTrue(expected.getMessage().contains("previous Loader has not closed")); - } - Assert.assertEquals(1, construction.constructed().size()); - Assert.assertSame(task, owners.get(9)); - } - } - - @Test(timeout = 15000) - public void testCapturedStopFinishesWhenResumeSaveFailsBeforeDispatch() throws Exception { - LoadTask restored = LoadTask.builder().id(9).options(new LoadOptions()) - .status(LoadStatus.PAUSED).lastDuration(0L).build(); - CountDownLatch updating = new CountDownLatch(1); - CountDownLatch selected = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - Map owners = new ConcurrentHashMap() { - @Override - public LoadTask get(Object id) { - LoadTask owner = super.get(id); - if (owner != null && Thread.currentThread().getName().equals("unscheduled-stop")) { - selected.countDown(); - } - return owner; - } - }; - RuntimeException primary = new RuntimeException("resume update rejected"); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.selectById(9)).thenReturn(restored); - Mockito.when(mapper.update(Mockito.eq(restored), Mockito.any())).thenAnswer(invocation -> { - updating.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - throw primary; - }); - LoadTaskService service = new LoadTaskService(); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - set(service, "mapper", mapper); - set(service, "taskExecutor", executor); - set(service, "runningTaskContainer", owners); - java.util.concurrent.atomic.AtomicReference resumeFailure = - new java.util.concurrent.atomic.AtomicReference<>(); - java.util.concurrent.atomic.AtomicReference stopFailure = - new java.util.concurrent.atomic.AtomicReference<>(); - Thread resume = new Thread(() -> { - try (MockedConstruction construction = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, ignored) -> { - LoadContext context = Mockito.mock(LoadContext.class); - Mockito.when(loader.context()).thenReturn(context); - Mockito.when(context.closed()).thenReturn(true); - })) { - service.resume(9, "session-token"); - } catch (Throwable failure) { - resumeFailure.set(failure); - } - }, "unscheduled-resume"); - Thread stop = new Thread(() -> { - try { - service.stop(9); - } catch (Throwable failure) { - stopFailure.set(failure); - } - }, "unscheduled-stop"); - stop.setDaemon(true); - try { - resume.start(); - Assert.assertTrue(updating.await(5, TimeUnit.SECONDS)); - stop.start(); - Assert.assertFalse("Stop waits for admission before selecting the unscheduled owner", - selected.await(300, TimeUnit.MILLISECONDS)); - Assert.assertSame(restored, owners.get(9)); - release.countDown(); - resume.join(5000); - stop.join(1500); - Assert.assertFalse(resume.isAlive()); - Assert.assertFalse("No unscheduled runner remains to finish this stop waiter", stop.isAlive()); - Assert.assertSame(primary, resumeFailure.get()); - Assert.assertNotNull("FAILED owners cannot accept a stop request", stopFailure.get()); - Assert.assertEquals(LoadStatus.FAILED, restored.getStatus()); - Assert.assertTrue(restored.isFinished()); - Assert.assertTrue(restored.context().closed()); - Assert.assertFalse(owners.containsKey(9)); - Mockito.verifyNoInteractions(executor); - } finally { - release.countDown(); - restored.setFinished(true); - resume.join(5000); - stop.join(5000); - } - } - - @Test - public void testPublishedRestartIsLockedUntilInitialized() throws Exception { - LoadTask restored = LoadTask.builder().id(9).options(new LoadOptions()) - .status(LoadStatus.PAUSED).lastDuration(0L).build(); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - Mockito.when(mapper.selectById(9)).thenReturn(restored); - Mockito.when(mapper.updateById(restored)).thenReturn(1); - set(service, "mapper", mapper); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - set(service, "taskExecutor", executor); - java.util.concurrent.atomic.AtomicBoolean locked = new java.util.concurrent.atomic.AtomicBoolean(); - Map owners = new ConcurrentHashMap() { - @Override - public LoadTask putIfAbsent(Integer id, LoadTask task) { - LoadTask previous = super.putIfAbsent(id, task); - if (previous == null) { - Thread observer = new Thread(() -> { - try { - Field field = LoadTask.class.getDeclaredField("lock"); - field.setAccessible(true); - java.util.concurrent.locks.ReentrantLock lock = - (java.util.concurrent.locks.ReentrantLock) field.get(task); - boolean acquired = lock.tryLock(); - if (acquired) { - lock.unlock(); - } - locked.set(!acquired); - } catch (ReflectiveOperationException e) { - throw new AssertionError(e); - } - }); - observer.start(); - try { - observer.join(5000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new AssertionError(e); - } - Assert.assertFalse(observer.isAlive()); - } - return previous; - } - }; - set(service, "runningTaskContainer", owners); - try (MockedConstruction construction = Mockito.mockConstruction(HugeGraphLoader.class)) { - service.resume(9, "session-token"); - Assert.assertTrue("The published owner must already hold its task lock", locked.get()); - Assert.assertSame(restored, owners.get(9)); - Assert.assertSame(construction.constructed().get(0), restored.getLoader()); - Assert.assertEquals(LoadStatus.RUNNING, restored.getStatus()); - Mockito.verify(executor).execute(Mockito.eq(restored), Mockito.any()); - } - } - - @Test - public void testClosedCallbackKeepsEqualButDistinctOwner() throws Exception { - LoadTask old = this.task(this.loader()); - old.setId(9); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.updateById(old)).thenReturn(1); - set(service, "mapper", mapper); - Map owners = new ConcurrentHashMap<>(); - set(service, "runningTaskContainer", owners); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - java.util.concurrent.atomic.AtomicReference callback = - new java.util.concurrent.atomic.AtomicReference<>(); - Mockito.doAnswer(invocation -> { - callback.set(invocation.getArgument(1)); - return null; - }).when(executor).execute(Mockito.eq(old), Mockito.any()); - set(service, "taskExecutor", executor); - dispatch(service, old); - old.run(); - LoadTask replacement = this.equalRuntimeReplacement(old); - owners.put(9, replacement); - callback.get().run(); - Assert.assertEquals(old, replacement); - Assert.assertSame(replacement, owners.get(9)); - Mockito.verify(mapper).updateById(old); - } - - @Test - public void testNoTransactionRemovalKeepsEqualButDistinctOwner() throws Exception { - LoadTask old = this.task(this.loader()); - old.setId(9); - old.run(); - LoadTask replacement = this.equalRuntimeReplacement(old); - LoadTaskService service = new LoadTaskService(); - Map owners = new ConcurrentHashMap<>(); - owners.put(9, replacement); - set(service, "runningTaskContainer", owners); - Assert.assertFalse(TransactionSynchronizationManager.isSynchronizationActive()); - Method remove = LoadTaskService.class.getDeclaredMethod("removeOwnerAfterCommit", LoadTask.class); - remove.setAccessible(true); - remove.invoke(service, old); - Assert.assertSame(replacement, owners.get(9)); - } - - @Test - public void testCommittedPollKeepsEqualButDistinctOwner() throws Exception { - this.assertPollCommitBoundary(false, true); - } - - private LoadTask equalRuntimeReplacement(LoadTask previous) { - HugeGraphLoader loader = this.loader(); - Mockito.when(loader.context().closed()).thenReturn(false); - LoadTask replacement = LoadTask.builder().id(previous.getId()).options(previous.getOptions()) - .loader(loader).finished(false).status(previous.getStatus()).fileReadLines(previous.getFileReadLines()) - .lastDuration(previous.getLastDuration()).currDuration(previous.getCurrDuration()).build(); - Assert.assertEquals(previous, replacement); - Assert.assertNotSame(previous, replacement); - Assert.assertNotSame(previous.getLoader(), replacement.getLoader()); - Assert.assertTrue(previous.isFinished()); - Assert.assertFalse(replacement.isFinished()); - return replacement; - } - - @Test - public void testPollTransactionRollbackRetainsBothFinishedOwners() throws Exception { - this.assertPollCommitBoundary(false); - } - - @Test - public void testPollCommitFailureRetainsBothFinishedOwners() throws Exception { - this.assertPollCommitBoundary(true); - } - - private void assertPollCommitBoundary(boolean commitFailure) throws Exception { - this.assertPollCommitBoundary(commitFailure, false); - } - - private void assertPollCommitBoundary(boolean commitFailure, boolean replaceBeforeCommit) throws Exception { - org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = - new org.springframework.jdbc.datasource.DriverManagerDataSource(); - dataSource.setDriverClassName("org.h2.Driver"); - dataSource.setUrl("jdbc:h2:mem:cleanup-poll-" + java.util.UUID.randomUUID()); - org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(dataSource); - try (java.sql.Connection keeper = dataSource.getConnection()) { - jdbc.execute("CREATE TABLE task_progress (id INT PRIMARY KEY, load_status VARCHAR(16))"); - jdbc.update("INSERT INTO task_progress VALUES (1, 'RUNNING'), (2, 'RUNNING')"); - LoadTask first = this.task(this.loader()); - LoadTask second = this.task(this.loader()); - first.setId(1); - second.setId(2); - first.run(); - second.run(); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenReturn(1); - java.util.concurrent.atomic.AtomicBoolean fail = new java.util.concurrent.atomic.AtomicBoolean(true); - RuntimeException failure = new RuntimeException("later owner SQL failed"); - Mockito.when(mapper.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - if (!commitFailure && task == second && fail.get()) { - throw failure; - } - return jdbc.update("UPDATE task_progress SET load_status=? WHERE id=?", - task.getStatus().name(), task.getId()); - }); - Map owners = new java.util.concurrent.ConcurrentSkipListMap<>(); - owners.put(1, first); - owners.put(2, second); - set(service, "mapper", mapper); - set(service, "runningTaskContainer", owners); - org.springframework.jdbc.datasource.DataSourceTransactionManager manager = - new org.springframework.jdbc.datasource.DataSourceTransactionManager(dataSource) { - @Override - protected void doCommit( - org.springframework.transaction.support.DefaultTransactionStatus status) { - if (commitFailure && fail.get()) { - throw failure; - } - super.doCommit(status); - } - }; - manager.setRollbackOnCommitFailure(true); - set(service, "transactionManager", manager); - org.springframework.transaction.support.TransactionTemplate transaction = - new org.springframework.transaction.support.TransactionTemplate(manager); - try { - transaction.execute(status -> { - service.updateLoadTaskProgress(); - if (!commitFailure) { - status.setRollbackOnly(); - } - return null; - }); - if (commitFailure) { - Assert.fail("commit must fail"); - } - } catch (RuntimeException actual) { - Assert.assertSame(failure, actual); - } - Assert.assertEquals("RUNNING", jdbc.queryForObject( - "SELECT load_status FROM task_progress WHERE id=1", String.class)); - Assert.assertEquals("RUNNING", jdbc.queryForObject( - "SELECT load_status FROM task_progress WHERE id=2", String.class)); - Assert.assertSame(first, owners.get(1)); - Assert.assertSame(second, owners.get(2)); - fail.set(false); - LoadTask replacement = replaceBeforeCommit ? this.equalRuntimeReplacement(first) : null; - transaction.execute(status -> { - service.updateLoadTaskProgress(); - Assert.assertSame(first, owners.get(1)); - Assert.assertSame(second, owners.get(2)); - if (replaceBeforeCommit) { - owners.put(1, replacement); - } - return null; - }); - if (replaceBeforeCommit) { - Assert.assertSame(replacement, owners.get(1)); - Assert.assertNull(owners.get(2)); - } else { - Assert.assertTrue(owners.isEmpty()); - } - Assert.assertEquals("FAILED", jdbc.queryForObject( - "SELECT load_status FROM task_progress WHERE id=1", String.class)); - Assert.assertEquals("FAILED", jdbc.queryForObject( - "SELECT load_status FROM task_progress WHERE id=2", String.class)); - } - } - - @Test(timeout = 15000) - public void testTaskAdmissionPreventsPauseResumeStatusAba() throws Exception { - org.springframework.jdbc.datasource.DriverManagerDataSource ds = - new org.springframework.jdbc.datasource.DriverManagerDataSource(); - ds.setDriverClassName("org.h2.Driver"); - ds.setUrl("jdbc:h2:mem:admission-" + java.util.UUID.randomUUID()); - org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(ds); - CountDownLatch saving = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - CountDownLatch competitors = new CountDownLatch(2); - java.util.concurrent.atomic.AtomicInteger reads = new java.util.concurrent.atomic.AtomicInteger(); - java.util.concurrent.atomic.AtomicReference stopFailure = - new java.util.concurrent.atomic.AtomicReference<>(); - java.util.concurrent.atomic.AtomicReference pauseFailure = - new java.util.concurrent.atomic.AtomicReference<>(); - java.util.concurrent.atomic.AtomicReference resumeFailure = - new java.util.concurrent.atomic.AtomicReference<>(); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - Map owners = new ConcurrentHashMap<>(); - set(service, "mapper", mapper); - set(service, "taskExecutor", executor); - set(service, "runningTaskContainer", owners); - try (java.sql.Connection keeper = ds.getConnection()) { - jdbc.execute("CREATE TABLE admission (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); - jdbc.update("INSERT INTO admission VALUES (9, ?)", LoadStatus.RUNNING.getValue()); - Mockito.when(mapper.selectById(9)).thenAnswer(invocation -> { - reads.incrementAndGet(); - byte code = jdbc.queryForObject("SELECT load_status FROM admission WHERE id=9", Byte.class); - LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) - .filter(value -> value.getValue() == code).findFirst().get(); - return LoadTask.builder().id(9).options(new LoadOptions()).status(status).build(); - }); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = - invocation.getArgument(1); - condition.getSqlSegment(); - Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Byte).findFirst().get(); - saving.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - return jdbc.update("UPDATE admission SET load_status=? WHERE id=? AND load_status=?", - task.getStatus().getValue(), task.getId(), expected); - }); - Thread stop = new Thread(() -> { - try { - service.stop(9); - } catch (Throwable error) { - stopFailure.set(error); - } - }); - Thread pause = new Thread(() -> { - competitors.countDown(); - try { - service.pause(9); - } catch (Throwable error) { - pauseFailure.set(error); - } - }); - Thread resume = new Thread(() -> { - competitors.countDown(); - try { - service.resume(9, "session-token"); - } catch (Throwable error) { - resumeFailure.set(error); - } - }); - try { - stop.start(); - Assert.assertTrue(saving.await(5, TimeUnit.SECONDS)); - pause.start(); - resume.start(); - Assert.assertTrue(competitors.await(5, TimeUnit.SECONDS)); - Thread.sleep(100); - Assert.assertTrue(pause.isAlive()); - Assert.assertTrue(resume.isAlive()); - Assert.assertEquals("Later controls cannot read an ABA snapshot", 1, reads.get()); - Assert.assertEquals(LoadStatus.RUNNING.getValue(), jdbc.queryForObject( - "SELECT load_status FROM admission WHERE id=9", Byte.class)); - Mockito.verifyNoInteractions(executor); - } finally { - release.countDown(); - stop.join(5000); - pause.join(5000); - resume.join(5000); - } - Assert.assertFalse(stop.isAlive()); - Assert.assertFalse(pause.isAlive()); - Assert.assertFalse(resume.isAlive()); - Assert.assertNull(stopFailure.get()); - Assert.assertTrue(pauseFailure.get() instanceof org.apache.hugegraph.exception.ExternalException); - Assert.assertTrue(resumeFailure.get() instanceof org.apache.hugegraph.exception.ExternalException); - Assert.assertEquals(LoadStatus.STOPPED.getValue(), jdbc.queryForObject( - "SELECT load_status FROM admission WHERE id=9", Byte.class)); - Assert.assertTrue(owners.isEmpty()); - Mockito.verifyNoInteractions(executor); - } - } - - @Test(timeout = 15000) - public void testInitialPublicationWaitsForTaskAdmissionAfterCommit() throws Exception { - LoadTaskService service = new LoadTaskService(); - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - set(service, "taskExecutor", executor); - Map owners = new ConcurrentHashMap<>(); - set(service, "runningTaskContainer", owners); - Field field = LoadTaskService.class.getDeclaredField("taskControls"); - field.setAccessible(true); - com.google.common.util.concurrent.Striped controls = - (com.google.common.util.concurrent.Striped) field.get(service); - java.util.concurrent.locks.Lock control = controls.get(9); - CountDownLatch committed = new CountDownLatch(1); - java.util.concurrent.atomic.AtomicReference failure = - new java.util.concurrent.atomic.AtomicReference<>(); - Thread publication = new Thread(() -> { - TransactionSynchronizationManager.initSynchronization(); - try { - dispatch(service, task); - TransactionSynchronization synchronization = - TransactionSynchronizationManager.getSynchronizations().get(0); - committed.countDown(); - synchronization.afterCommit(); - synchronization.afterCompletion(TransactionSynchronization.STATUS_COMMITTED); - } catch (Throwable error) { - failure.set(error); - } finally { - TransactionSynchronizationManager.clearSynchronization(); - } - }); - control.lock(); - try { - publication.start(); - Assert.assertTrue(committed.await(5, TimeUnit.SECONDS)); - Thread.sleep(100); - Assert.assertTrue(publication.isAlive()); - Assert.assertTrue(owners.isEmpty()); - Mockito.verifyNoInteractions(executor); - int independentId = 10; - while (controls.get(independentId) == control) { - independentId++; - } - LoadTask independent = this.task(this.loader()); - independent.setId(independentId); - dispatch(service, independent); - Assert.assertSame(independent, owners.get(independentId)); - Mockito.verify(executor).execute(Mockito.eq(independent), Mockito.any()); - } finally { - control.unlock(); - publication.join(5000); - loader.shutdown(); - } - Assert.assertFalse(publication.isAlive()); - Assert.assertNull(failure.get()); - Assert.assertSame(task, owners.get(9)); - Mockito.verify(executor).execute(Mockito.eq(task), Mockito.any()); - } - - @Test(timeout = 15000) - public void testInitialPublicationHonorsCommittedStop() throws Exception { - this.assertInitialPublicationHonorsTerminalRequest(false, false); - } - - @Test(timeout = 15000) - public void testInitialPublicationHonorsCommittedPause() throws Exception { - this.assertInitialPublicationHonorsTerminalRequest(true, false); - } - - @Test(timeout = 15000) - public void testCancelledInitialPublicationRetainsUnclosedOwner() throws Exception { - this.assertInitialPublicationHonorsTerminalRequest(false, true); - } - - @Test(timeout = 15000) - public void testCancelledInitialPublicationRetainsStillOpenContext() throws Exception { - this.assertInitialPublicationHonorsTerminalRequest(false, true, false); - } - - private void assertInitialPublicationHonorsTerminalRequest(boolean pause, boolean cleanupFailure) throws Exception { - this.assertInitialPublicationHonorsTerminalRequest(pause, cleanupFailure, true); - } - - private void assertInitialPublicationHonorsTerminalRequest(boolean pause, boolean cleanupFailure, - boolean throwsOnCleanup) throws Exception { - org.springframework.jdbc.datasource.DriverManagerDataSource ds = - new org.springframework.jdbc.datasource.DriverManagerDataSource(); - ds.setDriverClassName("org.h2.Driver"); - ds.setUrl("jdbc:h2:mem:late-publication-" + java.util.UUID.randomUUID()); - org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(ds); - HugeGraphLoader loader = this.loader(); - Mockito.when(loader.context().closed()).thenReturn(!cleanupFailure); - if (cleanupFailure && throwsOnCleanup) { - Mockito.doThrow(new RuntimeException("worker cleanup failed")).when(loader).shutdown(); - } - LoadTask initial = this.task(loader); - initial.setId(9); - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - Map owners = new ConcurrentHashMap<>(); - set(service, "mapper", mapper); - set(service, "taskExecutor", executor); - set(service, "runningTaskContainer", owners); - CountDownLatch registered = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - java.util.concurrent.atomic.AtomicReference failure = - new java.util.concurrent.atomic.AtomicReference<>(); - try (java.sql.Connection keeper = ds.getConnection()) { - jdbc.execute("CREATE TABLE initial_task (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); - jdbc.update("INSERT INTO initial_task VALUES (9, ?)", LoadStatus.RUNNING.getValue()); - Mockito.when(mapper.selectById(9)).thenAnswer(invocation -> { - byte code = jdbc.queryForObject("SELECT load_status FROM initial_task WHERE id=9", Byte.class); - LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) - .filter(value -> value.getValue() == code).findFirst().get(); - return LoadTask.builder().id(9).options(new LoadOptions()).status(status).build(); - }); - Mockito.when(mapper.update(Mockito.any(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = - invocation.getArgument(1); - condition.getSqlSegment(); - Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Byte).findFirst().get(); - return jdbc.update("UPDATE initial_task SET load_status=? WHERE id=? AND load_status=?", - task.getStatus().getValue(), task.getId(), expected); - }); - Thread publication = new Thread(() -> { - TransactionSynchronizationManager.initSynchronization(); - try { - dispatch(service, initial); - TransactionSynchronization synchronization = - TransactionSynchronizationManager.getSynchronizations().get(0); - registered.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - synchronization.afterCommit(); - synchronization.afterCompletion(TransactionSynchronization.STATUS_COMMITTED); - } catch (Throwable error) { - failure.set(error); - } finally { - TransactionSynchronizationManager.clearSynchronization(); - } - }); - try { - publication.start(); - Assert.assertTrue(registered.await(5, TimeUnit.SECONDS)); - LoadTask terminal = pause ? service.pause(9) : service.stop(9); - Assert.assertEquals(pause ? LoadStatus.PAUSED : LoadStatus.STOPPED, terminal.getStatus()); - Assert.assertNotSame(initial, terminal); - } finally { - release.countDown(); - publication.join(5000); - } - Assert.assertFalse(publication.isAlive()); - Assert.assertNull(failure.get()); - LoadStatus expected = pause ? LoadStatus.PAUSED : LoadStatus.STOPPED; - Assert.assertEquals(expected.getValue(), jdbc.queryForObject( - "SELECT load_status FROM initial_task WHERE id=9", Byte.class)); - Assert.assertEquals(cleanupFailure ? LoadStatus.FAILED : expected, initial.getStatus()); - Assert.assertTrue(initial.isFinished()); - Mockito.verify(loader).shutdown(); - Mockito.verify(loader, Mockito.never()).load(); - Mockito.verifyNoInteractions(executor); - if (cleanupFailure) { - Assert.assertSame(initial, owners.get(9)); - Assert.assertFalse(initial.context().closed()); - } else { - Assert.assertTrue(initial.context().closed()); - Assert.assertTrue(owners.isEmpty()); - } - } - } - - @Test - public void testStaleInitialPublicationDoesNotDispatchMissingOrTerminalRow() throws Exception { - for (LoadStatus status : new LoadStatus[]{null, LoadStatus.FAILED, LoadStatus.SUCCEED, LoadStatus.INIT}) { - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.selectById(9)).thenReturn(status == null ? null : - LoadTask.builder().id(9).status(status).build()); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - HugeGraphLoader loader = this.loader(); - LoadTask old = this.task(loader); - old.setId(9); - set(service, "mapper", mapper); - set(service, "taskExecutor", executor); - dispatch(service, old); - Assert.assertTrue(old.isFinished()); - Assert.assertTrue(loader.context().closed()); - Mockito.verify(loader, Mockito.never()).load(); - Mockito.verify(mapper, Mockito.never()).updateById(Mockito.any(LoadTask.class)); - Mockito.verify(mapper, Mockito.never()).update(Mockito.any(), Mockito.any()); - Mockito.verifyNoInteractions(executor); - } - } - - @Test - public void testStaleInitialPublicationClosesOnlyItsOwnLoader() throws Exception { - LoadTaskService service = new LoadTaskService(); - HugeGraphLoader oldLoader = this.loader(); - HugeGraphLoader newLoader = this.loader(); - LoadTask old = this.task(oldLoader); - old.setId(9); - LoadTask fresh = LoadTask.builder().id(9).options(old.getOptions()).lastDuration(0L) - .status(LoadStatus.RUNNING).loader(newLoader).build(); - Assert.assertEquals(old, fresh); - Assert.assertNotSame(old, fresh); - Map owners = new ConcurrentHashMap<>(); - owners.put(9, fresh); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.selectById(9)).thenReturn(fresh); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - set(service, "mapper", mapper); - set(service, "taskExecutor", executor); - set(service, "runningTaskContainer", owners); - dispatch(service, old); - Assert.assertSame(fresh, owners.get(9)); - Assert.assertEquals(LoadStatus.RUNNING, fresh.getStatus()); - Assert.assertFalse(fresh.isFinished()); - Assert.assertFalse(newLoader.context().closed()); - Assert.assertTrue(old.isFinished()); - Assert.assertTrue(oldLoader.context().closed()); - Mockito.verify(newLoader, Mockito.never()).shutdown(); - Mockito.verifyNoInteractions(executor); - Mockito.verify(mapper, Mockito.never()).update(Mockito.any(), Mockito.any()); - Mockito.verify(mapper, Mockito.never()).updateById(Mockito.any(LoadTask.class)); - } - - @Test - public void testInitialPublicationRejectsUnusableOwnLoader() throws Exception { - for (int state = 0; state < 4; state++) { - LoadTaskService service = new LoadTaskService(); - HugeGraphLoader loader = this.loader(); - LoadTask old = this.task(loader); - old.setId(9); - if (state == 0) { - old.setFinished(true); - } else if (state == 1) { - old.setLoader(null); - } else if (state == 2) { - Mockito.when(loader.context().closed()).thenReturn(true); - } else { - Mockito.when(loader.context().stopped()).thenReturn(true); - } - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.selectById(9)).thenReturn(LoadTask.builder().id(9).status(LoadStatus.RUNNING).build()); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - set(service, "mapper", mapper); - set(service, "taskExecutor", executor); - dispatch(service, old); - Assert.assertTrue(old.isFinished()); - Mockito.verifyNoInteractions(executor); - Mockito.verify(mapper, Mockito.never()).update(Mockito.any(), Mockito.any()); - } - } - - @Test - public void testDeleteRejectsTerminalSnapshotWithUnclosedOwner() throws Exception { - LoadTaskService service = new LoadTaskService(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.selectById(9)).thenReturn(LoadTask.builder().id(9).status(LoadStatus.PAUSED).build()); - LoadTask owner = this.task(this.loader()); - owner.setId(9); - Map owners = new ConcurrentHashMap<>(); - owners.put(9, owner); - set(service, "mapper", mapper); - set(service, "runningTaskContainer", owners); - org.apache.hugegraph.testutil.Assert.assertThrows(RuntimeException.class, () -> service.remove(9)); - Assert.assertSame(owner, owners.get(9)); - Mockito.verify(mapper, Mockito.never()).delete(Mockito.any()); - } - - @Test - public void testGraphDeletionCommitsTaskCleanupOutsideCallerTransaction() throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - fixture.enableGraphDeletion(); - new org.springframework.transaction.support.TransactionTemplate(fixture.manager).execute(status -> { - fixture.service.removeByGraph("space", "graph"); - Assert.assertTrue(TransactionSynchronizationManager.isActualTransactionActive()); - try (java.sql.Connection other = fixture.dataSource.getConnection(); - java.sql.Statement statement = other.createStatement(); - java.sql.ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM delete_tasks")) { - Assert.assertTrue(rows.next()); - Assert.assertEquals(0, rows.getInt(1)); - } catch (java.sql.SQLException failure) { - throw new RuntimeException(failure); - } - status.setRollbackOnly(); - return null; - }); - Assert.assertEquals(Integer.valueOf(0), fixture.jdbc.queryForObject( - "SELECT COUNT(*) FROM delete_jobs", Integer.class)); - Mockito.verify(fixture.jobs, Mockito.never()).deleteByMap(Mockito.anyMap()); - } - } - - @Test - public void testGraphDeletionKeepsNewJobsAndOtherGraphScopes() throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - fixture.enableGraphDeletion(); - fixture.jdbc.update("INSERT INTO delete_jobs VALUES (8, ?, 'other-space', 'graph'), " + - "(9, ?, 'space', 'other-graph')", 0, 0); - Mockito.doAnswer(invocation -> { - java.util.List snapshot = - Collections.singletonList(fixture.jobs.selectById(7)); - fixture.jdbc.update("INSERT INTO delete_jobs VALUES (11, ?, 'space', 'graph')", 0); - fixture.jdbc.update("INSERT INTO delete_tasks VALUES (12, ?, 11)", LoadStatus.RUNNING.getValue()); - return snapshot; - }).when(fixture.jobs).selectList(Mockito.any()); - - fixture.service.removeByGraph("space", "graph"); - - Assert.assertEquals(java.util.Arrays.asList(8, 9, 11), fixture.jdbc.queryForList( - "SELECT id FROM delete_jobs ORDER BY id", Integer.class)); - Assert.assertEquals(Collections.singletonList(12), fixture.jdbc.queryForList( - "SELECT id FROM delete_tasks", Integer.class)); - Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=12", Byte.class)); - Mockito.verify(fixture.jobs, Mockito.never()).deleteByMap(Mockito.anyMap()); - } - } - - @Test - public void testGraphDeletionFailureRetainsFailedJobAndCommittedStops() throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - fixture.enableGraphDeletion(); - fixture.jdbc.update("INSERT INTO delete_jobs VALUES (8, ?, 'space', 'graph')", 0); - fixture.jdbc.update("INSERT INTO delete_tasks VALUES (11, ?, 8)", LoadStatus.RUNNING.getValue()); - RuntimeException refusal = new IllegalStateException("metadata deletion rejected"); - Mockito.doThrow(refusal).when(fixture.jobs).deleteById(8); - - org.apache.hugegraph.testutil.Assert.assertThrows(IllegalStateException.class, - () -> fixture.service.removeByGraph("space", "graph")); - - Assert.assertEquals(Collections.singletonList(8), fixture.jdbc.queryForList( - "SELECT id FROM delete_jobs", Integer.class)); - Assert.assertEquals(Collections.singletonList(11), fixture.jdbc.queryForList( - "SELECT id FROM delete_tasks", Integer.class)); - Assert.assertEquals(LoadStatus.STOPPED.getValue(), fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=11", Byte.class)); - } - } - - @Test - public void testJobDeletionCommitsStopsBeforeCallerTransactionCompletes() throws Exception { - for (boolean scoped : new boolean[]{false, true}) { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - new org.springframework.transaction.support.TransactionTemplate(fixture.manager).execute(status -> { - fixture.delete(scoped); - Assert.assertTrue(TransactionSynchronizationManager.isActualTransactionActive()); - // Independent connections must see committed deletion before the caller commits. - try (java.sql.Connection other = fixture.dataSource.getConnection(); - java.sql.Statement statement = other.createStatement(); - java.sql.ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM delete_tasks")) { - Assert.assertTrue(rows.next()); - Assert.assertEquals(0, rows.getInt(1)); - } catch (java.sql.SQLException failure) { - throw new RuntimeException(failure); - } - status.setRollbackOnly(); - return null; - }); - Assert.assertEquals(0, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_tasks", Integer.class) - .intValue()); - Assert.assertEquals(0, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_jobs", Integer.class) - .intValue()); - } - } - } - - @Test - public void testMetadataRollbackKeepsCommittedStopsAndAllTaskRows() throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - RuntimeException primary = new RuntimeException("job metadata delete rejected"); - Mockito.doThrow(primary).when(fixture.jobs).deleteById(7); - try { - fixture.delete(true); - Assert.fail("metadata must roll back"); - } catch (RuntimeException actual) { - Assert.assertSame(primary, actual); - } - Assert.assertEquals(2, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_tasks", Integer.class) - .intValue()); - Assert.assertEquals(2, fixture.jdbc.queryForObject( - "SELECT COUNT(*) FROM delete_tasks WHERE load_status=?", Integer.class, - LoadStatus.STOPPED.getValue()).intValue()); - Assert.assertEquals(1, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_jobs", Integer.class) - .intValue()); - } - } - - @Test - public void testRetryWinningBeforeMetadataDeleteRollsBackEarlierTaskDeletion() throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - LoadTask fresh = this.task(this.loader()); - fresh.setId(9); - Mockito.doAnswer(invocation -> { - com.baomidou.mybatisplus.core.conditions.query.QueryWrapper condition = - invocation.getArgument(0); - condition.getSqlSegment(); - Integer id = (Integer) condition.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Integer).findFirst().get(); - Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Byte).findFirst().get(); - int deleted = fixture.jdbc.update("DELETE FROM delete_tasks WHERE id=? AND load_status=?", - id, expected); - if (id == 10) { - // A separately committed restart wins before the next task is admitted for deletion. - try (java.sql.Connection other = fixture.dataSource.getConnection(); - java.sql.PreparedStatement update = other.prepareStatement( - "UPDATE delete_tasks SET load_status=? WHERE id=9 AND load_status=?")) { - update.setByte(1, LoadStatus.RUNNING.getValue()); - update.setByte(2, LoadStatus.STOPPED.getValue()); - Assert.assertEquals(1, update.executeUpdate()); - } - fixture.owners.put(9, fresh); - } - return deleted; - }).when(fixture.tasks).delete(Mockito.any()); - org.apache.hugegraph.testutil.Assert.assertThrows(RuntimeException.class, () -> fixture.delete(false)); - Assert.assertSame(fresh, fixture.owners.get(9)); - Assert.assertFalse(fresh.context().closed()); - Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); - Assert.assertEquals(LoadStatus.STOPPED.getValue(), fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=10", Byte.class)); - Assert.assertEquals(1, fixture.jdbc.queryForObject("SELECT COUNT(*) FROM delete_jobs", Integer.class) - .intValue()); - } - } - - @Test - public void testDispatchCommitFailureKeepsClosedOwnerForConditionalRecovery() throws Exception { - for (LoadStatus advanced : new LoadStatus[]{LoadStatus.RUNNING, LoadStatus.SUCCEED, null}) { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - LoadTask task = this.task(this.loader()); - task.setId(9); - task.setJobId(7); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - Mockito.doThrow(new java.util.concurrent.RejectedExecutionException("queue full")) - .when(executor).execute(Mockito.any(), Mockito.any()); - org.springframework.jdbc.datasource.DataSourceTransactionManager rejecting = - new org.springframework.jdbc.datasource.DataSourceTransactionManager(fixture.dataSource) { - @Override - protected void doCommit( - org.springframework.transaction.support.DefaultTransactionStatus status) { - throw new org.springframework.transaction.TransactionSystemException("commit failed"); - } - }; - rejecting.setRollbackOnCommitFailure(true); - set(fixture.load, "taskExecutor", executor); - set(fixture.load, "transactionManager", rejecting); - set(fixture.load, "jobManagerMapper", fixture.jobs); - - dispatch(fixture.load, task); - - Assert.assertSame(task, fixture.owners.get(9)); - Assert.assertTrue(task.getLoader().context().closed()); - Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); - if (advanced == null) { - fixture.jdbc.update("DELETE FROM delete_tasks WHERE id=9"); - } else { - fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=9", advanced.getValue()); - } - set(fixture.load, "transactionManager", fixture.manager); - new org.springframework.transaction.support.TransactionTemplate(fixture.manager).execute(status -> { - fixture.load.updateLoadTaskProgress(); - Assert.assertSame("retire only after recovery commits", task, fixture.owners.get(9)); - return null; - }); - Assert.assertFalse(fixture.owners.containsKey(9)); - Assert.assertEquals(advanced == null ? Collections.emptyList() : Collections.singletonList( - advanced == LoadStatus.RUNNING ? LoadStatus.FAILED.getValue() : advanced.getValue()), - fixture.jdbc.queryForList("SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); - Assert.assertEquals(advanced == LoadStatus.RUNNING ? - org.apache.hugegraph.entity.enums.JobStatus.FAILED.getValue() : - org.apache.hugegraph.entity.enums.JobStatus.LOADING.getValue(), - fixture.jdbc.queryForObject("SELECT status FROM delete_jobs WHERE id=7", - Byte.class)); - } - } - } - - @Test - public void testPollSavepointRollbackFailureAbortsOuterTransaction() throws Exception { - org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = - new org.springframework.jdbc.datasource.DriverManagerDataSource( - "jdbc:h2:mem:poll-deadlock-" + java.util.UUID.randomUUID() + ";LOCK_TIMEOUT=3000", "sa", ""); - org.springframework.jdbc.core.JdbcTemplate jdbc = new org.springframework.jdbc.core.JdbcTemplate(dataSource); - org.springframework.jdbc.datasource.DataSourceTransactionManager manager = - new org.springframework.jdbc.datasource.DataSourceTransactionManager(dataSource); - try (java.sql.Connection keeper = dataSource.getConnection()) { - jdbc.execute("CREATE TABLE tasks (id INT PRIMARY KEY, load_status TINYINT, n INT)"); - jdbc.update("INSERT INTO tasks VALUES (9, ?, 0), (10, ?, 0), (11, ?, 0), (12, ?, 0)", - LoadStatus.RUNNING.getValue(), LoadStatus.RUNNING.getValue(), - LoadStatus.RUNNING.getValue(), LoadStatus.RUNNING.getValue()); - CountDownLatch otherLocked = new CountDownLatch(1); - CountDownLatch pollAtSecond = new CountDownLatch(1); - java.util.concurrent.atomic.AtomicReference otherFailure = - new java.util.concurrent.atomic.AtomicReference<>(); - Thread other = new Thread(() -> { - try { - new org.springframework.transaction.support.TransactionTemplate(manager).execute(status -> { - jdbc.update("UPDATE tasks SET n=1 WHERE id=10"); - jdbc.update("UPDATE tasks SET n=1 WHERE id=11"); - otherLocked.countDown(); - try { - Assert.assertTrue(pollAtSecond.await(10, TimeUnit.SECONDS)); - Thread.sleep(150); - } catch (InterruptedException failure) { - throw new RuntimeException(failure); - } - jdbc.update("UPDATE tasks SET n=1 WHERE id=9"); - return null; - }); - } catch (Throwable failure) { - otherFailure.set(failure); - } - }, "poll-deadlock-competitor"); - Map owners = new java.util.concurrent.ConcurrentSkipListMap<>(); - for (int id : new int[]{9, 10, 12}) { - owners.put(id, LoadTask.builder().id(id).finished(true).initialStateReadFailed(true) - .status(LoadStatus.FAILED).build()); - } - LoadTask first = owners.get(9); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.update(Mockito.nullable(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { - com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper query = invocation.getArgument(1); - query.getSqlSegment(); - int id = (Integer) query.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Integer).findFirst().get(); - if (id == 10) { - pollAtSecond.countDown(); - } - return jdbc.update("UPDATE tasks SET load_status=? WHERE id=? AND load_status=?", - LoadStatus.FAILED.getValue(), id, LoadStatus.RUNNING.getValue()); - }); - LoadTaskService service = new LoadTaskService(); - set(service, "mapper", mapper); - set(service, "transactionManager", manager); - set(service, "runningTaskContainer", owners); - org.springframework.aop.framework.ProxyFactory factory = - new org.springframework.aop.framework.ProxyFactory(service); - factory.setProxyTargetClass(true); - factory.addAdvice(new org.springframework.transaction.interceptor.TransactionInterceptor( - manager, new org.springframework.transaction.annotation.AnnotationTransactionAttributeSource())); - Throwable pollFailure = null; - other.start(); - try { - Assert.assertTrue(otherLocked.await(5, TimeUnit.SECONDS)); - try { - ((LoadTaskService) factory.getProxy()).updateLoadTaskProgress(); - } catch (Throwable failure) { - pollFailure = failure; - } - } finally { - pollAtSecond.countDown(); - other.join(5000); - } - Assert.assertFalse(other.isAlive()); - Assert.assertNull(otherFailure.get()); - Assert.assertTrue(pollFailure instanceof org.springframework.transaction.TransactionSystemException); - Assert.assertTrue(pollFailure.getCause() instanceof java.sql.SQLException); - Assert.assertEquals(90063, ((java.sql.SQLException) pollFailure.getCause()).getErrorCode()); - Assert.assertEquals(LoadStatus.RUNNING.getValue(), jdbc.queryForObject( - "SELECT load_status FROM tasks WHERE id=9", Byte.class)); - Assert.assertEquals(LoadStatus.RUNNING.getValue(), jdbc.queryForObject( - "SELECT load_status FROM tasks WHERE id=12", Byte.class)); - Assert.assertSame(first, owners.get(9)); - Assert.assertEquals(3, owners.size()); - Mockito.verify(mapper, Mockito.times(2)).update(Mockito.nullable(LoadTask.class), Mockito.any()); - } - } - - @Test - public void testPollMissingFinalRowDoesNotBlockOtherOwners() throws Exception { - this.assertPollFailureIsolation(true); - } - - @Test - public void testPollRollsBackFailedOwnerWriteAndContinuesOtherOwners() throws Exception { - this.assertPollFailureIsolation(false); - } - - @Test - public void testPollConstraintFailureContinuesOtherOwners() throws Exception { - this.assertPollFailureIsolation(false, true); - } - - private void assertPollFailureIsolation(boolean missing) throws Exception { - this.assertPollFailureIsolation(missing, false); - } - - private void assertPollFailureIsolation(boolean missing, boolean constraint) throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - LoadTask first = this.task(this.loader()); - first.setId(9); - first.run(); - LoadTask second = this.task(this.loader()); - second.setId(10); - second.run(); - Map owners = new java.util.concurrent.ConcurrentSkipListMap<>(); - owners.put(9, first); - owners.put(10, second); - set(fixture.load, "runningTaskContainer", owners); - set(fixture.load, "transactionManager", fixture.manager); - if (missing) { - fixture.jdbc.update("DELETE FROM delete_tasks WHERE id=9"); - } - if (constraint) { - fixture.jdbc.execute("ALTER TABLE delete_tasks ADD CONSTRAINT reject_first_failed CHECK " + - "(id <> 9 OR load_status <> " + LoadStatus.FAILED.getValue() + ")"); - } - Mockito.when(fixture.tasks.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - int count = fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=?", - task.getStatus().getValue(), task.getId()); - if (!missing && !constraint && task == first) { - throw new IllegalStateException("failed after partial SQL"); - } - return count; - }); - - new org.springframework.transaction.support.TransactionTemplate(fixture.manager).execute(status -> { - fixture.load.updateLoadTaskProgress(); - Assert.assertSame(second, owners.get(10)); - return null; - }); - - Assert.assertFalse(owners.containsKey(10)); - Assert.assertEquals(missing ? null : first, owners.get(9)); - Assert.assertEquals(LoadStatus.FAILED.getValue(), fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=10", Byte.class)); - if (!missing) { - Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); - } - } - } - - @Test - public void testOwnDispatchRejectionConditionallyMarksTaskAndJobFailed() throws Exception { - this.assertOwnDispatchFailureState(false); - } - - @Test - public void testOwnDispatchRejectionDoesNotOverwriteAdvancedTaskState() throws Exception { - this.assertOwnDispatchFailureState(true); - } - - private void assertOwnDispatchFailureState(boolean advanced) throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - task.setJobId(7); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - Mockito.doAnswer(invocation -> { - if (advanced) { - fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=9", - LoadStatus.SUCCEED.getValue()); - } - throw new java.util.concurrent.RejectedExecutionException("own dispatch rejected"); - }).when(executor).execute(Mockito.eq(task), Mockito.any()); - set(fixture.load, "taskExecutor", executor); - set(fixture.load, "transactionManager", fixture.manager); - set(fixture.load, "jobManagerMapper", fixture.jobs); - Method method = LoadTaskService.class.getDeclaredMethod("executeAfterCommit", LoadTask.class); - method.setAccessible(true); - method.invoke(fixture.load, task); - Assert.assertEquals(advanced ? LoadStatus.SUCCEED.getValue() : LoadStatus.FAILED.getValue(), - fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); - Assert.assertEquals(advanced ? org.apache.hugegraph.entity.enums.JobStatus.LOADING.getValue() : - org.apache.hugegraph.entity.enums.JobStatus.FAILED.getValue(), - fixture.jdbc.queryForObject("SELECT status FROM delete_jobs WHERE id=7", Byte.class)); - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - Assert.assertTrue(task.isFinished()); - Assert.assertTrue(loader.context().closed()); - Assert.assertFalse(fixture.owners.containsKey(9)); - Mockito.verify(loader, Mockito.never()).load(); - } - } - - @Test - public void testRecoveryReasonsAreRuntimeOnly() throws Exception { - LoadTask task = LoadTask.builder().status(LoadStatus.FAILED).lastDuration(0L).currDuration(0L) - .fileReadLines(0L).fileTotalLines(0L).build(); - task.setInitialStateReadFailed(true); - task.setDispatchFailurePending(true); - com.fasterxml.jackson.databind.ObjectMapper json = new com.fasterxml.jackson.databind.ObjectMapper(); - com.fasterxml.jackson.databind.JsonNode value = json.valueToTree(task); - Assert.assertFalse(value.has("initialStateReadFailed")); - Assert.assertFalse(value.has("initial_state_read_failed")); - Assert.assertFalse(value.has("dispatchFailurePending")); - Assert.assertFalse(value.has("dispatch_failure_pending")); - LoadTask restored = json.readValue( - "{\"initialStateReadFailed\":true,\"dispatchFailurePending\":true}", LoadTask.class); - Assert.assertFalse(restored.isInitialStateReadFailed()); - Assert.assertFalse(restored.isDispatchFailurePending()); - Field field = LoadTask.class.getDeclaredField("initialStateReadFailed"); - Assert.assertFalse(field.getAnnotation(com.baomidou.mybatisplus.annotation.TableField.class).exist()); - com.baomidou.mybatisplus.core.metadata.TableInfo table = - com.baomidou.mybatisplus.core.metadata.TableInfoHelper.initTableInfo( - new org.apache.ibatis.builder.MapperBuilderAssistant( - new com.baomidou.mybatisplus.core.MybatisConfiguration(), "cleanup-test"), - LoadTask.class); - Assert.assertFalse(table.getFieldList().stream() - .anyMatch(item -> item.getProperty().equals("initialStateReadFailed"))); - Assert.assertFalse(table.getFieldList().stream() - .anyMatch(item -> item.getProperty().equals("dispatchFailurePending"))); - } - - @Test - public void testInitialReadFailurePollPersistsFailedAndRetiresOwner() throws Exception { - this.assertInitialReadRecovery(LoadStatus.RUNNING, false, false); - } - - @Test - public void testInitialReadFailurePollPreservesSucceededTask() throws Exception { - this.assertInitialReadRecovery(LoadStatus.SUCCEED, false, false); - } - - @Test - public void testInitialReadFailurePollDoesNotRecreateDeletedRow() throws Exception { - this.assertInitialReadRecovery(null, false, false); - } - - @Test - public void testInitialReadFailurePollRetainsOwnerAfterSqlFailure() throws Exception { - this.assertInitialReadRecovery(LoadStatus.RUNNING, true, false); - } - - @Test - public void testInitialReadFailurePollRetainsOwnerAfterCommitFailure() throws Exception { - this.assertInitialReadRecovery(LoadStatus.RUNNING, false, true); - } - - @Test - public void testDifferentInitialOwnerCancelsBeforeDatabaseRead() throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - HugeGraphLoader oldLoader = this.loader(); - LoadTask old = this.task(oldLoader); - old.setId(9); - HugeGraphLoader newLoader = this.loader(); - LoadTask replacement = this.task(newLoader); - replacement.setId(9); - fixture.owners.put(9, replacement); - Mockito.doThrow(new RuntimeException("must not read metadata for a replaced initial owner")) - .when(fixture.tasks).selectById(9); - - dispatch(fixture.load, old); - - Assert.assertSame(replacement, fixture.owners.get(9)); - Assert.assertTrue(oldLoader.context().closed()); - Assert.assertFalse(newLoader.context().closed()); - Mockito.verify(fixture.tasks, Mockito.never()).selectById(9); - Mockito.verify(fixture.tasks, Mockito.never()).update(Mockito.any(), Mockito.any()); - Mockito.verify(newLoader, Mockito.never()).shutdown(); - newLoader.shutdown(); - } - } - - private void assertInitialReadRecovery(LoadStatus afterRead, boolean sqlFailure, - boolean commitFailure) throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - HugeGraphLoader loader = this.loader(); - LoadTask task = this.task(loader); - task.setId(9); - task.setJobId(7); - java.util.concurrent.atomic.AtomicBoolean first = new java.util.concurrent.atomic.AtomicBoolean(true); - RuntimeException readFailure = new RuntimeException("initial metadata read temporarily unavailable"); - Mockito.doAnswer(invocation -> { - if (first.getAndSet(false)) { - throw readFailure; - } - throw new AssertionError("initial failure recovery must use conditional SQL, not another snapshot"); - }).when(fixture.tasks).selectById(9); - - dispatch(fixture.load, task); - - Assert.assertSame("the closed initial owner must remain recoverable", task, fixture.owners.get(9)); - Assert.assertTrue(loader.context().closed()); - Assert.assertTrue(task.isFinished()); - Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); - if (afterRead == null) { - fixture.jdbc.update("DELETE FROM delete_tasks WHERE id=9"); - } else { - fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=9", afterRead.getValue()); - } - RuntimeException writeFailure = new RuntimeException("conditional failure write unavailable"); - java.util.concurrent.atomic.AtomicBoolean rejectSql = - new java.util.concurrent.atomic.AtomicBoolean(sqlFailure); - Mockito.doAnswer(invocation -> { - Assert.assertNull(invocation.getArgument(0)); - com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = - invocation.getArgument(1); - String sql = condition.getSqlSegment(); - Assert.assertTrue(sql.contains("id =")); - Assert.assertTrue(sql.contains("load_status =")); - Assert.assertTrue(condition.getParamNameValuePairs().containsValue(LoadStatus.RUNNING.getValue())); - Assert.assertTrue(condition.getSqlSet().contains("load_status=")); - if (rejectSql.getAndSet(false)) { - throw writeFailure; - } - return fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=9 AND load_status=?", - LoadStatus.FAILED.getValue(), LoadStatus.RUNNING.getValue()); - }).when(fixture.tasks).update(Mockito.nullable(LoadTask.class), Mockito.any()); - org.springframework.jdbc.datasource.DataSourceTransactionManager tx = fixture.manager; - if (commitFailure) { - java.util.concurrent.atomic.AtomicBoolean rejectCommit = - new java.util.concurrent.atomic.AtomicBoolean(true); - tx = new org.springframework.jdbc.datasource.DataSourceTransactionManager(fixture.dataSource) { - @Override - protected void doCommit(org.springframework.transaction.support.DefaultTransactionStatus status) { - if (rejectCommit.getAndSet(false)) { - throw new org.springframework.transaction.TransactionSystemException("commit unavailable"); - } - super.doCommit(status); - } - }; - tx.setRollbackOnCommitFailure(true); - } - org.springframework.transaction.support.TransactionTemplate transaction = - new org.springframework.transaction.support.TransactionTemplate(tx); - if (sqlFailure) { - transaction.execute(status -> { - fixture.load.updateLoadTaskProgress(); - return null; - }); - Assert.assertSame(task, fixture.owners.get(9)); - Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); - } else if (commitFailure) { - try { - transaction.execute(status -> { - fixture.load.updateLoadTaskProgress(); - return null; - }); - Assert.fail("the original SQL/commit failure must be propagated"); - } catch (RuntimeException failure) { - if (sqlFailure) { - Assert.assertSame(writeFailure, failure); - } else { - Assert.assertTrue( - failure instanceof org.springframework.transaction.TransactionSystemException); - } - } - Assert.assertSame(task, fixture.owners.get(9)); - Assert.assertEquals(LoadStatus.RUNNING.getValue(), fixture.jdbc.queryForObject( - "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class)); - } - transaction.execute(status -> { - fixture.load.updateLoadTaskProgress(); - Assert.assertSame("owner retirement waits for commit", task, fixture.owners.get(9)); - return null; - }); - Assert.assertFalse(fixture.owners.containsKey(9)); - java.util.List rows = fixture.jdbc.queryForList( - "SELECT load_status FROM delete_tasks WHERE id=9", Byte.class); - Assert.assertEquals(afterRead == null ? Collections.emptyList() : - Collections.singletonList(afterRead == LoadStatus.RUNNING ? - LoadStatus.FAILED.getValue() : afterRead.getValue()), rows); - Mockito.verify(fixture.tasks, Mockito.never()).updateById(Mockito.any(LoadTask.class)); - Mockito.verify(loader, Mockito.never()).load(); - } - } - - @Test - public void testDeleteAdmissionRejectsLaterControllerStart() throws Exception { - this.assertJobAdmission(true, false); - } - - @Test - public void testControllerStartAdmissionMakesDeleteEnumerateNewTask() throws Exception { - this.assertJobAdmission(false, false); - } - - @Test - public void testDeleteAdmissionRejectsLaterControllerCreate() throws Exception { - this.assertJobAdmission(true, true); - } - - private void assertJobAdmission(boolean deleteFirst, boolean createOnly) throws Exception { - try (JobDeleteFixture fixture = new JobDeleteFixture()) { - fixture.jdbc.update("UPDATE delete_jobs SET status=? WHERE id=7", - org.apache.hugegraph.entity.enums.JobStatus.SETTING.getValue()); - Mockito.doAnswer(invocation -> { - java.util.List rows = fixture.jdbc.queryForList( - "SELECT status FROM delete_jobs WHERE id=7", Byte.class); - if (rows.isEmpty()) { - return null; - } - return org.apache.hugegraph.entity.load.JobManager.builder().id(7) - .jobStatus(java.util.Arrays.stream(org.apache.hugegraph.entity.enums.JobStatus.values()) - .filter(status -> status.getValue().equals(rows.get(0))).findFirst().get()).build(); - }).when(fixture.jobs).selectById(7); - Mockito.doAnswer(invocation -> fixture.jobs.selectById(7)) - .when(fixture.jobs).selectOne(Mockito.any()); - CountDownLatch entered = new CountDownLatch(1); - CountDownLatch release = new CountDownLatch(1); - java.util.concurrent.atomic.AtomicReference startFailure = - new java.util.concurrent.atomic.AtomicReference<>(); - java.util.concurrent.atomic.AtomicReference deleteFailure = - new java.util.concurrent.atomic.AtomicReference<>(); - java.util.concurrent.atomic.AtomicInteger constructions = new java.util.concurrent.atomic.AtomicInteger(); - Mockito.doAnswer(invocation -> { - java.util.List snapshot = new java.util.ArrayList<>(); - for (Integer id : fixture.jdbc.queryForList("SELECT id FROM delete_tasks ORDER BY id", Integer.class)) { - snapshot.add(fixture.tasks.selectById(id)); - } - if (deleteFirst) { - entered.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - } - return snapshot; - }).when(fixture.tasks).selectList(Mockito.any()); - Mockito.when(fixture.tasks.insert(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - task.setId(11); - if (!deleteFirst) { - entered.countDown(); - Assert.assertTrue(release.await(5, TimeUnit.SECONDS)); - } - return fixture.jdbc.update("INSERT INTO delete_tasks VALUES (11, ?)", task.getStatus().getValue()); - }); - Mockito.when(fixture.tasks.updateById(Mockito.any(LoadTask.class))).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - return fixture.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=?", - task.getStatus().getValue(), task.getId()); - }); - LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); - Mockito.doAnswer(invocation -> { - new LoadTaskExecutor().execute(invocation.getArgument(0), invocation.getArgument(1)); - return null; - }).when(executor).execute(Mockito.any(), Mockito.any()); - set(fixture.load, "taskExecutor", executor); - FileMapping mapping = new FileMapping(); - mapping.setId(8); - mapping.setJobId(7); - mapping.setPath(this.folder.newFile("job-admission.csv").getAbsolutePath()); - mapping.setVertexMappings(Collections.emptySet()); - mapping.setEdgeMappings(Collections.emptySet()); - mapping.setLoadParameter(new LoadParameter()); - FileSetting setting = new FileSetting(); - setting.setColumnNames(Collections.singletonList("id")); - mapping.setFileSetting(setting); - org.apache.hugegraph.service.load.FileMappingService files = - Mockito.mock(org.apache.hugegraph.service.load.FileMappingService.class); - Mockito.when(files.get("space", "graph", 7, 8)).thenReturn(mapping); - org.apache.hugegraph.config.HugeConfig config = Mockito.mock(org.apache.hugegraph.config.HugeConfig.class); - Mockito.when(config.get(org.apache.hugegraph.options.HubbleOptions.PD_ENABLED)).thenReturn(false); - Mockito.when(config.get(org.apache.hugegraph.options.HubbleOptions.SERVER_URL)) - .thenReturn("http://127.0.0.1:19880"); - org.apache.hugegraph.controller.load.LoadTaskController controller = - new org.apache.hugegraph.controller.load.LoadTaskController(fixture.load) { - @Override - protected org.apache.hugegraph.driver.HugeClient requireGraphSpaceWrite(String graphSpace) { - return null; - } - - @Override - protected org.apache.hugegraph.driver.HugeClient authClient(String graphSpace, String graph) { - return null; - } - - @Override - protected String getToken() { - return "token"; - } - - @Override - protected String getUser() { - return null; - } - }; - Field jobField = org.apache.hugegraph.controller.load.LoadTaskController.class - .getDeclaredField("jobService"); - jobField.setAccessible(true); - jobField.set(controller, fixture.service); - Field filesField = org.apache.hugegraph.controller.load.LoadTaskController.class - .getDeclaredField("fmService"); - filesField.setAccessible(true); - filesField.set(controller, files); - Field configField = org.apache.hugegraph.controller.load.LoadTaskController.class - .getDeclaredField("config"); - configField.setAccessible(true); - configField.set(controller, config); - Thread start = new Thread(() -> { - try (MockedConstruction loaders = Mockito.mockConstruction( - HugeGraphLoader.class, (loader, context) -> { - constructions.incrementAndGet(); - LoadContext resources = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); - java.util.concurrent.atomic.AtomicBoolean closed = - new java.util.concurrent.atomic.AtomicBoolean(); - Mockito.when(loader.context()).thenReturn(resources); - Mockito.when(resources.closed()).thenAnswer(invocation -> closed.get()); - Mockito.when(resources.newProgress().totalInputRead()).thenReturn(0L); - Mockito.when(resources.summary().totalTime()).thenReturn(1L); - Mockito.when(loader.load()).thenReturn(true); - Mockito.doAnswer(invocation -> { - closed.set(true); - return null; - }).when(loader).shutdown(); - })) { - if (createOnly) { - controller.create("space", "graph", 7, LoadTask.builder().status(LoadStatus.INIT).build()); - } else { - controller.start("space", "graph", 7, Collections.singletonList(8)); - } - } catch (Throwable failure) { - startFailure.set(failure); - failure.printStackTrace(); - } - }); - Thread delete = new Thread(() -> { - try { - fixture.delete(true); - } catch (Throwable failure) { - deleteFailure.set(failure); - } - }); - Thread first = deleteFirst ? delete : start; - Thread second = deleteFirst ? start : delete; - first.start(); - Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); - second.start(); - try { - second.join(250); - System.out.println("JOB_ADMISSION deleteFirst=" + deleteFirst + " createOnly=" + createOnly + - " competitorWaiting=" + second.isAlive()); - Assert.assertTrue("same-job competitor must await admission", second.isAlive()); - } finally { - release.countDown(); - first.join(5000); - second.join(5000); - } - Assert.assertFalse(first.isAlive()); - Assert.assertFalse(second.isAlive()); - Assert.assertNull(deleteFailure.get()); - if (deleteFirst) { - Assert.assertTrue(startFailure.get() instanceof org.apache.hugegraph.exception.ExternalException); - Assert.assertEquals(0, constructions.get()); - Mockito.verifyNoInteractions(executor); - } else { - Assert.assertNull(startFailure.get()); - Assert.assertEquals(1, constructions.get()); - Mockito.verify(executor).execute(Mockito.any(), Mockito.any()); - } - Assert.assertEquals(Integer.valueOf(0), fixture.jdbc.queryForObject( - "SELECT COUNT(*) FROM delete_tasks", Integer.class)); - Assert.assertEquals(Integer.valueOf(0), fixture.jdbc.queryForObject( - "SELECT COUNT(*) FROM delete_jobs", Integer.class)); - Assert.assertTrue(fixture.owners.isEmpty()); - } - } - - private static class JobDeleteFixture implements AutoCloseable { - private final org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = - new org.springframework.jdbc.datasource.DriverManagerDataSource(); - private final org.springframework.jdbc.core.JdbcTemplate jdbc; - private final org.springframework.jdbc.datasource.DataSourceTransactionManager manager; - private final java.sql.Connection keeper; - private final LoadTaskMapper tasks = Mockito.mock(LoadTaskMapper.class); - private final org.apache.hugegraph.mapper.load.JobManagerMapper jobs = - Mockito.mock(org.apache.hugegraph.mapper.load.JobManagerMapper.class); - private final Map owners = new ConcurrentHashMap<>(); - private final org.apache.hugegraph.service.load.JobManagerService service; - private final LoadTaskService load = new LoadTaskService(); - - private JobDeleteFixture() throws Exception { - this.dataSource.setDriverClassName("org.h2.Driver"); - this.dataSource.setUrl("jdbc:h2:mem:job-delete-" + java.util.UUID.randomUUID()); - this.keeper = this.dataSource.getConnection(); - this.jdbc = new org.springframework.jdbc.core.JdbcTemplate(this.dataSource); - this.manager = new org.springframework.jdbc.datasource.DataSourceTransactionManager(this.dataSource); - this.jdbc.execute("CREATE TABLE delete_tasks (id INT PRIMARY KEY, load_status TINYINT NOT NULL)"); - this.jdbc.execute("CREATE TABLE delete_jobs (id INT PRIMARY KEY, status TINYINT NOT NULL)"); - this.jdbc.update("INSERT INTO delete_tasks VALUES (9, ?), (10, ?)", - LoadStatus.RUNNING.getValue(), LoadStatus.RUNNING.getValue()); - this.jdbc.update("INSERT INTO delete_jobs VALUES (7, ?)", - org.apache.hugegraph.entity.enums.JobStatus.LOADING.getValue()); - Mockito.when(this.tasks.selectById(Mockito.anyInt())).thenAnswer(invocation -> { - int id = invocation.getArgument(0); - java.util.List rows = this.jdbc.queryForList( - "SELECT load_status FROM delete_tasks WHERE id=?", Byte.class, id); - if (rows.isEmpty()) { - return null; - } - LoadStatus status = java.util.Arrays.stream(LoadStatus.values()) - .filter(value -> value.getValue().equals(rows.get(0))).findFirst().get(); - return LoadTask.builder().id(id).jobId(7).options(new LoadOptions()).status(status).build(); - }); - Mockito.when(this.tasks.selectList(Mockito.any())).thenAnswer(invocation -> - java.util.Arrays.asList(this.tasks.selectById(10), this.tasks.selectById(9))); - Mockito.when(this.tasks.update(Mockito.nullable(LoadTask.class), Mockito.any())).thenAnswer(invocation -> { - LoadTask task = invocation.getArgument(0); - com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper condition = - invocation.getArgument(1); - String where = condition.getSqlSegment(); - Byte expected = (Byte) condition.getParamNameValuePairs().entrySet().stream() - .filter(entry -> entry.getValue() instanceof Byte && where.contains("." + entry.getKey() + "}")) - .map(Map.Entry::getValue).findFirst().get(); - Integer id = (Integer) condition.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Integer).findFirst().get(); - Byte target = task == null ? LoadStatus.FAILED.getValue() : task.getStatus().getValue(); - return this.jdbc.update("UPDATE delete_tasks SET load_status=? WHERE id=? AND load_status=?", - target, id, expected); - }); - Mockito.when(this.tasks.delete(Mockito.any())).thenAnswer(invocation -> { - com.baomidou.mybatisplus.core.conditions.query.QueryWrapper condition = - invocation.getArgument(0); - condition.getSqlSegment(); - Integer id = (Integer) condition.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Integer).findFirst().get(); - Byte expected = (Byte) condition.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Byte).findFirst().get(); - return this.jdbc.update("DELETE FROM delete_tasks WHERE id=? AND load_status=?", id, expected); - }); - org.apache.hugegraph.entity.load.JobManager job = - org.apache.hugegraph.entity.load.JobManager.builder().id(7) - .jobStatus(org.apache.hugegraph.entity.enums.JobStatus.LOADING).build(); - Mockito.when(this.jobs.selectById(7)).thenReturn(job); - Mockito.when(this.jobs.selectOne(Mockito.any())).thenReturn(job); - Mockito.when(this.jobs.deleteById(7)).thenAnswer(invocation -> - this.jdbc.update("DELETE FROM delete_jobs WHERE id=7")); - Mockito.when(this.jobs.updateById(Mockito.any(org.apache.hugegraph.entity.load.JobManager.class))) - .thenAnswer(invocation -> { - org.apache.hugegraph.entity.load.JobManager updated = invocation.getArgument(0); - return this.jdbc.update("UPDATE delete_jobs SET status=? WHERE id=?", - updated.getJobStatus().getValue(), updated.getId()); - }); - set(this.load, "mapper", this.tasks); - set(this.load, "runningTaskContainer", this.owners); - set(this.load, "transactionManager", this.manager); - org.apache.hugegraph.service.load.JobManagerService target = - new org.apache.hugegraph.service.load.JobManagerService(); - set(target, "mapper", this.jobs); - set(target, "taskService", this.load); - set(target, "transactionManager", this.manager); - org.apache.hugegraph.service.load.FileMappingService files = - Mockito.mock(org.apache.hugegraph.service.load.FileMappingService.class); - Mockito.when(files.listByJob(7)).thenReturn(Collections.emptyList()); - set(target, "fileMappingService", files); - org.springframework.aop.framework.ProxyFactory factory = - new org.springframework.aop.framework.ProxyFactory(target); - factory.setProxyTargetClass(true); - factory.addAdvice(new org.springframework.transaction.interceptor.TransactionInterceptor( - this.manager, - new org.springframework.transaction.annotation.AnnotationTransactionAttributeSource())); - this.service = (org.apache.hugegraph.service.load.JobManagerService) factory.getProxy(); - } - - private void delete(boolean scoped) { - if (scoped) { - this.service.deleteJob("space", "graph", 7); - } else { - this.service.deleteJob(7); - } - } - - private void enableGraphDeletion() { - this.jdbc.execute("ALTER TABLE delete_jobs ADD graphspace VARCHAR(48) DEFAULT 'space'"); - this.jdbc.execute("ALTER TABLE delete_jobs ADD graph_name VARCHAR(48) DEFAULT 'graph'"); - this.jdbc.execute("ALTER TABLE delete_tasks ADD job_id INT DEFAULT 7"); - Mockito.when(this.jobs.selectList(Mockito.any())).thenAnswer(invocation -> { - Assert.assertFalse("the graph cleanup must suspend the caller transaction", - TransactionSynchronizationManager.isActualTransactionActive()); - com.baomidou.mybatisplus.core.conditions.query.QueryWrapper query = invocation.getArgument(0); - query.getSqlSegment(); - Assert.assertTrue(query.getSqlSegment().contains("graphspace")); - Assert.assertTrue(query.getSqlSegment().contains("graph")); - Assert.assertTrue(query.getParamNameValuePairs().containsValue("space")); - Assert.assertTrue(query.getParamNameValuePairs().containsValue("graph")); - java.util.List result = new java.util.ArrayList<>(); - for (Integer id : this.jdbc.queryForList( - "SELECT id FROM delete_jobs WHERE graphspace='space' AND graph_name='graph' ORDER BY id", - Integer.class)) { - result.add(this.jobs.selectById(id)); - } - return result; - }); - Mockito.when(this.jobs.selectById(Mockito.anyInt())).thenAnswer(invocation -> { - int id = invocation.getArgument(0); - return this.jdbc.queryForList("SELECT id FROM delete_jobs WHERE id=?", Integer.class, id).isEmpty() ? - null : org.apache.hugegraph.entity.load.JobManager.builder().id(id) - .graphSpace("space").graph("graph").build(); - }); - Mockito.when(this.jobs.selectOne(Mockito.any())).thenAnswer(invocation -> { - com.baomidou.mybatisplus.core.conditions.query.QueryWrapper query = invocation.getArgument(0); - query.getSqlSegment(); - int id = (Integer) query.getParamNameValuePairs().values().stream() - .filter(value -> value instanceof Integer).findFirst().get(); - return this.jobs.selectById(id); - }); - Mockito.when(this.jobs.deleteById(Mockito.anyInt())).thenAnswer(invocation -> - this.jdbc.update("DELETE FROM delete_jobs WHERE id=?", (Integer) invocation.getArgument(0))); - Mockito.when(this.jobs.deleteByMap(Mockito.anyMap())).thenAnswer(invocation -> - this.jdbc.update("DELETE FROM delete_jobs WHERE graphspace='space' AND graph_name='graph'")); - Mockito.doAnswer(invocation -> { - com.baomidou.mybatisplus.core.conditions.query.QueryWrapper query = invocation.getArgument(0); - query.getSqlSegment(); - int jobId = (Integer) query.getParamNameValuePairs().values().stream().findFirst().get(); - java.util.List result = new java.util.ArrayList<>(); - for (Integer id : this.jdbc.queryForList( - "SELECT id FROM delete_tasks WHERE job_id=? ORDER BY id", Integer.class, jobId)) { - result.add(this.tasks.selectById(id)); - } - return result; - }).when(this.tasks).selectList(Mockito.any()); - } - - @Override - public void close() throws Exception { - this.keeper.close(); - } - } - - // Separate admission instances intentionally keep the defensive old-tail SQL/identity - // controls concurrent. This is not a cross-Hubble lifecycle coordination guarantee. - private static LoadTaskService peer(LoadTaskService source) throws Exception { - LoadTaskService peer = new LoadTaskService(); - for (String name : new String[]{"mapper", "taskExecutor", "runningTaskContainer"}) { - Field field = LoadTaskService.class.getDeclaredField(name); - field.setAccessible(true); - set(peer, name, field.get(source)); - } - return peer; - } - - private static void set(Object object, String name, Object value) throws Exception { - Field field = object.getClass().getDeclaredField(name); - field.setAccessible(true); - field.set(object, value); - } - - private static void enablePollTransactions(LoadTaskService service) throws Exception { - org.springframework.jdbc.datasource.DriverManagerDataSource dataSource = - new org.springframework.jdbc.datasource.DriverManagerDataSource(); - dataSource.setUrl("jdbc:h2:mem:poll-" + java.util.UUID.randomUUID()); - set(service, "transactionManager", - new org.springframework.jdbc.datasource.DataSourceTransactionManager(dataSource)); - } - - private static void dispatch(LoadTaskService service, LoadTask task) throws Exception { - Field mapperField = LoadTaskService.class.getDeclaredField("mapper"); - mapperField.setAccessible(true); - LoadTaskMapper mapper = (LoadTaskMapper) mapperField.get(service); - if (mapper == null) { - mapper = Mockito.mock(LoadTaskMapper.class); - set(service, "mapper", mapper); - } - boolean configured = Mockito.mockingDetails(mapper).getStubbings().stream() - .anyMatch(stub -> stub.getInvocation().getMethod().getName().equals("selectById") && - java.util.Objects.equals(stub.getInvocation().getArguments()[0], task.getId())); - if (!configured) { - Mockito.when(mapper.selectById(task.getId())).thenReturn( - LoadTask.builder().id(task.getId()).status(LoadStatus.RUNNING).build()); - } - Method method = LoadTaskService.class.getDeclaredMethod("executeAfterCommit", LoadTask.class); - method.setAccessible(true); - method.invoke(service, task); - } - - private HugeGraphLoader loader() { - HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); - LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); - Mockito.when(loader.context()).thenReturn(context); - java.util.concurrent.atomic.AtomicBoolean closed = - new java.util.concurrent.atomic.AtomicBoolean(); - Mockito.when(context.closed()).thenAnswer(invocation -> closed.get()); - Mockito.doAnswer(invocation -> { - closed.set(true); - return null; - }).when(loader).shutdown(); - Mockito.when(context.newProgress().totalInputRead()).thenReturn(7L); - Mockito.when(context.summary().totalTime()).thenReturn(12L); - return loader; - } - - private LoadTask task(HugeGraphLoader loader) { - return LoadTask.builder().loader(loader).options(new LoadOptions()).status(LoadStatus.RUNNING) - .lastDuration(0L).build(); - } -} diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java index 11fd1bb0c..9d7407a19 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java @@ -33,6 +33,8 @@ import org.apache.hugegraph.entity.load.LoadParameter; import org.apache.hugegraph.entity.load.LoadTask; import org.apache.hugegraph.handler.LoadTaskExecutor; +import org.apache.hugegraph.loader.HugeGraphLoader; +import org.apache.hugegraph.loader.executor.LoadContext; import org.apache.hugegraph.loader.executor.LoadOptions; import org.apache.hugegraph.loader.source.file.FileSource; import org.apache.hugegraph.service.load.LoadTaskService; @@ -40,6 +42,7 @@ import org.apache.hugegraph.testutil.Assert; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import org.mockito.MockedConstruction; import org.springframework.transaction.support.TransactionSynchronization; import org.springframework.transaction.support.TransactionSynchronizationManager; @@ -47,6 +50,46 @@ public class LoadTaskServiceTest { + @Test + public void testRunnerClosesBeforePublishingCompletion() { + for (boolean success : new boolean[]{true, false}) { + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(loader.load()).thenReturn(success); + LoadTask task = LoadTask.builder().id(9).loader(loader).options(new LoadOptions()) + .status(LoadStatus.RUNNING).lastDuration(0L).build(); + Mockito.doAnswer(invocation -> { + Assert.assertFalse(task.isFinished()); + Assert.assertEquals(LoadStatus.RUNNING, task.getStatus()); + return null; + }).when(loader).shutdown(); + + task.run(); + + Mockito.verify(loader).shutdown(); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(success ? LoadStatus.SUCCEED : LoadStatus.FAILED, task.getStatus()); + } + } + + @Test + public void testPreviewAlwaysClosesLoader() { + for (boolean success : new boolean[]{true, false}) { + try (MockedConstruction construction = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, ignored) -> { + if (success) { + Mockito.when(loader.load()).thenReturn(true); + } else { + Mockito.when(loader.load()).thenThrow(new IllegalStateException("load failed")); + } + })) { + new LoadTaskService().loader(new LoadOptions()); + Mockito.verify(construction.constructed().get(0)).shutdown(); + } + } + } + @Test public void testListByIdsAppliesGraphScope() throws Exception { LoadTaskService service = new LoadTaskService(); @@ -84,13 +127,7 @@ public void testLoadExecutionWaitsForOuterTransactionCommit() LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); this.setField(service, "taskExecutor", executor); this.setField(service, "runningTaskContainer", new ConcurrentHashMap<>()); - org.apache.hugegraph.loader.HugeGraphLoader loader = - Mockito.mock(org.apache.hugegraph.loader.HugeGraphLoader.class); - Mockito.when(loader.context()).thenReturn(Mockito.mock(org.apache.hugegraph.loader.executor.LoadContext.class)); - LoadTask task = LoadTask.builder().id(9).status(LoadStatus.RUNNING).loader(loader).build(); - LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); - Mockito.when(mapper.selectById(9)).thenReturn(task); - this.setField(service, "mapper", mapper); + LoadTask task = LoadTask.builder().id(9).build(); Method method = LoadTaskService.class.getDeclaredMethod( "executeAfterCommit", LoadTask.class); method.setAccessible(true); @@ -120,8 +157,7 @@ public void testResumeRehydratesLoaderWithCurrentSessionToken() LoadTask task = Mockito.mock(LoadTask.class); LoadOptions options = new LoadOptions(); Mockito.when(mapper.selectById(9)).thenReturn(task); - Mockito.when(task.getId()).thenReturn(9); - Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenReturn(1); + Mockito.when(mapper.updateById(task)).thenReturn(1); Mockito.when(task.getStatus()).thenReturn(LoadStatus.PAUSED); Mockito.when(task.getOptions()).thenReturn(options); this.setField(service, "mapper", mapper); @@ -144,8 +180,7 @@ public void testResumeDoesNotPersistRunningStateWhenReconnectFails() LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); LoadTask task = Mockito.mock(LoadTask.class); Mockito.when(mapper.selectById(9)).thenReturn(task); - Mockito.when(task.getId()).thenReturn(9); - Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenReturn(1); + Mockito.when(mapper.updateById(task)).thenReturn(1); Mockito.when(task.getStatus()).thenReturn(LoadStatus.PAUSED); Mockito.when(task.getOptions()).thenReturn(new LoadOptions()); Mockito.doThrow(new IllegalArgumentException("missing token")) @@ -160,7 +195,6 @@ public void testResumeDoesNotPersistRunningStateWhenReconnectFails() }); Mockito.verify(mapper, Mockito.never()).updateById(task); - Mockito.verify(mapper, Mockito.never()).update(Mockito.eq(task), Mockito.any()); Mockito.verify(executor, Mockito.never()) .execute(Mockito.any(), Mockito.any()); } @@ -173,8 +207,7 @@ public void testRetryRehydratesLoaderWithCurrentSessionToken() LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); LoadTask task = Mockito.mock(LoadTask.class); Mockito.when(mapper.selectById(9)).thenReturn(task); - Mockito.when(task.getId()).thenReturn(9); - Mockito.when(mapper.update(Mockito.eq(task), Mockito.any())).thenReturn(1); + Mockito.when(mapper.updateById(task)).thenReturn(1); Mockito.when(task.getStatus()).thenReturn(LoadStatus.STOPPED); Mockito.when(task.getOptions()).thenReturn(new LoadOptions()); this.setField(service, "mapper", mapper); diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java index d7607498c..07800995c 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -86,7 +86,6 @@ GroovySchemaCompatibilityTest.class, JobManagerServiceTest.class, K8sTokenEndpointSecurityTest.class, - LoadTaskCleanupTest.class, LoadTaskServiceTest.class, LoaderScopeControllerTest.class, LoginAttemptGuardTest.class, diff --git a/hugegraph-loader/README.md b/hugegraph-loader/README.md index 2cf290e88..bcd19f1cd 100644 --- a/hugegraph-loader/README.md +++ b/hugegraph-loader/README.md @@ -7,6 +7,10 @@ hugegraph-loader is a customizable command line utility for loading small to medium size graph datasets into the HugeGraph database from multiple data sources with various input formats. +Embedded callers must call `shutdown()` after `load()`, including failed imports. Successful shutdown waits for this Loader's +workers, closes its clients and unregisters its JVM shutdown hook. Closing a Loader whose `load()` was never called +leaves the existing import checkpoint unchanged. + ## 1. Features - Multiple data sources, such as local file(path), HDFS file(path), MySQL @@ -171,22 +175,3 @@ Server 1.7 compatibility CI uses the pinned official ASF 1.7.0 binary archive. T hugegraph-loader is licensed under Apache 2.0 License. See the [Flink CDC Java 17 validation](docs/flink-cdc-java17.md) for its current limits and launcher syntax. - -### Embedded Loader shutdown - -Call `HugeGraphLoader.shutdown()` when an embedded load finishes or its dispatch -fails. Shutdown waits for that Loader's workers, closes its context, and removes -its own JVM shutdown hook once the context is closed. If workers cannot terminate, -the context and hook remain available for a later shutdown attempt. A hook already -running during JVM shutdown still closes its Loader. This does not shut down the -process-wide executor manager or coordinate graph mode across separate imports. - -Closing a Loader before `load()` starts does not write a new checkpoint. An actual -load attempt and direct `LoadContext.close()` keep their existing progress-save -behavior. Incremental loading keeps independent copies of the previous loaded -and loading offsets, including inputs paused before their first read, and adds -newly loaded counts to the completed counts. It also preserves mapped inputs -outside the selected file range; removed mappings do not retain source markers -in the next checkpoint. Non-incremental replay starts fresh. -An ongoing insert can still block the existing wait before shutdown -reaches cancellation; this change adds no load deadline. diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java index 45b023c3f..fa558e641 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java @@ -607,15 +607,6 @@ private void loadInputs(List structs) { private List prepareTaskItems(List structs, boolean scatter) { - if (this.context.options().incrementalMode) { - // Range filtering must not discard progress of other mapped inputs. - for (InputStruct struct : structs) { - if (this.context.newProgress().get(struct.id()) == null && - this.context.oldProgress().get(struct.id()) != null) { - this.context.newProgress().addStruct(struct, this.context.oldProgress().get(struct.id())); - } - } - } ArrayList tasks = new ArrayList<>(); ArrayList readers = new ArrayList<>(); int curFile = 0; @@ -912,67 +903,59 @@ private boolean reachedMaxReadLines() { * TODO: How to distinguish load task finished normally or abnormally */ private synchronized void stopThenShutdown() { - Throwable failure = null; try { - if (this.context.closed()) { - return; - } - LOG.info("Stop loading then shutdown HugeGraphLoader"); - boolean tasksStopped = this.manager == null; - boolean interrupted = Thread.currentThread().isInterrupted(); - try { - this.context.stopLoading(); - if (this.manager != null) { - // Wait all insert tasks stopped before exit - try { - this.manager.waitFinished(); - } finally { - this.manager.shutdown(); - tasksStopped = true; - } + this.drainThenClose(); + } finally { + if (this.context.closed() && this.shutdownHook != null) { + try { + Runtime.getRuntime().removeShutdownHook(this.shutdownHook); + } catch (IllegalStateException ignored) { + // JVM shutdown already started; this hook can finish closing. } - } finally { - interrupted |= Thread.currentThread().isInterrupted(); + } + } + } + + private void drainThenClose() { + if (this.context.closed()) { + return; + } + LOG.info("Stop loading then shutdown HugeGraphLoader"); + boolean tasksStopped = this.manager == null; + boolean interrupted = Thread.currentThread().isInterrupted(); + try { + this.context.stopLoading(); + if (this.manager != null) { + // Wait all insert tasks stopped before exit try { - // Keep dependencies alive if cancellation could not terminate the workers. - if (tasksStopped) { - try { - this.context.unsetLoadingMode(); - } finally { - this.context.close(this.loadStarted); - } - } + this.manager.waitFinished(); } finally { - if (interrupted) { - Thread.currentThread().interrupt(); - } + this.manager.shutdown(); + tasksStopped = true; } } - } catch (RuntimeException | Error e) { - failure = e; - throw e; } finally { - if (this.context.closed() && this.shutdownHook != null) { - try { - Runtime.getRuntime().removeShutdownHook(this.shutdownHook); - } catch (IllegalStateException ignored) { - // JVM shutdown already started; this hook can still finish closing. - } catch (RuntimeException | Error e) { - if (failure == null) { - throw e; - } - if (failure != e) { - failure.addSuppressed(e); + interrupted |= Thread.currentThread().isInterrupted(); + try { + // Keep dependencies alive if cancellation could not terminate the workers. + if (tasksStopped) { + try { + this.context.unsetLoadingMode(); + } finally { + this.context.close(this.loadStarted); } } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } } } } private void cleanupEmptyProgress() { Map inputProgressMap = this.context.newProgress().inputProgress(); - inputProgressMap.entrySet().removeIf(entry -> entry.getValue().loadedItems().isEmpty() && - entry.getValue().loadingItems().isEmpty()); + inputProgressMap.entrySet().removeIf(entry -> entry.getValue().loadedItems().isEmpty()); } private static class SplitInputStructs { diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java index cc0d82714..1023b967e 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/executor/LoadContext.java @@ -96,10 +96,6 @@ private LoadContext(LoadOptions options, SchemaCache schemaCache) { this.summary = new LoadSummary(); this.oldProgress = LoadProgress.parse(options); this.newProgress = new LoadProgress(); - if (options.incrementalMode) { - this.newProgress.plusVertexLoaded(this.oldProgress.vertexLoaded()); - this.newProgress.plusEdgeLoaded(this.oldProgress.edgeLoaded()); - } this.loggers = new ConcurrentHashMap<>(); this.client = schemaCache == null ? HugeClientHolder.create(options) : null; if (this.client != null && this.options.direct) { diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputItemProgress.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputItemProgress.java index 88dd50a2d..a1c78b71a 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputItemProgress.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputItemProgress.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -public abstract class InputItemProgress implements Cloneable { +public abstract class InputItemProgress { /** * NOTE: this offset refers to the parsed part in input instead of loaded. @@ -50,12 +50,4 @@ public void offset(long offset) { public void confirmOffset() { this.confirmOffset = this.offset; } - - public InputItemProgress copy() { - try { - return (InputItemProgress) super.clone(); - } catch (CloneNotSupportedException e) { - throw new AssertionError("Input progress must be cloneable", e); - } - } } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputProgress.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputProgress.java index 66c340384..66e2898ab 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputProgress.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/InputProgress.java @@ -50,20 +50,10 @@ public InputProgress(@JsonProperty("type") SourceType type, } public InputProgress(InputStruct struct) { - this(struct, null); - } - - public InputProgress(InputStruct struct, InputProgress previous) { this.type = struct.input().type(); this.loadedItems = Collections.synchronizedMap( InsertionOrderUtil.newMap()); this.loadingItems = new ConcurrentHashMap<>(); - if (previous != null && previous.type == this.type) { - synchronized (previous) { - previous.loadedItems.forEach((name, item) -> this.loadedItems.put(name, item.copy())); - previous.loadingItems.forEach((name, item) -> this.loadingItems.put(name, item.copy())); - } - } } public synchronized Map loadedItems() { diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/LoadProgress.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/LoadProgress.java index e78da8334..9fbeb4745 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/LoadProgress.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/progress/LoadProgress.java @@ -101,12 +101,8 @@ public long totalInputRead() { } public InputProgress addStruct(InputStruct struct) { - return this.addStruct(struct, null); - } - - public InputProgress addStruct(InputStruct struct, InputProgress previous) { E.checkNotNull(struct, "mapping mapping"); - this.inputProgress.put(struct.id(), new InputProgress(struct, previous)); + this.inputProgress.put(struct.id(), new InputProgress(struct)); return this.inputProgress.get(struct.id()); } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/AbstractReader.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/AbstractReader.java index 4221cfe55..6c0ecae2e 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/AbstractReader.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/AbstractReader.java @@ -34,8 +34,7 @@ public void progress(LoadContext context, InputStruct struct) { // Update loading vertex/edge mapping this.newProgress = context.newProgress().get(struct.id()); if (this.newProgress == null) { - this.newProgress = context.newProgress().addStruct( - struct, context.options().incrementalMode ? this.oldProgress : null); + this.newProgress = context.newProgress().addStruct(struct); } } } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/file/FileReader.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/file/FileReader.java index d961dc1ae..e6e32af01 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/file/FileReader.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/reader/file/FileReader.java @@ -238,14 +238,14 @@ private LoadStatus checkLastLoadStatus(Readable readable) { InputItemProgress loaded = this.oldProgress.matchLoadedItem(input); // The file has been loaded before, and it is not changed if (loaded != null) { - this.newProgress.addLoadedItem(readable.name(), loaded.copy()); + this.newProgress.addLoadedItem(readable.name(), loaded); return LoadStatus.LOADED; } InputItemProgress loading = this.oldProgress.matchLoadingItem(input); if (loading != null) { // The file has been loaded half before and it is not changed - this.newProgress.addLoadingItem(readable.name(), loading.copy()); + this.newProgress.addLoadingItem(readable.name(), loading); return LoadStatus.LOADED_HALF; } else { this.newProgress.addLoadingItem(readable.name(), input); diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java index edfa54deb..d59e54d39 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java @@ -25,12 +25,10 @@ import java.io.OutputStreamWriter; import java.io.StringWriter; import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.Arrays; -import java.util.List; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -671,218 +669,72 @@ public boolean requiresLayout() { } } - @Test - public void testIncrementalReaderInitializationCopiesCheckpointWithoutAliases() throws Exception { - this.assertReaderCheckpointInitialization(true); - } - - @Test - public void testReplayReaderInitializationDoesNotInheritCheckpoint() throws Exception { - this.assertReaderCheckpointInitialization(false); + private static HugeGraphLoader loader(LoadContext context, InputStruct... inputs) throws Exception { + HugeGraphLoader loader = allocate(HugeGraphLoader.class); + set(loader, "context", context); + set(loader, "options", context.options()); + set(loader, "mapping", new LoadMapping(Arrays.asList(inputs))); + return loader; } @Test - public void testIncrementalRangeFilteringPreservesOnlyCurrentMappingCheckpoints() throws Exception { - this.assertReaderCheckpointInitialization(true, true); - } - - private void assertReaderCheckpointInitialization(boolean incremental) throws Exception { - this.assertReaderCheckpointInitialization(incremental, false); - } - - private void assertReaderCheckpointInitialization(boolean incremental, boolean filtered) throws Exception { + public void testUnstartedShutdownPreservesCheckpoint() throws Exception { try (ModeServer server = new ModeServer()) { - File firstFile = this.folder.newFile("copied-loaded.csv"); - File secondFile = this.folder.newFile("copied-loading.csv"); - java.nio.file.Files.writeString(firstFile.toPath(), "a0\na1\n"); - java.nio.file.Files.writeString(secondFile.toPath(), "b0\nb1\n"); LoadOptions options = new LoadOptions(); options.host = "127.0.0.1"; options.port = server.server.getAddress().getPort(); - options.file = this.folder.newFile(incremental ? "copy-inc.json" : "copy-replay.json").getAbsolutePath(); - LoadProgress checkpoint = new LoadProgress(); - InputStruct[] structs = new InputStruct[2]; - File[] files = {firstFile, secondFile}; - for (int i = 0; i < files.length; i++) { - FileSource source = new FileSource(); - source.path(files[i].getAbsolutePath()); - source.header(new String[]{"id"}); - source.hasHeader(false); - InputStruct struct = new InputStruct(null, null); - struct.id(Integer.toString(i)); - struct.input(source); - structs[i] = struct; - org.apache.hugegraph.loader.progress.FileItemProgress item = - new org.apache.hugegraph.loader.progress.FileItemProgress( - files[i].getName(), files[i].lastModified(), - String.valueOf(org.apache.commons.io.FileUtils.checksumCRC32(files[i])), - i == 0 ? 2L : 1L); - org.apache.hugegraph.loader.progress.InputProgress progress = checkpoint.addStruct(struct); - if (i == 0) { - progress.addLoadedItem(files[i].getName(), item); - } else { - progress.addLoadingItem(files[i].getName(), item); - } - } - if (filtered) { - InputStruct removed = new InputStruct(null, null); - removed.id("removed"); - removed.input(structs[0].input()); - checkpoint.addStruct(removed).addLoadedItem(firstFile.getName(), - new org.apache.hugegraph.loader.progress.FileItemProgress( - firstFile.getName(), firstFile.lastModified(), "removed", 1L)); - options.startFile = 1; - options.endFile = 2; - } - checkpoint.plusVertexLoaded(5L); - checkpoint.plusEdgeLoaded(2L); - java.nio.file.Path saved = java.nio.file.Paths.get(LoadProgress.format(options, "20000101-000000")); - java.nio.file.Files.createDirectories(saved.getParent()); - java.nio.file.Files.writeString(saved, JsonUtil.toJson(checkpoint)); - options.incrementalMode = incremental; - LoadContext context = new LoadContext(options); - try { - if (filtered) { - HugeGraphLoader loader = loader(context, structs); - Method prepare = HugeGraphLoader.class.getDeclaredMethod( - "prepareTaskItems", List.class, boolean.class); - prepare.setAccessible(true); - @SuppressWarnings("unchecked") - List tasks = - (List) prepare.invoke(loader, Arrays.asList(structs), false); - Assert.assertEquals(1, tasks.size()); - for (HugeGraphLoader.InputTaskItem item : tasks) { - item.reader.close(); - } - Assert.assertNull(context.newProgress().get("removed")); - } else { - for (InputStruct struct : structs) { - org.apache.hugegraph.loader.reader.file.LocalFileReader scanner = - new org.apache.hugegraph.loader.reader.file.LocalFileReader( - (FileSource) struct.input()); - try (org.apache.hugegraph.loader.reader.InputReader reader = scanner.split().get(0)) { - reader.init(context, struct); - } - } - } - if (incremental) { - org.apache.hugegraph.loader.progress.InputItemProgress oldLoaded = - context.oldProgress().get("0").loadedItem(firstFile.getName()); - org.apache.hugegraph.loader.progress.InputItemProgress newLoaded = - context.newProgress().get("0").loadedItem(firstFile.getName()); - org.apache.hugegraph.loader.progress.InputItemProgress oldLoading = - context.oldProgress().get("1").loadingItem(secondFile.getName()); - org.apache.hugegraph.loader.progress.InputItemProgress newLoading = - context.newProgress().get("1").loadingItem(secondFile.getName()); - Assert.assertNotSame(oldLoaded, newLoaded); - Assert.assertNotSame(oldLoading, newLoading); - Assert.assertTrue(newLoaded instanceof org.apache.hugegraph.loader.progress.FileItemProgress); - Assert.assertEquals(oldLoaded, newLoaded); - newLoading.offset(2L); - newLoading.confirmOffset(); - Assert.assertEquals(1L, oldLoading.offset()); - newLoading.offset(1L); - newLoading.confirmOffset(); - HugeGraphLoader loader = loader(context, structs); - Method cleanup = HugeGraphLoader.class.getDeclaredMethod("cleanupEmptyProgress"); - cleanup.setAccessible(true); - cleanup.invoke(loader); - Assert.assertEquals(2, context.newProgress().inputProgress().size()); - Assert.assertEquals(3L, context.newProgress().totalInputRead()); - context.stopLoading(); - context.close(); - LoadProgress next = LoadProgress.parse(options); - Assert.assertEquals(3L, next.totalInputRead()); - Assert.assertEquals(5L, next.vertexLoaded()); - Assert.assertEquals(2L, next.edgeLoaded()); - Assert.assertNull(next.get("removed")); - } else { - Assert.assertEquals(0L, context.newProgress().totalInputRead()); - Assert.assertEquals(0L, context.newProgress().vertexLoaded()); - Assert.assertEquals(0L, context.newProgress().edgeLoaded()); - Assert.assertTrue(context.newProgress().get("0").loadedItems().isEmpty()); - Assert.assertTrue(context.newProgress().get("1").loadingItems().isEmpty()); - } - } finally { - context.close(false); - } - } - } + options.file = this.folder.newFile("unstarted.json").getAbsolutePath(); + options.incrementalMode = true; + File checkpoint = new File(LoadProgress.format(options, "2000-01-01")); + Files.createDirectories(checkpoint.toPath().getParent()); + String saved = "{\"vertex_progress\":7,\"edge_progress\":0,\"input_progress\":{}}"; + Files.writeString(checkpoint.toPath(), saved); + HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList())); + Assert.assertEquals(7L, loader.context().oldProgress().vertexLoaded()); - @Test - public void testDirectContextCloseStillSavesEmptyProgress() throws Exception { - try (ModeServer server = new ModeServer()) { - LoadOptions options = new LoadOptions(); - options.host = "127.0.0.1"; - options.port = server.server.getAddress().getPort(); - options.file = this.folder.newFile("direct-empty.json").getAbsolutePath(); - LoadContext context = new LoadContext(options); - context.close(); - Assert.assertTrue(context.closed()); - Assert.assertTrue(new File(LoadProgress.format(options, context.timestamp())).exists()); - Assert.assertTrue(context.newProgress().inputProgress().isEmpty()); - } - } + loader.shutdown(); - @Test - public void testShutdownUnregistersOwnNativeHook() throws Exception { - try (ModeServer server = new ModeServer()) { - LoadOptions options = new LoadOptions(); - options.host = "127.0.0.1"; - options.port = server.server.getAddress().getPort(); - options.file = this.folder.newFile("hook-success.json").getAbsolutePath(); - HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList())); - Field field = HugeGraphLoader.class.getDeclaredField("shutdownHook"); - field.setAccessible(true); - Thread hook = (Thread) field.get(loader); - try { - loader.shutdown(); - Assert.assertTrue(loader.context().closed()); - Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); - loader.shutdown(); - } finally { - Runtime.getRuntime().removeShutdownHook(hook); - } + Assert.assertTrue(loader.context().closed()); + Assert.assertEquals(saved, Files.readString(checkpoint.toPath())); + Assert.assertFalse(new File(LoadProgress.format(options, loader.context().timestamp())).exists()); } } @Test - public void testLoadFailureStillUnregistersClosedLoaderHook() throws Exception { - try (ModeServer server = new ModeServer()) { - LoadOptions options = new LoadOptions(); - options.host = "127.0.0.1"; - options.port = server.server.getAddress().getPort(); - options.file = this.folder.newFile("hook-failure.json").getAbsolutePath(); - HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList())); - Field field = HugeGraphLoader.class.getDeclaredField("shutdownHook"); - field.setAccessible(true); - Thread hook = (Thread) field.get(loader); - IllegalStateException failure = new IllegalStateException("original loading failure"); - loader.context().failLoading(failure); - try { + public void testClosedLoaderUnregistersOwnHook() throws Exception { + for (boolean failed : new boolean[]{false, true}) { + try (ModeServer server = new ModeServer()) { + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = this.folder.newFile("hook-" + failed + ".json").getAbsolutePath(); + HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList())); + Field field = HugeGraphLoader.class.getDeclaredField("shutdownHook"); + field.setAccessible(true); + Thread hook = (Thread) field.get(loader); + RuntimeException failure = new IllegalStateException("loading failed"); try { + if (failed) { + loader.context().failLoading(failure); + try { + loader.shutdown(); + Assert.fail("shutdown must retain the loading failure"); + } catch (RuntimeException actual) { + Assert.assertSame(failure, actual); + } + } else { + loader.shutdown(); + } + Assert.assertTrue(loader.context().closed()); + Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); loader.shutdown(); - Assert.fail("shutdown must retain the loading failure"); - } catch (IllegalStateException actual) { - Assert.assertSame(failure, actual); + } finally { + Runtime.getRuntime().removeShutdownHook(hook); } - Assert.assertTrue(loader.context().closed()); - Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); - loader.shutdown(); - } finally { - Runtime.getRuntime().removeShutdownHook(hook); } } } - private static HugeGraphLoader loader(LoadContext context, InputStruct... inputs) throws Exception { - HugeGraphLoader loader = allocate(HugeGraphLoader.class); - set(loader, "context", context); - set(loader, "options", context.options()); - set(loader, "mapping", new LoadMapping(Arrays.asList(inputs))); - return loader; - } - private static class ModeServer implements AutoCloseable { private final HttpServer server; diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/TaskManagerTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/TaskManagerTest.java index deb89442f..46eeeb91e 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/TaskManagerTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/TaskManagerTest.java @@ -593,9 +593,6 @@ private void assertInterruptedShutdown(boolean cancel, boolean timeout, boolean setField(loader, "context", context); setField(loader, "manager", manager); setField(loader, "options", options); - Thread hook = new Thread(loader::shutdown); - setField(loader, "shutdownHook", hook); - Runtime.getRuntime().addShutdownHook(hook); ExecutorService caller = Executors.newSingleThreadExecutor(); try { manager.submitBatch(struct, mapping, Arrays.asList(new Record("in-flight", new Edge("knows")))); @@ -623,11 +620,6 @@ private void assertInterruptedShutdown(boolean cancel, boolean timeout, boolean } Assert.assertTrue(shutdown.get(5, TimeUnit.SECONDS)); Assert.assertEquals(!timeout, context.closed()); - Assert.assertEquals(timeout, Runtime.getRuntime().removeShutdownHook(hook)); - if (timeout) { - // An unfinished owner keeps its native hook and can close on retry. - Runtime.getRuntime().addShutdownHook(hook); - } Assert.assertFalse(client.closedWhileRunning); Assert.assertFalse(context.noError()); if (!timeout) { @@ -642,14 +634,7 @@ private void assertInterruptedShutdown(boolean cancel, boolean timeout, boolean Assert.assertEquals(1L, client.closed.getCount()); } Assert.assertFalse(new File(LoadProgress.format(options, "test")).exists()); - if (timeout) { - graph.release.countDown(); - Assert.assertThrows(LoadException.class, loader::shutdown); - Assert.assertTrue(context.closed()); - Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); - } } finally { - Runtime.getRuntime().removeShutdownHook(hook); graph.release.countDown(); caller.shutdownNow(); caller.awaitTermination(5, TimeUnit.SECONDS); From 8757dc8886294ca21a81a00574567784fce74e62 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 18:17:34 +0800 Subject: [PATCH 08/14] docs(hubble): clarify initial task cleanup - limit unused-loader cleanup to initial task creation - retain task-exit and built-in example guidance - leave resume and retry failures in the deferred scope --- hugegraph-hubble/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index 727cbb129..99cd73efc 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -51,8 +51,8 @@ Use matching Server, PD, and Store minor versions in a distributed cluster. **Se ## Import resource cleanup -Import tasks close their Loader before publishing completion. The built-in example import also closes its Loader; -tasks that cannot be saved or dispatched, or whose transaction rolls back, close their unused Loader. +Import tasks close their Loader before publishing completion. The built-in example import also closes its Loader. +For new tasks, a failed initial save or dispatch, or rollback of the outer creation transaction, closes the unused Loader. Shutdown failures are reported as task failures. ## Local development feedback loop From 2b2d66c6e4d6ab62a6dfb85ba2e2887f71ad11bb Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 14 Oct 2026 05:30:21 +0800 Subject: [PATCH 09/14] fix(hubble): preserve failures during cleanup Close loaders when resume or retry dispatch fails. Preserve stopped states when shutdown rethrows the load failure. Report independent cleanup errors without replacing the primary cause. Cover native loader exception and dispatch boundaries. --- hugegraph-hubble/README.md | 7 +- .../hugegraph/entity/load/LoadTask.java | 53 +++- .../service/load/LoadTaskService.java | 30 +- .../hugegraph/unit/LoadTaskServiceTest.java | 277 ++++++++++++++++++ 4 files changed, 356 insertions(+), 11 deletions(-) diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index 99cd73efc..707b962c8 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -52,8 +52,11 @@ Use matching Server, PD, and Store minor versions in a distributed cluster. **Se ## Import resource cleanup Import tasks close their Loader before publishing completion. The built-in example import also closes its Loader. -For new tasks, a failed initial save or dispatch, or rollback of the outer creation transaction, closes the unused Loader. -Shutdown failures are reported as task failures. +A failed save/update or dispatch closes the unused Loader, including resumed and retried tasks. +Rollback of the outer creation transaction also closes the new Loader. +Dispatch rejection marks the task and its job as failed. An independent shutdown failure marks the task as failed; +check the logs and graph loading mode before retrying. A shutdown that only rethrows the existing load failure preserves +a user-requested paused or stopped status. ## Local development feedback loop diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java index c33ce30b7..3766e3d93 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java @@ -28,6 +28,7 @@ import org.apache.hugegraph.entity.enums.LoadStatus; import org.apache.hugegraph.exception.InternalException; import org.apache.hugegraph.loader.HugeGraphLoader; +import org.apache.hugegraph.loader.exception.LoadException; import org.apache.hugegraph.loader.executor.LoadContext; import org.apache.hugegraph.loader.executor.LoadOptions; import org.apache.hugegraph.util.Ex; @@ -185,14 +186,37 @@ public void run() { } catch (Throwable e) { failure = e; } finally { + Throwable recordedFailure = failure; + if (failure != null) { + // Only read the recorded error: shutdown can wrap a checked + // failure differently from load(), or rethrow it unchanged. + try { + this.context().throwIfFailed(); + } catch (Throwable recorded) { + recordedFailure = recorded; + if (recorded != failure) { + // Retain an asynchronously recorded error even when it + // differs from the error propagated by load(). + log.error("Recorded load failure for task {}", this.id, recorded); + } + } + } + Throwable recordedCause = recordedFailure == null ? null : recordedFailure.getCause(); + int suppressed = recordedFailure == null ? 0 : recordedFailure.getSuppressed().length; + int causeSuppressed = recordedCause == null ? 0 : recordedCause.getSuppressed().length; try { this.loader.shutdown(); } catch (Throwable e) { noError = false; - cleanupFailed = true; + cleanupFailed = !sameLoadFailure(recordedFailure, e) || + recordedFailure.getSuppressed().length != suppressed || + (recordedCause != null && recordedCause.getSuppressed().length != causeSuppressed); + if (cleanupFailed) { + log.error("Failed to close load task {}; check graph loading mode before retrying", this.id, e); + } if (failure == null) { failure = e; - } else if (failure != e) { + } else if (cleanupFailed && failure != e && !referencesFailure(e, failure)) { failure.addSuppressed(e); } } @@ -225,6 +249,31 @@ public void run() { } } + private static boolean sameLoadFailure(Throwable recorded, Throwable cleanup) { + if (recorded == null) { + return false; + } + if (recorded == cleanup) { + return true; + } + // LoadContext.throwIfFailed() wraps only non-runtime failures this way. + return recorded.getClass() == LoadException.class && cleanup.getClass() == LoadException.class && + recorded.getCause() != null && !(recorded.getCause() instanceof RuntimeException) && + recorded.getCause() == cleanup.getCause() && cleanup.getSuppressed().length == 0; + } + + private static boolean referencesFailure(Throwable cleanup, Throwable failure) { + if (cleanup.getCause() == failure) { + return true; + } + for (Throwable suppressed : cleanup.getSuppressed()) { + if (suppressed == failure) { + return true; + } + } + return false; + } + public void lock() { this.lock.lock(); } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java index 366c5074a..2615a86b1 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java @@ -269,6 +269,16 @@ private void executeSafely(LoadTask task) { } catch (RuntimeException e) { closeUnscheduled(task, e); this.markDispatchFailed(task, e); + } catch (Error failure) { + closeUnscheduled(task, failure); + try { + this.markDispatchFailed(task, failure); + } catch (Error persistenceFailure) { + if (persistenceFailure != failure) { + failure.addSuppressed(persistenceFailure); + } + } + throw failure; } } @@ -287,7 +297,7 @@ private static void closeUnscheduled(LoadTask task, Throwable failure) { } } - private void markDispatchFailed(LoadTask task, RuntimeException cause) { + private void markDispatchFailed(LoadTask task, Throwable cause) { log.error("Failed to dispatch load task {}, marking task and job " + "as FAILED", task.getId(), cause); task.setStatus(LoadStatus.FAILED); @@ -330,6 +340,16 @@ private void markDispatchFailed(LoadTask task, RuntimeException cause) { } } + private void updateAndExecute(LoadTask task) { + try { + this.update(task); + } catch (RuntimeException | Error failure) { + closeUnscheduled(task, failure); + throw failure; + } + this.executeSafely(task); + } + private void execute(LoadTask task) { // Executed in other threads this.taskExecutor.execute(task, () -> this.update(task)); @@ -378,9 +398,7 @@ public LoadTask resume(int taskId, String token) { task.getOptions().incrementalMode = true; task.reconnect(token); task.setStatus(LoadStatus.RUNNING); - this.update(task); - this.taskExecutor.execute(task, () -> this.update(task)); - this.runningTaskContainer.put(taskId, task); + this.updateAndExecute(task); } finally { task.unlock(); } @@ -431,9 +449,7 @@ public LoadTask retry(int taskId, String token) { task.setStatus(LoadStatus.RUNNING); task.setLastDuration(0L); task.setCurrDuration(0L); - this.update(task); - this.taskExecutor.execute(task, () -> this.update(task)); - this.runningTaskContainer.put(taskId, task); + this.updateAndExecute(task); } finally { task.unlock(); } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java index 9d7407a19..2e97180aa 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java @@ -18,12 +18,20 @@ package org.apache.hugegraph.unit; +import java.io.OutputStream; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collections; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.apache.hugegraph.entity.GraphConnection; import org.apache.hugegraph.entity.enums.LoadStatus; @@ -36,6 +44,9 @@ import org.apache.hugegraph.loader.HugeGraphLoader; import org.apache.hugegraph.loader.executor.LoadContext; import org.apache.hugegraph.loader.executor.LoadOptions; +import org.apache.hugegraph.loader.mapping.LoadMapping; +import org.apache.hugegraph.loader.util.LoadUtil; +import org.apache.hugegraph.loader.exception.LoadException; import org.apache.hugegraph.loader.source.file.FileSource; import org.apache.hugegraph.service.load.LoadTaskService; import org.apache.hugegraph.mapper.load.LoadTaskMapper; @@ -43,13 +54,19 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockito.MockedConstruction; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.SimpleTransactionStatus; import org.springframework.transaction.support.TransactionSynchronization; import org.springframework.transaction.support.TransactionSynchronizationManager; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.sun.net.httpserver.HttpServer; public class LoadTaskServiceTest { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + @Test public void testRunnerClosesBeforePublishingCompletion() { for (boolean success : new boolean[]{true, false}) { @@ -73,6 +90,266 @@ public void testRunnerClosesBeforePublishingCompletion() { } } + @Test + public void testCancelledLoadFailureDoesNotOverwriteRequestedStatus() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/versions", exchange -> { + byte[] body = "{\"versions\":{\"core\":\"1.8.0\",\"api\":\"0.74\"}}" + .getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(body); + } + }); + server.start(); + try { + for (LoadStatus status : new LoadStatus[]{LoadStatus.PAUSED, LoadStatus.STOPPED}) { + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.getAddress().getPort(); + options.file = this.folder.newFile("cancel-" + status + ".json").getAbsolutePath(); + HugeGraphLoader loader = Mockito.spy(new HugeGraphLoader( + options, new LoadMapping(Collections.emptyList()))); + IllegalStateException failure = new IllegalStateException("load failed"); + LoadTask task = LoadTask.builder().id(9).loader(loader).options(options) + .status(LoadStatus.RUNNING).lastDuration(0L).build(); + Mockito.doAnswer(invocation -> { + task.setStatus(status); + loader.context().failLoading(failure); + throw failure; + }).when(loader).load(); + Field field = HugeGraphLoader.class.getDeclaredField("shutdownHook"); + field.setAccessible(true); + Thread hook = (Thread) field.get(loader); + try { + task.run(); + + Assert.assertTrue(loader.context().closed()); + Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(status, task.getStatus()); + Assert.assertEquals(0, failure.getSuppressed().length); + } finally { + loader.shutdown(); + Runtime.getRuntime().removeShutdownHook(hook); + } + } + } finally { + server.stop(0); + } + } + + @Test + public void testCancelledLoadWithIndependentCleanupFailureIsFailed() { + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + IllegalStateException failure = new IllegalStateException("load failed"); + IllegalStateException cleanup = new IllegalStateException("cleanup failed"); + Mockito.when(loader.load()).thenThrow(failure); + Mockito.doThrow(cleanup).when(loader).shutdown(); + LoadTask task = LoadTask.builder().id(9).loader(loader).options(new LoadOptions()) + .status(LoadStatus.STOPPED).lastDuration(0L).build(); + + task.run(); + + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertEquals(1, failure.getSuppressed().length); + Assert.assertSame(cleanup, failure.getSuppressed()[0]); + } + + @Test + public void testCancelledRecordedLoadFailureRethrowKeepsRequestedStatus() { + for (Throwable original : new Throwable[]{new IllegalStateException("load failed"), + new Exception("load failed"), + new AssertionError("load failed"), + new UndeclaredThrowableException( + new RuntimeException(new IllegalStateException("wrapped")))}) { + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + RuntimeException loadFailure = LoadUtil.targetRuntimeException(original); + RuntimeException recorded = original instanceof RuntimeException ? (RuntimeException) original : + new LoadException("recorded failure", original); + RuntimeException rethrown = original instanceof RuntimeException ? (RuntimeException) original : + new LoadException("recorded failure", original); + Mockito.when(loader.load()).thenThrow(loadFailure); + Mockito.doThrow(recorded).when(context).throwIfFailed(); + Mockito.doThrow(rethrown).when(loader).shutdown(); + LoadTask task = LoadTask.builder().id(9).loader(loader).options(new LoadOptions()) + .status(LoadStatus.PAUSED).lastDuration(0L).build(); + + task.run(); + + Assert.assertEquals(LoadStatus.PAUSED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(0, loadFailure.getSuppressed().length); + } + } + + @Test + public void testCancelledRethrowWithNewSuppressedCleanupFailureIsFailed() { + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + IllegalStateException failure = new IllegalStateException("load failed"); + IllegalStateException cleanup = new IllegalStateException("close failed"); + Mockito.when(loader.load()).thenThrow(failure); + Mockito.doThrow(failure).when(context).throwIfFailed(); + Mockito.doAnswer(invocation -> { + failure.addSuppressed(cleanup); + throw failure; + }).when(loader).shutdown(); + LoadTask task = LoadTask.builder().id(9).loader(loader).options(new LoadOptions()) + .status(LoadStatus.STOPPED).lastDuration(0L).build(); + + task.run(); + + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + Assert.assertSame(cleanup, failure.getSuppressed()[0]); + } + + @Test + public void testAfterCommitDispatchErrorClosesLoaderAndPreservesError() throws Exception { + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadTask task = LoadTask.builder().id(9).loader(loader).options(new LoadOptions()) + .status(LoadStatus.RUNNING).build(); + Error failure = new AssertionError("dispatch failed"); + IllegalStateException cleanup = new IllegalStateException("close failed"); + Mockito.doThrow(failure).when(executor).execute(Mockito.eq(task), Mockito.any()); + Mockito.doThrow(cleanup).when(loader).shutdown(); + Mockito.when(mapper.update(Mockito.isNull(), Mockito.any())).thenReturn(1); + this.setField(service, "mapper", mapper); + this.setField(service, "taskExecutor", executor); + this.setField(service, "runningTaskContainer", new ConcurrentHashMap<>()); + PlatformTransactionManager transactions = Mockito.mock(PlatformTransactionManager.class); + Mockito.when(transactions.getTransaction(Mockito.any())).thenReturn(new SimpleTransactionStatus()); + this.setField(service, "transactionManager", transactions); + Method dispatch = LoadTaskService.class.getDeclaredMethod("executeAfterCommit", LoadTask.class); + dispatch.setAccessible(true); + TransactionSynchronizationManager.initSynchronization(); + try { + dispatch.invoke(service, task); + TransactionSynchronization synchronization = + TransactionSynchronizationManager.getSynchronizations().get(0); + try { + synchronization.afterCommit(); + Assert.fail("Expected dispatch Error"); + } catch (Error actual) { + Assert.assertSame(failure, actual); + } + synchronization.afterCompletion(TransactionSynchronization.STATUS_COMMITTED); + Mockito.verify(loader).shutdown(); + Mockito.verify(mapper).update(Mockito.isNull(), Mockito.any()); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertSame(cleanup, failure.getSuppressed()[0]); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + } + + @Test + public void testCleanupFailureWithoutLoadFailureStillMarksTaskFailed() { + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(loader.load()).thenReturn(true); + Mockito.doThrow(new IllegalStateException("cleanup failed")).when(loader).shutdown(); + LoadTask task = LoadTask.builder().id(9).loader(loader).options(new LoadOptions()) + .status(LoadStatus.PAUSED).lastDuration(0L).build(); + + task.run(); + + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + } + + @Test + public void testResumeAndRetryCloseLoaderOnUpdateFailure() throws Exception { + for (boolean resume : new boolean[]{true, false}) { + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadTask task = Mockito.spy(LoadTask.builder().id(9).loader(loader) + .options(new LoadOptions()) + .status(resume ? LoadStatus.PAUSED : LoadStatus.STOPPED) + .build()); + Mockito.doNothing().when(task).reconnect("current-token"); + Mockito.when(mapper.selectById(9)).thenReturn(task); + IllegalStateException failure = new IllegalStateException("update failed"); + IllegalStateException cleanup = new IllegalStateException("cleanup failed"); + Mockito.when(mapper.updateById(task)).thenThrow(failure); + Mockito.doThrow(cleanup).when(loader).shutdown(); + ConcurrentHashMap running = new ConcurrentHashMap<>(); + this.setField(service, "mapper", mapper); + this.setField(service, "taskExecutor", executor); + this.setField(service, "runningTaskContainer", running); + + try { + if (resume) { + service.resume(9, "current-token"); + } else { + service.retry(9, "current-token"); + } + Assert.fail("Expected update failure"); + } catch (IllegalStateException actual) { + Assert.assertSame(failure, actual); + Assert.assertEquals(1, actual.getSuppressed().length); + Assert.assertSame(cleanup, actual.getSuppressed()[0]); + } + Mockito.verify(loader).shutdown(); + Assert.assertTrue(task.isFinished()); + Assert.assertTrue(running.isEmpty()); + Mockito.verify(executor, Mockito.never()).execute(Mockito.any(), Mockito.any()); + } + } + + @Test + public void testResumeAndRetryCloseLoaderOnDispatchRejection() throws Exception { + for (boolean resume : new boolean[]{true, false}) { + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadTask task = Mockito.spy(LoadTask.builder().id(9).loader(loader) + .options(new LoadOptions()) + .status(resume ? LoadStatus.PAUSED : LoadStatus.STOPPED) + .build()); + Mockito.doNothing().when(task).reconnect("current-token"); + Mockito.when(mapper.selectById(9)).thenReturn(task); + Mockito.when(mapper.updateById(task)).thenReturn(1); + Mockito.doThrow(new RejectedExecutionException("queue full")) + .when(executor).execute(Mockito.eq(task), Mockito.any()); + ConcurrentHashMap running = new ConcurrentHashMap<>(); + this.setField(service, "mapper", mapper); + this.setField(service, "taskExecutor", executor); + this.setField(service, "runningTaskContainer", running); + PlatformTransactionManager transactions = Mockito.mock(PlatformTransactionManager.class); + Mockito.when(transactions.getTransaction(Mockito.any())) + .thenReturn(new SimpleTransactionStatus()); + Mockito.when(mapper.update(Mockito.isNull(), Mockito.any())).thenReturn(1); + this.setField(service, "transactionManager", transactions); + + LoadTask result = resume ? service.resume(9, "current-token") : + service.retry(9, "current-token"); + + Assert.assertSame(task, result); + Mockito.verify(loader).shutdown(); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertTrue(running.isEmpty()); + Mockito.verify(mapper).update(Mockito.isNull(), Mockito.any()); + } + } + @Test public void testPreviewAlwaysClosesLoader() { for (boolean success : new boolean[]{true, false}) { From 5e0389b364ec268336920a9cf2508ee6156d4bc3 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 14 Oct 2026 12:43:14 +0800 Subject: [PATCH 10/14] fix(loader): preserve mode reset failures - retain independent mode reset errors during shutdown - preserve direct fatal cleanup errors and identity - prevent cyclic cleanup exception diagnostics - cover actual loader resource cleanup and reset failures --- .../hugegraph/loader/HugeGraphLoader.java | 38 ++- .../loader/test/unit/LoadContextTest.java | 258 ++++++++++++++++++ 2 files changed, 294 insertions(+), 2 deletions(-) diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java index fa558e641..a9565c0d5 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java @@ -20,9 +20,11 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -941,9 +943,19 @@ private void drainThenClose() { if (tasksStopped) { try { this.context.unsetLoadingMode(); - } finally { - this.context.close(this.loadStarted); + } catch (Throwable resetFailure) { + try { + this.context.close(this.loadStarted); + } catch (Throwable closeFailure) { + if (closeFailure instanceof Error && !(resetFailure instanceof Error)) { + suppressCleanupFailure(closeFailure, resetFailure); + throw (Error) closeFailure; + } + suppressCleanupFailure(resetFailure, closeFailure); + } + throw resetFailure; } + this.context.close(this.loadStarted); } } finally { if (interrupted) { @@ -953,6 +965,28 @@ private void drainThenClose() { } } + private static void suppressCleanupFailure(Throwable primary, Throwable secondary) { + Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); + if (!containsFailure(secondary, primary, seen)) { + primary.addSuppressed(secondary); + } + } + + private static boolean containsFailure(Throwable failure, Throwable target, Set seen) { + if (failure == null || !seen.add(failure)) { + return false; + } + if (failure == target || containsFailure(failure.getCause(), target, seen)) { + return true; + } + for (Throwable suppressed : failure.getSuppressed()) { + if (containsFailure(suppressed, target, seen)) { + return true; + } + } + return false; + } + private void cleanupEmptyProgress() { Map inputProgressMap = this.context.newProgress().inputProgress(); inputProgressMap.entrySet().removeIf(entry -> entry.getValue().loadedItems().isEmpty()); diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java index d59e54d39..2fb806903 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java @@ -31,8 +31,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import org.apache.hugegraph.driver.GraphsManager; import org.apache.hugegraph.driver.HugeClient; import org.apache.hugegraph.driver.HugeClientBuilder; import org.apache.hugegraph.exception.ServerException; @@ -65,6 +69,8 @@ import com.sun.net.httpserver.HttpServer; +import okhttp3.OkHttpClient; + public class LoadContextTest { @Rule @@ -735,6 +741,258 @@ public void testClosedLoaderUnregistersOwnHook() throws Exception { } } + @Test + public void testLoaderResetFailurePreservesIndependentAndLoadingFailures() throws Exception { + this.assertLoaderShutdownFailure(true); + } + + @Test + public void testLoaderSuccessfulResetPreservesLoadingFailureIdentity() throws Exception { + this.assertLoaderShutdownFailure(false); + } + + private void assertLoaderShutdownFailure(boolean rejectReset) throws Exception { + try (ModeServer server = new ModeServer()) { + server.mode = GraphMode.NONE; + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = this.folder.newFile("reset-" + rejectReset + ".json").getAbsolutePath(); + HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList())); + Thread hook = (Thread) field(loader, "shutdownHook"); + LoadContext context = loader.context(); + Object manager = field(loader, "manager"); + ExecutorService batch = (ExecutorService) field(manager, "batchService"); + ExecutorService single = (ExecutorService) field(manager, "singleService"); + try { + // Start both actual native pools; shutdown must drain and terminate them. + batch.submit(() -> { }).get(); + single.submit(() -> { }).get(); + context.setLoadingMode(); + OkHttpClient transport = (OkHttpClient) field(field(context.client(), "client"), "client"); + RuntimeException original = new IllegalStateException("recorded loading failure"); + context.failLoading(original); + if (rejectReset) { + server.failure = "Independent reset denied"; + } + RuntimeException actual = null; + try { + loader.shutdown(); + Assert.fail("Shutdown must report the recorded failure"); + } catch (RuntimeException failure) { + actual = failure; + } + if (rejectReset) { + Assert.assertTrue("Reset failure must remain primary", actual instanceof LoadException); + Assert.assertNotSame(original, actual); + Assert.assertTrue(actual.getCause() instanceof ServerException); + Assert.assertEquals("Independent reset denied", actual.getCause().getMessage()); + Assert.assertEquals(1, actual.getSuppressed().length); + Assert.assertSame(original, actual.getSuppressed()[0]); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(Boolean.TRUE, field(context, "loadingModeSet")); + } else { + Assert.assertSame(original, actual); + Assert.assertEquals(0, actual.getSuppressed().length); + Assert.assertEquals(GraphMode.NONE, server.mode); + Assert.assertEquals(Boolean.FALSE, field(context, "loadingModeSet")); + } + assertAcyclic(actual, Collections.newSetFromMap(new IdentityHashMap<>())); + Assert.assertTrue(context.closed()); + Assert.assertTrue(batch.isTerminated()); + Assert.assertTrue(single.isTerminated()); + Assert.assertTrue(transport.dispatcher().executorService().isShutdown()); + Assert.assertEquals(0, transport.connectionPool().connectionCount()); + Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); + Assert.assertEquals(1, server.modeReads); + Assert.assertEquals(2, server.modeWrites); + loader.shutdown(); + Assert.assertEquals(1, server.modeReads); + Assert.assertEquals(2, server.modeWrites); + } finally { + server.failure = null; + try { + loader.shutdown(); + } catch (RuntimeException ignored) { + // Retain the original assertion while closing the actual native resources. + } + Runtime.getRuntime().removeShutdownHook(hook); + } + } + } + + @Test + public void testLoaderPromotesFatalCloseOverOrdinaryResetFailure() throws Exception { + this.assertFatalLoaderShutdown(false, false, false); + } + + @Test + public void testLoaderPreservesFatalResetAndRecordedLoadingFailure() throws Exception { + this.assertFatalLoaderShutdown(true, false, false); + } + + @Test + public void testLoaderDoesNotSuppressFatalFailureOnItself() throws Exception { + this.assertFatalLoaderShutdown(true, true, false); + } + + @Test + public void testLoaderDoesNotCreateFatalResetCauseCycle() throws Exception { + this.assertFatalLoaderShutdown(true, false, true); + } + + private void assertFatalLoaderShutdown(boolean fatalReset, boolean sameError, + boolean causeCycle) throws Exception { + try (ModeServer server = new ModeServer()) { + server.mode = GraphMode.NONE; + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = this.folder.newFile().getAbsolutePath(); + HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList())); + LoadContext context = loader.context(); + Thread hook = (Thread) field(loader, "shutdownHook"); + // Use real SDK clients; only the fatal accessor / post-close boundary is controlled. + BoundaryClient primary = new BoundaryClient(server); + HugeClient secondary = server.client(); + context.client().close(); + set(context, "client", primary); + set(context, "indirectClient", secondary); + OkHttpClient transport = (OkHttpClient) field(field(primary, "client"), "client"); + OkHttpClient secondaryTransport = (OkHttpClient) field(field(secondary, "client"), "client"); + Object manager = field(loader, "manager"); + ExecutorService batch = (ExecutorService) field(manager, "batchService"); + ExecutorService single = (ExecutorService) field(manager, "singleService"); + Error fatal = new AssertionError("fatal SDK cleanup boundary"); + RuntimeException original = causeCycle ? new IllegalStateException("loading failed", fatal) : + new IllegalStateException("recorded loading failure"); + try { + batch.submit(() -> { }).get(); + single.submit(() -> { }).get(); + context.setLoadingMode(); + if (fatalReset) { + primary.resetFailure = fatal; + if (!sameError) { + context.failLoading(original); + } + } else { + server.failure = "Independent reset denied"; + } + primary.closeFailure = !fatalReset || sameError ? fatal : null; + Throwable actual = null; + try { + loader.shutdown(); + } catch (RuntimeException | Error failure) { + actual = failure; + } + Assert.assertSame("First fatal Error must escape with its identity", fatal, actual); + if (causeCycle || sameError) { + Assert.assertEquals(0, fatal.getSuppressed().length); + } else { + Assert.assertEquals(1, fatal.getSuppressed().length); + if (fatalReset) { + Assert.assertSame(original, fatal.getSuppressed()[0]); + } else { + Assert.assertTrue(fatal.getSuppressed()[0] instanceof LoadException); + Assert.assertEquals("Independent reset denied", + fatal.getSuppressed()[0].getCause().getMessage()); + } + } + if (fatalReset && !sameError) { + try { + context.throwIfFailed(); + Assert.fail("Context must retain the original recorded failure"); + } catch (RuntimeException failure) { + Assert.assertSame(original, failure); + } + } + assertAcyclic(fatal, Collections.newSetFromMap(new IdentityHashMap<>())); + Assert.assertTrue(context.closed()); + Assert.assertTrue(batch.isTerminated()); + Assert.assertTrue(single.isTerminated()); + Assert.assertTrue(transport.dispatcher().executorService().isShutdown()); + Assert.assertEquals(0, transport.connectionPool().connectionCount()); + Assert.assertTrue(secondaryTransport.dispatcher().executorService().isShutdown()); + Assert.assertEquals(0, secondaryTransport.connectionPool().connectionCount()); + Assert.assertEquals(1, primary.closeCalls); + Assert.assertEquals(GraphMode.LOADING, server.mode); + Assert.assertEquals(Boolean.TRUE, field(context, "loadingModeSet")); + Assert.assertFalse(Runtime.getRuntime().removeShutdownHook(hook)); + int reads = server.modeReads; + int writes = server.modeWrites; + loader.shutdown(); + Assert.assertEquals(reads, server.modeReads); + Assert.assertEquals(writes, server.modeWrites); + Assert.assertEquals(1, primary.closeCalls); + } finally { + server.failure = null; + primary.resetFailure = null; + primary.closeFailure = null; + try { + loader.shutdown(); + } catch (RuntimeException | Error ignored) { + // Preserve assertions while tearing down the real native resources. + } + primary.close(); + secondary.close(); + Runtime.getRuntime().removeShutdownHook(hook); + } + } + } + + private static class BoundaryClient extends HugeClient { + + private Error resetFailure; + private Error closeFailure; + private int closeCalls; + + private BoundaryClient(ModeServer server) { + super(HugeClient.builder("http://127.0.0.1:" + server.server.getAddress().getPort(), "hugegraph")); + } + + @Override + public GraphsManager graphs() { + if (this.resetFailure != null) { + throw this.resetFailure; + } + return super.graphs(); + } + + @Override + public void close() { + this.closeCalls++; + super.close(); + if (this.closeFailure != null) { + throw this.closeFailure; + } + } + } + + private static void assertAcyclic(Throwable failure, Set path) { + if (failure == null) { + return; + } + Assert.assertTrue("Failure graph must not contain cycles", path.add(failure)); + assertAcyclic(failure.getCause(), path); + for (Throwable suppressed : failure.getSuppressed()) { + assertAcyclic(suppressed, path); + } + path.remove(failure); + } + + private static Object field(Object target, String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // SDK transport fields are declared in its existing superclass. + } + } + throw new NoSuchFieldException(name); + } + private static class ModeServer implements AutoCloseable { private final HttpServer server; From ffa63182fdd0acab220284d57010335f8c65f529 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 15 Oct 2026 02:00:33 +0800 Subject: [PATCH 11/14] fix(hubble): preserve resumable task cleanup - close embedded loaders without saving parsed offsets - wait for callbacks before short metadata deletions - preserve cleanup evidence and document operator recovery - verify pause and failed resume through native loading --- hugegraph-hubble/README.md | 47 + .../controller/ingest/IngestController.java | 2 +- .../load/FileMappingController.java | 22 +- .../controller/load/FileUploadController.java | 353 ++-- .../hugegraph/entity/load/LoadTask.java | 34 +- .../service/load/FileMappingService.java | 64 +- .../service/load/JobManagerService.java | 161 +- .../service/load/LoadTaskService.java | 908 ++++++-- .../IngestTransactionIntegrationTest.java | 640 +++++- .../load/JobAdmissionIntegrationTest.java | 1819 +++++++++++++++++ .../testutil/NativeCleanupTestSupport.java | 171 ++ .../unit/FileUploadControllerTest.java | 27 +- .../hugegraph/unit/JobManagerServiceTest.java | 205 +- .../unit/LoadTaskCheckpointTest.java | 259 +++ .../hugegraph/unit/LoadTaskServiceTest.java | 129 +- .../unit/LoaderScopeControllerTest.java | 4 + .../apache/hugegraph/unit/UnitTestSuite.java | 2 + .../hugegraph/loader/HugeGraphLoader.java | 17 +- .../loader/test/unit/LoadContextTest.java | 44 + 19 files changed, 4540 insertions(+), 368 deletions(-) create mode 100644 hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/JobAdmissionIntegrationTest.java create mode 100644 hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/testutil/NativeCleanupTestSupport.java create mode 100644 hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCheckpointTest.java diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index 707b962c8..6a4596064 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -57,6 +57,52 @@ Rollback of the outer creation transaction also closes the new Loader. Dispatch rejection marks the task and its job as failed. An independent shutdown failure marks the task as failed; check the logs and graph loading mode before retrying. A shutdown that only rethrows the existing load failure preserves a user-requested paused or stopped status. +Hubble closes its Loaders without saving parsed file offsets as checkpoints. Pause/resume and FAILED-task resume +therefore reread the source from its existing checkpoint (or the beginning), including rows that were parsed but never +inserted. Already inserted rows may be replayed, as before this cleanup change. The standalone Loader retains its +existing checkpoint behavior. Checkpoints written by older builds are not invalidated automatically. If their progress +may be unsafe, stop a paused task and use Retry to reread the entire source instead of resuming that checkpoint. +Deleting a job signals all accepted tasks to stop before waiting for their native cleanup and final status callbacks. +Single-task, job and source-file deletion wait for those callbacks before starting the short metadata transaction. +An internal caller that already owns a database transaction is refused before changing data if callbacks remain pending; +finish that transaction and retry. Completed tasks can still be deleted within the caller's existing transaction. +A final callback committed before deletion remains committed if that deletion later rolls back. A task can complete cleanup after the +task-list snapshot. Existing task IDs stay reserved +against retry/resume until the deletion transaction commits or rolls back. An independent cleanup failure preserves +the task metadata and runtime Loader, including when unused-Loader shutdown fails. Single-task deletion uses the same +stop and cleanup checks. Runtime handles remain until deletion commits. After rollback, an already stopped task stays +in memory while its persisted status may still be RUNNING or PAUSED. Ordinary stop can persist STOPPED using that +completed-cleanup certificate without closing the Loader again; another deletion can also reuse it. +Task creation and job deletion are serialized within the same Hubble process. A new task checks that its parent job +and graph still exist before constructing a Loader or inserting metadata; deletion includes tasks admitted before its +snapshot. Runtime ownership is published before asynchronous execution can begin. Multi-file requests keep their +existing per-file behavior; these guards do not coordinate separate Hubble processes. +The task-page deletion route uses the same job cleanup. Upload requests and source-file detach share the job +reservation, so deletion includes an admitted upload and a detached source cannot start from a stale mapping. +Different files of the same job can upload concurrently. Chunks for the same job and filename are serialized so a merge +only sees completed chunks. This limits concurrency within a single file, while retaining parallel I/O across files. +Tokens are opaque and bound to the job and filename. Committed job deletion invalidates even unused tokens; +committed mapping deletion invalidates its pending upload tokens. Rollback keeps them available. Background +cleanup also invalidates the affected upload before deleting its files, without cancelling same-name files in another job. +Clear-all takes the job admission gate before reading its mapping list, so it includes uploads already in progress. +For partial uploads, different names that the configured filesystem resolves to the same directory are rejected +before writing data. Native filesystem identity is used; distinct case-sensitive filenames remain allowed. +Token issuance waits for same-file failed-upload cleanup before returning a replacement token. +Token issuance does not create upload directories; a failed mapping reservation removes only its new empty directory. +Mappings needed by running, paused, failed or stopped tasks remain attached. Cleanup failures before a task INSERT +or after transaction rollback keep their runtime handle and prevent parent-job deletion even if no task row remains. +Retry/resume refuse to replace unfinished or cleanup-failed runtimes. After native cleanup, they wait for the previous +status callback; its failure is surfaced without starting a replacement Loader. Stop and drain all imports, have an administrator +inspect and recover the graph mode, then restart Hubble before retrying. + +Database failures during progress polling or after rejected dispatch require operator recovery; this change does not +add automatic per-task persistence repair. A failed final write can leave a task RUNNING or its job LOADING in metadata. +Hubble keeps failed callback/cleanup evidence and refuses replacement/deletion when cleanup cannot be confirmed. +Recover the metadata database first, stop Hubble and other imports, and have an administrator verify that no Loader +still uses the graph before repairing an orphaned graph mode through the Server graph-mode API. Restart Hubble with +the same metadata database and source files. If a persisted task is still RUNNING with no active runner, use the existing +Pause action, then Resume (or Retry for a terminal task) with current authentication. These actions recheck persisted +state; they do not repair a database that is still unavailable. Do not manually delete metadata rows to bypass cleanup. ## Local development feedback loop @@ -259,6 +305,7 @@ Hubble 1.8 requires Java 17 and uses Spring Boot 3 with H2 2.x for metadata. MyS The startup script includes `--add-opens=java.base/java.net=ALL-UNNAMED` for the embedded Loader's Hive ORC reader. Include this option when using a custom Java launch command. Packaged archives are generated under `target/` and exclude local runtime data. + ## Graph switching Query execution waits until the graph context matches the current route, including keyboard shortcuts during a graph switch. Standalone Server 1.5 supports core graph, schema and data operations; GraphSpace management and PD mode require a newer server. diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java index 3fadd9af0..dc1bd7791 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/ingest/IngestController.java @@ -378,7 +378,7 @@ public Response deleteTask(@PathVariable("id") int id) { .message("Task not found: " + id).build(); } this.requireJobWrite(job); - jobManagerService.remove(id); + jobManagerService.deleteJob(id); return Response.builder().status(Constant.STATUS_OK).build(); } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileMappingController.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileMappingController.java index 43381c508..c4ed8f67f 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileMappingController.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileMappingController.java @@ -127,16 +127,20 @@ public void clear(@PathVariable("graphspace") String graphSpace, @PathVariable("graph") String graph, @PathVariable("jobId") int jobId) { this.requireGraphSpaceWrite(graphSpace); - List mappings = this.service.listByJob(graphSpace, graph, - jobId); - Set fileIds = new HashSet<>(); - for (FileMapping mapping : mappings) { - fileIds.add(mapping.getId()); - } - // Check them all upfront, don't delete part of the mappings then fail - this.checkNoRunningTask(jobId, fileIds); + try (var reservation = this.jobService.reserveJob(jobId)) { + Ex.check(this.jobService.get(graphSpace, graph, jobId) != null, + "job-manager.not-exist.id", jobId); + List mappings = this.service.listByJob(graphSpace, graph, + jobId); + Set fileIds = new HashSet<>(); + for (FileMapping mapping : mappings) { + fileIds.add(mapping.getId()); + } + // Check them all upfront, don't delete part of the mappings then fail + this.checkNoRunningTask(jobId, fileIds); - this.jobService.deleteMappings(jobId, mappings); + this.jobService.deleteMappings(jobId, mappings); + } } @PostMapping("{id}/file-setting") diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileUploadController.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileUploadController.java index cbbe7cbc6..5af232d34 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileUploadController.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileUploadController.java @@ -23,6 +23,8 @@ import java.io.File; import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; @@ -88,30 +90,35 @@ public Map fileToken( @RequestParam("names") List fileNames) { this.requireGraphSpaceWrite(graphSpace); - Ex.check(this.jobService.get(graphSpace, graph, jobId) != null, - "job-manager.not-exist.id", jobId); - Ex.check(CollectionUtil.allUnique(fileNames), - "load.upload.file.duplicate-name"); - Map tokens = new HashMap<>(); - Map reservations = new HashMap<>(); - try { - for (String fileName : fileNames) { - this.checkFileNameValid(fileName); - String token = this.service.generateFileToken(fileName); - ReadWriteLock lock = new ReentrantReadWriteLock(); - // Atomic reservation to avoid a containsKey-then-put race - ReadWriteLock previous = this.uploadingTokenLocks() - .putIfAbsent(token, lock); - Ex.check(previous == null, "load.upload.file.token.existed"); - reservations.put(token, lock); - tokens.put(fileName, token); + try (var reservation = this.jobService.reserveUpload(jobId)) { + Ex.check(this.jobService.get(graphSpace, graph, jobId) != null, + "job-manager.not-exist.id", jobId); + Ex.check(CollectionUtil.allUnique(fileNames), + "load.upload.file.duplicate-name"); + Map tokens = new HashMap<>(); + Map reservations = new HashMap<>(); + try { + for (String fileName : fileNames) { + this.checkFileNameValid(fileName); + synchronized (this.service) { + String token = this.service.generateFileToken(jobId, fileName); + // Aliases for one physical file share its existing native lock. + ReadWriteLock lock = this.uploadingTokenLocks().entrySet().stream() + .filter(entry -> this.service.matchesFileToken(jobId, fileName, entry.getKey())) + .map(Map.Entry::getValue).findFirst().orElseGet(ReentrantReadWriteLock::new); + ReadWriteLock previous = this.uploadingTokenLocks().putIfAbsent(token, lock); + Ex.check(previous == null, "load.upload.file.token.existed"); + reservations.put(token, lock); + tokens.put(fileName, token); + } + } + return tokens; + } catch (RuntimeException e) { + reservations.forEach((token, lock) -> { + this.uploadingTokenLocks().remove(token, lock); + }); + throw e; } - return tokens; - } catch (RuntimeException e) { - reservations.forEach((token, lock) -> { - this.uploadingTokenLocks().remove(token, lock); - }); - throw e; } } @@ -128,100 +135,108 @@ public FileUploadResult upload(@PathVariable("graphspace") String graphSpace, @RequestParam("total") int total, @RequestParam("index") int index) { this.requireGraphSpaceWrite(graphSpace); - this.checkTotalAndIndexValid(total, index); - this.checkFileNameValid(fileName); - this.checkFileNameMatchToken(fileName, token); - JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); - Long sourceFileSize = this.resolveSourceFileSize(file, fileSize, - total, index); - this.checkFileValid(graphSpace, graph, jobId, jobEntity, file, fileName); - if (jobEntity.getJobStatus() == JobStatus.DEFAULT) { - jobEntity.setJobStatus(JobStatus.UPLOADING); - this.jobService.update(jobEntity); - } - // Ensure location exist and generate file path - String filePath = this.generateFilePath(graphSpace, graph, jobId, - fileName); - // Check this file deleted before - ReadWriteLock lock = this.uploadingTokenLocks().get(token); - FileUploadResult result; - if (lock == null) { - result = new FileUploadResult(); - result.setName(file.getOriginalFilename()); - result.setSize(file.getSize()); - result.setStatus(FileUploadResult.Status.FAILURE); - result.setCause("File has been deleted"); - return result; - } - - lock.readLock().lock(); - try { - FileMapping reservedMapping; - synchronized (this.service) { - reservedMapping = this.reserveUploadQuota(graphSpace, graph, jobId, - fileName, filePath, - sourceFileSize); + try (var reservation = this.jobService.reserveUpload(jobId)) { + this.checkTotalAndIndexValid(total, index); + this.checkFileNameValid(fileName); + this.checkFileNameMatchToken(jobId, fileName, token); + JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); + Long sourceFileSize = this.resolveSourceFileSize(file, fileSize, + total, index); + this.checkFileValid(graphSpace, graph, jobId, jobEntity, file, fileName); + // Check this file deleted before + ReadWriteLock lock = this.uploadingTokenLocks().get(token); + FileUploadResult result; + if (lock == null) { + return this.deletedUploadResult(file); } - result = this.service.uploadFile(file, fileName, index, filePath); - if (result.getStatus() == FileUploadResult.Status.FAILURE) { - return result; + + if (jobEntity.getJobStatus() == JobStatus.DEFAULT) { + jobEntity.setJobStatus(JobStatus.UPLOADING); + this.jobService.update(jobEntity); } - synchronized (this.service) { - // Verify the existence of fragmented files - FileMapping mapping = this.service.get(graphSpace, graph, jobId, - fileName); - if (mapping == null) { - mapping = reservedMapping; + lock.writeLock().lock(); + try { + if (this.uploadingTokenLocks().get(token) != lock) { + return this.deletedUploadResult(file); } - Ex.check(mapping != null, "load.file-mapping.not-exist.name", - fileName); - if (mapping.getFileStatus() == FileMappingStatus.COMPLETED) { - result.setId(mapping.getId()); - this.uploadingTokenLocks().remove(token); - return result; + // Do not recreate directories for a revoked token waiting on its lock. + String filePath = this.generateFilePath(graphSpace, graph, jobId, fileName); + FileMapping reservedMapping; + synchronized (this.service) { + reservedMapping = this.reserveUploadQuota(graphSpace, graph, jobId, + fileName, filePath, + sourceFileSize); } - mapping.setUpdateTime(HubbleUtil.nowDate()); - // Determine whether all the parts have been uploaded, then merge them - boolean merged = this.service.tryMergePartFiles(filePath, total); - if (!merged) { - this.service.update(mapping); + result = this.service.uploadFile(file, fileName, index, filePath); + if (result.getStatus() == FileUploadResult.Status.FAILURE) { return result; } - JobManager currentJob = this.jobService.get(graphSpace, graph, - jobId); - long actualFileSize; - try { - actualFileSize = this.resolveUploadedFileSize( - mapping.getPath()); - Ex.check(currentJob != null, "job-manager.not-exist.id", - jobId); - long reservedUploadingSize = - this.sumReservedUploadingSize(jobId, - mapping.getId()); - this.checkFileSizeLimit(actualFileSize, - currentJob.getJobSize(), - reservedUploadingSize); - } catch (RuntimeException e) { - this.cleanupFailedUpload(mapping, token); - throw e; + synchronized (this.service) { + // Verify the existence of fragmented files + FileMapping mapping = this.service.get(graphSpace, graph, jobId, + fileName); + if (mapping == null) { + mapping = reservedMapping; + } + Ex.check(mapping != null, "load.file-mapping.not-exist.name", + fileName); + if (mapping.getFileStatus() == FileMappingStatus.COMPLETED) { + result.setId(mapping.getId()); + this.service.removeUploadingTokens(jobId, fileName); + return result; + } + mapping.setUpdateTime(HubbleUtil.nowDate()); + // Determine whether all the parts have been uploaded, then merge them + boolean merged = this.service.tryMergePartFiles(filePath, total); + if (!merged) { + this.service.update(mapping); + return result; + } + JobManager currentJob = this.jobService.get(graphSpace, graph, + jobId); + long actualFileSize; + try { + actualFileSize = this.resolveUploadedFileSize( + mapping.getPath()); + Ex.check(currentJob != null, "job-manager.not-exist.id", + jobId); + long reservedUploadingSize = + this.sumReservedUploadingSize(jobId, + mapping.getId()); + this.checkFileSizeLimit(actualFileSize, + currentJob.getJobSize(), + reservedUploadingSize); + } catch (RuntimeException e) { + this.cleanupFailedUpload(mapping, token); + throw e; + } + // Read column names and values then fill it + this.service.extractColumns(mapping); + mapping.setFileStatus(FileMappingStatus.COMPLETED); + mapping.setTotalLines(FileUtil.countLines(mapping.getPath())); + mapping.setTotalSize(actualFileSize); + + this.finalizeUpload(mapping); + result.setId(mapping.getId()); + // Remove uploading file token + this.service.removeUploadingTokens(jobId, fileName); } - // Read column names and values then fill it - this.service.extractColumns(mapping); - mapping.setFileStatus(FileMappingStatus.COMPLETED); - mapping.setTotalLines(FileUtil.countLines(mapping.getPath())); - mapping.setTotalSize(actualFileSize); - - this.finalizeUpload(mapping); - result.setId(mapping.getId()); - // Remove uploading file token - this.uploadingTokenLocks().remove(token); + return result; + } finally { + lock.writeLock().unlock(); } - return result; - } finally { - lock.readLock().unlock(); } } + private FileUploadResult deletedUploadResult(MultipartFile file) { + FileUploadResult result = new FileUploadResult(); + result.setName(file.getOriginalFilename()); + result.setSize(file.getSize()); + result.setStatus(FileUploadResult.Status.FAILURE); + result.setCause("File has been deleted"); + return result; + } + private void finalizeUpload(FileMapping mapping) { String originalPath = mapping.getPath(); String movedPath = this.service.moveToNextLevelDir(mapping); @@ -256,31 +271,31 @@ public Boolean delete(@PathVariable("graphspace") String graphSpace, @RequestParam("name") String fileName, @RequestParam("token") String token) { this.requireGraphSpaceWrite(graphSpace); - this.checkFileNameValid(fileName); - JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); - Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); - Ex.check(jobEntity.getJobStatus() == JobStatus.UPLOADING || - jobEntity.getJobStatus() == JobStatus.MAPPING || - jobEntity.getJobStatus() == JobStatus.SETTING, - "deleted.file.no-permission"); - FileMapping mapping = this.service.get(graphSpace, graph, jobId, - fileName); - Ex.check(mapping != null, "load.file-mapping.not-exist.name", fileName); - - ReadWriteLock lock = this.uploadingTokenLocks().get(token); - if (lock != null) { - lock.writeLock().lock(); - } - try { - this.jobService.deleteMappings(jobId, - Collections.singletonList(mapping)); + try (var reservation = this.jobService.reserveJob(jobId)) { + this.checkFileNameValid(fileName); + this.checkFileNameMatchToken(jobId, fileName, token); + JobManager jobEntity = this.jobService.get(graphSpace, graph, jobId); + Ex.check(jobEntity != null, "job-manager.not-exist.id", jobId); + Ex.check(jobEntity.getJobStatus() == JobStatus.UPLOADING || + jobEntity.getJobStatus() == JobStatus.MAPPING || + jobEntity.getJobStatus() == JobStatus.SETTING, + "deleted.file.no-permission"); + FileMapping mapping = this.service.get(graphSpace, graph, jobId, + fileName); + Ex.check(mapping != null, "load.file-mapping.not-exist.name", fileName); + + ReadWriteLock lock = this.uploadingTokenLocks().get(token); if (lock != null) { - this.uploadingTokenLocks().remove(token); + lock.writeLock().lock(); } - return true; - } finally { - if (lock != null) { - lock.writeLock().unlock(); + try { + this.jobService.deleteMappings(jobId, + Collections.singletonList(mapping)); + return true; + } finally { + if (lock != null) { + lock.writeLock().unlock(); + } } } } @@ -317,9 +332,8 @@ private void checkTotalAndIndexValid(int total, int index) { } } - private void checkFileNameMatchToken(String fileName, String token) { - String md5Prefix = HubbleUtil.md5(fileName); - Ex.check(StringUtils.isNotEmpty(token) && token.startsWith(md5Prefix), + private void checkFileNameMatchToken(int jobId, String fileName, String token) { + Ex.check(this.service.matchesFileToken(jobId, fileName, token), "load.upload.file.name-token.unmatch"); } @@ -382,18 +396,75 @@ private FileMapping reserveUploadQuota(String graphSpace, String graph, this.checkFileSizeLimit(reservedFileSize, currentJob.getJobSize(), reservedUploadingSize); - if (mapping == null) { - mapping = new FileMapping(graphSpace, graph, fileName, filePath); - mapping.setJobId(jobId); + boolean createdDirectory = this.reservePhysicalUploadDirectory(jobId, fileName, filePath); + try { + if (mapping == null) { + mapping = new FileMapping(graphSpace, graph, fileName, filePath); + mapping.setJobId(jobId); + this.fillUploadingReservation(mapping, reservedFileSize); + this.service.save(mapping); + return mapping; + } + + mapping.setPath(filePath); this.fillUploadingReservation(mapping, reservedFileSize); - this.service.save(mapping); + this.service.update(mapping); return mapping; + } catch (RuntimeException | Error failure) { + if (createdDirectory) { + this.removeUnusedUploadDirectory(Path.of(filePath), failure); + } + throw failure; + } + } + + private boolean reservePhysicalUploadDirectory(int jobId, String fileName, String filePath) { + Path directory = this.service.requirePathUnderUploadRoot(filePath).toPath(); + boolean created = false; + try { + try { + Files.createDirectory(directory); + created = true; + } catch (FileAlreadyExistsException existing) { + if (!Files.isDirectory(directory)) { + throw existing; + } + } + // The leaf now exists, so use native filesystem identity rather than spelling rules. + directory = this.service.requirePathUnderUploadRoot(directory.toString()).toPath(); + for (FileMapping current : this.service.listByJob(jobId)) { + if (fileName.equals(current.getName()) || + (current.getFileStatus() != FileMappingStatus.UPLOADING && + current.getFileStatus() != FileMappingStatus.FAILURE)) { + continue; + } + Path existing = this.service.requirePathUnderUploadRoot(current.getPath()).toPath(); + Ex.check(!Files.exists(existing) || !Files.isSameFile(directory, existing), + "load.upload.file.existed", fileName); + } + return created; + } catch (IOException error) { + InternalException failure = new InternalException("Failed to reserve upload directory '%s'", error, + filePath); + if (created) { + this.removeUnusedUploadDirectory(directory, failure); + } + throw failure; + } catch (RuntimeException | Error failure) { + if (created) { + this.removeUnusedUploadDirectory(directory, failure); + } + throw failure; } + } - mapping.setPath(filePath); - this.fillUploadingReservation(mapping, reservedFileSize); - this.service.update(mapping); - return mapping; + private void removeUnusedUploadDirectory(Path directory, Throwable failure) { + try { + // Only a directory created by this request is eligible, and only if it remains empty. + Files.deleteIfExists(directory); + } catch (IOException cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } } private Long resolveSourceFileSize(MultipartFile file, Long fileSize, @@ -446,7 +517,7 @@ private long resolveUploadedFileSize(String filePath) { } private void cleanupFailedUpload(FileMapping mapping, String token) { - this.uploadingTokenLocks().remove(token); + this.service.removeUploadingTokens(mapping.getJobId(), mapping.getName()); this.service.cleanupMappings(Collections.singletonList(mapping)); } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java index 3766e3d93..6f2a57212 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/entity/load/LoadTask.java @@ -69,6 +69,14 @@ public class LoadTask implements Runnable { @JsonIgnore private transient volatile boolean finished; + @TableField(exist = false) + @JsonIgnore + private transient volatile Throwable cleanupFailure; + + @TableField(exist = false) + @JsonIgnore + private transient volatile boolean cleanupFailed; + @TableId(type = IdType.AUTO) @MergeProperty(useNew = false) @JsonProperty("id") @@ -170,7 +178,7 @@ public LoadTask(LoadOptions options, GraphConnection connection, this.currDuration = 0L; this.createTime = HubbleUtil.nowDate(); - this.loader = new HugeGraphLoader(this.options); + this.loader = new HugeGraphLoader(this.options, false); } @Override @@ -178,11 +186,13 @@ public void run() { Ex.check(this.options != null, "The load options shouldn't be null"); log.info("LoadTask is start running : {}", this.id); + this.cleanupFailure = null; boolean noError = false; - boolean cleanupFailed = false; + this.cleanupFailed = false; Throwable failure = null; try { - noError = this.loader.load(); + // A queued task can be paused before its worker starts. + noError = this.context().stopped() ? this.context().noError() : this.loader.load(); } catch (Throwable e) { failure = e; } finally { @@ -208,15 +218,17 @@ public void run() { this.loader.shutdown(); } catch (Throwable e) { noError = false; - cleanupFailed = !sameLoadFailure(recordedFailure, e) || - recordedFailure.getSuppressed().length != suppressed || - (recordedCause != null && recordedCause.getSuppressed().length != causeSuppressed); - if (cleanupFailed) { + this.cleanupFailed = !sameLoadFailure(recordedFailure, e) || + recordedFailure.getSuppressed().length != suppressed || + (recordedCause != null && + recordedCause.getSuppressed().length != causeSuppressed); + if (this.cleanupFailed) { + this.cleanupFailure = e; log.error("Failed to close load task {}; check graph loading mode before retrying", this.id, e); } if (failure == null) { failure = e; - } else if (cleanupFailed && failure != e && !referencesFailure(e, failure)) { + } else if (this.cleanupFailed && failure != e && !referencesFailure(e, failure)) { failure.addSuppressed(e); } } @@ -228,7 +240,7 @@ public void run() { try { // Pay attention to whether the user stops actively or // the program stops by itself - if (cleanupFailed || this.status.inRunning()) { + if (this.cleanupFailed || this.status.inRunning()) { if (noError) { this.status = LoadStatus.SUCCEED; } else { @@ -291,7 +303,7 @@ public void stop() { // pass } } - if (this.context().closed()) { + if (this.context().closed() && !this.cleanupFailed) { this.loader = null; } log.info("LoadTask {} stopped", this.id); @@ -307,7 +319,7 @@ public void reconnect(String token) { runtimeOptions.token = token; runtimeOptions.pdToken = null; runtimeOptions.trustStoreToken = null; - this.loader = new HugeGraphLoader(runtimeOptions); + this.loader = new HugeGraphLoader(runtimeOptions, false); this.finished = false; } catch (CloneNotSupportedException e) { throw new InternalException("Failed to prepare load task options", diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/FileMappingService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/FileMappingService.java index 0e531df83..5ae13458d 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/FileMappingService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/FileMappingService.java @@ -36,7 +36,6 @@ import java.util.Collections; import java.util.Date; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -96,6 +95,8 @@ public class FileMappingService { private FileMappingMapper mapper; @Autowired private JobManagerMapper jobManagerMapper; + @Autowired + private LoadTaskService taskService; private final Map uploadingTokenLocks; @@ -190,18 +191,17 @@ public void detachFromJob(int mappingId, int jobId) { } } - public String generateFileToken(String fileName) { - /* - * Append a short random suffix so that tokens cannot be guessed - * from the file name and upload time alone. Note that - * checkFileNameMatchToken() only checks the md5 prefix, so the - * token still passes that check. - */ - return this.fileTokenPrefix(fileName) + + public String generateFileToken(int jobId, String fileName) { + // Tokens are opaque to clients and bound to this job and filename. + return this.fileTokenPrefix(jobId, fileName) + HubbleUtil.nowTime().getEpochSecond() + "-" + randomTokenSuffix(); } + public boolean matchesFileToken(int jobId, String fileName, String token) { + return token != null && token.startsWith(this.fileTokenPrefix(jobId, fileName)); + } + private static String randomTokenSuffix() { StringBuilder suffix = new StringBuilder(TOKEN_SUFFIX_LENGTH); for (int i = 0; i < TOKEN_SUFFIX_LENGTH; i++) { @@ -529,6 +529,9 @@ private List listOrphanedJobFiles() { } private void tryCleanupMapping(FileMapping mapping) { + if (mapping.getJobId() != null) { + this.removeUploadingTokens(mapping.getJobId(), mapping.getName()); + } try { this.deleteDiskFile(mapping); this.removeCleanupRecord(mapping.getId()); @@ -585,6 +588,26 @@ private Path normalizePath(File file) { } private void tryDeleteUnfinishedMapping(FileMapping mapping) { + if (mapping.getJobId() == null) { + this.tryCleanupMapping(mapping); + return; + } + try (LoadTaskService.TaskReservation reservation = this.taskService.reserveJob(mapping.getJobId())) { + FileMapping current = this.get(mapping.getId()); + if (current == null || (current.getFileStatus() != FileMappingStatus.UPLOADING && + current.getFileStatus() != FileMappingStatus.FAILURE)) { + return; + } + long threshold = this.config.get(HubbleOptions.UPLOAD_FILE_MAX_TIME_CONSUMING) * 1000; + if (HubbleUtil.nowDate().getTime() - current.getUpdateTime().getTime() <= threshold) { + return; + } + this.removeUploadingTokens(mapping.getJobId(), mapping.getName()); + this.deleteUnfinishedMapping(current); + } + } + + private void deleteUnfinishedMapping(FileMapping mapping) { String filePath = mapping.getPath(); try { File file = this.requirePathUnderUploadRoot(filePath); @@ -601,22 +624,19 @@ private void tryDeleteUnfinishedMapping(FileMapping mapping) { return; } this.remove(mapping.getId()); - this.removeUploadingTokens(mapping.getName()); } - private void removeUploadingTokens(String fileName) { - String tokenPrefix = this.fileTokenPrefix(fileName); - Iterator> iter; - iter = this.uploadingTokenLocks.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry entry = iter.next(); - if (entry.getKey().startsWith(tokenPrefix)) { - iter.remove(); - } - } + public void removeUploadingTokens(int jobId, String fileName) { + String prefix = this.fileTokenPrefix(jobId, fileName); + this.uploadingTokenLocks.keySet().removeIf(token -> token.startsWith(prefix)); + } + + public void removeJobUploadingTokens(int jobId) { + String prefix = jobId + "-"; + this.uploadingTokenLocks.keySet().removeIf(token -> token.startsWith(prefix)); } - private String fileTokenPrefix(String fileName) { - return HubbleUtil.md5(fileName) + "-"; + private String fileTokenPrefix(int jobId, String fileName) { + return jobId + "-" + HubbleUtil.md5(fileName) + "-"; } } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java index deb05a8f7..12af1707a 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/JobManagerService.java @@ -42,12 +42,16 @@ import org.apache.hugegraph.exception.ExternalException; import org.apache.hugegraph.exception.InternalException; import org.apache.hugegraph.mapper.load.JobManagerMapper; +import org.apache.hugegraph.util.Ex; import org.apache.hugegraph.util.HubbleUtil; import org.apache.hugegraph.util.PageUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.annotation.Isolation; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionTemplate; import org.springframework.transaction.support.TransactionSynchronization; import org.springframework.transaction.support.TransactionSynchronizationManager; @@ -61,6 +65,8 @@ public class JobManagerService { private LoadTaskService taskService; @Autowired private FileMappingService fileMappingService; + @Autowired + private PlatformTransactionManager transactionManager; public int count() { return Math.toIntExact(this.mapper.selectCount(null)); @@ -228,27 +234,58 @@ public void completeUpload(FileMapping mapping) { } } - @Transactional(isolation = Isolation.READ_COMMITTED) public void deleteMappings(int jobId, List mappings) { - if (mappings.isEmpty()) { - return; - } - long releasedSize = 0L; - for (FileMapping mapping : mappings) { - if (mapping.getJobId() == null || mapping.getJobId() != jobId) { - throw new InternalException("File mapping does not belong to " + - "job %s", jobId); + try (LoadTaskService.TaskReservation reservation = this.reserveJob(jobId)) { + if (mappings.isEmpty()) { + return; } - if (mapping.getFileStatus() == FileMappingStatus.COMPLETED) { - releasedSize = Math.addExact(releasedSize, - mapping.getTotalSize()); + Ex.check(this.get(jobId) != null, "job-manager.not-exist.id", jobId); + List tasks = this.taskService.taskListByJob(jobId); + List currentMappings = new ArrayList<>(); + long releasedSize = 0L; + for (FileMapping mapping : mappings) { + if (mapping.getJobId() == null || mapping.getJobId() != jobId) { + throw new InternalException("File mapping does not belong to " + + "job %s", jobId); + } + FileMapping current = this.fileMappingService.get(mapping.getId()); + if (current == null || current.getJobId() == null || current.getJobId() != jobId) { + throw new InternalException("File mapping does not belong to job %s", jobId); + } + for (LoadTask task : tasks) { + LoadStatus status = task.getStatus(); + boolean needsSource = status == LoadStatus.RUNNING || status == LoadStatus.PAUSED || + status == LoadStatus.FAILED || status == LoadStatus.STOPPED; + Ex.check(!needsSource || !java.util.Objects.equals(task.getFileId(), current.getId()), + "load.file-mapping.delete.task-in-running", current.getId()); + if (java.util.Objects.equals(task.getFileId(), current.getId())) { + this.taskService.checkCallbackWaitAllowed(task.getId()); + } + } + if (current.getFileStatus() == FileMappingStatus.COMPLETED) { + releasedSize = Math.addExact(releasedSize, + current.getTotalSize()); + } + currentMappings.add(current); } - this.fileMappingService.detachFromJob(mapping.getId(), jobId); - } - if (this.mapper.decreaseJobSize(jobId, releasedSize) != 1) { - throw new InternalException("entity.update.failed", jobId); + for (LoadTask task : tasks) { + if (currentMappings.stream().anyMatch(mapping -> + java.util.Objects.equals(task.getFileId(), mapping.getId()))) { + this.taskService.awaitCallbacksForDelete(task.getId()); + } + } + long size = releasedSize; + this.inDeleteTransaction(() -> { + for (FileMapping current : currentMappings) { + this.fileMappingService.detachFromJob(current.getId(), jobId); + } + if (this.mapper.decreaseJobSize(jobId, size) != 1) { + throw new InternalException("entity.update.failed", jobId); + } + this.deleteDiskFilesAfterCommit(jobId, currentMappings, false); + }); + } - this.deleteDiskFilesAfterCommit(mappings); } @Transactional(isolation = Isolation.READ_COMMITTED) @@ -264,28 +301,69 @@ public void removeByGraph(String graphSpace, String graph) { "graph", graph)); } - @Transactional(isolation = Isolation.READ_COMMITTED) + public LoadTaskService.TaskReservation reserveJob(int id) { + return this.taskService.reserveJob(id); + } + + public LoadTaskService.TaskReservation reserveUpload(int id) { + return this.taskService.reserveUpload(id); + } + public void deleteJob(int id) { - JobManager job = this.get(id); - if (job == null) { - throw new ExternalException("job.manager.not-exist.id", id); - } + try (LoadTaskService.TaskReservation jobReservation = this.taskService.reserveJob(id)) { + JobManager job = this.get(id); + if (job == null) { + throw new ExternalException("job.manager.not-exist.id", id); + } - List loadTasks = this.taskService.taskListByJob(id); - for (LoadTask loadTask : loadTasks) { - if (loadTask.getStatus().inRunning() || - loadTask.getStatus() == LoadStatus.PAUSED) { - this.taskService.stop(loadTask.getId()); + this.taskService.ensureJobCleanupComplete(id); + List loadTasks = this.taskService.taskListByJob(id); + try (LoadTaskService.TaskReservation reservation = this.taskService.reserveTasks( + loadTasks.stream().map(LoadTask::getId).toList())) { + // The job gate keeps these IDs stable. Re-read after all task + // reservations, using selectById rather than the cached list query. + List currentTasks = loadTasks.stream() + .map(task -> this.taskService.get(task.getId())) + .toList(); + for (LoadTask task : loadTasks) { + this.taskService.checkCallbackWaitAllowed(task.getId()); + } + // Publish every stop intent before waiting for a queued owner. + this.taskService.requestJobStopForDelete(id); + for (LoadTask task : loadTasks) { + this.taskService.awaitCallbacksForDelete(task.getId()); + } + this.inDeleteTransaction(() -> { + for (int index = 0; index < loadTasks.size(); index++) { + LoadTask loadTask = loadTasks.get(index); + LoadTask current = currentTasks.get(index); + if (current == null) { + continue; + } + // An initially live row still needs its native certificate; + // a refreshed terminal status alone cannot provide one. + if (loadTask.getStatus().inRunning() || loadTask.getStatus() == LoadStatus.PAUSED || + current.getStatus().inRunning() || current.getStatus() == LoadStatus.PAUSED) { + this.taskService.stopForDelete(loadTask.getId()); + } + this.taskService.ensureRuntimeCleanupComplete(loadTask.getId()); + this.taskService.remove(loadTask.getId()); + } + + List mappings = this.fileMappingService.listByJob(id); + this.remove(id); + this.deleteDiskFilesAfterCommit(id, mappings, true); + }); } - this.taskService.remove(loadTask.getId()); } + } - List mappings = this.fileMappingService.listByJob(id); - this.remove(id); - this.deleteDiskFilesAfterCommit(mappings); + private void inDeleteTransaction(Runnable action) { + TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); + transaction.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED); + transaction.executeWithoutResult(status -> action.run()); } - @Transactional(isolation = Isolation.READ_COMMITTED) public void deleteJob(String graphSpace, String graph, int id) { JobManager job = this.get(graphSpace, graph, id); if (job == null) { @@ -294,14 +372,10 @@ public void deleteJob(String graphSpace, String graph, int id) { this.deleteJob(id); } - private void deleteDiskFilesAfterCommit(List mappings) { - if (mappings.isEmpty()) { - return; - } - + private void deleteDiskFilesAfterCommit(int jobId, List mappings, boolean wholeJob) { List copiedMappings = new ArrayList<>(mappings); if (!TransactionSynchronizationManager.isSynchronizationActive()) { - this.deleteDiskFiles(copiedMappings); + this.deleteDiskFiles(jobId, copiedMappings, wholeJob); return; } @@ -309,12 +383,19 @@ private void deleteDiskFilesAfterCommit(List mappings) { new TransactionSynchronization() { @Override public void afterCommit() { - deleteDiskFiles(copiedMappings); + deleteDiskFiles(jobId, copiedMappings, wholeJob); } }); } - private void deleteDiskFiles(List mappings) { + private void deleteDiskFiles(int jobId, List mappings, boolean wholeJob) { + if (wholeJob) { + this.fileMappingService.removeJobUploadingTokens(jobId); + } else { + for (FileMapping mapping : mappings) { + this.fileMappingService.removeUploadingTokens(jobId, mapping.getName()); + } + } this.fileMappingService.cleanupMappings(mappings); } } diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java index 2615a86b1..bc2a1796d 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/service/load/LoadTaskService.java @@ -54,6 +54,7 @@ import org.apache.hugegraph.loader.util.MappingUtil; import org.apache.hugegraph.loader.util.Printer; import org.apache.hugegraph.mapper.load.JobManagerMapper; +import org.apache.hugegraph.mapper.load.FileMappingMapper; import org.apache.hugegraph.mapper.load.LoadTaskMapper; import org.apache.hugegraph.service.schema.EdgeLabelService; import org.apache.hugegraph.service.schema.VertexLabelService; @@ -79,11 +80,21 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.Collections; +import java.util.Objects; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; + +import com.google.common.util.concurrent.Striped; +import java.util.concurrent.RejectedExecutionException; import org.apache.commons.lang3.StringUtils; @@ -98,6 +109,8 @@ public class LoadTaskService { @Autowired private LoadTaskMapper mapper; @Autowired + private FileMappingMapper fileMappingMapper; + @Autowired private JobManagerMapper jobManagerMapper; @Autowired private VertexLabelService vlService; @@ -112,6 +125,137 @@ public class LoadTaskService { private Map runningTaskContainer; + private final Object admissionLock = new Object(); + private final Set acceptedRuns = new HashSet<>(); + private boolean closing; + private final Striped taskControls = Striped.lock(Runtime.getRuntime().availableProcessors() << 2); + + private final Striped jobControls = + Striped.readWriteLock(Runtime.getRuntime().availableProcessors() << 2); + + TaskReservation reserveJob(int id) { + return this.reserve(Collections.singletonList(this.jobControls.get(id).writeLock())); + } + + TaskReservation reserveUpload(int id) { + return this.reserve(Collections.singletonList(this.jobControls.get(id).readLock())); + } + + TaskReservation reserveTasks(List ids) { + return this.reserve(this.taskControls, ids); + } + + private TaskReservation reserve(Striped controls, List ids) { + List locks = new ArrayList<>(); + // Guava orders by stripe identity, including colliding task IDs. + controls.bulkGet(ids).forEach(locks::add); + return this.reserve(locks); + } + + private TaskReservation reserve(List locks) { + int acquired = 0; + try { + for (Lock lock : locks) { + lock.lock(); + acquired++; + } + return new TaskReservation(locks); + } catch (RuntimeException | Error failure) { + for (int i = acquired - 1; i >= 0; i--) { + locks.get(i).unlock(); + } + throw failure; + } + } + + public final class TaskReservation implements AutoCloseable { + + private final List locks; + private boolean closed; + + private TaskReservation(List locks) { + this.locks = locks; + } + + private void unlock() { + for (int i = this.locks.size() - 1; i >= 0; i--) { + this.locks.get(i).unlock(); + } + } + + @Override + public void close() { + if (this.closed) { + return; + } + this.closed = true; + if (TransactionSynchronizationManager.isActualTransactionActive() && + TransactionSynchronizationManager.isSynchronizationActive()) { + try { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + unlock(); + } + }); + return; + } catch (RuntimeException | Error failure) { + this.unlock(); + throw failure; + } + } + this.unlock(); + } + } + + private void forgetRuntimeAfterCommit(int taskId, LoadTask retained) { + Runnable forget = () -> { + if (retained != null) { + this.forgetRuntime(taskId, retained); + } + }; + if (TransactionSynchronizationManager.isActualTransactionActive() && + TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + forget.run(); + } + }); + } else { + forget.run(); + } + } + + void ensureJobCleanupComplete(int jobId) { + List failures = new ArrayList<>(); + synchronized (this.admissionLock) { + for (AcceptedRun run : this.acceptedRuns) { + failures.add(run.task); + } + } + failures.addAll(this.runningTaskContainer.values()); + for (LoadTask task : failures) { + if (Objects.equals(task.getJobId(), jobId)) { + Ex.check(!task.isCleanupFailed(), + "Task cleanup is incomplete; inspect graph mode and recover before deleting the job"); + } + } + } + + void ensureRuntimeCleanupComplete(int taskId) { + LoadTask retained = this.runningTaskContainer.get(taskId); + if (retained == null) { + return; + } + retained.lock(); + try { + Ex.check(retained.isFinished() && !retained.isCleanupFailed(), + "Task cleanup is incomplete; stop imports, inspect graph mode and recover before retrying"); + } finally { + retained.unlock(); + } + } public LoadTaskService() { this.runningTaskContainer = new ConcurrentHashMap<>(); @@ -174,6 +318,21 @@ public List taskListByJob(int jobId) { @Transactional(isolation = Isolation.READ_COMMITTED) public void save(LoadTask entity) { + Integer jobId = entity.getJobId(); + Ex.check(jobId != null, "Load task has no job id"); + try (TaskReservation reservation = this.reserveJob(jobId)) { + this.ensureParentExists(jobId, entity.getGraphSpace(), entity.getGraph()); + this.insertTask(entity); + } + } + + private void ensureParentExists(int jobId, String graphSpace, String graph) { + JobManager parent = this.jobManagerMapper.selectById(jobId); + Ex.check(parent != null && Objects.equals(parent.getGraphSpace(), graphSpace) && + Objects.equals(parent.getGraph(), graph), "job-manager.not-exist.id", jobId); + } + + private void insertTask(LoadTask entity) { LoadOptions runtimeOptions = entity.getOptions(); entity.setOptions(withoutCredentials(runtimeOptions)); try { @@ -215,14 +374,34 @@ private static LoadOptions withoutCredentials(LoadOptions options) { } } - @Transactional(isolation = Isolation.READ_COMMITTED) public void remove(int id) { - this.runningTaskContainer.remove(id); - if (this.mapper.deleteById(id) != 1) { - throw new InternalException("entity.delete.failed", id); + try (TaskReservation reservation = this.reserveTasks(Collections.singletonList(id))) { + this.checkCallbackWaitAllowed(id); + LoadTask retained = this.runningTaskContainer.get(id); + if (retained != null) { + this.requestStopForDelete(retained); + } + this.awaitCallbacksForDelete(id); + this.ensureRuntimeCleanupComplete(id); + this.inDeleteTransaction(() -> { + LoadTask task = this.get(id); + if (task != null && (task.getStatus().inRunning() || task.getStatus() == LoadStatus.PAUSED)) { + this.stopForDelete(id); + } + if (this.mapper.deleteById(id) != 1) { + throw new InternalException("entity.delete.failed", id); + } + this.forgetRuntimeAfterCommit(id, retained); + }); } } + private void inDeleteTransaction(Runnable action) { + TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); + transaction.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED); + transaction.executeWithoutResult(status -> action.run()); + } + public List batchTasks(int jobId) { QueryWrapper query = Wrappers.query(); query.eq("job_id", jobId); @@ -231,14 +410,33 @@ public List batchTasks(int jobId) { public LoadTask start(GraphConnection connection, FileMapping fileMapping, HugeClient client) { - LoadTask task = this.buildLoadTask(connection, fileMapping, client); - try { - this.save(task); - this.executeAfterCommit(task); - return task; - } catch (RuntimeException | Error failure) { - closeUnscheduled(task, failure); - throw failure; + synchronized (this.admissionLock) { + this.checkAcceptingLoads(); + } + Integer jobId = fileMapping.getJobId(); + Ex.check(jobId != null, "File mapping has no job id"); + try (TaskReservation reservation = this.reserveJob(jobId)) { + this.ensureParentExists(jobId, connection.getGraphSpace(), connection.getGraph()); + Ex.check(Objects.equals(fileMapping.getGraphSpace(), connection.getGraphSpace()) && + Objects.equals(fileMapping.getGraph(), connection.getGraph()), + "File mapping does not belong to the load task graph"); + FileMapping current = this.fileMappingMapper.selectById(fileMapping.getId()); + Ex.check(current != null && Objects.equals(current.getJobId(), jobId) && + Objects.equals(current.getGraphSpace(), connection.getGraphSpace()) && + Objects.equals(current.getGraph(), connection.getGraph()), + "File mapping does not belong to the load task job"); + synchronized (this.admissionLock) { + this.checkAcceptingLoads(); + LoadTask task = this.buildLoadTask(connection, current, client); + try { + this.insertTask(task); + this.executeAfterCommit(task); + return task; + } catch (RuntimeException | Error failure) { + this.closeUnscheduled(task, failure); + throw failure; + } + } } } @@ -247,61 +445,58 @@ protected void executeAfterCommit(LoadTask task) { this.executeSafely(task); return; } - TransactionSynchronizationManager.registerSynchronization( - new TransactionSynchronization() { - @Override - public void afterCommit() { - executeSafely(task); - } + synchronized (this.admissionLock) { + AcceptedRun run = this.accept(task); + try { + TransactionSynchronizationManager.registerSynchronization( + new TransactionSynchronization() { + @Override + public void afterCommit() { + dispatchCommitted(run); + } - @Override - public void afterCompletion(int status) { - if (status != TransactionSynchronization.STATUS_COMMITTED) { - closeUnscheduled(task, null); - } - } - }); + @Override + public void afterCompletion(int status) { + if (status != STATUS_COMMITTED) { + discardRolledBack(run); + } + } + }); + } catch (RuntimeException | Error failure) { + this.release(run); + this.forgetRuntime(task.getId(), task); + throw failure; + } + } } private void executeSafely(LoadTask task) { try { this.execute(task); } catch (RuntimeException e) { - closeUnscheduled(task, e); - this.markDispatchFailed(task, e); - } catch (Error failure) { - closeUnscheduled(task, failure); - try { - this.markDispatchFailed(task, failure); - } catch (Error persistenceFailure) { - if (persistenceFailure != failure) { - failure.addSuppressed(persistenceFailure); - } - } - throw failure; + // execute() already completed native cleanup and its failure marker. + log.error("Failed to dispatch load task {}", task.getId(), e); } } - private static void closeUnscheduled(LoadTask task, Throwable failure) { + private boolean markDispatchFailed(LoadTask task, Throwable cause) { + log.error("Failed to dispatch load task {}, marking task and job " + + "as FAILED", task.getId(), cause); try { - if (task.getLoader() != null) { - task.getLoader().shutdown(); + this.cleanupUnstarted(task); + } catch (RuntimeException | Error cleanupFailure) { + if (cause != cleanupFailure) { + cause.addSuppressed(cleanupFailure); } - } catch (Throwable cleanup) { - if (failure != null && failure != cleanup) { - failure.addSuppressed(cleanup); - } - log.error("Failed to close unscheduled load task {}", task.getId(), cleanup); - } finally { - task.setFinished(true); + log.warn("Failed to close undispatched load task {}", task.getId(), cleanupFailure); } - } - - private void markDispatchFailed(LoadTask task, Throwable cause) { - log.error("Failed to dispatch load task {}, marking task and job " + - "as FAILED", task.getId(), cause); + task.setFinished(true); task.setStatus(LoadStatus.FAILED); - this.runningTaskContainer.remove(task.getId()); + if (task.isCleanupFailed()) { + this.runningTaskContainer.putIfAbsent(task.getId(), task); + } else { + this.forgetRuntime(task.getId(), task); + } try { TransactionTemplate transaction = @@ -332,11 +527,61 @@ private void markDispatchFailed(LoadTask task, Throwable cause) { } return null; }); - } catch (RuntimeException e) { + // TransactionTemplate returns only after REQUIRES_NEW actually commits. + return !task.isCleanupFailed(); + } catch (RuntimeException | Error e) { // The ingest transaction is already committed. Never turn a // dispatch failure into a retryable request failure. + if (cause != e) { + cause.addSuppressed(e); + } log.error("Failed to persist dispatch failure for load task {}", task.getId(), e); + return false; + } + } + + private void execute(LoadTask task) { + synchronized (this.admissionLock) { + AcceptedRun run = this.accept(task); + this.runningTaskContainer.put(task.getId(), task); + try { + this.taskExecutor.execute(task, () -> this.complete(run)); + } catch (RuntimeException | Error failure) { + this.failDispatch(run, failure); + throw failure; + } + } + } + + private AcceptedRun accept(LoadTask task) { + this.checkAcceptingLoads(); + AcceptedRun run = new AcceptedRun(task); + this.acceptedRuns.add(run); + return run; + } + + private void cleanupUnstarted(LoadTask task) { + task.lock(); + try { + if (task.getLoader() != null) { + try { + task.getLoader().shutdown(); + } catch (RuntimeException | Error failure) { + task.setCleanupFailed(true); + if (task.getCleanupFailure() == null) { + task.setCleanupFailure(failure); + } + throw failure; + } + // A repeated shutdown can return after context.close even when + // an earlier owner release failed. Keep that recovery handle. + if (!task.isCleanupFailed()) { + task.setLoader(null); + } + } + } finally { + task.unlock(); } } @@ -344,79 +589,312 @@ private void updateAndExecute(LoadTask task) { try { this.update(task); } catch (RuntimeException | Error failure) { - closeUnscheduled(task, failure); + this.closeUnscheduled(task, failure); throw failure; } - this.executeSafely(task); + // Frontend resume/retry must preserve a synchronous dispatch failure. + this.execute(task); } - private void execute(LoadTask task) { - // Executed in other threads - this.taskExecutor.execute(task, () -> this.update(task)); - // Save current load task - this.runningTaskContainer.put(task.getId(), task); + private void closeUnscheduled(LoadTask task, Throwable failure) { + try { + this.cleanupUnstarted(task); + } catch (RuntimeException | Error cleanupFailure) { + if (failure != cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + } finally { + task.setFinished(true); + } + if (!task.isCleanupFailed()) { + // The persistence operation never admitted this run. Once its + // native resources are closed, no shutdown responsibility remains. + return; + } + task.setStatus(LoadStatus.FAILED); + if (task.getId() != null) { + this.runningTaskContainer.compute(task.getId(), (id, current) -> + current == null || (current.isFinished() && !current.isCleanupFailed()) ? task : current); + } + // A failed insert can have no id. Keep its unresolved native cleanup + // in the same private ticket set without publishing a null map key. + for (AcceptedRun accepted : this.acceptedRuns) { + if (accepted.task == task) { + return; + } + } + AcceptedRun run = new AcceptedRun(task); + run.completed.completeExceptionally(failure); + this.acceptedRuns.add(run); } - public LoadTask pause(int taskId) { - /* - * The in-memory container is not repopulated on startup, so a task - * that was RUNNING before a restart is only visible in the database - */ - LoadTask task = this.runningTaskContainer.get(taskId); - boolean loading = task != null; - if (!loading) { - task = this.get(taskId); + private void forgetRuntime(Integer taskId, LoadTask expected) { + if (taskId != null) { + this.runningTaskContainer.computeIfPresent(taskId, (id, current) -> current == expected ? null : current); } - Ex.check(task != null, "load.task.not-exist.id", taskId); - Ex.check(task.getStatus() == LoadStatus.RUNNING, - "Can only pause the RUNNING task"); - // Mark status as paused, should set before context.stopLoading() - task.setStatus(LoadStatus.PAUSED); - if (loading) { - // Let HugeGraphLoader stop - task.stop(); + } + + private void discardRolledBack(AcceptedRun run) { + try { + this.cleanupUnstarted(run.task); + run.task.setFinished(true); + run.completed.complete(null); + } catch (RuntimeException | Error failure) { + run.task.setFinished(true); + run.completed.completeExceptionally(failure); + throw failure; + } finally { + this.forgetRuntime(run.task.getId(), run.task); + this.release(run); } + } - task.lock(); + private void dispatchCommitted(AcceptedRun run) { try { - this.update(task); - this.runningTaskContainer.remove(taskId); + // This run was admitted before close. The pause hook keeps the + // executor and metadata dependencies alive until its ticket ends. + this.runningTaskContainer.put(run.task.getId(), run.task); + this.taskExecutor.execute(run.task, () -> this.complete(run)); + } catch (RuntimeException | Error failure) { + this.failDispatch(run, failure); + if (failure instanceof Error) { + throw (Error) failure; + } + } + } + + private void failDispatch(AcceptedRun run, Throwable failure) { + boolean resolved = false; + try { + resolved = this.markDispatchFailed(run.task, failure); } finally { - task.unlock(); + if (resolved) { + // The request still receives its dispatch error, but native + // cleanup and its committed FAILED marker need no recovery. + run.completed.complete(null); + } else { + run.completed.completeExceptionally(failure); + } + this.release(run); + } + } + + public LoadTask pause(int taskId) { + try (TaskReservation reservation = this.reserveTasks(Collections.singletonList(taskId))) { + /* + * The in-memory container is not repopulated on startup, so a task + * that was RUNNING before a restart is only visible in the database + */ + LoadTask task = this.runningTaskContainer.get(taskId); + boolean loading = task != null; + if (!loading) { + task = this.get(taskId); + } + Ex.check(task != null, "load.task.not-exist.id", taskId); + Ex.check(task.getStatus() == LoadStatus.RUNNING, + "Can only pause the RUNNING task"); + // Mark status as paused, should set before context.stopLoading() + task.setStatus(LoadStatus.PAUSED); + if (loading) { + // Let HugeGraphLoader stop + task.stop(); + } + + task.lock(); + try { + this.update(task); + if (!task.isCleanupFailed()) { + this.forgetRuntime(taskId, task); + } + } finally { + task.unlock(); + } + return task; } - return task; } public LoadTask resume(int taskId, String token) { - LoadTask task = this.get(taskId); - Ex.check(task.getStatus() == LoadStatus.PAUSED || - task.getStatus() == LoadStatus.FAILED, - "Can only resume the PAUSED or FAILED task"); + try (TaskReservation reservation = this.reserveTasks(Collections.singletonList(taskId))) { + synchronized (this.admissionLock) { + this.checkAcceptingLoads(); + this.ensureRuntimeCleanupComplete(taskId); + } + // A stopped native runner may still own its final metadata write. + // Its callback does not take the task reservation; release admission + // and the entity lock before joining, then read fresh state below. + this.awaitTaskCallbacks(taskId); + synchronized (this.admissionLock) { + this.checkAcceptingLoads(); + this.ensureRuntimeCleanupComplete(taskId); + LoadTask task = this.get(taskId); + Ex.check(task != null, "load.task.not-exist.id", taskId); + Ex.check(task.getStatus() == LoadStatus.PAUSED || + task.getStatus() == LoadStatus.FAILED, + "Can only resume the PAUSED or FAILED task"); + task.lock(); + try { + // Set work mode in incrental mode, load from last breakpoint + task.getOptions().incrementalMode = true; + task.reconnect(token); + task.setStatus(LoadStatus.RUNNING); + this.updateAndExecute(task); + } finally { + task.unlock(); + } + return task; + } + } + } + + public LoadTask stop(int taskId) { + try (TaskReservation reservation = this.reserveTasks(Collections.singletonList(taskId))) { + return this.stop(taskId, false); + } + } + + LoadTask stopForDelete(int taskId) { + LoadTask retained = this.runningTaskContainer.get(taskId); + if (retained != null) { + this.requestStopForDelete(retained); + } + // Do not write the row while its callback still needs that same row. + // Neither admission nor the task entity lock is held during this join. + this.awaitCallbacksForDelete(taskId); + if (retained != null && !retained.isFinished()) { + // Retained work from callers predating ticket registration still + // needs the native completion certificate before deletion. + retained.stop(); + } + return this.stop(taskId, true); + } + + void requestJobStopForDelete(int jobId) { + Set tasks = Collections.newSetFromMap(new java.util.IdentityHashMap<>()); + synchronized (this.admissionLock) { + for (AcceptedRun run : this.acceptedRuns) { + if (Objects.equals(run.task.getJobId(), jobId)) { + tasks.add(run.task); + } + } + for (LoadTask task : this.runningTaskContainer.values()) { + if (Objects.equals(task.getJobId(), jobId)) { + tasks.add(task); + } + } + } + Throwable failure = null; + for (LoadTask task : tasks) { + try { + this.requestStopForDelete(task); + } catch (RuntimeException | Error stopFailure) { + failure = collectFailure(failure, stopFailure); + } + } + throwFailure(failure); + } + + private void requestStopForDelete(LoadTask task) { task.lock(); try { - // Set work mode in incrental mode, load from last breakpoint - task.getOptions().incrementalMode = true; - task.reconnect(token); - task.setStatus(LoadStatus.RUNNING); - this.updateAndExecute(task); + if (task.getStatus().inRunning() || task.getStatus() == LoadStatus.PAUSED) { + task.setStatus(LoadStatus.STOPPED); + if (task.getLoader() != null) { + task.context().stopLoading(); + } + } } finally { task.unlock(); } - return task; } - public LoadTask stop(int taskId) { + void awaitCallbacksForDelete(int taskId) { + this.awaitTaskCallbacks(taskId); + } + + void checkCallbackWaitAllowed(int taskId) { + if (!TransactionSynchronizationManager.isActualTransactionActive()) { + return; + } + synchronized (this.admissionLock) { + for (AcceptedRun run : this.acceptedRuns) { + Ex.check(!Objects.equals(run.task.getId(), taskId) || run.completed.isDone(), + "Task callbacks are still running; retry after the current transaction completes"); + } + } + } + + private void awaitTaskCallbacks(int taskId) { + this.checkCallbackWaitAllowed(taskId); + List runs = new ArrayList<>(); + synchronized (this.admissionLock) { + for (AcceptedRun run : this.acceptedRuns) { + if (Objects.equals(run.task.getId(), taskId)) { + runs.add(run); + } + } + } + Throwable failure = null; + for (AcceptedRun run : runs) { + try { + run.completed.join(); + } catch (CancellationException cancellation) { + failure = collectFailure(failure, cancellation); + } catch (CompletionException completion) { + failure = collectFailure(failure, completion.getCause() == null ? completion : completion.getCause()); + } + } + throwFailure(failure); + } + + private static void throwFailure(Throwable failure) { + if (failure instanceof RuntimeException) { + throw (RuntimeException) failure; + } + if (failure instanceof Error) { + throw (Error) failure; + } + if (failure != null) { + throw new CompletionException(failure); + } + } + + private LoadTask stop(int taskId, boolean deleting) { LoadTask task = this.runningTaskContainer.get(taskId); boolean loading = task != null; if (!loading) { task = this.get(taskId); } Ex.check(task != null, "load.task.not-exist.id", taskId); - LoadStatus status = task.getStatus(); - Ex.check(status == LoadStatus.RUNNING || status == LoadStatus.PAUSED, - "Can only stop the RUNNING or PAUSED task"); - // Mark status as stopped - task.setStatus(LoadStatus.STOPPED); + LoadStatus status; + task.lock(); + try { + status = task.getStatus(); + if (deleting && (status == LoadStatus.SUCCEED || status == LoadStatus.FAILED || + status == LoadStatus.STOPPED)) { + // Only a retained runtime task can certify completed cleanup. + Ex.check(loading && task.isFinished() && !task.isCleanupFailed(), + "Task cleanup is not confirmed; check graph loading mode before deleting"); + return task; + } + if (!deleting && loading && status == LoadStatus.STOPPED && + task.isFinished() && !task.isCleanupFailed()) { + // A failed deletion can roll metadata back after native stop. + // Reuse that safe runtime; do not stop or close its Loader twice. + LoadTask persisted = this.get(taskId); + Ex.check(persisted != null && + (persisted.getStatus() == LoadStatus.RUNNING || + persisted.getStatus() == LoadStatus.PAUSED), + "Can only stop the RUNNING or PAUSED task"); + this.update(task); + this.forgetRuntimeAfterCommit(taskId, task); + return task; + } + Ex.check(status == LoadStatus.RUNNING || status == LoadStatus.PAUSED, + "Can only stop the RUNNING or PAUSED task"); + task.setStatus(LoadStatus.STOPPED); + } finally { + task.unlock(); + } /* * Only a task taken from the in-memory container owns a loader, one * restored from the database has nothing running to interrupt, so @@ -428,8 +906,18 @@ public LoadTask stop(int taskId) { task.lock(); try { + if (deleting && loading) { + // stop() waits outside the lock used by the runner's final + // publication; independent cleanup errors must retain evidence. + Ex.check(task.isFinished() && !task.isCleanupFailed(), + "Task cleanup failed; check graph loading mode before deleting"); + } this.update(task); - this.runningTaskContainer.remove(taskId); + // Deletion may still roll back after native stop. Keep the real + // terminal runtime until remove() observes transaction commit. + if (!deleting && !task.isCleanupFailed()) { + this.forgetRuntime(taskId, task); + } } finally { task.unlock(); } @@ -437,23 +925,38 @@ public LoadTask stop(int taskId) { } public LoadTask retry(int taskId, String token) { - LoadTask task = this.get(taskId); - Ex.check(task.getStatus() == LoadStatus.FAILED || - task.getStatus() == LoadStatus.STOPPED, - "Can only retry the FAILED or STOPPED task"); - task.lock(); - try { - // Set work mode in normal mode, load from begin - task.getOptions().incrementalMode = false; - task.reconnect(token); - task.setStatus(LoadStatus.RUNNING); - task.setLastDuration(0L); - task.setCurrDuration(0L); - this.updateAndExecute(task); - } finally { - task.unlock(); + try (TaskReservation reservation = this.reserveTasks(Collections.singletonList(taskId))) { + synchronized (this.admissionLock) { + this.checkAcceptingLoads(); + this.ensureRuntimeCleanupComplete(taskId); + } + // A stopped native runner may still own its final metadata write. + // Its callback does not take the task reservation; release admission + // and the entity lock before joining, then read fresh state below. + this.awaitTaskCallbacks(taskId); + synchronized (this.admissionLock) { + this.checkAcceptingLoads(); + this.ensureRuntimeCleanupComplete(taskId); + LoadTask task = this.get(taskId); + Ex.check(task != null, "load.task.not-exist.id", taskId); + Ex.check(task.getStatus() == LoadStatus.FAILED || + task.getStatus() == LoadStatus.STOPPED, + "Can only retry the FAILED or STOPPED task"); + task.lock(); + try { + // Set work mode in normal mode, load from begin + task.getOptions().incrementalMode = false; + task.reconnect(token); + task.setStatus(LoadStatus.RUNNING); + task.setLastDuration(0L); + task.setCurrDuration(0L); + this.updateAndExecute(task); + } finally { + task.unlock(); + } + return task; + } } - return task; } public String readLoadFailedReason(FileMapping mapping) { @@ -493,11 +996,178 @@ public String readLoadFailedReason(FileMapping mapping) { } public void pauseAllTasks() { - List tasks = this.listAll(); - for (LoadTask task : tasks) { + List runs; + List runtimeTasks; + synchronized (this.admissionLock) { + this.closing = true; + runs = new ArrayList<>(this.acceptedRuns); + runtimeTasks = new ArrayList<>(this.runningTaskContainer.values()); + for (AcceptedRun run : runs) { + if (runtimeTasks.stream().noneMatch(task -> task == run.task)) { + runtimeTasks.add(run.task); + } + } + } + Set runtimeIds = new HashSet<>(); + for (LoadTask task : runtimeTasks) { + runtimeIds.add(task.getId()); + } + Throwable failure = null; + List failedTasks = new ArrayList<>(); + // Signal every accepted runner before waiting: a queued runner needs + // the active workers to finish before it can observe its stop intent. + for (AcceptedRun run : runs) { + try { + this.requestPause(run.task); + } catch (RuntimeException | Error e) { + failure = collectFailure(failure, e); + } + } + for (AcceptedRun run : runs) { + try { + // finished only covers LoadTask.run(); this ticket also covers + // its async callback and the final metadata update. + run.completed.join(); + } catch (CancellationException e) { + failure = collectFailure(failure, e); + failedTasks.add(run.task); + } catch (CompletionException e) { + Throwable cause = e.getCause() == null ? e : e.getCause(); + failure = collectFailure(failure, cause); + failedTasks.add(run.task); + } + } + for (LoadTask task : runtimeTasks) { + boolean handled = failedTasks.stream().noneMatch(failed -> failed == task); + try { + // Always attempt native cleanup before dropping a handle, + // including when a callback or earlier cleanup failed. + this.cleanupUnstarted(task); + } catch (RuntimeException | Error e) { + failure = collectFailure(failure, e); + handled = false; + } + if (task.isCleanupFailed() && handled) { + Throwable cleanup = task.getCleanupFailure(); + failure = collectFailure(failure, cleanup); + handled = false; + } + if (handled) { + this.forgetRuntime(task.getId(), task); + } else { + // Keep the unpersisted runtime and exceptional ticket visible. + if (task.getId() != null) { + this.runningTaskContainer.putIfAbsent(task.getId(), task); + } + } + } + try { + // Rows surviving a restart have no accepted runtime or Loader. + for (LoadTask task : this.listAll()) { + if (task.getStatus().inRunning() && + !runtimeIds.contains(task.getId())) { + try { + task.setStatus(LoadStatus.PAUSED); + this.update(task); + } catch (RuntimeException | Error e) { + failure = collectFailure(failure, e); + } + } + } + } catch (RuntimeException | Error e) { + failure = collectFailure(failure, e); + } + if (failure instanceof RuntimeException) { + throw (RuntimeException) failure; + } + if (failure instanceof Error) { + throw (Error) failure; + } + if (failure != null) { + throw new CompletionException(failure); + } + } + + private void checkAcceptingLoads() { + if (this.closing) { + throw new RejectedExecutionException("Hubble is closing"); + } + } + + private void complete(AcceptedRun run) { + Throwable failure = run.task.getCleanupFailure(); + run.task.lock(); + try { + // Native cleanup has already run in LoadTask.run(). Still attempt + // metadata persistence when cleanup failed, retaining both causes. + this.update(run.task); + } catch (RuntimeException | Error metadataFailure) { + if (failure == null) { + failure = metadataFailure; + } else if (failure != metadataFailure) { + failure.addSuppressed(metadataFailure); + } + } finally { + run.task.unlock(); + } + if (failure == null) { + run.completed.complete(null); + this.release(run); + return; + } + run.completed.completeExceptionally(failure); + // Exceptional tickets remain registered even when the callback ended + // before close, until their unresolved metadata/cleanup is reported. + if (failure instanceof RuntimeException) { + throw (RuntimeException) failure; + } + if (failure instanceof Error) { + throw (Error) failure; + } + throw new CompletionException(failure); + } + + private void release(AcceptedRun run) { + synchronized (this.admissionLock) { + if (!run.completed.isCompletedExceptionally()) { + this.acceptedRuns.remove(run); + } + } + } + + private void requestPause(LoadTask task) { + task.lock(); + try { if (task.getStatus().inRunning()) { - this.pause(task.getId()); + task.setStatus(LoadStatus.PAUSED); + // Native stopLoading() only publishes a stop reason. It does + // not wait for the queued/active runner or its callback. + if (task.getLoader() != null) { + task.context().stopLoading(); + } } + } finally { + task.unlock(); + } + } + + private static Throwable collectFailure(Throwable primary, Throwable failure) { + if (primary == null) { + return failure; + } + if (primary != failure) { + primary.addSuppressed(failure); + } + return primary; + } + + private static final class AcceptedRun { + + private final LoadTask task; + private final CompletableFuture completed = new CompletableFuture<>(); + + private AcceptedRun(LoadTask task) { + this.task = task; } } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java index 19f0e7afd..16986aa35 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/IngestTransactionIntegrationTest.java @@ -20,16 +20,22 @@ import java.util.Collections; import java.util.Date; +import java.util.List; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import javax.sql.DataSource; import org.junit.After; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -49,6 +55,12 @@ import org.springframework.transaction.support.TransactionTemplate; import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.controller.ingest.IngestController; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.test.util.ReflectionTestUtils; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.apache.hugegraph.driver.HugeClient; import org.apache.hugegraph.entity.GraphConnection; import org.apache.hugegraph.entity.enums.FileMappingStatus; @@ -63,10 +75,12 @@ import org.apache.hugegraph.handler.LoadTaskExecutor; import org.apache.hugegraph.loader.executor.LoadOptions; import org.apache.hugegraph.loader.HugeGraphLoader; +import org.apache.hugegraph.loader.executor.LoadContext; import org.apache.hugegraph.options.HubbleOptions; import org.apache.hugegraph.service.schema.EdgeLabelService; import org.apache.hugegraph.service.schema.VertexLabelService; import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.NativeCleanupTestSupport; @RunWith(SpringRunner.class) @SpringBootTest(classes = IngestTransactionIntegrationTest.TestConfiguration.class, @@ -92,6 +106,9 @@ }) public class IngestTransactionIntegrationTest { + @Rule + public TemporaryFolder nativeFolder = new TemporaryFolder(); + private static final String UPLOAD_ROOT = "/tmp/hubble-ingest-transaction-test"; @@ -122,6 +139,10 @@ public void setup() { this.cleanup(); this.executor.reset(); this.taskService.failTaskInsert(false); + this.taskService.afterTaskSnapshot(null); + this.taskService.afterStopCertificate(null); + this.taskService.stubReconnect(false); + this.taskService.beforeRestartRead(null); Mockito.when(this.config.get(HubbleOptions.UPLOAD_FILE_LOCATION)) .thenReturn(UPLOAD_ROOT); } @@ -149,10 +170,17 @@ public void testMappingInsertFailureRollsBackAllRowsAndDoesNotDispatch() { public void testTaskInsertFailureRollsBackAllRowsAndDoesNotDispatch() { this.taskService.failTaskInsert(true); - Assert.assertThrows(RuntimeException.class, () -> { + Throwable failure = Assert.assertThrows(RuntimeException.class, () -> { this.jobService.createIngestTask(this.job(), this.mapping(), this.connection(), this.client()); }); + Throwable cause = failure; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + Assert.assertTrue("The task INSERT must reach the real H2 constraint", cause instanceof java.sql.SQLException); + Assert.assertEquals("23502", ((java.sql.SQLException) cause).getSQLState()); + Assert.assertTrue(cause.getMessage().toLowerCase(java.util.Locale.ROOT).contains("file_id")); this.assertTableCounts(0, 0, 0); Assert.assertEquals(0, this.executor.executions()); @@ -344,7 +372,9 @@ public void testUnscheduledLoadersCloseAfterSaveFailureRollbackAndRejection() { } Assert.assertEquals(1, construction.constructed().size()); Mockito.verify(construction.constructed().get(0)).shutdown(); - Assert.assertTrue(this.taskService.createdTask().isFinished()); + if (outcome != 0) { + Assert.assertTrue(this.taskService.createdTask().isFinished()); + } Assert.assertEquals(outcome == 2 ? 1 : 0, this.count("load_task")); Assert.assertEquals(outcome == 2 ? 1 : 0, this.executor.executions()); } finally { @@ -355,6 +385,520 @@ public void testUnscheduledLoadersCloseAfterSaveFailureRollbackAndRejection() { } } + @Test + public void testDeleteJobAcceptsCompletionAfterActualDatabaseSnapshot() throws Exception { + this.deleteAfterSnapshot(false); + } + + @Test + public void testDeleteJobRetainsRowsAfterIndependentCleanupFailure() throws Exception { + this.deleteAfterSnapshot(true); + } + + private void deleteAfterSnapshot(boolean cleanupFailure) throws Exception { + FileMapping mapping = this.mapping(); + mapping.setPath(UPLOAD_ROOT + "/delete-race-" + UUID.randomUUID() + "/people.csv"); + LoadTask task = this.jobService.createIngestTask(this.job(), mapping, + this.connection(), this.client()); + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(loader.load()).thenReturn(!cleanupFailure); + if (cleanupFailure) { + Mockito.doThrow(new IllegalStateException("release failed")).when(loader).shutdown(); + } + task.setLoader(loader); + ExecutorService runner = Executors.newSingleThreadExecutor(); + this.taskService.afterTaskSnapshot(() -> { + try { + // Independent thread uses the real Spring/MyBatis callback path + // after the deletion transaction reads its RUNNING snapshot. + runner.submit(() -> { + task.run(); + this.executor.complete(task); + }).get(5, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + try { + if (cleanupFailure) { + Assert.assertThrows(RuntimeException.class, () -> this.jobService.deleteJob(task.getJobId())); + this.assertTableCounts(1, 1, 1); + Assert.assertEquals(LoadStatus.FAILED, this.taskService.get(task.getId()).getStatus()); + } else { + this.jobService.deleteJob(task.getJobId()); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertEquals(0, this.count("load_task")); + } + Mockito.verify(loader).shutdown(); + Assert.assertTrue(task.isFinished()); + } finally { + runner.shutdownNow(); + Assert.assertTrue(runner.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test(timeout = 15000) + public void testDeleteCertificateReservesIdentityAgainstFreshRetry() throws Exception { + FileMapping mapping = this.mapping(); + mapping.setPath(UPLOAD_ROOT + "/reserved-delete-" + UUID.randomUUID() + "/people.csv"); + LoadTask task = this.jobService.createIngestTask(this.job(), mapping, + this.connection(), this.client()); + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + Mockito.when(loader.context()).thenReturn(Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS)); + task.setLoader(loader); + this.taskService.stubReconnect(true); + ExecutorService callbacks = Executors.newSingleThreadExecutor(); + this.taskService.afterTaskSnapshot(() -> { + try { + callbacks.submit(() -> { + task.run(); + this.executor.complete(task); + }).get(5, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + CountDownLatch certified = new CountDownLatch(1); + CountDownLatch delete = new CountDownLatch(1); + this.taskService.afterStopCertificate(() -> { + certified.countDown(); + try { + Assert.assertTrue(delete.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + }); + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future deleting = requests.submit(() -> this.jobService.deleteJob(task.getJobId())); + Assert.assertTrue(certified.await(5, TimeUnit.SECONDS)); + CountDownLatch retryEntered = new CountDownLatch(1); + Future retry = requests.submit(() -> { + retryEntered.countDown(); + return this.taskService.retry(task.getId(), "current-session-token"); + }); + Assert.assertTrue(retryEntered.await(5, TimeUnit.SECONDS)); + try { + retry.get(300, TimeUnit.MILLISECONDS); + Assert.fail("retry crossed the completed-cleanup deletion reservation"); + } catch (TimeoutException expected) { + // Deletion owns the task identity through its transaction. + } + delete.countDown(); + deleting.get(5, TimeUnit.SECONDS); + Throwable rejected = Assert.assertThrows( + java.util.concurrent.ExecutionException.class, () -> retry.get(5, TimeUnit.SECONDS)); + Assert.assertTrue(rejected.getCause() instanceof RuntimeException); + Assert.assertEquals(1, this.executor.executions()); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertEquals(0, this.count("load_task")); + } finally { + delete.countDown(); + requests.shutdownNow(); + callbacks.shutdownNow(); + Assert.assertTrue(requests.awaitTermination(5, TimeUnit.SECONDS)); + Assert.assertTrue(callbacks.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + public void testNativeResetFailureCannotCertifyDeletionAfterLoadFailure() throws Exception { + FileMapping mapping = this.mapping(); + mapping.setPath(UPLOAD_ROOT + "/native-delete-" + UUID.randomUUID() + "/people.csv"); + LoadTask task = this.jobService.createIngestTask(this.job(), mapping, + this.connection(), this.client()); + ExecutorService callback = Executors.newSingleThreadExecutor(); + try (NativeCleanupTestSupport nativeCleanup = new NativeCleanupTestSupport( + task, this.nativeFolder.newFile("mode-reset.json").toPath(), true, LoadStatus.RUNNING)) { + this.taskService.afterTaskSnapshot(() -> { + try { + callback.submit(() -> { + task.run(); + this.executor.complete(task); + }).get(5, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + + Assert.assertThrows(RuntimeException.class, () -> this.jobService.deleteJob(task.getJobId())); + + this.assertTableCounts(1, 1, 1); + Assert.assertEquals(LoadStatus.FAILED, this.taskService.get(task.getId()).getStatus()); + Assert.assertTrue(task.isCleanupFailed()); + Assert.assertNotSame(nativeCleanup.original, nativeCleanup.shutdownFailure.get()); + Assert.assertEquals("LOADING", nativeCleanup.mode()); + nativeCleanup.assertResourcesClosed(); + try (MockedConstruction replacements = Mockito.mockConstruction(HugeGraphLoader.class)) { + Assert.assertThrows(RuntimeException.class, + () -> this.taskService.retry(task.getId(), "current-session-token")); + Assert.assertThrows(RuntimeException.class, + () -> this.taskService.resume(task.getId(), "current-session-token")); + Assert.assertTrue(replacements.constructed().isEmpty()); + Assert.assertSame(nativeCleanup.loader, task.getLoader()); + Assert.assertEquals(1, this.executor.executions()); + } + } finally { + callback.shutdownNow(); + Assert.assertTrue(callback.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test(timeout = 15000) + public void testDeletionReservationSurvivesOuterTransactionUntilCommit() throws Exception { + this.reserveUntilOuterCompletion(false); + } + + @Test(timeout = 15000) + public void testDeletionRollbackRetainsRuntimeAndReleasesReservation() throws Exception { + this.reserveUntilOuterCompletion(true); + } + + @Test(timeout = 15000) + public void testIngestDeleteReservationBlocksFreshRetryUntilOuterCommit() throws Exception { + this.reserveUntilOuterCompletion(false, true); + } + + private void reserveUntilOuterCompletion(boolean rollback) throws Exception { + this.reserveUntilOuterCompletion(rollback, false); + } + + private void reserveUntilOuterCompletion(boolean rollback, boolean endpoint) throws Exception { + FileMapping mapping = this.mapping(); + mapping.setPath(UPLOAD_ROOT + "/outer-delete-" + UUID.randomUUID() + "/people.csv"); + LoadTask task = this.jobService.createIngestTask(this.job(), mapping, + this.connection(), this.client()); + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + Mockito.when(loader.context()).thenReturn(Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS)); + task.setLoader(loader); + task.run(); + this.executor.complete(task); + this.taskService.stubReconnect(true); + CountDownLatch deleted = new CountDownLatch(1); + CountDownLatch complete = new CountDownLatch(1); + CountDownLatch restartRead = new CountDownLatch(1); + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future deleting = requests.submit(() -> new TransactionTemplate(this.transactionManager) + .execute(status -> { + if (endpoint) { + try { + this.ingestEndpoint().perform(delete(org.apache.hugegraph.common.Constant.API_VERSION + + "ingest/jobs/" + task.getId())) + .andExpect(status().isOk()); + } catch (Exception e) { + throw new AssertionError(e); + } + } else { + this.jobService.deleteJob(task.getJobId()); + } + deleted.countDown(); + try { + Assert.assertTrue(complete.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + if (rollback) { + status.setRollbackOnly(); + } + return null; + })); + Assert.assertTrue(deleted.await(5, TimeUnit.SECONDS)); + // remove() now legitimately reads the task as well. Observe only + // the fresh restart request after deletion has returned. + this.taskService.beforeRestartRead(restartRead::countDown); + Assert.assertSame(task, this.taskService.retainedTask(task.getId())); + CountDownLatch retryEntered = new CountDownLatch(1); + Future retry = requests.submit(() -> { + retryEntered.countDown(); + return this.taskService.retry(task.getId(), "current-session-token"); + }); + Assert.assertTrue(retryEntered.await(5, TimeUnit.SECONDS)); + Assert.assertFalse("restart read bypassed transaction reservation", + restartRead.await(300, TimeUnit.MILLISECONDS)); + complete.countDown(); + deleting.get(5, TimeUnit.SECONDS); + if (rollback) { + Assert.assertEquals(LoadStatus.RUNNING, retry.get(5, TimeUnit.SECONDS).getStatus()); + this.assertTableCounts(1, 1, 1); + Assert.assertEquals(2, this.executor.executions()); + } else { + Assert.assertThrows(java.util.concurrent.ExecutionException.class, + () -> retry.get(5, TimeUnit.SECONDS)); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertEquals(endpoint ? 1 : 0, this.count("job_manager")); + Assert.assertEquals(1, this.executor.executions()); + } + Assert.assertTrue(restartRead.await(5, TimeUnit.SECONDS)); + } finally { + complete.countDown(); + requests.shutdownNow(); + Assert.assertTrue(requests.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test + public void testIngestDeleteEndpointStopsActiveTaskBeforeRemovingRow() throws Exception { + LoadTask task = this.jobService.createIngestTask(this.job(), this.mapping(), + this.connection(), this.client()); + HugeGraphLoader loader = this.stopOnNativeSignal(task); + + this.ingestEndpoint().perform(delete(org.apache.hugegraph.common.Constant.API_VERSION + + "ingest/jobs/" + task.getId())) + .andExpect(status().isOk()); + + Mockito.verify(loader).shutdown(); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(LoadStatus.STOPPED, task.getStatus()); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertNull(this.taskService.retainedTask(task.getId())); + } + + @Test + public void testTaskPageDeletionStopsActiveChildAndCleansMetadata() throws Exception { + FileMapping mapping = this.mapping(); + mapping.setPath(UPLOAD_ROOT + "/legacy-delete-" + UUID.randomUUID() + "/people.csv"); + LoadTask task = this.jobService.createIngestTask(this.job(), mapping, + this.connection(), this.client()); + HugeGraphLoader loader = this.stopOnNativeSignal(task); + this.ingestEndpoint().perform(delete(org.apache.hugegraph.common.Constant.API_VERSION + + "ingest/tasks/" + task.getJobId())) + .andExpect(status().isOk()); + this.assertTableCounts(0, 0, 0); + Assert.assertNull(this.taskService.retainedTask(task.getId())); + Mockito.verify(loader).shutdown(); + } + + @Test + public void testIngestDeleteEndpointRetainsNativeCleanupFailure() throws Exception { + LoadTask task = this.jobService.createIngestTask(this.job(), this.mapping(), + this.connection(), this.client()); + try (NativeCleanupTestSupport nativeCleanup = new NativeCleanupTestSupport( + task, this.nativeFolder.newFile("endpoint-reset.json").toPath(), true, LoadStatus.RUNNING)) { + task.run(); + this.executor.complete(task); + + Assert.assertThrows(Exception.class, () -> this.ingestEndpoint().perform( + delete(org.apache.hugegraph.common.Constant.API_VERSION + "ingest/jobs/" + task.getId()))); + + this.assertTableCounts(1, 1, 1); + Assert.assertSame(task, this.taskService.retainedTask(task.getId())); + Assert.assertTrue(task.isCleanupFailed()); + Assert.assertEquals("LOADING", nativeCleanup.mode()); + nativeCleanup.assertResourcesClosed(); + } + } + + @Test + public void testActiveStopDeletionFailureRollsBackAndKeepsRuntimeForDeletionRetry() { + LoadTask task = this.jobService.createIngestTask(this.job(), this.mapping(), + this.connection(), this.client()); + HugeGraphLoader loader = this.stopOnNativeSignal(task); + // Fail the later job-row DELETE after native stop and task removal, + // using real H2 referential integrity rather than a mocked mapper. + this.jdbc.execute("CREATE TABLE delete_guard (job_id INT REFERENCES job_manager(id))"); + try { + this.jdbc.update("INSERT INTO delete_guard VALUES (?)", task.getJobId()); + Assert.assertThrows(RuntimeException.class, () -> this.jobService.deleteJob(task.getJobId())); + } finally { + this.jdbc.execute("DROP TABLE delete_guard"); + } + + this.assertTableCounts(1, 1, 1); + // The final callback commits before the short deletion transaction. + // A failed DELETE rolls back metadata removal, not that completed callback. + Assert.assertEquals(LoadStatus.STOPPED, this.taskService.get(task.getId()).getStatus()); + Assert.assertSame(task, this.taskService.retainedTask(task.getId())); + Assert.assertEquals(LoadStatus.STOPPED, task.getStatus()); + Assert.assertTrue(task.isFinished()); + Mockito.verify(loader).shutdown(); + // Periodic progress skips terminal runtimes; do not claim automatic + // DB reconciliation. A later deletion can reuse the real certificate. + this.jobService.deleteJob(task.getJobId()); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertNull(this.taskService.retainedTask(task.getId())); + Mockito.verify(loader).shutdown(); + } + + @Test + public void testPublicStopRecoversRunningRowAfterDeletionRollback() throws Exception { + this.assertPublicStopAfterDeletionRollback(LoadStatus.RUNNING); + } + + @Test + public void testPublicStopRecoversPausedRowAfterDeletionRollback() throws Exception { + this.assertPublicStopAfterDeletionRollback(LoadStatus.PAUSED); + } + + @Test + public void testRecoveredStopRetainsRuntimeOnOuterRollback() throws Exception { + this.assertPublicStopAfterDeletionRollback(LoadStatus.RUNNING, true); + } + + private void assertPublicStopAfterDeletionRollback(LoadStatus persisted) throws Exception { + this.assertPublicStopAfterDeletionRollback(persisted, false); + } + + private void assertPublicStopAfterDeletionRollback(LoadStatus persisted, boolean rollback) throws Exception { + LoadTask task = this.jobService.createIngestTask(this.job(), this.mapping(), + this.connection(), this.client()); + HugeGraphLoader loader = this.stopOnNativeSignal(task); + this.jdbc.execute("CREATE TABLE stop_recovery_guard (job_id INT REFERENCES job_manager(id))"); + try { + this.jdbc.update("INSERT INTO stop_recovery_guard VALUES (?)", task.getJobId()); + Assert.assertThrows(RuntimeException.class, () -> this.jobService.deleteJob(task.getJobId())); + } finally { + this.jdbc.execute("DROP TABLE stop_recovery_guard"); + } + this.jdbc.update("UPDATE load_task SET load_status = ? WHERE id = ?", persisted.getValue(), task.getId()); + Assert.assertEquals(persisted, this.taskService.get(task.getId()).getStatus()); + Assert.assertSame(task, this.taskService.retainedTask(task.getId())); + Assert.assertTrue(task.isFinished()); + Assert.assertEquals(LoadStatus.STOPPED, task.getStatus()); + + task.setCleanupFailed(true); + Assert.assertThrows(RuntimeException.class, () -> this.taskService.stop(task.getId())); + Assert.assertEquals(persisted, this.taskService.get(task.getId()).getStatus()); + Assert.assertSame(task, this.taskService.retainedTask(task.getId())); + task.setCleanupFailed(false); + task.setFinished(false); + Assert.assertThrows(RuntimeException.class, () -> this.taskService.stop(task.getId())); + Assert.assertEquals(persisted, this.taskService.get(task.getId()).getStatus()); + task.setFinished(true); + + if (rollback) { + new TransactionTemplate(this.transactionManager).execute(transaction -> { + this.taskService.stop(task.getId()); + Assert.assertEquals(LoadStatus.STOPPED, this.taskService.get(task.getId()).getStatus()); + Assert.assertSame(task, this.taskService.retainedTask(task.getId())); + transaction.setRollbackOnly(); + return null; + }); + Assert.assertEquals(persisted, this.taskService.get(task.getId()).getStatus()); + Assert.assertSame(task, this.taskService.retainedTask(task.getId())); + } + + String endpoint = org.apache.hugegraph.common.Constant.API_VERSION + + "graphspaces/DEFAULT/graphs/hugegraph/job-manager/" + task.getJobId() + "/load-tasks/stop"; + this.loadTaskEndpoint().perform(org.springframework.test.web.servlet.request.MockMvcRequestBuilders + .post(endpoint) + .param("task_id", task.getId().toString())) + .andExpect(status().isOk()); + Assert.assertEquals(LoadStatus.STOPPED, this.taskService.get(task.getId()).getStatus()); + Assert.assertNull(this.taskService.retainedTask(task.getId())); + Mockito.verify(loader).shutdown(); + Assert.assertThrows(Exception.class, () -> this.loadTaskEndpoint().perform( + org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post(endpoint) + .param("task_id", task.getId().toString()))); + this.taskService.stubReconnect(true); + int dispatched = this.executor.executions(); + this.taskService.retry(task.getId(), "fixture-current-token"); + Assert.assertEquals(dispatched + 1, this.executor.executions()); + Assert.assertEquals(LoadStatus.RUNNING, this.taskService.get(task.getId()).getStatus()); + Mockito.verify(loader).shutdown(); + } + + private MockMvc loadTaskEndpoint() { + org.apache.hugegraph.controller.load.LoadTaskController controller = + new org.apache.hugegraph.controller.load.LoadTaskController(this.taskService) { + @Override + protected HugeClient requireGraphSpaceWrite(String graphSpace) { + return null; + } + }; + ReflectionTestUtils.setField(controller, "jobService", this.jobService); + return MockMvcBuilders.standaloneSetup(controller).build(); + } + + @Test + public void testRejectedDispatchNativeCleanupFailureRetainsRowsAndBlocksReplacement() throws Exception { + this.executor.reject(true); + AtomicReference created = new AtomicReference<>(); + AtomicReference boundary = + new AtomicReference<>(); + try { + new TransactionTemplate(this.transactionManager).execute(status -> { + LoadTask task = this.jobService.createIngestTask(this.job(), this.mapping(), + this.connection(), this.client()); + created.set(task); + try { + NativeCleanupTestSupport nativeCleanup = new NativeCleanupTestSupport( + task, this.nativeFolder.newFile("dispatch-reset.json").toPath(), true, LoadStatus.RUNNING); + boundary.set(nativeCleanup); + nativeCleanup.loader.context().failLoading(nativeCleanup.original); + } catch (Exception e) { + throw new AssertionError(e); + } + return null; + }); + LoadTask task = created.get(); + NativeCleanupTestSupport nativeCleanup = boundary.get(); + Assert.assertTrue(task.isFinished()); + Assert.assertTrue(task.isCleanupFailed()); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertSame(task, this.taskService.retainedTask(task.getId())); + this.assertTableCounts(1, 1, 1); + Assert.assertEquals(LoadStatus.FAILED, this.taskService.get(task.getId()).getStatus()); + Assert.assertEquals("LOADING", nativeCleanup.mode()); + nativeCleanup.assertResourcesClosed(); + try (MockedConstruction replacements = Mockito.mockConstruction(HugeGraphLoader.class)) { + Assert.assertThrows(RuntimeException.class, () -> this.taskService.retry(task.getId(), + "current-token")); + Assert.assertThrows(RuntimeException.class, () -> this.taskService.resume(task.getId(), + "current-token")); + Assert.assertThrows(RuntimeException.class, () -> this.taskService.remove(task.getId())); + Assert.assertTrue(replacements.constructed().isEmpty()); + this.assertTableCounts(1, 1, 1); + Assert.assertEquals(1, this.executor.executions()); + } + } finally { + if (boundary.get() != null) { + boundary.get().close(); + } + } + } + + @Test + public void testMissingTaskDeletionKeepsExistingContracts() throws Exception { + this.ingestEndpoint().perform(delete(org.apache.hugegraph.common.Constant.API_VERSION + "ingest/jobs/-1")) + .andExpect(status().isOk()) + .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers + .jsonPath("$.status").value(404)); + Assert.assertThrows(org.apache.hugegraph.exception.InternalException.class, + () -> this.taskService.remove(-1)); + this.assertTableCounts(0, 0, 0); + } + + private HugeGraphLoader stopOnNativeSignal(LoadTask task) { + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + Mockito.doAnswer(invocation -> { + task.run(); + this.executor.complete(task); + return null; + }).when(context).stopLoading(); + task.setLoader(loader); + return loader; + } + + private MockMvc ingestEndpoint() { + IngestController controller = new IngestController() { + @Override + protected HugeClient requireGraphSpaceWrite(String graphSpace) { + // Only authentication is isolated: routing, controller, Spring + // service proxy, transaction and MyBatis persistence are real. + return null; + } + }; + ReflectionTestUtils.setField(controller, "loadTaskService", this.taskService); + ReflectionTestUtils.setField(controller, "jobManagerService", this.jobService); + return MockMvcBuilders.standaloneSetup(controller).build(); + } + private JobManager job() { Date now = new Date(); return JobManager.builder() @@ -503,13 +1047,81 @@ public static class TestLoadTaskService extends LoadTaskService { private boolean failTaskInsert; private LoadTask createdTask; + private Runnable afterTaskSnapshot; + private Runnable afterStopCertificate; + private boolean stubReconnect; + private Runnable beforeRestartRead; + + public LoadTask retainedTask(int id) { + try { + java.lang.reflect.Field field = LoadTaskService.class.getDeclaredField("runningTaskContainer"); + field.setAccessible(true); + @SuppressWarnings("unchecked") + java.util.Map running = (java.util.Map) field.get(this); + return running.get(id); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + } + + public void beforeRestartRead(Runnable action) { + this.beforeRestartRead = action; + } + + public void afterStopCertificate(Runnable action) { + this.afterStopCertificate = action; + } + + public void stubReconnect(boolean enabled) { + this.stubReconnect = enabled; + } + + @Override + public LoadTask get(int id) { + if (this.stubReconnect && this.beforeRestartRead != null) { + this.beforeRestartRead.run(); + } + LoadTask task = super.get(id); + if (this.stubReconnect && task != null) { + task = Mockito.spy(task); + Mockito.doNothing().when(task).reconnect(Mockito.anyString()); + } + return task; + } + + @Override + LoadTask stopForDelete(int id) { + LoadTask task = super.stopForDelete(id); + Runnable action = this.afterStopCertificate; + this.afterStopCertificate = null; + if (action != null) { + action.run(); + } + return task; + } + + public void afterTaskSnapshot(Runnable action) { + this.afterTaskSnapshot = action; + } + + @Override + public List taskListByJob(int jobId) { + List snapshot = super.taskListByJob(jobId); + Runnable action = this.afterTaskSnapshot; + this.afterTaskSnapshot = null; + if (action != null) { + action.run(); + } + return snapshot; + } public void failTaskInsert(boolean fail) { this.failTaskInsert = fail; } public LoadTask startLoader(GraphConnection connection, FileMapping mapping, HugeClient client) { - return super.start(connection, mapping, client); + this.createdTask = super.start(connection, mapping, client); + return this.createdTask; } @Override @@ -525,14 +1137,13 @@ public LoadTask createdTask() { @Override public LoadTask start(GraphConnection connection, FileMapping mapping, HugeClient client) { - String graph = this.failTaskInsert ? null : connection.getGraph(); LoadOptions options = new LoadOptions(); options.token = connection.getToken(); LoadTask task = LoadTask.builder() .graphSpace(connection.getGraphSpace()) - .graph(graph) + .graph(connection.getGraph()) .jobId(mapping.getJobId()) - .fileId(mapping.getId()) + .fileId(this.failTaskInsert ? null : mapping.getId()) .fileName(mapping.getName()) .options(options) .vertices(Collections.emptySet()) @@ -553,6 +1164,8 @@ public LoadTask start(GraphConnection connection, FileMapping mapping, public static class RecordingLoadTaskExecutor extends LoadTaskExecutor { private final AtomicInteger executions = new AtomicInteger(); + private final java.util.Map callbacks = + Collections.synchronizedMap(new java.util.IdentityHashMap<>()); private volatile boolean reject; @Override @@ -562,6 +1175,20 @@ public void execute(LoadTask task, Runnable callback) { throw new org.springframework.core.task.TaskRejectedException( "test executor queue is full"); } + this.callbacks.put(task, callback); + } + + public void complete(LoadTask task) { + Runnable callback = this.callbacks.remove(task); + Assert.assertNotNull("A recorded dispatch must run its actual service callback", callback); + try { + callback.run(); + } catch (RuntimeException | Error failure) { + if (!task.isCleanupFailed()) { + throw failure; + } + Assert.assertSame(task.getCleanupFailure(), failure); + } } public int executions() { @@ -574,6 +1201,7 @@ public void reject(boolean reject) { public void reset() { this.executions.set(0); + this.callbacks.clear(); this.reject = false; } } diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/JobAdmissionIntegrationTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/JobAdmissionIntegrationTest.java new file mode 100644 index 000000000..5188b42e6 --- /dev/null +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/JobAdmissionIntegrationTest.java @@ -0,0 +1,1819 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.service.load; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import javax.sql.DataSource; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.MockedConstruction; +import org.mockito.Mockito; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.aop.support.AopUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.transaction.support.TransactionTemplate; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.controller.load.LoadTaskController; +import org.apache.hugegraph.controller.load.FileUploadController; +import org.apache.hugegraph.controller.load.FileMappingController; +import org.apache.hugegraph.driver.HugeClient; +import org.apache.hugegraph.entity.GraphConnection; +import org.apache.hugegraph.entity.enums.FileMappingStatus; +import org.apache.hugegraph.entity.enums.JobStatus; +import org.apache.hugegraph.entity.enums.LoadStatus; +import org.apache.hugegraph.entity.load.FileMapping; +import org.apache.hugegraph.entity.load.FileSetting; +import org.apache.hugegraph.entity.load.JobManager; +import org.apache.hugegraph.entity.load.ListFormat; +import org.apache.hugegraph.entity.load.LoadParameter; +import org.apache.hugegraph.entity.load.LoadTask; +import org.apache.hugegraph.handler.LoadTaskExecutor; +import org.apache.hugegraph.loader.HugeGraphLoader; +import org.apache.hugegraph.loader.executor.LoadContext; +import org.apache.hugegraph.loader.executor.LoadOptions; +import org.apache.hugegraph.loader.builder.SchemaCache; +import org.apache.hugegraph.loader.task.TaskManager; +import org.springframework.scheduling.annotation.Async; +import org.apache.hugegraph.options.HubbleOptions; +import org.apache.hugegraph.service.schema.EdgeLabelService; +import org.apache.hugegraph.service.schema.VertexLabelService; +import org.apache.hugegraph.testutil.Assert; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = JobAdmissionIntegrationTest.Configuration.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = { + "spring.datasource.url=jdbc:h2:mem:job-admission;DB_CLOSE_DELAY=-1", + "spring.datasource.driver-class-name=org.h2.Driver", + "spring.datasource.username=sa", "spring.datasource.password=", + "spring.sql.init.schema-locations=classpath:database/ingest-transaction-schema.sql", + "spring.sql.init.mode=always", "spring.datasource.hikari.maximum-pool-size=3", + "spring.autoconfigure.exclude=" + + "org.mybatis.spring.boot.autoconfigure.MybatisAutoConfiguration," + + "org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchClientAutoConfiguration," + + "org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration", + "mybatis-plus.configuration.map-underscore-to-camel-case=true", + "mybatis-plus.configuration.use-generated-keys=true", "mybatis-plus.global-config.banner=false" + }) +public class JobAdmissionIntegrationTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + @Autowired + private GateTaskService tasks; + @Autowired + private JobManagerService jobs; + @Autowired + private GateMappingService mappings; + @Autowired + private HugeConfig config; + @Autowired + private DataSource dataSource; + @Autowired + private PlatformTransactionManager transactionManager; + @Autowired + private ThreadPoolTaskExecutor executor; + @Autowired + private GateLoadTaskExecutor taskExecutor; + @MockBean + private VertexLabelService vertexLabels; + @MockBean + private EdgeLabelService edgeLabels; + + private JdbcTemplate jdbc; + private Path uploads; + + @Before + public void setup() throws Exception { + this.jdbc = new JdbcTemplate(this.dataSource); + this.clearRows(); + this.mappings.getUploadingTokenLocks().clear(); + this.tasks.afterSnapshot(null); + this.tasks.queuedFirst(null); + this.tasks.prepareRestart(null); + this.tasks.afterReservation(null); + CallbackWriteFailure.thread = null; + this.taskExecutor.callbackGate(null); + this.taskExecutor.callbackFinished(null); + this.mappings.afterSnapshot(null); + this.uploads = this.folder.newFolder("uploads").toPath(); + Mockito.when(this.config.get(HubbleOptions.PD_ENABLED)).thenReturn(false); + Mockito.when(this.config.get(HubbleOptions.SERVER_URL)).thenReturn("http://127.0.0.1:65530"); + Mockito.when(this.config.get(HubbleOptions.UPLOAD_FILE_LOCATION)).thenReturn(this.uploads.toString()); + Mockito.when(this.config.get(HubbleOptions.UPLOAD_FILE_FORMAT_LIST)).thenReturn(List.of("csv")); + Mockito.when(this.config.get(HubbleOptions.UPLOAD_SINGLE_FILE_SIZE_LIMIT)).thenReturn(1024L); + Mockito.when(this.config.get(HubbleOptions.UPLOAD_TOTAL_FILE_SIZE_LIMIT)).thenReturn(4096L); + } + + @After + public void teardown() throws Exception { + this.executor.getThreadPoolExecutor().submit(() -> { }).get(5, TimeUnit.SECONDS); + this.clearRows(); + } + + @Test(timeout = 15000) + public void testDeleteSnapshotRejectsLaterControllerStartBeforeLoaderConstruction() throws Exception { + FileMapping mapping = this.seed(); + CountDownLatch snapshot = new CountDownLatch(1); + CountDownLatch delete = new CountDownLatch(1); + this.tasks.afterSnapshot(() -> { + snapshot.countDown(); + await(delete); + }); + AtomicInteger constructions = new AtomicInteger(); + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future deleting = requests.submit(() -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertTrue(snapshot.await(5, TimeUnit.SECONDS)); + CountDownLatch entering = new CountDownLatch(1); + Future starting = requests.submit(() -> { + try (MockedConstruction ignored = Mockito.mockConstruction( + HugeGraphLoader.class, (loader, context) -> constructions.incrementAndGet())) { + entering.countDown(); + return this.controller().start("DEFAULT", "hugegraph", mapping.getJobId(), + List.of(mapping.getId())); + } + }); + Assert.assertTrue(entering.await(5, TimeUnit.SECONDS)); + assertBlocked(starting); + Assert.assertEquals(0, constructions.get()); + Assert.assertEquals(0, this.count("load_task")); + delete.countDown(); + deleting.get(5, TimeUnit.SECONDS); + Assert.assertThrows(java.util.concurrent.ExecutionException.class, () -> starting.get(5, TimeUnit.SECONDS)); + Assert.assertEquals(0, constructions.get()); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertEquals(0, this.count("load_task")); + } finally { + delete.countDown(); + stop(requests); + } + } + + @Test(timeout = 15000) + public void testControllerStartBeforeDeleteMustFinishNativeCleanupBeforeCommit() throws Exception { + FileMapping mapping = this.seed(); + CountDownLatch loading = new CountDownLatch(1); + CountDownLatch load = new CountDownLatch(1); + CountDownLatch cleaning = new CountDownLatch(1); + CountDownLatch cleanup = new CountDownLatch(1); + AtomicReference constructed = new AtomicReference<>(); + ExecutorService requests = Executors.newSingleThreadExecutor(); + try (MockedConstruction ignored = Mockito.mockConstruction(HugeGraphLoader.class, + (loader, construction) -> { + constructed.set(loader); + LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(loader.load()).thenAnswer(call -> { + loading.countDown(); + await(load); + return false; + }); + Mockito.doAnswer(call -> { + load.countDown(); + return null; + }).when(context).stopLoading(); + Mockito.doAnswer(call -> { + cleaning.countDown(); + await(cleanup); + return null; + }).when(loader).shutdown(); + })) { + LoadTask task = this.controller().start("DEFAULT", "hugegraph", mapping.getJobId(), + List.of(mapping.getId())).get(0); + Assert.assertTrue(AopUtils.isAopProxy(this.tasks)); + Assert.assertTrue(AopUtils.isAopProxy(this.taskExecutor)); + Assert.assertTrue(loading.await(5, TimeUnit.SECONDS)); + Assert.assertSame(task, this.tasks.retained(task.getId())); + Future deleting = requests.submit(() -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertTrue(cleaning.await(5, TimeUnit.SECONDS)); + assertBlocked(deleting); + Assert.assertFalse(task.isFinished()); + Assert.assertEquals(1, this.count("job_manager")); + Assert.assertEquals(1, this.count("load_task")); + cleanup.countDown(); + deleting.get(5, TimeUnit.SECONDS); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertNull(this.tasks.retained(task.getId())); + Mockito.verify(constructed.get()).shutdown(); + } finally { + load.countDown(); + cleanup.countDown(); + stop(requests); + } + } + + @Test(timeout = 15000) + public void testPublicSaveCannotInsertAfterDeleteSnapshot() throws Exception { + this.publicSaveAfterDeletion(false); + } + + @Test(timeout = 15000) + public void testJobReservationSurvivesDeleteMethodAndReleasesOnOuterRollback() throws Exception { + this.publicSaveAfterDeletion(true); + } + + private void publicSaveAfterDeletion(boolean rollback) throws Exception { + FileMapping mapping = this.seed(); + LoadTask task = this.metadata(mapping); + CountDownLatch deleted = new CountDownLatch(1); + CountDownLatch complete = new CountDownLatch(1); + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + TransactionTemplate transaction = new TransactionTemplate(this.transactionManager); + Future deleting = requests.submit(() -> transaction.execute(status -> { + this.jobs.deleteJob(mapping.getJobId()); + deleted.countDown(); + await(complete); + if (rollback) { + status.setRollbackOnly(); + } + return null; + })); + Assert.assertTrue(deleted.await(5, TimeUnit.SECONDS)); + CountDownLatch entering = new CountDownLatch(1); + Future saving = requests.submit(() -> { + entering.countDown(); + this.tasks.save(task); + }); + Assert.assertTrue(entering.await(5, TimeUnit.SECONDS)); + assertBlocked(saving); + Assert.assertEquals(0, this.count("load_task")); + complete.countDown(); + deleting.get(5, TimeUnit.SECONDS); + if (rollback) { + saving.get(5, TimeUnit.SECONDS); + Assert.assertEquals(1, this.count("job_manager")); + Assert.assertEquals(1, this.count("load_task")); + } else { + Assert.assertThrows(java.util.concurrent.ExecutionException.class, () -> saving.get(5, + TimeUnit.SECONDS)); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertEquals(0, this.count("load_task")); + } + } finally { + complete.countDown(); + stop(requests); + } + } + + @Test + public void testPublicStartChecksFreshParentIdentityBeforeConstructingLoader() throws Exception { + FileMapping mapping = this.seed(); + GraphConnection wrongGraph = this.connection(); + wrongGraph.setGraph("another-graph"); + try (MockedConstruction constructed = Mockito.mockConstruction(HugeGraphLoader.class)) { + Assert.assertThrows(RuntimeException.class, () -> this.tasks.start(wrongGraph, mapping, this.client())); + LoadTask wrongTask = this.metadata(mapping); + wrongTask.setGraphSpace("another-space"); + Assert.assertThrows(RuntimeException.class, () -> this.tasks.save(wrongTask)); + this.jobs.deleteJob(mapping.getJobId()); + Assert.assertThrows(RuntimeException.class, () -> this.tasks.start(this.connection(), mapping, + this.client())); + Assert.assertTrue(constructed.constructed().isEmpty()); + Assert.assertEquals(0, this.count("load_task")); + } + } + + @Test + public void testPublicStartAcceptsIngestLoadingParentAndClosesAfterOuterRollback() throws Exception { + FileMapping mapping = this.seed(); + JobManager job = this.jobs.get(mapping.getJobId()); + job.setJobStatus(JobStatus.LOADING); + this.jobs.update(job); + AtomicReference created = new AtomicReference<>(); + try (MockedConstruction constructed = Mockito.mockConstruction(HugeGraphLoader.class)) { + new TransactionTemplate(this.transactionManager).execute(status -> { + created.set(this.tasks.start(this.connection(), mapping, this.client())); + status.setRollbackOnly(); + return null; + }); + Assert.assertEquals(1, constructed.constructed().size()); + Mockito.verify(constructed.constructed().get(0)).shutdown(); + Assert.assertTrue(created.get().isFinished()); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertEquals(1, this.count("job_manager")); + } + // The outer rollback must have released the job stripe. + this.tasks.save(this.metadata(mapping)); + Assert.assertEquals(1, this.count("load_task")); + } + + @Test(timeout = 15000) + public void testRuntimePublishedBeforeExecutorSubmissionCanBecomeObservable() throws Exception { + FileMapping mapping = this.seed(); + LoadTaskExecutor original = this.taskExecutor; + AtomicInteger submits = new AtomicInteger(); + LoadTaskExecutor observing = new LoadTaskExecutor() { + @Override + public void execute(LoadTask task, Runnable callback) { + submits.incrementAndGet(); + Assert.assertSame("runtime must be registered before submission", task, + JobAdmissionIntegrationTest.this.tasks.retained(task.getId())); + super.execute(task, callback); + } + }; + ReflectionTestUtils.setField(this.tasks, "taskExecutor", observing); + try (MockedConstruction ignored = Mockito.mockConstruction(HugeGraphLoader.class, + (loader, construction) -> Mockito.when(loader.context()).thenReturn( + Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS)))) { + LoadTask task = this.tasks.start(this.connection(), mapping, this.client()); + Assert.assertEquals(1, submits.get()); + Assert.assertTrue(task.isFinished()); + Assert.assertSame(task, this.tasks.retained(task.getId())); + } finally { + ReflectionTestUtils.setField(this.tasks, "taskExecutor", original); + } + } + + @Test(timeout = 15000) + public void testClearEmptySnapshotIncludesAnAlreadyAdmittedUpload() throws Exception { + this.assertClearIncludesAdmittedUpload(false); + } + + @Test(timeout = 15000) + public void testClearNonemptySnapshotIncludesAnAlreadyAdmittedUpload() throws Exception { + this.assertClearIncludesAdmittedUpload(true); + } + + private void assertClearIncludesAdmittedUpload(boolean nonempty) throws Exception { + JobManager job = nonempty ? this.jobs.get(this.seed().getJobId()) : this.seedJob(); + FileUploadController upload = this.uploadController(); + FileMappingController clear = this.mappingController(); + CountDownLatch admitted = new CountDownLatch(1); + CountDownLatch createMapping = new CountDownLatch(1); + CountDownLatch snapshot = new CountDownLatch(1); + FileMappingService observed = Mockito.mock(FileMappingService.class, call -> { + Object result = call.getMethod().invoke(this.mappings, call.getArguments()); + if (call.getMethod().getName().equals("listByJob") && call.getArguments().length == 3) { + snapshot.countDown(); + } + return result; + }); + ReflectionTestUtils.setField(clear, "service", observed); + java.util.concurrent.atomic.AtomicBoolean firstSize = new java.util.concurrent.atomic.AtomicBoolean(true); + MockMultipartFile part = new MockMultipartFile("file", "admitted.csv", "text/csv", "name\nalice\n".getBytes()) { + @Override + public long getSize() { + if (firstSize.compareAndSet(true, false)) { + admitted.countDown(); + await(createMapping); + } + return super.getSize(); + } + }; + String token = upload.fileToken("DEFAULT", "hugegraph", job.getId(), List.of("admitted.csv")) + .get("admitted.csv"); + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future uploading = requests.submit(() -> upload.upload("DEFAULT", "hugegraph", job.getId(), + part, "admitted.csv", 20L, token, 2, 0)); + Assert.assertTrue(admitted.await(5, TimeUnit.SECONDS)); + Future clearing = requests.submit(() -> clear.clear("DEFAULT", "hugegraph", job.getId())); + boolean prematureSnapshot = snapshot.await(500, TimeUnit.MILLISECONDS); + assertBlocked(clearing); + createMapping.countDown(); + uploading.get(5, TimeUnit.SECONDS); + clearing.get(5, TimeUnit.SECONDS); + Assert.assertTrue(this.mappings.listByJob(job.getId()).isEmpty()); + Assert.assertFalse(prematureSnapshot); + } finally { + createMapping.countDown(); + stop(requests); + } + } + + @Test + public void testNativeCaseAliasCannotOverwriteAnExistingPartialUpload() throws Exception { + this.assertNativeAliasProtection("name.csv", "NAME.csv"); + } + + @Test + public void testNativeUnicodeAliasCannotOverwriteAnExistingPartialUpload() throws Exception { + this.assertNativeAliasProtection("café.csv", "café.csv"); + } + + private void assertNativeAliasProtection(String name, String alias) throws Exception { + JobManager job = this.seedJob(); + FileUploadController controller = this.uploadController(); + var tokens = controller.fileToken("DEFAULT", "hugegraph", job.getId(), List.of(name, alias)); + MockMultipartFile first = new MockMultipartFile("file", name, "text/csv", "name\nalpha\n".getBytes()); + controller.upload("DEFAULT", "hugegraph", job.getId(), first, name, 20L, tokens.get(name), 2, 0); + FileMapping mapping = this.mappings.get("DEFAULT", "hugegraph", job.getId(), name); + Path directory = Path.of(mapping.getPath()); + Path aliasDirectory = directory.resolveSibling(alias); + boolean equivalent = Files.exists(aliasDirectory) && Files.isSameFile(directory, aliasDirectory); + AtomicInteger aliasWrites = new AtomicInteger(); + MockMultipartFile second = new MockMultipartFile("file", alias, "text/csv", "other\nbeta\n".getBytes()) { + @Override + public void transferTo(File destination) throws IOException { + aliasWrites.incrementAndGet(); + super.transferTo(destination); + } + }; + if (equivalent) { + Assert.assertThrows(RuntimeException.class, () -> controller.upload("DEFAULT", "hugegraph", job.getId(), + second, alias, 20L, tokens.get(alias), 2, 0)); + Assert.assertEquals(0, aliasWrites.get()); + Assert.assertTrue(this.mappings.get("DEFAULT", "hugegraph", job.getId(), alias) == null); + Assert.assertEquals("name\nalpha\n", Files.readString(directory.resolve(name + "-0"))); + } else { + var accepted = controller.upload("DEFAULT", "hugegraph", job.getId(), second, alias, 20L, + tokens.get(alias), 2, 0); + Assert.assertFalse(accepted.getStatus() == + org.apache.hugegraph.entity.load.FileUploadResult.Status.FAILURE); + Assert.assertFalse(Files.isSameFile(directory, aliasDirectory)); + Assert.assertEquals("name\nalpha\n", Files.readString(directory.resolve(name + "-0"))); + Assert.assertEquals("other\nbeta\n", Files.readString(aliasDirectory.resolve(alias + "-0"))); + } + } + + @Test + public void testFailedMappingInsertRemovesOnlyItsNewEmptyPartDirectory() throws Exception { + JobManager job = this.seedJob(); + FileUploadController controller = this.uploadController(); + String token = controller.fileToken("DEFAULT", "hugegraph", job.getId(), List.of("rejected.csv")) + .get("rejected.csv"); + try (var paths = Files.walk(this.uploads)) { + Assert.assertEquals(1L, paths.count()); + } + String constraint = "reject_upload_fixture"; + this.jdbc.execute("ALTER TABLE file_mapping ADD CONSTRAINT " + constraint + + " CHECK (job_id <> " + job.getId() + ")"); + try { + Assert.assertThrows(RuntimeException.class, () -> controller.upload("DEFAULT", "hugegraph", job.getId(), + this.file("rejected.csv"), "rejected.csv", 20L, token, 2, 0)); + Assert.assertTrue(this.mappings.listByJob(job.getId()).isEmpty()); + Path directory = this.uploads.resolve("graph-connection-DEFAULT-hugegraph") + .resolve("job-" + job.getId()).resolve("rejected.csv"); + Assert.assertFalse(Files.exists(directory)); + } finally { + this.jdbc.execute("ALTER TABLE file_mapping DROP CONSTRAINT " + constraint); + } + } + + @Test + public void testClearRejectsWrongGraphRouteBeforeTakingAMappingSnapshot() throws Exception { + FileMapping mapping = this.seed(); + Assert.assertThrows(RuntimeException.class, + () -> this.mappingController().clear("DEFAULT", "other", mapping.getJobId())); + Assert.assertEquals((Object) mapping.getJobId(), this.mappings.get(mapping.getId()).getJobId()); + Assert.assertTrue(Files.isRegularFile(Path.of(mapping.getPath()))); + } + + @Test(timeout = 15000) + public void testSameJobFilesWriteConcurrentlyAndDeleteWaitsForBoth() throws Exception { + JobManager job = this.seedJob(); + FileUploadController controller = this.uploadController(); + var tokens = controller.fileToken("DEFAULT", "hugegraph", job.getId(), List.of("a.csv", "b.csv")); + CountDownLatch writing = new CountDownLatch(2); + CountDownLatch finish = new CountDownLatch(1); + ExecutorService requests = Executors.newFixedThreadPool(3); + try { + Future first = requests.submit(() -> controller.upload("DEFAULT", "hugegraph", job.getId(), + this.blockedFile("a.csv", writing, finish), "a.csv", 20L, tokens.get("a.csv"), 2, 0)); + Future second = requests.submit(() -> controller.upload("DEFAULT", "hugegraph", job.getId(), + this.blockedFile("b.csv", writing, finish), "b.csv", 20L, tokens.get("b.csv"), 2, 0)); + Assert.assertTrue(writing.await(2, TimeUnit.SECONDS)); + Future deleting = requests.submit(() -> this.jobs.deleteJob(job.getId())); + assertBlocked(deleting); + finish.countDown(); + first.get(5, TimeUnit.SECONDS); + second.get(5, TimeUnit.SECONDS); + deleting.get(5, TimeUnit.SECONDS); + Assert.assertEquals(0, this.count("file_mapping")); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().isEmpty()); + } finally { + finish.countDown(); + stop(requests); + } + } + + @Test(timeout = 15000) + public void testSameFileTokenAliasesCompleteBeforeMergeAndPreserveContent() throws Exception { + JobManager job = this.seedJob(); + FileUploadController controller = this.uploadController(); + String token = controller.fileToken("DEFAULT", "hugegraph", job.getId(), List.of("chunks.csv")) + .get("chunks.csv"); + String alias = controller.fileToken("DEFAULT", "hugegraph", job.getId(), List.of("chunks.csv")) + .get("chunks.csv"); + JobManager other = this.seedJob(); + String otherToken = controller.fileToken("DEFAULT", "hugegraph", other.getId(), List.of("chunks.csv")) + .get("chunks.csv"); + Assert.assertSame(this.mappings.getUploadingTokenLocks().get(token), + this.mappings.getUploadingTokenLocks().get(alias)); + CountDownLatch writing = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + AtomicInteger secondWrites = new AtomicInteger(); + MockMultipartFile firstPart = new MockMultipartFile("file", "chunks.csv", "text/csv", "name\n".getBytes()) { + @Override + public void transferTo(File destination) throws IOException { + super.transferTo(destination); + writing.countDown(); + await(finish); + } + }; + MockMultipartFile secondPart = new MockMultipartFile("file", "chunks.csv", "text/csv", "alice\n".getBytes()) { + @Override + public void transferTo(File destination) throws IOException { + secondWrites.incrementAndGet(); + super.transferTo(destination); + } + }; + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future first = requests.submit(() -> controller.upload("DEFAULT", "hugegraph", job.getId(), + firstPart, "chunks.csv", 11L, token, 2, 0)); + Assert.assertTrue(writing.await(5, TimeUnit.SECONDS)); + Future second = requests.submit(() -> controller.upload("DEFAULT", "hugegraph", job.getId(), + secondPart, "chunks.csv", 11L, alias, 2, 1)); + assertBlocked(second); + Assert.assertEquals(0, secondWrites.get()); + finish.countDown(); + first.get(5, TimeUnit.SECONDS); + second.get(5, TimeUnit.SECONDS); + FileMapping mapping = this.mappings.get("DEFAULT", "hugegraph", job.getId(), "chunks.csv"); + Assert.assertEquals(FileMappingStatus.COMPLETED, mapping.getFileStatus()); + Assert.assertEquals("name\nalice\n", Files.readString(Path.of(mapping.getPath()))); + Assert.assertEquals(11L, (long) mapping.getTotalSize()); + Assert.assertEquals(List.of("col-1"), mapping.getFileSetting().getColumnNames()); + Assert.assertEquals(List.of("name"), mapping.getFileSetting().getColumnValues()); + Assert.assertEquals(11L, (long) this.jobs.get(job.getId()).getJobSize()); + Assert.assertFalse(this.mappings.getUploadingTokenLocks().containsKey(token)); + Assert.assertFalse(this.mappings.getUploadingTokenLocks().containsKey(alias)); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().containsKey(otherToken)); + } finally { + finish.countDown(); + stop(requests); + } + } + + @Test + public void testMappingIdDeleteRevokesOldMultipartTokenOnlyInItsJob() throws Exception { + this.assertMappingDeleteRevokesToken(false); + } + + @Test + public void testMappingClearRevokesOldMultipartTokenOnlyInItsJob() throws Exception { + this.assertMappingDeleteRevokesToken(true); + } + + private void assertMappingDeleteRevokesToken(boolean clear) throws Exception { + JobManager job = this.seedJob(); + JobManager other = this.seedJob(); + FileUploadController controller = this.uploadController(); + String token = controller.fileToken("DEFAULT", "hugegraph", job.getId(), List.of("partial.csv")) + .get("partial.csv"); + String otherToken = controller.fileToken("DEFAULT", "hugegraph", other.getId(), List.of("partial.csv")) + .get("partial.csv"); + controller.upload("DEFAULT", "hugegraph", job.getId(), this.file("partial.csv"), + "partial.csv", 20L, token, 2, 0); + FileMapping partial = this.mappings.get("DEFAULT", "hugegraph", job.getId(), "partial.csv"); + FileMappingController mappingController = this.mappingController(); + if (clear) { + mappingController.clear("DEFAULT", "hugegraph", job.getId()); + } else { + mappingController.delete("DEFAULT", "hugegraph", job.getId(), partial.getId()); + } + var late = controller.upload("DEFAULT", "hugegraph", job.getId(), this.file("partial.csv"), + "partial.csv", 20L, token, 2, 1); + Assert.assertEquals(org.apache.hugegraph.entity.load.FileUploadResult.Status.FAILURE, late.getStatus()); + Assert.assertTrue(this.mappings.get("DEFAULT", "hugegraph", job.getId(), "partial.csv") == null); + Assert.assertFalse(Files.exists(Path.of(partial.getPath()))); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().containsKey(otherToken)); + var accepted = controller.upload("DEFAULT", "hugegraph", other.getId(), this.file("partial.csv"), + "partial.csv", 20L, otherToken, 2, 0); + Assert.assertFalse(accepted.getStatus() == + org.apache.hugegraph.entity.load.FileUploadResult.Status.FAILURE); + } + + @Test + public void testTokenOnlyJobDeleteInvalidatesTokensAfterCommitButNotRollback() { + JobManager job = this.seedJob(); + FileUploadController controller = this.uploadController(); + String token = controller.fileToken("DEFAULT", "hugegraph", job.getId(), List.of("pending.csv")) + .get("pending.csv"); + new TransactionTemplate(this.transactionManager).execute(status -> { + this.jobs.deleteJob(job.getId()); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().containsKey(token)); + status.setRollbackOnly(); + return null; + }); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().containsKey(token)); + Assert.assertTrue(this.jobs.get(job.getId()) != null); + this.jobs.deleteJob(job.getId()); + Assert.assertFalse(this.mappings.getUploadingTokenLocks().containsKey(token)); + Assert.assertTrue(this.jobs.get(job.getId()) == null); + } + + @Test + public void testTokenDeleteRollbackKeepsPartialUploadAndItsToken() { + JobManager job = this.seedJob(); + FileUploadController controller = this.uploadController(); + String token = controller.fileToken("DEFAULT", "hugegraph", job.getId(), List.of("rollback.csv")) + .get("rollback.csv"); + controller.upload("DEFAULT", "hugegraph", job.getId(), this.file("rollback.csv"), + "rollback.csv", 20L, token, 2, 0); + FileMapping partial = this.mappings.get("DEFAULT", "hugegraph", job.getId(), "rollback.csv"); + new TransactionTemplate(this.transactionManager).execute(status -> { + controller.delete("DEFAULT", "hugegraph", job.getId(), "rollback.csv", token); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().containsKey(token)); + status.setRollbackOnly(); + return null; + }); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().containsKey(token)); + Assert.assertEquals(job.getId(), this.mappings.get(partial.getId()).getJobId()); + Assert.assertTrue(Files.exists(Path.of(partial.getPath()))); + } + + @Test + public void testExpiredUploadRevokesOnlyItsJobEvenWhenDiskCleanupFails() throws Exception { + JobManager job = this.seedJob(); + JobManager other = this.seedJob(); + FileUploadController controller = this.uploadController(); + String token = controller.fileToken("DEFAULT", "hugegraph", job.getId(), List.of("expired.csv")) + .get("expired.csv"); + String otherToken = controller.fileToken("DEFAULT", "hugegraph", other.getId(), List.of("expired.csv")) + .get("expired.csv"); + controller.upload("DEFAULT", "hugegraph", job.getId(), this.file("expired.csv"), + "expired.csv", 20L, token, 2, 0); + FileMapping mapping = this.mappings.get("DEFAULT", "hugegraph", job.getId(), "expired.csv"); + mapping.setPath(this.folder.newFolder("outside-upload-root").getAbsolutePath()); + mapping.setUpdateTime(new Date(System.currentTimeMillis() - 10000)); + this.mappings.update(mapping); + Mockito.when(this.config.get(HubbleOptions.UPLOAD_FILE_MAX_TIME_CONSUMING)).thenReturn(1L); + // Invoke the target synchronously: the production scheduled operation and filesystem checks are unchanged. + GateMappingService target = org.springframework.test.util.AopTestUtils.getUltimateTargetObject(this.mappings); + target.deleteUnfinishedFile(); + Assert.assertFalse(this.mappings.getUploadingTokenLocks().containsKey(token)); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().containsKey(otherToken)); + Assert.assertTrue(this.mappings.get(mapping.getId()) != null); + } + + @Test(timeout = 15000) + public void testTokenIssuanceHoldsAdmissionThroughRegistration() throws Exception { + JobManager job = this.seedJob(); + FileUploadController controller = this.uploadController(); + CountDownLatch parentRead = new CountDownLatch(1); + CountDownLatch register = new CountDownLatch(1); + JobManagerService observed = Mockito.mock(JobManagerService.class, call -> { + Object result = call.getMethod().invoke(this.jobs, call.getArguments()); + if (call.getMethod().getName().equals("get") && call.getArguments().length == 3) { + parentRead.countDown(); + await(register); + } + return result; + }); + ReflectionTestUtils.setField(controller, "jobService", observed); + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future issuing = requests.submit(() -> controller.fileToken("DEFAULT", "hugegraph", job.getId(), + List.of("pending.csv"))); + Assert.assertTrue(parentRead.await(5, TimeUnit.SECONDS)); + Future deleting = requests.submit(() -> this.jobs.deleteJob(job.getId())); + assertBlocked(deleting); + register.countDown(); + issuing.get(5, TimeUnit.SECONDS); + deleting.get(5, TimeUnit.SECONDS); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().isEmpty()); + } finally { + register.countDown(); + stop(requests); + } + } + + @Test(timeout = 15000) + public void testTokenIssuanceWaitsForDeletionThenRejectsMissingParent() throws Exception { + JobManager job = this.seedJob(); + CountDownLatch snapshot = new CountDownLatch(1); + CountDownLatch commit = new CountDownLatch(1); + this.mappings.afterSnapshot(() -> { + snapshot.countDown(); + await(commit); + }); + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future deleting = requests.submit(() -> this.jobs.deleteJob(job.getId())); + Assert.assertTrue(snapshot.await(5, TimeUnit.SECONDS)); + Future issuing = requests.submit(() -> this.uploadController().fileToken("DEFAULT", "hugegraph", + job.getId(), List.of("pending.csv"))); + assertBlocked(issuing); + commit.countDown(); + deleting.get(5, TimeUnit.SECONDS); + Assert.assertThrows(java.util.concurrent.ExecutionException.class, () -> issuing.get(5, TimeUnit.SECONDS)); + Assert.assertTrue(this.mappings.getUploadingTokenLocks().isEmpty()); + } finally { + commit.countDown(); + stop(requests); + } + } + + @Test(timeout = 25000) + public void testJobDeleteRefreshesStoppedSnapshotAfterRetryReleasesTaskReservation() throws Exception { + // Two Spring workers let the replacement actually start while the old + // callback returns; the submit observer keeps the task stripe until then. + this.executor.setMaxPoolSize(2); + this.executor.setCorePoolSize(2); + FileMapping mapping = this.seed(); + LoadTask oldTask = this.metadata(mapping); + oldTask.setStatus(LoadStatus.RUNNING); + CountDownLatch oldLoading = new CountDownLatch(1); + NativeRun oldRun = this.nativeRun(oldTask, oldLoading, null); + this.tasks.save(oldTask); + CountDownLatch oldCallbackEntered = new CountDownLatch(1); + CountDownLatch releaseOldCallback = new CountDownLatch(1); + this.taskExecutor.callbackGate(() -> { + oldCallbackEntered.countDown(); + await(releaseOldCallback); + }); + CountDownLatch retryReserved = new CountDownLatch(1); + CountDownLatch stoppedSnapshot = new CountDownLatch(1); + CountDownLatch newLoading = new CountDownLatch(1); + CountDownLatch cleanupEntered = new CountDownLatch(1); + CountDownLatch releaseCleanup = new CountDownLatch(1); + CountDownLatch newCallbackEntered = new CountDownLatch(1); + CountDownLatch releaseNewCallback = new CountDownLatch(1); + AtomicReference newRun = new AtomicReference<>(); + AtomicReference newTask = new AtomicReference<>(); + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + this.tasks.executeAfterCommit(oldTask); + Assert.assertTrue(oldLoading.await(5, TimeUnit.SECONDS)); + this.tasks.stop(oldTask.getId()); + Assert.assertTrue(oldCallbackEntered.await(5, TimeUnit.SECONDS)); + oldRun.assertClosed(); + Assert.assertNull(this.tasks.retained(oldTask.getId())); + this.tasks.prepareRestart(fresh -> { + try { + NativeRun owned = this.nativeRun(fresh, newLoading, null); + Mockito.doAnswer(call -> { + cleanupEntered.countDown(); + await(releaseCleanup); + return call.callRealMethod(); + }).when(owned.loader).shutdown(); + newRun.set(owned); + newTask.set(fresh); + } catch (Exception failure) { + throw new AssertionError(failure); + } + }); + LoadTaskExecutor observing = new LoadTaskExecutor() { + @Override + public void execute(LoadTask task, Runnable callback) { + JobAdmissionIntegrationTest.this.taskExecutor.execute(task, callback); + if (task != oldTask) { + await(newLoading); + } + } + }; + ReflectionTestUtils.setField(this.tasks, "taskExecutor", observing); + this.tasks.afterReservation(retryReserved::countDown); + Future retry = requests.submit(() -> this.tasks.retry(oldTask.getId(), "fixture-session")); + Assert.assertTrue(retryReserved.await(5, TimeUnit.SECONDS)); + this.tasks.afterReservation(null); + this.tasks.afterSnapshot(() -> { + Assert.assertEquals(LoadStatus.STOPPED.getValue().intValue(), this.jdbc.queryForObject( + "SELECT load_status FROM load_task WHERE id = ?", Integer.class, oldTask.getId()).intValue()); + stoppedSnapshot.countDown(); + }); + Future deleting = requests.submit(() -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertTrue("Delete must actually snapshot STOPPED before acquiring the held stripe", + stoppedSnapshot.await(5, TimeUnit.SECONDS)); + assertBlocked(deleting); + this.taskExecutor.callbackGate(() -> { + newCallbackEntered.countDown(); + await(releaseNewCallback); + }); + releaseOldCallback.countDown(); + Assert.assertEquals(oldTask.getId(), retry.get(5, TimeUnit.SECONDS).getId()); + Assert.assertTrue(newLoading.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("Delete must signal the replacement's actual native cleanup", + cleanupEntered.await(5, TimeUnit.SECONDS)); + Assert.assertFalse(newTask.get().isFinished()); + Assert.assertEquals(LoadStatus.RUNNING.getValue().intValue(), this.jdbc.queryForObject( + "SELECT load_status FROM load_task WHERE id = ?", Integer.class, oldTask.getId()).intValue()); + assertBlocked(deleting); + Assert.assertEquals(1, this.count("load_task")); + Assert.assertEquals(1, this.count("job_manager")); + releaseCleanup.countDown(); + Assert.assertTrue(newCallbackEntered.await(5, TimeUnit.SECONDS)); + newRun.get().assertClosed(); + Assert.assertTrue(newTask.get().isFinished()); + assertBlocked(deleting); + Assert.assertEquals("Deletion must join the replacement's actual callback before SQL delete", + 1, this.count("load_task")); + releaseNewCallback.countDown(); + deleting.get(5, TimeUnit.SECONDS); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertEquals(0, this.count("file_mapping")); + } finally { + this.taskExecutor.callbackGate(null); + releaseOldCallback.countDown(); + releaseCleanup.countDown(); + releaseNewCallback.countDown(); + oldRun.stop.countDown(); + NativeRun replacement = newRun.get(); + if (replacement != null) { + replacement.stop.countDown(); + } + requests.shutdown(); + Assert.assertTrue(requests.awaitTermination(8, TimeUnit.SECONDS)); + this.executor.getThreadPoolExecutor().submit(() -> { }).get(5, TimeUnit.SECONDS); + oldRun.close(); + if (replacement != null) { + replacement.close(); + } + this.tasks.prepareRestart(null); + this.tasks.afterReservation(null); + ReflectionTestUtils.setField(this.tasks, "taskExecutor", this.taskExecutor); + this.executor.setCorePoolSize(1); + this.executor.setMaxPoolSize(1); + } + } + + @Test(timeout = 20000) + public void testRetryWaitsForStoppedRunsActualCallbackBeforeWritingRunning() throws Exception { + this.retryAfterCallback(false); + } + + @Test(timeout = 20000) + public void testRetryRefusesFailedActualCallbackWithoutCreatingAnotherLoader() throws Exception { + this.retryAfterCallback(true); + } + + private void retryAfterCallback(boolean callbackFails) throws Exception { + FileMapping mapping = this.seed(); + LoadTask task = this.metadata(mapping); + task.setStatus(LoadStatus.RUNNING); + CountDownLatch oldLoading = new CountDownLatch(1); + NativeRun oldRun = this.nativeRun(task, oldLoading, null); + this.tasks.save(task); + CountDownLatch callbackEntered = new CountDownLatch(1); + CountDownLatch releaseCallback = new CountDownLatch(1); + CountDownLatch callbackFinished = new CountDownLatch(1); + this.taskExecutor.callbackGate(() -> { + CallbackWriteFailure.thread = Thread.currentThread(); + callbackEntered.countDown(); + await(releaseCallback); + }); + this.taskExecutor.callbackFinished(callbackFinished); + ExecutorService requests = Executors.newSingleThreadExecutor(); + AtomicInteger reconnected = new AtomicInteger(); + AtomicReference restarted = new AtomicReference<>(); + CountDownLatch newLoading = new CountDownLatch(1); + CountDownLatch reserved = new CountDownLatch(1); + boolean trigger = false; + try { + this.tasks.executeAfterCommit(task); + Assert.assertTrue(oldLoading.await(5, TimeUnit.SECONDS)); + this.tasks.stop(task.getId()); + Assert.assertTrue(callbackEntered.await(5, TimeUnit.SECONDS)); + Assert.assertTrue(task.isFinished()); + oldRun.assertClosed(); + Assert.assertNull(this.tasks.retained(task.getId())); + Assert.assertEquals(LoadStatus.STOPPED.getValue().intValue(), this.jdbc.queryForObject( + "SELECT load_status FROM load_task WHERE id = ?", Integer.class, task.getId()).intValue()); + if (callbackFails) { + this.jdbc.execute("CREATE TRIGGER reject_old_callback BEFORE UPDATE ON load_task FOR EACH ROW CALL '" + + CallbackWriteFailure.class.getName() + "'"); + trigger = true; + } + this.tasks.prepareRestart(fresh -> { + reconnected.incrementAndGet(); + try { + restarted.set(this.nativeRun(fresh, newLoading, null)); + } catch (Exception failure) { + throw new AssertionError(failure); + } + }); + this.tasks.afterReservation(reserved::countDown); + Future retry = requests.submit(() -> this.tasks.retry(task.getId(), "fixture-session")); + Assert.assertTrue("Retry must actually own the task reservation", reserved.await(5, TimeUnit.SECONDS)); + assertBlocked(retry); + Assert.assertEquals("An unfinished old callback must precede any reconnect", 0, reconnected.get()); + Assert.assertEquals(LoadStatus.STOPPED.getValue().intValue(), this.jdbc.queryForObject( + "SELECT load_status FROM load_task WHERE id = ?", Integer.class, task.getId()).intValue()); + this.taskExecutor.callbackGate(null); + releaseCallback.countDown(); + Assert.assertTrue(callbackFinished.await(5, TimeUnit.SECONDS)); + if (callbackFails) { + Throwable rejected = Assert.assertThrows(java.util.concurrent.ExecutionException.class, + () -> retry.get(5, TimeUnit.SECONDS)); + Throwable original = this.tasks.callbackFailure(task.getId()); + Assert.assertNotNull(original); + Assert.assertSame("Restart must preserve the actual failed metadata callback cause", + original, rejected.getCause()); + Assert.assertEquals(0, reconnected.get()); + Assert.assertEquals(LoadStatus.STOPPED.getValue().intValue(), this.jdbc.queryForObject( + "SELECT load_status FROM load_task WHERE id = ?", Integer.class, task.getId()).intValue()); + Assert.assertSame(original, Assert.assertThrows(RuntimeException.class, + () -> this.tasks.retry(task.getId(), "fixture-session"))); + Assert.assertThrows(RuntimeException.class, () -> this.jobs.deleteJob(mapping.getJobId())); + } else { + LoadTask running = retry.get(5, TimeUnit.SECONDS); + Assert.assertTrue(newLoading.await(5, TimeUnit.SECONDS)); + Assert.assertEquals(1, reconnected.get()); + Assert.assertEquals(LoadStatus.RUNNING, running.getStatus()); + Assert.assertEquals("Completed old callbacks cannot overwrite the restarted H2 row", + LoadStatus.RUNNING.getValue().intValue(), this.jdbc.queryForObject( + "SELECT load_status FROM load_task WHERE id = ?", Integer.class, task.getId()).intValue()); + this.tasks.stop(task.getId()); + this.jobs.deleteJob(mapping.getJobId()); + Assert.assertEquals(0, this.count("load_task")); + restarted.get().assertClosed(); + } + } finally { + this.taskExecutor.callbackGate(null); + releaseCallback.countDown(); + oldRun.stop.countDown(); + NativeRun newRun = restarted.get(); + if (newRun != null) { + newRun.stop.countDown(); + } + requests.shutdown(); + Assert.assertTrue(requests.awaitTermination(8, TimeUnit.SECONDS)); + this.executor.getThreadPoolExecutor().submit(() -> { }).get(5, TimeUnit.SECONDS); + oldRun.close(); + if (newRun != null) { + newRun.close(); + } + if (trigger) { + this.jdbc.execute("DROP TRIGGER reject_old_callback"); + } + CallbackWriteFailure.thread = null; + this.tasks.prepareRestart(null); + this.tasks.afterReservation(null); + this.taskExecutor.callbackFinished(null); + } + } + + public static class CallbackWriteFailure implements org.h2.api.Trigger { + private static volatile Thread thread; + + @Override + public void fire(java.sql.Connection connection, Object[] oldRow, Object[] newRow) + throws java.sql.SQLException { + if (Thread.currentThread() == thread) { + throw new java.sql.SQLException("controlled actual old callback write failure", "45000"); + } + } + } + + @Test + public void testDirectStartDispatchErrorRetainsOneRecoveryTicket() throws Exception { + FileMapping mapping = this.seed(); + Error dispatchFailure = new AssertionError("controlled direct dispatch Error"); + RuntimeException cleanupFailure = new IllegalStateException("controlled unscheduled cleanup failure"); + AtomicReference observed = new AtomicReference<>(); + LoadTaskExecutor rejecting = Mockito.mock(LoadTaskExecutor.class); + Mockito.doAnswer(call -> { + observed.set(call.getArgument(0)); + throw dispatchFailure; + }).when(rejecting).execute(Mockito.any(LoadTask.class), Mockito.any()); + ReflectionTestUtils.setField(this.tasks, "taskExecutor", rejecting); + try (MockedConstruction constructed = Mockito.mockConstruction(HugeGraphLoader.class, + (loader, construction) -> Mockito.doThrow(cleanupFailure).when(loader).shutdown())) { + Assert.assertSame(dispatchFailure, Assert.assertThrows(Error.class, + () -> this.tasks.start(this.connection(), mapping, this.client()))); + LoadTask task = observed.get(); + Assert.assertNotNull(task); + Assert.assertTrue(task.isFinished()); + Assert.assertTrue(task.isCleanupFailed()); + Assert.assertSame(task, this.tasks.retained(task.getId())); + Assert.assertEquals(LoadStatus.FAILED, this.tasks.get(task.getId()).getStatus()); + java.util.Set runs = (java.util.Set) ReflectionTestUtils.getField(this.tasks, "acceptedRuns"); + long matching = runs.stream().filter(run -> ReflectionTestUtils.getField(run, "task") == task).count(); + Assert.assertEquals("The dispatch and start catch paths own the same accepted run", 1L, matching); + Assert.assertThrows(RuntimeException.class, () -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertEquals(1, this.count("job_manager")); + Mockito.verify(constructed.constructed().get(0), Mockito.atLeastOnce()).shutdown(); + } finally { + ReflectionTestUtils.setField(this.tasks, "taskExecutor", this.taskExecutor); + } + } + + @Test(timeout = 20000) + public void testDeleteWaitsForActualCallbackAfterNativeFinished() throws Exception { + FileMapping mapping = this.seed(); + LoadTask task = this.metadata(mapping); + task.setStatus(LoadStatus.RUNNING); + CountDownLatch callbackEntered = new CountDownLatch(1); + CountDownLatch releaseCallback = new CountDownLatch(1); + CountDownLatch callbackFinished = new CountDownLatch(1); + NativeRun nativeRun = this.nativeRun(task, null, null); + this.tasks.save(task); + this.taskExecutor.callbackGate(() -> { + callbackEntered.countDown(); + await(releaseCallback); + }); + this.taskExecutor.callbackFinished(callbackFinished); + ExecutorService requests = Executors.newSingleThreadExecutor(); + CountDownLatch deleteSnapshot = new CountDownLatch(1); + this.tasks.afterSnapshot(deleteSnapshot::countDown); + try { + this.tasks.executeAfterCommit(task); + Assert.assertTrue(callbackEntered.await(5, TimeUnit.SECONDS)); + Assert.assertTrue(task.isFinished()); + nativeRun.assertClosed(); + Future deletion = requests.submit(() -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertTrue(deleteSnapshot.await(5, TimeUnit.SECONDS)); + assertBlocked(deletion); + com.zaxxer.hikari.HikariDataSource dataSource = + (com.zaxxer.hikari.HikariDataSource) this.jdbc.getDataSource(); + Assert.assertEquals("Callback waiting must leave metadata connections available", 0, + dataSource.getHikariPoolMXBean().getActiveConnections()); + Assert.assertEquals("Native finished is not a callback certificate", 1, this.count("load_task")); + Assert.assertEquals(1, this.count("job_manager")); + releaseCallback.countDown(); + deletion.get(5, TimeUnit.SECONDS); + Assert.assertTrue(callbackFinished.await(5, TimeUnit.SECONDS)); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertEquals(0, this.count("file_mapping")); + } finally { + releaseCallback.countDown(); + this.executor.getThreadPoolExecutor().submit(() -> { }).get(5, TimeUnit.SECONDS); + nativeRun.close(); + stop(requests); + this.taskExecutor.callbackGate(null); + this.taskExecutor.callbackFinished(null); + } + } + + @Test(timeout = 20000) + public void testReplacementUploadTokenWaitsForFailedUploadCleanup() throws Exception { + FileMapping parent = this.seed(); + FileUploadController controller = this.uploadController(); + int jobId = parent.getJobId(); + String name = "retry.csv"; + String token = controller.fileToken("DEFAULT", "hugegraph", jobId, List.of(name)).get(name); + CountDownLatch cleaning = new CountDownLatch(1); + CountDownLatch releaseCleanup = new CountDownLatch(1); + this.mappings.beforeCleanup(() -> { + cleaning.countDown(); + await(releaseCleanup); + }); + MockMultipartFile upload = new MockMultipartFile("file", name, "text/csv", "name\nnew\n".getBytes()) { + @Override + public void transferTo(File destination) throws IOException { + super.transferTo(destination); + // The merged result fails the final quota recheck. + Mockito.when(config.get(HubbleOptions.UPLOAD_TOTAL_FILE_SIZE_LIMIT)).thenReturn(1L); + } + }; + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future failing = requests.submit(() -> Assert.assertThrows(RuntimeException.class, + () -> controller.upload("DEFAULT", "hugegraph", jobId, upload, name, null, token, 1, 0))); + Assert.assertTrue(cleaning.await(5, TimeUnit.SECONDS)); + Future replacement = requests.submit(() -> + controller.fileToken("DEFAULT", "hugegraph", jobId, List.of(name)).get(name)); + assertBlocked(replacement); + Mockito.when(this.config.get(HubbleOptions.UPLOAD_TOTAL_FILE_SIZE_LIMIT)).thenReturn(4096L); + releaseCleanup.countDown(); + failing.get(5, TimeUnit.SECONDS); + String nextToken = replacement.get(5, TimeUnit.SECONDS); + MockMultipartFile next = new MockMultipartFile("file", name, "text/csv", "name\nnew\n".getBytes()); + controller.upload("DEFAULT", "hugegraph", jobId, next, name, null, nextToken, 1, 0); + FileMapping completed = this.mappings.get("DEFAULT", "hugegraph", jobId, name); + Assert.assertNotNull(completed); + Assert.assertEquals(FileMappingStatus.COMPLETED, completed.getFileStatus()); + Assert.assertTrue(Files.isRegularFile(Path.of(completed.getPath()))); + } finally { + releaseCleanup.countDown(); + this.mappings.beforeCleanup(null); + stop(requests); + } + } + + @Test(timeout = 20000) + public void testDeleteRejectsPendingCallbackInsideExistingTransaction() throws Exception { + FileMapping mapping = this.seed(); + LoadTask task = this.metadata(mapping); + task.setStatus(LoadStatus.RUNNING); + CountDownLatch callbackEntered = new CountDownLatch(1); + CountDownLatch releaseCallback = new CountDownLatch(1); + NativeRun nativeRun = this.nativeRun(task, null, null); + this.tasks.save(task); + this.taskExecutor.callbackGate(() -> { + callbackEntered.countDown(); + await(releaseCallback); + }); + try { + this.tasks.executeAfterCommit(task); + Assert.assertTrue(callbackEntered.await(5, TimeUnit.SECONDS)); + new TransactionTemplate(this.transactionManager).executeWithoutResult(status -> { + Throwable failure = Assert.assertThrows(RuntimeException.class, + () -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertTrue(failure.getMessage().contains("retry after the current transaction")); + Assert.assertEquals(1, this.count("load_task")); + Assert.assertEquals(1, this.count("job_manager")); + Assert.assertEquals(1, this.count("file_mapping")); + Assert.assertEquals(LoadStatus.RUNNING, this.tasks.get(task.getId()).getStatus()); + }); + releaseCallback.countDown(); + this.executor.getThreadPoolExecutor().submit(() -> { }).get(5, TimeUnit.SECONDS); + new TransactionTemplate(this.transactionManager).executeWithoutResult(status -> + this.jobs.deleteJob(mapping.getJobId())); + Assert.assertEquals(0, this.count("job_manager")); + nativeRun.assertClosed(); + } finally { + releaseCallback.countDown(); + this.executor.getThreadPoolExecutor().submit(() -> { }).get(5, TimeUnit.SECONDS); + nativeRun.close(); + this.taskExecutor.callbackGate(null); + } + } + + @Test(timeout = 25000) + public void testQueuedFirstJobDeleteSignalsEveryNativeOwnerBeforeWaiting() throws Exception { + FileMapping mapping = this.seed(); + this.executor.setMaxPoolSize(8); + this.executor.setCorePoolSize(8); + CountDownLatch active = new CountDownLatch(8); + CountDownLatch stopRequests = new CountDownLatch(9); + List nativeRuns = new ArrayList<>(); + List tasks = new ArrayList<>(); + ExecutorService requests = Executors.newSingleThreadExecutor(); + CountDownLatch deleteSnapshot = new CountDownLatch(1); + try { + for (int i = 0; i < 9; i++) { + LoadTask task = this.metadata(mapping); + task.setStatus(LoadStatus.RUNNING); + NativeRun nativeRun = this.nativeRun(task, active, stopRequests); + nativeRuns.add(nativeRun); + tasks.add(task); + this.tasks.save(task); + this.tasks.executeAfterCommit(task); + if (i == 7) { + Assert.assertTrue(active.await(5, TimeUnit.SECONDS)); + } + } + LoadTask queued = tasks.get(8); + Assert.assertFalse(queued.isFinished()); + Assert.assertEquals(1, this.executor.getThreadPoolExecutor().getQueue().size()); + this.tasks.queuedFirst(queued.getId()); + this.tasks.afterSnapshot(deleteSnapshot::countDown); + Future deletion = requests.submit(() -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertTrue(deleteSnapshot.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("Every actual native stop intent must precede queued-first wait", + stopRequests.await(5, TimeUnit.SECONDS)); + deletion.get(5, TimeUnit.SECONDS); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertEquals(0, this.count("job_manager")); + for (int i = 0; i < tasks.size(); i++) { + Assert.assertTrue(tasks.get(i).isFinished()); + nativeRuns.get(i).assertClosed(); + } + Mockito.verify(nativeRuns.get(8).loader, Mockito.never()).load(); + } finally { + for (NativeRun run : nativeRuns) { + run.stop.countDown(); + } + requests.shutdown(); + Assert.assertTrue(requests.awaitTermination(8, TimeUnit.SECONDS)); + for (NativeRun run : nativeRuns) { + run.close(); + } + this.executor.getThreadPoolExecutor().submit(() -> { }).get(5, TimeUnit.SECONDS); + this.executor.setCorePoolSize(1); + this.executor.setMaxPoolSize(1); + this.tasks.queuedFirst(null); + } + } + + private NativeRun nativeRun(LoadTask task, CountDownLatch active, CountDownLatch stopRequests) throws Exception { + LoadOptions options = task.getOptions(); + LoadContext context = Mockito.spy(LoadContext.forOffline(options, new SchemaCache(null))); + TaskManager manager = new TaskManager(context); + Class unsafeType = Class.forName("sun.misc.Unsafe"); + java.lang.reflect.Field field = unsafeType.getDeclaredField("theUnsafe"); + field.setAccessible(true); + HugeGraphLoader raw = (HugeGraphLoader) unsafeType.getMethod("allocateInstance", Class.class) + .invoke(field.get(null), HugeGraphLoader.class); + ReflectionTestUtils.setField(raw, "context", context); + ReflectionTestUtils.setField(raw, "manager", manager); + ReflectionTestUtils.setField(raw, "options", options); + HugeGraphLoader loader = Mockito.spy(raw); + CountDownLatch stop = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicBoolean requested = new java.util.concurrent.atomic.AtomicBoolean(); + Mockito.doAnswer(call -> { + Object value = call.callRealMethod(); + stop.countDown(); + if (stopRequests != null && requested.compareAndSet(false, true)) { + stopRequests.countDown(); + } + return value; + }).when(context).stopLoading(); + Mockito.doAnswer(call -> { + if (active != null) { + active.countDown(); + await(stop); + } + return true; + }).when(loader).load(); + task.setLoader(loader); + return new NativeRun(loader, context, manager, stop); + } + + private static final class NativeRun implements AutoCloseable { + private final HugeGraphLoader loader; + private final LoadContext context; + private final TaskManager manager; + private final CountDownLatch stop; + private NativeRun(HugeGraphLoader loader, LoadContext context, TaskManager manager, CountDownLatch stop) { + this.loader = loader; + this.context = context; + this.manager = manager; + this.stop = stop; + } + + private void assertClosed() { + Assert.assertTrue(this.context.closed()); + Assert.assertTrue(((ExecutorService) ReflectionTestUtils.getField(this.manager, "batchService")) + .isTerminated()); + Assert.assertTrue(((ExecutorService) ReflectionTestUtils.getField(this.manager, "singleService")) + .isTerminated()); + Mockito.verify(this.context).close(Mockito.anyBoolean()); + } + + @Override + public void close() { + this.stop.countDown(); + this.loader.shutdown(); + } + } + + private MockMultipartFile file(String name) { + return new MockMultipartFile("file", name, "text/csv", "name\nperson\n".getBytes()); + } + + private MockMultipartFile blockedFile(String name, CountDownLatch writing, CountDownLatch finish) { + return new MockMultipartFile("file", name, "text/csv", "name\nperson\n".getBytes()) { + @Override + public void transferTo(File destination) throws IOException { + writing.countDown(); + await(finish); + super.transferTo(destination); + } + }; + } + + private JobManager seedJob() { + JobManager job = JobManager.builder().graphSpace("DEFAULT").graph("hugegraph") + .jobName(UUID.randomUUID().toString()).jobStatus(JobStatus.SETTING) + .createTime(new Date()).updateTime(new Date()).build(); + this.jobs.save(job); + return job; + } + + private FileMappingController mappingController() { + FileMappingController controller = new FileMappingController() { + @Override + protected HugeClient requireGraphSpaceWrite(String graphSpace) { + return client(); + } + }; + ReflectionTestUtils.setField(controller, "service", this.mappings); + ReflectionTestUtils.setField(controller, "jobService", this.jobs); + ReflectionTestUtils.setField(controller, "taskService", this.tasks); + return controller; + } + + @Test(timeout = 15000) + public void testUploadCannotCreateMappingAfterDeleteSnapshot() throws Exception { + FileMapping mapping = this.seed(); + FileUploadController controller = this.uploadController(); + String token = controller.fileToken("DEFAULT", "hugegraph", mapping.getJobId(), + List.of("later.csv")).get("later.csv"); + CountDownLatch snapshot = new CountDownLatch(1); + CountDownLatch complete = new CountDownLatch(1); + this.mappings.afterSnapshot(() -> { + snapshot.countDown(); + await(complete); + }); + AtomicInteger writes = new AtomicInteger(); + MockMultipartFile file = new MockMultipartFile("file", "later.csv", "text/csv", "name\nnew\n".getBytes()) { + @Override + public void transferTo(File destination) throws IOException { + writes.incrementAndGet(); + super.transferTo(destination); + } + }; + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future deleting = requests.submit(() -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertTrue(snapshot.await(5, TimeUnit.SECONDS)); + CountDownLatch entered = new CountDownLatch(1); + Future uploading = requests.submit(() -> { + entered.countDown(); + return controller.upload("DEFAULT", "hugegraph", mapping.getJobId(), file, + "later.csv", null, token, 1, 0); + }); + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + assertBlocked(uploading); + complete.countDown(); + deleting.get(5, TimeUnit.SECONDS); + Assert.assertThrows(java.util.concurrent.ExecutionException.class, + () -> uploading.get(5, TimeUnit.SECONDS)); + Assert.assertEquals(0, writes.get()); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertEquals(0, this.count("file_mapping")); + try (var files = Files.walk(this.uploads)) { + Assert.assertFalse(files.anyMatch(Files::isRegularFile)); + } + } finally { + complete.countDown(); + stop(requests); + } + } + + @Test(timeout = 15000) + public void testDeleteWaitsForUploadingFileAndCleansCommittedMapping() throws Exception { + FileMapping mapping = this.seed(); + FileUploadController controller = this.uploadController(); + String token = controller.fileToken("DEFAULT", "hugegraph", mapping.getJobId(), + List.of("later.csv")).get("later.csv"); + CountDownLatch writing = new CountDownLatch(1); + CountDownLatch complete = new CountDownLatch(1); + MockMultipartFile file = new MockMultipartFile("file", "later.csv", "text/csv", "name\nnew\n".getBytes()) { + @Override + public void transferTo(File destination) throws IOException { + writing.countDown(); + await(complete); + super.transferTo(destination); + } + }; + ExecutorService requests = Executors.newFixedThreadPool(2); + try { + Future uploading = requests.submit(() -> controller.upload("DEFAULT", "hugegraph", mapping.getJobId(), + file, "later.csv", null, token, 1, 0)); + Assert.assertTrue(writing.await(5, TimeUnit.SECONDS)); + CountDownLatch entered = new CountDownLatch(1); + Future deleting = requests.submit(() -> { + entered.countDown(); + this.jobs.deleteJob(mapping.getJobId()); + }); + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + assertBlocked(deleting); + Assert.assertEquals(1, this.count("job_manager")); + complete.countDown(); + uploading.get(5, TimeUnit.SECONDS); + deleting.get(5, TimeUnit.SECONDS); + Assert.assertEquals(0, this.count("job_manager")); + Assert.assertEquals(0, this.count("file_mapping")); + try (var files = Files.walk(this.uploads)) { + Assert.assertFalse(files.anyMatch(Files::isRegularFile)); + } + } finally { + complete.countDown(); + stop(requests); + } + } + + @Test + public void testDetachedMappingCannotStartFromStaleControllerSnapshot() throws Exception { + FileMapping mapping = this.seed(); + this.jobs.deleteMappings(mapping.getJobId(), List.of(mapping)); + Assert.assertTrue(this.mappings.get(mapping.getId()) == null || + this.mappings.get(mapping.getId()).getJobId() == null); + try (MockedConstruction constructed = Mockito.mockConstruction(HugeGraphLoader.class)) { + Assert.assertThrows(RuntimeException.class, + () -> this.tasks.start(this.connection(), mapping, this.client())); + Assert.assertTrue(constructed.constructed().isEmpty()); + Assert.assertEquals(0, this.count("load_task")); + } + } + + @Test + public void testRunningTaskPreventsMappingDetach() throws Exception { + FileMapping mapping = this.seed(); + LoadTask task = this.metadata(mapping); + task.setStatus(LoadStatus.RUNNING); + this.tasks.save(task); + Assert.assertThrows(RuntimeException.class, + () -> this.jobs.deleteMappings(mapping.getJobId(), List.of(mapping))); + Assert.assertEquals(mapping.getJobId(), this.mappings.get(mapping.getId()).getJobId()); + Assert.assertTrue(Files.isRegularFile(Path.of(mapping.getPath()))); + } + + @Test + public void testCleanupFailureBeforeGeneratedTaskIdProtectsParent() throws Exception { + FileMapping mapping = this.seed(); + RuntimeException cleanup = new IllegalStateException("fixture native cleanup failed"); + String constraint = "reject_fixture_task"; + this.jdbc.execute("ALTER TABLE load_task ADD CONSTRAINT " + constraint + + " CHECK (job_id <> " + mapping.getJobId() + ")"); + try (MockedConstruction constructed = Mockito.mockConstruction(HugeGraphLoader.class, + (loader, context) -> Mockito.doThrow(cleanup).when(loader).shutdown())) { + Throwable insert = Assert.assertThrows(RuntimeException.class, + () -> this.tasks.start(this.connection(), mapping, this.client())); + Assert.assertTrue(insert instanceof org.springframework.dao.DataIntegrityViolationException); + Assert.assertEquals(1, constructed.constructed().size()); + Mockito.verify(constructed.constructed().get(0)).shutdown(); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertThrows(RuntimeException.class, () -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertEquals(1, this.count("job_manager")); + Assert.assertEquals(1, this.count("file_mapping")); + Assert.assertTrue(Files.isRegularFile(Path.of(mapping.getPath()))); + Assert.assertTrue(java.util.Arrays.asList(insert.getSuppressed()).contains(cleanup)); + } finally { + this.jdbc.execute("ALTER TABLE load_task DROP CONSTRAINT " + constraint); + } + } + + @Test + public void testRollbackCleanupFailureProtectsParentWithoutTaskRow() throws Exception { + FileMapping mapping = this.seed(); + RuntimeException cleanup = new IllegalStateException("fixture rollback cleanup failed"); + AtomicReference created = new AtomicReference<>(); + try (MockedConstruction constructed = Mockito.mockConstruction(HugeGraphLoader.class, + (loader, context) -> Mockito.doThrow(cleanup).when(loader).shutdown())) { + new TransactionTemplate(this.transactionManager).execute(status -> { + created.set(this.tasks.start(this.connection(), mapping, this.client())); + status.setRollbackOnly(); + return null; + }); + Assert.assertNotNull(created.get().getId()); + Assert.assertTrue(created.get().isCleanupFailed()); + Assert.assertTrue(created.get().isFinished()); + Mockito.verify(constructed.constructed().get(0)).shutdown(); + Assert.assertEquals(0, this.count("load_task")); + Assert.assertThrows(RuntimeException.class, () -> this.jobs.deleteJob(mapping.getJobId())); + Assert.assertEquals(1, this.count("job_manager")); + Assert.assertEquals(1, this.count("file_mapping")); + } + } + + private FileMapping seed() throws Exception { + JobManager job = JobManager.builder().graphSpace("DEFAULT").graph("hugegraph") + .jobName(UUID.randomUUID().toString()).jobStatus(JobStatus.SETTING) + .createTime(new Date()).updateTime(new Date()).build(); + this.jobs.save(job); + Path dir = Files.createDirectories(this.uploads.resolve(UUID.randomUUID().toString())); + Path input = dir.resolve("people.csv"); + Files.writeString(input, "name\nperson\n"); + FileMapping mapping = new FileMapping(); + mapping.setGraphSpace("DEFAULT"); + mapping.setGraph("hugegraph"); + mapping.setJobId(job.getId()); + mapping.setName("people.csv"); + mapping.setPath(input.toString()); + mapping.setTotalLines(2L); + mapping.setTotalSize(12L); + mapping.setFileStatus(FileMappingStatus.COMPLETED); + mapping.setFileSetting(FileSetting.builder().columnNames(List.of("name")).format("CSV").delimiter(",") + .charset("UTF-8").dateFormat("yyyy-MM-dd HH:mm:ss").timeZone("GMT+8") + .skippedLine("(^#|^//).*").listFormat(new ListFormat()).build()); + mapping.setVertexMappings(Collections.emptySet()); + mapping.setEdgeMappings(Collections.emptySet()); + mapping.setLoadParameter(new LoadParameter()); + mapping.setCreateTime(new Date()); + mapping.setUpdateTime(new Date()); + this.mappings.save(mapping); + return mapping; + } + + private LoadTask metadata(FileMapping mapping) { + return LoadTask.builder().graphSpace("DEFAULT").graph("hugegraph").jobId(mapping.getJobId()) + .fileId(mapping.getId()).fileName(mapping.getName()).options(new LoadOptions()) + .vertices(Collections.emptySet()).edges(Collections.emptySet()).fileTotalLines(2L) + .fileReadLines(0L).lastDuration(0L).currDuration(0L).createTime(new Date()) + .status(LoadStatus.STOPPED).build(); + } + + private LoadTaskController controller() { + LoadTaskController controller = new LoadTaskController(this.tasks) { + @Override + protected HugeClient requireGraphSpaceWrite(String graphSpace) { + return client(); + } + + @Override + protected HugeClient authClient(String space, String graph) { + return client(); + } + + @Override + protected String getToken() { + return "fixture-current-token"; + } + + @Override + protected String getUser() { + return "fixture-user"; + } + }; + ReflectionTestUtils.setField(controller, "jobService", this.jobs); + ReflectionTestUtils.setField(controller, "fmService", this.mappings); + ReflectionTestUtils.setField(controller, "config", this.config); + return controller; + } + + private FileUploadController uploadController() { + FileUploadController controller = new FileUploadController() { + @Override + protected HugeClient requireGraphSpaceWrite(String graphSpace) { + return client(); + } + }; + ReflectionTestUtils.setField(controller, "service", this.mappings); + ReflectionTestUtils.setField(controller, "jobService", this.jobs); + ReflectionTestUtils.setField(controller, "config", this.config); + return controller; + } + + private GraphConnection connection() { + GraphConnection connection = new GraphConnection(); + connection.setGraphSpace("DEFAULT"); + connection.setGraph("hugegraph"); + return connection; + } + + private HugeClient client() { + return Mockito.mock(HugeClient.class); + } + + private int count(String table) { + return this.jdbc.queryForObject("SELECT COUNT(*) FROM " + table, Integer.class); + } + + private void clearRows() { + this.jdbc.update("DELETE FROM load_task"); + this.jdbc.update("DELETE FROM file_mapping"); + this.jdbc.update("DELETE FROM job_manager"); + } + + private static void await(CountDownLatch latch) { + try { + Assert.assertTrue(latch.await(8, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + + private static void assertBlocked(Future request) throws Exception { + try { + request.get(300, TimeUnit.MILLISECONDS); + Assert.fail("request bypassed lifecycle reservation"); + } catch (TimeoutException expected) { + // The owning transaction still holds the reservation. + } + } + + private static void stop(ExecutorService requests) throws Exception { + requests.shutdownNow(); + Assert.assertTrue(requests.awaitTermination(5, TimeUnit.SECONDS)); + } + + public static class GateTaskService extends LoadTaskService { + private Runnable afterSnapshot; + private Integer queuedFirst; + private java.util.function.Consumer prepareRestart; + private Runnable afterReservation; + + public void prepareRestart(java.util.function.Consumer prepare) { + this.prepareRestart = prepare; + } + + public void afterReservation(Runnable action) { + this.afterReservation = action; + } + + @Override + TaskReservation reserveTasks(List ids) { + TaskReservation reservation = super.reserveTasks(ids); + Runnable action = this.afterReservation; + if (action != null) { + action.run(); + } + return reservation; + } + + @Override + public LoadTask get(int id) { + LoadTask task = super.get(id); + java.util.function.Consumer prepare = this.prepareRestart; + if (task != null && prepare != null) { + LoadTask controlled = Mockito.spy(task); + Mockito.doAnswer(call -> { + prepare.accept(controlled); + return null; + }).when(controlled).reconnect("fixture-session"); + return controlled; + } + return task; + } + + public Throwable callbackFailure(int taskId) { + java.util.Set accepted = (java.util.Set) ReflectionTestUtils.getField(this, "acceptedRuns"); + for (Object run : accepted) { + LoadTask task = (LoadTask) ReflectionTestUtils.getField(run, "task"); + if (task.getId() != null && task.getId() == taskId) { + java.util.concurrent.CompletableFuture completed = (java.util.concurrent.CompletableFuture) + ReflectionTestUtils.getField(run, "completed"); + try { + completed.join(); + } catch (java.util.concurrent.CompletionException failure) { + return failure.getCause(); + } + } + } + return null; + } + + public void afterSnapshot(Runnable action) { + this.afterSnapshot = action; + } + + public void queuedFirst(Integer id) { + this.queuedFirst = id; + } + + @Override + public List taskListByJob(int id) { + List snapshot = super.taskListByJob(id); + if (this.queuedFirst != null) { + snapshot.sort(Comparator.comparing(task -> !task.getId().equals(this.queuedFirst))); + } + Runnable action = this.afterSnapshot; + this.afterSnapshot = null; + if (action != null) { + action.run(); + } + return snapshot; + } + + public LoadTask retained(int id) { + @SuppressWarnings("unchecked") + java.util.Map running = (java.util.Map) + ReflectionTestUtils.getField(this, "runningTaskContainer"); + return running.get(id); + } + } + + public static class GateMappingService extends FileMappingService { + private Runnable beforeCleanup; + + public void beforeCleanup(Runnable action) { + this.beforeCleanup = action; + } + + @Override + public void cleanupMappings(List mappings) { + Runnable action = this.beforeCleanup; + this.beforeCleanup = null; + if (action != null) { + action.run(); + } + super.cleanupMappings(mappings); + } + + private Runnable afterSnapshot; + public void afterSnapshot(Runnable action) { + this.afterSnapshot = action; + } + + @Override + public List listByJob(int id) { + List snapshot = super.listByJob(id); + Runnable action = this.afterSnapshot; + this.afterSnapshot = null; + if (action != null) { + action.run(); + } + return snapshot; + } + } + + public static class GateLoadTaskExecutor extends LoadTaskExecutor { + private volatile Runnable callbackGate; + private volatile CountDownLatch callbackFinished; + + public void callbackGate(Runnable action) { + this.callbackGate = action; + } + + public void callbackFinished(CountDownLatch signal) { + this.callbackFinished = signal; + } + + @Async + @Override + public void execute(LoadTask task, Runnable callback) { + super.execute(task, () -> { + Runnable action = this.callbackGate; + if (action != null) { + action.run(); + } + try { + callback.run(); + } finally { + CountDownLatch signal = this.callbackFinished; + if (signal != null) { + signal.countDown(); + } + } + }); + } + } + + @SpringBootConfiguration + @EnableAutoConfiguration + @EnableTransactionManagement + @EnableAsync + @MapperScan("org.apache.hugegraph.mapper.load") + public static class Configuration { + @Bean + public GateTaskService loadTaskService() { + return new GateTaskService(); + } + + @Bean + public JobManagerService jobManagerService() { + return new JobManagerService(); + } + + @Bean + public GateMappingService fileMappingService() { + return new GateMappingService(); + } + + @Bean + public GateLoadTaskExecutor loadTaskExecutor() { + return new GateLoadTaskExecutor(); + } + + @Bean + public HugeConfig hugeConfig() { + return Mockito.mock(HugeConfig.class); + } + + @Bean + public ThreadPoolTaskExecutor taskExecutor() { + ThreadPoolTaskExecutor pool = new ThreadPoolTaskExecutor(); + pool.setCorePoolSize(1); + pool.setMaxPoolSize(1); + pool.setQueueCapacity(4); + pool.setThreadNamePrefix("job-admission-fixture-"); + pool.setWaitForTasksToCompleteOnShutdown(true); + pool.setAwaitTerminationSeconds(5); + return pool; + } + } +} diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/testutil/NativeCleanupTestSupport.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/testutil/NativeCleanupTestSupport.java new file mode 100644 index 000000000..97b48efaa --- /dev/null +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/testutil/NativeCleanupTestSupport.java @@ -0,0 +1,171 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.testutil; + +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.hugegraph.entity.enums.LoadStatus; +import org.apache.hugegraph.entity.load.LoadTask; +import org.apache.hugegraph.loader.HugeGraphLoader; +import org.apache.hugegraph.loader.executor.LoadOptions; +import org.apache.hugegraph.loader.mapping.LoadMapping; +import org.apache.hugegraph.loader.task.TaskManager; +import org.apache.hugegraph.util.JsonUtil; +import org.mockito.Mockito; + +import com.sun.net.httpserver.HttpServer; +import okhttp3.OkHttpClient; + +/** Real native shutdown and HTTP reset boundary; only load failure entry is controlled. */ +public final class NativeCleanupTestSupport implements AutoCloseable { + + public final HugeGraphLoader loader; + public final RuntimeException original = new IllegalStateException("recorded load failure"); + public final AtomicReference shutdownFailure = new AtomicReference<>(); + private final HttpServer server; + private volatile String mode = "NONE"; + + public NativeCleanupTestSupport(LoadTask task, Path input, boolean rejectReset, + LoadStatus requestedStatus) throws Exception { + this.server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + this.server.createContext("/", exchange -> { + Object response; + int status = 200; + if (exchange.getRequestURI().getPath().equals("/versions")) { + response = Map.of("versions", Map.of("core", "1.8.0", "api", "0.74")); + } else if (exchange.getRequestURI().getPath().endsWith("/mode")) { + if (exchange.getRequestMethod().equals("GET")) { + response = Map.of("mode", this.mode); + } else { + String value = JsonUtil.fromJson(new String(exchange.getRequestBody().readAllBytes(), + StandardCharsets.UTF_8), String.class); + if (rejectReset && value.equals("NONE")) { + status = 500; + response = Map.of("exception", "class java.lang.IllegalStateException", + "message", "independent mode reset failed"); + } else { + this.mode = value; + response = Map.of(); + } + } + } else { + status = 404; + response = Map.of("message", "unsupported fixture path"); + } + byte[] body = JsonUtil.toJson(response).getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, body.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(body); + } + }); + this.server.start(); + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = this.server.getAddress().getPort(); + options.file = input.toString(); + this.loader = Mockito.spy(new HugeGraphLoader(options, new LoadMapping(Collections.emptyList()))); + task.setLoader(this.loader); + task.setOptions(options); + this.loader.context().setLoadingMode(); + Mockito.doAnswer(invocation -> { + task.setStatus(requestedStatus); + this.loader.context().failLoading(this.original); + throw this.original; + }).when(this.loader).load(); + Mockito.doAnswer(invocation -> { + try { + return invocation.callRealMethod(); + } catch (Throwable failure) { + this.shutdownFailure.set(failure); + throw failure; + } + }).when(this.loader).shutdown(); + } + + public String mode() { + return this.mode; + } + + public void assertResourcesClosed() throws Exception { + Assert.assertTrue(this.loader.context().closed()); + TaskManager manager = (TaskManager) field(this.loader, HugeGraphLoader.class, "manager"); + Assert.assertTrue(((ExecutorService) field(manager, TaskManager.class, "batchService")).isTerminated()); + Assert.assertTrue(((ExecutorService) field(manager, TaskManager.class, "singleService")).isTerminated()); + Assert.assertFalse(Runtime.getRuntime().removeShutdownHook( + (Thread) field(this.loader, HugeGraphLoader.class, "shutdownHook"))); + OkHttpClient transport = (OkHttpClient) fieldAny(fieldAny(this.loader.context().client(), "client"), "client"); + Assert.assertTrue(transport.dispatcher().executorService().isShutdown()); + Assert.assertEquals(0, transport.connectionPool().connectionCount()); + Set active = Collections.newSetFromMap(new IdentityHashMap<>()); + Assert.assertTrue(containsOriginalWithoutCycle(this.shutdownFailure.get(), active)); + } + + private boolean containsOriginalWithoutCycle(Throwable failure, Set active) { + if (failure == null) { + return false; + } + Assert.assertTrue("cyclic throwable graph", active.add(failure)); + boolean found = failure == this.original; + found |= this.containsOriginalWithoutCycle(failure.getCause(), active); + for (Throwable suppressed : failure.getSuppressed()) { + found |= this.containsOriginalWithoutCycle(suppressed, active); + } + active.remove(failure); + return found; + } + + private static Object fieldAny(Object object, String name) throws Exception { + for (Class type = object.getClass(); type != null; type = type.getSuperclass()) { + try { + return field(object, type, name); + } catch (NoSuchFieldException ignored) { + // Continue through the SDK transport hierarchy. + } + } + throw new NoSuchFieldException(name); + } + + private static Object field(Object object, Class type, String name) throws Exception { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(object); + } + + @Override + public void close() throws Exception { + try { + this.loader.shutdown(); + } finally { + Runtime.getRuntime().removeShutdownHook( + (Thread) field(this.loader, HugeGraphLoader.class, "shutdownHook")); + this.server.stop(0); + } + } +} diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/FileUploadControllerTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/FileUploadControllerTest.java index 269d71d07..f31349c85 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/FileUploadControllerTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/FileUploadControllerTest.java @@ -30,7 +30,9 @@ import java.util.List; import org.apache.commons.io.FileUtils; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.springframework.mock.web.MockMultipartFile; @@ -50,6 +52,9 @@ public class FileUploadControllerTest { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + @Test public void testCheckFileValidAcceptsUppercaseTrimmedFormat() throws Exception { @@ -118,6 +123,11 @@ public void testReserveUploadQuotaUsesGraphRouteForNewMapping() throws Exception { FileUploadController controller = this.controller("csv"); FileMappingService service = Mockito.mock(FileMappingService.class); + Path uploadRoot = this.folder.newFolder("quota-upload").toPath(); + Path uploadPath = uploadRoot.resolve("data.csv"); + FileMappingService pathValidator = this.fileMappingService(uploadRoot); + Mockito.when(service.requirePathUnderUploadRoot(Mockito.anyString())) + .thenAnswer(call -> pathValidator.requirePathUnderUploadRoot(call.getArgument(0))); JobManagerService jobService = Mockito.mock(JobManagerService.class); JobManager job = JobManager.builder().id(1).jobSize(20L).build(); Mockito.when(jobService.get("DEFAULT", "hugegraph", 1)) @@ -131,7 +141,7 @@ public void testReserveUploadQuotaUsesGraphRouteForNewMapping() FileMapping mapping = this.reserveUploadQuota(controller, "DEFAULT", "hugegraph", 1, "data.csv", - "upload/data.csv", 10L); + uploadPath.toString(), 10L); ArgumentCaptor captor = ArgumentCaptor.forClass( FileMapping.class); @@ -140,7 +150,8 @@ public void testReserveUploadQuotaUsesGraphRouteForNewMapping() Assert.assertEquals("DEFAULT", mapping.getGraphSpace()); Assert.assertEquals("hugegraph", mapping.getGraph()); Assert.assertEquals("data.csv", mapping.getName()); - Assert.assertEquals("upload/data.csv", mapping.getPath()); + Assert.assertEquals(uploadPath.toString(), mapping.getPath()); + Assert.assertTrue(Files.isDirectory(uploadPath)); Assert.assertEquals(10L, mapping.getTotalSize()); Assert.assertEquals(FileMappingStatus.UPLOADING, mapping.getFileStatus()); @@ -177,6 +188,11 @@ public void testReserveUploadQuotaUpdatesExistingUploadingMapping() throws Exception { FileUploadController controller = this.controller("csv"); FileMappingService service = Mockito.mock(FileMappingService.class); + Path uploadRoot = this.folder.newFolder("quota-upload").toPath(); + Path uploadPath = uploadRoot.resolve("data.csv"); + FileMappingService pathValidator = this.fileMappingService(uploadRoot); + Mockito.when(service.requirePathUnderUploadRoot(Mockito.anyString())) + .thenAnswer(call -> pathValidator.requirePathUnderUploadRoot(call.getArgument(0))); JobManagerService jobService = Mockito.mock(JobManagerService.class); JobManager job = JobManager.builder().id(1).jobSize(20L).build(); FileMapping existing = new FileMapping("DEFAULT", "hugegraph", @@ -199,10 +215,11 @@ public void testReserveUploadQuotaUpdatesExistingUploadingMapping() FileMapping mapping = this.reserveUploadQuota(controller, "DEFAULT", "hugegraph", 1, "data.csv", - "upload/data.csv", 10L); + uploadPath.toString(), 10L); Assert.assertEquals(existing, mapping); - Assert.assertEquals("upload/data.csv", mapping.getPath()); + Assert.assertEquals(uploadPath.toString(), mapping.getPath()); + Assert.assertTrue(Files.isDirectory(uploadPath)); Assert.assertEquals(10L, mapping.getTotalSize()); Assert.assertEquals(FileMappingStatus.UPLOADING, mapping.getFileStatus()); @@ -414,7 +431,7 @@ private void checkFileValid(FileUploadController controller, JobManager job, String.class, int.class, JobManager.class, - org.springframework.web.multipart.MultipartFile.class, + org.springframework.web.multipart.MultipartFile.class, String.class); method.setAccessible(true); try { diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/JobManagerServiceTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/JobManagerServiceTest.java index c2bb8d8a3..95db52f72 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/JobManagerServiceTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/JobManagerServiceTest.java @@ -21,6 +21,16 @@ import java.lang.reflect.Field; import java.util.Collections; import java.util.Arrays; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.exception.ExternalException; + +import org.apache.hugegraph.loader.HugeGraphLoader; +import org.apache.hugegraph.loader.executor.LoadContext; +import org.apache.hugegraph.loader.executor.LoadOptions; +import org.apache.hugegraph.mapper.load.LoadTaskMapper; import org.junit.Test; import org.mockito.Mockito; @@ -176,8 +186,199 @@ public void testCreateIngestTaskPropagatesLoadBuildFailure() Mockito.verify(taskService).start(connection, mapping, client); } - private JobManagerService service() { - return new JobManagerService(); + @Test + public void testDeleteJobAllowsSucceededTaskAfterRunningSnapshot() throws Exception { + this.deleteCompletedTask(LoadStatus.SUCCEED); + } + + @Test + public void testDeleteJobAllowsFailedLoadAfterRunningSnapshot() throws Exception { + this.deleteCompletedTask(LoadStatus.FAILED); + } + + @Test + public void testDeleteJobAllowsStoppedTaskAfterRunningSnapshot() throws Exception { + this.deleteCompletedTask(LoadStatus.STOPPED); + } + + private void deleteCompletedTask(LoadStatus terminal) throws Exception { + DeleteFixture fixture = this.deleteFixture(terminal, null); + + fixture.jobs.deleteJob(7); + + Assert.assertEquals(terminal, fixture.task.getStatus()); + Assert.assertTrue(fixture.task.isFinished()); + Mockito.verify(fixture.loader).shutdown(); + Mockito.verify(fixture.tasks).deleteById(9); + Mockito.verify(fixture.mapper).deleteById(7); + Assert.assertTrue(fixture.running.isEmpty()); + } + + @Test(timeout = 10000) + public void testDeleteJobPreservesTaskWhenCleanupFailsDuringStopWait() throws Exception { + DeleteFixture fixture = this.deleteFixture(LoadStatus.FAILED, + new IllegalStateException("release failed")); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch stopped = new CountDownLatch(1); + Mockito.when(fixture.loader.load()).thenAnswer(invocation -> { + entered.countDown(); + Assert.assertTrue(stopped.await(5, TimeUnit.SECONDS)); + return false; + }); + Mockito.doAnswer(invocation -> { + stopped.countDown(); + return null; + }).when(fixture.context).stopLoading(); + LoadTask snapshot = LoadTask.builder().id(9).status(LoadStatus.RUNNING).build(); + Mockito.doReturn(Collections.singletonList(snapshot)).when(fixture.service).taskListByJob(7); + Mockito.when(fixture.tasks.updateById(Mockito.any(LoadTask.class))).thenReturn(1); + Thread runner = new Thread(fixture.task, "delete-job-cleanup-race-test"); + runner.start(); + try { + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + + Assert.assertThrows(RuntimeException.class, () -> fixture.jobs.deleteJob(7)); + + Mockito.verify(fixture.tasks, Mockito.never()).deleteById(Mockito.anyInt()); + Mockito.verify(fixture.mapper, Mockito.never()).deleteById(Mockito.anyInt()); + Assert.assertSame(fixture.task, fixture.running.get(9)); + Assert.assertEquals(LoadStatus.FAILED, fixture.task.getStatus()); + } finally { + stopped.countDown(); + runner.join(5000); + Assert.assertFalse(runner.isAlive()); + } + } + + @Test + public void testDeleteJobPreservesTaskWhenCleanupFailedAfterSnapshot() throws Exception { + DeleteFixture fixture = this.deleteFixture(LoadStatus.FAILED, + new IllegalStateException("release failed")); + + Assert.assertThrows(RuntimeException.class, () -> fixture.jobs.deleteJob(7)); + + Mockito.verify(fixture.tasks, Mockito.never()).deleteById(Mockito.anyInt()); + Mockito.verify(fixture.mapper, Mockito.never()).deleteById(Mockito.anyInt()); + Mockito.verify(fixture.mappings, Mockito.never()).listByJob(Mockito.anyInt()); + Assert.assertEquals(LoadStatus.FAILED, fixture.task.getStatus()); + Assert.assertSame(fixture.task, fixture.running.get(9)); + } + + @Test + public void testDeleteJobDoesNotAssumeDatabaseTerminalCleanupSucceeded() throws Exception { + DeleteFixture fixture = this.deleteFixture(LoadStatus.FAILED, null); + fixture.running.clear(); + LoadTask snapshot = LoadTask.builder().id(9).status(LoadStatus.RUNNING).build(); + Mockito.doReturn(Collections.singletonList(snapshot)).when(fixture.service).taskListByJob(7); + Mockito.when(fixture.tasks.selectById(9)) + .thenReturn(LoadTask.builder().id(9).status(LoadStatus.FAILED).build()); + + Assert.assertThrows(ExternalException.class, () -> fixture.jobs.deleteJob(7)); + + Mockito.verify(fixture.tasks, Mockito.never()).deleteById(Mockito.anyInt()); + Mockito.verify(fixture.mapper, Mockito.never()).deleteById(Mockito.anyInt()); + } + + @Test + public void testOrdinaryStopStillRejectsCompletedTasks() throws Exception { + for (LoadStatus terminal : new LoadStatus[]{LoadStatus.SUCCEED, LoadStatus.FAILED, LoadStatus.STOPPED}) { + DeleteFixture fixture = this.deleteFixture(terminal, null); + fixture.task.run(); + + Assert.assertThrows(ExternalException.class, () -> fixture.service.stop(9)); + + Assert.assertEquals(terminal, fixture.task.getStatus()); + Mockito.verify(fixture.tasks, Mockito.never()).updateById(Mockito.any(LoadTask.class)); + } + } + + @Test + public void testDeleteJobDoesNotRemoveMetadataWhenStopPersistenceFails() throws Exception { + DeleteFixture fixture = this.deleteFixture(LoadStatus.PAUSED, null); + fixture.running.clear(); + LoadTask paused = LoadTask.builder().id(9).status(LoadStatus.PAUSED).build(); + Mockito.doReturn(Collections.singletonList(paused)).when(fixture.service).taskListByJob(7); + Mockito.when(fixture.tasks.selectById(9)).thenReturn(paused); + IllegalStateException failure = new IllegalStateException("persist stop failed"); + Mockito.when(fixture.tasks.updateById(Mockito.any(LoadTask.class))).thenThrow(failure); + + Assert.assertSame(failure, Assert.assertThrows(IllegalStateException.class, + () -> fixture.jobs.deleteJob(7))); + + Mockito.verify(fixture.tasks, Mockito.never()).deleteById(Mockito.anyInt()); + Mockito.verify(fixture.mapper, Mockito.never()).deleteById(Mockito.anyInt()); + Mockito.verify(fixture.mappings, Mockito.never()).listByJob(Mockito.anyInt()); + } + + private DeleteFixture deleteFixture(LoadStatus terminal, RuntimeException cleanup) throws Exception { + DeleteFixture fixture = new DeleteFixture(); + fixture.jobs = this.service(); + fixture.service = Mockito.spy(new LoadTaskService()); + org.springframework.transaction.PlatformTransactionManager transactions = + Mockito.mock(org.springframework.transaction.PlatformTransactionManager.class); + Mockito.when(transactions.getTransaction(Mockito.any())) + .thenReturn(new org.springframework.transaction.support.SimpleTransactionStatus()); + this.setField(fixture.service, "transactionManager", transactions); + fixture.mapper = Mockito.mock(JobManagerMapper.class); + fixture.tasks = Mockito.mock(LoadTaskMapper.class); + fixture.mappings = Mockito.mock(FileMappingService.class); + fixture.loader = Mockito.mock(HugeGraphLoader.class); + fixture.context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); + Mockito.when(fixture.loader.context()).thenReturn(fixture.context); + fixture.task = LoadTask.builder().id(9).jobId(7).loader(fixture.loader) + .options(new LoadOptions()).status(LoadStatus.RUNNING).lastDuration(0L).build(); + Mockito.when(fixture.loader.load()).thenAnswer(invocation -> { + if (terminal == LoadStatus.STOPPED) { + fixture.task.setStatus(LoadStatus.STOPPED); + } + return terminal == LoadStatus.SUCCEED; + }); + if (cleanup != null) { + Mockito.doThrow(cleanup).when(fixture.loader).shutdown(); + } + fixture.running = new ConcurrentHashMap<>(); + fixture.running.put(9, fixture.task); + this.setField(fixture.service, "runningTaskContainer", fixture.running); + this.setField(fixture.service, "mapper", fixture.tasks); + this.setField(fixture.jobs, "mapper", fixture.mapper); + this.setField(fixture.jobs, "taskService", fixture.service); + this.setField(fixture.jobs, "fileMappingService", fixture.mappings); + Mockito.when(fixture.mapper.selectById(7)).thenReturn(JobManager.builder().id(7).build()); + Mockito.when(fixture.mapper.deleteById(7)).thenReturn(1); + Mockito.when(fixture.tasks.deleteById(9)).thenReturn(1); + Mockito.when(fixture.mappings.listByJob(7)).thenReturn(Collections.emptyList()); + LoadTask snapshot = LoadTask.builder().id(9).status(LoadStatus.RUNNING).build(); + Mockito.when(fixture.tasks.selectById(9)).thenAnswer(invocation -> + LoadTask.builder().id(9).status(fixture.task.getStatus()).build()); + Mockito.doAnswer(invocation -> { + // The real runner finishes cleanup after the database snapshot, + // before deleteJob reaches the real stop implementation. + fixture.task.run(); + return Collections.singletonList(snapshot); + }).when(fixture.service).taskListByJob(7); + return fixture; + } + + private static class DeleteFixture { + private JobManagerService jobs; + private LoadTaskService service; + private JobManagerMapper mapper; + private LoadTaskMapper tasks; + private FileMappingService mappings; + private HugeGraphLoader loader; + private LoadContext context; + private LoadTask task; + private ConcurrentHashMap running; + } + + private JobManagerService service() throws Exception { + JobManagerService service = new JobManagerService(); + org.springframework.transaction.PlatformTransactionManager transactions = + Mockito.mock(org.springframework.transaction.PlatformTransactionManager.class); + Mockito.when(transactions.getTransaction(Mockito.any())) + .thenReturn(new org.springframework.transaction.support.SimpleTransactionStatus()); + this.setField(service, "transactionManager", transactions); + return service; } private void setField(Object object, String name, Object value) diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCheckpointTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCheckpointTest.java new file mode 100644 index 000000000..9285789ce --- /dev/null +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskCheckpointTest.java @@ -0,0 +1,259 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.unit; + +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; + +import org.apache.hugegraph.entity.GraphConnection; +import org.apache.hugegraph.entity.enums.LoadStatus; +import org.apache.hugegraph.entity.load.FileMapping; +import org.apache.hugegraph.entity.load.LoadTask; +import org.apache.hugegraph.loader.HugeGraphLoader; +import org.apache.hugegraph.loader.executor.LoadOptions; +import org.apache.hugegraph.loader.progress.LoadProgress; +import org.apache.hugegraph.util.JsonUtil; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import com.sun.net.httpserver.HttpServer; + +/** Real CSV/parser/semaphore/insert/cleanup pipeline against a controlled HTTP endpoint. */ +public class LoadTaskCheckpointTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test(timeout = 20000) + public void testPauseAtBatchSemaphoreThenResumeDoesNotSkipUnsubmittedRows() throws Exception { + try (InsertServer server = new InsertServer()) { + server.holdFirstBatch = true; + LoadTask task = this.task(server); + HugeGraphLoader first = task.getLoader(); + Object manager = field(first, "manager"); + Semaphore semaphore = (Semaphore) field(manager, "batchSemaphore"); + ExecutorService runner = Executors.newSingleThreadExecutor(); + try { + Future running = runner.submit(task); + Assert.assertTrue(server.firstBatch.await(5, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!semaphore.hasQueuedThreads() && System.nanoTime() < deadline) { + Thread.sleep(10); + } + Assert.assertTrue("The actual parser must wait on the native batch semaphore", + semaphore.hasQueuedThreads()); + task.setStatus(LoadStatus.PAUSED); + task.context().stopLoading(); + server.releaseBatch.countDown(); + running.get(5, TimeUnit.SECONDS); + Assert.assertEquals(LoadStatus.PAUSED, task.getStatus()); + Assert.assertTrue(first.context().closed()); + Assert.assertTrue("The parsed waiting batch was never inserted", server.rows.size() < 10); + this.resume(task); + Assert.assertEquals(LoadStatus.SUCCEED, task.getStatus()); + Assert.assertEquals("Resume must include every unsubmitted row", 10, server.rows.size()); + this.assertNoCheckpoint(task, first); + Assert.assertTrue(task.context().closed()); + } finally { + server.releaseBatch.countDown(); + runner.shutdownNow(); + Assert.assertTrue(runner.awaitTermination(5, TimeUnit.SECONDS)); + first.shutdown(); + task.getLoader().shutdown(); + } + } + } + + @Test(timeout = 20000) + public void testFailedSingleInsertThenResumeDoesNotSkipFailedRow() throws Exception { + try (InsertServer server = new InsertServer()) { + server.rejectName = "person2"; + LoadTask task = this.task(server); + HugeGraphLoader first = task.getLoader(); + task.run(); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertEquals(9, server.rows.size()); + Assert.assertFalse(server.rows.contains("person2")); + Assert.assertTrue(first.context().closed()); + server.rejectName = null; + this.resume(task); + Assert.assertEquals(LoadStatus.SUCCEED, task.getStatus()); + Assert.assertEquals("The failed row must be read and inserted on resume", 10, server.rows.size()); + Assert.assertTrue(server.rows.contains("person2")); + this.assertNoCheckpoint(task, first); + Assert.assertTrue(task.context().closed()); + } + } + + private LoadTask task(InsertServer server) throws Exception { + Path input = this.folder.newFile("people.csv").toPath(); + StringBuilder csv = new StringBuilder(); + for (int i = 0; i < 10; i++) { + csv.append("person").append(i).append('\n'); + } + Files.writeString(input, csv); + Path mapping = this.folder.newFile("mapping.json").toPath(); + Files.writeString(mapping, JsonUtil.toJson(Map.of("vertices", List.of(Map.of( + "label", "person", "input", Map.of("type", "FILE", "path", input.toString(), + "format", "CSV", "header", List.of("name"), "charset", "UTF-8")))))); + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = mapping.toString(); + options.batchSize = server.holdFirstBatch ? 4 : 1; + options.batchInsertThreads = 1; + options.singleInsertThreads = 1; + options.retryTimes = 0; + GraphConnection connection = new GraphConnection(); + connection.setId(1); + FileMapping file = new FileMapping().setId(1).setJobId(1).setName("people.csv").setTotalLines(10); + return new LoadTask(options, connection, file); + } + + private void resume(LoadTask task) { + // The same native reconnect path used after LoadTaskService joins the old callback. + task.getOptions().incrementalMode = true; + task.reconnect("current-token"); + task.setStatus(LoadStatus.RUNNING); + task.run(); + } + + private void assertNoCheckpoint(LoadTask task, HugeGraphLoader loader) { + Assert.assertFalse("Hubble must not persist parsed offsets as inserted-row checkpoints", + Files.exists(Path.of(LoadProgress.format(task.getOptions(), loader.context().timestamp())))); + } + + private static Object field(Object target, String name) throws Exception { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } + + private static class InsertServer implements AutoCloseable { + + private final HttpServer server; + private final ExecutorService handlers = Executors.newCachedThreadPool(); + private final Set rows = ConcurrentHashMap.newKeySet(); + private final CountDownLatch firstBatch = new CountDownLatch(1); + private final CountDownLatch releaseBatch = new CountDownLatch(1); + private volatile boolean holdFirstBatch; + private volatile String rejectName; + private String mode = "NONE"; + + private InsertServer() throws Exception { + this.server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + this.server.setExecutor(this.handlers); + this.server.createContext("/", exchange -> { + Object response = Map.of(); + int status = 200; + String path = exchange.getRequestURI().getPath(); + if (path.equals("/versions")) { + response = Map.of("versions", Map.of("core", "1.8.0", "api", "0.74")); + } else if (path.endsWith("/mode")) { + if (exchange.getRequestMethod().equals("PUT")) { + this.mode = JsonUtil.fromJson(new String(exchange.getRequestBody().readAllBytes(), + StandardCharsets.UTF_8), String.class); + } else { + response = Map.of("mode", this.mode); + } + } else if (path.endsWith("/schema/propertykeys")) { + response = Map.of("propertykeys", List.of(Map.of("id", 1, "name", "name", "data_type", "TEXT", + "cardinality", "SINGLE", "aggregate_type", "NONE", "properties", List.of()))); + } else if (path.endsWith("/schema/vertexlabels")) { + response = Map.of("vertexlabels", List.of(Map.of("id", 1, "name", "person", + "id_strategy", "PRIMARY_KEY", "primary_keys", List.of("name"), + "properties", List.of("name"), "nullable_keys", List.of(), "enable_label_index", true))); + } else if (path.endsWith("/schema/edgelabels")) { + response = Map.of("edgelabels", List.of()); + } else if (path.endsWith("/vertices/batch") || path.endsWith("/vertices")) { + if (path.endsWith("/batch") && this.holdFirstBatch) { + this.firstBatch.countDown(); + try { + if (!this.releaseBatch.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Batch release deadline expired"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + InputStream input = exchange.getRequestBody(); + if ("gzip".equals(exchange.getRequestHeaders().getFirst("Content-Encoding"))) { + input = new GZIPInputStream(input); + } + String body = new String(input.readAllBytes(), StandardCharsets.UTF_8); + List vertices = path.endsWith("/batch") ? JsonUtil.fromJson(body, List.class) : + Collections.singletonList(JsonUtil.fromJson(body, Map.class)); + List names = new ArrayList<>(); + for (Object value : vertices) { + Map vertex = (Map) value; + names.add((String) ((Map) vertex.get("properties")).get("name")); + } + if (this.rejectName != null && names.contains(this.rejectName)) { + status = 500; + response = Map.of("exception", "class java.lang.IllegalStateException", + "message", "controlled insert failure"); + } else { + this.rows.addAll(names); + response = path.endsWith("/batch") ? names : Map.of("id", names.get(0), "label", "person", + "properties", Map.of("name", names.get(0))); + } + } else { + status = 404; + response = Map.of("message", "Unsupported test endpoint " + path); + } + byte[] body = JsonUtil.toJson(response).getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, body.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(body); + } + }); + this.server.start(); + } + + @Override + public void close() throws Exception { + this.releaseBatch.countDown(); + this.server.stop(0); + this.handlers.shutdownNow(); + Assert.assertTrue(this.handlers.awaitTermination(5, TimeUnit.SECONDS)); + } + } +} diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java index 2e97180aa..54130b8db 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoadTaskServiceTest.java @@ -51,6 +51,7 @@ import org.apache.hugegraph.service.load.LoadTaskService; import org.apache.hugegraph.mapper.load.LoadTaskMapper; import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.NativeCleanupTestSupport; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockito.MockedConstruction; @@ -166,7 +167,7 @@ public void testCancelledRecordedLoadFailureRethrowKeepsRequestedStatus() { new Exception("load failed"), new AssertionError("load failed"), new UndeclaredThrowableException( - new RuntimeException(new IllegalStateException("wrapped")))}) { + new RuntimeException(new IllegalStateException("wrapped")))}) { HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); LoadContext context = Mockito.mock(LoadContext.class, Mockito.RETURNS_DEEP_STUBS); Mockito.when(loader.context()).thenReturn(context); @@ -227,7 +228,8 @@ public void testAfterCommitDispatchErrorClosesLoaderAndPreservesError() throws E Mockito.when(mapper.update(Mockito.isNull(), Mockito.any())).thenReturn(1); this.setField(service, "mapper", mapper); this.setField(service, "taskExecutor", executor); - this.setField(service, "runningTaskContainer", new ConcurrentHashMap<>()); + ConcurrentHashMap running = new ConcurrentHashMap<>(); + this.setField(service, "runningTaskContainer", running); PlatformTransactionManager transactions = Mockito.mock(PlatformTransactionManager.class); Mockito.when(transactions.getTransaction(Mockito.any())).thenReturn(new SimpleTransactionStatus()); this.setField(service, "transactionManager", transactions); @@ -250,6 +252,12 @@ public void testAfterCommitDispatchErrorClosesLoaderAndPreservesError() throws E Assert.assertTrue(task.isFinished()); Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); Assert.assertSame(cleanup, failure.getSuppressed()[0]); + Assert.assertTrue(task.isCleanupFailed()); + Assert.assertSame(task, running.get(9)); + Assert.assertThrows(RuntimeException.class, () -> service.retry(9, "current-token")); + Assert.assertThrows(RuntimeException.class, () -> service.resume(9, "current-token")); + Assert.assertSame(failure, Assert.assertThrows(Error.class, () -> service.remove(9))); + Mockito.verify(mapper, Mockito.never()).deleteById(Mockito.anyInt()); } finally { TransactionSynchronizationManager.clearSynchronization(); } @@ -307,7 +315,12 @@ public void testResumeAndRetryCloseLoaderOnUpdateFailure() throws Exception { } Mockito.verify(loader).shutdown(); Assert.assertTrue(task.isFinished()); - Assert.assertTrue(running.isEmpty()); + Assert.assertTrue(task.isCleanupFailed()); + Assert.assertSame(task, running.get(9)); + Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); + Assert.assertThrows(RuntimeException.class, () -> service.retry(9, "current-token")); + Assert.assertThrows(RuntimeException.class, () -> service.resume(9, "current-token")); + Mockito.verify(task).reconnect("current-token"); Mockito.verify(executor, Mockito.never()).execute(Mockito.any(), Mockito.any()); } } @@ -326,8 +339,8 @@ public void testResumeAndRetryCloseLoaderOnDispatchRejection() throws Exception Mockito.doNothing().when(task).reconnect("current-token"); Mockito.when(mapper.selectById(9)).thenReturn(task); Mockito.when(mapper.updateById(task)).thenReturn(1); - Mockito.doThrow(new RejectedExecutionException("queue full")) - .when(executor).execute(Mockito.eq(task), Mockito.any()); + RejectedExecutionException rejected = new RejectedExecutionException("queue full"); + Mockito.doThrow(rejected).when(executor).execute(Mockito.eq(task), Mockito.any()); ConcurrentHashMap running = new ConcurrentHashMap<>(); this.setField(service, "mapper", mapper); this.setField(service, "taskExecutor", executor); @@ -338,10 +351,14 @@ public void testResumeAndRetryCloseLoaderOnDispatchRejection() throws Exception Mockito.when(mapper.update(Mockito.isNull(), Mockito.any())).thenReturn(1); this.setField(service, "transactionManager", transactions); - LoadTask result = resume ? service.resume(9, "current-token") : - service.retry(9, "current-token"); - - Assert.assertSame(task, result); + Throwable actual = Assert.assertThrows(RejectedExecutionException.class, () -> { + if (resume) { + service.resume(9, "current-token"); + } else { + service.retry(9, "current-token"); + } + }); + Assert.assertSame(rejected, actual); Mockito.verify(loader).shutdown(); Assert.assertTrue(task.isFinished()); Assert.assertEquals(LoadStatus.FAILED, task.getStatus()); @@ -499,6 +516,100 @@ public void testRetryRehydratesLoaderWithCurrentSessionToken() Mockito.verify(executor).execute(Mockito.eq(task), Mockito.any()); } + @Test + public void testRetryAndResumeRejectUnfinishedOrCleanupFailedRuntime() throws Exception { + for (boolean resume : new boolean[]{false, true}) { + for (boolean cleanupFailed : new boolean[]{false, true}) { + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + LoadTaskExecutor executor = Mockito.mock(LoadTaskExecutor.class); + LoadTask persisted = Mockito.mock(LoadTask.class); + Mockito.when(persisted.getStatus()).thenReturn(LoadStatus.FAILED); + Mockito.when(persisted.getOptions()).thenReturn(new LoadOptions()); + Mockito.when(mapper.selectById(9)).thenReturn(persisted); + Mockito.when(mapper.updateById(persisted)).thenReturn(1); + LoadTask retained = LoadTask.builder().id(9).status(LoadStatus.FAILED) + .finished(cleanupFailed).cleanupFailed(cleanupFailed).build(); + ConcurrentHashMap running = new ConcurrentHashMap<>(); + running.put(9, retained); + this.setField(service, "mapper", mapper); + this.setField(service, "taskExecutor", executor); + this.setField(service, "runningTaskContainer", running); + + Assert.assertThrows(RuntimeException.class, () -> { + if (resume) { + service.resume(9, "current-session-token"); + } else { + service.retry(9, "current-session-token"); + } + }); + + Assert.assertSame(retained, running.get(9)); + Mockito.verify(persisted, Mockito.never()).reconnect(Mockito.anyString()); + Mockito.verify(mapper, Mockito.never()).updateById(Mockito.any(LoadTask.class)); + Mockito.verify(executor, Mockito.never()).execute(Mockito.any(), Mockito.any()); + } + } + } + + @Test + public void testNativeResetFailureAndPurePauseStopCloseResources() throws Exception { + for (LoadStatus status : new LoadStatus[]{LoadStatus.PAUSED, LoadStatus.STOPPED}) { + for (boolean rejectReset : new boolean[]{false, true}) { + LoadTask task = LoadTask.builder().id(9).status(LoadStatus.RUNNING).lastDuration(0L).build(); + try (NativeCleanupTestSupport nativeCleanup = new NativeCleanupTestSupport( + task, this.folder.newFile("native-" + status + rejectReset + ".json").toPath(), + rejectReset, status)) { + task.run(); + + Assert.assertEquals(rejectReset ? LoadStatus.FAILED : status, task.getStatus()); + Assert.assertEquals(rejectReset, task.isCleanupFailed()); + if (rejectReset) { + Assert.assertNotSame(nativeCleanup.original, nativeCleanup.shutdownFailure.get()); + } else { + Assert.assertSame(nativeCleanup.original, nativeCleanup.shutdownFailure.get()); + } + Assert.assertEquals(rejectReset ? "LOADING" : "NONE", nativeCleanup.mode()); + nativeCleanup.assertResourcesClosed(); + } + } + } + } + + @Test + public void testPauseAndStopKeepCleanupFailedRuntimeAndLoader() throws Exception { + for (boolean pause : new boolean[]{false, true}) { + LoadTaskService service = new LoadTaskService(); + LoadTaskMapper mapper = Mockito.mock(LoadTaskMapper.class); + HugeGraphLoader loader = Mockito.mock(HugeGraphLoader.class); + LoadContext context = Mockito.mock(LoadContext.class); + Mockito.when(loader.context()).thenReturn(context); + Mockito.when(context.closed()).thenReturn(true); + LoadTask task = LoadTask.builder().id(9).status(LoadStatus.RUNNING).loader(loader).build(); + Mockito.doAnswer(invocation -> { + task.setStatus(LoadStatus.FAILED); + task.setCleanupFailed(true); + task.setFinished(true); + return null; + }).when(context).stopLoading(); + Mockito.when(mapper.updateById(task)).thenReturn(1); + ConcurrentHashMap running = new ConcurrentHashMap<>(); + running.put(9, task); + this.setField(service, "mapper", mapper); + this.setField(service, "runningTaskContainer", running); + + if (pause) { + service.pause(9); + } else { + service.stop(9); + } + + Assert.assertSame(task, running.get(9)); + Assert.assertSame(loader, task.getLoader()); + Assert.assertTrue(task.isCleanupFailed()); + } + } + @Test public void testLoadOptionsUseTokenWithoutCredentialPassword() throws Exception { diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoaderScopeControllerTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoaderScopeControllerTest.java index 9c3901d08..a719e10a9 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoaderScopeControllerTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/LoaderScopeControllerTest.java @@ -15,6 +15,7 @@ * License for the specific language governing permissions and limitations * under the License. */ + package org.apache.hugegraph.unit; import java.util.Collections; @@ -82,6 +83,8 @@ public void testFileClearUsesNestedScope() { JobManagerService jobService = Mockito.mock(JobManagerService.class); LoadTaskService taskService = Mockito.mock(LoadTaskService.class); FileMapping mapping = FileMapping.builder().id(11).build(); + Mockito.when(jobService.get("space-a", "graph-a", 7)) + .thenReturn(JobManager.builder().id(7).graphSpace("space-a").graph("graph-a").build()); Mockito.when(service.listByJob("space-a", "graph-a", 7)) .thenReturn(Collections.singletonList(mapping)); Mockito.when(taskService.taskListByJob(7)) @@ -93,6 +96,7 @@ public void testFileClearUsesNestedScope() { controller.clear("space-a", "graph-a", 7); + Mockito.verify(jobService).get("space-a", "graph-a", 7); Mockito.verify(jobService).deleteMappings( 7, Collections.singletonList(mapping)); Mockito.verify(service, Mockito.never()).deleteDiskFile(mapping); diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java index 07800995c..b745912fd 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -77,6 +77,7 @@ HubbleOptionsTest.class, IngestControllerTest.class, IngestTransactionIntegrationTest.class, + org.apache.hugegraph.service.load.JobAdmissionIntegrationTest.class, LangChainControllerSecurityTest.class, LegacyFacadeRemovalTest.class, MessageSourceHandlerTest.class, @@ -87,6 +88,7 @@ JobManagerServiceTest.class, K8sTokenEndpointSecurityTest.class, LoadTaskServiceTest.class, + LoadTaskCheckpointTest.class, LoaderScopeControllerTest.class, LoginAttemptGuardTest.class, OltpAlgoControllerTest.class, diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java index a9565c0d5..7325e50f6 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/HugeGraphLoader.java @@ -94,6 +94,7 @@ public final class HugeGraphLoader { private final LoadOptions options; private final Thread shutdownHook; private volatile boolean loadStarted; + private final boolean saveProgress; public static class InputTaskItem { @@ -141,13 +142,23 @@ public HugeGraphLoader(String[] args) { } public HugeGraphLoader(LoadOptions options) { - this(options, LoadMapping.of(options.file)); + this(options, true); + } + + /** Embedded callers may close resources without persisting parsed offsets. */ + public HugeGraphLoader(LoadOptions options, boolean saveProgress) { + this(options, LoadMapping.of(options.file), saveProgress); // Set concurrency GlobalExecutorManager.setBatchThreadCount(options.batchInsertThreads); GlobalExecutorManager.setSingleThreadCount(options.singleInsertThreads); } public HugeGraphLoader(LoadOptions options, LoadMapping mapping) { + this(options, mapping, true); + } + + public HugeGraphLoader(LoadOptions options, LoadMapping mapping, boolean saveProgress) { + this.saveProgress = saveProgress; this.context = new LoadContext(options); this.options = options; this.mapping = mapping; @@ -945,7 +956,7 @@ private void drainThenClose() { this.context.unsetLoadingMode(); } catch (Throwable resetFailure) { try { - this.context.close(this.loadStarted); + this.context.close(this.loadStarted && this.saveProgress); } catch (Throwable closeFailure) { if (closeFailure instanceof Error && !(resetFailure instanceof Error)) { suppressCleanupFailure(closeFailure, resetFailure); @@ -955,7 +966,7 @@ private void drainThenClose() { } throw resetFailure; } - this.context.close(this.loadStarted); + this.context.close(this.loadStarted && this.saveProgress); } } finally { if (interrupted) { diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java index 2fb806903..6841945c2 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/LoadContextTest.java @@ -706,6 +706,50 @@ public void testUnstartedShutdownPreservesCheckpoint() throws Exception { } } + @Test + public void testEmbeddedShutdownHookDoesNotReplaceExistingCheckpoint() throws Exception { + try (ModeServer server = new ModeServer()) { + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = this.folder.newFile("embedded.json").getAbsolutePath(); + options.incrementalMode = true; + File checkpoint = new File(LoadProgress.format(options, "2000-01-01")); + Files.createDirectories(checkpoint.toPath().getParent()); + String saved = "{\"vertex_progress\":7,\"edge_progress\":0,\"input_progress\":{}}"; + Files.writeString(checkpoint.toPath(), saved); + HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList()), false); + Assert.assertEquals(7L, loader.context().oldProgress().vertexLoaded()); + set(loader, "loadStarted", true); + loader.context().newProgress().plusVertexLoaded(99); + + ((Thread) field(loader, "shutdownHook")).run(); + + Assert.assertTrue(loader.context().closed()); + Assert.assertEquals(saved, Files.readString(checkpoint.toPath())); + Assert.assertFalse(new File(LoadProgress.format(options, loader.context().timestamp())).exists()); + } + } + + @Test + public void testDefaultLoaderShutdownStillWritesProgress() throws Exception { + try (ModeServer server = new ModeServer()) { + LoadOptions options = new LoadOptions(); + options.host = "127.0.0.1"; + options.port = server.server.getAddress().getPort(); + options.file = this.folder.newFile("standalone.json").getAbsolutePath(); + HugeGraphLoader loader = new HugeGraphLoader(options, new LoadMapping(Collections.emptyList())); + set(loader, "loadStarted", true); + loader.context().newProgress().plusVertexLoaded(7); + + loader.shutdown(); + + File checkpoint = new File(LoadProgress.format(options, loader.context().timestamp())); + Assert.assertTrue(loader.context().closed()); + Assert.assertEquals(7L, LoadProgress.read(checkpoint).vertexLoaded()); + } + } + @Test public void testClosedLoaderUnregistersOwnHook() throws Exception { for (boolean failed : new boolean[]{false, true}) { From 3dbdd8d8b2d8ae70047baed2c339400dc34213ac Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 15 Oct 2026 02:03:25 +0800 Subject: [PATCH 12/14] docs(hubble): clarify deletion recovery boundary - allow metadata deletion after the old runtime has ended - separate Server graph recovery from Hubble metadata deletion - retain current-process cleanup failure responsibility --- hugegraph-hubble/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index 6a4596064..75ffe2c1d 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -97,12 +97,14 @@ inspect and recover the graph mode, then restart Hubble before retrying. Database failures during progress polling or after rejected dispatch require operator recovery; this change does not add automatic per-task persistence repair. A failed final write can leave a task RUNNING or its job LOADING in metadata. -Hubble keeps failed callback/cleanup evidence and refuses replacement/deletion when cleanup cannot be confirmed. -Recover the metadata database first, stop Hubble and other imports, and have an administrator verify that no Loader -still uses the graph before repairing an orphaned graph mode through the Server graph-mode API. Restart Hubble with -the same metadata database and source files. If a persisted task is still RUNNING with no active runner, use the existing -Pause action, then Resume (or Retry for a terminal task) with current authentication. These actions recheck persisted -state; they do not repair a database that is still unavailable. Do not manually delete metadata rows to bypass cleanup. +While a runtime remains in this Hubble process, unresolved callback/cleanup evidence blocks unsafe replacement or deletion. +After an abnormal process exit, the old runtime is gone: a leftover RUNNING task can be deleted directly. This removes +Hubble metadata only; it does not restore Server graph mode or release any leftover Server loading owner. +Server recovery is a separate operation. Restore the metadata database; before starting new imports, stop related imports +and have an administrator inspect and recover the Server graph mode/owner. If keeping the task for another attempt, +restart Hubble with the same metadata and source files, use the existing Pause action for a persisted RUNNING row, +then Resume (or Stop followed by Retry when old checkpoints are unsafe) with current authentication. These actions +do not repair a database that is still unavailable. ## Local development feedback loop From cff6333730dc7f2e50a261ffcfe1e3389b82b7f5 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 15 Oct 2026 05:56:35 +0800 Subject: [PATCH 13/14] fix(hubble): order queued deletion test stops - hold active mock runs until every stop is requested - reuse the existing latch to preserve queued-first ordering - retain cleanup assertions and production behavior --- .../hugegraph/service/load/JobAdmissionIntegrationTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/JobAdmissionIntegrationTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/JobAdmissionIntegrationTest.java index 5188b42e6..5d9e13343 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/JobAdmissionIntegrationTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/service/load/JobAdmissionIntegrationTest.java @@ -1263,6 +1263,9 @@ private NativeRun nativeRun(LoadTask task, CountDownLatch active, CountDownLatch if (active != null) { active.countDown(); await(stop); + if (stopRequests != null) { + await(stopRequests); + } } return true; }).when(loader).load(); From a7d3a0cea241e22c1126cd8cd719d7a9f191ddbe Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 16 Oct 2026 01:25:33 +0800 Subject: [PATCH 14/14] fix(hubble): reuse upload directory reservations - reuse existing uploading directories for later chunks - retain first reservation alias checks and upload quotas - verify real chunks, alias rejection and operation counts --- .../controller/load/FileUploadController.java | 7 +- .../unit/FileUploadControllerTest.java | 100 ++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileUploadController.java b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileUploadController.java index 5af232d34..540ec31dd 100644 --- a/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileUploadController.java +++ b/hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/load/FileUploadController.java @@ -396,7 +396,12 @@ private FileMapping reserveUploadQuota(String graphSpace, String graph, this.checkFileSizeLimit(reservedFileSize, currentJob.getJobSize(), reservedUploadingSize); - boolean createdDirectory = this.reservePhysicalUploadDirectory(jobId, fileName, filePath); + Path directory = this.service.requirePathUnderUploadRoot(filePath).toPath(); + // Later chunks reuse the directory already reserved by this UPLOADING mapping. + boolean needsReservation = mapping == null || !filePath.equals(mapping.getPath()) || + !Files.isDirectory(directory); + boolean createdDirectory = needsReservation && + this.reservePhysicalUploadDirectory(jobId, fileName, filePath); try { if (mapping == null) { mapping = new FileMapping(graphSpace, graph, fileName, filePath); diff --git a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/FileUploadControllerTest.java b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/FileUploadControllerTest.java index f31349c85..e607469d9 100644 --- a/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/FileUploadControllerTest.java +++ b/hugegraph-hubble/hubble-be/src/test/java/org/apache/hugegraph/unit/FileUploadControllerTest.java @@ -155,6 +155,14 @@ public void testReserveUploadQuotaUsesGraphRouteForNewMapping() Assert.assertEquals(10L, mapping.getTotalSize()); Assert.assertEquals(FileMappingStatus.UPLOADING, mapping.getFileStatus()); + + Path failedPath = uploadRoot.resolve("failed.csv"); + Mockito.doThrow(new InternalException("database failure")).when(service).save(Mockito.any()); + Assert.assertThrows(InternalException.class, () -> + this.reserveUploadQuota(controller, "DEFAULT", "hugegraph", 1, + "failed.csv", failedPath.toString(), 10L)); + Assert.assertFalse(Files.exists(failedPath)); + Assert.assertTrue(Files.isDirectory(uploadPath)); } @Test @@ -227,6 +235,98 @@ public void testReserveUploadQuotaUpdatesExistingUploadingMapping() Mockito.verify(service, Mockito.never()).save(Mockito.any()); } + @Test + public void testChunkReservationsReuseDirectoryAndStillEnforceQuota() + throws Exception { + FileUploadController controller = this.controller("csv"); + Path uploadRoot = this.folder.newFolder("chunk-upload").toPath(); + Path uploadPath = uploadRoot.resolve("data.csv"); + FileMappingService service = Mockito.spy(this.fileMappingService(uploadRoot)); + JobManagerService jobService = Mockito.mock(JobManagerService.class); + JobManager job = JobManager.builder().id(1).jobSize(20L).build(); + Mockito.when(jobService.get("DEFAULT", "hugegraph", 1)).thenReturn(job); + List mappings = new ArrayList<>(); + Path otherPath = Files.createDirectory(uploadRoot.resolve("other.csv")); + FileMapping other = new FileMapping("DEFAULT", "hugegraph", "other.csv", otherPath.toString()); + other.setId(3); + other.setFileStatus(FileMappingStatus.UPLOADING); + other.setTotalSize(5L); + mappings.add(other); + Mockito.doReturn(mappings).when(service).listByJob(1); + Mockito.doAnswer(call -> mappings.size() == 1 ? null : mappings.get(1)) + .when(service).get("DEFAULT", "hugegraph", 1, "data.csv"); + Mockito.doAnswer(call -> { + FileMapping mapping = call.getArgument(0); + mapping.setId(2); + mappings.add(mapping); + return null; + }).when(service).save(Mockito.any()); + Mockito.doNothing().when(service).update(Mockito.any()); + this.setField(controller, "service", service); + this.setField(controller, "jobService", jobService); + + for (int i = 0; i < 3; i++) { + this.reserveUploadQuota(controller, "DEFAULT", "hugegraph", 1, + "data.csv", uploadPath.toString(), 10L); + MockMultipartFile chunk = new MockMultipartFile( + "file", "data.csv", "text/csv", ("part" + i).getBytes()); + service.uploadFile(chunk, "data.csv", i, uploadPath.toString()); + } + // One quota scan per chunk and one physical alias scan for the first chunk. + Mockito.verify(service, Mockito.times(4)).listByJob(1); + Mockito.verify(service, Mockito.times(6)) + .requirePathUnderUploadRoot(Mockito.anyString()); + Mockito.verify(service).requirePathUnderUploadRoot(otherPath.toString()); + Path outsidePath = uploadRoot.getParent().resolve("outside.csv"); + Assert.assertThrows(InternalException.class, () -> + this.reserveUploadQuota(controller, "DEFAULT", "hugegraph", 1, + "data.csv", outsidePath.toString(), 10L)); + Assert.assertTrue(service.tryMergePartFiles(uploadPath.toString(), 3)); + Assert.assertEquals("part0part1part2", Files.readString(uploadPath)); + + // Increasing the declared size and a concurrently increased job size are still checked. + Assert.assertThrows(ExternalException.class, () -> + this.reserveUploadQuota(controller, "DEFAULT", "hugegraph", 1, + "data.csv", uploadPath.toString(), 1025L)); + job.setJobSize(2040L); + Assert.assertThrows(ExternalException.class, () -> + this.reserveUploadQuota(controller, "DEFAULT", "hugegraph", 1, + "data.csv", uploadPath.toString(), 10L)); + Assert.assertEquals(10L, mappings.get(1).getTotalSize()); + } + + @Test + public void testFirstReservationRejectsNativeAliases() throws Exception { + Path uploadRoot = this.folder.newFolder("alias-upload").toPath(); + for (String[] names : new String[][]{{"data.csv", "DATA.csv"}, + {"caf\u00e9.csv", "cafe\u0301.csv"}}) { + Path original = Files.createDirectory(uploadRoot.resolve(names[0])); + Path alias = uploadRoot.resolve(names[1]); + // Exercise case/Unicode aliases where the native filesystem actually aliases them. + if (!Files.exists(alias)) { + Files.createSymbolicLink(alias, original); + } + Assert.assertTrue(Files.isSameFile(original, alias)); + FileUploadController controller = this.controller("csv"); + FileMappingService service = Mockito.spy(this.fileMappingService(uploadRoot)); + FileMapping existing = new FileMapping("DEFAULT", "hugegraph", names[0], original.toString()); + existing.setFileStatus(FileMappingStatus.UPLOADING); + Mockito.doReturn(null).when(service).get("DEFAULT", "hugegraph", 1, names[1]); + Mockito.doReturn(Collections.singletonList(existing)).when(service).listByJob(1); + JobManagerService jobService = Mockito.mock(JobManagerService.class); + Mockito.when(jobService.get("DEFAULT", "hugegraph", 1)) + .thenReturn(JobManager.builder().id(1).jobSize(0L).build()); + this.setField(controller, "service", service); + this.setField(controller, "jobService", jobService); + + Assert.assertThrows(ExternalException.class, () -> + this.reserveUploadQuota(controller, "DEFAULT", "hugegraph", 1, + names[1], alias.toString(), 10L)); + Mockito.verify(service, Mockito.never()).save(Mockito.any()); + Assert.assertTrue(Files.isDirectory(original)); + } + } + @Test public void testCheckTotalAndIndexValidRejectsIndexAtTotal() throws Exception {