diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java index 22ed58cb608a7..2092bb9239e61 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -88,7 +89,9 @@ public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { sourceAttributes.put("source.realtime.mode", "log"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("capture.tree", "true"); + sourceAttributes.put("mode.double-living", "true"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -173,7 +176,9 @@ private void testSinkFormat(final String format, final boolean isAsyncLoad) thro final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("capture.tree", "true"); + sourceAttributes.put("mode.double-living", "true"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -213,7 +218,11 @@ private void testSinkFormat(final String format, final boolean isAsyncLoad) thro handleFailure); Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("testPipe").getCode()); + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq("testPipe").setIsTableModel(false)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq("testPipe").setIsTableModel(true)).getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), @@ -260,7 +269,9 @@ public void testWriteBackSink() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("capture.tree", "true"); + sourceAttributes.put("mode.double-living", "true"); sourceAttributes.put("forwarding-pipe-requests", "false"); sourceAttributes.put("source.database-name", "test.*"); sourceAttributes.put("source.table-name", "test.*"); @@ -377,7 +388,9 @@ private void doTest(BiConsumer>, Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("capture.tree", "true"); + sourceAttributes.put("mode.double-living", "true"); sourceAttributes.put("source.database-name", "test.*"); sourceAttributes.put("source.table-name", "test.*"); sourceAttributes.put("user", "root"); @@ -734,7 +747,9 @@ public void testLoadTsFileWithoutVerify() throws Exception { sourceAttributes.put("source.realtime.mode", "batch"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("capture.tree", "true"); + sourceAttributes.put("mode.double-living", "true"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java index 6924f17cedc43..d0ce0bdad46d4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java @@ -20,10 +20,15 @@ package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; +import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; @@ -42,6 +47,9 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.junit.Assert.fail; @@ -245,7 +253,7 @@ public void testReadPipeIsolation() { } @Test - public void testCaptureTreeAndTableIsolation() throws Exception { + public void testCaptureTreeAndTableIgnoredByDialectIsolation() throws Exception { final String treePipeName = "tree_a2b"; final String tablePipeName = "table_a2b"; @@ -272,7 +280,7 @@ public void testCaptureTreeAndTableIsolation() throws Exception { Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session - Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); // 2. Create table pipe by table session try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -292,30 +300,105 @@ public void testCaptureTreeAndTableIsolation() throws Exception { } // Show tree pipe by tree session - Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session - Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); // 3. Drop pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(false)).getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) - .getCode()); + client.dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(true)).getCode()); + } + } + + @Test + public void testSameNameTreeOnlyAndTableOnlyPipeIsolation() throws Exception { + final String pipeName = "same_name_pipe"; + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + pipeName, receiverDataNode.getIpAndPortString())); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + pipeName, receiverDataNode.getIpAndPortString())); + } + + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("stop pipe " + pipeName); + } + + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("drop pipe " + pipeName); + } + + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute("start pipe " + pipeName); + statement.execute("drop pipe " + pipeName); } + + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); } @Test - public void testCaptureCornerCases() { + public void testSameNamePipeWithCaptureAttributesStillIsolated() throws Exception { + final String pipeName = "same_name_conflict_pipe"; final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - // 1. Create tree pipe but capture table data + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + pipeName, receiverDataNode.getIpAndPortString())); + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source ('capture.tree'='true','capture.table'='true')" + + " with sink ('node-urls'='%s')", + pipeName, receiverDataNode.getIpAndPortString())); + } + + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + } + + @Test + public void testCaptureAttributesAreIgnoredByDialect() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe with capture attributes pointing to table data try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( @@ -333,12 +416,12 @@ public void testCaptureCornerCases() { } // Show tree pipe by tree session - Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session - Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); - // 2. Create table pipe but capture tree data + // 2. Create table pipe with capture attributes pointing to tree data try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( @@ -373,14 +456,114 @@ public void testCaptureCornerCases() { + " with sink (" + "'node-urls'='%s')", "p3", receiverDataNode.getIpAndPortString())); - fail(); - } catch (final SQLException ignored) { + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); } // Show tree pipe by tree session - Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); } + + @Test + public void testDirectRpcCreationDialectCompatibility() throws Exception { + final String pipeName = "rpc_same_name_pipe"; + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final Map sinkAttributes = new HashMap<>(); + sinkAttributes.put("sink", "iotdb-thrift-sink"); + sinkAttributes.put("sink.ip", receiverDataNode.getIp()); + sinkAttributes.put("sink.port", String.valueOf(receiverDataNode.getPort())); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.createPipe(new TCreatePipeReq(pipeName, sinkAttributes)).getCode()); + + Assert.assertEquals(1, showPipes(client, false).size()); + Assert.assertEquals(0, showPipes(client, true).size()); + Assert.assertTrue( + showPipes(client, false) + .get(0) + .pipeExtractor + .contains( + SystemConstant.SQL_DIALECT_KEY + "=" + SystemConstant.SQL_DIALECT_TREE_VALUE)); + + final Map tableSourceAttributes = new HashMap<>(); + tableSourceAttributes.put( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq(pipeName, sinkAttributes) + .setExtractorAttributes(tableSourceAttributes)) + .getCode()); + + Assert.assertEquals(1, showPipes(client, false).size()); + Assert.assertEquals(1, showPipes(client, true).size()); + Assert.assertTrue( + showPipes(client, true) + .get(0) + .pipeExtractor + .contains( + SystemConstant.SQL_DIALECT_KEY + "=" + SystemConstant.SQL_DIALECT_TABLE_VALUE)); + + Assert.assertNotEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.createPipe(new TCreatePipeReq(pipeName, sinkAttributes)).getCode()); + } + } + + @Test + public void testLegacyLifecycleRpcPrefersTreePipeThenTablePipe() throws Exception { + final String pipeName = "legacy_same_name_pipe"; + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + pipeName, receiverDataNode.getIpAndPortString())); + } + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + pipeName, receiverDataNode.getIpAndPortString())); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + Assert.assertEquals(1, showPipes(client, false).size()); + Assert.assertEquals(1, showPipes(client, true).size()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe(pipeName).getCode()); + Assert.assertEquals(0, showPipes(client, false).size()); + Assert.assertEquals(1, showPipes(client, true).size()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe(pipeName).getCode()); + Assert.assertEquals(0, showPipes(client, false).size()); + Assert.assertEquals(0, showPipes(client, true).size()); + } + } + + private List showPipes( + final SyncConfigNodeIServiceClient client, final boolean isTableModel) throws Exception { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(isTableModel) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + return showPipeResult; + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java index 8ea9888be882f..afa6b38775e62 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java @@ -80,6 +80,7 @@ public void testLifeCycleWithHistoryEnabled() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -144,6 +145,7 @@ public void testLifeCycleWithHistoryDisabled() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("user", "root"); sourceAttributes.put("source.inclusion", "data.insert"); sourceAttributes.put("source.inclusion.exclusion", ""); @@ -201,6 +203,7 @@ public void testLifeCycleLogMode() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("source.mode", "forced-log"); sourceAttributes.put("user", "root"); @@ -257,6 +260,7 @@ public void testLifeCycleFileMode() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("mode.streaming", "false"); sourceAttributes.put("user", "root"); @@ -311,6 +315,7 @@ public void testLifeCycleHybridMode() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("source.mode", "hybrid"); sourceAttributes.put("user", "root"); @@ -363,6 +368,7 @@ public void testLifeCycleWithClusterRestart() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -426,6 +432,7 @@ public void testReceiverRestartWhenTransferring() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -505,6 +512,7 @@ public void testReceiverAlreadyHaveTimeSeries() throws Exception { final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); @@ -565,6 +573,7 @@ public void testDoubleLiving() throws Exception { // Add this property to avoid to make self cycle. sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("forwarding-pipe-requests", "false"); sourceAttributes.put("user", "root"); @@ -598,6 +607,7 @@ public void testDoubleLiving() throws Exception { // Add this property to avoid to make self cycle. sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("capture.tree", "true"); sourceAttributes.put("forwarding-pipe-requests", "false"); sourceAttributes.put("user", "root"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeMemoryIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeMemoryIT.java index 757ddcda36aa2..49c7a6e62642d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeMemoryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeMemoryIT.java @@ -77,6 +77,7 @@ public void testCreatePipeMemoryManage() { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector", "iotdb-thrift-connector"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java index f233596fc60cd..64e5f81f9498a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipePermissionIT.java @@ -290,6 +290,7 @@ public void testReceiverPermission() throws Exception { sourceAttributes.put("source.inclusion", "all"); sourceAttributes.put("source.capture.tree", "false"); sourceAttributes.put("source.capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "iotdb-thrift-sink"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java index 3fce11bf51f6d..fec72eb370fb0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java @@ -211,6 +211,7 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); @@ -254,6 +255,7 @@ public void testPipeOnBothSenderAndReceiver() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("database-name", "test.*"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); @@ -303,6 +305,7 @@ private void doTest() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("database-name", "test.*"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); @@ -427,6 +430,7 @@ private void doTestUseNodeUrls(String connectorName) throws Exception { connectorAttributes.put("connector.node-urls", nodeUrlsBuilder.toString()); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("database-name", "test.*"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java index 022d71d548775..b7b6e7f66f988 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSourceIT.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.MultiEnvFactory; @@ -110,7 +111,9 @@ public void testMatchingMultipleDatabases() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("extractor.database-name", "test"); extractorAttributes.put("extractor.table-name", "test"); extractorAttributes.put("extractor.pattern", "root.db1"); @@ -160,9 +163,17 @@ public void testMatchingMultipleDatabases() throws Exception { Thread.sleep(10000); Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p1").getCode()); + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq("p1").setIsTableModel(false)).getCode()); Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipe("p2").getCode()); + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq("p1").setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq("p2").setIsTableModel(false)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq("p2").setIsTableModel(true)).getCode()); TestUtils.executeNonQueries( senderEnv, @@ -246,7 +257,9 @@ public void testHistoryAndRealtime() throws Exception { connectorAttributes.put("connector.port", Integer.toString(receiverPort)); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("extractor.database-name", "test"); extractorAttributes.put("extractor.table-name", "test2"); extractorAttributes.put("extractor.inclusion", "data.insert"); @@ -365,7 +378,9 @@ public void testHistoryStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws E final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("extractor.database-name", "test"); extractorAttributes.put("extractor.table-name", "test1"); extractorAttributes.put("extractor.pattern", "root.db.d1"); @@ -461,7 +476,9 @@ public void testExtractorTimeRangeMatch() throws Exception { connectorAttributes.put("connector.port", Integer.toString(receiverPort)); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.capture.tree", "true"); + extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("source.inclusion", "data.insert"); extractorAttributes.put("source.start-time", "2"); extractorAttributes.put("source.end-time", "4"); @@ -565,7 +582,9 @@ public void testSourceStartTimeAndEndTimeWorkingWithOrWithoutPattern() throws Ex final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("source.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("source.capture.tree", "true"); + extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("source.pattern", "root.db.d1"); extractorAttributes.put("source.table-name", "test1"); extractorAttributes.put("source.inclusion", "data.insert"); @@ -669,7 +688,9 @@ public void testHistoryLooseRange() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("source.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("source.capture.tree", "true"); + extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("source.table-name", "test1"); extractorAttributes.put("source.path", "root.db.d1.at1"); extractorAttributes.put("source.inclusion", "data.insert"); @@ -722,7 +743,9 @@ public void testRealtimeLooseRange() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("source.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("source.capture.tree", "true"); + extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("source.table-name", "test1"); extractorAttributes.put("source.path", "root.db.d1.at1"); extractorAttributes.put("source.inclusion", "data.insert"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java index 53243cffaf3ae..a759a714a7faf 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java @@ -67,6 +67,7 @@ public void testPipeSwitchStatus() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); @@ -165,6 +166,7 @@ public void testPipeIllegallySwitchStatus() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); @@ -251,6 +253,7 @@ public void testDropPipeAndCreateAgain() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); @@ -318,6 +321,7 @@ public void testWrongPipeName() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("database-name", "test"); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("inclusion", "data.insert"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java index 6dfede5a8ecec..04e3e38167a56 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -23,6 +23,7 @@ import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; @@ -152,7 +153,11 @@ public void testTsFileDecompositionWithMods() { String.format( "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true', 'capture.table'='true', 'inclusion'='data.insert,data.delete') WITH CONNECTOR('ip'='%s', 'port'='%s', 'username'='root', 'format'='tablet')", receiverEnv.getDataNodeWrapperList().get(0).getIp(), - receiverEnv.getDataNodeWrapperList().get(0).getPort())); + receiverEnv.getDataNodeWrapperList().get(0).getPort()), + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + null, + BaseEnv.TABLE_SQL_DIALECT); TestUtils.assertDataEventuallyOnEnv( receiverEnv, diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java index 57b5846fc4bf3..507286af3830c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java @@ -107,6 +107,7 @@ public void testReceiverNotLoadDeletedTimeseries() throws Exception { // Enable mods transfer extractorAttributes.put("mods", "true"); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector.batch.enable", "false"); @@ -151,6 +152,7 @@ public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); extractorAttributes.put("user", "root"); @@ -235,6 +237,7 @@ public void testReceiverAutoExtendIdColumn() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); extractorAttributes.put("user", "root"); @@ -316,6 +319,7 @@ public void testLoadWhenIncomingIdColumnsArePrefixOfExisting() throws Exception final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); extractorAttributes.put("user", "root"); @@ -389,6 +393,7 @@ public void testLoadAutoCreateWithTableDeletion() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); extractorAttributes.put("user", "root"); @@ -438,6 +443,7 @@ public void testLoadAutoCreateWithoutInsertPermission() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.realtime.mode", "file"); extractorAttributes.put("user", "root"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java index e0945b120a0c0..e7a7cd508afed 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java @@ -86,6 +86,7 @@ public void testTableNamePattern() throws Exception { extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector", "iotdb-thrift-connector"); @@ -153,6 +154,7 @@ public void testTableNamePatternByHistoryTSFile() throws Exception { extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.start-time", "0"); extractorAttributes.put("extractor.end-time", "49"); extractorAttributes.put("user", "root"); @@ -208,6 +210,7 @@ public void testTableNamePatternByRealTime() throws Exception { extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.start-time", "100"); extractorAttributes.put("extractor.end-time", "149"); extractorAttributes.put("user", "root"); @@ -277,6 +280,7 @@ public void testDataBasePattern() throws Exception { extractorAttributes.put("extractor.database-name", "pattern.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector", "iotdb-thrift-connector"); @@ -346,6 +350,7 @@ public void testDataBasePatternByHistoryTSFile() throws Exception { extractorAttributes.put("extractor.database-name", "pattern.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector", "iotdb-thrift-connector"); @@ -398,6 +403,7 @@ public void testDataBasePatternByRealtime() throws Exception { extractorAttributes.put("extractor.database-name", "pattern.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector", "iotdb-thrift-connector"); @@ -463,6 +469,7 @@ public void testIoTDBPatternWithDataBaseAndTable() throws Exception { extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector", "iotdb-thrift-connector"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java index de0f0c460eed8..f0bf52b934bff 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java @@ -111,6 +111,7 @@ public void testDoubleLivingAutoConflict() throws Exception { extractorAttributes.put("source.inclusion", "data.insert"); extractorAttributes.put("source.inclusion.exclusion", ""); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); // Add this property to avoid making self cycle. extractorAttributes.put("source.forwarding-pipe-requests", "false"); extractorAttributes.put("user", "root"); @@ -143,6 +144,7 @@ public void testDoubleLivingAutoConflict() throws Exception { extractorAttributes.put("source.inclusion", "data.insert"); extractorAttributes.put("source.inclusion.exclusion", ""); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); // Add this property to avoid to make self cycle. extractorAttributes.put("source.forwarding-pipe-requests", "false"); extractorAttributes.put("user", "root"); @@ -209,6 +211,7 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { extractorAttributes.put("source.inclusion.exclusion", ""); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("source.forwarding-pipe-requests", "false"); extractorAttributes.put("user", "root"); @@ -241,6 +244,7 @@ public void testDoubleLivingAutoConflictTemplate() throws Exception { extractorAttributes.put("source.inclusion.exclusion", ""); extractorAttributes.put("table-name", "test.*"); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("source.forwarding-pipe-requests", "false"); extractorAttributes.put("user", "root"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java index ff906d8a5afcb..9348e3ccad60f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java @@ -176,6 +176,7 @@ public void testAutoDropInHistoricalTransferWithTimeRange() throws Exception { sourceAttributes.put("mode.snapshot", "true"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("start-time", "0"); sourceAttributes.put("end-time", "49"); sourceAttributes.put("user", "root"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java index 27297c064719a..9fceb9618d672 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java @@ -144,6 +144,7 @@ private void testMachineDowntime(String sink) { sourceAttributes.put("source", "iotdb-source"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("user", "root"); processorAttributes.put("processor", "do-nothing-processor"); @@ -212,6 +213,7 @@ private void testWithAllParameters(final String realtimeMode) throws Exception { sourceAttributes.put("source", "iotdb-source"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("database-name", "test"); sourceAttributes.put("table-name", "test"); sourceAttributes.put("start-time", "0"); @@ -290,6 +292,7 @@ public void testPipeAfterDataRegionLeaderStop() { sourceAttributes.put("source", "iotdb-source"); sourceAttributes.put("database-name", "test"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test"); sourceAttributes.put("start-time", "0"); sourceAttributes.put("end-time", "300"); @@ -350,6 +353,7 @@ public void testPipeAfterDataRegionLeaderStop() { sourceAttributes.put("database-name", "test1"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test1"); sourceAttributes.put("start-time", "0"); sourceAttributes.put("end-time", "300"); @@ -554,6 +558,7 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { sourceAttributes.put("database-name", "test"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test"); sourceAttributes.put("user", "root"); @@ -607,6 +612,7 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { sourceAttributes.put("database-name", "test1"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test1"); sourceAttributes.put("user", "root"); @@ -650,6 +656,7 @@ public void testCreatePipeWhenRegisteringNewDataNode() throws Exception { sourceAttributes.put("database-name", "test1"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test1"); sourceAttributes.put("user", "root"); @@ -721,6 +728,7 @@ public void testRegisteringNewDataNodeWhenTransferringData() throws Exception { sourceAttributes.put("database-name", "test"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test"); sourceAttributes.put("user", "root"); @@ -800,6 +808,7 @@ public void testRegisteringNewDataNodeAfterTransferringData() throws Exception { sourceAttributes.put("database-name", "test"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test"); sourceAttributes.put("user", "root"); @@ -867,6 +876,7 @@ public void testSenderRestartWhenTransferring() throws Exception { sourceAttributes.put("database-name", "test"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test"); sourceAttributes.put("user", "root"); @@ -917,6 +927,7 @@ public void testConcurrentlyCreatePipeOfSameName() throws Exception { sourceAttributes.put("database-name", "test"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test"); sourceAttributes.put("user", "root"); @@ -1026,6 +1037,7 @@ private void testCreatePipesWithSameConnector(final int pipeCount) throws Except sourceAttributes.put("database-name", "test"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test"); sourceAttributes.put("user", "root"); @@ -1108,6 +1120,7 @@ public void testNegativeTimestamp() throws Exception { sourceAttributes.put("source", "iotdb-source"); sourceAttributes.put("database-name", "test"); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("table-name", "test"); sourceAttributes.put("user", "root"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java index cd9a1c193fd2c..26f7dd9db9ca5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java @@ -20,6 +20,7 @@ package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; @@ -60,7 +61,7 @@ public void setUp() { } @Test - public void testDoubleLivingInvalidParameter() throws Exception { + public void testDoubleLivingInvalidForwardingParameter() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -74,8 +75,6 @@ public void testDoubleLivingInvalidParameter() throws Exception { + " with sink (" + "'node-urls'='%s')", "p1", receiverDataNode.getIpAndPortString())); - fail(); - } catch (final SQLException ignored) { } try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -89,8 +88,6 @@ public void testDoubleLivingInvalidParameter() throws Exception { + " with sink (" + "'node-urls'='%s')", "p2", receiverDataNode.getIpAndPortString())); - fail(); - } catch (final SQLException ignored) { } try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -115,7 +112,7 @@ public void testDoubleLivingInvalidParameter() throws Exception { new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)) .pipeInfoList; showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); - Assert.assertEquals(0, showPipeResult.size()); + Assert.assertEquals(2, showPipeResult.size()); } } @@ -338,5 +335,60 @@ public void testDoubleLivingIsolation() throws Exception { .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) .getCode()); } + + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + } + + @Test + public void testDoubleLivingCreatesSameNamePipesWithSeparateDialects() throws Exception { + final String pipeName = "double_living_same_name_pipe"; + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source ('mode.double-living'='true')" + + " with sink ('node-urls'='%s')", + pipeName, receiverDataNode.getIpAndPortString())); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List treePipes = showPipes(client, false); + final List tablePipes = showPipes(client, true); + + Assert.assertEquals(1, treePipes.size()); + Assert.assertEquals(1, tablePipes.size()); + Assert.assertEquals(pipeName, treePipes.get(0).id); + Assert.assertEquals(pipeName, tablePipes.get(0).id); + Assert.assertNotEquals(treePipes.get(0).creationTime, tablePipes.get(0).creationTime); + Assert.assertTrue( + treePipes + .get(0) + .pipeExtractor + .contains( + SystemConstant.SQL_DIALECT_KEY + "=" + SystemConstant.SQL_DIALECT_TREE_VALUE)); + Assert.assertTrue( + tablePipes + .get(0) + .pipeExtractor + .contains( + SystemConstant.SQL_DIALECT_KEY + "=" + SystemConstant.SQL_DIALECT_TABLE_VALUE)); + } + } + + private List showPipes( + final SyncConfigNodeIServiceClient client, final boolean isTableModel) throws Exception { + final List showPipeResult = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(isTableModel) + .setUserName(SessionConfig.DEFAULT_USER)) + .pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + return showPipeResult; } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java index 6c22d41822694..e56e0e9e47b39 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java @@ -231,6 +231,7 @@ private void testTableConfigIdempotent(final List beforeSqlList, final S sourceAttributes.put("source.inclusion.exclusion", ""); sourceAttributes.put("source.forwarding-pipe-requests", "false"); sourceAttributes.put("source.capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("source.capture.tree", "false"); sourceAttributes.put("user", "root"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java index f26aa171ae11b..27dccd5b7fa17 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java @@ -76,6 +76,7 @@ public void testTableSync() throws Exception { extractorAttributes.put("extractor.inclusion", "all"); extractorAttributes.put("extractor.capture.tree", "false"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.database-name", "test"); extractorAttributes.put("extractor.table-name", "t.*[0-9]"); extractorAttributes.put("user", "root"); @@ -212,6 +213,7 @@ public void testNoTree() throws Exception { extractorAttributes.put("extractor.inclusion", "all"); extractorAttributes.put("extractor.capture.tree", "false"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector", "iotdb-thrift-connector"); @@ -330,6 +332,7 @@ public void testAuth() throws Exception { extractorAttributes.put("extractor.inclusion", "all"); extractorAttributes.put("extractor.capture.tree", "false"); extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.database-name", "test"); extractorAttributes.put("extractor.table-name", "t.*[0-9]"); extractorAttributes.put("user", "root"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java index 9d635cd809528..5470b61c368cc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java @@ -88,6 +88,7 @@ private void testInsertNullValueTemplate( connectorAttributes.put("connector.port", Integer.toString(receiverPort)); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("realtime-mode", realtime); extractorAttributes.put("user", "root"); if (withParsing) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java index 547db349e2fa7..1b82df7ea61b1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkCompressionIT.java @@ -160,7 +160,9 @@ private void doTest( extractorAttributes.put("extractor", "iotdb-extractor"); extractorAttributes.put("extractor.realtime.mode", realtimeMode); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("user", "root"); processorAttributes.put("processor", "do-nothing-processor"); @@ -248,7 +250,7 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p1" - + " with extractor ('extractor.pattern'='root.db.d1.s1','table-name'='test1','capture.table'='true','capture.tree'='true')" + + " with extractor ('extractor.pattern'='root.db.d1.s1','table-name'='test1','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + " with connector (" + "'connector.ip'='%s'," + "'connector.port'='%s'," @@ -265,7 +267,7 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p2" - + " with extractor ('extractor.pattern'='root.db.d1.s2','table-name'='test2','capture.table'='true','capture.tree'='true')" + + " with extractor ('extractor.pattern'='root.db.d1.s2','table-name'='test2','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + " with connector (" + "'connector.ip'='%s'," + "'connector.port'='%s'," @@ -283,7 +285,7 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p3" - + " with extractor ('extractor.pattern'='root.db.d1.s3','table-name'='test3','capture.table'='true','capture.tree'='true')" + + " with extractor ('extractor.pattern'='root.db.d1.s3','table-name'='test3','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + " with connector (" + "'connector.ip'='%s'," + "'connector.port'='%s'," @@ -300,7 +302,7 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p4" - + " with extractor ('extractor.pattern'='root.db.d1.s4','table-name'='test4','capture.table'='true','capture.tree'='true')" + + " with extractor ('extractor.pattern'='root.db.d1.s4','table-name'='test4','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + " with connector (" + "'connector.ip'='%s'," + "'connector.port'='%s'," @@ -318,7 +320,7 @@ public void testZstdCompressorLevel() throws Exception { statement.execute( String.format( "create pipe p5" - + " with extractor ('extractor.pattern'='root.db.d1.s5','table-name'='test5','capture.table'='true','capture.tree'='true')" + + " with extractor ('extractor.pattern'='root.db.d1.s5','table-name'='test5','capture.table'='true','capture.tree'='true','mode.double-living'='true')" + " with connector (" + "'connector.ip'='%s'," + "'connector.port'='%s'," diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkParallelIT.java index ae466349d72d5..6749d3f8aff25 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkParallelIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeSinkParallelIT.java @@ -74,6 +74,8 @@ public void testIoTConnectorParallel() throws Exception { final Map connectorAttributes = new HashMap<>(); extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("__system.sql-dialect", "table"); + extractorAttributes.put("mode.double-living", "true"); extractorAttributes.put("user", "root"); connectorAttributes.put("connector", "iotdb-thrift-connector"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java index b1d0a4dda7371..f787c8f924af2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java @@ -311,6 +311,7 @@ public void testOPCUASinkInTableModel() throws Exception { final Map sourceAttributes = new HashMap<>(); final Map sinkAttributes = new HashMap<>(); sourceAttributes.put("capture.table", "true"); + sourceAttributes.put("__system.sql-dialect", "table"); sourceAttributes.put("user", "root"); sinkAttributes.put("sink", "opc-ua-sink"); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/AlterPipePlanV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/AlterPipePlanV2.java index e380218991793..cd8b1f6a6d1ee 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/AlterPipePlanV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/AlterPipePlanV2.java @@ -27,9 +27,11 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Objects; public class AlterPipePlanV2 extends ConfigPhysicalPlan { + private PipeStaticMeta currentPipeStaticMeta; private PipeStaticMeta pipeStaticMeta; private PipeRuntimeMeta pipeRuntimeMeta; @@ -39,10 +41,25 @@ public AlterPipePlanV2() { public AlterPipePlanV2(PipeStaticMeta pipeStaticMeta, PipeRuntimeMeta pipeRuntimeMeta) { super(ConfigPhysicalPlanType.AlterPipeV2); + this.currentPipeStaticMeta = pipeStaticMeta; this.pipeStaticMeta = pipeStaticMeta; this.pipeRuntimeMeta = pipeRuntimeMeta; } + public AlterPipePlanV2( + PipeStaticMeta currentPipeStaticMeta, + PipeStaticMeta pipeStaticMeta, + PipeRuntimeMeta pipeRuntimeMeta) { + super(ConfigPhysicalPlanType.AlterPipeV2); + this.currentPipeStaticMeta = currentPipeStaticMeta; + this.pipeStaticMeta = pipeStaticMeta; + this.pipeRuntimeMeta = pipeRuntimeMeta; + } + + public PipeStaticMeta getCurrentPipeStaticMeta() { + return currentPipeStaticMeta; + } + public PipeStaticMeta getPipeStaticMeta() { return pipeStaticMeta; } @@ -56,11 +73,45 @@ protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); pipeStaticMeta.serialize(stream); pipeRuntimeMeta.serialize(stream); + currentPipeStaticMeta.serialize(stream); } @Override protected void deserializeImpl(ByteBuffer buffer) throws IOException { pipeStaticMeta = PipeStaticMeta.deserialize(buffer); pipeRuntimeMeta = PipeRuntimeMeta.deserialize(buffer); + currentPipeStaticMeta = + buffer.hasRemaining() ? PipeStaticMeta.deserialize(buffer) : pipeStaticMeta; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + AlterPipePlanV2 that = (AlterPipePlanV2) obj; + return currentPipeStaticMeta.equals(that.currentPipeStaticMeta) + && pipeStaticMeta.equals(that.pipeStaticMeta) + && pipeRuntimeMeta.equals(that.pipeRuntimeMeta); + } + + @Override + public int hashCode() { + return Objects.hash(currentPipeStaticMeta, pipeStaticMeta, pipeRuntimeMeta); + } + + @Override + public String toString() { + return "AlterPipePlanV2{" + + "currentPipeStaticMeta='" + + currentPipeStaticMeta + + "', pipeStaticMeta='" + + pipeStaticMeta + + "', pipeRuntimeMeta='" + + pipeRuntimeMeta + + "'}"; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/DropPipePlanV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/DropPipePlanV2.java index 8397e572f8984..43a9614173fad 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/DropPipePlanV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/DropPipePlanV2.java @@ -31,29 +31,41 @@ public class DropPipePlanV2 extends ConfigPhysicalPlan { private String pipeName; + private boolean isTableModel; public DropPipePlanV2() { super(ConfigPhysicalPlanType.DropPipeV2); } public DropPipePlanV2(String pipeName) { + this(pipeName, false); + } + + public DropPipePlanV2(String pipeName, boolean isTableModel) { super(ConfigPhysicalPlanType.DropPipeV2); this.pipeName = pipeName; + this.isTableModel = isTableModel; } public String getPipeName() { return pipeName; } + public boolean isTableModel() { + return isTableModel; + } + @Override protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); BasicStructureSerDeUtil.write(pipeName, stream); + stream.writeBoolean(isTableModel); } @Override protected void deserializeImpl(ByteBuffer buffer) throws IOException { pipeName = BasicStructureSerDeUtil.readString(buffer); + isTableModel = buffer.hasRemaining() && buffer.get() != 0; } @Override @@ -65,16 +77,16 @@ public boolean equals(Object obj) { return false; } DropPipePlanV2 that = (DropPipePlanV2) obj; - return pipeName.equals(that.pipeName); + return isTableModel == that.isTableModel && pipeName.equals(that.pipeName); } @Override public int hashCode() { - return Objects.hash(pipeName); + return Objects.hash(pipeName, isTableModel); } @Override public String toString() { - return "DropPipePlanV2{" + "pipeName='" + pipeName + "'}"; + return "DropPipePlanV2{" + "pipeName='" + pipeName + "', isTableModel=" + isTableModel + "'}"; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusPlanV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusPlanV2.java index 7f417702dd385..25550b2ce7ef9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusPlanV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusPlanV2.java @@ -34,15 +34,21 @@ public class SetPipeStatusPlanV2 extends ConfigPhysicalPlan { private String pipeName; private PipeStatus status; + private boolean isTableModel; public SetPipeStatusPlanV2() { super(ConfigPhysicalPlanType.SetPipeStatusV2); } public SetPipeStatusPlanV2(String pipeName, PipeStatus status) { + this(pipeName, status, false); + } + + public SetPipeStatusPlanV2(String pipeName, PipeStatus status, boolean isTableModel) { super(ConfigPhysicalPlanType.SetPipeStatusV2); this.pipeName = pipeName; this.status = status; + this.isTableModel = isTableModel; } public String getPipeName() { @@ -53,17 +59,23 @@ public PipeStatus getPipeStatus() { return status; } + public boolean isTableModel() { + return isTableModel; + } + @Override protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ReadWriteIOUtils.write(pipeName, stream); ReadWriteIOUtils.write(status.getType(), stream); + ReadWriteIOUtils.write(isTableModel, stream); } @Override protected void deserializeImpl(ByteBuffer buffer) throws IOException { pipeName = ReadWriteIOUtils.readString(buffer); status = PipeStatus.getPipeStatus(ReadWriteIOUtils.readByte(buffer)); + isTableModel = buffer.hasRemaining() && ReadWriteIOUtils.readBool(buffer); } @Override @@ -75,16 +87,25 @@ public boolean equals(Object obj) { return false; } SetPipeStatusPlanV2 that = (SetPipeStatusPlanV2) obj; - return pipeName.equals(that.pipeName) && status.equals(that.status); + return isTableModel == that.isTableModel + && pipeName.equals(that.pipeName) + && status.equals(that.status); } @Override public int hashCode() { - return Objects.hash(pipeName, status); + return Objects.hash(pipeName, status, isTableModel); } @Override public String toString() { - return "SetPipeStatusPlanV2{" + "pipeName='" + pipeName + "', status='" + status + "'}"; + return "SetPipeStatusPlanV2{" + + "pipeName='" + + pipeName + + "', status='" + + status + + "', isTableModel=" + + isTableModel + + "'}"; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusWithStoppedByRuntimeExceptionPlanV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusWithStoppedByRuntimeExceptionPlanV2.java index 35ee503d5c5fc..7c8126ee7ac8d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusWithStoppedByRuntimeExceptionPlanV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusWithStoppedByRuntimeExceptionPlanV2.java @@ -35,6 +35,7 @@ public class SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 extends ConfigPhys private String pipeName; private PipeStatus status; private boolean stoppedByRuntimeException; + private boolean isTableModel; public SetPipeStatusWithStoppedByRuntimeExceptionPlanV2() { super(ConfigPhysicalPlanType.SetPipeStatusWithStoppedByRuntimeExceptionV2); @@ -42,10 +43,19 @@ public SetPipeStatusWithStoppedByRuntimeExceptionPlanV2() { public SetPipeStatusWithStoppedByRuntimeExceptionPlanV2( final String pipeName, final PipeStatus status, final boolean stoppedByRuntimeException) { + this(pipeName, status, stoppedByRuntimeException, false); + } + + public SetPipeStatusWithStoppedByRuntimeExceptionPlanV2( + final String pipeName, + final PipeStatus status, + final boolean stoppedByRuntimeException, + final boolean isTableModel) { super(ConfigPhysicalPlanType.SetPipeStatusWithStoppedByRuntimeExceptionV2); this.pipeName = pipeName; this.status = status; this.stoppedByRuntimeException = stoppedByRuntimeException; + this.isTableModel = isTableModel; } public String getPipeName() { @@ -60,12 +70,17 @@ public boolean isStoppedByRuntimeException() { return stoppedByRuntimeException; } + public boolean isTableModel() { + return isTableModel; + } + @Override protected void serializeImpl(final DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ReadWriteIOUtils.write(pipeName, stream); ReadWriteIOUtils.write(status.getType(), stream); ReadWriteIOUtils.write(stoppedByRuntimeException, stream); + ReadWriteIOUtils.write(isTableModel, stream); } @Override @@ -73,6 +88,7 @@ protected void deserializeImpl(final ByteBuffer buffer) throws IOException { pipeName = ReadWriteIOUtils.readString(buffer); status = PipeStatus.getPipeStatus(ReadWriteIOUtils.readByte(buffer)); stoppedByRuntimeException = ReadWriteIOUtils.readBool(buffer); + isTableModel = buffer.hasRemaining() && ReadWriteIOUtils.readBool(buffer); } @Override @@ -86,13 +102,14 @@ public boolean equals(final Object obj) { final SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 that = (SetPipeStatusWithStoppedByRuntimeExceptionPlanV2) obj; return stoppedByRuntimeException == that.stoppedByRuntimeException + && isTableModel == that.isTableModel && pipeName.equals(that.pipeName) && status.equals(that.status); } @Override public int hashCode() { - return Objects.hash(pipeName, status, stoppedByRuntimeException); + return Objects.hash(pipeName, status, stoppedByRuntimeException, isTableModel); } @Override @@ -104,6 +121,8 @@ public String toString() { + status + "', stoppedByRuntimeException='" + stoppedByRuntimeException + + "', isTableModel='" + + isTableModel + "'}"; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java index 49f10a79e8ac3..6a63ca506903f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java @@ -108,12 +108,25 @@ public PipeTableResp filter( final String pipeName, final boolean isTableModel, final String userName) { - final PipeTableResp resp = filter(whereClause, pipeName); - resp.allPipeMeta.removeIf( - meta -> - !meta.getStaticMeta().visibleUnder(isTableModel) - || !isVisible4User(userName, meta.getStaticMeta())); - return resp; + return new PipeTableResp( + status, + allPipeMeta.stream() + .filter( + meta -> + meta.getStaticMeta().visibleUnder(isTableModel) + && isVisible4User(userName, meta.getStaticMeta())) + .collect(Collectors.toList())) + .filter(whereClause, pipeName); + } + + public PipeTableResp filter( + final Boolean whereClause, final String pipeName, final String userName) { + return new PipeTableResp( + status, + allPipeMeta.stream() + .filter(meta -> isVisible4User(userName, meta.getStaticMeta())) + .collect(Collectors.toList())) + .filter(whereClause, pipeName); } public boolean isVisible4User(final String userName, final PipeStaticMeta meta) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 2beeea6b4af4f..9874467bc7568 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -1585,7 +1585,8 @@ public TSStatus alterPipe(final TAlterPipeReq req) { public TSStatus startConsensusPipe(String pipeName) { try { - StartPipeProcedureV2 procedure = new StartPipeProcedureV2(pipeName); + StartPipeProcedureV2 procedure = + new StartPipeProcedureV2(pipeName, resolveIsTableModel(pipeName)); executor.submitProcedure(procedure); return handleConsensusPipeProcedure(procedure); } catch (Exception e) { @@ -1594,8 +1595,12 @@ public TSStatus startConsensusPipe(String pipeName) { } public TSStatus startPipe(String pipeName) { + return startPipe(pipeName, false); + } + + public TSStatus startPipe(String pipeName, boolean isTableModel) { try { - StartPipeProcedureV2 procedure = new StartPipeProcedureV2(pipeName); + StartPipeProcedureV2 procedure = new StartPipeProcedureV2(pipeName, isTableModel); executor.submitProcedure(procedure); TSStatus status = waitingProcedureFinished(procedure); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -1611,7 +1616,8 @@ public TSStatus startPipe(String pipeName) { public TSStatus stopConsensusPipe(String pipeName) { try { - StopPipeProcedureV2 procedure = new StopPipeProcedureV2(pipeName); + StopPipeProcedureV2 procedure = + new StopPipeProcedureV2(pipeName, resolveIsTableModel(pipeName)); executor.submitProcedure(procedure); return handleConsensusPipeProcedure(procedure); } catch (Exception e) { @@ -1620,8 +1626,12 @@ public TSStatus stopConsensusPipe(String pipeName) { } public TSStatus stopPipe(String pipeName) { + return stopPipe(pipeName, false); + } + + public TSStatus stopPipe(String pipeName, boolean isTableModel) { try { - StopPipeProcedureV2 procedure = new StopPipeProcedureV2(pipeName); + StopPipeProcedureV2 procedure = new StopPipeProcedureV2(pipeName, isTableModel); executor.submitProcedure(procedure); TSStatus status = waitingProcedureFinished(procedure); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -1637,7 +1647,8 @@ public TSStatus stopPipe(String pipeName) { public TSStatus dropConsensusPipe(String pipeName) { try { - DropPipeProcedureV2 procedure = new DropPipeProcedureV2(pipeName); + DropPipeProcedureV2 procedure = + new DropPipeProcedureV2(pipeName, resolveIsTableModel(pipeName)); executor.submitProcedure(procedure); return handleConsensusPipeProcedure(procedure); } catch (Exception e) { @@ -1651,7 +1662,8 @@ public TSStatus dropConsensusPipe(String pipeName) { */ public void dropConsensusPipeAsync(String pipeName) { try { - DropPipeProcedureV2 procedure = new DropPipeProcedureV2(pipeName); + DropPipeProcedureV2 procedure = + new DropPipeProcedureV2(pipeName, resolveIsTableModel(pipeName)); executor.submitProcedure(procedure); LOGGER.info(ManagerMessages.SUBMITTED_ASYNC_CONSENSUS_PIPE_DROP, pipeName); } catch (Exception e) { @@ -1661,8 +1673,12 @@ public void dropConsensusPipeAsync(String pipeName) { } public TSStatus dropPipe(String pipeName) { + return dropPipe(pipeName, false); + } + + public TSStatus dropPipe(String pipeName, boolean isTableModel) { try { - final DropPipeProcedureV2 procedure = new DropPipeProcedureV2(pipeName); + final DropPipeProcedureV2 procedure = new DropPipeProcedureV2(pipeName, isTableModel); executor.submitProcedure(procedure); final TSStatus status = waitingProcedureFinished(procedure, PROCEDURE_WAIT_TIME_OUT << 1); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -1676,6 +1692,10 @@ public TSStatus dropPipe(String pipeName) { } } + private boolean resolveIsTableModel(String pipeName) { + return configManager.getPipeManager().getPipeTaskCoordinator().resolveIsTableModel(pipeName); + } + private TSStatus handleConsensusPipeProcedure(Procedure procedure) { TSStatus status = waitingProcedureFinished(procedure); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java index 58fd18cf854f8..033e8fbcf1cba 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/task/PipeConfigNodeTaskAgent.java @@ -60,6 +60,20 @@ public class PipeConfigNodeTaskAgent extends PipeTaskAgent { private static final Logger LOGGER = LoggerFactory.getLogger(PipeConfigNodeTaskAgent.class); + public static boolean trySetPushPipeMetaRespExceptionMessageCreationTime( + final TPushPipeMetaRespExceptionMessage exceptionMessage, final long creationTime) { + try { + exceptionMessage + .getClass() + .getMethod("setCreationTime", long.class) + .invoke(exceptionMessage, creationTime); + return true; + } catch (final Exception ignored) { + // The field is absent when compiling against an old thrift-datanode artifact. + return false; + } + } + @Override protected boolean isShutdown() { return PipeConfigNodeAgent.runtime().isShutdown(); @@ -108,7 +122,7 @@ protected void createPipeTask( } pipeMetaKeeper - .getPipeMeta(pipeStaticMeta.getPipeName()) + .getPipeMeta(pipeStaticMeta) .getRuntimeMeta() .getConsensusGroupId2TaskMetaMap() .put(consensusGroupId, pipeTaskMeta); @@ -122,10 +136,14 @@ protected TPushPipeMetaRespExceptionMessage handleSinglePipeMetaChangesInternal( ? super.handleSinglePipeMetaChangesInternal(pipeMetaFromCoordinator.deepCopy4TaskAgent()) : null; } catch (final Exception e) { - return new TPushPipeMetaRespExceptionMessage( - pipeMetaFromCoordinator.getStaticMeta().getPipeName(), - e.getMessage(), - System.currentTimeMillis()); + final TPushPipeMetaRespExceptionMessage exceptionMessage = + new TPushPipeMetaRespExceptionMessage( + pipeMetaFromCoordinator.getStaticMeta().getPipeName(), + e.getMessage(), + System.currentTimeMillis()); + trySetPushPipeMetaRespExceptionMessageCreationTime( + exceptionMessage, pipeMetaFromCoordinator.getStaticMeta().getCreationTime()); + return exceptionMessage; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java index 97c6795daf4db..31f4119caaaef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java @@ -31,7 +31,6 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTemporaryMetaInCoordinator; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; -import org.apache.iotdb.confignode.consensus.response.pipe.task.PipeTableResp; import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.manager.ConfigManager; import org.apache.iotdb.confignode.manager.pipe.resource.PipeConfigNodeResourceManager; @@ -177,7 +176,7 @@ private void parseHeartbeatAndSaveMetaChangeLocally( temporaryMeta.getGlobalRemainingEvents(), temporaryMeta.getGlobalRemainingTime(), staticMeta); - pipeTaskInfo.get().removePipeMeta(staticMeta.getPipeName()); + pipeTaskInfo.get().removePipeMeta(staticMeta); PipeLogger.log( LOGGER::info, ManagerMessages.DETECTED_COMPLETION_OF_PIPE_STATIC_META_REMOVE_IT, @@ -279,36 +278,44 @@ private void parseHeartbeatAndSaveMetaChangeLocally( } if (exception instanceof PipeRuntimeSinkCriticalException) { - ((PipeTableResp) pipeTaskInfo.get().showPipes()) - .filter(true, pipeName).getAllPipeMeta().stream() - .filter(pipeMeta -> !pipeMeta.getStaticMeta().getPipeName().equals(pipeName)) - .map(PipeMeta::getRuntimeMeta) - .filter( - runtimeMeta -> !runtimeMeta.getStatus().get().equals(PipeStatus.STOPPED)) - .forEach( - runtimeMeta -> { - // Record the connector exception for each pipe affected - Map exceptionMap = - runtimeMeta.getNodeId2PipeRuntimeExceptionMap(); - if (!exceptionMap.containsKey(nodeId) - || exceptionMap.get(nodeId).getTimeStamp() - < exception.getTimeStamp()) { - exceptionMap.put(nodeId, exception); - } - runtimeMeta.getStatus().set(PipeStatus.STOPPED); - runtimeMeta.setIsStoppedByRuntimeException(true); - - needWriteConsensusOnConfigNodes.set(true); - needPushPipeMetaToDataNodes.set(false); - - PipeLogger.log( - LOGGER::warn, - exception, - ManagerMessages - .DETECT_PIPERUNTIMESINKCRITICALEXCEPTION_FROM_AGENT_STOP_PIPE, - exception, - pipeName); - }); + pipeTaskInfo + .get() + .getPipeMetaList() + .forEach( + pipeMeta -> { + final PipeStaticMeta affectedStaticMeta = pipeMeta.getStaticMeta(); + if (!affectedStaticMeta + .getSinkParameters() + .equals(pipeMetaFromCoordinator.getStaticMeta().getSinkParameters()) + || affectedStaticMeta.equals(pipeMetaFromCoordinator.getStaticMeta())) { + return; + } + + final PipeRuntimeMeta runtimeMeta = pipeMeta.getRuntimeMeta(); + if (!runtimeMeta.getStatus().get().equals(PipeStatus.STOPPED)) { + // Record the connector exception for each pipe affected + Map exceptionMap = + runtimeMeta.getNodeId2PipeRuntimeExceptionMap(); + if (!exceptionMap.containsKey(nodeId) + || exceptionMap.get(nodeId).getTimeStamp() + < exception.getTimeStamp()) { + exceptionMap.put(nodeId, exception); + } + runtimeMeta.getStatus().set(PipeStatus.STOPPED); + runtimeMeta.setIsStoppedByRuntimeException(true); + + needWriteConsensusOnConfigNodes.set(true); + needPushPipeMetaToDataNodes.set(false); + + PipeLogger.log( + LOGGER::warn, + exception, + ManagerMessages + .DETECT_PIPERUNTIMESINKCRITICALEXCEPTION_FROM_AGENT_STOP_PIPE, + exception, + pipeName); + } + }); } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index 490c3f0389237..aa3a9ab87eacd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -22,6 +22,8 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.confignode.consensus.request.read.pipe.task.ShowPipePlanV2; import org.apache.iotdb.confignode.consensus.response.pipe.task.PipeTableResp; @@ -38,6 +40,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -45,7 +48,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -104,6 +109,8 @@ public TSStatus createPipe(TCreatePipeReq req) { final TSStatus status; if (req.getPipeName().startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().createConsensusPipe(req); + } else if (isDoubleLivingPipe(req)) { + status = createDoubleLivingPipe(req); } else { status = configManager.getProcedureManager().createPipe(req); } @@ -113,12 +120,89 @@ public TSStatus createPipe(TCreatePipeReq req) { return status; } + private boolean isDoubleLivingPipe(final TCreatePipeReq req) { + return new PipeParameters(req.getExtractorAttributes()) + .getBooleanOrDefault( + Arrays.asList( + PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + } + + private TSStatus createDoubleLivingPipe(final TCreatePipeReq req) { + final PipeParameters sourceParameters = new PipeParameters(req.getExtractorAttributes()); + final String currentDialect = + sourceParameters.getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); + final String firstDialect = + SystemConstant.SQL_DIALECT_TREE_VALUE.equals(currentDialect) + ? SystemConstant.SQL_DIALECT_TREE_VALUE + : SystemConstant.SQL_DIALECT_TABLE_VALUE; + final String secondDialect = + SystemConstant.SQL_DIALECT_TREE_VALUE.equals(firstDialect) + ? SystemConstant.SQL_DIALECT_TABLE_VALUE + : SystemConstant.SQL_DIALECT_TREE_VALUE; + + final TCreatePipeReq firstReq = + cloneCreatePipeRequestWithDialect(req, sourceParameters, firstDialect); + final TCreatePipeReq secondReq = + cloneCreatePipeRequestWithDialect(req, sourceParameters, secondDialect); + try { + pipeTaskInfo.checkBeforeCreatePipe(firstReq); + pipeTaskInfo.checkBeforeCreatePipe(secondReq); + } catch (final Exception e) { + return RpcUtils.getStatus(TSStatusCode.PIPE_ERROR, e.getMessage()); + } + + final TSStatus firstStatus = configManager.getProcedureManager().createPipe(firstReq); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != firstStatus.getCode()) { + return firstStatus; + } + final TSStatus secondStatus = configManager.getProcedureManager().createPipe(secondReq); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == secondStatus.getCode()) { + return secondStatus; + } + if (!firstReq.isSetIfNotExistsCondition() || !firstReq.isIfNotExistsCondition()) { + configManager + .getProcedureManager() + .dropPipe( + firstReq.getPipeName(), SystemConstant.SQL_DIALECT_TABLE_VALUE.equals(firstDialect)); + } + return secondStatus; + } + + private TCreatePipeReq cloneCreatePipeRequestWithDialect( + final TCreatePipeReq req, final PipeParameters sourceParameters, final String sqlDialect) { + final Map sourceAttributes = new HashMap<>(sourceParameters.getAttribute()); + sourceAttributes.put(SystemConstant.SQL_DIALECT_KEY, sqlDialect); + + final TCreatePipeReq clonedReq = + new TCreatePipeReq() + .setPipeName(req.getPipeName()) + .setExtractorAttributes(sourceAttributes) + .setProcessorAttributes(cloneAttributes(req.getProcessorAttributes())) + .setConnectorAttributes(cloneAttributes(req.getConnectorAttributes())); + if (req.isSetIfNotExistsCondition()) { + clonedReq.setIfNotExistsCondition(req.isIfNotExistsCondition()); + } + if (req.isSetNeedManuallyStart()) { + clonedReq.setNeedManuallyStart(req.isNeedManuallyStart()); + } + return clonedReq; + } + + private Map cloneAttributes(final Map attributes) { + return new HashMap<>(attributes == null ? Collections.emptyMap() : attributes); + } + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus alterPipe(TAlterPipeReq req) { final String pipeName = req.getPipeName(); final boolean isSetIfExistsCondition = req.isSetIfExistsCondition() && req.isIfExistsCondition(); - if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + final boolean isTableModel = + resolveIsTableModel(pipeName, req.isSetIsTableModel(), req.isTableModel); + if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { return isSetIfExistsCondition ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) : RpcUtils.getStatus( @@ -126,6 +210,7 @@ public TSStatus alterPipe(TAlterPipeReq req) { String.format( "Failed to alter pipe %s. Failures: %s does not exist.", pipeName, pipeName)); } + req.setIsTableModel(isTableModel); final TSStatus status = configManager.getProcedureManager().alterPipe(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn(ManagerMessages.FAILED_TO_ALTER_PIPE_RESULT_STATUS, req.getPipeName(), status); @@ -135,11 +220,15 @@ public TSStatus alterPipe(TAlterPipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ private TSStatus startPipe(String pipeName) { + return startPipe(pipeName, false); + } + + private TSStatus startPipe(String pipeName, boolean isTableModel) { final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().startConsensusPipe(pipeName); } else { - status = configManager.getProcedureManager().startPipe(pipeName); + status = configManager.getProcedureManager().startPipe(pipeName, isTableModel); } if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn(ManagerMessages.FAILED_TO_START_PIPE_RESULT_STATUS, pipeName, status); @@ -150,22 +239,28 @@ private TSStatus startPipe(String pipeName) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus startPipe(TStartPipeReq req) { final String pipeName = req.getPipeName(); - if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + final boolean isTableModel = + resolveIsTableModel(pipeName, req.isSetIsTableModel(), req.isTableModel); + if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { return RpcUtils.getStatus( TSStatusCode.PIPE_NOT_EXIST_ERROR, String.format( "Failed to start pipe %s. Failures: %s does not exist.", pipeName, pipeName)); } - return startPipe(pipeName); + return startPipe(pipeName, isTableModel); } /** Caller should ensure that the method is called in the lock {@link #lock()}. */ private TSStatus stopPipe(String pipeName) { + return stopPipe(pipeName, false); + } + + private TSStatus stopPipe(String pipeName, boolean isTableModel) { final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().stopConsensusPipe(pipeName); } else { - status = configManager.getProcedureManager().stopPipe(pipeName); + status = configManager.getProcedureManager().stopPipe(pipeName, isTableModel); } if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn(ManagerMessages.FAILED_TO_STOP_PIPE_RESULT_STATUS, pipeName, status); @@ -176,13 +271,15 @@ private TSStatus stopPipe(String pipeName) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus stopPipe(TStopPipeReq req) { final String pipeName = req.getPipeName(); - if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + final boolean isTableModel = + resolveIsTableModel(pipeName, req.isSetIsTableModel(), req.isTableModel); + if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { return RpcUtils.getStatus( TSStatusCode.PIPE_NOT_EXIST_ERROR, String.format( "Failed to stop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); } - return stopPipe(pipeName); + return stopPipe(pipeName, isTableModel); } /** Caller should ensure that the method is called in the lock {@link #lock()}. */ @@ -190,7 +287,9 @@ public TSStatus dropPipe(TDropPipeReq req) { final String pipeName = req.getPipeName(); final boolean isSetIfExistsCondition = req.isSetIfExistsCondition() && req.isIfExistsCondition(); - if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + final boolean isTableModel = + resolveIsTableModel(pipeName, req.isSetIsTableModel(), req.isTableModel); + if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { return isSetIfExistsCondition ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) : RpcUtils.getStatus( @@ -202,7 +301,7 @@ public TSStatus dropPipe(TDropPipeReq req) { if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().dropConsensusPipe(pipeName); } else { - status = configManager.getProcedureManager().dropPipe(pipeName); + status = configManager.getProcedureManager().dropPipe(pipeName, isTableModel); } if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn(ManagerMessages.FAILED_TO_DROP_PIPE_RESULT_STATUS, pipeName, status); @@ -210,10 +309,25 @@ public TSStatus dropPipe(TDropPipeReq req) { return status; } + private boolean resolveIsTableModel( + final String pipeName, final boolean isSetIsTableModel, final boolean isTableModel) { + return isSetIsTableModel + ? isTableModel + : !pipeTaskInfo.isPipeExisted(pipeName, false) + && pipeTaskInfo.isPipeExisted(pipeName, true); + } + + public boolean resolveIsTableModel(final String pipeName) { + return resolveIsTableModel(pipeName, false, false); + } + public TShowPipeResp showPipes(final TShowPipeReq req) { try { - return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2())) - .filter(req.whereClause, req.pipeName, req.isTableModel, req.userName) + final PipeTableResp pipeTableResp = + (PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2()); + return (req.isSetIsTableModel() + ? pipeTableResp.filter(req.whereClause, req.pipeName, req.isTableModel, req.userName) + : pipeTableResp.filter(req.whereClause, req.pipeName, req.userName)) .convertToTShowPipeResp(); } catch (final ConsensusException e) { PipeLogger.log( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java index bef26386afd7c..c20d3215d1a40 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.confignode.consensus.request.write.pipe.runtime.PipeHandleLeaderChangePlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.runtime.PipeHandleMetaChangePlan; @@ -82,15 +83,14 @@ public PipeTaskInfo getPipeTaskInfo() { public TSStatus createPipe(final CreatePipePlanV2 plan) { try { final Optional pipeMetaBeforeCreation = - Optional.ofNullable( - pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeStaticMeta().getPipeName())); + Optional.ofNullable(pipeTaskInfo.getPipeMetaByPipeStaticMeta(plan.getPipeStaticMeta())); pipeTaskInfo.createPipe(plan); final TPushPipeMetaRespExceptionMessage message = PipeConfigNodeAgent.task() .handleSinglePipeMetaChanges( - pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeStaticMeta().getPipeName())); + pipeTaskInfo.getPipeMetaByPipeStaticMeta(plan.getPipeStaticMeta())); if (message == null) { pipeMetaBeforeCreation.orElseGet( () -> { @@ -122,7 +122,8 @@ public TSStatus setPipeStatus(final SetPipeStatusPlanV2 plan) { pipeTaskInfo.setPipeStatus(plan); PipeConfigNodeAgent.task() - .handleSinglePipeMetaChanges(pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeName())); + .handleSinglePipeMetaChanges( + pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeName(), plan.isTableModel())); PipeTemporaryMetaInCoordinatorMetrics.getInstance() .handleTemporaryMetaChanges(pipeTaskInfo.getPipeMetaList()); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); @@ -139,7 +140,8 @@ public TSStatus setPipeStatusWithStoppedByRuntimeException( pipeTaskInfo.setPipeStatusWithStoppedByRuntimeException(plan); PipeConfigNodeAgent.task() - .handleSinglePipeMetaChanges(pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeName())); + .handleSinglePipeMetaChanges( + pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeName(), plan.isTableModel())); PipeTemporaryMetaInCoordinatorMetrics.getInstance() .handleTemporaryMetaChanges(pipeTaskInfo.getPipeMetaList()); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); @@ -156,12 +158,19 @@ public TSStatus setPipeStatusWithStoppedByRuntimeException( public TSStatus dropPipe(final DropPipePlanV2 plan) { try { final Optional pipeMetaBeforeDrop = - Optional.ofNullable(pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeName())); + Optional.ofNullable( + pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeName(), plan.isTableModel())); pipeTaskInfo.dropPipe(plan); final TPushPipeMetaRespExceptionMessage message = - PipeConfigNodeAgent.task().handleDropPipe(plan.getPipeName()); + pipeMetaBeforeDrop + .map( + meta -> { + meta.getRuntimeMeta().getStatus().set(PipeStatus.DROPPED); + return PipeConfigNodeAgent.task().handleSinglePipeMetaChanges(meta); + }) + .orElse(null); if (message == null) { pipeMetaBeforeDrop.ifPresent( meta -> { @@ -191,14 +200,14 @@ public TSStatus alterPipe(final AlterPipePlanV2 plan) { try { final Optional pipeMetaBeforeAlter = Optional.ofNullable( - pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeStaticMeta().getPipeName())); + pipeTaskInfo.getPipeMetaByPipeStaticMeta(plan.getCurrentPipeStaticMeta())); pipeTaskInfo.alterPipe(plan); final TPushPipeMetaRespExceptionMessage message = PipeConfigNodeAgent.task() .handleSinglePipeMetaChanges( - pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeStaticMeta().getPipeName())); + pipeTaskInfo.getPipeMetaByPipeStaticMeta(plan.getPipeStaticMeta())); if (message == null) { PipeConfigNodeAgent.runtime() .increaseListenerReference(plan.getPipeStaticMeta().getSourceParameters()); @@ -273,7 +282,7 @@ public TSStatus handleMetaChanges(final PipeHandleMetaChangePlan plan) { final List pipeMetaListFromCoordinator = new ArrayList<>(); for (final PipeMeta pipeMeta : plan.getPipeMetaList()) { pipeMetaListFromCoordinator.add( - pipeTaskInfo.getPipeMetaByPipeName(pipeMeta.getStaticMeta().getPipeName())); + pipeTaskInfo.getPipeMetaByPipeStaticMeta(pipeMeta.getStaticMeta())); } PipeConfigNodeAgent.task().handlePipeMetaChanges(pipeMetaListFromCoordinator); PipeTemporaryMetaInCoordinatorMetrics.getInstance() diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 247f803152bd9..d37441704465b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -40,6 +40,7 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; @@ -59,6 +60,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp; +import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaRespExceptionMessage; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; @@ -78,6 +80,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; @@ -93,6 +96,20 @@ public class PipeTaskInfo implements SnapshotProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(PipeTaskInfo.class); private static final String SNAPSHOT_FILE_NAME = "pipe_task_info.bin"; + private static OptionalLong tryGetPushPipeMetaRespExceptionMessageCreationTime( + final TPushPipeMetaRespExceptionMessage exceptionMessage) { + try { + if (!Boolean.TRUE.equals( + exceptionMessage.getClass().getMethod("isSetCreationTime").invoke(exceptionMessage))) { + return OptionalLong.empty(); + } + return OptionalLong.of( + (long) exceptionMessage.getClass().getMethod("getCreationTime").invoke(exceptionMessage)); + } catch (final Exception ignored) { + return OptionalLong.empty(); + } + } + private final PipeMetaKeeper pipeMetaKeeper; // Pure in-memory object, not involved in snapshot serialization and deserialization. @@ -184,7 +201,14 @@ public boolean checkBeforeCreatePipe(final TCreatePipeReq createPipeRequest) private boolean checkBeforeCreatePipeInternal(final TCreatePipeReq createPipeRequest) throws PipeException { - if (!isPipeExisted(createPipeRequest.getPipeName())) { + final PipeStaticMeta pipeStaticMeta = + new PipeStaticMeta( + createPipeRequest.getPipeName(), + System.currentTimeMillis(), + createPipeRequest.getExtractorAttributes(), + createPipeRequest.getProcessorAttributes(), + createPipeRequest.getConnectorAttributes()); + if (!pipeMetaKeeper.containsPipeMetaOverlapped(pipeStaticMeta)) { return true; } @@ -213,7 +237,7 @@ public void checkAndUpdateRequestBeforeAlterPipe(final TAlterPipeReq alterPipeRe private void checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq alterPipeRequest) throws PipeException { - if (!isPipeExisted(alterPipeRequest.getPipeName())) { + if (!isPipeExisted(alterPipeRequest.getPipeName(), alterPipeRequest.isTableModel)) { final String exceptionMessage = String.format( "Failed to alter pipe %s, %s", alterPipeRequest.getPipeName(), PIPE_NOT_EXIST_MSG); @@ -222,7 +246,8 @@ private void checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq al } final PipeStaticMeta pipeStaticMetaFromCoordinator = - getPipeMetaByPipeName(alterPipeRequest.getPipeName()).getStaticMeta(); + getPipeMetaByPipeName(alterPipeRequest.getPipeName(), alterPipeRequest.isTableModel) + .getStaticMeta(); // deep copy current pipe static meta final PipeStaticMeta copiedPipeStaticMetaFromCoordinator = new PipeStaticMeta( @@ -249,6 +274,7 @@ private void checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq al .getAttribute()); } } + keepExtractorDialectConsistentWithTargetModel(alterPipeRequest); if (!alterPipeRequest.isReplaceAllProcessorAttributes) { // modify mode if (alterPipeRequest.getProcessorAttributes().isEmpty()) { @@ -277,6 +303,33 @@ private void checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq al .getAttribute()); } } + + final PipeStaticMeta updatedPipeStaticMeta = + new PipeStaticMeta( + alterPipeRequest.getPipeName(), + System.currentTimeMillis(), + alterPipeRequest.getExtractorAttributes(), + alterPipeRequest.getProcessorAttributes(), + alterPipeRequest.getConnectorAttributes()); + if (pipeMetaKeeper.containsPipeMetaOverlapped( + updatedPipeStaticMeta, pipeStaticMetaFromCoordinator)) { + final String exceptionMessage = + String.format( + "Failed to alter pipe %s, %s", + alterPipeRequest.getPipeName(), PIPE_ALREADY_EXIST_MSG); + LOGGER.info(exceptionMessage); + throw new PipeException(exceptionMessage); + } + } + + private void keepExtractorDialectConsistentWithTargetModel(final TAlterPipeReq alterPipeRequest) { + alterPipeRequest + .getExtractorAttributes() + .put( + SystemConstant.SQL_DIALECT_KEY, + alterPipeRequest.isTableModel + ? SystemConstant.SQL_DIALECT_TABLE_VALUE + : SystemConstant.SQL_DIALECT_TREE_VALUE); } public void checkBeforeStartPipe(final String pipeName) throws PipeException { @@ -288,6 +341,16 @@ public void checkBeforeStartPipe(final String pipeName) throws PipeException { } } + public void checkBeforeStartPipe(final String pipeName, final boolean isTableModel) + throws PipeException { + acquireReadLock(); + try { + checkBeforeStartPipeInternal(pipeName, isTableModel); + } finally { + releaseReadLock(); + } + } + private void checkBeforeStartPipeInternal(final String pipeName) throws PipeException { if (!isPipeExisted(pipeName)) { final String exceptionMessage = @@ -305,6 +368,24 @@ private void checkBeforeStartPipeInternal(final String pipeName) throws PipeExce } } + private void checkBeforeStartPipeInternal(final String pipeName, final boolean isTableModel) + throws PipeException { + if (!isPipeExisted(pipeName, isTableModel)) { + final String exceptionMessage = + String.format("Failed to start pipe %s, %s", pipeName, PIPE_NOT_EXIST_MSG); + LOGGER.warn(exceptionMessage); + throw new PipeException(exceptionMessage); + } + + final PipeStatus pipeStatus = getPipeStatus(pipeName, isTableModel); + if (pipeStatus == PipeStatus.DROPPED) { + final String exceptionMessage = + String.format("Failed to start pipe %s, the pipe is already dropped", pipeName); + LOGGER.warn(exceptionMessage); + throw new PipeException(exceptionMessage); + } + } + public void checkBeforeStopPipe(final String pipeName) throws PipeException { acquireReadLock(); try { @@ -314,6 +395,16 @@ public void checkBeforeStopPipe(final String pipeName) throws PipeException { } } + public void checkBeforeStopPipe(final String pipeName, final boolean isTableModel) + throws PipeException { + acquireReadLock(); + try { + checkBeforeStopPipeInternal(pipeName, isTableModel); + } finally { + releaseReadLock(); + } + } + private void checkBeforeStopPipeInternal(final String pipeName) throws PipeException { if (!isPipeExisted(pipeName)) { final String exceptionMessage = @@ -331,6 +422,24 @@ private void checkBeforeStopPipeInternal(final String pipeName) throws PipeExcep } } + private void checkBeforeStopPipeInternal(final String pipeName, final boolean isTableModel) + throws PipeException { + if (!isPipeExisted(pipeName, isTableModel)) { + final String exceptionMessage = + String.format("Failed to stop pipe %s, %s", pipeName, PIPE_NOT_EXIST_MSG); + LOGGER.warn(exceptionMessage); + throw new PipeException(exceptionMessage); + } + + final PipeStatus pipeStatus = getPipeStatus(pipeName, isTableModel); + if (pipeStatus == PipeStatus.DROPPED) { + final String exceptionMessage = + String.format("Failed to stop pipe %s, the pipe is already dropped", pipeName); + LOGGER.warn(exceptionMessage); + throw new PipeException(exceptionMessage); + } + } + public void checkBeforeDropPipe(final String pipeName) { acquireReadLock(); try { @@ -377,6 +486,15 @@ private PipeStatus getPipeStatus(final String pipeName) { } } + private PipeStatus getPipeStatus(final String pipeName, final boolean isTableModel) { + acquireReadLock(); + try { + return pipeMetaKeeper.getPipeMeta(pipeName, isTableModel).getRuntimeMeta().getStatus().get(); + } finally { + releaseReadLock(); + } + } + public boolean isPipeRunning(final String pipeName) { acquireReadLock(); try { @@ -387,6 +505,16 @@ public boolean isPipeRunning(final String pipeName) { } } + public boolean isPipeRunning(final String pipeName, final boolean isTableModel) { + acquireReadLock(); + try { + return pipeMetaKeeper.containsPipeMeta(pipeName, isTableModel) + && PipeStatus.RUNNING.equals(getPipeStatus(pipeName, isTableModel)); + } finally { + releaseReadLock(); + } + } + public boolean isPipeStoppedByUser(final String pipeName) { acquireReadLock(); try { @@ -398,6 +526,17 @@ public boolean isPipeStoppedByUser(final String pipeName) { } } + public boolean isPipeStoppedByUser(final String pipeName, final boolean isTableModel) { + acquireReadLock(); + try { + return pipeMetaKeeper.containsPipeMeta(pipeName, isTableModel) + && PipeStatus.STOPPED.equals(getPipeStatus(pipeName, isTableModel)) + && !isStoppedByRuntimeException(pipeName, isTableModel); + } finally { + releaseReadLock(); + } + } + public void validatePipePluginUsageByPipe(String pluginName) { acquireReadLock(); try { @@ -514,8 +653,8 @@ public TSStatus alterPipe(final AlterPipePlanV2 plan) { try { enrichPipeMetaWithRootUserForCompatibility(plan.getPipeStaticMeta()); final PipeTemporaryMeta temporaryMeta = - pipeMetaKeeper.getPipeMeta(plan.getPipeStaticMeta().getPipeName()).getTemporaryMeta(); - pipeMetaKeeper.removePipeMeta(plan.getPipeStaticMeta().getPipeName()); + pipeMetaKeeper.getPipeMeta(plan.getCurrentPipeStaticMeta()).getTemporaryMeta(); + pipeMetaKeeper.removePipeMeta(plan.getCurrentPipeStaticMeta()); pipeMetaKeeper.addPipeMeta( new PipeMeta(plan.getPipeStaticMeta(), plan.getPipeRuntimeMeta(), temporaryMeta)); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); @@ -528,7 +667,7 @@ public TSStatus setPipeStatus(final SetPipeStatusPlanV2 plan) { acquireWriteLock(); try { pipeMetaKeeper - .getPipeMeta(plan.getPipeName()) + .getPipeMeta(plan.getPipeName(), plan.isTableModel()) .getRuntimeMeta() .getStatus() .set(plan.getPipeStatus()); @@ -543,12 +682,12 @@ public TSStatus setPipeStatusWithStoppedByRuntimeException( acquireWriteLock(); try { pipeMetaKeeper - .getPipeMeta(plan.getPipeName()) + .getPipeMeta(plan.getPipeName(), plan.isTableModel()) .getRuntimeMeta() .getStatus() .set(plan.getPipeStatus()); pipeMetaKeeper - .getPipeMeta(plan.getPipeName()) + .getPipeMeta(plan.getPipeName(), plan.isTableModel()) .getRuntimeMeta() .setIsStoppedByRuntimeException(plan.isStoppedByRuntimeException()); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); @@ -560,7 +699,7 @@ public TSStatus setPipeStatusWithStoppedByRuntimeException( public TSStatus dropPipe(final DropPipePlanV2 plan) { acquireWriteLock(); try { - pipeMetaKeeper.removePipeMeta(plan.getPipeName()); + pipeMetaKeeper.removePipeMeta(plan.getPipeName(), plan.isTableModel()); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } finally { releaseWriteLock(); @@ -597,6 +736,39 @@ public PipeMeta getPipeMetaByPipeName(final String pipeName) { } } + public PipeMeta getPipeMetaByPipeName(final String pipeName, final boolean isTableModel) { + acquireReadLock(); + try { + return pipeMetaKeeper.getPipeMetaByPipeName(pipeName, isTableModel); + } finally { + releaseReadLock(); + } + } + + public PipeMeta getPipeMetaByPipeStaticMeta(final PipeStaticMeta pipeStaticMeta) { + acquireReadLock(); + try { + return pipeMetaKeeper.getPipeMeta(pipeStaticMeta); + } finally { + releaseReadLock(); + } + } + + public long generateUniqueCreationTime(final String pipeName) { + acquireReadLock(); + try { + long creationTime = System.currentTimeMillis(); + for (final PipeMeta pipeMeta : pipeMetaKeeper.getPipeMetaList()) { + if (pipeMeta.getStaticMeta().getPipeName().equals(pipeName)) { + creationTime = Math.max(creationTime, pipeMeta.getStaticMeta().getCreationTime() + 1); + } + } + return creationTime; + } finally { + releaseReadLock(); + } + } + public Map getConsensusPipeStatusMap() { acquireReadLock(); try { @@ -747,11 +919,29 @@ public boolean isStoppedByRuntimeException(final String pipeName) { } } + public boolean isStoppedByRuntimeException(final String pipeName, final boolean isTableModel) { + acquireReadLock(); + try { + return isStoppedByRuntimeExceptionInternal(pipeName, isTableModel); + } finally { + releaseReadLock(); + } + } + private boolean isStoppedByRuntimeExceptionInternal(final String pipeName) { return pipeMetaKeeper.containsPipeMeta(pipeName) && pipeMetaKeeper.getPipeMeta(pipeName).getRuntimeMeta().getIsStoppedByRuntimeException(); } + private boolean isStoppedByRuntimeExceptionInternal( + final String pipeName, final boolean isTableModel) { + return pipeMetaKeeper.containsPipeMeta(pipeName, isTableModel) + && pipeMetaKeeper + .getPipeMeta(pipeName, isTableModel) + .getRuntimeMeta() + .getIsStoppedByRuntimeException(); + } + /** * Clear the exceptions of a pipe locally after it starts successfully. * @@ -769,13 +959,35 @@ public void clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(final String } } + public void clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse( + final String pipeName, final boolean isTableModel) { + acquireWriteLock(); + try { + clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal(pipeName, isTableModel); + } finally { + releaseWriteLock(); + } + } + private void clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal( final String pipeName) { - if (!pipeMetaKeeper.containsPipeMeta(pipeName)) { + clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal( + pipeMetaKeeper.getPipeMeta(pipeName)); + } + + private void clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal( + final String pipeName, final boolean isTableModel) { + clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal( + pipeMetaKeeper.getPipeMeta(pipeName, isTableModel)); + } + + private void clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal( + final PipeMeta pipeMeta) { + if (pipeMeta == null) { return; } - final PipeRuntimeMeta runtimeMeta = pipeMetaKeeper.getPipeMeta(pipeName).getRuntimeMeta(); + final PipeRuntimeMeta runtimeMeta = pipeMeta.getRuntimeMeta(); // To avoid unnecessary retries, we set the isStoppedByRuntimeException flag to false runtimeMeta.setIsStoppedByRuntimeException(false); @@ -852,35 +1064,40 @@ private boolean recordDataNodePushPipeMetaExceptionsInternal( continue; } - resp.getExceptionMessages().stream() - .filter(message -> pipeMetaKeeper.containsPipeMeta(message.getPipeName())) - .forEach( - message -> { - final PipeRuntimeMeta runtimeMeta = - pipeMetaKeeper.getPipeMeta(message.getPipeName()).getRuntimeMeta(); - - // Keep user-stopped pipes out of the auto-restart flow. Otherwise, a failed - // STOPPED meta sync can turn a manually stopped pipe into a runtime-stopped one - // and the next PipeMetaSyncer round will restart it automatically. - if (PipeStatus.STOPPED.equals(runtimeMeta.getStatus().get()) - && !runtimeMeta.getIsStoppedByRuntimeException()) { - return; - } - - // Mark the status of the pipe with exception as stopped - runtimeMeta.getStatus().set(PipeStatus.STOPPED); - runtimeMeta.setIsStoppedByRuntimeException(true); - - final Map exceptionMap = - runtimeMeta.getNodeId2PipeRuntimeExceptionMap(); - if (!exceptionMap.containsKey(dataNodeId) - || exceptionMap.get(dataNodeId).getTimeStamp() < message.getTimeStamp()) { - exceptionMap.put( - dataNodeId, - new PipeRuntimeCriticalException( - message.getMessage(), message.getTimeStamp())); - } - }); + for (final TPushPipeMetaRespExceptionMessage message : resp.getExceptionMessages()) { + final OptionalLong creationTime = + tryGetPushPipeMetaRespExceptionMessageCreationTime(message); + final PipeMeta pipeMeta = + creationTime.isPresent() + ? pipeMetaKeeper.getPipeMeta(message.getPipeName(), creationTime.getAsLong()) + : pipeMetaKeeper.getPipeMeta(message.getPipeName()); + if (pipeMeta == null) { + continue; + } + + final PipeRuntimeMeta runtimeMeta = pipeMeta.getRuntimeMeta(); + + // Keep user-stopped pipes out of the auto-restart flow. Otherwise, a failed STOPPED meta + // sync can turn a manually stopped pipe into a runtime-stopped one and the next + // PipeMetaSyncer round will restart it automatically. + if (PipeStatus.STOPPED.equals(runtimeMeta.getStatus().get()) + && !runtimeMeta.getIsStoppedByRuntimeException()) { + continue; + } + + // Mark the status of the pipe with exception as stopped + runtimeMeta.getStatus().set(PipeStatus.STOPPED); + runtimeMeta.setIsStoppedByRuntimeException(true); + + final Map exceptionMap = + runtimeMeta.getNodeId2PipeRuntimeExceptionMap(); + if (!exceptionMap.containsKey(dataNodeId) + || exceptionMap.get(dataNodeId).getTimeStamp() < message.getTimeStamp()) { + exceptionMap.put( + dataNodeId, + new PipeRuntimeCriticalException(message.getMessage(), message.getTimeStamp())); + } + } } } @@ -946,8 +1163,7 @@ private void handleSuccessfulRestartInternal() { .forEach( pipeMeta -> { if (pipeMeta.getRuntimeMeta().getStatus().get().equals(PipeStatus.RUNNING)) { - clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse( - pipeMeta.getStaticMeta().getPipeName()); + clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalseInternal(pipeMeta); } }); } @@ -961,10 +1177,23 @@ public void removePipeMeta(final String pipeName) { } } + public void removePipeMeta(final PipeStaticMeta pipeStaticMeta) { + acquireWriteLock(); + try { + removePipeMetaInternal(pipeStaticMeta); + } finally { + releaseWriteLock(); + } + } + private void removePipeMetaInternal(final String pipeName) { pipeMetaKeeper.removePipeMeta(pipeName); } + private void removePipeMetaInternal(final PipeStaticMeta pipeStaticMeta) { + pipeMetaKeeper.removePipeMeta(pipeStaticMeta); + } + /////////////////////////////// Snapshot /////////////////////////////// @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java index 9ce0c9f72a3c8..4def6fc78b383 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java @@ -36,7 +36,9 @@ import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; @@ -80,8 +82,6 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_IOTDB_PORT_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_REALTIME_FIRST_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_CAPTURE_TABLE_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_CONSENSUS_GROUP_ID_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_CONSENSUS_RECEIVER_DATANODE_ID_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_CONSENSUS_SENDER_DATANODE_ID_KEY; @@ -667,6 +667,11 @@ private TCreatePipeReq buildConsensusPipeReq( ConsensusPipeName pipeName = new ConsensusPipeName(dataRegionId, senderNodeId, receiverNodeId); String replicateMode = CONF.getIotConsensusV2Mode(); + String database = configManager.getPartitionManager().getRegionDatabase(regionId); + String sqlDialect = + database != null && PathUtils.isTableModelDatabase(database) + ? SystemConstant.SQL_DIALECT_TABLE_VALUE + : SystemConstant.SQL_DIALECT_TREE_VALUE; Map extractorAttributes = new HashMap<>(); extractorAttributes.put(EXTRACTOR_KEY, BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName()); @@ -677,8 +682,7 @@ private TCreatePipeReq buildConsensusPipeReq( extractorAttributes.put( EXTRACTOR_CONSENSUS_RECEIVER_DATANODE_ID_KEY, String.valueOf(receiverNodeId)); extractorAttributes.put(EXTRACTOR_REALTIME_MODE_KEY, replicateMode); - extractorAttributes.put(EXTRACTOR_CAPTURE_TABLE_KEY, String.valueOf(true)); - extractorAttributes.put(EXTRACTOR_CAPTURE_TREE_KEY, String.valueOf(true)); + extractorAttributes.put(SystemConstant.SQL_DIALECT_KEY, sqlDialect); extractorAttributes.put( EXTRACTOR_IOTDB_USER_KEY, CommonDescriptor.getInstance().getConfig().getDefaultAdminName()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java index 24a38b156fef9..d2bcd4e953943 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.manager.pipe.metric.overview.PipeProcedureMetrics; @@ -567,6 +568,22 @@ protected Map pushSinglePipeMetaToDataNodes( .serialize()); } + protected Map pushSinglePipeMetaToDataNodes( + String pipeName, boolean isTableModel, ConfigNodeProcedureEnv env) throws IOException { + return env.pushSinglePipeMetaToDataNodes( + copyAndFilterOutNonWorkingDataRegionPipeTasks( + pipeTaskInfo.get().getPipeMetaByPipeName(pipeName, isTableModel)) + .serialize()); + } + + protected Map pushSinglePipeMetaToDataNodes( + PipeStaticMeta pipeStaticMeta, ConfigNodeProcedureEnv env) throws IOException { + return env.pushSinglePipeMetaToDataNodes( + copyAndFilterOutNonWorkingDataRegionPipeTasks( + pipeTaskInfo.get().getPipeMetaByPipeStaticMeta(pipeStaticMeta)) + .serialize()); + } + protected Map pushSinglePipeMetaToDataNodes4Realtime( String pipeName, ConfigNodeProcedureEnv env) throws IOException { final PipeMeta pipeMeta = pipeTaskInfo.get().getPipeMetaByPipeName(pipeName); @@ -585,6 +602,42 @@ protected Map pushSinglePipeMetaToDataNodes4Realtime copyAndFilterOutNonWorkingDataRegionPipeTasks(pipeMeta).serialize()); } + protected Map pushSinglePipeMetaToDataNodes4Realtime( + String pipeName, boolean isTableModel, ConfigNodeProcedureEnv env) throws IOException { + final PipeMeta pipeMeta = pipeTaskInfo.get().getPipeMetaByPipeName(pipeName, isTableModel); + // Note that although the altered pipe has progress in it, + // if we alter it to realtime we should ignore the previous data + if (!pipeMeta.getStaticMeta().isSourceExternal()) { + pipeMeta + .getStaticMeta() + .getSourceParameters() + .addOrReplaceEquivalentAttributes( + new PipeParameters( + Collections.singletonMap( + SystemConstant.RESTART_OR_NEWLY_ADDED_KEY, Boolean.FALSE.toString()))); + } + return env.pushSinglePipeMetaToDataNodes( + copyAndFilterOutNonWorkingDataRegionPipeTasks(pipeMeta).serialize()); + } + + protected Map pushSinglePipeMetaToDataNodes4Realtime( + PipeStaticMeta pipeStaticMeta, ConfigNodeProcedureEnv env) throws IOException { + final PipeMeta pipeMeta = pipeTaskInfo.get().getPipeMetaByPipeStaticMeta(pipeStaticMeta); + // Note that although the altered pipe has progress in it, + // if we alter it to realtime we should ignore the previous data + if (!pipeMeta.getStaticMeta().isSourceExternal()) { + pipeMeta + .getStaticMeta() + .getSourceParameters() + .addOrReplaceEquivalentAttributes( + new PipeParameters( + Collections.singletonMap( + SystemConstant.RESTART_OR_NEWLY_ADDED_KEY, Boolean.FALSE.toString()))); + } + return env.pushSinglePipeMetaToDataNodes( + copyAndFilterOutNonWorkingDataRegionPipeTasks(pipeMeta).serialize()); + } + /** * Drop a pipe on all the dataNodes. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java index b0d0bce219bde..ee42f4dc9f3b0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.schema.table.Audit; import org.apache.iotdb.commons.utils.TestOnly; @@ -44,6 +45,7 @@ import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; @@ -55,7 +57,10 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -155,7 +160,9 @@ public void executeFromCalculateInfoForTask(final ConfigNodeProcedureEnv env) { alterPipeRequest.getPipeName()); final PipeMeta currentPipeMeta = - pipeTaskInfo.get().getPipeMetaByPipeName(alterPipeRequest.getPipeName()); + pipeTaskInfo + .get() + .getPipeMetaByPipeName(alterPipeRequest.getPipeName(), alterPipeRequest.isTableModel); currentPipeStaticMeta = currentPipeMeta.getStaticMeta(); currentPipeRuntimeMeta = currentPipeMeta.getRuntimeMeta(); @@ -166,7 +173,7 @@ public void executeFromCalculateInfoForTask(final ConfigNodeProcedureEnv env) { updatedPipeStaticMeta = new PipeStaticMeta( alterPipeRequest.getPipeName(), - System.currentTimeMillis(), + pipeTaskInfo.get().generateUniqueCreationTime(alterPipeRequest.getPipeName()), new HashMap<>(alterPipeRequest.getExtractorAttributes()), new HashMap<>(alterPipeRequest.getProcessorAttributes()), new HashMap<>(alterPipeRequest.getConnectorAttributes())); @@ -253,7 +260,9 @@ public void executeFromCalculateInfoForTask(final ConfigNodeProcedureEnv env) { // If the pipe's previous status was user stopped, then after the alter operation, the pipe's // status remains user stopped; otherwise, it becomes running. - if (!pipeTaskInfo.get().isPipeStoppedByUser(alterPipeRequest.getPipeName())) { + if (!pipeTaskInfo + .get() + .isPipeStoppedByUser(alterPipeRequest.getPipeName(), alterPipeRequest.isTableModel)) { updatedPipeRuntimeMeta.getStatus().set(PipeStatus.RUNNING); } } @@ -270,7 +279,9 @@ public void executeFromWriteConfigNodeConsensus(final ConfigNodeProcedureEnv env response = env.getConfigManager() .getConsensusManager() - .write(new AlterPipePlanV2(updatedPipeStaticMeta, updatedPipeRuntimeMeta)); + .write( + new AlterPipePlanV2( + currentPipeStaticMeta, updatedPipeStaticMeta, updatedPipeRuntimeMeta)); } catch (final ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -287,12 +298,7 @@ public void executeFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) thro LOGGER.info(ProcedureMessages.ALTERPIPEPROCEDUREV2_EXECUTEFROMOPERATEONDATANODES, pipeName); final String exceptionMessage = - parsePushPipeMetaExceptionForPipe( - pipeName, - !PipeTaskAgent.isRealtimeOnlyPipe(currentPipeStaticMeta.getSourceParameters()) - && PipeTaskAgent.isRealtimeOnlyPipe(updatedPipeStaticMeta.getSourceParameters()) - ? pushSinglePipeMetaToDataNodes4Realtime(pipeName, env) - : pushSinglePipeMetaToDataNodes(pipeName, env)); + parsePushPipeMetaExceptionForPipe(pipeName, pushAlteredPipeMetaToDataNodes(env)); if (!exceptionMessage.isEmpty()) { LOGGER.warn( ProcedureMessages.FAILED_TO_ALTER_PIPE_DETAILS_METADATA_WILL_BE_SYNCHRONIZED_LATER, @@ -301,6 +307,35 @@ public void executeFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) thro } } + private Map pushAlteredPipeMetaToDataNodes( + final ConfigNodeProcedureEnv env) throws IOException { + final List pipeMetaBinaryList = new ArrayList<>(); + + final PipeMeta droppedCurrentPipeMeta = + new PipeMeta(currentPipeStaticMeta, currentPipeRuntimeMeta).deepCopy4TaskAgent(); + droppedCurrentPipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.DROPPED); + pipeMetaBinaryList.add( + copyAndFilterOutNonWorkingDataRegionPipeTasks(droppedCurrentPipeMeta).serialize()); + + final PipeMeta updatedPipeMeta = + pipeTaskInfo.get().getPipeMetaByPipeStaticMeta(updatedPipeStaticMeta); + if (!PipeTaskAgent.isRealtimeOnlyPipe(currentPipeStaticMeta.getSourceParameters()) + && PipeTaskAgent.isRealtimeOnlyPipe(updatedPipeStaticMeta.getSourceParameters()) + && !updatedPipeMeta.getStaticMeta().isSourceExternal()) { + updatedPipeMeta + .getStaticMeta() + .getSourceParameters() + .addOrReplaceEquivalentAttributes( + new PipeParameters( + Collections.singletonMap( + SystemConstant.RESTART_OR_NEWLY_ADDED_KEY, Boolean.FALSE.toString()))); + } + pipeMetaBinaryList.add( + copyAndFilterOutNonWorkingDataRegionPipeTasks(updatedPipeMeta).serialize()); + + return env.pushMultiPipeMetaToDataNodes(pipeMetaBinaryList); + } + @Override public void rollbackFromValidateTask(final ConfigNodeProcedureEnv env) { LOGGER.info( @@ -327,7 +362,9 @@ public void rollbackFromWriteConfigNodeConsensus(final ConfigNodeProcedureEnv en response = env.getConfigManager() .getConsensusManager() - .write(new AlterPipePlanV2(currentPipeStaticMeta, currentPipeRuntimeMeta)); + .write( + new AlterPipePlanV2( + updatedPipeStaticMeta, currentPipeStaticMeta, currentPipeRuntimeMeta)); } catch (final ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -408,6 +445,7 @@ public void serialize(final DataOutputStream stream) throws IOException { } ReadWriteIOUtils.write(alterPipeRequest.isReplaceAllExtractorAttributes, stream); } + ReadWriteIOUtils.write(alterPipeRequest.isTableModel, stream); } @Override @@ -458,6 +496,8 @@ public void deserialize(final ByteBuffer byteBuffer) { alterPipeRequest.setExtractorAttributes(new HashMap<>()); alterPipeRequest.isReplaceAllExtractorAttributes = false; } + alterPipeRequest.isTableModel = + byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer); } @Override @@ -470,6 +510,7 @@ public boolean equals(final Object o) { } AlterPipeProcedureV2 that = (AlterPipeProcedureV2) o; return this.alterPipeRequest.getPipeName().equals(that.alterPipeRequest.getPipeName()) + && this.alterPipeRequest.isTableModel == that.alterPipeRequest.isTableModel && this.alterPipeRequest .getExtractorAttributes() .toString() @@ -488,6 +529,7 @@ public boolean equals(final Object o) { public int hashCode() { return Objects.hash( alterPipeRequest.getPipeName(), + alterPipeRequest.isTableModel, alterPipeRequest.getExtractorAttributes(), alterPipeRequest.getProcessorAttributes(), alterPipeRequest.getConnectorAttributes()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java index 3cb0665600a09..e731ce44ea6c9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java @@ -92,7 +92,7 @@ public CreatePipeProcedureV2() { public CreatePipeProcedureV2(final TCreatePipeReq createPipeRequest) throws PipeException { super(); - this.createPipeRequest = createPipeRequest; + this.createPipeRequest = normalizeCreatePipeRequest(createPipeRequest); } /** This is only used when the pipe task info lock is held by another procedure. */ @@ -101,7 +101,20 @@ public CreatePipeProcedureV2( throws PipeException { super(); this.pipeTaskInfo = pipeTaskInfo; - this.createPipeRequest = createPipeRequest; + this.createPipeRequest = normalizeCreatePipeRequest(createPipeRequest); + } + + private TCreatePipeReq normalizeCreatePipeRequest(final TCreatePipeReq createPipeRequest) { + if (createPipeRequest.getExtractorAttributes() == null) { + createPipeRequest.setExtractorAttributes(new HashMap<>()); + } + if (createPipeRequest.getProcessorAttributes() == null) { + createPipeRequest.setProcessorAttributes(new HashMap<>()); + } + if (createPipeRequest.getConnectorAttributes() == null) { + createPipeRequest.setConnectorAttributes(new HashMap<>()); + } + return createPipeRequest; } /** @@ -112,6 +125,14 @@ public String getPipeName() { return createPipeRequest.getPipeName(); } + /** + * This should be called after {@link #executeFromValidateTask} and {@link + * #executeFromCalculateInfoForTask}. + */ + public PipeStaticMeta getPipeStaticMeta() { + return pipeStaticMeta; + } + /** * This should be called after {@link #executeFromValidateTask} and {@link * #executeFromCalculateInfoForTask}. @@ -268,7 +289,7 @@ public void executeFromCalculateInfoForTask(final ConfigNodeProcedureEnv env) { pipeStaticMeta = new PipeStaticMeta( createPipeRequest.getPipeName(), - System.currentTimeMillis(), + pipeTaskInfo.get().generateUniqueCreationTime(createPipeRequest.getPipeName()), createPipeRequest.getExtractorAttributes(), createPipeRequest.getProcessorAttributes(), createPipeRequest.getConnectorAttributes()); @@ -386,7 +407,8 @@ public void executeFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) thro LOGGER.info(ProcedureMessages.CREATEPIPEPROCEDUREV2_EXECUTEFROMOPERATEONDATANODES, pipeName); final String exceptionMessage = - parsePushPipeMetaExceptionForPipe(pipeName, pushSinglePipeMetaToDataNodes(pipeName, env)); + parsePushPipeMetaExceptionForPipe( + pipeName, pushSinglePipeMetaToDataNodes(pipeStaticMeta, env)); if (!exceptionMessage.isEmpty()) { LOGGER.warn( ProcedureMessages.FAILED_TO_CREATE_PIPE_DETAILS_METADATA_WILL_BE_SYNCHRONIZED_LATER, @@ -421,7 +443,9 @@ public void rollbackFromWriteConfigNodeConsensus(final ConfigNodeProcedureEnv en response = env.getConfigManager() .getConsensusManager() - .write(new DropPipePlanV2(createPipeRequest.getPipeName())); + .write( + new DropPipePlanV2( + createPipeRequest.getPipeName(), pipeStaticMeta.visibleUnderTableModel())); } catch (final ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2.java index 530ca4c1956c6..3c5665d526d14 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2.java @@ -20,6 +20,8 @@ package org.apache.iotdb.confignode.procedure.impl.pipe.task; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.DropPipePlanV2; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.i18n.ProcedureMessages; @@ -47,21 +49,36 @@ public class DropPipeProcedureV2 extends AbstractOperatePipeProcedureV2 { private static final Logger LOGGER = LoggerFactory.getLogger(DropPipeProcedureV2.class); private String pipeName; + private boolean isTableModel; + private PipeMeta pipeMetaToDrop; public DropPipeProcedureV2() { super(); } public DropPipeProcedureV2(String pipeName) throws PipeException { + this(pipeName, false); + } + + public DropPipeProcedureV2(String pipeName, boolean isTableModel) throws PipeException { super(); this.pipeName = pipeName; + this.isTableModel = isTableModel; } /** This is only used when the pipe task info lock is held by another procedure. */ public DropPipeProcedureV2(String pipeName, AtomicReference pipeTaskInfo) throws PipeException { + this(pipeName, false, pipeTaskInfo); + } + + /** This is only used when the pipe task info lock is held by another procedure. */ + public DropPipeProcedureV2( + String pipeName, boolean isTableModel, AtomicReference pipeTaskInfo) + throws PipeException { super(); this.pipeName = pipeName; + this.isTableModel = isTableModel; this.pipeTaskInfo = pipeTaskInfo; } @@ -69,6 +86,16 @@ public String getPipeName() { return pipeName; } + public boolean isTableModel() { + return pipeMetaToDrop == null + ? isTableModel + : pipeMetaToDrop.getStaticMeta().visibleUnderTableModel(); + } + + public PipeMeta getPipeMetaToDrop() { + return pipeMetaToDrop; + } + @Override protected PipeTaskOperation getOperation() { return PipeTaskOperation.DROP_PIPE; @@ -86,7 +113,7 @@ public boolean executeFromValidateTask(ConfigNodeProcedureEnv env) throws PipeEx @Override public void executeFromCalculateInfoForTask(ConfigNodeProcedureEnv env) throws PipeException { LOGGER.info(ProcedureMessages.DROPPIPEPROCEDUREV2_EXECUTEFROMCALCULATEINFOFORTASK, pipeName); - // Do nothing + pipeMetaToDrop = pipeTaskInfo.get().getPipeMetaByPipeName(pipeName, isTableModel); } @Override @@ -96,7 +123,10 @@ public void executeFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) thro TSStatus response; try { - response = env.getConfigManager().getConsensusManager().write(new DropPipePlanV2(pipeName)); + response = + env.getConfigManager() + .getConsensusManager() + .write(new DropPipePlanV2(pipeName, isTableModel)); } catch (ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -111,8 +141,22 @@ public void executeFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) thro public void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) { LOGGER.info(ProcedureMessages.DROPPIPEPROCEDUREV2_EXECUTEFROMOPERATEONDATANODES, pipeName); - final String exceptionMessage = - parsePushPipeMetaExceptionForPipe(pipeName, dropSinglePipeOnDataNodes(pipeName, env)); + String exceptionMessage; + try { + if (pipeMetaToDrop == null) { + exceptionMessage = + parsePushPipeMetaExceptionForPipe(pipeName, pushPipeMetaToDataNodes(env)); + } else { + final PipeMeta droppedPipeMeta = + copyAndFilterOutNonWorkingDataRegionPipeTasks(pipeMetaToDrop); + droppedPipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.DROPPED); + exceptionMessage = + parsePushPipeMetaExceptionForPipe( + pipeName, env.pushSinglePipeMetaToDataNodes(droppedPipeMeta.serialize())); + } + } catch (final IOException e) { + exceptionMessage = e.getMessage(); + } if (!exceptionMessage.isEmpty()) { LOGGER.warn( ProcedureMessages.FAILED_TO_DROP_PIPE_DETAILS_METADATA_WILL_BE_SYNCHRONIZED_LATER, @@ -151,12 +195,23 @@ public void serialize(DataOutputStream stream) throws IOException { stream.writeShort(ProcedureType.DROP_PIPE_PROCEDURE_V2.getTypeCode()); super.serialize(stream); ReadWriteIOUtils.write(pipeName, stream); + ReadWriteIOUtils.write(isTableModel, stream); + if (pipeMetaToDrop == null) { + ReadWriteIOUtils.write(false, stream); + } else { + ReadWriteIOUtils.write(true, stream); + pipeMetaToDrop.serialize(stream); + } } @Override public void deserialize(ByteBuffer byteBuffer) { super.deserialize(byteBuffer); pipeName = ReadWriteIOUtils.readString(byteBuffer); + isTableModel = byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer); + if (byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer)) { + pipeMetaToDrop = PipeMeta.deserialize4Coordinator(byteBuffer); + } } @Override @@ -171,11 +226,12 @@ public boolean equals(Object o) { return getProcId() == that.getProcId() && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() + && isTableModel == that.isTableModel && pipeName.equals(that.pipeName); } @Override public int hashCode() { - return Objects.hash(getProcId(), getCurrentState(), getCycles(), pipeName); + return Objects.hash(getProcId(), getCurrentState(), getCycles(), pipeName, isTableModel); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java index 7c9bff96f8fe6..4b03706ebafb7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java @@ -20,6 +20,7 @@ package org.apache.iotdb.confignode.procedure.impl.pipe.task; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; @@ -46,14 +47,20 @@ public class StartPipeProcedureV2 extends AbstractOperatePipeProcedureV2 { private static final Logger LOGGER = LoggerFactory.getLogger(StartPipeProcedureV2.class); private String pipeName; + private boolean isTableModel; public StartPipeProcedureV2() { super(); } public StartPipeProcedureV2(String pipeName) throws PipeException { + this(pipeName, false); + } + + public StartPipeProcedureV2(String pipeName, boolean isTableModel) throws PipeException { super(); this.pipeName = pipeName; + this.isTableModel = isTableModel; } @Override @@ -65,10 +72,10 @@ protected PipeTaskOperation getOperation() { public boolean executeFromValidateTask(ConfigNodeProcedureEnv env) throws PipeException { LOGGER.info(ProcedureMessages.STARTPIPEPROCEDUREV2_EXECUTEFROMVALIDATETASK, pipeName); - pipeTaskInfo.get().checkBeforeStartPipe(pipeName); + pipeTaskInfo.get().checkBeforeStartPipe(pipeName, isTableModel); - return !pipeTaskInfo.get().isPipeRunning(pipeName) - || pipeTaskInfo.get().isStoppedByRuntimeException(pipeName); + return !pipeTaskInfo.get().isPipeRunning(pipeName, isTableModel) + || pipeTaskInfo.get().isStoppedByRuntimeException(pipeName, isTableModel); } @Override @@ -87,7 +94,7 @@ public void executeFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) thro response = env.getConfigManager() .getConsensusManager() - .write(new SetPipeStatusPlanV2(pipeName, PipeStatus.RUNNING)); + .write(new SetPipeStatusPlanV2(pipeName, PipeStatus.RUNNING, isTableModel)); } catch (ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -102,8 +109,11 @@ public void executeFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) thro public void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) throws IOException { LOGGER.info(ProcedureMessages.STARTPIPEPROCEDUREV2_EXECUTEFROMOPERATEONDATANODES, pipeName); + final PipeStaticMeta pipeStaticMeta = + pipeTaskInfo.get().getPipeMetaByPipeName(pipeName, isTableModel).getStaticMeta(); final String exceptionMessage = - parsePushPipeMetaExceptionForPipe(pipeName, pushSinglePipeMetaToDataNodes(pipeName, env)); + parsePushPipeMetaExceptionForPipe( + pipeName, pushSinglePipeMetaToDataNodes(pipeStaticMeta, env)); if (!exceptionMessage.isEmpty()) { LOGGER.warn( ProcedureMessages.FAILED_TO_START_PIPE_DETAILS_METADATA_WILL_BE_SYNCHRONIZED_LATER, @@ -114,7 +124,9 @@ public void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) throws IOE // Clear exceptions and set isStoppedByRuntimeException to false if the pipe is // started successfully on all data nodes - pipeTaskInfo.get().clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(pipeName); + pipeTaskInfo + .get() + .clearExceptionsAndSetIsStoppedByRuntimeExceptionToFalse(pipeName, isTableModel); } @Override @@ -139,7 +151,7 @@ public void rollbackFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) { response = env.getConfigManager() .getConsensusManager() - .write(new SetPipeStatusPlanV2(pipeName, PipeStatus.STOPPED)); + .write(new SetPipeStatusPlanV2(pipeName, PipeStatus.STOPPED, isTableModel)); } catch (ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -170,12 +182,14 @@ public void serialize(DataOutputStream stream) throws IOException { stream.writeShort(ProcedureType.START_PIPE_PROCEDURE_V2.getTypeCode()); super.serialize(stream); ReadWriteIOUtils.write(pipeName, stream); + ReadWriteIOUtils.write(isTableModel, stream); } @Override public void deserialize(ByteBuffer byteBuffer) { super.deserialize(byteBuffer); pipeName = ReadWriteIOUtils.readString(byteBuffer); + isTableModel = byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer); } @Override @@ -190,11 +204,12 @@ public boolean equals(Object o) { return getProcId() == that.getProcId() && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() + && isTableModel == that.isTableModel && pipeName.equals(that.pipeName); } @Override public int hashCode() { - return Objects.hash(getProcId(), getCurrentState(), getCycles(), pipeName); + return Objects.hash(getProcId(), getCurrentState(), getCycles(), pipeName, isTableModel); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2.java index 4410be01d3fdd..10f50b3ef1622 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2.java @@ -20,6 +20,7 @@ package org.apache.iotdb.confignode.procedure.impl.pipe.task; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusWithStoppedByRuntimeExceptionPlanV2; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; @@ -46,6 +47,7 @@ public class StopPipeProcedureV2 extends AbstractOperatePipeProcedureV2 { private static final Logger LOGGER = LoggerFactory.getLogger(StopPipeProcedureV2.class); private String pipeName; + private boolean isTableModel; private boolean isStoppedByRuntimeExceptionBeforeStop; public StopPipeProcedureV2() { @@ -53,8 +55,13 @@ public StopPipeProcedureV2() { } public StopPipeProcedureV2(String pipeName) throws PipeException { + this(pipeName, false); + } + + public StopPipeProcedureV2(String pipeName, boolean isTableModel) throws PipeException { super(); this.pipeName = pipeName; + this.isTableModel = isTableModel; } @Override @@ -66,16 +73,16 @@ protected PipeTaskOperation getOperation() { public boolean executeFromValidateTask(ConfigNodeProcedureEnv env) throws PipeException { LOGGER.info(ProcedureMessages.STOPPIPEPROCEDUREV2_EXECUTEFROMVALIDATETASK, pipeName); - pipeTaskInfo.get().checkBeforeStopPipe(pipeName); + pipeTaskInfo.get().checkBeforeStopPipe(pipeName, isTableModel); - return !pipeTaskInfo.get().isPipeStoppedByUser(pipeName); + return !pipeTaskInfo.get().isPipeStoppedByUser(pipeName, isTableModel); } @Override public void executeFromCalculateInfoForTask(ConfigNodeProcedureEnv env) throws PipeException { LOGGER.info(ProcedureMessages.STOPPIPEPROCEDUREV2_EXECUTEFROMCALCULATEINFOFORTASK, pipeName); isStoppedByRuntimeExceptionBeforeStop = - pipeTaskInfo.get().isStoppedByRuntimeException(pipeName); + pipeTaskInfo.get().isStoppedByRuntimeException(pipeName, isTableModel); } @Override @@ -90,7 +97,7 @@ public void executeFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) thro .getConsensusManager() .write( new SetPipeStatusWithStoppedByRuntimeExceptionPlanV2( - pipeName, PipeStatus.STOPPED, false)); + pipeName, PipeStatus.STOPPED, false, isTableModel)); } catch (ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -105,8 +112,11 @@ public void executeFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) thro public void executeFromOperateOnDataNodes(ConfigNodeProcedureEnv env) throws IOException { LOGGER.info(ProcedureMessages.STOPPIPEPROCEDUREV2_EXECUTEFROMOPERATEONDATANODES, pipeName); + final PipeStaticMeta pipeStaticMeta = + pipeTaskInfo.get().getPipeMetaByPipeName(pipeName, isTableModel).getStaticMeta(); final String exceptionMessage = - parsePushPipeMetaExceptionForPipe(pipeName, pushSinglePipeMetaToDataNodes(pipeName, env)); + parsePushPipeMetaExceptionForPipe( + pipeName, pushSinglePipeMetaToDataNodes(pipeStaticMeta, env)); if (!exceptionMessage.isEmpty()) { LOGGER.warn( ProcedureMessages.FAILED_TO_STOP_PIPE_DETAILS_METADATA_WILL_BE_SYNCHRONIZED_LATER, @@ -138,7 +148,10 @@ public void rollbackFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) { .getConsensusManager() .write( new SetPipeStatusWithStoppedByRuntimeExceptionPlanV2( - pipeName, PipeStatus.RUNNING, isStoppedByRuntimeExceptionBeforeStop)); + pipeName, + PipeStatus.RUNNING, + isStoppedByRuntimeExceptionBeforeStop, + isTableModel)); } catch (ConsensusException e) { LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -170,6 +183,7 @@ public void serialize(DataOutputStream stream) throws IOException { super.serialize(stream); ReadWriteIOUtils.write(pipeName, stream); ReadWriteIOUtils.write(isStoppedByRuntimeExceptionBeforeStop, stream); + ReadWriteIOUtils.write(isTableModel, stream); } @Override @@ -179,6 +193,7 @@ public void deserialize(ByteBuffer byteBuffer) { // Legacy persisted procedures do not carry this field. isStoppedByRuntimeExceptionBeforeStop = byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer); + isTableModel = byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer); } @Override @@ -193,6 +208,7 @@ public boolean equals(Object o) { return getProcId() == that.getProcId() && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() + && isTableModel == that.isTableModel && isStoppedByRuntimeExceptionBeforeStop == that.isStoppedByRuntimeExceptionBeforeStop && pipeName.equals(that.pipeName); } @@ -204,6 +220,7 @@ public int hashCode() { getCurrentState(), getCycles(), pipeName, + isTableModel, isStoppedByRuntimeExceptionBeforeStop); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/AbstractOperateSubscriptionAndPipeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/AbstractOperateSubscriptionAndPipeProcedure.java index 5c77fa3a5e4a3..8e922783fab1e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/AbstractOperateSubscriptionAndPipeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/AbstractOperateSubscriptionAndPipeProcedure.java @@ -20,9 +20,12 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.subscription; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.impl.pipe.task.DropPipeProcedureV2; import org.apache.iotdb.confignode.procedure.impl.subscription.AbstractOperateSubscriptionProcedure; import org.apache.iotdb.confignode.procedure.state.ProcedureLockState; import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp; @@ -133,19 +136,20 @@ protected void releaseLock(ConfigNodeProcedureEnv configNodeProcedureEnv) { /** * Pushing multiple pipeMetas to all the dataNodes, forcing an update to the pipes' runtime state. * - * @param pipeNames pipe names of the pipes to push + * @param pipeStaticMetas pipe static metas of the pipes to push * @param env ConfigNodeProcedureEnv * @return The responseMap after pushing pipe meta * @throws IOException Exception when Serializing to byte buffer */ protected Map pushMultiPipeMetaToDataNodes( - List pipeNames, ConfigNodeProcedureEnv env) throws IOException { + List pipeStaticMetas, ConfigNodeProcedureEnv env) throws IOException { final List pipeMetaBinaryList = new ArrayList<>(); - for (String pipeName : pipeNames) { - PipeMeta pipeMeta = pipeTaskInfo.get().getPipeMetaByPipeName(pipeName); + for (PipeStaticMeta pipeStaticMeta : pipeStaticMetas) { + PipeMeta pipeMeta = pipeTaskInfo.get().getPipeMetaByPipeStaticMeta(pipeStaticMeta); if (pipeMeta == null) { LOGGER.warn( - ProcedureMessages.PIPE_NOT_FOUND_IN_PIPETASKINFO_CAN_NOT_PUSH_ITS_META, pipeName); + ProcedureMessages.PIPE_NOT_FOUND_IN_PIPETASKINFO_CAN_NOT_PUSH_ITS_META, + pipeStaticMeta.getPipeName()); continue; } pipeMetaBinaryList.add(copyAndFilterOutNonWorkingDataRegionPipeTasks(pipeMeta).serialize()); @@ -157,12 +161,28 @@ protected Map pushMultiPipeMetaToDataNodes( /** * Drop multiple pipes on all the dataNodes. * - * @param pipeNames pipe names of the pipes to drop + * @param dropPipeProcedures drop pipe procedures that captured the pipe metas to drop * @param env ConfigNodeProcedureEnv * @return The responseMap after pushing pipe meta */ protected Map dropMultiPipeOnDataNodes( - List pipeNames, ConfigNodeProcedureEnv env) { - return env.dropMultiPipeOnDataNodes(pipeNames); + List dropPipeProcedures, ConfigNodeProcedureEnv env) throws IOException { + final List pipeMetaBinaryList = new ArrayList<>(); + for (final DropPipeProcedureV2 dropPipeProcedure : dropPipeProcedures) { + final PipeMeta pipeMetaToDrop = dropPipeProcedure.getPipeMetaToDrop(); + if (pipeMetaToDrop == null) { + LOGGER.warn( + ProcedureMessages.PIPE_NOT_FOUND_IN_PIPETASKINFO_CAN_NOT_PUSH_ITS_META, + dropPipeProcedure.getPipeName()); + continue; + } + + final PipeMeta droppedPipeMeta = + copyAndFilterOutNonWorkingDataRegionPipeTasks(pipeMetaToDrop); + droppedPipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.DROPPED); + pipeMetaBinaryList.add(droppedPipeMeta.serialize()); + } + + return env.pushMultiPipeMetaToDataNodes(pipeMetaBinaryList); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java index 57e60c9cdda28..c01ae03de38a9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java @@ -134,7 +134,7 @@ protected boolean executeFromValidate(final ConfigNodeProcedureEnv env) if (!subscriptionInfo.get().isTopicSubscribedByConsumerGroup(topicName, consumerGroupId) // even if there existed subscription meta, if there is no corresponding pipe meta, it // will try to create the pipe - || !pipeTaskInfo.get().isPipeExisted(pipeName)) { + || !pipeTaskInfo.get().isPipeExisted(pipeName, topicMeta.visibleUnderTableModel())) { createPipeProcedures.add( new CreatePipeProcedureV2( new TCreatePipeReq() @@ -210,20 +210,22 @@ protected void executeFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) // Push pipe meta to data nodes (only for non-consensus pipe-based topics) if (!createPipeProcedures.isEmpty()) { - final List pipeNames = + final List pipeStaticMetas = createPipeProcedures.stream() - .map(CreatePipeProcedureV2::getPipeName) + .map(CreatePipeProcedureV2::getPipeStaticMeta) .collect(Collectors.toList()); final String exceptionMessage = AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( - null, pushMultiPipeMetaToDataNodes(pipeNames, env)); + null, pushMultiPipeMetaToDataNodes(pipeStaticMetas, env)); if (!exceptionMessage.isEmpty()) { // throw exception instead of logging warn, do not rely on metadata synchronization throw new SubscriptionException( String.format( ProcedureMessages .FAILED_TO_CREATE_PIPES_WHEN_CREATING_SUBSCRIPTION_WITH_REQUEST_DETAILS, - pipeNames, + pipeStaticMetas.stream() + .map(PipeStaticMeta::getPipeName) + .collect(Collectors.toList()), subscribeReq, exceptionMessage)); } @@ -243,7 +245,11 @@ protected void rollbackFromOperateOnConfigNodes(final ConfigNodeProcedureEnv env // Rollback CreatePipeProcedureV2s final List dropPipePlans = createPipeProcedures.stream() - .map(procedure -> new DropPipePlanV2(procedure.getPipeName())) + .map( + procedure -> + new DropPipePlanV2( + procedure.getPipeName(), + procedure.getPipeStaticMeta().visibleUnderTableModel())) .collect(Collectors.toList()); TSStatus response; try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java index 8db599402090d..2dad91bf7cf81 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java @@ -124,6 +124,7 @@ protected boolean executeFromValidate(final ConfigNodeProcedureEnv env) new DropPipeProcedureV2( PipeStaticMeta.generateSubscriptionPipeName( topic, unsubscribeReq.getConsumerGroupId()), + topicMeta.visibleUnderTableModel(), pipeTaskInfo)); } } @@ -147,7 +148,7 @@ protected void executeFromOperateOnConfigNodes(final ConfigNodeProcedureEnv env) // Execute DropPipeProcedureV2s final List dropPipePlans = dropPipeProcedures.stream() - .map(proc -> new DropPipePlanV2(proc.getPipeName())) + .map(proc -> new DropPipePlanV2(proc.getPipeName(), proc.isTableModel())) .collect(Collectors.toList()); TSStatus response; try { @@ -178,20 +179,18 @@ protected void executeFromOperateOnDataNodes(final ConfigNodeProcedureEnv env) LOGGER.info(ProcedureMessages.DROPSUBSCRIPTIONPROCEDURE_EXECUTEFROMOPERATEONDATANODES); // Push pipe meta to data nodes - final List pipeNames = - dropPipeProcedures.stream() - .map(DropPipeProcedureV2::getPipeName) - .collect(Collectors.toList()); final String exceptionMessage = AbstractOperatePipeProcedureV2.parsePushPipeMetaExceptionForPipe( - null, dropMultiPipeOnDataNodes(pipeNames, env)); + null, dropMultiPipeOnDataNodes(dropPipeProcedures, env)); if (!exceptionMessage.isEmpty()) { // throw exception instead of logging warn, do not rely on metadata synchronization throw new SubscriptionException( String.format( ProcedureMessages .FAILED_TO_DROP_PIPES_WHEN_DROPPING_SUBSCRIPTION_WITH_REQUEST_BECAUSE, - pipeNames, + dropPipeProcedures.stream() + .map(DropPipeProcedureV2::getPipeName) + .collect(Collectors.toList()), unsubscribeReq, exceptionMessage)); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index ea35be6c5d7e1..8f467e0bc1e4c 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -971,6 +971,8 @@ public void AlterPipePlanV2Test() throws IOException { final AlterPipePlanV2 alterPipePlanV21 = (AlterPipePlanV2) ConfigPhysicalPlan.Factory.create(alterPipePlanV2.serializeToByteBuffer()); + Assert.assertEquals( + alterPipePlanV2.getCurrentPipeStaticMeta(), alterPipePlanV21.getCurrentPipeStaticMeta()); Assert.assertEquals(alterPipePlanV2.getPipeStaticMeta(), alterPipePlanV21.getPipeStaticMeta()); Assert.assertEquals( alterPipePlanV2.getPipeRuntimeMeta(), alterPipePlanV21.getPipeRuntimeMeta()); @@ -980,12 +982,13 @@ public void AlterPipePlanV2Test() throws IOException { public void SetPipeStatusPlanV2Test() throws IOException { final SetPipeStatusPlanV2 setPipeStatusPlanV2 = new SetPipeStatusPlanV2( - "pipe", org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus.RUNNING); + "pipe", org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus.RUNNING, true); final SetPipeStatusPlanV2 setPipeStatusPlanV21 = (SetPipeStatusPlanV2) ConfigPhysicalPlan.Factory.create(setPipeStatusPlanV2.serializeToByteBuffer()); Assert.assertEquals(setPipeStatusPlanV2.getPipeName(), setPipeStatusPlanV21.getPipeName()); Assert.assertEquals(setPipeStatusPlanV2.getPipeStatus(), setPipeStatusPlanV21.getPipeStatus()); + Assert.assertEquals(setPipeStatusPlanV2.isTableModel(), setPipeStatusPlanV21.isTableModel()); } @Test @@ -993,7 +996,10 @@ public void SetPipeStatusWithStoppedByRuntimeExceptionPlanV2Test() throws IOExce final SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 setPipeStatusWithStoppedByRuntimeExceptionPlanV2 = new SetPipeStatusWithStoppedByRuntimeExceptionPlanV2( - "pipe", org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus.STOPPED, true); + "pipe", + org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus.STOPPED, + true, + true); final SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 setPipeStatusWithStoppedByRuntimeExceptionPlanV21 = (SetPipeStatusWithStoppedByRuntimeExceptionPlanV2) @@ -1008,14 +1014,18 @@ public void SetPipeStatusWithStoppedByRuntimeExceptionPlanV2Test() throws IOExce Assert.assertEquals( setPipeStatusWithStoppedByRuntimeExceptionPlanV2.isStoppedByRuntimeException(), setPipeStatusWithStoppedByRuntimeExceptionPlanV21.isStoppedByRuntimeException()); + Assert.assertEquals( + setPipeStatusWithStoppedByRuntimeExceptionPlanV2.isTableModel(), + setPipeStatusWithStoppedByRuntimeExceptionPlanV21.isTableModel()); } @Test public void DropPipePlanV2Test() throws IOException { - final DropPipePlanV2 dropPipePlanV2 = new DropPipePlanV2("demo"); + final DropPipePlanV2 dropPipePlanV2 = new DropPipePlanV2("demo", true); final DropPipePlanV2 dropPipePlanV21 = (DropPipePlanV2) ConfigPhysicalPlan.Factory.create(dropPipePlanV2.serializeToByteBuffer()); Assert.assertEquals(dropPipePlanV2.getPipeName(), dropPipePlanV21.getPipeName()); + Assert.assertEquals(dropPipePlanV2.isTableModel(), dropPipePlanV21.isTableModel()); } @Test diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/response/pipe/PipeTableRespTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/response/pipe/PipeTableRespTest.java index 94189a19d9977..f6a75d26ee4a8 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/response/pipe/PipeTableRespTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/response/pipe/PipeTableRespTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.confignode.consensus.response.pipe.task.PipeTableResp; import org.apache.iotdb.rpc.TSStatusCode; @@ -118,4 +119,80 @@ public void testFilter() { PipeTableResp allPipeTableResp = pipeTableResp.filter(true, null); Assert.assertEquals(3, allPipeTableResp.getAllPipeMeta().size()); } + + @Test + public void testFilterByModelBeforeWhereClause() { + TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + List pipeMetaList = new ArrayList<>(); + + pipeMetaList.add(constructPipeMeta("sameNamePipe", 121, new HashMap<>(), "127.0.0.1")); + + Map tableExtractorAttributes = new HashMap<>(); + tableExtractorAttributes.put( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); + pipeMetaList.add( + constructPipeMeta("sameNamePipe", 122, tableExtractorAttributes, "172.30.30.30")); + pipeMetaList.add( + constructPipeMeta("tablePeerPipe", 123, tableExtractorAttributes, "172.30.30.30")); + + PipeTableResp pipeTableResp = new PipeTableResp(status, pipeMetaList); + + PipeTableResp filteredTablePipeTableResp = + pipeTableResp.filter(true, "sameNamePipe", true, null); + Assert.assertEquals(2, filteredTablePipeTableResp.getAllPipeMeta().size()); + Assert.assertTrue( + filteredTablePipeTableResp.getAllPipeMeta().stream() + .allMatch(pipeMeta -> pipeMeta.getStaticMeta().visibleUnderTableModel())); + + PipeTableResp filteredTreePipeTableResp = + pipeTableResp.filter(true, "sameNamePipe", false, null); + Assert.assertEquals(1, filteredTreePipeTableResp.getAllPipeMeta().size()); + Assert.assertFalse( + filteredTreePipeTableResp.getAllPipeMeta().get(0).getStaticMeta().visibleUnderTableModel()); + } + + @Test + public void testFilterWithoutModelKeepsLegacyVisibility() { + TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + List pipeMetaList = new ArrayList<>(); + + pipeMetaList.add(constructPipeMeta("sameNamePipe", 121, new HashMap<>(), "127.0.0.1")); + + Map tableExtractorAttributes = new HashMap<>(); + tableExtractorAttributes.put( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); + pipeMetaList.add( + constructPipeMeta("sameNamePipe", 122, tableExtractorAttributes, "172.30.30.30")); + pipeMetaList.add( + constructPipeMeta("tablePeerPipe", 123, tableExtractorAttributes, "172.30.30.30")); + + PipeTableResp pipeTableResp = new PipeTableResp(status, pipeMetaList); + + Assert.assertEquals( + 2, pipeTableResp.filter(false, "sameNamePipe", null).getAllPipeMeta().size()); + Assert.assertEquals(3, pipeTableResp.filter(null, null, null).getAllPipeMeta().size()); + } + + private PipeMeta constructPipeMeta( + final String pipeName, + final long creationTime, + final Map extractorAttributes, + final String host) { + Map processorAttributes = new HashMap<>(); + Map connectorAttributes = new HashMap<>(); + + processorAttributes.put("processor", "do-nothing-processor"); + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("host", host); + connectorAttributes.put("port", "6667"); + + PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1); + ConcurrentMap pipeTasks = new ConcurrentHashMap<>(); + pipeTasks.put(1, pipeTaskMeta); + PipeStaticMeta pipeStaticMeta = + new PipeStaticMeta( + pipeName, creationTime, extractorAttributes, processorAttributes, connectorAttributes); + PipeRuntimeMeta pipeRuntimeMeta = new PipeRuntimeMeta(pipeTasks); + return new PipeMeta(pipeStaticMeta, pipeRuntimeMeta); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java index 7b78f59253dc8..d68ca45d64a9b 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2; import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp; @@ -42,6 +43,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.OptionalLong; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -50,10 +52,39 @@ public class PipeTaskInfoAutoRestartTest { private static final int DATA_NODE_ID = 1; private PipeTaskInfo pipeTaskInfo; + private long creationTime; + + private static boolean trySetPushPipeMetaRespExceptionMessageCreationTime( + final TPushPipeMetaRespExceptionMessage exceptionMessage, final long creationTime) { + try { + exceptionMessage + .getClass() + .getMethod("setCreationTime", long.class) + .invoke(exceptionMessage, creationTime); + return true; + } catch (final Exception ignored) { + return false; + } + } + + private static OptionalLong tryGetPushPipeMetaRespExceptionMessageCreationTime( + final TPushPipeMetaRespExceptionMessage exceptionMessage) { + try { + if (!Boolean.TRUE.equals( + exceptionMessage.getClass().getMethod("isSetCreationTime").invoke(exceptionMessage))) { + return OptionalLong.empty(); + } + return OptionalLong.of( + (long) exceptionMessage.getClass().getMethod("getCreationTime").invoke(exceptionMessage)); + } catch (final Exception ignored) { + return OptionalLong.empty(); + } + } @Before public void setUp() { pipeTaskInfo = new PipeTaskInfo(); + creationTime = System.currentTimeMillis(); } @Test @@ -92,6 +123,39 @@ public void testRecordDataNodePushPipeMetaExceptionsKeepsUserStoppedPipeOutOfAut Assert.assertEquals(PipeStatus.STOPPED, runtimeMeta.getStatus().get()); } + @Test + public void testRecordDataNodePushPipeMetaExceptionsTargetsSameNamePipeByCreationTime() { + final String pipeName = "sameNamePipe"; + final PipeStaticMeta treePipeStaticMeta = createPipe(pipeName, PipeStatus.RUNNING, false); + final PipeStaticMeta tablePipeStaticMeta = createPipe(pipeName, PipeStatus.RUNNING, true); + + final TPushPipeMetaRespExceptionMessage probe = + new TPushPipeMetaRespExceptionMessage(pipeName, "probe", System.currentTimeMillis()); + trySetPushPipeMetaRespExceptionMessageCreationTime( + probe, tablePipeStaticMeta.getCreationTime()); + org.junit.Assume.assumeTrue( + tryGetPushPipeMetaRespExceptionMessageCreationTime(probe).isPresent()); + + Assert.assertTrue( + pipeTaskInfo.recordDataNodePushPipeMetaExceptions( + createErrorRespMap(pipeName, tablePipeStaticMeta.getCreationTime()))); + + final PipeRuntimeMeta treeRuntimeMeta = + pipeTaskInfo.getPipeMetaByPipeName(pipeName, false).getRuntimeMeta(); + final PipeRuntimeMeta tableRuntimeMeta = + pipeTaskInfo.getPipeMetaByPipeName(pipeName, true).getRuntimeMeta(); + + Assert.assertEquals(PipeStatus.RUNNING, treeRuntimeMeta.getStatus().get()); + Assert.assertFalse(treeRuntimeMeta.getIsStoppedByRuntimeException()); + Assert.assertEquals(PipeStatus.STOPPED, tableRuntimeMeta.getStatus().get()); + Assert.assertTrue(tableRuntimeMeta.getIsStoppedByRuntimeException()); + Assert.assertTrue( + tableRuntimeMeta.getNodeId2PipeRuntimeExceptionMap().containsKey(DATA_NODE_ID)); + + Assert.assertNotEquals( + treePipeStaticMeta.getCreationTime(), tablePipeStaticMeta.getCreationTime()); + } + @Test public void testEnrichOldUserPipeWithRootUserForCompatibility() { final String rootUserName = CommonDescriptor.getInstance().getConfig().getDefaultAdminName(); @@ -232,9 +296,17 @@ public void testEnrichLoadedPipeMetasWithRootUserForCompatibility() { } private Map createErrorRespMap(final String pipeName) { + return createErrorRespMap(pipeName, null); + } + + private Map createErrorRespMap( + final String pipeName, final Long creationTime) { final TPushPipeMetaRespExceptionMessage exceptionMessage = new TPushPipeMetaRespExceptionMessage( pipeName, "failed to push pipe meta", System.currentTimeMillis()); + if (creationTime != null) { + trySetPushPipeMetaRespExceptionMessageCreationTime(exceptionMessage, creationTime); + } final TPushPipeMetaResp resp = new TPushPipeMetaResp() .setStatus(new TSStatus(TSStatusCode.PIPE_PUSH_META_ERROR.getStatusCode())) @@ -242,24 +314,36 @@ private Map createErrorRespMap(final String pipeName return Collections.singletonMap(DATA_NODE_ID, resp); } - private void createPipe(final String pipeName, final PipeStatus initialStatus) { + private PipeStaticMeta createPipe(final String pipeName, final PipeStatus initialStatus) { + return createPipe(pipeName, initialStatus, false); + } + + private PipeStaticMeta createPipe( + final String pipeName, final PipeStatus initialStatus, final boolean isTableModel) { final Map extractorAttributes = new HashMap<>(); extractorAttributes.put("extractor", "iotdb-source"); - createPipeWithSourceAttributes(pipeName, extractorAttributes); + if (isTableModel) { + extractorAttributes.put( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); + } + final PipeStaticMeta pipeStaticMeta = + createPipeWithSourceAttributes(pipeName, extractorAttributes); if (PipeStatus.RUNNING.equals(initialStatus)) { - pipeTaskInfo.setPipeStatus(new SetPipeStatusPlanV2(pipeName, PipeStatus.RUNNING)); + pipeTaskInfo.setPipeStatus( + new SetPipeStatusPlanV2(pipeName, PipeStatus.RUNNING, isTableModel)); } + return pipeStaticMeta; } - private void createPipeWithSourceAttributes( + private PipeStaticMeta createPipeWithSourceAttributes( final String pipeName, final Map extractorAttributes) { final Map connectorAttributes = new HashMap<>(); connectorAttributes.put("connector", "iotdb-thrift-sink"); - createPipeWithAttributes(pipeName, extractorAttributes, connectorAttributes); + return createPipeWithAttributes(pipeName, extractorAttributes, connectorAttributes); } - private void createPipeWithAttributes( + private PipeStaticMeta createPipeWithAttributes( final String pipeName, final Map extractorAttributes, final Map connectorAttributes) { @@ -273,11 +357,12 @@ private void createPipeWithAttributes( final PipeStaticMeta pipeStaticMeta = new PipeStaticMeta( pipeName, - System.currentTimeMillis(), + ++creationTime, extractorAttributes, processorAttributes, connectorAttributes); final PipeRuntimeMeta pipeRuntimeMeta = new PipeRuntimeMeta(pipeTasks); pipeTaskInfo.createPipe(new CreatePipePlanV2(pipeStaticMeta, pipeRuntimeMeta)); + return pipeStaticMeta; } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2Test.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2Test.java index cf51ad9878005..6b574dd4e48b7 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2Test.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2Test.java @@ -53,6 +53,7 @@ public void serializeDeserializeTest() { new TAlterPipeReq("testPipe", processorAttributes, connectorAttributes, false, true); req.setExtractorAttributes(extractorAttributes); req.setIsReplaceAllExtractorAttributes(false); + req.setIsTableModel(true); AlterPipeProcedureV2 proc = new AlterPipeProcedureV2(req, ProcedureType.ALTER_PIPE_PROCEDURE_V2); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV3Test.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV3Test.java index 3e1c460f85563..1a418c56ea952 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV3Test.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV3Test.java @@ -55,6 +55,7 @@ public void serializeDeserializeTest() { new TAlterPipeReq("testPipe", processorAttributes, connectorAttributes, false, true); req.setExtractorAttributes(extractorAttributes); req.setIsReplaceAllExtractorAttributes(false); + req.setIsTableModel(true); AlterPipeProcedureV2 proc = new AlterPipeProcedureV2(req); try { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2Test.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2Test.java index ac6ca8c740c36..013376571ec2e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2Test.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2Test.java @@ -77,6 +77,26 @@ public void serializeDeserializeTest() { } } + @Test + public void serializeDeserializeWithMissingOptionalAttributesTest() throws Exception { + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + final Map connectorAttributes = new HashMap<>(); + connectorAttributes.put("connector", "iotdb-thrift-connector"); + + final CreatePipeProcedureV2 proc = + new CreatePipeProcedureV2(new TCreatePipeReq("testPipe", connectorAttributes)); + + proc.serialize(outputStream); + final ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + final CreatePipeProcedureV2 proc2 = + (CreatePipeProcedureV2) ProcedureFactory.getInstance().create(buffer); + + assertEquals(proc, proc2); + } + @Test public void testCheckAndEnrichSourceAuthenticationWithEncryptedPassword() { final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2Test.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2Test.java index 032288df93b24..e2932c12ec658 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2Test.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2Test.java @@ -36,7 +36,7 @@ public void serializeDeserializeTest() { PublicBAOS byteArrayOutputStream = new PublicBAOS(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); - DropPipeProcedureV2 proc = new DropPipeProcedureV2("testPipe"); + DropPipeProcedureV2 proc = new DropPipeProcedureV2("testPipe", true); try { proc.serialize(outputStream); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2Test.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2Test.java index 4584d75753ac0..c0f77b1ccf7a2 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2Test.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2Test.java @@ -36,7 +36,7 @@ public void serializeDeserializeTest() { PublicBAOS byteArrayOutputStream = new PublicBAOS(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); - StartPipeProcedureV2 proc = new StartPipeProcedureV2("testPipe"); + StartPipeProcedureV2 proc = new StartPipeProcedureV2("testPipe", true); try { proc.serialize(outputStream); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2Test.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2Test.java index e3da356059b5e..7d598870d31b4 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2Test.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2Test.java @@ -91,7 +91,7 @@ public void serializeDeserializeTest() { PublicBAOS byteArrayOutputStream = new PublicBAOS(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); - StopPipeProcedureV2 proc = new StopPipeProcedureV2("testPipe"); + StopPipeProcedureV2 proc = new StopPipeProcedureV2("testPipe", true); try { proc.serialize(outputStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index 033264b831fd4..de8b3f009c726 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -198,7 +198,7 @@ protected void createPipeTask( } pipeMetaKeeper - .getPipeMeta(pipeStaticMeta.getPipeName()) + .getPipeMeta(pipeStaticMeta) .getRuntimeMeta() .getConsensusGroupId2TaskMetaMap() .put(consensusGroupId, pipeTaskMeta); @@ -546,13 +546,19 @@ protected void collectPipeMetaListInternal( ///////////////////////// Terminate Logic ///////////////////////// public void markCompleted(final String pipeName, final int regionId) { + markCompleted(pipeName, 0, regionId); + } + + public void markCompleted(final String pipeName, final long creationTime, final int regionId) { acquireWriteLock(); try { - if (pipeMetaKeeper.containsPipeMeta(pipeName)) { + final PipeMeta pipeMeta = + creationTime == 0 + ? pipeMetaKeeper.getPipeMeta(pipeName) + : pipeMetaKeeper.getPipeMeta(pipeName, creationTime); + if (pipeMeta != null) { final PipeDataNodeTask pipeDataNodeTask = - ((PipeDataNodeTask) - pipeTaskManager.getPipeTask( - pipeMetaKeeper.getPipeMeta(pipeName).getStaticMeta(), regionId)); + ((PipeDataNodeTask) pipeTaskManager.getPipeTask(pipeMeta.getStaticMeta(), regionId)); if (Objects.nonNull(pipeDataNodeTask)) { pipeDataNodeTask.markCompleted(); } @@ -565,8 +571,8 @@ public void markCompleted(final String pipeName, final int regionId) { ///////////////////////// Utils ///////////////////////// public Set getPipeTaskRegionIdSet(final String pipeName, final long creationTime) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); - return pipeMeta == null || pipeMeta.getStaticMeta().getCreationTime() != creationTime + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); + return pipeMeta == null ? Collections.emptySet() : pipeMeta.getRuntimeMeta().getConsensusGroupId2TaskMetaMap().keySet(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java index 12385c52b0665..a4cb28b6203ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java @@ -172,7 +172,7 @@ public void markCompleted() { // To avoid deadlock if (shouldMark) { terminateExecutor.submit( - () -> PipeDataNodeAgent.task().markCompleted(pipeName, dataRegionId)); + () -> PipeDataNodeAgent.task().markCompleted(pipeName, creationTime, dataRegionId)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index 6d4c3580bd281..8f664761aa032 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -378,11 +378,16 @@ public long getExtractTime() { @Override public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) { extractTime = System.nanoTime(); + final String pipeTsFileResourcePipeName = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, creationTime); try { - tsFile = PipeDataNodeResourceManager.tsfile().increaseFileReference(tsFile, true, pipeName); + tsFile = + PipeDataNodeResourceManager.tsfile() + .increaseFileReference(tsFile, true, pipeTsFileResourcePipeName); if (isWithMod) { modFile = - PipeDataNodeResourceManager.tsfile().increaseFileReference(modFile, false, pipeName); + PipeDataNodeResourceManager.tsfile() + .increaseFileReference(modFile, false, pipeTsFileResourcePipeName); } return true; } catch (final Exception e) { @@ -402,10 +407,14 @@ public boolean internallyIncreaseResourceReferenceCount(final String holderMessa @Override public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) { + final String pipeTsFileResourcePipeName = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, creationTime); try { - PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile, pipeName); + PipeDataNodeResourceManager.tsfile() + .decreaseFileReference(tsFile, pipeTsFileResourcePipeName); if (isWithMod) { - PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile, pipeName); + PipeDataNodeResourceManager.tsfile() + .decreaseFileReference(modFile, pipeTsFileResourcePipeName); } close(); return true; @@ -667,7 +676,9 @@ private Set getDeviceSet() throws IOException { PipeDataNodeResourceManager.tsfile() .getDeviceIsAlignedMapFromCache( PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( - resource.getTsFile(), pipeName), + resource.getTsFile(), + PipeTsFileResourceManager.getPipeTsFileResourcePipeName( + pipeName, creationTime)), false); if (Objects.nonNull(deviceIsAlignedMap)) { return deviceIsAlignedMap.keySet(); @@ -939,6 +950,7 @@ public PipeEventResource eventResourceBuilder() { this.isReleased, this.referenceCount, this.pipeName, + this.creationTime, this.tsFile, this.isWithMod, this.modFile, @@ -954,11 +966,13 @@ private static class PipeTsFileInsertionEventResource extends PipeEventResource private final File sharedModFile; // unused now private final AtomicReference eventParser; private final String pipeName; + private final long creationTime; private PipeTsFileInsertionEventResource( final AtomicBoolean isReleased, final AtomicInteger referenceCount, final String pipeName, + final long creationTime, final File tsFile, final boolean isWithMod, final File modFile, @@ -966,6 +980,7 @@ private PipeTsFileInsertionEventResource( final AtomicReference eventParser) { super(isReleased, referenceCount); this.pipeName = pipeName; + this.creationTime = creationTime; this.tsFile = tsFile; this.isWithMod = isWithMod; this.modFile = modFile; @@ -976,10 +991,14 @@ private PipeTsFileInsertionEventResource( @Override protected void finalizeResource() { try { + final String pipeTsFileResourcePipeName = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, creationTime); // decrease reference count - PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile, pipeName); + PipeDataNodeResourceManager.tsfile() + .decreaseFileReference(tsFile, pipeTsFileResourcePipeName); if (isWithMod) { - PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile, pipeName); + PipeDataNodeResourceManager.tsfile() + .decreaseFileReference(modFile, pipeTsFileResourcePipeName); } // close event parser diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java index 540e15e29bfd8..1599fe78ebaea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; +import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; import org.apache.iotdb.db.pipe.source.dataregion.IoTDBDataRegionSource; import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; import org.apache.iotdb.metrics.AbstractMetricService; @@ -67,6 +68,9 @@ private void createMetrics(final String pipeID) { private void createAutoGauge(final String pipeID) { final PipeDataNodeRemainingEventAndTimeOperator operator = remainingEventAndTimeOperatorMap.get(pipeID); + final String pipeTsFileResourcePipeName = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName( + operator.getPipeName(), operator.getCreationTime()); metricService.createAutoGauge( Metric.PIPE_DATANODE_REMAINING_EVENT_COUNT.toString(), MetricLevel.IMPORTANT, @@ -91,7 +95,7 @@ private void createAutoGauge(final String pipeID) { Metric.PIPE_FLOATING_MEMORY_USAGE.toString(), MetricLevel.IMPORTANT, PipeDataNodeAgent.task(), - a -> a.getFloatingMemoryUsageInByte(operator.getPipeName()), + a -> a.getFloatingMemoryUsageInByte(operator.getPipeName(), operator.getCreationTime()), Tag.NAME.toString(), operator.getPipeName(), Tag.CREATION_TIME.toString(), @@ -100,7 +104,7 @@ private void createAutoGauge(final String pipeID) { Metric.PIPE_LINKED_TSFILE_COUNT.toString(), MetricLevel.IMPORTANT, PipeDataNodeResourceManager.tsfile(), - a -> a.getLinkedTsFileCount(operator.getPipeName()), + a -> a.getLinkedTsFileCount(pipeTsFileResourcePipeName), Tag.NAME.toString(), operator.getPipeName(), Tag.CREATION_TIME.toString(), @@ -109,7 +113,7 @@ private void createAutoGauge(final String pipeID) { Metric.PIPE_LINKED_TSFILE_SIZE.toString(), MetricLevel.IMPORTANT, PipeDataNodeResourceManager.tsfile(), - a -> a.getTotalLinkedTsFileSize(operator.getPipeName()), + a -> a.getTotalLinkedTsFileSize(pipeTsFileResourcePipeName), Tag.NAME.toString(), operator.getPipeName(), Tag.CREATION_TIME.toString(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java index 90cd17539f67b..d6ba6188eff0d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java @@ -56,6 +56,11 @@ public class PipeTsFileResourceManager { hardlinkOrCopiedFileToPipeTsFileResourceMap = new ConcurrentHashMap<>(); private final PipeTsFileResourceSegmentLock segmentLock = new PipeTsFileResourceSegmentLock(); + public static String getPipeTsFileResourcePipeName( + final @Nullable String pipeName, final long creationTime) { + return Objects.isNull(pipeName) ? null : pipeName + "_" + creationTime; + } + public File increaseFileReference( final File file, final boolean isTsFile, final @Nullable String pipeName) throws IOException { return increaseFileReference(file, isTsFile, pipeName, null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index 813d55e4ef508..c5dc7662bbc72 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -23,8 +23,6 @@ import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; -import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; -import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePatternOperations; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; import org.apache.iotdb.commons.pipe.source.IoTDBSource; @@ -55,7 +53,6 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; -import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.iotdb.pipe.api.exception.PipePasswordCheckException; import org.apache.iotdb.rpc.TSStatusCode; @@ -85,12 +82,9 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODE_STRICT_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_ENABLE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_MODS_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATH_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATTERN_FORMAT_IOTDB_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATTERN_FORMAT_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATTERN_FORMAT_PREFIX_VALUE; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATTERN_INCLUSION_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_PATTERN_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_REALTIME_ENABLE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_REALTIME_LOOSE_RANGE_KEY; @@ -119,10 +113,7 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_MODE_STRICT_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_MODS_ENABLE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_MODS_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_PATH_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_PATTERN_FORMAT_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_PATTERN_INCLUSION_KEY; -import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_PATTERN_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_REALTIME_ENABLE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_REALTIME_LOOSE_RANGE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_REALTIME_MODE_KEY; @@ -151,71 +142,6 @@ public class IoTDBDataRegionSource extends IoTDBSource { public void validate(final PipeParameterValidator validator) throws Exception { super.validate(validator); - final boolean isTreeDialect = - validator - .getParameters() - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); - // Validate whether the pipe needs to extract table model data or tree model data - final boolean isCaptureTree = - validator - .getParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeSourceConstant.SOURCE_CAPTURE_TREE_KEY), - isTreeDialect); - final boolean isCaptureTable = - validator - .getParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeSourceConstant.SOURCE_CAPTURE_TABLE_KEY), - !isTreeDialect); - if (!isCaptureTree && !isCaptureTable) { - throw new PipeParameterNotValidException( - DataNodePipeMessages.CAPTURE_TREE_AND_CAPTURE_TABLE_CAN_NOT); - } - - final boolean isDoubleLiving = - validator - .getParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, - PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); - final boolean isTreeModelDataAllowedToBeCaptured = isDoubleLiving || isCaptureTree; - final boolean isTableModelDataAllowedToBeCaptured = isDoubleLiving || isCaptureTable; - if (!isTreeModelDataAllowedToBeCaptured - && validator - .getParameters() - .hasAnyAttributes( - EXTRACTOR_PATH_KEY, - SOURCE_PATH_KEY, - EXTRACTOR_PATTERN_KEY, - SOURCE_PATTERN_KEY, - EXTRACTOR_PATTERN_INCLUSION_KEY, - SOURCE_PATTERN_INCLUSION_KEY)) { - throw new PipeException(DataNodePipeMessages.THE_PIPE_CANNOT_EXTRACT_TREE_MODEL_DATA); - } - if (!isTableModelDataAllowedToBeCaptured - && validator - .getParameters() - .hasAnyAttributes( - EXTRACTOR_DATABASE_NAME_KEY, - SOURCE_DATABASE_NAME_KEY, - EXTRACTOR_TABLE_NAME_KEY, - SOURCE_TABLE_NAME_KEY, - EXTRACTOR_DATABASE_KEY, - SOURCE_DATABASE_KEY, - EXTRACTOR_TABLE_KEY, - SOURCE_TABLE_KEY)) { - throw new PipeException(DataNodePipeMessages.THE_PIPE_CANNOT_EXTRACT_TABLE_MODEL_DATA); - } - final Pair insertionDeletionListeningOptionPair = DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair( validator.getParameters()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java index cb02df50a3956..36b84e1e12712 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java @@ -52,6 +52,7 @@ import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.processor.iotconsensusv2.IoTConsensusV2Processor; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; +import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter; import org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeTsFileEpochProgressIndexKeeper; import org.apache.iotdb.db.storageengine.StorageEngine; @@ -127,6 +128,7 @@ public class PipeHistoricalDataRegionTsFileAndDeletionSource private String pipeName; private long creationTime; + private String pipeNameWithCreationTime; private String tsFileDedupScopeID; private PipeTaskMeta pipeTaskMeta; @@ -314,6 +316,8 @@ public void customize( pipeName = environment.getPipeName(); creationTime = environment.getCreationTime(); + pipeNameWithCreationTime = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, creationTime); if (environment instanceof PipeTaskSourceRuntimeEnvironment) { pipeTaskMeta = ((PipeTaskSourceRuntimeEnvironment) environment).getPipeTaskMeta(); if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { @@ -615,7 +619,7 @@ private void extractTsFiles( // Will unpin it after the PipeTsFileInsertionEvent is created and pinned. try { PipeDataNodeResourceManager.tsfile() - .pinTsFileResource(resource, shouldTransferModFile, pipeName); + .pinTsFileResource(resource, shouldTransferModFile, pipeNameWithCreationTime); return false; } catch (final IOException e) { ++statistics.pinFailedCount; @@ -949,7 +953,7 @@ protected boolean consumeSkippedHistoricalTsFileEventIfNecessary(final TsFileRes } finally { try { PipeDataNodeResourceManager.tsfile() - .unpinTsFileResource(resource, shouldTransferModFile, pipeName); + .unpinTsFileResource(resource, shouldTransferModFile, pipeNameWithCreationTime); } catch (final IOException e) { LOGGER.warn( DataNodePipeMessages.PIPE_FAILED_TO_UNPIN_SKIPPED_HISTORICAL_TSFILERESOURCE, @@ -1044,7 +1048,7 @@ protected Event supplyTsFileEvent(final TsFileResource resource) { if (shouldUnpinResource) { try { PipeDataNodeResourceManager.tsfile() - .unpinTsFileResource(resource, shouldTransferModFile, pipeName); + .unpinTsFileResource(resource, shouldTransferModFile, pipeNameWithCreationTime); } catch (final IOException e) { LOGGER.warn( DataNodePipeMessages.PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_CREATING, @@ -1155,7 +1159,7 @@ public synchronized void close() { try { PipeDataNodeResourceManager.tsfile() .unpinTsFileResource( - (TsFileResource) resource, shouldTransferModFile, pipeName); + (TsFileResource) resource, shouldTransferModFile, pipeNameWithCreationTime); } catch (final IOException e) { LOGGER.warn( DataNodePipeMessages.PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_DROPPING, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionHybridSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionHybridSource.java index c219acbc69743..e3c0a032d888e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionHybridSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionHybridSource.java @@ -179,7 +179,7 @@ private void extractTsFileInsertion(final PipeRealtimeEvent event) { // tablets. private boolean canNotUseTabletAnymore(final PipeRealtimeEvent event) { final long floatingMemoryUsageInByte = - PipeDataNodeAgent.task().getFloatingMemoryUsageInByte(pipeName); + PipeDataNodeAgent.task().getFloatingMemoryUsageInByte(pipeName, creationTime); final long pipeCount = PipeDataNodeAgent.task().getPipeCount(); long totalFloatingMemorySizeInBytes = PipeDataNodeResourceManager.memory().getTotalFloatingMemorySizeInBytes(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 6eda402c2eb2c..91d3cf84ad4bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -71,6 +71,7 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.pipe.datastructure.visibility.Visibility; import org.apache.iotdb.commons.pipe.datastructure.visibility.VisibilityUtils; import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapPseudoTPipeTransferRequest; @@ -2199,14 +2200,35 @@ public SettableFuture createPipe( return future; } + final PipeParameters sourcePipeParameters = + new PipeParameters(createPipeStatement.getSourceAttributes()); + final PipeParameters sinkPipeParameters = + new PipeParameters(createPipeStatement.getSinkAttributes()); + // Validate pipe plugin before creation try { - PipeDataNodeAgent.plugin() - .validate( - pipeName, - createPipeStatement.getSourceAttributes(), - createPipeStatement.getProcessorAttributes(), - createPipeStatement.getSinkAttributes()); + if (isDoubleLivingPipe(sourcePipeParameters)) { + validatePipePlugin( + pipeName, + cloneSourceParametersWithDialect( + sourcePipeParameters, SystemConstant.SQL_DIALECT_TREE_VALUE) + .getAttribute(), + createPipeStatement.getProcessorAttributes(), + createPipeStatement.getSinkAttributes()); + validatePipePlugin( + pipeName, + cloneSourceParametersWithDialect( + sourcePipeParameters, SystemConstant.SQL_DIALECT_TABLE_VALUE) + .getAttribute(), + createPipeStatement.getProcessorAttributes(), + createPipeStatement.getSinkAttributes()); + } else { + validatePipePlugin( + pipeName, + createPipeStatement.getSourceAttributes(), + createPipeStatement.getProcessorAttributes(), + createPipeStatement.getSinkAttributes()); + } } catch (final Exception e) { future.setException( new IoTDBException( @@ -2219,118 +2241,15 @@ public SettableFuture createPipe( // Syntactic sugar: if full-sync mode is detected (i.e. not snapshot mode, or both realtime // and history are true), the pipe is split into history-only and realtime–only modes. - final PipeParameters sourcePipeParameters = - new PipeParameters(createPipeStatement.getSourceAttributes()); - final PipeParameters sinkPipeParameters = - new PipeParameters(createPipeStatement.getSinkAttributes()); try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - if (PipeConfig.getInstance().getPipeAutoSplitFullEnabled() - && PipeDataNodeAgent.task().isFullSync(sourcePipeParameters)) { - // 1. Send request to create the real-time data synchronization pipeline - final TCreatePipeReq realtimeReq = - new TCreatePipeReq() - // Append suffix to the pipeline name for real-time data - .setPipeName(pipeName + "_realtime") - // NOTE: set if not exists always to true to handle partial failure - .setIfNotExistsCondition(true) - // Use extractor parameters for real-time data - .setExtractorAttributes( - sourcePipeParameters - .addOrReplaceEquivalentAttributesWithClone( - new PipeParameters( - ImmutableMap.of( - PipeSourceConstant.EXTRACTOR_REALTIME_ENABLE_KEY, - Boolean.toString(true), - PipeSourceConstant.EXTRACTOR_HISTORY_ENABLE_KEY, - Boolean.toString(false)))) - .getAttribute()) - .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) - .setConnectorAttributes(createPipeStatement.getSinkAttributes()); - - final TSStatus realtimeTsStatus = configNodeClient.createPipe(realtimeReq); - // If creation fails, immediately return with exception - // If the procedure is still running, it's probably stuck on DataNode - // The pipe creation can ignore this situation and succeed, thus we do not need to skip in - // this case - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != realtimeTsStatus.getCode() - && TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() - != realtimeTsStatus.getCode()) { - future.setException(new IoTDBException(realtimeTsStatus)); - return future; - } - - // 2. Send request to create the historical data synchronization pipeline - final Map historySinkAttributes = - sinkPipeParameters.hasAnyAttributes( - PipeSinkConstant.SINK_ENABLE_SEND_TSFILE_LIMIT, - PipeSinkConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT) - ? createPipeStatement.getSinkAttributes() - : sinkPipeParameters - .addOrReplaceEquivalentAttributesWithClone( - new PipeParameters( - Collections.singletonMap( - PipeSinkConstant.SINK_ENABLE_SEND_TSFILE_LIMIT, - Boolean.TRUE.toString()))) - .getAttribute(); - - final TCreatePipeReq historyReq = - new TCreatePipeReq() - // Append suffix to the pipeline name for historical data - .setPipeName(pipeName + "_history") - .setIfNotExistsCondition(createPipeStatement.hasIfNotExistsCondition()) - // Use source parameters for historical data - .setExtractorAttributes( - sourcePipeParameters - .addOrReplaceEquivalentAttributesWithClone( - new PipeParameters( - ImmutableMap.of( - PipeSourceConstant.EXTRACTOR_REALTIME_ENABLE_KEY, - Boolean.toString(false), - PipeSourceConstant.EXTRACTOR_HISTORY_ENABLE_KEY, - Boolean.toString(true), - PipeSourceConstant.EXTRACTOR_MODE_KEY, - PipeSourceConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE, - // We force the historical pipe to transfer data (and maybe - // deletion) only - // Thus we can transfer schema only once - // And may drop the historical pipe on successfully transferred - PipeSourceConstant.SOURCE_INCLUSION_KEY, - DataRegionListeningFilter - .parseInsertionDeletionListeningOptionPair( - sourcePipeParameters) - .getRight() - ? "data" - : PipeSourceConstant.EXTRACTOR_INCLUSION_DEFAULT_VALUE, - PipeSourceConstant.SOURCE_EXCLUSION_KEY, - PipeSourceConstant.EXTRACTOR_EXCLUSION_DEFAULT_VALUE))) - .getAttribute()) - .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) - .setConnectorAttributes(historySinkAttributes); - - final TSStatus historyTsStatus = configNodeClient.createPipe(historyReq); - // If creation fails, immediately return with exception - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != historyTsStatus.getCode()) { - future.setException(new IoTDBException(historyTsStatus)); - return future; - } - - // 3. Set success status only if both pipelines are created successfully - future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + final TSStatus tsStatus = + createPipeInternal( + configNodeClient, createPipeStatement, sourcePipeParameters, sinkPipeParameters); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { + future.setException(new IoTDBException(tsStatus)); } else { - final TCreatePipeReq req = - new TCreatePipeReq() - .setPipeName(pipeName) - .setIfNotExistsCondition(createPipeStatement.hasIfNotExistsCondition()) - .setExtractorAttributes(createPipeStatement.getSourceAttributes()) - .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) - .setConnectorAttributes(createPipeStatement.getSinkAttributes()); - final TSStatus tsStatus = configNodeClient.createPipe(req); - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { - future.setException(new IoTDBException(tsStatus)); - } else { - future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); - } + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); } } catch (final Exception e) { future.setException(e); @@ -2338,6 +2257,156 @@ public SettableFuture createPipe( return future; } + private void validatePipePlugin( + final String pipeName, + final Map sourceAttributes, + final Map processorAttributes, + final Map sinkAttributes) + throws Exception { + PipeDataNodeAgent.plugin() + .validate( + pipeName, + cloneAttributes(sourceAttributes), + cloneAttributes(processorAttributes), + cloneAttributes(sinkAttributes)); + } + + private boolean isDoubleLivingPipe(final PipeParameters sourcePipeParameters) { + return sourcePipeParameters.getBooleanOrDefault( + Arrays.asList( + PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + } + + private PipeParameters cloneSourceParametersWithDialect( + final PipeParameters sourcePipeParameters, final String sqlDialect) { + final Map sourceAttributes = new HashMap<>(sourcePipeParameters.getAttribute()); + sourceAttributes.put(SystemConstant.SQL_DIALECT_KEY, sqlDialect); + return new PipeParameters(sourceAttributes); + } + + private Map cloneAttributes(final Map attributes) { + return new HashMap<>(attributes == null ? Collections.emptyMap() : attributes); + } + + private TSStatus createPipeInternal( + final ConfigNodeClient configNodeClient, + final CreatePipeStatement createPipeStatement, + final PipeParameters sourcePipeParameters, + final PipeParameters sinkPipeParameters) + throws TException, IllegalPathException { + final String pipeName = createPipeStatement.getPipeName(); + if (PipeConfig.getInstance().getPipeAutoSplitFullEnabled() + && PipeDataNodeAgent.task().isFullSync(sourcePipeParameters)) { + // 1. Send request to create the real-time data synchronization pipeline + final TCreatePipeReq realtimeReq = + new TCreatePipeReq() + // Append suffix to the pipeline name for real-time data + .setPipeName(pipeName + "_realtime") + // NOTE: set if not exists always to true to handle partial failure + .setIfNotExistsCondition(true) + // Use extractor parameters for real-time data + .setExtractorAttributes( + sourcePipeParameters + .addOrReplaceEquivalentAttributesWithClone( + new PipeParameters( + ImmutableMap.of( + SystemConstant.SQL_DIALECT_KEY, + sourcePipeParameters.getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, + SystemConstant.SQL_DIALECT_TREE_VALUE), + PipeSourceConstant.EXTRACTOR_REALTIME_ENABLE_KEY, + Boolean.toString(true), + PipeSourceConstant.EXTRACTOR_HISTORY_ENABLE_KEY, + Boolean.toString(false)))) + .getAttribute()) + .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) + .setConnectorAttributes(createPipeStatement.getSinkAttributes()); + + final TSStatus realtimeTsStatus = configNodeClient.createPipe(realtimeReq); + // If creation fails, immediately return with exception + // If the procedure is still running, it's probably stuck on DataNode + // The pipe creation can ignore this situation and succeed, thus we do not need to skip in + // this case + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != realtimeTsStatus.getCode() + && TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() + != realtimeTsStatus.getCode()) { + return realtimeTsStatus; + } + + // 2. Send request to create the historical data synchronization pipeline + final Map historySinkAttributes = + sinkPipeParameters.hasAnyAttributes( + PipeSinkConstant.SINK_ENABLE_SEND_TSFILE_LIMIT, + PipeSinkConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT) + ? createPipeStatement.getSinkAttributes() + : sinkPipeParameters + .addOrReplaceEquivalentAttributesWithClone( + new PipeParameters( + Collections.singletonMap( + PipeSinkConstant.SINK_ENABLE_SEND_TSFILE_LIMIT, + Boolean.TRUE.toString()))) + .getAttribute(); + + final TCreatePipeReq historyReq = + new TCreatePipeReq() + // Append suffix to the pipeline name for historical data + .setPipeName(pipeName + "_history") + .setIfNotExistsCondition(createPipeStatement.hasIfNotExistsCondition()) + // Use source parameters for historical data + .setExtractorAttributes( + sourcePipeParameters + .addOrReplaceEquivalentAttributesWithClone( + new PipeParameters( + ImmutableMap.of( + SystemConstant.SQL_DIALECT_KEY, + sourcePipeParameters.getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, + SystemConstant.SQL_DIALECT_TREE_VALUE), + PipeSourceConstant.EXTRACTOR_REALTIME_ENABLE_KEY, + Boolean.toString(false), + PipeSourceConstant.EXTRACTOR_HISTORY_ENABLE_KEY, + Boolean.toString(true), + PipeSourceConstant.EXTRACTOR_MODE_KEY, + PipeSourceConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE, + // We force the historical pipe to transfer data (and maybe + // deletion) only + // Thus we can transfer schema only once + // And may drop the historical pipe on successfully transferred + PipeSourceConstant.SOURCE_INCLUSION_KEY, + DataRegionListeningFilter + .parseInsertionDeletionListeningOptionPair( + sourcePipeParameters) + .getRight() + ? "data" + : PipeSourceConstant.EXTRACTOR_INCLUSION_DEFAULT_VALUE, + PipeSourceConstant.SOURCE_EXCLUSION_KEY, + PipeSourceConstant.EXTRACTOR_EXCLUSION_DEFAULT_VALUE))) + .getAttribute()) + .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) + .setConnectorAttributes(historySinkAttributes); + + final TSStatus historyTsStatus = configNodeClient.createPipe(historyReq); + // If creation fails, immediately return with exception + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != historyTsStatus.getCode()) { + return historyTsStatus; + } + + // 3. Set success status only if both pipelines are created successfully + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + + final TCreatePipeReq req = + new TCreatePipeReq() + .setPipeName(pipeName) + .setIfNotExistsCondition(createPipeStatement.hasIfNotExistsCondition()) + .setExtractorAttributes(sourcePipeParameters.getAttribute()) + .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) + .setConnectorAttributes(createPipeStatement.getSinkAttributes()); + return configNodeClient.createPipe(req); + } + @Override public SettableFuture alterPipe(final AlterPipeStatement alterPipeStatement) { final SettableFuture future = SettableFuture.create(); @@ -2374,9 +2443,12 @@ public SettableFuture alterPipe(final AlterPipeStatement alter .filter( pipeMeta -> pipeMeta - .getStaticMeta() - .getPipeName() - .equals(alterPipeStatement.getPipeName())) + .getStaticMeta() + .getPipeName() + .equals(alterPipeStatement.getPipeName()) + && pipeMeta + .getStaticMeta() + .visibleUnder(alterPipeStatement.isTableModel())) .findFirst() .orElse(null); if (pipeMetaFromCoordinator == null) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index c30a50ac4958a..b2da23ae0b489 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -49,6 +49,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -59,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -161,6 +163,39 @@ private long convertMsToCeilSeconds(final long timeOutInMs) { return Math.max(1L, (Math.max(0L, timeOutInMs) + 999L) / 1000L); } + public static boolean trySetPushPipeMetaRespExceptionMessageCreationTime( + final TPushPipeMetaRespExceptionMessage exceptionMessage, final long creationTime) { + try { + exceptionMessage + .getClass() + .getMethod("setCreationTime", long.class) + .invoke(exceptionMessage, creationTime); + return true; + } catch (final IllegalAccessException + | InvocationTargetException + | NoSuchMethodException + | SecurityException e) { + return false; + } + } + + public static OptionalLong tryGetPushPipeMetaRespExceptionMessageCreationTime( + final TPushPipeMetaRespExceptionMessage exceptionMessage) { + try { + if (!Boolean.TRUE.equals( + exceptionMessage.getClass().getMethod("isSetCreationTime").invoke(exceptionMessage))) { + return OptionalLong.empty(); + } + return OptionalLong.of( + (long) exceptionMessage.getClass().getMethod("getCreationTime").invoke(exceptionMessage)); + } catch (final IllegalAccessException + | InvocationTargetException + | NoSuchMethodException + | SecurityException e) { + return OptionalLong.empty(); + } + } + ////////////////////////// Pipe Task Management Entry ////////////////////////// public TPushPipeMetaRespExceptionMessage handleSinglePipeMetaChanges( @@ -191,8 +226,11 @@ protected TPushPipeMetaRespExceptionMessage handleSinglePipeMetaChangesInternal( pipeName, e.getMessage()); LOGGER.warn(PipeMessages.FAILED_TO_HANDLE_SINGLE_PIPE_META_CHANGES, pipeName, e); - return new TPushPipeMetaRespExceptionMessage( - pipeName, errorMessage, System.currentTimeMillis()); + final TPushPipeMetaRespExceptionMessage exceptionMessage = + new TPushPipeMetaRespExceptionMessage(pipeName, errorMessage, System.currentTimeMillis()); + trySetPushPipeMetaRespExceptionMessageCreationTime( + exceptionMessage, pipeMetaFromCoordinator.getStaticMeta().getCreationTime()); + return exceptionMessage; } } @@ -201,6 +239,7 @@ protected TPushPipeMetaRespExceptionMessage handleSinglePipeMetaChangesInternal( private void executeSinglePipeMetaChanges(final PipeMeta metaFromCoordinator) throws IllegalPathException { final String pipeName = metaFromCoordinator.getStaticMeta().getPipeName(); + final long creationTime = metaFromCoordinator.getStaticMeta().getCreationTime(); // Do nothing with the subscription pipe if disable subscription if (PipeStaticMeta.isSubscriptionPipe(pipeName) @@ -208,13 +247,18 @@ private void executeSinglePipeMetaChanges(final PipeMeta metaFromCoordinator) return; } - final PipeMeta metaInAgent = pipeMetaKeeper.getPipeMeta(pipeName); + if (metaFromCoordinator.getRuntimeMeta().getStatus().get() == PipeStatus.DROPPED) { + dropPipe(pipeName, creationTime); + return; + } + + final PipeMeta metaInAgent = pipeMetaKeeper.getPipeMeta(metaFromCoordinator.getStaticMeta()); // If pipe meta does not exist on local agent, create a new pipe if (metaInAgent == null) { if (createPipe(metaFromCoordinator)) { // If the status recorded in coordinator is RUNNING, start the pipe - startPipe(pipeName, metaFromCoordinator.getStaticMeta().getCreationTime()); + startPipe(pipeName, creationTime); } // If the status recorded in coordinator is STOPPED or DROPPED, do nothing return; @@ -226,7 +270,7 @@ private void executeSinglePipeMetaChanges(final PipeMeta metaFromCoordinator) // First check if pipe static meta has changed, if so, drop the pipe and create a new one if (!staticMetaInAgent.equals(staticMetaFromCoordinator)) { - dropPipe(pipeName); + dropPipe(pipeName, staticMetaInAgent.getCreationTime()); if (createPipe(metaFromCoordinator)) { startPipe(pipeName, metaFromCoordinator.getStaticMeta().getCreationTime()); } @@ -425,9 +469,12 @@ protected List handlePipeMetaChangesInternal( PipeMessages.FAILED_TO_HANDLE_PIPE_META_CHANGES_FORMAT, pipeName, e.getMessage()); PipeLogger.log( LOGGER::warn, e, PipeMessages.FAILED_TO_HANDLE_PIPE_META_CHANGES_LOG, pipeName); - exceptionMessages.add( + final TPushPipeMetaRespExceptionMessage exceptionMessage = new TPushPipeMetaRespExceptionMessage( - pipeName, errorMessage, System.currentTimeMillis())); + pipeName, errorMessage, System.currentTimeMillis()); + trySetPushPipeMetaRespExceptionMessageCreationTime( + exceptionMessage, metaFromCoordinator.getStaticMeta().getCreationTime()); + exceptionMessages.add(exceptionMessage); } } // If the number of successful changes to pipe meta is 0, it means that the failure has @@ -438,16 +485,18 @@ protected List handlePipeMetaChangesInternal( } // Check if there are pipes on local agent that do not exist on coordinator, if so, drop them - final Set pipeNamesFromCoordinator = + final Set pipeStaticMetasFromCoordinator = pipeMetaListFromCoordinator.stream() - .map(meta -> meta.getStaticMeta().getPipeName()) + .map(PipeMeta::getStaticMeta) .collect(Collectors.toSet()); for (final PipeMeta metaInAgent : pipeMetaKeeper.getPipeMetaList()) { final String pipeName = metaInAgent.getStaticMeta().getPipeName(); try { - if (!pipeNamesFromCoordinator.contains(pipeName)) { - dropPipe(metaInAgent.getStaticMeta().getPipeName()); + if (!pipeStaticMetasFromCoordinator.contains(metaInAgent.getStaticMeta())) { + dropPipe( + metaInAgent.getStaticMeta().getPipeName(), + metaInAgent.getStaticMeta().getCreationTime()); } } catch (final Exception e) { // Report the exception message for CN to sense the failure of meta sync @@ -456,9 +505,12 @@ protected List handlePipeMetaChangesInternal( PipeMessages.FAILED_TO_HANDLE_PIPE_META_CHANGES_FORMAT, pipeName, e.getMessage()); PipeLogger.log( LOGGER::warn, e, PipeMessages.FAILED_TO_HANDLE_PIPE_META_CHANGES_LOG, pipeName); - exceptionMessages.add( + final TPushPipeMetaRespExceptionMessage exceptionMessage = new TPushPipeMetaRespExceptionMessage( - pipeName, errorMessage, System.currentTimeMillis())); + pipeName, errorMessage, System.currentTimeMillis()); + trySetPushPipeMetaRespExceptionMessageCreationTime( + exceptionMessage, metaInAgent.getStaticMeta().getCreationTime()); + exceptionMessages.add(exceptionMessage); } } @@ -514,7 +566,8 @@ protected boolean createPipe(final PipeMeta pipeMetaFromCoordinator) throws Ille pipeMetaFromCoordinator.getStaticMeta().getProcessorParameters(), pipeMetaFromCoordinator.getStaticMeta().getSinkParameters()); - final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + final PipeMeta existedPipeMeta = + pipeMetaKeeper.getOverlappedPipeMeta(pipeMetaFromCoordinator.getStaticMeta()); if (existedPipeMeta != null) { if (!checkBeforeCreatePipe(existedPipeMeta, pipeName, creationTime)) { return false; @@ -567,7 +620,7 @@ protected abstract Map buildPipeTasks(final PipeMeta pipeMeta * @return {@code true} if a pipe has indeed been dropped, otherwise {@code false}. */ protected boolean dropPipe(final String pipeName, final long creationTime) { - final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); if (!checkBeforeDropPipe(existedPipeMeta, pipeName, creationTime)) { return false; @@ -592,7 +645,7 @@ protected boolean dropPipe(final String pipeName, final long creationTime) { LOGGER.info(PipeMessages.DROP_ALL_PIPE_TASKS, pipeName, System.currentTimeMillis() - startTime); // Remove pipe meta from pipe meta keeper - pipeMetaKeeper.removePipeMeta(pipeName); + pipeMetaKeeper.removePipeMeta(existedPipeMeta.getStaticMeta()); return true; } @@ -626,13 +679,13 @@ protected boolean dropPipe(final String pipeName) { LOGGER.info(PipeMessages.DROP_ALL_PIPE_TASKS, pipeName, System.currentTimeMillis() - startTime); // Remove pipe meta from pipe meta keeper - pipeMetaKeeper.removePipeMeta(pipeName); + pipeMetaKeeper.removePipeMeta(existedPipeMeta.getStaticMeta()); return true; } protected void startPipe(final String pipeName, final long creationTime) { - final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); if (!checkBeforeStartPipe(existedPipeMeta, pipeName, creationTime)) { return; @@ -665,7 +718,7 @@ protected void startPipe(final String pipeName, final long creationTime) { } private void stopPipe(final String pipeName, final long creationTime) { - final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + final PipeMeta existedPipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); if (!checkBeforeStopPipe(existedPipeMeta, pipeName, creationTime)) { return; @@ -887,7 +940,7 @@ protected abstract void createPipeTask( private void dropPipeTask(final int consensusGroupId, final PipeStaticMeta pipeStaticMeta) { pipeMetaKeeper - .getPipeMeta(pipeStaticMeta.getPipeName()) + .getPipeMeta(pipeStaticMeta) .getRuntimeMeta() .getConsensusGroupId2TaskMetaMap() .remove(consensusGroupId); @@ -1095,8 +1148,12 @@ public long getPipeCreationTime(final String pipeName) { return pipeMeta == null ? 0 : pipeMeta.getStaticMeta().getCreationTime(); } + public boolean isPipeExisted(final String pipeName, final long creationTime) { + return pipeMetaKeeper.getPipeMeta(pipeName, creationTime) != null; + } + public String getPipeNameWithCreationTime(final String pipeName, final long creationTime) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); return pipeMeta == null ? pipeName + "_" + creationTime : ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()).getPipeNameWithCreationTime(); @@ -1104,7 +1161,7 @@ public String getPipeNameWithCreationTime(final String pipeName, final long crea public CommitterKey getCommitterKey( final String pipeName, final long creationTime, final int regionId, final int restartTime) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); return pipeMeta == null ? new CommitterKey(pipeName, creationTime, regionId, restartTime) : ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()) @@ -1130,9 +1187,16 @@ public long getFloatingMemoryUsageInByte(final String pipeName) { : ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()).getFloatingMemoryUsageInByte(); } + public long getFloatingMemoryUsageInByte(final String pipeName, final long creationTime) { + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); + return pipeMeta == null + ? 0 + : ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()).getFloatingMemoryUsageInByte(); + } + public void addFloatingMemoryUsageInByte( final String pipeName, final long creationTime, final long sizeInByte) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); // To avoid stale pipe before alter if (Objects.nonNull(pipeMeta) && pipeMeta.getStaticMeta().getCreationTime() == creationTime) { ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()) @@ -1142,7 +1206,7 @@ public void addFloatingMemoryUsageInByte( public void decreaseFloatingMemoryUsageInByte( final String pipeName, final long creationTime, final long sizeInByte) { - final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName); + final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName, creationTime); // To avoid stale pipe before alter if (Objects.nonNull(pipeMeta) && pipeMeta.getStaticMeta().getCreationTime() == creationTime) { ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta()) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java index 828cb69afa9fb..51e7d89bb245c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java @@ -19,6 +19,9 @@ package org.apache.iotdb.commons.pipe.agent.task.meta; +import org.apache.iotdb.commons.pipe.datastructure.visibility.Visibility; +import org.apache.iotdb.commons.pipe.datastructure.visibility.VisibilityUtils; + import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.FileInputStream; @@ -26,6 +29,7 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -70,27 +74,71 @@ public void releaseWriteLock() { ///////////////////////////////// PipeMeta ///////////////////////////////// public void addPipeMeta(PipeMeta pipeMeta) { - pipeNameToPipeMetaMap.put(pipeMeta.getStaticMeta().getPipeName(), pipeMeta); + pipeNameToPipeMetaMap.put(generatePipeMetaKeeperKey(pipeMeta.getStaticMeta()), pipeMeta); } public PipeMeta getPipeMeta(String pipeName) { - return pipeNameToPipeMetaMap.get(pipeName); + return getPipeMetaOptional(pipeName).orElse(null); + } + + public PipeMeta getPipeMeta(String pipeName, boolean isTableModel) { + return getPipeMetaOptional(pipeName, isTableModel).orElse(null); + } + + public PipeMeta getPipeMeta(PipeStaticMeta pipeStaticMeta) { + return pipeNameToPipeMetaMap.get(generatePipeMetaKeeperKey(pipeStaticMeta)); + } + + public PipeMeta getOverlappedPipeMeta(PipeStaticMeta pipeStaticMeta) { + return pipeNameToPipeMetaMap.values().stream() + .filter( + pipeMeta -> + pipeMeta.getStaticMeta().getPipeName().equals(pipeStaticMeta.getPipeName()) + && isVisibilityOverlapped(pipeMeta.getStaticMeta(), pipeStaticMeta)) + .findFirst() + .orElse(null); + } + + public PipeMeta getPipeMeta(String pipeName, long creationTime) { + return getPipeMetaOptional(pipeName, creationTime).orElse(null); } public void removePipeMeta(String pipeName) { - pipeNameToPipeMetaMap.remove(pipeName); + pipeMetaKeeperKeyOptional(pipeName).ifPresent(pipeNameToPipeMetaMap::remove); + } + + public void removePipeMeta(String pipeName, boolean isTableModel) { + pipeMetaKeeperKeyOptional(pipeName, isTableModel).ifPresent(pipeNameToPipeMetaMap::remove); + } + + public void removePipeMeta(PipeStaticMeta pipeStaticMeta) { + pipeNameToPipeMetaMap.remove(generatePipeMetaKeeperKey(pipeStaticMeta)); } public boolean containsPipeMeta(String pipeName) { - return pipeNameToPipeMetaMap.containsKey(pipeName); + return pipeMetaKeeperKeyOptional(pipeName).isPresent(); } public boolean containsPipeMeta(String pipeName, boolean isTableModel) { - final PipeMeta pipeMeta = pipeNameToPipeMetaMap.get(pipeName); - if (Objects.isNull(pipeMeta)) { - return false; - } - return pipeMeta.getStaticMeta().visibleUnder(isTableModel); + return pipeMetaKeeperKeyOptional(pipeName, isTableModel).isPresent(); + } + + public boolean containsPipeMeta(PipeStaticMeta pipeStaticMeta) { + return pipeNameToPipeMetaMap.containsKey(generatePipeMetaKeeperKey(pipeStaticMeta)); + } + + public boolean containsPipeMetaOverlapped(PipeStaticMeta pipeStaticMeta) { + return containsPipeMetaOverlapped(pipeStaticMeta, null); + } + + public boolean containsPipeMetaOverlapped( + PipeStaticMeta pipeStaticMeta, PipeStaticMeta excludedPipeStaticMeta) { + return pipeNameToPipeMetaMap.values().stream() + .anyMatch( + pipeMeta -> + pipeMeta.getStaticMeta().getPipeName().equals(pipeStaticMeta.getPipeName()) + && !pipeMeta.getStaticMeta().equals(excludedPipeStaticMeta) + && isVisibilityOverlapped(pipeMeta.getStaticMeta(), pipeStaticMeta)); } public Iterable getPipeMetaList() { @@ -102,7 +150,11 @@ public int getPipeMetaCount() { } public PipeMeta getPipeMetaByPipeName(String pipeName) { - return pipeNameToPipeMetaMap.get(pipeName); + return getPipeMeta(pipeName); + } + + public PipeMeta getPipeMetaByPipeName(String pipeName, boolean isTableModel) { + return getPipeMeta(pipeName, isTableModel); } public void clear() { @@ -118,7 +170,7 @@ public boolean isEmpty() { public void processTakeSnapshot(FileOutputStream fileOutputStream) throws IOException { ReadWriteIOUtils.write(pipeNameToPipeMetaMap.size(), fileOutputStream); for (Map.Entry entry : pipeNameToPipeMetaMap.entrySet()) { - ReadWriteIOUtils.write(entry.getKey(), fileOutputStream); + ReadWriteIOUtils.write(entry.getValue().getStaticMeta().getPipeName(), fileOutputStream); entry.getValue().serialize(fileOutputStream); } } @@ -128,8 +180,9 @@ public void processLoadSnapshot(FileInputStream fileInputStream) throws IOExcept final int size = ReadWriteIOUtils.readInt(fileInputStream); for (int i = 0; i < size; i++) { - final String pipeName = ReadWriteIOUtils.readString(fileInputStream); - pipeNameToPipeMetaMap.put(pipeName, PipeMeta.deserialize(fileInputStream)); + ReadWriteIOUtils.readString(fileInputStream); + final PipeMeta pipeMeta = PipeMeta.deserialize(fileInputStream); + pipeNameToPipeMetaMap.put(generatePipeMetaKeeperKey(pipeMeta.getStaticMeta()), pipeMeta); } } @@ -188,4 +241,98 @@ public long exceptionStoppedPipeCount() { && pipeMeta.getRuntimeMeta().getIsStoppedByRuntimeException()) .count(); } + + ///////////////////////////////// Tree & Table Isolation ///////////////////////////////// + + private Optional getPipeMetaOptional(String pipeName) { + return pipeMetaKeeperKeyOptional(pipeName).map(pipeNameToPipeMetaMap::get); + } + + private Optional getPipeMetaOptional(String pipeName, boolean isTableModel) { + return pipeMetaKeeperKeyOptional(pipeName, isTableModel).map(pipeNameToPipeMetaMap::get); + } + + private Optional getPipeMetaOptional(String pipeName, long creationTime) { + final PipeMeta pipeMeta = pipeNameToPipeMetaMap.get(pipeName); + if (Objects.nonNull(pipeMeta) && pipeMeta.getStaticMeta().getCreationTime() == creationTime) { + return Optional.of(pipeMeta); + } + + return pipeNameToPipeMetaMap.values().stream() + .filter( + meta -> + meta.getStaticMeta().getPipeName().equals(pipeName) + && meta.getStaticMeta().getCreationTime() == creationTime) + .findFirst(); + } + + private Optional pipeMetaKeeperKeyOptional(String pipeName, boolean isTableModel) { + final String modelKey = generatePipeMetaKeeperKey(pipeName, isTableModel); + if (pipeNameToPipeMetaMap.containsKey(modelKey)) { + return Optional.of(modelKey); + } + + final String bothModelKey = generatePipeMetaKeeperKeyForVisibility(pipeName, Visibility.BOTH); + return pipeNameToPipeMetaMap.containsKey(bothModelKey) + ? Optional.of(bothModelKey) + : Optional.empty(); + } + + private Optional pipeMetaKeeperKeyOptional(String pipeName) { + final String treeModelKey = generatePipeMetaKeeperKey(pipeName, false); + if (pipeNameToPipeMetaMap.containsKey(treeModelKey)) { + return Optional.of(treeModelKey); + } + + final String bothModelKey = generatePipeMetaKeeperKeyForVisibility(pipeName, Visibility.BOTH); + if (pipeNameToPipeMetaMap.containsKey(bothModelKey)) { + return Optional.of(bothModelKey); + } + + final String tableModelKey = generatePipeMetaKeeperKey(pipeName, true); + if (pipeNameToPipeMetaMap.containsKey(tableModelKey)) { + return Optional.of(tableModelKey); + } + + final String noneModelKey = generatePipeMetaKeeperKeyForVisibility(pipeName, Visibility.NONE); + return pipeNameToPipeMetaMap.containsKey(noneModelKey) + ? Optional.of(noneModelKey) + : Optional.empty(); + } + + private static String generatePipeMetaKeeperKey(final PipeStaticMeta pipeStaticMeta) { + final String pipeName = pipeStaticMeta.getPipeName(); + final Visibility visibility = + VisibilityUtils.calculateFromExtractorParameters(pipeStaticMeta.getSourceParameters()); + if (Objects.equals(Visibility.TREE_ONLY, visibility)) { + return generatePipeMetaKeeperKey(pipeName, false); + } + if (Objects.equals(Visibility.TABLE_ONLY, visibility)) { + return generatePipeMetaKeeperKey(pipeName, true); + } + if (Objects.equals(Visibility.NONE, visibility)) { + return generatePipeMetaKeeperKeyForVisibility(pipeName, Visibility.NONE); + } + return generatePipeMetaKeeperKeyForVisibility(pipeName, Visibility.BOTH); + } + + private static String generatePipeMetaKeeperKey( + final String pipeName, final boolean isTableModel) { + return (isTableModel ? "table:" : "tree:") + pipeName.length() + ":" + pipeName; + } + + private static String generatePipeMetaKeeperKeyForVisibility( + final String pipeName, final Visibility visibility) { + return visibility.name().toLowerCase() + ":" + pipeName.length() + ":" + pipeName; + } + + private static boolean isVisibilityOverlapped( + final PipeStaticMeta leftPipeStaticMeta, final PipeStaticMeta rightPipeStaticMeta) { + final Visibility leftVisibility = + VisibilityUtils.calculateFromExtractorParameters(leftPipeStaticMeta.getSourceParameters()); + final Visibility rightVisibility = + VisibilityUtils.calculateFromExtractorParameters(rightPipeStaticMeta.getSourceParameters()); + return VisibilityUtils.isCompatible(leftVisibility, rightVisibility) + || VisibilityUtils.isCompatible(rightVisibility, leftVisibility); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeStaticMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeStaticMeta.java index 9855552113c67..f798266e7d281 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeStaticMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeStaticMeta.java @@ -309,4 +309,8 @@ public boolean visibleUnder(final boolean isTableModel) { VisibilityUtils.calculateFromExtractorParameters(sourceParameters); return VisibilityUtils.isCompatible(visibility, isTableModel); } + + public boolean visibleUnderTableModel() { + return visibleUnder(true); + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java index d9b134b9acb76..bd950cd82ccb9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java @@ -20,7 +20,6 @@ package org.apache.iotdb.commons.pipe.datastructure.pattern; import org.apache.iotdb.commons.i18n.PipeMessages; -import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.exception.PipeException; @@ -134,19 +133,9 @@ public static TablePattern parsePipePatternFromSourceParameters( } public static boolean isTableModelDataAllowToBeCaptured(final PipeParameters sourceParameters) { - return sourceParameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, - PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE) - || sourceParameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeSourceConstant.SOURCE_CAPTURE_TABLE_KEY), - !sourceParameters - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); + return sourceParameters + .getStringOrDefault(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TABLE_VALUE); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java index 48c4c64abaa30..2da5f838227c7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.i18n.PipeMessages; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; -import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; @@ -791,19 +790,9 @@ private static int findMatchingParenthesis(final String text, final int openPare } public static boolean isTreeModelDataAllowToBeCaptured(final PipeParameters sourceParameters) { - return sourceParameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, - PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE) - || sourceParameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeSourceConstant.SOURCE_CAPTURE_TREE_KEY), - sourceParameters - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); + return sourceParameters + .getStringOrDefault(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); } /** diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/visibility/VisibilityUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/visibility/VisibilityUtils.java index 235a22e23adb2..d09941148194e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/visibility/VisibilityUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/visibility/VisibilityUtils.java @@ -19,23 +19,17 @@ package org.apache.iotdb.commons.pipe.datastructure.visibility; -import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.pipe.api.annotation.TableModel; import org.apache.iotdb.pipe.api.annotation.TreeModel; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.rpc.subscription.config.TopicConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.Arrays; import java.util.Objects; public class VisibilityUtils { - private static final Logger LOGGER = LoggerFactory.getLogger(VisibilityUtils.class); - private VisibilityUtils() { // forbidding instantiation } @@ -86,50 +80,12 @@ public static Visibility calculateFromPluginClass(final Class pipePluginClass public static Visibility calculateFromExtractorParameters( final PipeParameters extractorParameters) { - // visible under all model when 'mode.double-living' is set to true - final boolean isDoubleLiving = - extractorParameters.getBooleanOrDefault( - Arrays.asList( - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, - PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), - PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); - if (isDoubleLiving) { - return Visibility.BOTH; - } - final boolean isTreeDialect = extractorParameters .getStringOrDefault( SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); - final Boolean _isCaptureTree = - extractorParameters.getBooleanByKeys( - PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeSourceConstant.SOURCE_CAPTURE_TREE_KEY); - final boolean isCaptureTree = Objects.nonNull(_isCaptureTree) ? _isCaptureTree : isTreeDialect; - final Boolean _isCaptureTable = - extractorParameters.getBooleanByKeys( - PipeSourceConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeSourceConstant.SOURCE_CAPTURE_TABLE_KEY); - final boolean isCaptureTable = - Objects.nonNull(_isCaptureTable) ? _isCaptureTable : !isTreeDialect; - - // visible under specific tree or table model <-> actually capture tree or table data - if (isCaptureTree && isCaptureTable) { - return Visibility.BOTH; - } - if (isCaptureTree) { - return Visibility.TREE_ONLY; - } - if (isCaptureTable) { - return Visibility.TABLE_ONLY; - } - - // UNREACHABLE CODE - LOGGER.error( - "BROKEN INVARIANT: DETECT INVISIBLE EXTRACTOR PARAMETERS {}", - extractorParameters.getAttribute()); - return Visibility.NONE; + return isTreeDialect ? Visibility.TREE_ONLY : Visibility.TABLE_ONLY; } public static Visibility calculateFromTopicConfig(final TopicConfig config) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java index 71b97d0b8b37a..2701e5bb23702 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java @@ -84,7 +84,7 @@ private boolean tryAcquireWithPipeCheck(final RateLimiter rateLimiter, final int TimeUnit.MILLISECONDS)) { final PipeTaskAgent finalTaskAgent = taskAgent; if (Objects.nonNull(finalTaskAgent) - && finalTaskAgent.getPipeCreationTime(pipeName) != creationTime) { + && !finalTaskAgent.isPipeExisted(pipeName, creationTime)) { return false; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java index 33acc06123e76..9051a1eb5079a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java @@ -124,26 +124,6 @@ private void validateDoubleLiving(final PipeParameters parameters) { return; } - // check 'capture.tree' - final Boolean isCaptureTree = - parameters.getBooleanByKeys( - PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeSourceConstant.SOURCE_CAPTURE_TREE_KEY); - if (Objects.nonNull(isCaptureTree) && !isCaptureTree) { - throw new PipeParameterNotValidException( - "capture.tree can not be specified to false when double living is enabled"); - } - - // check 'capture.table' - final Boolean isCaptureTable = - parameters.getBooleanByKeys( - PipeSourceConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeSourceConstant.SOURCE_CAPTURE_TABLE_KEY); - if (Objects.nonNull(isCaptureTable) && !isCaptureTable) { - throw new PipeParameterNotValidException( - "capture.table can not be specified to false when double living is enabled"); - } - // check 'forwarding-pipe-requests' final Boolean isForwardingPipeRequests = parameters.getBooleanByKeys( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java index c44692f63588d..6f3cdf0175a78 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java @@ -492,6 +492,10 @@ public boolean visibleUnder(final boolean isTableModel) { return VisibilityUtils.isCompatible(visibility, isTableModel); } + public boolean visibleUnderTableModel() { + return visibleUnder(true); + } + ////////////////////////////////////// Object //////////////////////////////// @Override diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/visibility/VisibilityUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/visibility/VisibilityUtilsTest.java new file mode 100644 index 0000000000000..c3a5bdde89ac3 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/visibility/VisibilityUtilsTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.datastructure.visibility; + +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class VisibilityUtilsTest { + + @Test + public void testCalculateFromExtractorParametersUsesDialectOnly() { + Assert.assertEquals( + Visibility.TREE_ONLY, + VisibilityUtils.calculateFromExtractorParameters(new PipeParameters(new HashMap<>()))); + + final Map treeAttributes = new HashMap<>(); + treeAttributes.put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); + treeAttributes.put(PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY, "false"); + treeAttributes.put(PipeSourceConstant.EXTRACTOR_CAPTURE_TABLE_KEY, "true"); + treeAttributes.put(PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, "true"); + Assert.assertEquals( + Visibility.TREE_ONLY, + VisibilityUtils.calculateFromExtractorParameters(new PipeParameters(treeAttributes))); + + final Map tableAttributes = new HashMap<>(); + tableAttributes.put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); + tableAttributes.put(PipeSourceConstant.SOURCE_CAPTURE_TREE_KEY, "true"); + tableAttributes.put(PipeSourceConstant.SOURCE_CAPTURE_TABLE_KEY, "false"); + tableAttributes.put(PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY, "true"); + Assert.assertEquals( + Visibility.TABLE_ONLY, + VisibilityUtils.calculateFromExtractorParameters(new PipeParameters(tableAttributes))); + } + + @Test + public void testCaptureAttributesDoNotCreateBothOrNoneVisibility() { + final Map captureBothAttributes = new HashMap<>(); + captureBothAttributes.put(PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY, "true"); + captureBothAttributes.put(PipeSourceConstant.EXTRACTOR_CAPTURE_TABLE_KEY, "true"); + Assert.assertEquals( + Visibility.TREE_ONLY, + VisibilityUtils.calculateFromExtractorParameters( + new PipeParameters(captureBothAttributes))); + + final Map captureNoneAttributes = new HashMap<>(); + captureNoneAttributes.put(PipeSourceConstant.SOURCE_CAPTURE_TREE_KEY, "false"); + captureNoneAttributes.put(PipeSourceConstant.SOURCE_CAPTURE_TABLE_KEY, "false"); + Assert.assertEquals( + Visibility.TREE_ONLY, + VisibilityUtils.calculateFromExtractorParameters( + new PipeParameters(captureNoneAttributes))); + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaKeeperTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaKeeperTest.java new file mode 100644 index 0000000000000..d38ce7a093e19 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/PipeMetaKeeperTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.task; + +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMetaKeeper; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class PipeMetaKeeperTest { + + @Test + public void testSameNameTreeAndTablePipesCanCoexist() { + final String pipeName = "p"; + final PipeMetaKeeper keeper = new PipeMetaKeeper(); + final PipeStaticMeta treeStaticMeta = + createStaticMeta(pipeName, 1, SystemConstant.SQL_DIALECT_TREE_VALUE); + final PipeStaticMeta tableStaticMeta = + createStaticMeta(pipeName, 2, SystemConstant.SQL_DIALECT_TABLE_VALUE); + + Assert.assertFalse(keeper.containsPipeMetaOverlapped(treeStaticMeta)); + keeper.addPipeMeta(new PipeMeta(treeStaticMeta, new PipeRuntimeMeta())); + + Assert.assertTrue(keeper.containsPipeMetaOverlapped(treeStaticMeta)); + Assert.assertFalse(keeper.containsPipeMetaOverlapped(tableStaticMeta)); + keeper.addPipeMeta(new PipeMeta(tableStaticMeta, new PipeRuntimeMeta())); + + Assert.assertEquals(2, keeper.getPipeMetaCount()); + Assert.assertSame(treeStaticMeta, keeper.getPipeMeta(pipeName, false).getStaticMeta()); + Assert.assertSame(tableStaticMeta, keeper.getPipeMeta(pipeName, true).getStaticMeta()); + Assert.assertSame(treeStaticMeta, keeper.getPipeMeta(pipeName).getStaticMeta()); + } + + @Test + public void testCaptureAttributesDoNotAffectSameNameConflictScope() { + final String pipeName = "p"; + final PipeMetaKeeper keeper = new PipeMetaKeeper(); + final PipeStaticMeta treeStaticMeta = + createStaticMetaWithCaptureAttributes(pipeName, 1, SystemConstant.SQL_DIALECT_TREE_VALUE); + final PipeStaticMeta tableStaticMeta = + createStaticMetaWithCaptureAttributes(pipeName, 2, SystemConstant.SQL_DIALECT_TABLE_VALUE); + + keeper.addPipeMeta(new PipeMeta(treeStaticMeta, new PipeRuntimeMeta())); + + Assert.assertFalse(keeper.containsPipeMetaOverlapped(tableStaticMeta)); + Assert.assertTrue( + keeper.containsPipeMetaOverlapped( + createStaticMeta(pipeName, 3, SystemConstant.SQL_DIALECT_TREE_VALUE))); + } + + @Test + public void testNoDialectPipeDefaultsToTreeScope() { + final String pipeName = "p"; + final PipeMetaKeeper keeper = new PipeMetaKeeper(); + final PipeStaticMeta oldTreeStaticMeta = + new PipeStaticMeta(pipeName, 1, new HashMap<>(), new HashMap<>(), new HashMap<>()); + final PipeStaticMeta tableStaticMeta = + createStaticMeta(pipeName, 2, SystemConstant.SQL_DIALECT_TABLE_VALUE); + + keeper.addPipeMeta(new PipeMeta(oldTreeStaticMeta, new PipeRuntimeMeta())); + + Assert.assertSame(oldTreeStaticMeta, keeper.getPipeMeta(pipeName, false).getStaticMeta()); + Assert.assertNull(keeper.getPipeMeta(pipeName, true)); + Assert.assertFalse(keeper.containsPipeMetaOverlapped(tableStaticMeta)); + } + + private PipeStaticMeta createStaticMeta( + final String pipeName, final long creationTime, final String dialect) { + final Map attributes = new HashMap<>(); + attributes.put(SystemConstant.SQL_DIALECT_KEY, dialect); + return new PipeStaticMeta(pipeName, creationTime, attributes, new HashMap<>(), new HashMap<>()); + } + + private PipeStaticMeta createStaticMetaWithCaptureAttributes( + final String pipeName, final long creationTime, final String dialect) { + final Map attributes = new HashMap<>(); + attributes.put(SystemConstant.SQL_DIALECT_KEY, dialect); + attributes.put(PipeSourceConstant.EXTRACTOR_CAPTURE_TREE_KEY, "true"); + attributes.put(PipeSourceConstant.EXTRACTOR_CAPTURE_TABLE_KEY, "true"); + return new PipeStaticMeta(pipeName, creationTime, attributes, new HashMap<>(), new HashMap<>()); + } +} diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 11de2e4cba232..8316aa0ec5bc6 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -545,6 +545,7 @@ struct TPushPipeMetaRespExceptionMessage { 1: required string pipeName 2: required string message 3: required i64 timeStamp + 4: optional i64 creationTime } struct TPushSinglePipeMetaReq {