Skip to content

Remove compaction related TODOs #6491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Amemeda wants to merge 7 commits into
base: main
Choose a base branch
from
+16 −10
Open

Remove compaction related TODOs #6491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8aa479a
Update ExternalCompactionMetricsIT to verify compactors are done runn…
Amemeda Jul 27, 2026
3e740e6
Rework test to make sure all compactions have completed jobs to remov…
Amemeda Jul 28, 2026
ddee6c0
Fix flaky test ConfigurableScanServerSelectorTest.testServerSetChanges
Amemeda Jul 29, 2026
8d232a5
Confirm compactors are not running compaction for user tables, revert…
Amemeda Jul 31, 2026
8425839
Add check to verify compactors are not running compactions for user t…
Amemeda Aug 3, 2026
90cd207
Simplfy the runnign compaction check by moving assert into consumer
Amemeda Aug 5, 2026
2f684f5
Clean up consumer code
Amemeda Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ public void testDeleteTableCancelsSleepingExternalCompaction() throws Exception
try (AccumuloClient client =
Accumulo.newClient().from(getCluster().getClientProperties()).build()) {

createTable(client, table1, "cs4"); // TODO which service
createTable(client, table1, "cs4");
TableId tid = getCluster().getServerContext().getTableId(table1);
writeData(client, table1);

Expand Down
24 changes: 15 additions & 9 deletions test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.verify;
import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.writeData;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.Collection;
import java.util.List;
Expand All @@ -46,12 +47,14 @@
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.SystemTables;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
Expand All @@ -61,6 +64,7 @@
import org.apache.accumulo.test.metrics.TestStatsDRegistryFactory;
import org.apache.accumulo.test.metrics.TestStatsDSink;
import org.apache.accumulo.test.metrics.TestStatsDSink.Metric;
import org.apache.accumulo.test.util.Wait;
import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
Expand Down Expand Up @@ -214,20 +218,22 @@ public void testMetrics() throws Exception {
thread.join();

// Wait for all external compactions to complete
long count;
do {
// TODO: Change this from waiting to verifying that all compactors are done running jobs,
// not just check that the jobs have been polled off the queues.
Comment thread
Amemeda marked this conversation as resolved.
UtilWaitThread.sleep(10000);
Wait.waitFor(() -> {
try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets()
.forLevel(DataLevel.USER).fetch(ColumnType.ECOMP).build()) {
count = tm.stream().mapToLong(t -> t.getExternalCompactions().keySet().size()).sum();
return tm.stream().allMatch(t -> t.getExternalCompactions().isEmpty());
}
} while (count > 0);
});
Comment thread
Amemeda marked this conversation as resolved.

// verify any running compactions contains only system tables
ExternalCompactionUtil.getCompactionsRunningOnCompactors(getCluster().getServerContext(),
externalCompaction -> {
var tableId = KeyExtent.fromThrift(externalCompaction.getJob().getExtent()).tableId();
assertTrue(SystemTables.tableIds().contains(tableId));
});

verify(client, table1, 7);
verify(client, table2, 13);

}
}

Expand Down