Add LegacySSTableTest method testVerifyOldTupleSSTables for frozen tuples

src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java

@@ -197,13 +197,16 @@ public PartitionIndexIterator indexIterator(Descriptor descriptor, TableMetadata
     static class BigVersion extends Version
     {
         public static final String current_version = "nb";
-        public static final String earliest_supported_version = "ma";
+        public static final String earliest_supported_version = "me";
 
         // ma (3.0.0): swap bf hash order
         //             store rows natively
         // mb (3.0.7, 3.7): commit log lower bound included
         // mc (3.0.8, 3.9): commit log intervals included
         // md (3.0.18, 3.11.4): corrected sstable min/max clustering
+
+        // only `me` and above, and upgrades from 3.0.25, 3.11.11, and dse-5.1 are supported (see CassandraDaemon.checkForCompatibleUpgrades())
+
         // me (3.0.25, 3.11.11): added hostId of the node from which the sstable originated
 
         // na (4.0-rc1): uncompressed chunks, pending repair session, isTransient, checksummed sstable metadata file, new Bloomfilter format
@@ -368,7 +371,7 @@ public boolean hasMaxColumnValueLengths()
         @Override
         public boolean hasImplicitlyFrozenTuples()
         {
-            return false;
+            return true;
         }
 
         @Override

src/java/org/apache/cassandra/io/sstable/format/trieindex/TrieIndexFormat.java

@@ -334,7 +334,7 @@ static class TrieIndexVersion extends Version
         private final boolean hasMaxColumnValueLengths;
 
         private final int correspondingMessagingVersion;
-        private final boolean hasExplicitlyFrozenTuples;
+        private final boolean hasImplicitlyFrozenTuples;
 
         private final ByteComparable.Version byteComparableVersion;
 
@@ -348,7 +348,7 @@ static class TrieIndexVersion extends Version
             hasOriginatingHostId = version.matches("(a[d-z])|(b[b-z])") || version.compareTo("ca") >= 0;
             hasMaxColumnValueLengths = version.matches("b[a-z]"); // DSE only field
             correspondingMessagingVersion = version.compareTo("ca") >= 0 ? MessagingService.VERSION_SG_10 : MessagingService.VERSION_3014;
-            hasExplicitlyFrozenTuples = version.compareTo("cc") < 0 || version.compareTo("da") >= 0; // we don't know if what DA is going to be eventually, but it is almost certain it will not include explicitly frozen tuples
+            hasImplicitlyFrozenTuples = version.compareTo("cc") < 0 || version.compareTo("da") >= 0; // `da` is found in C* 5.0 and CC `main-5.0`, and both have implicitly frozen tuples
             byteComparableVersion = version.compareTo("ca") >= 0 ? ByteComparable.Version.OSS41 : ByteComparable.Version.LEGACY;
         }
 
@@ -479,7 +479,7 @@ public boolean hasIsTransient()
         @Override
         public boolean hasImplicitlyFrozenTuples()
         {
-            return hasExplicitlyFrozenTuples;
+            return hasImplicitlyFrozenTuples;
         }
 
         @Override

src/java/org/apache/cassandra/service/CassandraDaemon.java

@@ -34,6 +34,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.cassandra.utils.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -64,7 +65,6 @@
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.SSTableHeaderFix;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.PathUtils;
@@ -80,13 +80,6 @@
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.security.ThreadAwareSecurityManager;
 import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.JMXServerUtils;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.MBeanWrapper;
-import org.apache.cassandra.utils.Mx4jTool;
-import org.apache.cassandra.utils.INativeLibrary;
-import org.apache.cassandra.utils.WindowsTimer;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_FOREGROUND;
@@ -292,8 +285,6 @@ protected void setup()
 
         LegacySystemKeyspaceToNodes.convertToNodesFilesIfNecessary();
 
-        SSTableHeaderFix.fixNonFrozenUDTIfUpgradeFrom30();
-
         // clean up debris in the rest of the keyspaces
         for (String keyspaceName : Schema.instance.getKeyspaces())
         {

src/java/org/apache/cassandra/service/StartupChecks.java

@@ -64,9 +64,10 @@
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.JavaUtils;
 import org.apache.cassandra.utils.INativeLibrary;
+import org.apache.cassandra.utils.JavaUtils;
 import org.apache.cassandra.utils.SigarLibrary;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_PORT;
@@ -101,7 +102,8 @@ public class StartupChecks
     // The default set of pre-flight checks to run. Order is somewhat significant in that we probably
     // always want the system keyspace check run last, as this actually loads the schema for that
     // keyspace. All other checks should not require any schema initialization.
-    private final List<StartupCheck> DEFAULT_TESTS = ImmutableList.of(checkJemalloc,
+    private final List<StartupCheck> DEFAULT_TESTS = ImmutableList.of(checkPreviousVersion,
+                                                                      checkJemalloc,
                                                                       checkLz4Native,
                                                                       checkValidLaunchDate,
                                                                       checkJMXPorts,
@@ -144,6 +146,25 @@ public void verify() throws StartupException
             test.execute();
     }
 
+    public static final StartupCheck checkPreviousVersion = () -> {
+        String previousVersionString = FBUtilities.getPreviousReleaseVersionString();
+        if (null != previousVersionString)
+        {
+            CassandraVersion previousVersion = new CassandraVersion(previousVersionString);
+            boolean incompatible = previousVersion.major < 3;
+
+            if (previousVersion.major == 3)
+            {
+                incompatible |= previousVersion.minor == 0 && previousVersion.patch < 25;
+                incompatible |= previousVersion.minor == 11 && previousVersion.patch < 11;
+                incompatible |= previousVersion.minor != 0 && previousVersion.minor != 11;
+            }
+            if (incompatible)
+                throw new StartupException(StartupException.ERR_WRONG_DISK_STATE,
+                                           String.format("Only upgrades from >=3.0.25 and >=3.11.11 are supported (and sstables >= `me` format), found %s", previousVersionString));
+        }
+    };
+
     public static final StartupCheck checkJemalloc = new StartupCheck()
     {
         public void execute()

src/java/org/apache/cassandra/tools/StandaloneScrubber.java

@@ -46,7 +46,6 @@
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableHeaderFix;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.schema.Schema;
@@ -129,66 +128,6 @@ public static void main(String args[])
             }
             System.out.println(String.format("Pre-scrub sstables snapshotted into snapshot %s", snapshotName));
 
-            if (options.headerFixMode != Options.HeaderFixMode.OFF)
-            {
-                // Run the frozen-UDT checks _before_ the sstables are opened
-
-                List<String> logOutput = new ArrayList<>();
-
-                SSTableHeaderFix.Builder headerFixBuilder = SSTableHeaderFix.builder()
-                                                                            .logToList(logOutput)
-                                                                            .schemaCallback(() -> Schema.instance::getTableMetadata);
-                if (options.headerFixMode == Options.HeaderFixMode.VALIDATE)
-                    headerFixBuilder = headerFixBuilder.dryRun();
-
-                for (Pair<Descriptor, Set<Component>> p : listResult)
-                    headerFixBuilder.withPath(p.left.fileFor(Component.DATA).toPath());
-
-                SSTableHeaderFix headerFix = headerFixBuilder.build();
-                try
-                {
-                    headerFix.execute();
-                }
-                catch (Exception e)
-                {
-                    JVMStabilityInspector.inspectThrowable(e);
-                    if (options.debug)
-                        e.printStackTrace(System.err);
-                }
-
-                if (headerFix.hasChanges() || headerFix.hasError())
-                    logOutput.forEach(System.out::println);
-
-                if (headerFix.hasError())
-                {
-                    System.err.println("Errors in serialization-header detected, aborting.");
-                    System.exit(1);
-                }
-
-                switch (options.headerFixMode)
-                {
-                    case VALIDATE_ONLY:
-                    case FIX_ONLY:
-                        System.out.printf("Not continuing with scrub, since '--%s %s' was specified.%n",
-                                          HEADERFIX_OPTION,
-                                          options.headerFixMode.asCommandLineOption());
-                        System.exit(0);
-                    case VALIDATE:
-                        if (headerFix.hasChanges())
-                        {
-                            System.err.printf("Unfixed, but fixable errors in serialization-header detected, aborting. " +
-                                              "Use a non-validating mode ('-e %s' or '-e %s') for --%s%n",
-                                              Options.HeaderFixMode.FIX.asCommandLineOption(),
-                                              Options.HeaderFixMode.FIX_ONLY.asCommandLineOption(),
-                                              HEADERFIX_OPTION);
-                            System.exit(2);
-                        }
-                        break;
-                    case FIX:
-                        break;
-                }
-            }
-
             List<SSTableReader> sstables = new ArrayList<>();
 
             // Open sstables
@@ -286,9 +225,9 @@ private static class Options
         public boolean skipCorrupted;
         public boolean noValidate;
         public boolean reinserOverflowedTTL;
-        public HeaderFixMode headerFixMode = HeaderFixMode.VALIDATE;
 
-        enum HeaderFixMode
+        @Deprecated // sstables that need the fix are no longer supported.  kept for cmdline compatibility (can remove in 6.0)
+        private enum HeaderFixMode
         {
             VALIDATE_ONLY,
             VALIDATE,
@@ -300,11 +239,6 @@ static HeaderFixMode fromCommandLine(String value)
             {
                 return valueOf(value.replace('-', '_').toUpperCase().trim());
             }
-
-            String asCommandLineOption()
-            {
-                return name().toLowerCase().replace('_', '-');
-            }
         }
 
         private Options(String keyspaceName, String cfName)
@@ -347,11 +281,11 @@ public static Options parseArgs(String cmdArgs[])
                 opts.skipCorrupted = cmd.hasOption(SKIP_CORRUPTED_OPTION);
                 opts.noValidate = cmd.hasOption(NO_VALIDATE_OPTION);
                 opts.reinserOverflowedTTL = cmd.hasOption(REINSERT_OVERFLOWED_TTL_OPTION);
-                if (cmd.hasOption(HEADERFIX_OPTION))
-                {
+                if (cmd.hasOption(HEADERFIX_OPTION)) {
                     try
                     {
-                        opts.headerFixMode = HeaderFixMode.fromCommandLine(cmd.getOptionValue(HEADERFIX_OPTION));
+                        HeaderFixMode.fromCommandLine(cmd.getOptionValue(HEADERFIX_OPTION));
+                        System.out.println(String.format("The option %s is deprecated (and does nothing)", HEADERFIX_OPTION));
                     }
                     catch (Exception e)
                     {
@@ -384,22 +318,7 @@ private static CmdLineOptions getCmdLineOptions()
             options.addOption("m",  MANIFEST_CHECK_OPTION, "only check and repair the leveled manifest, without actually scrubbing the sstables");
             options.addOption("s",  SKIP_CORRUPTED_OPTION, "skip corrupt rows in counter tables");
             options.addOption("n",  NO_VALIDATE_OPTION,    "do not validate columns using column validator");
-            options.addOption("e",  HEADERFIX_OPTION,      true, "Option whether and how to perform a " +
-                                                                 "check of the sstable serialization-headers and fix known, " +
-                                                                 "fixable issues.\n" +
-                                                                 "Possible argument values:\n" +
-                                                                 "- validate-only: validate the serialization-headers, " +
-                                                                 "but do not fix those. Do not continue with scrub - " +
-                                                                 "i.e. only validate the header (dry-run of fix-only).\n" +
-                                                                 "- validate: (default) validate the serialization-headers, " +
-                                                                 "but do not fix those and only continue with scrub if no " +
-                                                                 "error were detected.\n" +
-                                                                 "- fix-only: validate and fix the serialization-headers, " +
-                                                                 "don't continue with scrub.\n" +
-                                                                 "- fix: validate and fix the serialization-headers, do not " +
-                                                                 "fix and do not continue with scrub if the serialization-header " +
-                                                                 "check encountered errors.\n" +
-                                                                 "- off: don't perform the serialization-header checks.");
+            options.addOption("e",  HEADERFIX_OPTION,      true, "Deprecated Option.");
             options.addOption("r", REINSERT_OVERFLOWED_TTL_OPTION, REINSERT_OVERFLOWED_TTL_OPTION_DESCRIPTION);
             return options;
         }

test/data/legacy-sstables/bb/legacy_tables/legacy_bb_tuple/bb-1-bti-Digest.crc32

@@ -0,0 +1 @@
+1407325515

test/data/legacy-sstables/ma/legacy_tables/legacy_ma_clust_counter/ma-1-big-TOC.txt → test/data/legacy-sstables/bb/legacy_tables/legacy_bb_tuple/bb-1-bti-TOC.txt

@@ -1,8 +1,8 @@
-Filter.db
-Digest.crc32
-Data.db
-Index.db
-Summary.db
 TOC.txt
 Statistics.db
+Data.db
+Partitions.db
+Rows.db
+Filter.db
+Digest.crc32
 CompressionInfo.db

test/data/legacy-sstables/cc/legacy_tables/legacy_cc_tuple/cc-3glk_0z9v_2f0ps1zlpy7dco1gdh-bti-Digest.crc32

@@ -0,0 +1 @@
+645653999

test/data/legacy-sstables/cc/legacy_tables/legacy_cc_tuple/cc-3glk_0z9v_2f0ps1zlpy7dco1gdh-bti-TOC.txt

@@ -0,0 +1,8 @@
+Filter.db
+Statistics.db
+TOC.txt
+Rows.db
+Digest.crc32
+Partitions.db
+Data.db
+CompressionInfo.db