SQUASH – raise minimum sstable format to me and restrict upgrades o…

…nly from versions >=3.0.25, >=3.11.11, and dse-5.1

this removes the need for, and all bugs related to, SSTableHeaderFix
startup with fail hard if detecting a previous version <3.0.25 or <3.11.11
remove all test data for now unsupported sstable formats

src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java

@@ -197,13 +197,16 @@ public PartitionIndexIterator indexIterator(Descriptor descriptor, TableMetadata
     static class BigVersion extends Version
     {
         public static final String current_version = "nb";
-        public static final String earliest_supported_version = "ma";
+        public static final String earliest_supported_version = "me";
 
         // ma (3.0.0): swap bf hash order
         //             store rows natively
         // mb (3.0.7, 3.7): commit log lower bound included
         // mc (3.0.8, 3.9): commit log intervals included
         // md (3.0.18, 3.11.4): corrected sstable min/max clustering
+
+        // only `me` and above, and upgrades from 3.0.25, 3.11.11, and dse-5.1 are supported (see CassandraDaemon.checkForCompatibleUpgrades())
+
         // me (3.0.25, 3.11.11): added hostId of the node from which the sstable originated
 
         // na (4.0-rc1): uncompressed chunks, pending repair session, isTransient, checksummed sstable metadata file, new Bloomfilter format
@@ -368,7 +371,7 @@ public boolean hasMaxColumnValueLengths()
         @Override
         public boolean hasImplicitlyFrozenTuples()
         {
-            return version.compareTo("me") <= 0;
+            return true;
         }
 
         @Override

src/java/org/apache/cassandra/io/sstable/format/trieindex/TrieIndexFormat.java

@@ -348,7 +348,7 @@ static class TrieIndexVersion extends Version
             hasOriginatingHostId = version.matches("(a[d-z])|(b[b-z])") || version.compareTo("ca") >= 0;
             hasMaxColumnValueLengths = version.matches("b[a-z]"); // DSE only field
             correspondingMessagingVersion = version.compareTo("ca") >= 0 ? MessagingService.VERSION_SG_10 : MessagingService.VERSION_3014;
-            hasImplicitlyFrozenTuples = version.compareTo("cc") < 0 || version.compareTo("da") >= 0; // we don't know if what DA is going to be eventually, but it is almost certain it will not include implicitly frozen tuples
+            hasImplicitlyFrozenTuples = version.compareTo("cc") < 0 || version.compareTo("da") >= 0; // `da` is found in C* 5.0 and CC `main-5.0`, and both have implicitly frozen tuples
             byteComparableVersion = version.compareTo("ca") >= 0 ? ByteComparable.Version.OSS41 : ByteComparable.Version.LEGACY;
         }
 

src/java/org/apache/cassandra/service/CassandraDaemon.java

@@ -34,6 +34,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.cassandra.utils.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -64,7 +65,6 @@
 import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.SSTableHeaderFix;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.PathUtils;
@@ -80,13 +80,6 @@
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.security.ThreadAwareSecurityManager;
 import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.JMXServerUtils;
-import org.apache.cassandra.utils.JVMStabilityInspector;
-import org.apache.cassandra.utils.MBeanWrapper;
-import org.apache.cassandra.utils.Mx4jTool;
-import org.apache.cassandra.utils.INativeLibrary;
-import org.apache.cassandra.utils.WindowsTimer;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_FOREGROUND;
@@ -292,8 +285,6 @@ protected void setup()
 
         LegacySystemKeyspaceToNodes.convertToNodesFilesIfNecessary();
 
-        SSTableHeaderFix.fixNonFrozenUDTIfUpgradeFrom30();
-
         // clean up debris in the rest of the keyspaces
         for (String keyspaceName : Schema.instance.getKeyspaces())
         {

src/java/org/apache/cassandra/service/StartupChecks.java

@@ -64,9 +64,10 @@
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.JavaUtils;
 import org.apache.cassandra.utils.INativeLibrary;
+import org.apache.cassandra.utils.JavaUtils;
 import org.apache.cassandra.utils.SigarLibrary;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_PORT;
@@ -101,7 +102,8 @@ public class StartupChecks
     // The default set of pre-flight checks to run. Order is somewhat significant in that we probably
     // always want the system keyspace check run last, as this actually loads the schema for that
     // keyspace. All other checks should not require any schema initialization.
-    private final List<StartupCheck> DEFAULT_TESTS = ImmutableList.of(checkJemalloc,
+    private final List<StartupCheck> DEFAULT_TESTS = ImmutableList.of(checkPreviousVersion,
+                                                                      checkJemalloc,
                                                                       checkLz4Native,
                                                                       checkValidLaunchDate,
                                                                       checkJMXPorts,
@@ -144,6 +146,25 @@ public void verify() throws StartupException
             test.execute();
     }
 
+    public static final StartupCheck checkPreviousVersion = () -> {
+        String previousVersionString = FBUtilities.getPreviousReleaseVersionString();
+        if (null != previousVersionString)
+        {
+            CassandraVersion previousVersion = new CassandraVersion(previousVersionString);
+            boolean incompatible = previousVersion.major < 3;
+
+            if (previousVersion.major == 3)
+            {
+                incompatible |= previousVersion.minor == 0 && previousVersion.patch < 25;
+                incompatible |= previousVersion.minor == 11 && previousVersion.patch < 11;
+                incompatible |= previousVersion.minor != 0 && previousVersion.minor != 11;
+            }
+            if (incompatible)
+                throw new StartupException(StartupException.ERR_WRONG_DISK_STATE,
+                                           String.format("Only upgrades from >=3.0.25 and >=3.11.11 are supported (and sstables >= `me` format), found %s", previousVersionString));
+        }
+    };
+
     public static final StartupCheck checkJemalloc = new StartupCheck()
     {
         public void execute()