Added handling of the USING clause in CREATE INDEX for dummy-vector-backend

.github/scripts/label_promoted_commits.py

@@ -54,34 +54,35 @@ def main():
     # Print commit information
     for commit in commits:
         print(f'Commit sha is: {commit.sha}')
-        pr_last_line = commit.commit.message.splitlines()[-1]
-        match = pr_pattern.search(pr_last_line)
-        if match:
-            pr_number = int(match.group(1))
-            if pr_number in processed_prs:
-                continue
-            if target_branch:
-                pr = repo.get_pull(pr_number)
-                branch_name = target_branch[1]
-                refs_pr = re.findall(r'Parent PR: (?:#|https.*?)(\d+)', pr.body)
-                if refs_pr:
-                    print(f'branch-{target_branch.group(1)}, pr number is: {pr_number}')
-                    # 1. change the backport label of the parent PR to note that
-                    #    we've merged the corresponding backport PR
-                    # 2. close the backport PR and leave a comment on it to note
-                    #    that it has been merged with a certain git commit.
-                    ref_pr_number = refs_pr[0]
-                    mark_backport_done(repo, ref_pr_number, branch_name)
-                    comment = f'Closed via {commit.sha}'
-                    add_comment_and_close_pr(pr, comment)
-            else:
-                try:
+        pr_last_line = commit.commit.message.splitlines()
+        for line in reversed(pr_last_line):
+            match = pr_pattern.search(line)
+            if match:
+                pr_number = int(match.group(1))
+                if pr_number in processed_prs:
+                    continue
+                if target_branch:
                     pr = repo.get_pull(pr_number)
-                    pr.add_to_labels('promoted-to-master')
-                    print(f'master branch, pr number is: {pr_number}')
-                except UnknownObjectException:
-                    print(f'{pr_number} is not a PR but an issue, no need to add label')
-            processed_prs.add(pr_number)
+                    branch_name = target_branch[1]
+                    refs_pr = re.findall(r'Parent PR: (?:#|https.*?)(\d+)', pr.body)
+                    if refs_pr:
+                        print(f'branch-{target_branch.group(1)}, pr number is: {pr_number}')
+                        # 1. change the backport label of the parent PR to note that
+                        #    we've merged the corresponding backport PR
+                        # 2. close the backport PR and leave a comment on it to note
+                        #    that it has been merged with a certain git commit.
+                        ref_pr_number = refs_pr[0]
+                        mark_backport_done(repo, ref_pr_number, branch_name)
+                        comment = f'Closed via {commit.sha}'
+                        add_comment_and_close_pr(pr, comment)
+                else:
+                    try:
+                        pr = repo.get_pull(pr_number)
+                        pr.add_to_labels('promoted-to-master')
+                        print(f'master branch, pr number is: {pr_number}')
+                    except UnknownObjectException:
+                        print(f'{pr_number} is not a PR but an issue, no need to add label')
+                processed_prs.add(pr_number)
 
 
 if __name__ == "__main__":

cql3/statements/create_index_statement.cc

@@ -26,6 +26,7 @@
 #include "cql3/statements/index_prop_defs.hh"
 #include "index/secondary_index_manager.hh"
 #include "mutation/mutation.hh"
+#include "view_info.hh"
 
 #include <boost/range/adaptor/transformed.hpp>
 #include <boost/algorithm/string/join.hpp>
@@ -69,13 +70,12 @@ create_index_statement::validate(query_processor& qp, const service::client_stat
     if (_raw_targets.empty() && !_properties->is_custom) {
         throw exceptions::invalid_request_exception("Only CUSTOM indexes can be created without specifying a target column");
     }
-
     _properties->validate();
 }
 
 std::vector<::shared_ptr<index_target>> create_index_statement::validate_while_executing(data_dictionary::database db) const {
     auto schema = validation::validate_column_family(db, keyspace(), column_family());
-
+    auto classes = db.find_column_family(schema).get_index_manager().supported_custom_classes(db.features());
     if (schema->is_counter()) {
         throw exceptions::invalid_request_exception("Secondary indexes are not supported on counter tables");
     }
@@ -90,7 +90,11 @@ std::vector<::shared_ptr<index_target>> create_index_statement::validate_while_e
     }
 
     validate_for_local_index(*schema);
-
+
+    if (_properties && _properties->custom_class && !classes.contains(*(_properties->custom_class))) {
+        throw exceptions::invalid_request_exception("Non-supported custom class provided");
+    }
+
     std::vector<::shared_ptr<index_target>> targets;
     for (auto& raw_target : _raw_targets) {
         targets.emplace_back(raw_target->prepare(*schema));
@@ -343,9 +347,11 @@ std::optional<create_index_statement::base_schema_with_new_index> create_index_s
     }
     index_metadata_kind kind;
     index_options_map index_options;
+    if (_properties->custom_class || _properties->is_custom) {
+        index_options = _properties->get_options();
+    }
     if (_properties->is_custom) {
         kind = index_metadata_kind::custom;
-        index_options = _properties->get_options();
     } else {
         kind = schema->is_compound() ? index_metadata_kind::composites : index_metadata_kind::keys;
     }

cql3/statements/index_prop_defs.cc

@@ -11,6 +11,7 @@
 #include <set>
 #include <seastar/core/print.hh>
 #include "index_prop_defs.hh"
+#include "gms/feature_service.hh"
 #include "index/secondary_index.hh"
 #include "exceptions/exceptions.hh"
 
@@ -22,10 +23,14 @@ void cql3::statements::index_prop_defs::validate() {
     if (is_custom && !custom_class) {
         throw exceptions::invalid_request_exception("CUSTOM index requires specifying the index class");
     }
+<<<<<<< HEAD
 
-    if (!is_custom && custom_class) {
-        throw exceptions::invalid_request_exception("Cannot specify index class for a non-CUSTOM index");
+    if (!is_custom && custom_class && *custom_class != "dummy-vector-backend") {
+        throw exceptions::invalid_request_exception("Cannot specify non-vector index class for a non-CUSTOM index");
     }
+=======
+
+>>>>>>> 2ff52fbeb8 (cql3/statements: make use of supported custom classes set instead of hardcoding)
     if (!is_custom && !_properties.empty()) {
         throw exceptions::invalid_request_exception("Cannot specify options for a non-CUSTOM index");
     }
@@ -36,15 +41,6 @@ void cql3::statements::index_prop_defs::validate() {
                         db::index::secondary_index::custom_index_option_name));
     }
 
-    // Currently, Scylla does not support *any* class of custom index
-    // implementation. If in the future we do (e.g., SASI, or something
-    // new), we'll need to check for valid values here.
-    if (is_custom && custom_class) {
-        throw exceptions::invalid_request_exception(
-                format("Unsupported CUSTOM INDEX class {}. Note that currently, Scylla does not support SASI or any other CUSTOM INDEX class.",
-                        *custom_class));
-
-    }
 }
 
 index_options_map

cql3/statements/index_prop_defs.hh

@@ -10,6 +10,7 @@
 
 #pragma once
 
+#include "gms/feature_service.hh"
 #include "property_definitions.hh"
 #include <seastar/core/sstring.hh>
 

dist/common/scripts/scylla_raid_setup

@@ -16,6 +16,7 @@ import sys
 import stat
 import logging
 import pyudev
+import psutil
 from pathlib import Path
 from scylla_util import *
 from subprocess import run, SubprocessError
@@ -92,6 +93,15 @@ class UdevInfo:
     def id_links(self):
         return [l for l in self.device.device_links if l.startswith('/dev/disk/by-id')]
 
+
+def is_selinux_enabled():
+    partitions = psutil.disk_partitions(all=True)
+    for p in partitions:
+        if p.fstype == 'selinuxfs':
+            if os.path.exists(p.mountpoint + '/enforce'):
+                return True
+    return False
+
 if __name__ == '__main__':
     if os.getuid() > 0:
         print('Requires root permission.')
@@ -335,17 +345,41 @@ WantedBy=local-fs.target
         udev_info.dump_variables()
 
     if is_redhat_variant():
-        if not shutil.which('getenforce'):
-            pkg_install('libselinux-utils')
+        offline_skip_relabel = False
+        has_semanage = True
+        if not shutil.which('matchpathcon'):
+            offline_skip_relabel = True
+            pkg_install('libselinux-utils', offline_exit=False)
         if not shutil.which('restorecon'):
-            pkg_install('policycoreutils')
+            offline_skip_relabel = True
+            pkg_install('policycoreutils', offline_exit=False)
         if not shutil.which('semanage'):
-            pkg_install('policycoreutils-python-utils')
-        selinux_status = out('getenforce')
+            if is_offline():
+                has_semanage = False
+            else:
+                pkg_install('policycoreutils-python-utils')
+        if is_offline() and offline_skip_relabel:
+            print('Unable to find SELinux tools, skip relabeling.')
+            sys.exit(0)
+
         selinux_context = out('matchpathcon -n /var/lib/systemd/coredump')
         selinux_type = selinux_context.split(':')[2]
-        run(f'semanage fcontext -a -t {selinux_type} "{root}/coredump(/.*)?"', shell=True, check=True)
-        if selinux_status != 'Disabled':
+        if has_semanage:
+            run(f'semanage fcontext -a -t {selinux_type} "{root}/coredump(/.*)?"', shell=True, check=True)
+        else:
+            # without semanage, we need to update file_contexts directly,
+            # and compile it to binary format (.bin file)
+            try:
+                with open('/etc/selinux/targeted/contexts/files/file_contexts.local', 'a') as f:
+                    spacer = ''
+                    if f.tell() != 0:
+                        spacer = '\n'
+                    f.write(f'{spacer}{root}/coredump(/.*)?   {selinux_context}\n')
+            except FileNotFoundError as e:
+                print('Unable to find SELinux policy files, skip relabeling.')
+                sys.exit(0)
+            run('sefcontext_compile /etc/selinux/targeted/contexts/files/file_contexts.local', shell=True, check=True)
+        if is_selinux_enabled():
             run(f'restorecon -F -v -R {root}', shell=True, check=True)
         else:
             Path('/.autorelabel').touch(exist_ok=True)

dist/common/scripts/scylla_util.py

@@ -293,13 +293,14 @@ def swap_exists():
     swaps = out('swapon --noheadings --raw')
     return True if swaps != '' else False
 
-def pkg_error_exit(pkg):
+def pkg_error_exit(pkg, offline_exit=True):
     print(f'Package "{pkg}" required.')
-    sys.exit(1)
+    if offline_exit:
+        sys.exit(1)
 
-def yum_install(pkg):
+def yum_install(pkg, offline_exit=True):
     if is_offline():
-        pkg_error_exit(pkg)
+        pkg_error_exit(pkg, offline_exit)
     return run(f'yum install -y {pkg}', shell=True, check=True)
 
 def apt_is_updated():
@@ -313,9 +314,9 @@ def apt_is_updated():
 
 APT_GET_UPDATE_NUM_RETRY = 30
 APT_GET_UPDATE_RETRY_INTERVAL = 10
-def apt_install(pkg):
+def apt_install(pkg, offline_exit=True):
     if is_offline():
-        pkg_error_exit(pkg)
+        pkg_error_exit(pkg, offline_exit)
 
     # The lock for update and install/remove are different, and
     # DPkg::Lock::Timeout will only wait for install/remove lock.
@@ -344,14 +345,14 @@ def apt_install(pkg):
     apt_env['DEBIAN_FRONTEND'] = 'noninteractive'
     return run(f'apt-get -o DPkg::Lock::Timeout=300 install -y {pkg}', shell=True, check=True, env=apt_env)
 
-def emerge_install(pkg):
+def emerge_install(pkg, offline_exit=True):
     if is_offline():
-        pkg_error_exit(pkg)
+        pkg_error_exit(pkg, offline_exit)
     return run(f'emerge -uq {pkg}', shell=True, check=True)
 
-def zypper_install(pkg):
+def zypper_install(pkg, offline_exit=True):
     if is_offline():
-        pkg_error_exit(pkg)
+        pkg_error_exit(pkg, offline_exit)
     return run(f'zypper install -y {pkg}', shell=True, check=True)
 
 def pkg_distro():
@@ -364,18 +365,20 @@ def pkg_distro():
     else:
         return distro.id()
 
-pkg_xlat = {'cpupowerutils': {'debian': 'linux-cpupower', 'gentoo':'sys-power/cpupower', 'arch':'cpupower', 'suse': 'cpupower'}}
-def pkg_install(pkg):
+pkg_xlat = {'cpupowerutils': {'debian': 'linux-cpupower', 'gentoo':'sys-power/cpupower', 'arch':'cpupower', 'suse': 'cpupower'},
+            'policycoreutils-python-utils': {'amzn2': 'policycoreutils-python'}}
+
+def pkg_install(pkg, offline_exit=True):
     if pkg in pkg_xlat and pkg_distro() in pkg_xlat[pkg]:
         pkg = pkg_xlat[pkg][pkg_distro()]
     if is_redhat_variant():
-        return yum_install(pkg)
+        return yum_install(pkg, offline_exit)
     elif is_debian_variant():
-        return apt_install(pkg)
+        return apt_install(pkg, offline_exit)
     elif is_gentoo():
-        return emerge_install(pkg)
+        return emerge_install(pkg, offline_exit)
     elif is_suse_variant():
-        return zypper_install(pkg)
+        return zypper_install(pkg, offline_exit)
     else:
         pkg_error_exit(pkg)
 

index/secondary_index_manager.cc

@@ -8,20 +8,25 @@
  * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0)
  */
 
+#include <optional>
+#include <ranges>
 #include <seastar/core/shared_ptr.hh>
 
 #include "index/secondary_index_manager.hh"
 
 #include "cql3/statements/index_target.hh"
 #include "cql3/expr/expression.hh"
 #include "index/target_parser.hh"
+#include "schema/schema.hh"
 #include "schema/schema_builder.hh"
 #include "db/view/view.hh"
 #include "concrete_types.hh"
 #include "db/tags/extension.hh"
 
 #include <boost/range/adaptor/map.hpp>
 #include <boost/algorithm/string/predicate.hpp>
+#include <set>
+#include <string_view>
 
 namespace secondary_index {
 
@@ -344,4 +349,25 @@ bool secondary_index_manager::is_global_index(const schema& s) const {
     });
 }
 
+std::optional<sstring> secondary_index_manager::custom_index_class(const schema& s) const {
+    auto range = _indices | std::views::values;
+    auto idx = std::ranges::find_if(range, [&s] (const index& i) {
+        return i.metadata().kind() != index_metadata_kind::custom && i.metadata().options().contains(cql3::statements::index_target::custom_index_option_name) && s.cf_name() == index_table_name(i.metadata().name());
+    });
+    if (idx == std::ranges::end(range)) {
+        return std::nullopt;
+    } else {
+        return (*idx).metadata().options().at(cql3::statements::index_target::custom_index_option_name);
+    }
+}
+
+std::set<std::string_view> secondary_index_manager::supported_custom_classes(const gms::feature_service& fs) const {
+    using namespace std::literals;
+    // TEMP: when actual vector backend will be added, this will create the set from features
+    std::set<std::string_view> classes = {
+        "dummy-vector-backend"sv,
+    };
+    return classes;
+}
+
 }

index/secondary_index_manager.hh

@@ -10,11 +10,14 @@
 
 #pragma once
 
+#include "gms/feature_service.hh"
 #include "schema/schema.hh"
 
 #include "data_dictionary/data_dictionary.hh"
 #include "cql3/statements/index_target.hh"
 
+#include <set>
+#include <string_view>
 #include <vector>
 
 namespace cql3::expr {
@@ -99,6 +102,8 @@ public:
     bool is_index(view_ptr) const;
     bool is_index(const schema& s) const;
     bool is_global_index(const schema& s) const;
+    std::optional<sstring> custom_index_class(const schema& s) const;
+    std::set<std::string_view> supported_custom_classes(const gms::feature_service& fs) const;
 private:
     void add_index(const index_metadata& im);
 };