K8s stuff (do not merge)

.dockerignore

@@ -0,0 +1 @@
+.git

.ruby-version

@@ -1 +1 @@
-2.6.9
+3.1.2

Gemfile

@@ -10,7 +10,7 @@ gem 'bundler', File.read(File.join(__dir__, '.bundler-version')).strip
 
 # Dependencies for connectors
 gem 'activesupport', '~>5.2.6'
-gem 'mime-types', '= 3.1'
+gem 'mime-types', '~> 3.3'
 gem 'tzinfo-data'
 gem 'tzinfo'
 gem 'fugit', '~> 1.5.3'
@@ -26,6 +26,8 @@ gem 'dry-inflector', '= 0.2.1'
 gem 'dry-schema',  '= 1.8.0'
 gem 'dry-validation', '= 1.7.0'
 
+gem 'rack', '>= 2.2.3.1'
+
 group :test do
   gem 'rspec-collection_matchers', '~> 1.2.0'
   gem 'rspec-core', '~> 3.10.1'
@@ -34,9 +36,8 @@ group :test do
   gem 'rubocop-performance', '1.11.5'
   gem 'rspec-mocks'
   gem 'webmock'
-  gem 'rack', '>= 2.2.3.1'
   gem 'rack-test'
-  gem 'ruby-debug-ide'
+  # gem 'ruby-debug-ide'
   gem 'pry-remote'
   gem 'pry-nav'
   gem 'debase', '0.2.5.beta2'

Gemfile.lock

@@ -105,7 +105,7 @@ GEM
       concurrent-ruby (~> 1.0)
     jwt (2.3.0)
     method_source (1.0.0)
-    mime-types (3.1)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
     minitest (5.16.1)
@@ -134,7 +134,6 @@ GEM
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rainbow (3.1.1)
-    rake (13.0.6)
     regexp_parser (2.2.1)
     remedy (0.3.0)
     rexml (3.2.5)
@@ -165,8 +164,6 @@ GEM
     rubocop-performance (1.11.5)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
-    ruby-debug-ide (0.7.3)
-      rake (>= 0.8.1)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
     signet (0.16.0)
@@ -226,7 +223,7 @@ DEPENDENCIES
   fugit (~> 1.5.3)
   hashie (~> 5.0.0)
   httpclient (~> 2.8.3)
-  mime-types (= 3.1)
+  mime-types (~> 3.3)
   mongo (~> 2.18)
   pry-nav
   pry-remote
@@ -239,7 +236,6 @@ DEPENDENCIES
   rspec_junit_formatter
   rubocop (= 1.18.4)
   rubocop-performance (= 1.11.5)
-  ruby-debug-ide
   signet (~> 0.16.0)
   simplecov
   simplecov-material
@@ -249,7 +245,7 @@ DEPENDENCIES
   webmock
 
 RUBY VERSION
-   ruby 2.6.9p207
+   ruby 3.1.2p20
 
 BUNDLED WITH
    2.3.15

Makefile

@@ -71,6 +71,10 @@ install:
 	- gem install bundler -v $(shell cat .bundler-version) && rbenv rehash
 	bundle _$(shell cat .bundler-version)_ install --jobs 1
 
+install_for_production:
+	- gem install bundler -v $(shell cat .bundler-version) && rbenv rehash
+	bundle _$(shell cat .bundler-version)_ install --without development test --jobs 1
+
 build-docker:
 	docker build -t connectors .
 
@@ -80,6 +84,12 @@ run-docker:
 exec_app:
 	cd lib/app; bundle _$(shell cat .bundler-version)_ exec ruby app.rb
 
+run_producer:
+	cd lib/app; bundle _$(shell cat .bundler-version)_ exec ruby producer_app.rb
+
+run_consumers:
+	cd lib/app; bundle _$(shell cat .bundler-version)_ exec ruby consumer_app.rb
+
 exec_cli:
 	cd lib/app; bundle _$(shell cat .bundler-version)_ exec ruby console_app.rb
 

k8s/Dockerfile

@@ -0,0 +1,29 @@
+FROM ubuntu:focal
+
+# installing all system dependencies, yq, ruby-build and rbenv
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends uuid-runtime curl ca-certificates git make build-essential libssl-dev libreadline-dev zlib1g-dev && \
+    rm -rf /var/lib/apt/lists/* && \
+    curl -L https://github.com/mikefarah/yq/releases/download/v4.24.2/yq_linux_amd64.tar.gz | tar -xzvf - && mv yq_linux_amd64 /usr/bin/yq && \
+    git clone https://github.com/rbenv/rbenv.git ~/.rbenv && \
+    curl -L https://github.com/sstephenson/ruby-build/archive/v20220324.tar.gz | tar -zxvf - -C /tmp/ && \
+    cd /tmp/ruby-build-* && ./install.sh
+
+# set the env
+ENV PATH /root/.rbenv/bin:/root/.rbenv/shims:$PATH
+ENV RUBY_VERSION 2.6.9
+RUN echo 'eval "$(rbenv init -)"' >> .bashrc
+RUN echo 'eval "$(rbenv init -)"' >> /etc/profile.d/rbenv.sh # or /etc/profile
+
+RUN RUBY_CONFIGURE_OPTS=--disable-install-doc rbenv install 2.6.9
+
+ENV ENT_SEARCH_CONFIG_PATH /app/k8s/stackpack/config/enterprise_search.yml
+ENV CONNECTORS_CONFIG /app/config/connectors.yml
+
+WORKDIR /app
+COPY . /app
+
+# run the make file to install the app
+RUN make install_for_production
+
+CMD ["make", "exec_app"]

k8s/Makefile

@@ -0,0 +1,10 @@
+CURDIR = $(shell pwd)
+
+ build:
+ 	docker build -t docker.elastic.co/enterprise-search/connectors-python .
+
+ run:
+ 	docker run --name connectors_python --rm -v $(CURDIR)/stackpack:/app docker.elastic.co/enterprise-search/connectors-python
+
+ shell:
+ 	docker exec -u root -it connectors_python /bin/bash

k8s/configmaps/connectors-python.yml

@@ -0,0 +1,32 @@
+elasticsearch:
+  host: http://host.docker.internal:9200
+  username: elastic
+  password: changeme
+  ssl: true
+  bulk_queue_max_size: 1024
+  bulk_display_every: 100
+  bulk_chunk_size: 150
+  request_timeout: 120
+  max_wait_duration: 120
+  initial_backoff_duration: 1
+  backoff_multiplier: 2
+  log_level: info
+
+service:
+  idling: 30
+  heartbeat: 300
+  max_errors: 20
+  max_errors_span: 600
+  trace_mem: false
+
+native_service_types:
+  - mysql
+
+ # some id
+ #connector_id: '1'
+
+sources:
+  mongodb: connectors.sources.mongo:MongoDataSource
+  s3: connectors.sources.aws:S3DataSource
+  dir: connectors.sources.directory:DirectoryDataSource
+  mysql: connectors.sources.mysql:MySqlDataSource

k8s/configmaps/connectors.yml

@@ -0,0 +1,25 @@
+# general metadata
+version: "8.6.0.0-20221123T101025Z"
+repository: "[email protected]:elastic/connectors.git"
+revision: "5452054864ad0863e9b185d25b28f1ff2f214849"
+elasticsearch:
+  # cloud_id: CHANGEME
+  hosts: http://localhost:9200
+  api_key: ZTdpTmhZUUJFMXlUVTl1eUcyTks6UTVwdDBaVlBSSXVCM2doMWFxMlYtQQ==
+  retry_on_failure: 3
+  request_timeout: 120
+  disable_warnings: true
+  trace: false
+  log: false
+thread_pool:
+  min_threads: 0
+  max_threads: 5
+  max_queue: 100
+log_level: info
+ecs_logging: false
+poll_interval: 3
+termination_timeout: 60
+heartbeat_interval: 1800
+native_mode: true
+connector_id: CHANGEME
+service_type: CHANGEME

k8s/configmaps/enterprise_search.yml

@@ -0,0 +1,13 @@
+ent_search:
+  listen_host: 0.0.0.0
+  ssl:
+    enabled: false
+
+elasticsearch:
+  # host: http://0.0.0.0:9200
+  host: http://host.docker.internal:9200
+  username: elastic
+  password: changeme
+
+allow_es_settings_modification: true
+secret_management.encryption_keys: [1603f8d7f94483625f252111ba7b4d10049a9c13bc4d4838ea2039516f6f1493]

k8s/configmaps/kustomization.yml

@@ -0,0 +1,8 @@
+---
+configMapGenerator:
+- name: enterprise-search-python-connectors-configuration
+  files:
+  - connectors-python.yml
+- name: enterprise-search-configuration
+  files:
+  - enterprise_search.yml

k8s/deployment.yml

@@ -0,0 +1,121 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: connectors-ruby-producer
+  labels:
+    service.type: connectors-ruby
+    service.group: producer
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      service.type: connectors-ruby
+      service.group: producer
+  template:
+    metadata:
+      labels:
+        service.type: connectors-ruby
+        service.group: producer
+    spec:
+      containers:
+      - name: connectors-ruby
+        image: connectors:latest
+        command: ["make"]
+        args: ["run_producer"]
+        imagePullPolicy: Never
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "250m"
+          limits:
+            memory: "256Mi"
+            cpu: "500m"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: connectors-ruby-consumer
+  labels:
+    service.type: ruby
+    service.group: consumer
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      service.type: connectors-ruby
+      service.group: consumer
+  template:
+    metadata:
+      labels:
+        service.type: connectors-ruby
+        service.group: consumer
+    spec:
+      containers:
+      - name: connectors-ruby
+        imagePullPolicy: Never
+        image: connectors:latest
+        command: ["make"]
+        args: ["run_consumers"]
+        volumeMounts:
+        - name: enterprise-search-configuration
+          mountPath: /app/config/ent-search
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "250m"
+          limits:
+            memory: "256Mi"
+            cpu: "500m"
+        imagePullPolicy: Never
+      volumes:
+      - name: enterprise-search-configuration
+        configMap:
+          name: enterprise-search-configuration-4bf99k7m9b
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: connectors-python
+  labels:
+    app: python
+    group: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      service.type: connectors-python
+      service.group: default
+  template:
+    metadata:
+      labels:
+        service.type: connectors-python
+        service.group: default
+    spec:
+      containers:
+      - name: connectors-python
+        image: connectors-python:latest
+        command: ["elastic-ingest"]
+        args: ["-c", "/app/config/connectors/connectors-python.yml", "--filebeat", '--debug']
+        env:
+        - name: ENT_SEARCH_CONFIG_PATH
+          value: /app/config/ent-search/enterprise_search.yml
+        volumeMounts:
+        - name: enterprise-search-configuration
+          mountPath: /app/config/ent-search/
+        - name: enterprise-search-python-connectors-configuration
+          mountPath: /app/config/connectors/
+        imagePullPolicy: Never
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "250m"
+          limits:
+            memory: "256Mi"
+            cpu: "500m"
+      volumes:
+      - name: enterprise-search-configuration
+        configMap:
+          name: enterprise-search-configuration-4bf99k7m9b
+      - name: enterprise-search-python-connectors-configuration
+        configMap:
+          name: enterprise-search-python-connectors-configuration-5gdd5tk82b

k8s/hpa.yml

@@ -0,0 +1,13 @@
+apiVersion: autoscaling/v1
+kind: HorizontalPodAutoscaler
+metadata:
+ name: hpa-connectors-ruby-consumer
+spec:
+ scaleTargetRef:
+   apiVersion: apps/v1
+   kind: Deployment
+   name: connectors-ruby-consumer
+ minReplicas: 1
+ maxReplicas: 5
+ targetCPUUtilizationPercentage: 30
+

k8s/namespace.yml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: enterprise-search
+  labels:
+    name: enterprise-search