diff --git a/.tekton/.currency/currency-pipeline.yaml b/.tekton/.currency/currency-pipeline.yaml new file mode 100644 index 00000000..ba5c3e85 --- /dev/null +++ b/.tekton/.currency/currency-pipeline.yaml @@ -0,0 +1,36 @@ +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: ruby-currency-pipeline +spec: + params: + - name: revision + type: string + workspaces: + - name: ruby-currency-pvc + tasks: + - name: clone-repo + params: + - name: revision + value: $(params.revision) + taskRef: + name: ruby-git-clone-task + workspaces: + - name: task-pvc + workspace: ruby-currency-pvc + - name: generate-currency-report + runAfter: + - clone-repo + taskRef: + name: ruby-generate-currency-report-task + workspaces: + - name: task-pvc + workspace: ruby-currency-pvc + - name: upload-currency-report + runAfter: + - generate-currency-report + taskRef: + name: ruby-upload-currency-report-task + workspaces: + - name: task-pvc + workspace: ruby-currency-pvc diff --git a/.tekton/.currency/currency-pipelinerun.yaml b/.tekton/.currency/currency-pipelinerun.yaml new file mode 100644 index 00000000..f9b59e4c --- /dev/null +++ b/.tekton/.currency/currency-pipelinerun.yaml @@ -0,0 +1,20 @@ +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: ruby-currency-pipelinerun +spec: + params: + - name: revision + value: "master" + pipelineRef: + name: ruby-currency-pipeline + serviceAccountName: currency-serviceaccount + workspaces: + - name: ruby-currency-pvc + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi diff --git a/.tekton/.currency/currency-rbac.yaml b/.tekton/.currency/currency-rbac.yaml new file mode 100644 index 00000000..aca210e4 --- /dev/null +++ b/.tekton/.currency/currency-rbac.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: currency-serviceaccount +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: currency-clusterrole +rules: +- apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list"] +- apiGroups: ["tekton.dev"] + resources: ["taskruns"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: currency-clusterrolebinding +subjects: +- kind: ServiceAccount + name: currency-serviceaccount + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: currency-clusterrole diff --git a/.tekton/.currency/currency-scheduled-eventlistener.yaml b/.tekton/.currency/currency-scheduled-eventlistener.yaml new file mode 100644 index 00000000..d73b60d5 --- /dev/null +++ b/.tekton/.currency/currency-scheduled-eventlistener.yaml @@ -0,0 +1,56 @@ +apiVersion: triggers.tekton.dev/v1beta1 +kind: EventListener +metadata: + name: ruby-currency-cron-listener +spec: + serviceAccountName: tekton-triggers-eventlistener-serviceaccount + triggers: + - name: currency-cron-trigger + template: + ref: ruby-currency-trigger-template +--- +apiVersion: triggers.tekton.dev/v1beta1 +kind: TriggerTemplate +metadata: + name: ruby-currency-trigger-template +spec: + resourcetemplates: + - apiVersion: tekton.dev/v1beta1 + kind: PipelineRun + metadata: + generateName: ruby-currency- + spec: + pipelineRef: + name: ruby-currency-pipeline + serviceAccountName: currency-serviceaccount + params: + - name: revision + value: "master" + workspaces: + - name: ruby-currency-pvc + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ruby-currency-cronjob +spec: + schedule: "5 3 * * Mon-Fri" + jobTemplate: + spec: + template: + spec: + containers: + - name: http-request-to-el-svc + # curlimages/curl:8.6.0 + image: curlimages/curl@sha256:f2237028bed58de91f62aea74260bb2a299cf12fbcabc23cfaf125fef276c884 + imagePullPolicy: IfNotPresent + args: ["curl", "-X", "POST", "--data", "{}", "el-ruby-currency-cron-listener.default.svc.cluster.local:8080"] + restartPolicy: OnFailure +--- diff --git a/.tekton/.currency/currency-tasks.yaml b/.tekton/.currency/currency-tasks.yaml new file mode 100644 index 00000000..4b8ea091 --- /dev/null +++ b/.tekton/.currency/currency-tasks.yaml @@ -0,0 +1,94 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: ruby-git-clone-task +spec: + params: + - name: revision + type: string + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: clone-repo + # alpine/git:2.43.0 + image: alpine/git@sha256:6ff4de047dcc8f0c7d75d2efff63fbc189e87d2f458305f2cc8f165ff83309cf + script: | + #!/bin/sh + echo "Cloning repo" + cd /workspace && git clone --filter=blob:none --sparse --depth 1 https://github.com/instana/ruby-sensor -b $(params.revision) + cd ruby-sensor + git sparse-checkout add .tekton/.currency + ls -lah /workspace/ruby-sensor +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: ruby-generate-currency-report-task +spec: + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: generate-currency-report + # 3.10.13-bookworm + image: python@sha256:c970ff53939772f47b0672e380328afb50d8fd1c0568ed4f82c22effc54244fc + script: | + #!/usr/bin/env bash + + cd /workspace/ruby-sensor/.tekton/.currency + + python -m venv /tmp/venv + source /tmp/venv/bin/activate + pip install -r resources/requirements.txt + + python scripts/generate_report.py + if [ $? -ne 0 ]; then + echo "Error occured while generating the ruby tracer currency report." >&2 + exit 1 + fi + cat docs/report.md +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: ruby-upload-currency-report-task +spec: + params: + - name: github-token-secret + default: instanacd-github-api-token + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: upload-currency-report + # alpine/git:2.43.0 + image: alpine/git@sha256:6ff4de047dcc8f0c7d75d2efff63fbc189e87d2f458305f2cc8f165ff83309cf + env: + - name: GH_ENTERPRISE_TOKEN + valueFrom: + secretKeyRef: + name: $(params.github-token-secret) + key: "GH_ENTERPRISE_TOKEN" + script: | + #!/bin/sh + + cd /workspace + git clone https://oauth2:$GH_ENTERPRISE_TOKEN@github.ibm.com/instana/tracer-reports.git + + if [ $? -ne 0 ]; then + echo "The attempt to clone the tracer-reports repository failed, preventing the upload of ruby tracer currency report." >&2 + exit 1 + fi + + cd tracer-reports + + cp ../ruby-sensor/.tekton/.currency/docs/report.md ./automated/currency/ruby/report.md + + git config user.name "Instanacd PAT for GitHub Enterprise" + git config user.email instana.ibm.github.enterprise@ibm.com + + git add . + + git commit -m "chore: Updated ruby currency report" + git push origin main diff --git a/.tekton/.currency/docs/report.md b/.tekton/.currency/docs/report.md new file mode 100644 index 00000000..0794c6ca --- /dev/null +++ b/.tekton/.currency/docs/report.md @@ -0,0 +1,19 @@ +##### This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [ruby tracer](https://github.com/instana/ruby-sensor) repo. +## Ruby supported packages and versions +| Package name | Support Policy | Beta version | Last Supported Version | Latest version | Up-to-date | Cloud Native | +|:---------------|:-----------------|:---------------|:-------------------------|:-----------------|:-------------|:---------------| +| Cuba | On demand | No | 4.0.3 | 4.0.3 | Yes | No | +| Rack | 0-day | No | 3.1.3 | 3.1.3 | Yes | No | +| Rails | 30-days | No | 7.1.3.4 | 7.1.3.4 | Yes | No | +| Rails::API | 30-days | No | 0.4.1 | 0.4.1 | Yes | No | +| Rails LTS | On demand | No | 5.2 | 5.2 | Yes | No | +| Roda | 30-days | No | 3.81.0 | 3.81.0 | Yes | No | +| Sinatra | 30-days | No | 4.0.0 | 4.0.0 | Yes | No | +| Excon | 30-days | No | 0.110.0 | 0.110.0 | Yes | Yes | +| gRPC | 30-days | No | 1.64.0 | 1.64.0 | Yes | No | +| Net::HTTP | 0-day | No | 0.4.1 | 0.4.1 | Yes | No | +| Rest-Client | 30-days | No | 2.1.0 | 2.1.0 | Yes | No | +| Dalli | 30-days | No | 3.2.8 | 3.2.8 | Yes | No | +| Resque | 30-days | No | 2.6.0 | 2.6.0 | Yes | Yes | +| Sidekiq | 30-days | No | 7.2.4 | 7.2.4 | Yes | Yes | +| GraphQL | 30-days | No | 2.3.5 | 2.3.5 | Yes | Yes | \ No newline at end of file diff --git a/.tekton/.currency/resources/requirements.txt b/.tekton/.currency/resources/requirements.txt new file mode 100644 index 00000000..06d8600c --- /dev/null +++ b/.tekton/.currency/resources/requirements.txt @@ -0,0 +1,5 @@ +requests +pandas +beautifulsoup4 +tabulate +kubernetes diff --git a/.tekton/.currency/resources/table.json b/.tekton/.currency/resources/table.json new file mode 100644 index 00000000..2bfd508f --- /dev/null +++ b/.tekton/.currency/resources/table.json @@ -0,0 +1,94 @@ +{ + "table": [ + { + "Package name": "Cuba", + "Support Policy": "On demand", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Rack", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Rails", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Rails::API", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Rails LTS", + "Support Policy": "On demand", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Roda", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Sinatra", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Excon", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "gRPC", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Net::HTTP", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Rest-Client", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Dalli", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Resque", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Sidekiq", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "GraphQL", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + } + ] +} diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py new file mode 100644 index 00000000..fb3a7f84 --- /dev/null +++ b/.tekton/.currency/scripts/generate_report.py @@ -0,0 +1,228 @@ +# Standard Libraries +import re +from json import load + +# Third Party +from requests import get +from pandas import DataFrame +from bs4 import BeautifulSoup +from kubernetes import client, config + +JSON_FILE = "resources/table.json" +REPORT_FILE = "docs/report.md" +API_V1_ENDPOINT = "https://rubygems.org/api/v1/versions/" + +def filter_taskruns(taskrun_filter, taskruns): + filtered_taskruns = list(filter(taskrun_filter, taskruns)) + filtered_taskruns.sort( + key=lambda tr: tr["metadata"]["creationTimestamp"], reverse=True + ) + + return filtered_taskruns + + +def get_taskruns(namespace, task_name): + group = "tekton.dev" + version = "v1" + plural = "taskruns" + + # access the custom resource from tekton + tektonV1 = client.CustomObjectsApi() + taskruns = tektonV1.list_namespaced_custom_object( + group, + version, + namespace, + plural, + label_selector=f"{group}/task={task_name}, triggers.tekton.dev/trigger=ruby-tracer-scheduled-pipeline-triggger", + )["items"] + + return taskruns + + +def process_taskrun_logs( + taskruns, core_v1_client, namespace, library, tekton_ci_output +): + for tr in taskruns: + pod_name = tr["status"]["podName"] + taskrun_name = tr["metadata"]["name"] + logs = core_v1_client.read_namespaced_pod_log( + pod_name, namespace, container="step-unittest" + ) + if "Installing" not in logs: + print( + f"Unable to retrieve logs from taskrun pod {pod_name} of taskrun {taskrun_name} for gem {library}." + ) + continue + + print( + f"Retrieving logs from taskrun pod {pod_name} of taskrun {taskrun_name} for gem {library}.." + ) + + match = re.search(f"Installing ({library} [^\s]+)", logs) + tekton_ci_output += f"{match[1]}\n" + break + + return tekton_ci_output + + +def get_tekton_ci_output(): + config.load_incluster_config() + + namespace = "default" + core_v1_client = client.CoreV1Api() + + ruby_33_prefix = "unittest-default-ruby-33-" + ruby_31_prefix = "unittest-default-ruby-31-" + + default_libraries_dict = { + "cuba": f"{ruby_33_prefix}1", + "excon": f"{ruby_33_prefix}4", + "graphql": f"{ruby_33_prefix}6", + "grpc": f"{ruby_33_prefix}7", + "rack": f"{ruby_33_prefix}10", + "rest-client": f"{ruby_33_prefix}11", + "roda": f"{ruby_33_prefix}13", + "sinatra": f"{ruby_33_prefix}16", + "net-http": f"{ruby_31_prefix}8", + } + + tekton_ci_output = "" + task_name = "ruby-tracer-unittest-default-libraries-task" + default_taskruns = get_taskruns(namespace, task_name) + + for library, pattern in default_libraries_dict.items(): + taskrun_filter = ( + lambda tr: tr["metadata"]["name"].endswith(pattern) + and tr["status"]["conditions"][0]["type"] == "Succeeded" + ) + filtered_default_taskruns = filter_taskruns(taskrun_filter, default_taskruns) + + tekton_ci_output = process_taskrun_logs( + filtered_default_taskruns, + core_v1_client, + namespace, + library, + tekton_ci_output, + ) + + other_libraries_dict = { + "rails": { + "pattern": "rails-postgres-11", + "task_name": "ruby-tracer-unittest-rails-postgres-task", + }, + "dalli": { + "pattern": "memcached-11", + "task_name": "ruby-tracer-unittest-memcached-libraries-task", + }, + "resque": { + "pattern": "unittest-redis-ruby-32-33-9", + "task_name": "ruby-tracer-unittest-redis-libraries-task", + }, + "sidekiq": { + "pattern": "unittest-redis-ruby-32-33-18", + "task_name": "ruby-tracer-unittest-redis-libraries-task", + }, + } + + for library, inner_dict in other_libraries_dict.items(): + pattern = inner_dict["pattern"] + task_name = inner_dict["task_name"] + taskrun_filter = ( + lambda tr: tr["metadata"]["name"].endswith(pattern) + and tr["status"]["conditions"][0]["type"] == "Succeeded" + ) + other_taskruns = get_taskruns(namespace, task_name) + filtered_other_taskruns = filter_taskruns(taskrun_filter, other_taskruns) + + tekton_ci_output = process_taskrun_logs( + filtered_other_taskruns, + core_v1_client, + namespace, + library, + tekton_ci_output + ) + + return tekton_ci_output + + +def get_upstream_version(dependency): + """get the latest version available upstream""" + if dependency != "rails lts": + response = get(f"{API_V1_ENDPOINT}/{dependency}/latest.json") + response_json = response.json() + latest_version = response_json["version"] + else: + url = "https://makandracards.com/railslts/16137-installing-rails-lts/read" + page = get(url) + soup = BeautifulSoup(page.text, "html.parser") + text = soup.findAll("li")[-1].text + pattern = "(\d+\.\d+\.?\d*)" + latest_version = re.search(pattern, text)[1] + return latest_version + + +def get_last_supported_version(tekton_ci_output, dependency): + """get up-to-date supported version""" + pattern = r" ([^\s]+)" + + last_supported_version = re.search(dependency + pattern, tekton_ci_output, flags=re.I | re.M) + + return last_supported_version[1] + + +def isUptodate(last_supported_version, latest_version): + if last_supported_version == latest_version: + up_to_date = "Yes" + else: + up_to_date = "No" + + return up_to_date + +def main(): + # Read the JSON file + with open(JSON_FILE) as file: + data = load(file) + + tekton_ci_output = get_tekton_ci_output() + + items = data["table"] + + for item in items: + package = item["Package name"] + package = package.lower().replace("::", "-") + + latest_version = get_upstream_version(package) + + if not package in ["rails lts", "rails-api"]: + last_supported_version = get_last_supported_version(tekton_ci_output, package) + else: + last_supported_version = latest_version + + up_to_date = isUptodate(last_supported_version, latest_version) + + item.update( + { + "Last Supported Version": last_supported_version, + "Latest version": latest_version, + "Up-to-date": up_to_date, + }, + ) + + # Create a DataFrame from the list of dictionaries + df = DataFrame(items) + df.insert(len(df.columns) - 1, "Cloud Native", df.pop("Cloud Native")) + + # Convert dataframe to markdown + markdown_table = df.to_markdown(index=False) + + disclaimer = "##### This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [ruby tracer](https://github.com/instana/ruby-sensor) repo." + title = "## Ruby supported packages and versions" + + # Combine disclaimer, title, and markdown table with line breaks + final_markdown = disclaimer + "\n" + title + "\n" + markdown_table + + with open(REPORT_FILE, "w") as file: + file.write(final_markdown) + +if __name__ == "__main__": + main() diff --git a/.tekton/run_unittests.sh b/.tekton/run_unittests.sh index 619a91fd..0fb4c54d 100755 --- a/.tekton/run_unittests.sh +++ b/.tekton/run_unittests.sh @@ -50,9 +50,6 @@ echo "with dependencies in '${BUNDLE_GEMFILE}'" gem update --system > /dev/null echo "Gem version $(gem --version)" -# List the built-in gem version of "net-http" -gem list | grep net-http - # Configure Bundler bundler --version bundle config set path '/tmp/vendor/bundle'