From 60c82b45c4aa680be725e6382648c88dc497c03f Mon Sep 17 00:00:00 2001 From: catttam <caterinaalarconm@gmail.com> Date: Wed, 10 Jul 2024 09:32:53 +0200 Subject: [PATCH] Added creation of index.html --- create_index.py | 104 ++++++++++++++++++++++++++++++++++ extract_goaccess_metrics.sh | 4 +- extract_prometheus_metrics.sh | 2 +- metrics_prom.py | 2 +- 4 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 create_index.py diff --git a/create_index.py b/create_index.py new file mode 100644 index 0000000..53e2c86 --- /dev/null +++ b/create_index.py @@ -0,0 +1,104 @@ +import os + +cluster_id = os.getenv("CLUSTER_ID") +# Configuration +folder_path = '/home/calarcon/Documents/accounting_tools/s3_example_folders' +s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/' +assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets' # Local path to assets + +INDEX="index.html" +SUB_INDEX="_index.html" +# HTML template parts + +html_header = f"""<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0'> + <style> + + </style> + <link rel="shortcut icon" href="{assets_base_url}/images/logo.png" type="image/webp"> + <link rel="apple-touch-icon" sizes="180x180" href="{assets_base_url}/images/logo.png"> + <link rel="icon" href="{assets_base_url}/images/favicon.png" type="image/webp"> + <title>OSCAR metrics</title> + <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet"> + </head> + <body> + <div class="min-h-full flex-h-center" id="background_div"> + <input type="hidden" value="https://bio.link" id="app-url"> + <input type="hidden" value="null" id="is-featured"> + <canvas id="bg-canvas" class="background-overlay"></canvas> + </input> + </input> + + <div class="mt-48 page-full-wrap relative "> + <input type="hidden" value="creator-page" id="page-type"> + <img class="display-image m-auto" data-src="{assets_base_url}/images/logo.png" src="{assets_base_url}/images/logo.png" alt="OSCAR metrics"/> + <h2 class="page-title page-text-color page-text-font mt-24 text-center text-fs-18">OSCAR-ai4eosc metrics index</h2> + + <div class="mt-24"> +""" + +html_footer = """ </div> + </div> + </div> + </body> +</html> +""" +html_css = """ + +""" +html_file_entry_template = """ <div class="page-item-wrap relative"> + <div class="page-item flex-both-center absolute"></div> + <a target="_blank" class="page-item-each py-10 flex-both-center" href="{url}" data-id="{id}" data-type="page_item"> + <img class="link-each-image" data-src="{icon}" src="{icon}" alt="{filename}"/> + <span class="item-title text-center">{filename}</span> + </a> + </div> +""" +# Determine the correct icon based on file type +def get_icon(file_name): + if file_name.endswith('.html'): + return f"{assets_base_url}/images/dashboard.png" + elif file_name.endswith('.csv'): + return f"{assets_base_url}/images/file.png" + elif os.path.isdir(os.path.join(folder_path, file_name)): + return f"{assets_base_url}/images/folder.png" + else: + return f"{assets_base_url}/images/file.png" + +def generate_html(out_file, dir_path): + # Generate HTML content + html_content = html_header + + for i, file_name in enumerate(os.listdir(dir_path)): + file_path = os.path.join(dir_path, file_name) + if os.path.isfile(file_path) or os.path.isdir(file_path): + #file_url = file_path.replace("\\", "/") + if os.path.isdir(file_path): + relative_url=file_name+".html" + generate_html(relative_url, file_path) + file_url = s3_path+relative_url + else: + file_url = s3_path+file_name + icon = get_icon(file_name) + file_entry = html_file_entry_template.format(url=file_url, id=i, icon=icon, filename=file_name) + html_content += file_entry + + html_content += html_footer + + # Write the generated HTML to a file + with open(out_file, 'w') as f: + f.write(html_content) + + print(f"HTML file '{out_file}' has been generated.") + + + + +def main(): + generate_html(INDEX, folder_path) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh index a9ae585..dd8a1fe 100755 --- a/extract_goaccess_metrics.sh +++ b/extract_goaccess_metrics.sh @@ -93,8 +93,8 @@ if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then echo "Error: Failed to create html report." exit 1 fi -goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/index.html" +goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/dashboard.html" # Upload metrics to s3 -aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}" +aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/goaccess_csv" aws s3 cp --recursive "${METRICS_PATH}" s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/rawmetrics/ \ No newline at end of file diff --git a/extract_prometheus_metrics.sh b/extract_prometheus_metrics.sh index a8bc3cd..ca7ade3 100755 --- a/extract_prometheus_metrics.sh +++ b/extract_prometheus_metrics.sh @@ -1,4 +1,4 @@ #!/bin/bash cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}' python3 metrics_prom.py $ENDPOINT $VO $cluster_auth -aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}" \ No newline at end of file +aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv" \ No newline at end of file diff --git a/metrics_prom.py b/metrics_prom.py index 41f3a9a..b8593f4 100644 --- a/metrics_prom.py +++ b/metrics_prom.py @@ -7,7 +7,7 @@ from oscar_python.client import Client QUERY_ENDPOINT = "/api/v1/query?query=" -TIME = "30d" +TIME = "5d" parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)