Merge pull request #73 from eUgEntOptIc44/main

update pip dependency Markdown and add line numbers
twfgcicdbot · Nov 13, 2023 · 2d00ac3 · 2d00ac3
2 parents 6e46d0d + 96df9ad
commit 2d00ac3
Show file tree

Hide file tree

Showing 10 changed files with 2,479 additions and 292 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -0,0 +1,24 @@
+{
+    "image": "mcr.microsoft.com/devcontainers/universal:2",
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "tamasfe.even-better-toml",
+                "eamodio.gitlens",
+                "DavidAnson.vscode-markdownlint",
+                "ms-python.python",
+                "ms-python.vscode-pylance",
+                "ms-python.black-formatter",
+                "charliermarsh.ruff",
+                "timonwong.shellcheck",
+                "SonarSource.sonarlint-vscode"
+            ],
+            "settings": {
+                "sonarlint.disableTelemetry": true,
+                "gitlens.telemetry.enabled": false,
+                "redhat.telemetry.enabled": false
+            }
+        }
+    },
+    "postCreateCommand": "sudo apt update && sudo apt upgrade -y && sudo apt install -y apktool dexdump ripgrep && virtualenv venv && source venv/bin/activate && python -m pip install --upgrade pip && pip install -r requirements.txt && python __init__.py"
+}
diff --git a/.github/workflows/exodusscan.yml b/.github/workflows/exodusscan.yml
@@ -22,11 +22,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: clone remote repo
-        run: git clone https://github.com/twfgcicdbot/TinyWeatherForecastGermanyMirror TinyWeatherForecastGermanyMirror
-
       - name: clone this repo
         run: git clone https://github.com/$GITHUB_REPOSITORY TinyWeatherForecastGermanyScan
+
+      - name: clone remote repo
+        run: cd TinyWeatherForecastGermanyScan && git clone https://github.com/twfgcicdbot/TinyWeatherForecastGermanyMirror TinyWeatherForecastGermanyMirror || true
 
       - name: update packages
         run: timeout 120s sudo apt update || true
@@ -68,7 +68,7 @@ jobs:
 
       - name: list directory contents
         run: ls -lisha
-      
+
       - name: install python requirements
         run: pip install -r TinyWeatherForecastGermanyScan/requirements.txt
 

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 ### TWFG ###
 TinyWeatherForecastGermanyScan/
 TinyWeatherForecastGermanyApk/
+TinyWeatherForecastGermanyMirror/
 
 ### Git ###
 # Created by git for backups. To disable backups in Git:

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # TinyWeatherForecastGermanyScan
 
-[![exodus-privacy scan](https://img.shields.io/github/workflow/status/twfgcicdbot/TinyWeatherForecastGermanyScan/exodus-privacy%20scan?label=exodus-privacy%20scan&logo=github&style=for-the-badge)](https://github.com/twfgcicdbot/TinyWeatherForecastGermanyScan/actions/workflows/exodusscan.yml) [![github pages page](https://img.shields.io/badge/-github%20pages-green?style=for-the-badge&logo=github)](https://twfgcicdbot.github.io/TinyWeatherForecastGermanyScan/index.html) [![F-Droid Store release version](https://img.shields.io/f-droid/v/de.kaffeemitkoffein.tinyweatherforecastgermany?color=%23efbb24&logo=fdroid&style=for-the-badge)](https://f-droid.org/packages/de.kaffeemitkoffein.tinyweatherforecastgermany) [![code license](https://img.shields.io/github/license/twfgcicdbot/TinyWeatherForecastGermanyMirror?style=for-the-badge&logo=github)](https://github.com/twfgcicdbot/TinyWeatherForecastGermanyMirror/blob/master/COPYING) 
+[![exodus-privacy scan](https://img.shields.io/github/workflow/status/twfgcicdbot/TinyWeatherForecastGermanyScan/exodus-privacy%20scan?label=exodus-privacy%20scan&logo=github&style=for-the-badge)](https://github.com/twfgcicdbot/TinyWeatherForecastGermanyScan/actions/workflows/exodusscan.yml) [![github pages page](https://img.shields.io/badge/-github%20pages-green?style=for-the-badge&logo=github)](https://twfgcicdbot.github.io/TinyWeatherForecastGermanyScan/index.html) [![F-Droid Store release version](https://img.shields.io/f-droid/v/de.kaffeemitkoffein.tinyweatherforecastgermany?color=%23efbb24&logo=fdroid&style=for-the-badge)](https://f-droid.org/packages/de.kaffeemitkoffein.tinyweatherforecastgermany) [![code license](https://img.shields.io/github/license/twfgcicdbot/TinyWeatherForecastGermanyMirror?style=for-the-badge&logo=github)](https://github.com/twfgcicdbot/TinyWeatherForecastGermanyMirror/blob/master/COPYING)
 
 tl;dr -> scan code of [**TinyWeatherForecastGermany**](https://codeberg.org/Starfish/TinyWeatherForecastGermany) (TWFG) for tracker signatures provided by [**ExodusPrivacy**](https://exodus-privacy.eu.org/en/) using the python module [exodus-core](https://github.com/Exodus-Privacy/exodus-core/)
 
@@ -9,6 +9,7 @@ tl;dr -> scan code of [**TinyWeatherForecastGermany**](https://codeberg.org/Star
 The DWD is Germany's **national weather agency** (similiar to **N**ational **O**ceanic **a**nd **A**tmospheric **A**dministration ([NOAA](https://www.noaa.gov/about-our-agency)) in the US).
 
 **Note**:
+
 * *'TWFG'* is an inofficial abbreviation for ***T**iny **W**eather **F**orecast **G**ermany.*
 * *[ExodusPrivacy](https://github.com/Exodus-Privacy)* also partially uses the alternative spelling *'εxodus'*. To maintain readability the author sticks to the first version without the leading greek character.
 
@@ -66,7 +67,7 @@ This project is **not affialited** with any of the following organizations: Clas
 ## Contributing
 
 * All contributions to **Tiny Weather Forecast Germany** are managed at the ['main'](https://codeberg.org/Starfish/TinyWeatherForecastGermany) code repository at [codeberg.org](https://codeberg.org/Starfish/TinyWeatherForecastGermany)
-* [**Translations**](https://translate.codeberg.org/engage/tiny-weather-forecast-germany/) are managed at the [**weblate** instance](https://translate.codeberg.org/projects/tiny-weather-forecast-germany/) provided by Codeberg e.V. 
+* [**Translations**](https://translate.codeberg.org/engage/tiny-weather-forecast-germany/) are managed at the [**weblate** instance](https://translate.codeberg.org/projects/tiny-weather-forecast-germany/) provided by Codeberg e.V.
 * Feel free to contribute to this script by opening issues and/or merge requests.
 * Please also see the automatically generated *javadoc* **code documentation** of Tiny Weather Forecast Germany [at GitLab](https://gitlab.com/tinyweatherforecastgermanygroup/twfg-javadoc).
 * For cybersec, privacy and/or copyright related issues regarding this repository please directly contact the maintainer [**Jean-Luc Tibaux** (@eUgEntOptIc44)](https://codeberg.org/eUgEntOptIc44) or our [CI/CD bot](https://github.com/twfgcicdbot).
diff --git a/__init__.py b/__init__.py
@@ -12,7 +12,7 @@
 
 ## Disclaimer
 
-No warranty or guarantee of any kind provided. Use at your own risk only.
+No warranty of any kind provided. Use at your own risk only.
 Not meant to be used in commercial or in general critical/productive environments at all.
 
 """
@@ -69,7 +69,7 @@
 
 user_agent = str(cryptogen.choice(user_agents))
 
-logging.debug(f"querying data as '{user_agent}' ")
+logging.debug(f"querying data as '{user_agent}'")
 
 headers = {
     "User-Agent": user_agent,
@@ -86,7 +86,7 @@
     # pprint(searchResultCodebergJson)
     logging.debug("fetched Codeberg data")
 except Exception as error_msg:
-    logging.error(f"codeberg api request failed! -> error: {error_msg} ")
+    logging.error(f"codeberg api request failed! -> error: {error_msg}")
 
 if len(search_cb_json) == 1 and search_cb_json is not None:
     twfg_json = search_cb_json[0]
@@ -95,8 +95,7 @@
 
     if twfg_json is None:
         logging.error(
-            "content of key 'results' in codeberg json response is 'None' ")
-
+            "content of key 'results' in codeberg json response is 'None'")
         try:
             pprint(str(search_cb_req.headers))
             pprint(str(search_cb_req.text))
@@ -535,10 +534,26 @@ def print_classes_tree(tree, result, level):
                                         if leaf_name in classes_dict:
                                             class_path = classes_dict[leaf_name]
                                             if class_path[0:3] == 'de/' or 'nodomain/freeyourgadget' in class_path or 'org/astronomie' in class_path:
-                                                source_a = ' -> <a class="subclass-source" title="open source at codeberg.org" target="_blank" href="https://codeberg.org/Starfish/TinyWeatherForecastGermany/src/branch/master/app/src/main/java/' + \
-                                                    str(class_path)+'/' + \
-                                                    str(leaf_name) + \
-                                                    '.java">source</a>'
+                                                c_java_p = Path(f"app/src/main/java/{class_path}/{leaf_name}.java")
+                                                c_java_p_local = "TinyWeatherForecastGermanyMirror" /  c_java_p
+
+                                                # parsing files to get line numbers
+                                                if c_java_p_local.exists():
+                                                    try:
+                                                        java_lines = []
+                                                        with open(c_java_p_local, "r", encoding="utf-8") as file_handle:
+                                                            java_lines = file_handle.read().split("\n")
+                                                        for jl_index, java_line in enumerate(java_lines):
+                                                            if f"class {c_java_p_local.stem}" in java_line:
+                                                                logging.debug(f"found class in line #{jl_index+1} -> '{java_line}'")
+                                                                c_java_p = f"{c_java_p}#L{jl_index+1}"
+                                                                break
+                                                    except Exception as error_msg:
+                                                        logging.error(f"while searching line number of class '{c_java_p_local.stem}' in {c_java_p_local} -> error: {error_msg}")
+
+                                                source_a = ' -> <a class="subclass-source" title="open source at codeberg.org" target="_blank" href="https://codeberg.org/Starfish/TinyWeatherForecastGermany/src/branch/master/' + \
+                                                    str(c_java_p) + \
+                                                    '">source</a>'
                                                 docs_a = ' -> <a class="subclass-docs" title="open javadocs" target="_blank" href="https://tinyweatherforecastgermanygroup.gitlab.io/twfg-javadoc/' + \
                                                     str(class_path)+'/' + \
                                                     str(leaf_name) + \

diff --git a/androguard-tests.py b/androguard-tests.py
@@ -41,8 +41,8 @@ def addLeafs(t, List):
     try:
         classPartsTemp = list(embeddedClassTemp.split("/"))
         addLeafs(classesTree, classPartsTemp)
-    except Exception as e:
-        print("ERROR:" + str(e))
+    except Exception as error_msg:
+        print(f"ERROR: {error_msg}")
 
 #pprint(classesTree)
 
@@ -72,5 +72,5 @@ def printClassesTree(tree, result, level):
 
 printClassesResult = str(BeautifulSoup(printClassesResult, features="html.parser").prettify())
 
-with open(str(Path("index.html").absolute()), "w+", encoding="utf-8") as fh:
-    fh.write(printClassesResult)
+with open(str(Path("index.html").absolute()), "w+", encoding="utf-8") as file_handle:
+    file_handle.write(printClassesResult)
diff --git a/logtohtml.py b/logtohtml.py
@@ -14,44 +14,46 @@
 
 ## Disclaimer
 
-No warranty or guarantee of any kind provided. Use at your own risk only.
+No warranty of any kind provided. Use at your own risk only.
 Not meant to be used in commercial or in general critical/productive environments at all.
 
 """
 
 import logging
-from pathlib import Path
 import sys
+from pathlib import Path
 
-from bs4 import BeautifulSoup  # for html parsing
 import htmlmin  # html minifier
 import regex  # extends feature set of 're' -> regular expressions
+from bs4 import BeautifulSoup  # for html parsing
 
 from pygments import highlight  # python code syntax hightlighter
-from pygments.lexers import get_lexer_by_name
 from pygments.formatters import HtmlFormatter
+from pygments.lexers import get_lexer_by_name
 
 working_dir = Path("TinyWeatherForecastGermanyScan")
 # create directory if not exists
 working_dir.mkdir(parents=True, exist_ok=True)
 
 log_p2 = working_dir / "debug2.log"
 try:
-    logging.basicConfig(format='%(asctime)-s %(levelname)s [%(name)s]: %(message)s',
-                        level=logging.DEBUG,
-                        handlers=[
-                            logging.FileHandler(str(log_p2), encoding="utf-8"),
-                            logging.StreamHandler()
-                        ])
+    logging.basicConfig(
+        format="%(asctime)-s %(levelname)s [%(name)s]: %(message)s",
+        level=logging.DEBUG,
+        handlers=[
+            logging.FileHandler(str(log_p2), encoding="utf-8"),
+            logging.StreamHandler(),
+        ],
+    )
 except Exception as error_msg:
-    logging.error(f"while logger init! -> error: {error_msg} ")
+    logging.error(f"while logger init! -> error: {error_msg}")
 
 log_p1 = working_dir / "debug.log"
 try:
     with open(str(log_p1), "r", encoding="utf-8") as file_handle:
         code = str(file_handle.read())
 except Exception as error_msg:
-    logging.error(f"failed to open '{log_p1.absolute()}' -> error: {error_msg} ")
+    logging.error(f"failed to open '{log_p1.absolute()}' -> error: {error_msg}")
     sys.exit("FATAL ERROR script execution aborted!")
 
 codeLines = str(code).split("\n")
@@ -62,31 +64,40 @@
     line_tmp = str(codeLines[lineIndex]).lower()
 
     if "error:" in line_tmp or "warning:" in line_tmp:
-        hlLinesIndices.append(lineIndex+1)
-        logging.debug(f"identified 'error:' match in line #{lineIndex+1} ")
+        hlLinesIndices.append(lineIndex + 1)
+        logging.debug(f"identified 'error:' match in line #{lineIndex+1}")
 
 # using 'logtalk' as no dedicated python log lexer exists
 lexer = get_lexer_by_name("logtalk", stripall=True)
 logging.debug("lexer init completed")
 
-formatter = HtmlFormatter(linenos=True, cssclass="sourcecode", full=True,
-                          style="perldoc", title="debug.log | Tiny Weather Forecast Germany",
-                          lineanchors="debuglog", lineseparator="<br>",
-                          hl_lines=hlLinesIndices, wrapcode=True)
+formatter = HtmlFormatter(
+    linenos=True,
+    cssclass="sourcecode",
+    full=True,
+    style="perldoc",
+    title="debug.log | Tiny Weather Forecast Germany",
+    lineanchors="debuglog",
+    lineseparator="<br>",
+    hl_lines=hlLinesIndices,
+    wrapcode=True,
+)
 logging.debug("HtmlFormatter init completed")
 
 result = highlight(code, lexer, formatter)
 logging.debug("highlight finished")
 
 # parse html to modify elements
-debug_file_soup = BeautifulSoup(str(result), features='html.parser')
+debug_file_soup = BeautifulSoup(str(result), features="html.parser")
 
-for metaTemp in debug_file_soup.select('head > meta'):
+for metaTemp in debug_file_soup.select("head > meta"):
     metaTemp.decompose()
 
-head_html = ('\n\n<meta http-equiv="content-type" content="text/html; charset=UTF-8" />\n'
-            '<meta charset="utf-8">\n<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'
-            '<meta name="robots" content="noindex, nofollow">')
+head_html = (
+    '\n\n<meta http-equiv="content-type" content="text/html; charset=UTF-8" />\n'
+    '<meta charset="utf-8">\n<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'
+    '<meta name="robots" content="noindex, nofollow">'
+)
 
 head_html += """\n
 <link rel="apple-touch-icon" sizes="180x180" href="images/apple-touch-icon.png">
@@ -105,10 +116,11 @@
 <meta name="thumbnail" content="images/icon.png">
 """
 
-debug_file_soup.title.insert_after(BeautifulSoup(
-    head_html, features='html.parser'))  # parse html to modify elements
+debug_file_soup.title.insert_after(
+    BeautifulSoup(head_html, features="html.parser")
+)  # parse html to modify elements
 
-logging.debug("added additional 'meta' tags ")
+logging.debug("added additional 'meta' tags")
 
 first_css_el = debug_file_soup.select("head style")
 if len(first_css_el) > 0:
@@ -143,23 +155,27 @@
 """
 
 css_txt += css_str
-css_txt = regex.sub(r'(?im)[\r\t\n]+', '', css_txt)
-css_txt = regex.sub(r'(?im)( )*(  ){2}', ' ', css_txt)
-css_txt = regex.sub(r'(?m)\/\*([^\*]+)\*\/', '', css_txt)
+css_txt = regex.sub(r"(?im)[\r\t\n]+", "", css_txt)
+css_txt = regex.sub(r"(?im)( )*(  ){2}", " ", css_txt)
+css_txt = regex.sub(r"(?m)\/\*([^\*]+)\*\/", "", css_txt)
 
 # parse html to modify elements
-debug_file_soup.head.append(BeautifulSoup(f'<style type=\"text/css\">{css_txt}</style>',
-                                          features='html.parser'))
+debug_file_soup.head.append(
+    BeautifulSoup(f'<style type="text/css">{css_txt}</style>', features="html.parser")
+)
 
 logging.debug("added additional 'css' tag ")
 
 debug_html_file = str(Path(working_dir / "debug.html").absolute())
 try:
     with open(debug_html_file, "w+", encoding="utf-8") as file_handle:
-        file_handle.write(htmlmin.minify(str(debug_file_soup),
-                 remove_empty_space=True, remove_comments=True))
+        file_handle.write(
+            htmlmin.minify(
+                str(debug_file_soup), remove_empty_space=True, remove_comments=True
+            )
+        )
 except Exception as error_msg:
-    logging.error(f"minification of '{debug_html_file}' failed -> error: {error_msg} ")
+    logging.error(f"minification of '{debug_html_file}' failed -> error: {error_msg}")
     with open(debug_html_file, "w+", encoding="utf-8") as file_handle:
         file_handle.write(str(debug_file_soup))