openhub/: Use openhub_django pypi package

.ci/build.sh

@@ -27,6 +27,7 @@ python manage.py fetch_deployed_data _site $ISSUES_JSON \
 
 python manage.py migrate
 python manage.py import_contributors_data
+python manage.py create_org_cluster_map_and_activity_graph org_map
 python manage.py import_issues_data
 python manage.py import_merge_requests_data
 python manage.py create_config_data
@@ -43,6 +44,8 @@ fi
 
 # Run meta review system
 python manage.py run_meta_review_system
+# Run openhub system
+python manage.py import_openhub_data
 
 rm _site/$ISSUES_JSON
 

.coafile

@@ -1,6 +1,6 @@
 [all]
 files = **.py, **.js, **.sh
-ignore = .git/**, **/__pycache__/**, gci/client.py, */migrations/**, private/*
+ignore = .git/**, **/__pycache__/**, gci/client.py, */migrations/**, private/*, **/leaflet_dist/**
 max_line_length = 80
 use_spaces = True
 preferred_quotation = '
@@ -42,6 +42,7 @@ files = static/**/*.js
 bears = JSHintBear
 allow_unused_variables = True
 javascript_strictness = False
+environment_jquery = True
 
 [all.yml]
 bears = YAMLLintBear
@@ -58,7 +59,7 @@ shell = bash
 # for use by other organizations.
 files = **
 # .coverage crashes AnnotationBear
-ignore += .coafile, *requirements.txt, .travis.yml, LICENSE, .nocover.yaml, .moban.yaml, .moban.dt/community-*.jj2, public/**, _site/**, .ci/check_moban.sh, .coverage
+ignore += .coafile, *requirements.txt, .travis.yml, LICENSE, .nocover.yaml, .moban.yaml, .moban.dt/community-*.jj2, public/**, _site/**, .ci/check_moban.sh, .coverage, static/js/main.js
 bears = KeywordBear
 language = python 3
 keywords = coala

.gitignore

@@ -63,6 +63,7 @@ coverage.xml
 *.log
 local_settings.py
 db.sqlite3
+db.sqlite3-journal
 
 # Flask stuff:
 instance/
@@ -272,6 +273,7 @@ flycheck_*.el
 
 # Session
 Session.vim
+Sessionx.vim
 
 # Temporary
 .netrwhist
@@ -429,11 +431,7 @@ DerivedData/
 *.perspectivev3
 !default.perspectivev3
 
-## Xcode Patch
-*.xcodeproj/*
-!*.xcodeproj/project.pbxproj
-!*.xcodeproj/xcshareddata/
-!*.xcworkspace/contents.xcworkspacedata
+## Gcc Patch
 /*.gcno
 
 # Eclipse rules

.moban.yaml

@@ -9,13 +9,12 @@ packages:
   - gci
   - gsoc
   - gamification
-  - log
+  - ci_build
   - meta_review
-  - model
-  - twitter
   - unassigned_issues
 
 dependencies:
+  - getorg~=0.3.1
   - git+https://gitlab.com/coala/coala-utils.git
   - git-url-parse
   - django>2.1,<2.2
@@ -24,6 +23,7 @@ dependencies:
   - git+https://gitlab.com/gitmate/open-source/IGitt.git@1fa5a0a21ea4fb8739d467c06972f748717adbdc
   - requests
   - git+https://github.com/andrewda/trav.git@ce805d12d3d1db0a51b1aa26bba9cd9ecc0d96b8
+  - openhub-django~=0.1.0
   - python-dateutil
   - pillow
   - ruamel.yaml

.nocover.yaml

@@ -8,11 +8,8 @@ nocover_file_globs:
   - community/git.py
   - gci/*.py
   - gsoc/*.py
-  - log/*.py
+  - ci_build/*.py
   - meta_review/handler.py
-  - model/*.py
-  - openhub/*.py
-  - twitter/*.py
   # Optional coverage. Once off scripts.
   - inactive_issues/inactive_issues_scraper.py
   - unassigned_issues/unassigned_issues_scraper.py

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-python: 3.6
+python: 3.6.3
 
 cache:
   pip: true

activity/scraper.py

@@ -136,7 +136,7 @@ def get_data(self):
         return self.data
 
 
-def activity_json(request):
+def activity_json(filename):
 
     org_name = get_org_name()
 
@@ -152,4 +152,5 @@ def activity_json(request):
     real_data = Scraper(parsed_json['issues'], datetime.datetime.today())
     real_data = real_data.get_data()
 
-    return HttpResponse(json.dumps(real_data))
+    with open(filename, 'w+') as f:
+        json.dump(real_data, f, indent=4)

ci_build/view_log.py

@@ -0,0 +1,131 @@
+import re
+import json
+import os
+import sys
+
+from django.views.generic import TemplateView
+
+from community.views import get_header_and_footer
+from community.git import (
+    get_org_name,
+    get_owner,
+    get_deploy_url,
+    get_upstream_deploy_url
+)
+
+
+class BuildLogsView(TemplateView):
+    template_name = 'build_logs.html'
+
+    def copy_build_logs_json(self, ci_build_jsons):
+        """
+        :param ci_build_jsons: A dict of directories path
+        :return: A boolean, whether the build file is copied
+        """
+        if os.path.isfile(ci_build_jsons['public_path']):
+            if sys.platform == 'linux':
+                os.popen('cp {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['public_path']))
+                os.popen('cp {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['static_path']))
+            else:
+                os.popen('copy {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['public_path']))
+                os.popen('copy {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['static_path']))
+            return True
+        return False
+
+    def create_and_copy_build_logs_json(self, logs, level_specific_logs):
+        """
+        Create a build logs detailed json file in ./_site directory and copy
+        that file in the ./static and ./public/static directories
+        :param logs: A list of all lines in build log file
+        :param level_specific_logs: A dict containing logs divided in their
+        respective categories
+        :return: A boolean, whether the files were copied or not
+        """
+        ci_build_jsons = {
+            'site_path': './_site/ci-build-detailed-logs.json',
+            'public_path': './public/static/ci-build-detailed-logs.json',
+            'static_path': './static/ci-build-detailed-logs.json'
+        }
+        with open(ci_build_jsons['site_path'], 'w+') as build_logs_file:
+            data = {
+                'logs': logs,
+                'logs_level_Specific': level_specific_logs
+            }
+            json.dump(data, build_logs_file, indent=4)
+        return self.copy_build_logs_json(ci_build_jsons)
+
+    def get_build_logs(self, log_file_path):
+        """
+        :param log_file_path: build logs file path
+        :return: a tuple of two where the first element in tuple refers to
+        a list of build logs in the file, and the second element is a dict
+        which categorizes the build logs into 5 categories - INFO, DEBUG,
+        WARNING, ERROR nad CRITICAL
+        """
+        log_lines = []
+        log_level_specific_lines = {
+            'INFO': [],
+            'DEBUG': [],
+            'WARNING': [],
+            'ERROR': [],
+            'CRITICAL': []
+        }
+        with open(log_file_path) as log_file:
+            previous_found_level = None
+            for line in log_file:
+                log_lines.append(line)
+                levels = re.findall(r'\[[A-Z]+]', line)
+                if levels:
+                    level = levels[0]
+                    level = previous_found_level = level[1:-1]
+                    log_level_specific_lines[level].append(line)
+                elif previous_found_level:
+                    log_level_specific_lines[previous_found_level].append(
+                        line)
+        return log_lines, log_level_specific_lines
+
+    def check_build_logs_stored(self):
+        """
+        Check whether the build logs json file is copied to _site and public
+        directories or not
+        :return: A Boolean
+        """
+        log_file_path = './_site/community.log'
+        log_file_exists = os.path.isfile(log_file_path)
+        if log_file_exists:
+            logs, level_specific_logs = self.get_build_logs(log_file_path)
+            return self.create_and_copy_build_logs_json(logs,
+                                                        level_specific_logs)
+        return False
+
+    def get_build_info(self):
+        """
+        Get the information about build, like who deployed the website i.e.
+        owner, name of the organization or user etc.
+        :return: A dict having information about build related details
+        """
+        data = {
+            'Org name': get_org_name(),
+            'Owner': get_owner(),
+            'Deploy URL': get_deploy_url(),
+        }
+        try:
+            data['Upstream deploy URL'] = get_upstream_deploy_url()
+        except RuntimeError:
+            data['Upstream deploy URL'] = 'Not found'
+        return data
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context = get_header_and_footer(context)
+        context['build_info'] = self.get_build_info()
+        context['logs_stored'] = self.check_build_logs_stored()
+        return context