[TECH-272] Make mpyl work in the monorepo

mpyl-example.py

@@ -23,6 +23,7 @@ def main(log: Logger, args: argparse.Namespace):
         run_config=MpylRunConfig(config=config, run_properties=run_properties),
         parameters=MpylCliParameters(
             local=args.local,
+            pull_main=True,
             verbose=args.verbose,
             all=args.all
         )

mpyl_config.example.yml

@@ -39,13 +39,19 @@ jenkins:
   defaultPipeline: 'mpyl-test'
 sbt:
   command: 'sbt'
+  clientCommand: 'sbtn'
   testWithCoverage: !ENV ${SBT_RUN_WITH_COVERAGE:false}
   verbose: false
   javaOpts: '-Xmx4G -Xms4G -XX:+UseG1GC -XX:+CMSClassUnloadingEnabled -Xss2M'
   sbtOpts: 'user.timezone=GMT jline.terminal=jline.UnixTerminal'
   clientMode:
     build: false
     test: false
+whiteLists:
+  default: [ "VPN" ]
+  addresses:
+    - name: "VPN"
+      values: [ "10.0.0.1" ]
 kubernetes:
   rancher:
     cluster:
@@ -105,3 +111,16 @@ project: # default values
       metrics:
         path: '/metrics'
         enabled: true
+    traefik:
+      enabled: true
+      hosts:
+        - host:
+            pr: "Host(`{SERVICE-NAME}-{PR-NUMBER}.test-backend.nl`)"
+            test: "Host(`{namespace}-{SERVICE-NAME}.test-backend.nl`)"
+            acceptance: "Host(`{namespace}-{SERVICE-NAME}.acce-backend.nl`)"
+            production: "Host(`{namespace}-{SERVICE-NAME}.prod-backend.nl`)"
+          tls:
+            all: "le-custom-prod-wildcard-cert"
+          insecure: false
+          whitelists:
+            all: [ "VPN" ]

run_properties.yml

@@ -2,7 +2,7 @@ build:
   run:
     id: !ENV ${BUILD_ID:1}
     run_url: !ENV ${RUN_DISPLAY_URL:http://localhost:3000/}
-    change_url: !ENV ${GIT_URL:http://localhost:3000/}
+    change_url: !ENV ${CHANGE_URL:http://localhost:3000/}
     tests_url: !ENV ${RUN_TESTS_DISPLAY_URL:http://localhost:3000/}
     user: !ENV ${BUILD_USER:anonymous}
     user_email: !ENV ${BUILD_USER_EMAIL}

src/mpyl/cli/build.py

@@ -53,7 +53,7 @@ def run(obj: CliContext, properties, ci, all_):  # pylint: disable=invalid-name
     run_properties = RunProperties.from_configuration(parse_config(properties), obj.config) if ci \
         else RunProperties.for_local_run(obj.config, obj.repo.get_sha, obj.repo.get_branch)
 
-    parameters = MpylCliParameters(local=not ci, all=all_, verbose=obj.verbose)
+    parameters = MpylCliParameters(local=not ci, pull_main=all_, all=all_, verbose=obj.verbose)
     obj.console.log(parameters)
     run_parameters = MpylRunParameters(
         run_config=MpylRunConfig(config=obj.config, run_properties=run_properties),

src/mpyl/cli/commands/build/mpyl.py

@@ -15,7 +15,7 @@
 from ....stages.discovery import for_stage, find_invalidated_projects_per_stage
 from ....steps.models import RunProperties
 from ....steps.run import RunResult
-from ....steps.steps import Steps
+from ....steps.steps import Steps, ExecutionException
 from ....utilities.repo import Repository, RepoConfig
 
 
@@ -28,6 +28,7 @@ class MpylRunConfig:
 @dataclass(frozen=True)
 class MpylCliParameters:
     local: bool
+    pull_main: bool = False
     verbose: bool = False
     all: bool = False
 
@@ -44,9 +45,13 @@ class MpylRunParameters:
 def get_build_plan(logger: logging.Logger, repo: Repository, mpyl_run_parameters: MpylRunParameters) -> RunResult:
     params = mpyl_run_parameters.parameters
     logger.info(f"Running with {params}")
-    if not params.local:
-        pull_result = repo.pull_main_branch()
-        logger.info(f'Pulled `{pull_result[0].remote_ref_path.strip()}` to local')
+    if params.pull_main:
+        if repo.main_branch_pulled:
+            logger.info(f'Branch {repo.main_branch} already present locally. Skipping pull.')
+        else:
+            logger.info(f'Pulling {repo.main_branch} from {repo.get_remote_url}')
+            pull_result = repo.pull_main_branch()
+            logger.info(f'Pulled `{pull_result[0].remote_ref_path.strip()}` to local')
 
     changes_in_branch = repo.changes_in_branch_including_local() if params.local else repo.changes_in_branch()
     logger.debug(f'Changes: {changes_in_branch}')
@@ -80,11 +85,11 @@ def run_mpyl(mpyl_run_parameters: MpylRunParameters, reporter: Optional[Reporter
             run_result: RunResult = run_plan
             try:
                 steps = Steps(logger=logger, properties=mpyl_run_parameters.run_config.run_properties)
-                run_result = run_build(run_plan, steps, reporter)
-            except Exception as exc:  # pylint: disable=broad-except
+                run_result = run_build(run_plan, steps, reporter, mpyl_run_parameters.parameters.local)
+            except ExecutionException as exc:  # pylint: disable=broad-except
+                run_result.exception = exc
                 console.log(f'Exception during build execution: {exc}')
                 console.print_exception()
-                run_result.exception = exc
 
             console.print(Markdown(run_result_to_markdown(run_result)))
             return run_result
@@ -107,10 +112,10 @@ def find_build_set(repo: Repository, changes_in_branch, build_all: bool) -> dict
     return find_invalidated_projects_per_stage(all_projects, changes_in_branch)
 
 
-def run_build(accumulator: RunResult, executor: Steps, reporter: Optional[Reporter] = None):
+def run_build(accumulator: RunResult, executor: Steps, reporter: Optional[Reporter] = None, dry_run: bool = True):
     for stage, projects in accumulator.run_plan.items():
         for proj in projects:
-            result = executor.execute(stage, proj, True)
+            result = executor.execute(stage, proj, dry_run)
             accumulator.append(result)
             if reporter:
                 reporter.send_report(accumulator)

src/mpyl/project.py

@@ -234,13 +234,15 @@ def from_config(values: dict):
 @dataclass(frozen=True)
 class Host:
     host: TargetProperty[str]
+    service_port: Optional[int]
     tls: TargetProperty[str]
     whitelists: TargetProperty[list[str]]
 
     @staticmethod
     def from_config(values: dict):
         return Host(
             host=TargetProperty.from_config(values.get('host', {})),
+            service_port=values.get('servicePort'),
             tls=TargetProperty.from_config(values.get('tls', {})),
             whitelists=TargetProperty.from_config(values.get('whitelists', {}))
         )

src/mpyl/reporting/formatting/markdown.py

@@ -6,6 +6,8 @@
 from junitparser import TestSuite
 
 from ...project import Stage, Project
+from ...steps import Output, ArtifactType
+from ...steps.deploy.kubernetes import DEPLOYED_SERVICE_KEY
 from ...steps.run import RunResult
 from ...steps.steps import StepResult, collect_test_results
 from ...utilities.junit import TestRunSummary, sum_suites
@@ -16,14 +18,24 @@ def summary_to_markdown(summary: TestRunSummary):
            f"💔 {summary.errors} 🙈 {summary.skipped}"
 
 
+def __add_link(name: str, output: Output) -> str:
+    if output.produced_artifact and output.produced_artifact.artifact_type == ArtifactType.DEPLOYED_HELM_APP:
+        url = output.produced_artifact.spec.get(DEPLOYED_SERVICE_KEY)
+        if url:
+            return f'[{name}]({url}/swagger/index.html)'
+
+    return name
+
+
 def __to_oneliner(result: list[StepResult], plan: set[Project]) -> str:
     project_names: list[str] = []
     if plan:
         sorted_plan = sorted(plan, key=operator.attrgetter('name'))
         for proj in sorted_plan:
             found_result = next((r for r in result if r.project.name == proj.name), None)
             if found_result:
-                project_names.append(f'*{proj.name}*' if found_result.output.success else f'~~{proj.name}~~')
+                project_names.append(f'*{__add_link(proj.name, found_result.output)}*' if found_result.output.success
+                                     else f'~~{proj.name}~~')
             else:
                 project_names.append(f'_{proj.name}_')
     else:
@@ -42,20 +54,29 @@ def stage_to_icon(stage: Stage):
     return '➡️'
 
 
+def markdown_for_stage(run_result: RunResult, stage: Stage):
+    step_results: list[StepResult] = run_result.results_for_stage(stage)
+    plan: set[Project] = run_result.plan_for_stage(stage)
+    result = ""
+    if step_results or plan:
+        result += f"{stage_to_icon(stage)}  {__to_oneliner(step_results, plan)}  \n"
+        test_results = collect_test_results(step_results)
+        if test_results:
+            result += to_markdown_test_report(
+                test_results) + f' [link]({run_result.run_properties.details.tests_url}) \n'
+
+    return result
+
+
 def run_result_to_markdown(run_result: RunResult) -> str:
-    result: str = f'{run_result.status_line}  \n' if run_result.is_finished and run_result.run_plan else ""
-    if run_result.exception:
-        result += f"\n```\n{run_result.exception}\n```\n"
+    result: str = f'{run_result.status_line}  \n'
+    exception = run_result.exception
+    if exception:
+        result += f"For _{exception.executor}_ on _{exception.project_name}_ at _{exception.stage}_ \n"
+        result += f"\n```\n{exception}\n```\n"
 
     for stage in Stage:
-        step_results: list[StepResult] = run_result.results_for_stage(stage)
-        plan: set[Project] = run_result.plan_for_stage(stage)
-        if step_results or plan:
-            result += f"{stage_to_icon(stage)}  {__to_oneliner(step_results, plan)}  \n"
-            test_results = collect_test_results(step_results)
-            if test_results:
-                result += to_markdown_test_report(
-                    test_results) + f' [link]({run_result.run_properties.details.tests_url}) \n'
+        result += markdown_for_stage(run_result, stage)
 
     return result
 

src/mpyl/reporting/targets/github.py

@@ -120,9 +120,9 @@ def send_report(self, results: RunResult, text: Optional[str] = None) -> None:
             access_token = integration.get_access_token(install.id)
             github = Github(login_or_token=access_token.token)
             repo = github.get_repo(self._github_config.repository)
-            if self._check_run_id:
+            if self._check_run_id and results:
                 run = repo.get_check_run(self._check_run_id)
-                conclusion = 'success' if results.is_success else 'failure'
+                conclusion = 'success' if results is None or results.is_success else 'failure'
                 self._logger.info(f'Setting check to {conclusion}')
                 run.edit(completed_at=datetime.now(), conclusion=conclusion, output=self._to_output(results))
             else:

src/mpyl/reporting/targets/jira.py

@@ -32,6 +32,8 @@
 from atlassian import Jira
 
 from . import Reporter
+from ..formatting.markdown import markdown_for_stage
+from ...project import Stage
 from ...steps.run import RunResult
 
 
@@ -128,7 +130,7 @@ def to_github_markdown(jira_markdown: str, jira_url: str) -> str:
 
 
 def to_markdown_summary(ticket: JiraTicket, run_result: RunResult) -> str:
-    description_markdown = to_github_markdown(ticket.description, ticket.ticket_url)
+    description_markdown = to_github_markdown(ticket.description, ticket.ticket_url) if ticket.description else ""
     lines = description_markdown.splitlines()
     max_message_length = 288
     if len(lines) > max_message_length:
@@ -137,7 +139,7 @@ def to_markdown_summary(ticket: JiraTicket, run_result: RunResult) -> str:
     details = run_result.run_properties.details
 
     build_status = f"🏗️ Build [{details.build_id}]({details.run_url}) {run_result.status_line}, " \
-                   f"started by _{details.user}_"
+                   f"started by _{details.user}_  \n{markdown_for_stage(run_result, Stage.DEPLOY)}"
     return f"## 📕 [{ticket.ticket_id}]({ticket.ticket_url}) {ticket.summary} " \
            f"![{ticket.user_email}]({ticket.user_avatar}) \n" \
            f"{description_markdown}\n" \

src/mpyl/reporting/targets/slack.py

@@ -45,7 +45,7 @@
 
 def to_slack_markdown(markdown: str) -> str:
     regex_replace = (
-        (re.compile(r'\[(.*)\]\((.*)\)', flags=re.M), r'<\2|\1>'),
+        (re.compile(r'\[(.*?)\]\((.*?)\)'), r'<\2|\1>'),
         (re.compile(r'^- ', flags=re.M), '• '),
         (re.compile(r'^  - ', flags=re.M), '  ◦ '),
         (re.compile(r'^    - ', flags=re.M), '    ⬩ '),
@@ -54,8 +54,8 @@ def to_slack_markdown(markdown: str) -> str:
         (re.compile(r'\*\*'), '*'),
         (re.compile(r'~~'), '~'),
     )
-    for regex, replacement in regex_replace:
-        markdown = regex.sub(replacement, markdown)
+    for pattern, replacement in regex_replace:
+        markdown = re.sub(pattern, replacement, markdown)
     return markdown
 
 

src/mpyl/schema/mpyl_config.schema.yml

@@ -9,6 +9,8 @@ definitions:
         "$ref": "#/definitions/CVS"
       docker:
         "$ref": "#/definitions/Docker"
+      whiteLists:
+        "$ref": "#/definitions/Whitelists"
       kubernetes:
         "$ref": "#/definitions/Kubernetes"
       project:
@@ -25,6 +27,36 @@ definitions:
       - cvs
       - docker
     title: MPyL global configuration
+  Whitelists:
+    type: object
+    required:
+      - default
+      - addresses
+    properties:
+      default:
+        description: "Default whitelist for all environments"
+        type: array
+        items:
+          minItems: 1
+          type: string
+      addresses:
+        type: array
+        minItems: 1
+        items:
+          type: object
+          additionalProperties: false
+          required:
+            - name
+            - values
+          properties:
+            name:
+              type: string
+            values:
+              type: array
+              items:
+                minItems: 1
+                type: string
+
   Kubernetes:
     type: object
     additionalProperties: false
@@ -164,6 +196,8 @@ definitions:
     properties:
       command:
         type: string
+      clientCommand:
+        type: string
       verbose:
         default: false
         type: boolean
@@ -185,6 +219,7 @@ definitions:
             default: true
     required:
       - command
+      - clientCommand
       - javaOpts
       - sbtOpts
     title: Sbt
@@ -196,6 +231,8 @@ definitions:
         type: object
         additionalProperties: false
         properties:
+          traefik:
+            "$ref": "project.schema.yml#/definitions/traefik"
           kubernetes:
             "$ref": "project.schema.yml#/definitions/kubernetes"
   Slack: