fix(test): Fix licenseRef.json

Replace common unicode with ASCII. Signed-off-by: Gaurav Mishra <[email protected]>
ag4ums · Oct 29, 2021 · ee0a491 · ee0a491
1 parent 31dd1c4
commit ee0a491
Show file tree

Hide file tree

Showing 6 changed files with 182 additions and 142 deletions.
diff --git a/.github/workflows/test-checks.yml b/.github/workflows/test-checks.yml
@@ -1,5 +1,6 @@
-# Copyright 2021 Anupam Ghosh <[email protected]>
-# SPDX-License-Identifier: GPL-2.0 AND LGPL-2.
+# Copyright 2021 Anupam Ghosh <[email protected]>,
+#   Gaurav Mishra <[email protected]>
+# SPDX-License-Identifier: GPL-2.0 AND LGPL-2.1
 
 name: c tests
 
@@ -12,13 +13,16 @@ on:
 jobs:
   C-tests:
     runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        php: ['7.4'] # Can be extended in future
     services:
       postgres:
         image: postgres:12
         env:
-          POSTGRES_DB: fossology
-          POSTGRES_PASSWORD:  fossy
-          POSTGRES_USER: fossy
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
         ports:
           - 5432:5432
         # Set health checks to wait until postgres has started
@@ -32,21 +36,35 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 50
-      
+
       - name: Install Dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y libcppunit-dev
           sudo ./utils/fo-installdeps -y -e
           export PATH="/usr/lib/ccache/:$PATH"
 
+      - name: Install PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php }}
+          extensions: mbstring, gd, xml, zip, json, sqlite3, curl
+        env:
+          fail-fast: true
+
       - name: Setup test database
         env:
-          DATABASE_HOST: 127.0.0.1
-          DATABASE_PORT: 5432
+          PGHOST: 127.0.0.1
+          PGPORT: 5432
         run: |
-          sudo ./utils/prepare-test
+          ./utils/prepare-test -afty
 
-          #remove test all as delagent is failing  
+          #remove test all as delagent is failing
       - name: Unit tests
-        run: make all phpvendors test-cli test-copyright test-lib test-nomos test-ojo test-reuser test-scheduler test-spdx2 test-unifiedreport test-www
+        env:
+          PGHOST: 127.0.0.1
+          PGPORT: 5432
+        run: |
+          make all phpvendors test-cli test-copyright test-lib test-monk \
+          test-nomos test-ojo test-reuser test-scheduler test-spdx2 \
+          test-unifiedreport test-www
diff --git a/install/db/licenseRef.json b/install/db/licenseRef.json
diff --git a/src/cli/exportLicenseRefUsingSPDX.php b/src/cli/exportLicenseRefUsingSPDX.php
@@ -1,6 +1,6 @@
 <?php
 /***********************************************************
- Copyright (C) 2019 Siemens AG
+ Copyright (C) 2019,2021 Siemens AG
 
  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
@@ -42,6 +42,10 @@ function startProcessingLicenseData()
     $updateExisting = '';
     $addNewLicense = '';
     $newLicenseRefData = array();
+    $scanList = array(
+      'licenses' => 'https://spdx.org/licenses/licenses.json',
+      'exceptions' => 'https://spdx.org/licenses/exceptions.json'
+    );
     $usage = "Usage: " . basename($argv[0]) . " [options]
 
       Create new licenseref.json file.  Options are:
@@ -64,10 +68,6 @@ function startProcessingLicenseData()
           For type 'licenses' URL is : $scanList[licenses]
 
           For type 'exceptions' URL is : $scanList[exceptions]";
-    $scanList = array(
-    'licenses' => 'https://spdx.org/licenses/licenses.json',
-    'exceptions' => 'https://spdx.org/licenses/exceptions.json'
-    );
 
     $options = getopt("hcEen", array("type:","url:"));
     /* get type and url if exists if not set them to empty */
@@ -203,10 +203,10 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
             !empty($updateExisting)
           )
          ) {
-        $existingLicenseRefData[$licenseIdCheck]['rf_fullname'] = $getCurrentData[$this->mapArrayData[$type][2]];
-        $existingLicenseRefData[$licenseIdCheck]['rf_text'] = $getCurrentData[$this->mapArrayData[$type][1]];
+        $existingLicenseRefData[$licenseIdCheck]['rf_fullname'] = $this->replaceUnicode($getCurrentData[$this->mapArrayData[$type][2]]);
+        $existingLicenseRefData[$licenseIdCheck]['rf_text'] = $this->replaceUnicode($getCurrentData[$this->mapArrayData[$type][1]]);
         $existingLicenseRefData[$licenseIdCheck]['rf_url'] = $getCurrentData['seeAlso'][0];
-        $existingLicenseRefData[$licenseIdCheck]['rf_notes'] = (array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : $existingLicenseRefData[$licenseIdCheck]['rf_notes']);
+        $existingLicenseRefData[$licenseIdCheck]['rf_notes'] = $this->replaceUnicode((array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : $existingLicenseRefData[$licenseIdCheck]['rf_notes']));
         echo "INFO: license ".$getCurrentData[$this->mapArrayData[$type][0]]." updated\n\n";
       }
       if (!is_numeric($licenseIdCheck) &&
@@ -218,7 +218,7 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
          ) {
         $existingLicenseRefData[] = array(
           'rf_shortname' => $getCurrentData[$this->mapArrayData[$type][0]],
-          'rf_text' =>  $getCurrentData[$this->mapArrayData[$type][1]],
+          'rf_text' =>  $this->replaceUnicode($getCurrentData[$this->mapArrayData[$type][1]]),
           'rf_url' =>  $getCurrentData['seeAlso'][0],
           'rf_add_date' => null,
           'rf_copyleft' => null,
@@ -227,7 +227,7 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
           'rf_FSFfree' => null,
           'rf_GPLv2compatible' => null,
           'rf_GPLv3compatible' => null,
-          'rf_notes' => (array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : null),
+          'rf_notes' => $this->replaceUnicode((array_key_exists("licenseComments", $getCurrentData) ? $getCurrentData['licenseComments'] : null)),
           'rf_Fedora' => null,
           'marydone' => "f",
           'rf_active' => "t",
@@ -243,6 +243,35 @@ function getListSPDX($type, $URL, $updateWithNew, $updateExisting, $addNewLicens
     }
     return $existingLicenseRefData;
   }
+
+  /**
+   * Replace common unicode characters with ASCII for consistent results.
+   *
+   * @param string $text Input text
+   * @return string Input with characters replaced
+   */
+  private function replaceUnicode($text)
+  {
+    $search = [
+      '\u00a0',  // no break space
+      '\u2018',  // Left single quote
+      '\u2019',  // Right single quote
+      '\u201c',  // Left double quote
+      '\u201d',  // Right double quote
+      '\u2013',  // em dash
+    ];
+
+    $replace = [
+      " ",
+      "'",
+      "'",
+      '"',
+      '"',
+      "-",
+    ];
+
+    return str_replace($search, $replace, $text);
+  }
 }
 $obj = new exportLicenseRef();
 echo $obj->startProcessingLicenseData();
diff --git a/src/monk/agent_tests/Functional/bulkTest.php b/src/monk/agent_tests/Functional/bulkTest.php
@@ -138,15 +138,6 @@ private function getHeartCount($output)
     }
   }
 
-  /** @group Functional */
-  public function testDatabaseSetup()
-  {
-    $this->setUpTables();
-    $this->setUpRepo();
-
-    $this->rmRepo();
-  }
-
   /** @group Functional */
   public function testRunTwoIndependentMonkBulkScans()
   {

diff --git a/src/scheduler/agent_tests/Functional/Functional.py b/src/scheduler/agent_tests/Functional/Functional.py
@@ -32,7 +32,7 @@
 from xml.dom.minidom import parseString
 from xml.dom import Node
 from optparse import OptionParser
-import ConfigParser
+import configparser
 import subprocess
 import functools
 import signal
@@ -121,7 +121,7 @@ def __init__(self, node):
     self.defines['pids'] = {}
 
     # get variable definitions
-    for i in xrange(definitions.length):
+    for i in range(definitions.length):
       if definitions.item(i).name not in self.defines:
         self.defines[definitions.item(i).name] = self.substitute(definitions.item(i).value, defNode)
 
@@ -275,7 +275,7 @@ def createAction(self, node):
     Returns the new action
     """
     def action_wrapper(action, node, doc, dest):
-      print '.',# node.nodeName,
+      print('.', end=' ')# node.nodeName,
       return action(node, doc, dest)
 
     if not hasattr(self, node.nodeName):
@@ -398,8 +398,8 @@ def loadConf(self, node, doc, dest):
     """
     dir = self.required(node, 'directory')
 
-    config = ConfigParser.ConfigParser()
-    config.readfp(open(dir + "/fossology.conf"))
+    config = configparser.ConfigParser()
+    config.read_file(open(dir + "/fossology.conf"))
 
     self.defines["FOSSOLOGY"] = {}
     self.defines["BUILD"] = {}
@@ -408,7 +408,7 @@ def loadConf(self, node, doc, dest):
     self.defines["FOSSOLOGY"]["path"]  = config.get("FOSSOLOGY", "path")
     self.defines["FOSSOLOGY"]["depth"] = config.get("FOSSOLOGY", "depth")
 
-    config.readfp(open(dir + "/VERSION"))
+    config.read_file(open(dir + "/VERSION"))
 
     self.defines["BUILD"]["VERSION"] = config.get("BUILD", "VERSION")
     self.defines["BUILD"]["COMMIT_HASH"] = config.get("BUILD", "COMMIT_HASH")
@@ -448,7 +448,7 @@ def loop(self, node, doc, dest):
           tests  += ret[0]
           failed += ret[1]
     else:
-      for i in xrange(int(iterations)):
+      for i in range(int(iterations)):
         self.defines[varname] = str(i)
         for action in actions:
           ret = action(doc, dest)
@@ -598,12 +598,12 @@ def performTests(self, suiteNode, document, fname):
     tests        = 0
     totalasserts = 0
 
-    print "start up",
+    print("start up", end=' ')
     for action in self.setup:
       while action(None, None)[1] != 0:
         time.sleep(5)
 
-    print "tests",
+    print("tests", end=' ')
     for test in self.tests:
       assertions = 0
       testNode = document.createElement("testcase")
@@ -627,7 +627,7 @@ def performTests(self, suiteNode, document, fname):
 
       suiteNode.appendChild(testNode)
 
-    print " clean up",
+    print(" clean up", end=' ')
     for action in self.cleanup:
       action(None, None)
 
@@ -637,7 +637,7 @@ def performTests(self, suiteNode, document, fname):
     suiteNode.setAttribute("failures", str(failures))
     suiteNode.setAttribute("tests", str(tests))
     suiteNode.setAttribute("assertions", str(totalasserts))
-    print
+    print()
 
 ################################################################################
 ### MAIN #######################################################################
@@ -684,10 +684,10 @@ def main():
     if options.specific and suite.getAttribute("name") != options.specific:
       continue
     if suite.hasAttribute("disable"):
-      print suite.getAttribute("name"),'::','disabled'
+      print(suite.getAttribute("name"),'::','disabled')
       continue
     if options.skipLongTests and suite.hasAttribute("longest") and int(suite.getAttribute("longest"))>int(options.skipLongTests):
-      print suite.getAttribute("name"),'::','expected to run',suite.getAttribute("longest"),'time units'
+      print(suite.getAttribute("name"),'::','expected to run',suite.getAttribute("longest"),'time units')
       continue
     suiteNode = resultsDoc.createElement("testsuite")
     errors = 0
@@ -706,7 +706,7 @@ def main():
       curr.cleanup = cleanup + curr.cleanup
 
       starttime = time.time()
-      print "{0: >15} ::".format(suite.getAttribute("name")),
+      print("{0: >15} ::".format(suite.getAttribute("name")), end=' ')
       if not timeout(functools.partial(curr.performTests, suiteNode, resultsDoc, testFile.name), maxRuntime):
         errors += 1
         errorNode = resultsDoc.createElement("error")

diff --git a/utils/prepare-test b/utils/prepare-test
@@ -92,20 +92,22 @@ fi
 echo
 
 if echo ${WarnAccept} | grep -q -e "^[yY]$" ; then
+  uid=$(id -u)
+  gid=$(id -g)
   sudo mkdir -p /var/local/cache/fossology
-  sudo chown "$(whoami)" /var/local/cache/fossology
+  sudo chown "${uid}:${gid}" /var/local/cache/fossology
   sudo mkdir -p /srv/fossologyTestRepo
-  sudo chown "$(whoami)" /srv/fossologyTestRepo
-  sudo su postgres -c "echo 'CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";'|psql -d template1" || true
+  sudo chown "${uid}:${gid}" /srv/fossologyTestRepo
+  sudo --preserve-env su postgres -c "echo 'CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";'|psql -d template1" || true
   if [[ "${CreateFossy}" == "Y" ]]; then
-    sudo su postgres -c "echo \"CREATE USER fossy WITH PASSWORD 'fossy' CREATEDB;\"|psql" || true >/dev/null
+    sudo --preserve-env su postgres -c "echo \"CREATE USER fossy WITH PASSWORD 'fossy' CREATEDB;\"|psql" || true >/dev/null
     echo "localhost:*:*:fossy:fossy" >> ~/.pgpass
     chmod 0600 ~/.pgpass
-    sudo su postgres -c "echo 'CREATE DATABASE fossology; GRANT ALL PRIVILEGES ON DATABASE fossology TO fossy;'|psql" || true
+    sudo --preserve-env su postgres -c "echo 'CREATE DATABASE fossology; GRANT ALL PRIVILEGES ON DATABASE fossology TO fossy;'|psql" || true
   fi
   if [[ "${CreateFossyTest}" == "Y" ]]; then
     testDbPassword=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13)
-    sudo su postgres -c "echo \"CREATE USER fossologytest WITH CREATEDB LOGIN PASSWORD '${testDbPassword}';\"|psql" || true >/dev/null
+    sudo --preserve-env su postgres -c "echo \"CREATE USER fossologytest WITH CREATEDB LOGIN PASSWORD '${testDbPassword}';\"|psql" || true >/dev/null
     echo "localhost:*:*:fossologytest:${testDbPassword}" >> ~/.pgpass
     chmod 0600 ~/.pgpass
   fi